libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  10 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  11
  12 /*
  13 =================================================================
  14
  15   Minimal set of definitions from libcurl
  16
  17   WARNING: for a matter of simplicity, several pointer types are
  18   casted to "void*", and most enumerated values are not included
  19
  20 =================================================================
  21 */
  22
  23 typedef struct CURL_s CURL;
  24 typedef struct CURLM_s CURLM;
  25 typedef struct curl_slist curl_slist;
  26 typedef enum
  27 {
  28         CURLE_OK = 0
  29 }
  30 CURLcode;
  31 typedef enum
  32 {
  33         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  34         CURLM_OK = 0
  35 }
  36 CURLMcode;
  37 #define CURL_GLOBAL_NOTHING 0
  38 #define CURL_GLOBAL_SSL 1
  39 #define CURL_GLOBAL_WIN32 2
  40 #define CURLOPTTYPE_LONG          0
  41 #define CURLOPTTYPE_OBJECTPOINT   10000
  42 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  43 #define CURLOPTTYPE_OFF_T         30000
  44 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  45 typedef enum
  46 {
  47         CINIT(WRITEDATA, OBJECTPOINT, 1),
  48         CINIT(URL,  OBJECTPOINT, 2),
  49         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  50         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  51         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  52         CINIT(REFERER, OBJECTPOINT, 16),
  53         CINIT(USERAGENT, OBJECTPOINT, 18),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56         CINIT(RESUME_FROM, LONG, 21),
  57         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  58         CINIT(POST, LONG, 47),         /* HTTP POST method */
  59         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  60         CINIT(POSTFIELDSIZE, LONG, 60),
  61         CINIT(PRIVATE, OBJECTPOINT, 103),
  62         CINIT(PROTOCOLS, LONG, 181),
  63         CINIT(REDIR_PROTOCOLS, LONG, 182)
  64 }
  65 CURLoption;
  66 #define CURLPROTO_HTTP   (1<<0)
  67 #define CURLPROTO_HTTPS  (1<<1)
  68 #define CURLPROTO_FTP    (1<<2)
  69 typedef enum
  70 {
  71         CURLINFO_TEXT = 0,
  72         CURLINFO_HEADER_IN,    /* 1 */
  73         CURLINFO_HEADER_OUT,   /* 2 */
  74         CURLINFO_DATA_IN,      /* 3 */
  75         CURLINFO_DATA_OUT,     /* 4 */
  76         CURLINFO_SSL_DATA_IN,  /* 5 */
  77         CURLINFO_SSL_DATA_OUT, /* 6 */
  78         CURLINFO_END
  79 }
  80 curl_infotype;
  81 #define CURLINFO_STRING   0x100000
  82 #define CURLINFO_LONG     0x200000
  83 #define CURLINFO_DOUBLE   0x300000
  84 #define CURLINFO_SLIST    0x400000
  85 #define CURLINFO_MASK     0x0fffff
  86 #define CURLINFO_TYPEMASK 0xf00000
  87 typedef enum
  88 {
  89         CURLINFO_NONE, /* first, never use this */
  90         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  91         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  92         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  93         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  94         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  95         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  96         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  97         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  98         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  99         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 100         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 101         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 102         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 103         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 104         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 105         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 106         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 107         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 108         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 109         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 110         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 111         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 112         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 113         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 114         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 115         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 116         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 117 }
 118 CURLINFO;
 119
 120 typedef enum
 121 {
 122         CURLMSG_NONE, /* first, not used */
 123         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 124                                          the CURLcode of the transfer */
 125         CURLMSG_LAST
 126 }
 127 CURLMSG;
 128 typedef struct
 129 {
 130         CURLMSG msg;       /* what this message means */
 131         CURL *easy_handle; /* the handle it concerns */
 132         union
 133         {
 134                 void *whatever;    /* message-specific data */
 135                 CURLcode result;   /* return code for transfer */
 136         }
 137         data;
 138 }
 139 CURLMsg;
 140
 141 static void (*qcurl_global_init) (long flags);
 142 static void (*qcurl_global_cleanup) (void);
 143
 144 static CURL * (*qcurl_easy_init) (void);
 145 static void (*qcurl_easy_cleanup) (CURL *handle);
 146 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 147 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 148 static const char * (*qcurl_easy_strerror) (CURLcode);
 149
 150 static CURLM * (*qcurl_multi_init) (void);
 151 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 152 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 153 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 154 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 155 static void (*qcurl_multi_cleanup) (CURLM *);
 156 static const char * (*qcurl_multi_strerror) (CURLcode);
 157 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 158 static void (*qcurl_slist_free_all) (curl_slist *list);
 159
 160 static dllfunction_t curlfuncs[] =
 161 {
 162         {"curl_global_init",            (void **) &qcurl_global_init},
 163         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 164         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 165         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 166         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 167         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 168         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 169         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 170         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 171         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 172         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 173         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 174         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 175         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 176         {"curl_slist_append",           (void **) &qcurl_slist_append},
 177         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 178         {NULL, NULL}
 179 };
 180
 181 // Handle for CURL DLL
 182 static dllhandle_t curl_dll = NULL;
 183 // will be checked at many places to find out if qcurl calls are allowed
 184
 185 typedef struct downloadinfo_s
 186 {
 187         char filename[MAX_OSPATH];
 188         char url[1024];
 189         char referer[256];
 190         qfile_t *stream;
 191         fs_offset_t startpos;
 192         CURL *curle;
 193         qboolean started;
 194         qboolean ispak;
 195         unsigned long bytes_received; // for buffer
 196         double bytes_received_curl; // for throttling
 197         double bytes_sent_curl; // for throttling
 198         struct downloadinfo_s *next, *prev;
 199         qboolean forthismap;
 200         double maxspeed;
 201         curl_slist *slist; // http headers
 202
 203         unsigned char *buffer;
 204         size_t buffersize;
 205         curl_callback_t callback;
 206         void *callback_data;
 207
 208         const unsigned char *postbuf;
 209         size_t postbufsize;
 210         const char *post_content_type;
 211         const char *extraheaders;
 212 }
 213 downloadinfo;
 214 static downloadinfo *downloads = NULL;
 215 static int numdownloads = 0;
 216
 217 static qboolean noclear = FALSE;
 218
 219 static int numdownloads_fail = 0;
 220 static int numdownloads_success = 0;
 221 static int numdownloads_added = 0;
 222 static char command_when_done[256] = "";
 223 static char command_when_error[256] = "";
 224
 225 /*
 226 ====================
 227 Curl_CommandWhenDone
 228
 229 Sets the command which is to be executed when the last download completes AND
 230 all downloads since last server connect ended with a successful status.
 231 Setting the command to NULL clears it.
 232 ====================
 233 */
 234 void Curl_CommandWhenDone(const char *cmd)
 235 {
 236         if(!curl_dll)
 237                 return;
 238         if(cmd)
 239                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 240         else
 241                 *command_when_done = 0;
 242 }
 243
 244 /*
 245 FIXME
 246 Do not use yet. Not complete.
 247 Problem: what counts as an error?
 248 */
 249
 250 void Curl_CommandWhenError(const char *cmd)
 251 {
 252         if(!curl_dll)
 253                 return;
 254         if(cmd)
 255                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 256         else
 257                 *command_when_error = 0;
 258 }
 259
 260 /*
 261 ====================
 262 Curl_Clear_forthismap
 263
 264 Clears the "will disconnect on failure" flags.
 265 ====================
 266 */
 267 void Curl_Clear_forthismap(void)
 268 {
 269         downloadinfo *di;
 270         if(noclear)
 271                 return;
 272         for(di = downloads; di; di = di->next)
 273                 di->forthismap = false;
 274         Curl_CommandWhenError(NULL);
 275         Curl_CommandWhenDone(NULL);
 276         numdownloads_fail = 0;
 277         numdownloads_success = 0;
 278         numdownloads_added = 0;
 279 }
 280
 281 /*
 282 ====================
 283 Curl_Have_forthismap
 284
 285 Returns true if a download needed for the current game is running.
 286 ====================
 287 */
 288 qboolean Curl_Have_forthismap(void)
 289 {
 290         return numdownloads_added != 0;
 291 }
 292
 293 void Curl_Register_predownload(void)
 294 {
 295         Curl_CommandWhenDone("cl_begindownloads");
 296         Curl_CommandWhenError("cl_begindownloads");
 297 }
 298
 299 /*
 300 ====================
 301 Curl_CheckCommandWhenDone
 302
 303 Checks if a "done command" is to be executed.
 304 All downloads finished, at least one success since connect, no single failure
 305 -> execute the command.
 306 */
 307 static void Curl_CheckCommandWhenDone(void)
 308 {
 309         if(!curl_dll)
 310                 return;
 311         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 312         {
 313                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 314                 Cbuf_AddText("\n");
 315                 Cbuf_AddText(command_when_done);
 316                 Cbuf_AddText("\n");
 317                 Curl_Clear_forthismap();
 318         }
 319         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 320         {
 321                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 322                 Cbuf_AddText("\n");
 323                 Cbuf_AddText(command_when_error);
 324                 Cbuf_AddText("\n");
 325                 Curl_Clear_forthismap();
 326         }
 327 }
 328
 329 /*
 330 ====================
 331 CURL_CloseLibrary
 332
 333 Load the cURL DLL
 334 ====================
 335 */
 336 static qboolean CURL_OpenLibrary (void)
 337 {
 338         const char* dllnames [] =
 339         {
 340 #if defined(WIN32)
 341                 "libcurl-4.dll",
 342                 "libcurl-3.dll",
 343 #elif defined(MACOSX)
 344                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 345                 "libcurl.3.dylib", // Mac OS X Tiger
 346                 "libcurl.2.dylib", // Mac OS X Panther
 347 #else
 348                 "libcurl.so.4",
 349                 "libcurl.so.3",
 350                 "libcurl.so", // FreeBSD
 351 #endif
 352                 NULL
 353         };
 354
 355         // Already loaded?
 356         if (curl_dll)
 357                 return true;
 358
 359         // Load the DLL
 360         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 361 }
 362
 363
 364 /*
 365 ====================
 366 CURL_CloseLibrary
 367
 368 Unload the cURL DLL
 369 ====================
 370 */
 371 static void CURL_CloseLibrary (void)
 372 {
 373         Sys_UnloadLibrary (&curl_dll);
 374 }
 375
 376
 377 static CURLM *curlm = NULL;
 378 static double bytes_received = 0; // used for bandwidth throttling
 379 static double bytes_sent = 0; // used for bandwidth throttling
 380 static double curltime = 0;
 381
 382 /*
 383 ====================
 384 CURL_fwrite
 385
 386 fwrite-compatible function that writes the data to a file. libcurl can call
 387 this.
 388 ====================
 389 */
 390 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 391 {
 392         fs_offset_t ret = -1;
 393         size_t bytes = size * nmemb;
 394         downloadinfo *di = (downloadinfo *) vdi;
 395
 396         if(di->buffer)
 397         {
 398                 if(di->bytes_received + bytes <= di->buffersize)
 399                 {
 400                         memcpy(di->buffer + di->bytes_received, data, bytes);
 401                         ret = bytes;
 402                 }
 403                 // otherwise: buffer overrun, ret stays -1
 404         }
 405
 406         if(di->stream)
 407         {
 408                 ret = FS_Write(di->stream, data, bytes);
 409         }
 410
 411         di->bytes_received += bytes;
 412
 413         return ret; // why not ret / nmemb?
 414 }
 415
 416 typedef enum
 417 {
 418         CURL_DOWNLOAD_SUCCESS = 0,
 419         CURL_DOWNLOAD_FAILED,
 420         CURL_DOWNLOAD_ABORTED,
 421         CURL_DOWNLOAD_SERVERERROR
 422 }
 423 CurlStatus;
 424
 425 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 426 {
 427         downloadinfo *di = (downloadinfo *) cbdata;
 428         switch(status)
 429         {
 430                 case CURLCBSTATUS_OK:
 431                         Con_DPrintf("Download of %s: OK\n", di->filename);
 432                         break;
 433                 case CURLCBSTATUS_FAILED:
 434                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 435                         break;
 436                 case CURLCBSTATUS_ABORTED:
 437                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 438                         break;
 439                 case CURLCBSTATUS_SERVERERROR:
 440                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 441                         break;
 442                 case CURLCBSTATUS_UNKNOWN:
 443                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 444                         break;
 445                 default:
 446                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 447                         break;
 448         }
 449 }
 450
 451 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 452 {
 453         curl_default_callback(status, length_received, buffer, cbdata);
 454 }
 455
 456 /*
 457 ====================
 458 Curl_EndDownload
 459
 460 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 461 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 462 code from libcurl, or 0, if another error has occurred.
 463 ====================
 464 */
 465 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 466 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 467 {
 468         qboolean ok = false;
 469         if(!curl_dll)
 470                 return;
 471         switch(status)
 472         {
 473                 case CURL_DOWNLOAD_SUCCESS:
 474                         ok = true;
 475                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 476                         break;
 477                 case CURL_DOWNLOAD_FAILED:
 478                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 479                         break;
 480                 case CURL_DOWNLOAD_ABORTED:
 481                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 482                         break;
 483                 case CURL_DOWNLOAD_SERVERERROR:
 484                         // reopen to enforce it to have zero bytes again
 485                         if(di->stream)
 486                         {
 487                                 FS_Close(di->stream);
 488                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 489                         }
 490
 491                         if(di->callback)
 492                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 493                         break;
 494                 default:
 495                         if(di->callback)
 496                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 497                         break;
 498         }
 499
 500         if(di->curle)
 501         {
 502                 qcurl_multi_remove_handle(curlm, di->curle);
 503                 qcurl_easy_cleanup(di->curle);
 504                 if(di->slist)
 505                         qcurl_slist_free_all(di->slist);
 506         }
 507
 508         if(!di->callback && ok && !di->bytes_received)
 509         {
 510                 Con_Printf("ERROR: empty file\n");
 511                 ok = false;
 512         }
 513
 514         if(di->stream)
 515                 FS_Close(di->stream);
 516
 517         if(ok && di->ispak)
 518         {
 519                 ok = FS_AddPack(di->filename, NULL, true);
 520                 if(!ok)
 521                 {
 522                         // pack loading failed?
 523                         // this is critical
 524                         // better clear the file again...
 525                         di->stream = FS_OpenRealFile(di->filename, "wb", false);
 526                         FS_Close(di->stream);
 527
 528                         if(di->startpos && !di->callback)
 529                         {
 530                                 // this was a resume?
 531                                 // then try to redownload it without reporting the error
 532                                 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
 533                                 di->forthismap = false; // don't count the error
 534                         }
 535                 }
 536         }
 537
 538         if(di->prev)
 539                 di->prev->next = di->next;
 540         else
 541                 downloads = di->next;
 542         if(di->next)
 543                 di->next->prev = di->prev;
 544
 545         --numdownloads;
 546         if(di->forthismap)
 547         {
 548                 if(ok)
 549                         ++numdownloads_success;
 550                 else
 551                         ++numdownloads_fail;
 552         }
 553         Z_Free(di);
 554 }
 555
 556 /*
 557 ====================
 558 CleanURL
 559
 560 Returns a "cleaned up" URL for display (to strip login data)
 561 ====================
 562 */
 563 static const char *CleanURL(const char *url)
 564 {
 565         static char urlbuf[1024];
 566         const char *p, *q, *r;
 567
 568         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 569         p = strstr(url, "://");
 570         if(p)
 571         {
 572                 q = strchr(p + 3, '@');
 573                 if(q)
 574                 {
 575                         r = strchr(p + 3, '/');
 576                         if(!r || q < r)
 577                         {
 578                                 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s%s", (int)(p - url + 3), url, q + 1);
 579                                 return urlbuf;
 580                         }
 581                 }
 582         }
 583
 584         return url;
 585 }
 586
 587 /*
 588 ====================
 589 CheckPendingDownloads
 590
 591 checks if there are free download slots to start new downloads in.
 592 To not start too many downloads at once, only one download is added at a time,
 593 up to a maximum number of cl_curl_maxdownloads are running.
 594 ====================
 595 */
 596 static void CheckPendingDownloads(void)
 597 {
 598         const char *h;
 599         if(!curl_dll)
 600                 return;
 601         if(numdownloads < cl_curl_maxdownloads.integer)
 602         {
 603                 downloadinfo *di;
 604                 for(di = downloads; di; di = di->next)
 605                 {
 606                         if(!di->started)
 607                         {
 608                                 if(!di->buffer)
 609                                 {
 610                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url), di->filename);
 611
 612                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 613                                         if(!di->stream)
 614                                         {
 615                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 616                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 617                                                 return;
 618                                         }
 619                                         FS_Seek(di->stream, 0, SEEK_END);
 620                                         di->startpos = FS_Tell(di->stream);
 621
 622                                         if(di->startpos > 0)
 623                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 624                                         Con_Print("...\n");
 625                                 }
 626                                 else
 627                                 {
 628                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url));
 629                                         di->startpos = 0;
 630                                 }
 631
 632                                 di->curle = qcurl_easy_init();
 633                                 di->slist = NULL;
 634                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 635                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 636                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 637                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 638                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 639                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 640                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 641                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 642                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 643                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 644                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 645                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 646                                 {
 647                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 648                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 649                                 }
 650                                 if(di->post_content_type)
 651                                 {
 652                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 653                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 654                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 655                                         di->slist = qcurl_slist_append(di->slist, va("Content-Type: %s", di->post_content_type));
 656                                 }
 657
 658                                 // parse extra headers into slist
 659                                 // \n separated list!
 660                                 h = di->extraheaders;
 661                                 while(h)
 662                                 {
 663                                         const char *hh = strchr(h, '\n');
 664                                         if(hh)
 665                                         {
 666                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 667                                                 memcpy(buf, h, hh - h);
 668                                                 buf[hh - h] = 0;
 669                                                 di->slist = qcurl_slist_append(di->slist, buf);
 670                                                 h = hh + 1;
 671                                         }
 672                                         else
 673                                         {
 674                                                 di->slist = qcurl_slist_append(di->slist, h);
 675                                                 h = NULL;
 676                                         }
 677                                 }
 678
 679                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 680
 681                                 qcurl_multi_add_handle(curlm, di->curle);
 682                                 di->started = true;
 683                                 ++numdownloads;
 684                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 685                                         break;
 686                         }
 687                 }
 688         }
 689 }
 690
 691 /*
 692 ====================
 693 Curl_Init
 694
 695 this function MUST be called before using anything else in this file.
 696 On Win32, this must be called AFTER WSAStartup has been done!
 697 ====================
 698 */
 699 void Curl_Init(void)
 700 {
 701         CURL_OpenLibrary();
 702         if(!curl_dll)
 703                 return;
 704         qcurl_global_init(CURL_GLOBAL_NOTHING);
 705         curlm = qcurl_multi_init();
 706 }
 707
 708 /*
 709 ====================
 710 Curl_Shutdown
 711
 712 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 713 ====================
 714 */
 715 void Curl_ClearRequirements(void);
 716 void Curl_Shutdown(void)
 717 {
 718         if(!curl_dll)
 719                 return;
 720         Curl_ClearRequirements();
 721         Curl_CancelAll();
 722         CURL_CloseLibrary();
 723         curl_dll = NULL;
 724 }
 725
 726 /*
 727 ====================
 728 Curl_Find
 729
 730 Finds the internal information block for a download given by file name.
 731 ====================
 732 */
 733 static downloadinfo *Curl_Find(const char *filename)
 734 {
 735         downloadinfo *di;
 736         if(!curl_dll)
 737                 return NULL;
 738         for(di = downloads; di; di = di->next)
 739                 if(!strcasecmp(di->filename, filename))
 740                         return di;
 741         return NULL;
 742 }
 743
 744 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 745 {
 746         downloadinfo *di;
 747         if(!curl_dll)
 748                 return;
 749         for(di = downloads; di; )
 750         {
 751                 if(di->callback == callback && di->callback_data == cbdata)
 752                 {
 753                         di->callback = curl_quiet_callback; // do NOT call the callback
 754                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 755                         di = downloads;
 756                 }
 757                 else
 758                         di = di->next;
 759         }
 760 }
 761
 762 /*
 763 ====================
 764 Curl_Begin
 765
 766 Starts a download of a given URL to the file name portion of this URL (or name
 767 if given) in the "dlcache/" folder.
 768 ====================
 769 */
 770 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 771 {
 772         if(!curl_dll)
 773         {
 774                 return false;
 775         }
 776         else
 777         {
 778                 char fn[MAX_OSPATH];
 779                 char urlbuf[1024];
 780                 const char *p, *q;
 781                 size_t length;
 782                 downloadinfo *di;
 783
 784                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 785                 p = strchr(URL, ':');
 786                 if(p)
 787                 {
 788                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 789                         {
 790                                 char addressstring[128];
 791                                 *addressstring = 0;
 792                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 793                                 q = strchr(addressstring, ':');
 794                                 if(!q)
 795                                         q = addressstring + strlen(addressstring);
 796                                 if(*addressstring)
 797                                 {
 798                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 799                                         URL = urlbuf;
 800                                 }
 801                         }
 802                 }
 803
 804                 // Note: This extraction of the file name portion is NOT entirely correct.
 805                 //
 806                 // It does the following:
 807                 //
 808                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 809                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 810                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 811                 //
 812                 // However, I'd like to keep this "buggy" behavior so that PHP script
 813                 // authors can write download scripts without having to enable
 814                 // AcceptPathInfo on Apache. They just have to ensure that their script
 815                 // can be called with such a "fake" path name like
 816                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 817                 //
 818                 // By the way, such PHP scripts should either send the file or a
 819                 // "Location:" redirect; PHP code example:
 820                 //
 821                 //   header("Location: http://www.example.com/");
 822                 //
 823                 // By the way, this will set User-Agent to something like
 824                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 825                 // dp://serverhost:serverport/ so you can filter on this; an example
 826                 // httpd log file line might be:
 827                 //
 828                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 829
 830                 if(!name)
 831                         name = CleanURL(URL);
 832
 833                 if(!buf)
 834                 {
 835                         p = strrchr(name, '/');
 836                         p = p ? (p+1) : name;
 837                         q = strchr(p, '?');
 838                         length = q ? (size_t)(q - p) : strlen(p);
 839                         dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 840
 841                         name = fn; // make it point back
 842
 843                         // already downloading the file?
 844                         {
 845                                 downloadinfo *di = Curl_Find(fn);
 846                                 if(di)
 847                                 {
 848                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url));
 849
 850                                         // however, if it was not for this map yet...
 851                                         if(forthismap && !di->forthismap)
 852                                         {
 853                                                 di->forthismap = true;
 854                                                 // this "fakes" a download attempt so the client will wait for
 855                                                 // the download to finish and then reconnect
 856                                                 ++numdownloads_added;
 857                                         }
 858
 859                                         return false;
 860                                 }
 861                         }
 862
 863                         if(ispak && FS_FileExists(fn))
 864                         {
 865                                 qboolean already_loaded;
 866                                 if(FS_AddPack(fn, &already_loaded, true))
 867                                 {
 868                                         Con_DPrintf("%s already exists, not downloading!\n", fn);
 869                                         if(already_loaded)
 870                                                 Con_DPrintf("(pak was already loaded)\n");
 871                                         else
 872                                         {
 873                                                 if(forthismap)
 874                                                 {
 875                                                         ++numdownloads_added;
 876                                                         ++numdownloads_success;
 877                                                 }
 878                                         }
 879
 880                                         return false;
 881                                 }
 882                                 else
 883                                 {
 884                                         qfile_t *f = FS_OpenRealFile(fn, "rb", false);
 885                                         if(f)
 886                                         {
 887                                                 char buf[4] = {0};
 888                                                 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 889
 890                                                 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 891                                                 {
 892                                                         Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 893                                                         FS_Close(f);
 894                                                         f = FS_OpenRealFile(fn, "wb", false);
 895                                                         if(f)
 896                                                                 FS_Close(f);
 897                                                 }
 898                                                 else
 899                                                 {
 900                                                         // OK
 901                                                         FS_Close(f);
 902                                                 }
 903                                         }
 904                                 }
 905                         }
 906                 }
 907
 908                 // if we get here, we actually want to download... so first verify the
 909                 // URL scheme (so one can't read local files using file://)
 910                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
 911                 {
 912                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
 913                         return false;
 914                 }
 915
 916                 if(forthismap)
 917                         ++numdownloads_added;
 918                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 919                 strlcpy(di->filename, name, sizeof(di->filename));
 920                 strlcpy(di->url, URL, sizeof(di->url));
 921                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 922                 di->forthismap = forthismap;
 923                 di->stream = NULL;
 924                 di->startpos = 0;
 925                 di->curle = NULL;
 926                 di->started = false;
 927                 di->ispak = (ispak && !buf);
 928                 di->maxspeed = maxspeed;
 929                 di->bytes_received = 0;
 930                 di->bytes_received_curl = 0;
 931                 di->bytes_sent_curl = 0;
 932                 di->extraheaders = extraheaders;
 933                 di->next = downloads;
 934                 di->prev = NULL;
 935                 if(di->next)
 936                         di->next->prev = di;
 937
 938                 di->buffer = buf;
 939                 di->buffersize = bufsize;
 940                 if(callback == NULL)
 941                 {
 942                         di->callback = curl_default_callback;
 943                         di->callback_data = di;
 944                 }
 945                 else
 946                 {
 947                         di->callback = callback;
 948                         di->callback_data = cbdata;
 949                 }
 950
 951                 if(post_content_type)
 952                 {
 953                         di->post_content_type = post_content_type;
 954                         di->postbuf = postbuf;
 955                         di->postbufsize = postbufsize;
 956                 }
 957                 else
 958                 {
 959                         di->post_content_type = NULL;
 960                         di->postbuf = NULL;
 961                         di->postbufsize = 0;
 962                 }
 963
 964                 downloads = di;
 965                 return true;
 966         }
 967 }
 968
 969 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
 970 {
 971         return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
 972 }
 973 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 974 {
 975         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
 976 }
 977 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 978 {
 979         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
 980 }
 981
 982 /*
 983 ====================
 984 Curl_Run
 985
 986 call this regularily as this will always download as much as possible without
 987 blocking.
 988 ====================
 989 */
 990 void Curl_Run(void)
 991 {
 992         double maxspeed;
 993         downloadinfo *di;
 994
 995         noclear = FALSE;
 996
 997         if(!cl_curl_enabled.integer)
 998                 return;
 999
1000         if(!curl_dll)
1001                 return;
1002
1003         Curl_CheckCommandWhenDone();
1004
1005         if(!downloads)
1006                 return;
1007
1008         if(realtime < curltime) // throttle
1009                 return;
1010
1011         {
1012                 int remaining;
1013                 CURLMcode mc;
1014
1015                 do
1016                 {
1017                         mc = qcurl_multi_perform(curlm, &remaining);
1018                 }
1019                 while(mc == CURLM_CALL_MULTI_PERFORM);
1020
1021                 for(di = downloads; di; di = di->next)
1022                 {
1023                         double b = 0;
1024                         if(di->curle)
1025                         {
1026                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1027                                 bytes_sent += (b - di->bytes_sent_curl);
1028                                 di->bytes_sent_curl = b;
1029                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1030                                 bytes_sent += (b - di->bytes_received_curl);
1031                                 di->bytes_received_curl = b;
1032                         }
1033                 }
1034
1035                 for(;;)
1036                 {
1037                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1038                         if(!msg)
1039                                 break;
1040                         if(msg->msg == CURLMSG_DONE)
1041                         {
1042                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1043                                 CURLcode result;
1044                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1045                                 result = msg->data.result;
1046                                 if(result)
1047                                 {
1048                                         failed = CURL_DOWNLOAD_FAILED;
1049                                 }
1050                                 else
1051                                 {
1052                                         long code;
1053                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1054                                         switch(code / 100)
1055                                         {
1056                                                 case 4: // e.g. 404?
1057                                                 case 5: // e.g. 500?
1058                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1059                                                         result = (CURLcode) code;
1060                                                         break;
1061                                         }
1062                                 }
1063
1064                                 Curl_EndDownload(di, failed, result);
1065                         }
1066                 }
1067         }
1068
1069         CheckPendingDownloads();
1070
1071         // when will we curl the next time?
1072         // we will wait a bit to ensure our download rate is kept.
1073         // we now know that realtime >= curltime... so set up a new curltime
1074
1075         // use the slowest allowing download to derive the maxspeed... this CAN
1076         // be done better, but maybe later
1077         maxspeed = cl_curl_maxspeed.value;
1078         for(di = downloads; di; di = di->next)
1079                 if(di->maxspeed > 0)
1080                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1081                                 maxspeed = di->maxspeed;
1082
1083         if(maxspeed > 0)
1084         {
1085                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1086                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1087                 bytes_sent = 0;
1088                 bytes_received = 0;
1089         }
1090         else
1091                 curltime = realtime;
1092 }
1093
1094 /*
1095 ====================
1096 Curl_CancelAll
1097
1098 Stops ALL downloads.
1099 ====================
1100 */
1101 void Curl_CancelAll(void)
1102 {
1103         if(!curl_dll)
1104                 return;
1105
1106         while(downloads)
1107         {
1108                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1109                 // INVARIANT: downloads will point to the next download after that!
1110         }
1111 }
1112
1113 /*
1114 ====================
1115 Curl_Running
1116
1117 returns true iff there is a download running.
1118 ====================
1119 */
1120 qboolean Curl_Running(void)
1121 {
1122         if(!curl_dll)
1123                 return false;
1124
1125         return downloads != NULL;
1126 }
1127
1128 /*
1129 ====================
1130 Curl_GetDownloadAmount
1131
1132 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1133 for the given download.
1134 ====================
1135 */
1136 static double Curl_GetDownloadAmount(downloadinfo *di)
1137 {
1138         if(!curl_dll)
1139                 return -2;
1140         if(di->curle)
1141         {
1142                 double length;
1143                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1144                 if(length > 0)
1145                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1146                 else
1147                         return 0;
1148         }
1149         else
1150                 return -1;
1151 }
1152
1153 /*
1154 ====================
1155 Curl_GetDownloadSpeed
1156
1157 returns the speed of the given download in bytes per second
1158 ====================
1159 */
1160 static double Curl_GetDownloadSpeed(downloadinfo *di)
1161 {
1162         if(!curl_dll)
1163                 return -2;
1164         if(di->curle)
1165         {
1166                 double speed;
1167                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1168                 return speed;
1169         }
1170         else
1171                 return -1;
1172 }
1173
1174 /*
1175 ====================
1176 Curl_Info_f
1177
1178 prints the download list
1179 ====================
1180 */
1181 // TODO rewrite using Curl_GetDownloadInfo?
1182 static void Curl_Info_f(void)
1183 {
1184         downloadinfo *di;
1185         if(!curl_dll)
1186                 return;
1187         if(Curl_Running())
1188         {
1189                 Con_Print("Currently running downloads:\n");
1190                 for(di = downloads; di; di = di->next)
1191                 {
1192                         double speed, percent;
1193                         Con_Printf("  %s -> %s ",  CleanURL(di->url), di->filename);
1194                         percent = 100.0 * Curl_GetDownloadAmount(di);
1195                         speed = Curl_GetDownloadSpeed(di);
1196                         if(percent >= 0)
1197                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1198                         else
1199                                 Con_Print("(queued)\n");
1200                 }
1201         }
1202         else
1203         {
1204                 Con_Print("No downloads running.\n");
1205         }
1206 }
1207
1208 /*
1209 ====================
1210 Curl_Curl_f
1211
1212 implements the "curl" console command
1213
1214 curl --info
1215 curl --cancel
1216 curl --cancel filename
1217 curl url
1218
1219 For internal use:
1220
1221 curl [--pak] [--forthismap] [--for filename filename...] url
1222         --pak: after downloading, load the package into the virtual file system
1223         --for filename...: only download of at least one of the named files is missing
1224         --forthismap: don't reconnect on failure
1225
1226 curl --clear_autodownload
1227         clears the download success/failure counters
1228
1229 curl --finish_autodownload
1230         if at least one download has been started, disconnect and drop to the menu
1231         once the last download completes successfully, reconnect to the current server
1232 ====================
1233 */
1234 void Curl_Curl_f(void)
1235 {
1236         double maxspeed = 0;
1237         int i;
1238         int end;
1239         qboolean pak = false;
1240         qboolean forthismap = false;
1241         const char *url;
1242         const char *name = 0;
1243
1244         if(!curl_dll)
1245         {
1246                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1247                 return;
1248         }
1249
1250         if(!cl_curl_enabled.integer)
1251         {
1252                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1253                 return;
1254         }
1255
1256         if(Cmd_Argc() < 2)
1257         {
1258                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1259                 return;
1260         }
1261
1262         url = Cmd_Argv(Cmd_Argc() - 1);
1263         end = Cmd_Argc();
1264
1265         for(i = 1; i != end; ++i)
1266         {
1267                 const char *a = Cmd_Argv(i);
1268                 if(!strcmp(a, "--info"))
1269                 {
1270                         Curl_Info_f();
1271                         return;
1272                 }
1273                 else if(!strcmp(a, "--cancel"))
1274                 {
1275                         if(i == end - 1) // last argument
1276                                 Curl_CancelAll();
1277                         else
1278                         {
1279                                 downloadinfo *di = Curl_Find(url);
1280                                 if(di)
1281                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1282                                 else
1283                                         Con_Print("download not found\n");
1284                         }
1285                         return;
1286                 }
1287                 else if(!strcmp(a, "--pak"))
1288                 {
1289                         pak = true;
1290                 }
1291                 else if(!strcmp(a, "--for")) // must be last option
1292                 {
1293                         for(i = i + 1; i != end - 1; ++i)
1294                         {
1295                                 if(!FS_FileExists(Cmd_Argv(i)))
1296                                         goto needthefile; // why can't I have a "double break"?
1297                         }
1298                         // if we get here, we have all the files...
1299                         return;
1300                 }
1301                 else if(!strcmp(a, "--forthismap"))
1302                 {
1303                         forthismap = true;
1304                 }
1305                 else if(!strcmp(a, "--as"))
1306                 {
1307                         if(i < end - 1)
1308                         {
1309                                 ++i;
1310                                 name = Cmd_Argv(i);
1311                         }
1312                 }
1313                 else if(!strcmp(a, "--clear_autodownload"))
1314                 {
1315                         // mark all running downloads as "not for this map", so if they
1316                         // fail, it does not matter
1317                         Curl_Clear_forthismap();
1318                         return;
1319                 }
1320                 else if(!strcmp(a, "--finish_autodownload"))
1321                 {
1322                         if(numdownloads_added)
1323                         {
1324                                 char donecommand[256];
1325                                 if(cls.netcon)
1326                                 {
1327                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1328                                         {
1329                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1330                                                 Curl_CommandWhenDone(donecommand);
1331                                                 noclear = TRUE;
1332                                                 CL_Disconnect();
1333                                                 noclear = FALSE;
1334                                                 Curl_CheckCommandWhenDone();
1335                                         }
1336                                         else
1337                                                 Curl_Register_predownload();
1338                                 }
1339                         }
1340                         return;
1341                 }
1342                 else if(!strncmp(a, "--maxspeed=", 11))
1343                 {
1344                         maxspeed = atof(a + 11);
1345                 }
1346                 else if(*a == '-')
1347                 {
1348                         Con_Printf("curl: invalid option %s\n", a);
1349                         // but we ignore the option
1350                 }
1351         }
1352
1353 needthefile:
1354         Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1355 }
1356
1357 /*
1358 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1359 {
1360         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1361         Z_Free(buffer);
1362 }
1363
1364 void Curl_CurlCat_f(void)
1365 {
1366         unsigned char *buf;
1367         const char *url = Cmd_Argv(1);
1368         buf = Z_Malloc(16384);
1369         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1370 }
1371 */
1372
1373 /*
1374 ====================
1375 Curl_Init_Commands
1376
1377 loads the commands and cvars this library uses
1378 ====================
1379 */
1380 void Curl_Init_Commands(void)
1381 {
1382         Cvar_RegisterVariable (&cl_curl_enabled);
1383         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1384         Cvar_RegisterVariable (&cl_curl_maxspeed);
1385         Cvar_RegisterVariable (&sv_curl_defaulturl);
1386         Cvar_RegisterVariable (&sv_curl_serverpackages);
1387         Cvar_RegisterVariable (&sv_curl_maxspeed);
1388         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1389         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1390 }
1391
1392 /*
1393 ====================
1394 Curl_GetDownloadInfo
1395
1396 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1397 The number of elements in the array is returned in int *nDownloads.
1398 const char **additional_info may be set to a string of additional user
1399 information, or to NULL if no such display shall occur. The returned
1400 array must be freed later using Z_Free.
1401 ====================
1402 */
1403 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1404 {
1405         int i;
1406         downloadinfo *di;
1407         Curl_downloadinfo_t *downinfo;
1408         static char addinfo[128];
1409
1410         if(!curl_dll)
1411         {
1412                 *nDownloads = 0;
1413                 if(additional_info)
1414                         *additional_info = NULL;
1415                 return NULL;
1416         }
1417
1418         i = 0;
1419         for(di = downloads; di; di = di->next)
1420                 ++i;
1421
1422         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1423         i = 0;
1424         for(di = downloads; di; di = di->next)
1425         {
1426                 // do not show infobars for background downloads
1427                 if(developer.integer <= 0)
1428                         if(di->buffer)
1429                                 continue;
1430                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1431                 if(di->curle)
1432                 {
1433                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1434                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1435                         downinfo[i].queued = false;
1436                 }
1437                 else
1438                 {
1439                         downinfo[i].queued = true;
1440                 }
1441                 ++i;
1442         }
1443
1444         if(additional_info)
1445         {
1446                 // TODO: can I clear command_when_done as soon as the first download fails?
1447                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1448                 {
1449                         if(!strncmp(command_when_done, "connect ", 8))
1450                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1451                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1452                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1453                         else
1454                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1455                         *additional_info = addinfo;
1456                 }
1457                 else
1458                         *additional_info = NULL;
1459         }
1460
1461         *nDownloads = i;
1462         return downinfo;
1463 }
1464
1465
1466 /*
1467 ====================
1468 Curl_FindPackURL
1469
1470 finds the URL where to find a given package.
1471
1472 For this, it reads a file "curl_urls.txt" of the following format:
1473
1474         data*.pk3       -
1475         revdm*.pk3      http://revdm/downloads/are/here/
1476         *                       http://any/other/stuff/is/here/
1477
1478 The URLs should end in /. If not, downloads will still work, but the cached files
1479 can't be just put into the data directory with the same download configuration
1480 (you might want to do this if you want to tag downloaded files from your
1481 server, but you should not). "-" means "don't download".
1482
1483 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1484 location instead.
1485
1486 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1487 this file for obvious reasons.
1488 ====================
1489 */
1490 static const char *Curl_FindPackURL(const char *filename)
1491 {
1492         static char foundurl[1024];
1493         fs_offset_t filesize;
1494         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1495         if(buf && filesize)
1496         {
1497                 // read lines of format "pattern url"
1498                 char *p = buf;
1499                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1500                 qboolean eof = false;
1501
1502                 pattern = p;
1503                 while(!eof)
1504                 {
1505                         switch(*p)
1506                         {
1507                                 case 0:
1508                                         eof = true;
1509                                         // fallthrough
1510                                 case '\n':
1511                                 case '\r':
1512                                         if(pattern && url && patternend)
1513                                         {
1514                                                 if(!urlend)
1515                                                         urlend = p;
1516                                                 *patternend = 0;
1517                                                 *urlend = 0;
1518                                                 if(matchpattern(filename, pattern, true))
1519                                                 {
1520                                                         strlcpy(foundurl, url, sizeof(foundurl));
1521                                                         Z_Free(buf);
1522                                                         return foundurl;
1523                                                 }
1524                                         }
1525                                         pattern = NULL;
1526                                         patternend = NULL;
1527                                         url = NULL;
1528                                         urlend = NULL;
1529                                         break;
1530                                 case ' ':
1531                                 case '\t':
1532                                         if(pattern && !patternend)
1533                                                 patternend = p;
1534                                         else if(url && !urlend)
1535                                                 urlend = p;
1536                                         break;
1537                                 default:
1538                                         if(!pattern)
1539                                                 pattern = p;
1540                                         else if(pattern && patternend && !url)
1541                                                 url = p;
1542                                         break;
1543                         }
1544                         ++p;
1545                 }
1546         }
1547         if(buf)
1548                 Z_Free(buf);
1549         return sv_curl_defaulturl.string;
1550 }
1551
1552 typedef struct requirement_s
1553 {
1554         struct requirement_s *next;
1555         char filename[MAX_OSPATH];
1556 }
1557 requirement;
1558 static requirement *requirements = NULL;
1559
1560
1561 /*
1562 ====================
1563 Curl_RequireFile
1564
1565 Adds the given file to the list of requirements.
1566 ====================
1567 */
1568 void Curl_RequireFile(const char *filename)
1569 {
1570         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1571         req->next = requirements;
1572         strlcpy(req->filename, filename, sizeof(req->filename));
1573         requirements = req;
1574 }
1575
1576 /*
1577 ====================
1578 Curl_ClearRequirements
1579
1580 Clears the list of required files for playing on the current map.
1581 This should be called at every map change.
1582 ====================
1583 */
1584 void Curl_ClearRequirements(void)
1585 {
1586         while(requirements)
1587         {
1588                 requirement *req = requirements;
1589                 requirements = requirements->next;
1590                 Z_Free(req);
1591         }
1592 }
1593
1594 /*
1595 ====================
1596 Curl_SendRequirements
1597
1598 Makes the current host_clients download all files he needs.
1599 This is done by sending him the following console commands:
1600
1601         curl --clear_autodownload
1602         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1603         curl --finish_autodownload
1604 ====================
1605 */
1606 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1607 {
1608         const char *p;
1609         const char *thispack = FS_WhichPack(filename);
1610         const char *packurl;
1611
1612         if(!thispack)
1613                 return false;
1614
1615         p = strrchr(thispack, '/');
1616         if(p)
1617                 thispack = p + 1;
1618
1619         packurl = Curl_FindPackURL(thispack);
1620
1621         if(packurl && *packurl && strcmp(packurl, "-"))
1622         {
1623                 if(!foundone)
1624                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1625
1626                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1627                 strlcat(sendbuffer, thispack, sendbuffer_len);
1628                 if(sv_curl_maxspeed.value > 0)
1629                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1630                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1631                 strlcat(sendbuffer, filename, sendbuffer_len);
1632                 strlcat(sendbuffer, " ", sendbuffer_len);
1633                 strlcat(sendbuffer, packurl, sendbuffer_len);
1634                 strlcat(sendbuffer, thispack, sendbuffer_len);
1635                 strlcat(sendbuffer, "\n", sendbuffer_len);
1636
1637                 return true;
1638         }
1639
1640         return false;
1641 }
1642 void Curl_SendRequirements(void)
1643 {
1644         // for each requirement, find the pack name
1645         char sendbuffer[4096] = "";
1646         requirement *req;
1647         qboolean foundone = false;
1648         const char *p;
1649
1650         for(req = requirements; req; req = req->next)
1651                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1652
1653         p = sv_curl_serverpackages.string;
1654         while(COM_ParseToken_Simple(&p, false, false))
1655                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1656
1657         if(foundone)
1658                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1659
1660         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1661                 Host_ClientCommands("%s", sendbuffer);
1662         else
1663                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1664 }