libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  10 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  11
  12 /*
  13 =================================================================
  14
  15   Minimal set of definitions from libcurl
  16
  17   WARNING: for a matter of simplicity, several pointer types are
  18   casted to "void*", and most enumerated values are not included
  19
  20 =================================================================
  21 */
  22
  23 typedef struct CURL_s CURL;
  24 typedef struct CURLM_s CURLM;
  25 typedef struct curl_slist curl_slist;
  26 typedef enum
  27 {
  28         CURLE_OK = 0
  29 }
  30 CURLcode;
  31 typedef enum
  32 {
  33         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  34         CURLM_OK = 0
  35 }
  36 CURLMcode;
  37 #define CURL_GLOBAL_NOTHING 0
  38 #define CURL_GLOBAL_SSL 1
  39 #define CURL_GLOBAL_WIN32 2
  40 #define CURLOPTTYPE_LONG          0
  41 #define CURLOPTTYPE_OBJECTPOINT   10000
  42 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  43 #define CURLOPTTYPE_OFF_T         30000
  44 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  45 typedef enum
  46 {
  47         CINIT(WRITEDATA, OBJECTPOINT, 1),
  48         CINIT(URL,  OBJECTPOINT, 2),
  49         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  50         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  51         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  52         CINIT(REFERER, OBJECTPOINT, 16),
  53         CINIT(USERAGENT, OBJECTPOINT, 18),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56         CINIT(RESUME_FROM, LONG, 21),
  57         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  58         CINIT(POST, LONG, 47),         /* HTTP POST method */
  59         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  60         CINIT(POSTFIELDSIZE, LONG, 60),
  61         CINIT(PRIVATE, OBJECTPOINT, 103),
  62         CINIT(PROTOCOLS, LONG, 181),
  63         CINIT(REDIR_PROTOCOLS, LONG, 182)
  64 }
  65 CURLoption;
  66 #define CURLPROTO_HTTP   (1<<0)
  67 #define CURLPROTO_HTTPS  (1<<1)
  68 #define CURLPROTO_FTP    (1<<2)
  69 typedef enum
  70 {
  71         CURLINFO_TEXT = 0,
  72         CURLINFO_HEADER_IN,    /* 1 */
  73         CURLINFO_HEADER_OUT,   /* 2 */
  74         CURLINFO_DATA_IN,      /* 3 */
  75         CURLINFO_DATA_OUT,     /* 4 */
  76         CURLINFO_SSL_DATA_IN,  /* 5 */
  77         CURLINFO_SSL_DATA_OUT, /* 6 */
  78         CURLINFO_END
  79 }
  80 curl_infotype;
  81 #define CURLINFO_STRING   0x100000
  82 #define CURLINFO_LONG     0x200000
  83 #define CURLINFO_DOUBLE   0x300000
  84 #define CURLINFO_SLIST    0x400000
  85 #define CURLINFO_MASK     0x0fffff
  86 #define CURLINFO_TYPEMASK 0xf00000
  87 typedef enum
  88 {
  89         CURLINFO_NONE, /* first, never use this */
  90         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  91         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  92         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  93         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  94         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  95         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  96         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  97         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  98         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  99         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 100         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 101         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 102         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 103         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 104         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 105         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 106         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 107         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 108         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 109         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 110         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 111         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 112         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 113         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 114         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 115         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 116         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 117 }
 118 CURLINFO;
 119
 120 typedef enum
 121 {
 122         CURLMSG_NONE, /* first, not used */
 123         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 124                                          the CURLcode of the transfer */
 125         CURLMSG_LAST
 126 }
 127 CURLMSG;
 128 typedef struct
 129 {
 130         CURLMSG msg;       /* what this message means */
 131         CURL *easy_handle; /* the handle it concerns */
 132         union
 133         {
 134                 void *whatever;    /* message-specific data */
 135                 CURLcode result;   /* return code for transfer */
 136         }
 137         data;
 138 }
 139 CURLMsg;
 140
 141 static void (*qcurl_global_init) (long flags);
 142 static void (*qcurl_global_cleanup) (void);
 143
 144 static CURL * (*qcurl_easy_init) (void);
 145 static void (*qcurl_easy_cleanup) (CURL *handle);
 146 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 147 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 148 static const char * (*qcurl_easy_strerror) (CURLcode);
 149
 150 static CURLM * (*qcurl_multi_init) (void);
 151 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 152 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 153 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 154 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 155 static void (*qcurl_multi_cleanup) (CURLM *);
 156 static const char * (*qcurl_multi_strerror) (CURLcode);
 157 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 158 static void (*qcurl_slist_free_all) (curl_slist *list);
 159
 160 static dllfunction_t curlfuncs[] =
 161 {
 162         {"curl_global_init",            (void **) &qcurl_global_init},
 163         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 164         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 165         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 166         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 167         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 168         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 169         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 170         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 171         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 172         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 173         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 174         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 175         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 176         {"curl_slist_append",           (void **) &qcurl_slist_append},
 177         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 178         {NULL, NULL}
 179 };
 180
 181 // Handle for CURL DLL
 182 static dllhandle_t curl_dll = NULL;
 183 // will be checked at many places to find out if qcurl calls are allowed
 184
 185 typedef struct downloadinfo_s
 186 {
 187         char filename[MAX_OSPATH];
 188         char url[1024];
 189         char referer[256];
 190         qfile_t *stream;
 191         fs_offset_t startpos;
 192         CURL *curle;
 193         qboolean started;
 194         qboolean ispak;
 195         unsigned long bytes_received; // for buffer
 196         double bytes_received_curl; // for throttling
 197         double bytes_sent_curl; // for throttling
 198         struct downloadinfo_s *next, *prev;
 199         qboolean forthismap;
 200         double maxspeed;
 201         curl_slist *slist; // http headers
 202
 203         unsigned char *buffer;
 204         size_t buffersize;
 205         curl_callback_t callback;
 206         void *callback_data;
 207
 208         const unsigned char *postbuf;
 209         size_t postbufsize;
 210         const char *post_content_type;
 211 }
 212 downloadinfo;
 213 static downloadinfo *downloads = NULL;
 214 static int numdownloads = 0;
 215
 216 static qboolean noclear = FALSE;
 217
 218 static int numdownloads_fail = 0;
 219 static int numdownloads_success = 0;
 220 static int numdownloads_added = 0;
 221 static char command_when_done[256] = "";
 222 static char command_when_error[256] = "";
 223
 224 /*
 225 ====================
 226 Curl_CommandWhenDone
 227
 228 Sets the command which is to be executed when the last download completes AND
 229 all downloads since last server connect ended with a successful status.
 230 Setting the command to NULL clears it.
 231 ====================
 232 */
 233 void Curl_CommandWhenDone(const char *cmd)
 234 {
 235         if(!curl_dll)
 236                 return;
 237         if(cmd)
 238                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 239         else
 240                 *command_when_done = 0;
 241 }
 242
 243 /*
 244 FIXME
 245 Do not use yet. Not complete.
 246 Problem: what counts as an error?
 247 */
 248
 249 void Curl_CommandWhenError(const char *cmd)
 250 {
 251         if(!curl_dll)
 252                 return;
 253         if(cmd)
 254                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 255         else
 256                 *command_when_error = 0;
 257 }
 258
 259 /*
 260 ====================
 261 Curl_Clear_forthismap
 262
 263 Clears the "will disconnect on failure" flags.
 264 ====================
 265 */
 266 void Curl_Clear_forthismap(void)
 267 {
 268         downloadinfo *di;
 269         if(noclear)
 270                 return;
 271         for(di = downloads; di; di = di->next)
 272                 di->forthismap = false;
 273         Curl_CommandWhenError(NULL);
 274         Curl_CommandWhenDone(NULL);
 275         numdownloads_fail = 0;
 276         numdownloads_success = 0;
 277         numdownloads_added = 0;
 278 }
 279
 280 /*
 281 ====================
 282 Curl_Have_forthismap
 283
 284 Returns true if a download needed for the current game is running.
 285 ====================
 286 */
 287 qboolean Curl_Have_forthismap(void)
 288 {
 289         return numdownloads_added != 0;
 290 }
 291
 292 void Curl_Register_predownload(void)
 293 {
 294         Curl_CommandWhenDone("cl_begindownloads");
 295         Curl_CommandWhenError("cl_begindownloads");
 296 }
 297
 298 /*
 299 ====================
 300 Curl_CheckCommandWhenDone
 301
 302 Checks if a "done command" is to be executed.
 303 All downloads finished, at least one success since connect, no single failure
 304 -> execute the command.
 305 */
 306 static void Curl_CheckCommandWhenDone(void)
 307 {
 308         if(!curl_dll)
 309                 return;
 310         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 311         {
 312                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 313                 Cbuf_AddText("\n");
 314                 Cbuf_AddText(command_when_done);
 315                 Cbuf_AddText("\n");
 316                 Curl_Clear_forthismap();
 317         }
 318         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 319         {
 320                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 321                 Cbuf_AddText("\n");
 322                 Cbuf_AddText(command_when_error);
 323                 Cbuf_AddText("\n");
 324                 Curl_Clear_forthismap();
 325         }
 326 }
 327
 328 /*
 329 ====================
 330 CURL_CloseLibrary
 331
 332 Load the cURL DLL
 333 ====================
 334 */
 335 static qboolean CURL_OpenLibrary (void)
 336 {
 337         const char* dllnames [] =
 338         {
 339 #if defined(WIN32)
 340                 "libcurl-4.dll",
 341                 "libcurl-3.dll",
 342 #elif defined(MACOSX)
 343                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 344                 "libcurl.3.dylib", // Mac OS X Tiger
 345                 "libcurl.2.dylib", // Mac OS X Panther
 346 #else
 347                 "libcurl.so.4",
 348                 "libcurl.so.3",
 349                 "libcurl.so", // FreeBSD
 350 #endif
 351                 NULL
 352         };
 353
 354         // Already loaded?
 355         if (curl_dll)
 356                 return true;
 357
 358         // Load the DLL
 359         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 360 }
 361
 362
 363 /*
 364 ====================
 365 CURL_CloseLibrary
 366
 367 Unload the cURL DLL
 368 ====================
 369 */
 370 static void CURL_CloseLibrary (void)
 371 {
 372         Sys_UnloadLibrary (&curl_dll);
 373 }
 374
 375
 376 static CURLM *curlm = NULL;
 377 static double bytes_received = 0; // used for bandwidth throttling
 378 static double bytes_sent = 0; // used for bandwidth throttling
 379 static double curltime = 0;
 380
 381 /*
 382 ====================
 383 CURL_fwrite
 384
 385 fwrite-compatible function that writes the data to a file. libcurl can call
 386 this.
 387 ====================
 388 */
 389 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 390 {
 391         fs_offset_t ret = -1;
 392         size_t bytes = size * nmemb;
 393         downloadinfo *di = (downloadinfo *) vdi;
 394
 395         if(di->buffer)
 396         {
 397                 if(di->bytes_received + bytes <= di->buffersize)
 398                 {
 399                         memcpy(di->buffer + di->bytes_received, data, bytes);
 400                         ret = bytes;
 401                 }
 402                 // otherwise: buffer overrun, ret stays -1
 403         }
 404
 405         if(di->stream)
 406         {
 407                 ret = FS_Write(di->stream, data, bytes);
 408         }
 409
 410         di->bytes_received += bytes;
 411
 412         return ret; // why not ret / nmemb?
 413 }
 414
 415 typedef enum
 416 {
 417         CURL_DOWNLOAD_SUCCESS = 0,
 418         CURL_DOWNLOAD_FAILED,
 419         CURL_DOWNLOAD_ABORTED,
 420         CURL_DOWNLOAD_SERVERERROR
 421 }
 422 CurlStatus;
 423
 424 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 425 {
 426         downloadinfo *di = (downloadinfo *) cbdata;
 427         switch(status)
 428         {
 429                 case CURLCBSTATUS_OK:
 430                         Con_DPrintf("Download of %s: OK\n", di->filename);
 431                         break;
 432                 case CURLCBSTATUS_FAILED:
 433                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 434                         break;
 435                 case CURLCBSTATUS_ABORTED:
 436                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 437                         break;
 438                 case CURLCBSTATUS_SERVERERROR:
 439                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 440                         break;
 441                 case CURLCBSTATUS_UNKNOWN:
 442                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 443                         break;
 444                 default:
 445                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 446                         break;
 447         }
 448 }
 449
 450 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 451 {
 452         curl_default_callback(status, length_received, buffer, cbdata);
 453 }
 454
 455 /*
 456 ====================
 457 Curl_EndDownload
 458
 459 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 460 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 461 code from libcurl, or 0, if another error has occurred.
 462 ====================
 463 */
 464 static qboolean Curl_Begin(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 465 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 466 {
 467         qboolean ok = false;
 468         if(!curl_dll)
 469                 return;
 470         switch(status)
 471         {
 472                 case CURL_DOWNLOAD_SUCCESS:
 473                         ok = true;
 474                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 475                         break;
 476                 case CURL_DOWNLOAD_FAILED:
 477                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 478                         break;
 479                 case CURL_DOWNLOAD_ABORTED:
 480                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 481                         break;
 482                 case CURL_DOWNLOAD_SERVERERROR:
 483                         // reopen to enforce it to have zero bytes again
 484                         if(di->stream)
 485                         {
 486                                 FS_Close(di->stream);
 487                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 488                         }
 489
 490                         if(di->callback)
 491                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 492                         break;
 493                 default:
 494                         if(di->callback)
 495                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 496                         break;
 497         }
 498
 499         if(di->curle)
 500         {
 501                 qcurl_multi_remove_handle(curlm, di->curle);
 502                 qcurl_easy_cleanup(di->curle);
 503                 if(di->slist)
 504                         qcurl_slist_free_all(di->slist);
 505         }
 506
 507         if(!di->callback && ok && !di->bytes_received)
 508         {
 509                 Con_Printf("ERROR: empty file\n");
 510                 ok = false;
 511         }
 512
 513         if(di->stream)
 514                 FS_Close(di->stream);
 515
 516         if(ok && di->ispak)
 517         {
 518                 ok = FS_AddPack(di->filename, NULL, true);
 519                 if(!ok)
 520                 {
 521                         // pack loading failed?
 522                         // this is critical
 523                         // better clear the file again...
 524                         di->stream = FS_OpenRealFile(di->filename, "wb", false);
 525                         FS_Close(di->stream);
 526
 527                         if(di->startpos && !di->callback && !di->post_content_type)
 528                         {
 529                                 // this was a resume?
 530                                 // then try to redownload it without reporting the error
 531                                 Curl_Begin(di->url, di->maxspeed, di->filename, di->ispak, di->forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
 532                                 di->forthismap = false; // don't count the error
 533                         }
 534                 }
 535         }
 536
 537         if(di->prev)
 538                 di->prev->next = di->next;
 539         else
 540                 downloads = di->next;
 541         if(di->next)
 542                 di->next->prev = di->prev;
 543
 544         --numdownloads;
 545         if(di->forthismap)
 546         {
 547                 if(ok)
 548                         ++numdownloads_success;
 549                 else
 550                         ++numdownloads_fail;
 551         }
 552         Z_Free(di);
 553 }
 554
 555 /*
 556 ====================
 557 CleanURL
 558
 559 Returns a "cleaned up" URL for display (to strip login data)
 560 ====================
 561 */
 562 static const char *CleanURL(const char *url)
 563 {
 564         static char urlbuf[1024];
 565         const char *p, *q, *r;
 566
 567         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 568         p = strstr(url, "://");
 569         if(p)
 570         {
 571                 q = strchr(p + 3, '@');
 572                 if(q)
 573                 {
 574                         r = strchr(p + 3, '/');
 575                         if(!r || q < r)
 576                         {
 577                                 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s%s", (int)(p - url + 3), url, q + 1);
 578                                 return urlbuf;
 579                         }
 580                 }
 581         }
 582
 583         return url;
 584 }
 585
 586 /*
 587 ====================
 588 CheckPendingDownloads
 589
 590 checks if there are free download slots to start new downloads in.
 591 To not start too many downloads at once, only one download is added at a time,
 592 up to a maximum number of cl_curl_maxdownloads are running.
 593 ====================
 594 */
 595 static void CheckPendingDownloads(void)
 596 {
 597         if(!curl_dll)
 598                 return;
 599         if(numdownloads < cl_curl_maxdownloads.integer)
 600         {
 601                 downloadinfo *di;
 602                 for(di = downloads; di; di = di->next)
 603                 {
 604                         if(!di->started)
 605                         {
 606                                 if(!di->buffer)
 607                                 {
 608                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url), di->filename);
 609
 610                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 611                                         if(!di->stream)
 612                                         {
 613                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 614                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 615                                                 return;
 616                                         }
 617                                         FS_Seek(di->stream, 0, SEEK_END);
 618                                         di->startpos = FS_Tell(di->stream);
 619
 620                                         if(di->startpos > 0)
 621                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 622                                         Con_Print("...\n");
 623                                 }
 624                                 else
 625                                 {
 626                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url));
 627                                         di->startpos = 0;
 628                                 }
 629
 630                                 di->curle = qcurl_easy_init();
 631                                 di->slist = NULL;
 632                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 633                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 634                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 635                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 636                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 637                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 638                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 639                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 640                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 641                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 642                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 643                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 644                                 {
 645                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 646                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 647                                 }
 648                                 if(di->post_content_type)
 649                                 {
 650                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 651                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 652                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 653                                         di->slist = qcurl_slist_append(di->slist, va("Content-Type: %s", di->post_content_type));
 654                                 }
 655                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 656
 657                                 qcurl_multi_add_handle(curlm, di->curle);
 658                                 di->started = true;
 659                                 ++numdownloads;
 660                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 661                                         break;
 662                         }
 663                 }
 664         }
 665 }
 666
 667 /*
 668 ====================
 669 Curl_Init
 670
 671 this function MUST be called before using anything else in this file.
 672 On Win32, this must be called AFTER WSAStartup has been done!
 673 ====================
 674 */
 675 void Curl_Init(void)
 676 {
 677         CURL_OpenLibrary();
 678         if(!curl_dll)
 679                 return;
 680         qcurl_global_init(CURL_GLOBAL_NOTHING);
 681         curlm = qcurl_multi_init();
 682 }
 683
 684 /*
 685 ====================
 686 Curl_Shutdown
 687
 688 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 689 ====================
 690 */
 691 void Curl_ClearRequirements(void);
 692 void Curl_Shutdown(void)
 693 {
 694         if(!curl_dll)
 695                 return;
 696         Curl_ClearRequirements();
 697         Curl_CancelAll();
 698         CURL_CloseLibrary();
 699         curl_dll = NULL;
 700 }
 701
 702 /*
 703 ====================
 704 Curl_Find
 705
 706 Finds the internal information block for a download given by file name.
 707 ====================
 708 */
 709 static downloadinfo *Curl_Find(const char *filename)
 710 {
 711         downloadinfo *di;
 712         if(!curl_dll)
 713                 return NULL;
 714         for(di = downloads; di; di = di->next)
 715                 if(!strcasecmp(di->filename, filename))
 716                         return di;
 717         return NULL;
 718 }
 719
 720 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 721 {
 722         downloadinfo *di;
 723         if(!curl_dll)
 724                 return;
 725         for(di = downloads; di; )
 726         {
 727                 if(di->callback == callback && di->callback_data == cbdata)
 728                 {
 729                         di->callback = curl_quiet_callback; // do NOT call the callback
 730                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 731                         di = downloads;
 732                 }
 733                 else
 734                         di = di->next;
 735         }
 736 }
 737
 738 /*
 739 ====================
 740 Curl_Begin
 741
 742 Starts a download of a given URL to the file name portion of this URL (or name
 743 if given) in the "dlcache/" folder.
 744 ====================
 745 */
 746 static qboolean Curl_Begin(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 747 {
 748         if(!curl_dll)
 749         {
 750                 return false;
 751         }
 752         else
 753         {
 754                 char fn[MAX_OSPATH];
 755                 char urlbuf[1024];
 756                 const char *p, *q;
 757                 size_t length;
 758                 downloadinfo *di;
 759
 760                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 761                 p = strchr(URL, ':');
 762                 if(p)
 763                 {
 764                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 765                         {
 766                                 char addressstring[128];
 767                                 *addressstring = 0;
 768                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 769                                 q = strchr(addressstring, ':');
 770                                 if(!q)
 771                                         q = addressstring + strlen(addressstring);
 772                                 if(*addressstring)
 773                                 {
 774                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 775                                         URL = urlbuf;
 776                                 }
 777                         }
 778                 }
 779
 780                 // Note: This extraction of the file name portion is NOT entirely correct.
 781                 //
 782                 // It does the following:
 783                 //
 784                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 785                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 786                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 787                 //
 788                 // However, I'd like to keep this "buggy" behavior so that PHP script
 789                 // authors can write download scripts without having to enable
 790                 // AcceptPathInfo on Apache. They just have to ensure that their script
 791                 // can be called with such a "fake" path name like
 792                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 793                 //
 794                 // By the way, such PHP scripts should either send the file or a
 795                 // "Location:" redirect; PHP code example:
 796                 //
 797                 //   header("Location: http://www.example.com/");
 798                 //
 799                 // By the way, this will set User-Agent to something like
 800                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 801                 // dp://serverhost:serverport/ so you can filter on this; an example
 802                 // httpd log file line might be:
 803                 //
 804                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 805
 806                 if(!name)
 807                         name = CleanURL(URL);
 808
 809                 if(!buf)
 810                 {
 811                         p = strrchr(name, '/');
 812                         p = p ? (p+1) : name;
 813                         q = strchr(p, '?');
 814                         length = q ? (size_t)(q - p) : strlen(p);
 815                         dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 816
 817                         name = fn; // make it point back
 818
 819                         // already downloading the file?
 820                         {
 821                                 downloadinfo *di = Curl_Find(fn);
 822                                 if(di)
 823                                 {
 824                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url));
 825
 826                                         // however, if it was not for this map yet...
 827                                         if(forthismap && !di->forthismap)
 828                                         {
 829                                                 di->forthismap = true;
 830                                                 // this "fakes" a download attempt so the client will wait for
 831                                                 // the download to finish and then reconnect
 832                                                 ++numdownloads_added;
 833                                         }
 834
 835                                         return false;
 836                                 }
 837                         }
 838
 839                         if(ispak && FS_FileExists(fn))
 840                         {
 841                                 qboolean already_loaded;
 842                                 if(FS_AddPack(fn, &already_loaded, true))
 843                                 {
 844                                         Con_DPrintf("%s already exists, not downloading!\n", fn);
 845                                         if(already_loaded)
 846                                                 Con_DPrintf("(pak was already loaded)\n");
 847                                         else
 848                                         {
 849                                                 if(forthismap)
 850                                                 {
 851                                                         ++numdownloads_added;
 852                                                         ++numdownloads_success;
 853                                                 }
 854                                         }
 855
 856                                         return false;
 857                                 }
 858                                 else
 859                                 {
 860                                         qfile_t *f = FS_OpenRealFile(fn, "rb", false);
 861                                         if(f)
 862                                         {
 863                                                 char buf[4] = {0};
 864                                                 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 865
 866                                                 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 867                                                 {
 868                                                         Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 869                                                         FS_Close(f);
 870                                                         f = FS_OpenRealFile(fn, "wb", false);
 871                                                         if(f)
 872                                                                 FS_Close(f);
 873                                                 }
 874                                                 else
 875                                                 {
 876                                                         // OK
 877                                                         FS_Close(f);
 878                                                 }
 879                                         }
 880                                 }
 881                         }
 882                 }
 883
 884                 // if we get here, we actually want to download... so first verify the
 885                 // URL scheme (so one can't read local files using file://)
 886                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
 887                 {
 888                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
 889                         return false;
 890                 }
 891
 892                 if(forthismap)
 893                         ++numdownloads_added;
 894                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 895                 strlcpy(di->filename, name, sizeof(di->filename));
 896                 strlcpy(di->url, URL, sizeof(di->url));
 897                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 898                 di->forthismap = forthismap;
 899                 di->stream = NULL;
 900                 di->startpos = 0;
 901                 di->curle = NULL;
 902                 di->started = false;
 903                 di->ispak = (ispak && !buf);
 904                 di->maxspeed = maxspeed;
 905                 di->bytes_received = 0;
 906                 di->bytes_received_curl = 0;
 907                 di->bytes_sent_curl = 0;
 908                 di->next = downloads;
 909                 di->prev = NULL;
 910                 if(di->next)
 911                         di->next->prev = di;
 912
 913                 di->buffer = buf;
 914                 di->buffersize = bufsize;
 915                 if(callback == NULL)
 916                 {
 917                         di->callback = curl_default_callback;
 918                         di->callback_data = di;
 919                 }
 920                 else
 921                 {
 922                         di->callback = callback;
 923                         di->callback_data = cbdata;
 924                 }
 925
 926                 if(post_content_type)
 927                 {
 928                         di->post_content_type = post_content_type;
 929                         di->postbuf = postbuf;
 930                         di->postbufsize = postbufsize;
 931                 }
 932                 else
 933                 {
 934                         di->post_content_type = NULL;
 935                         di->postbuf = NULL;
 936                         di->postbufsize = 0;
 937                 }
 938
 939                 downloads = di;
 940                 return true;
 941         }
 942 }
 943
 944 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
 945 {
 946         return Curl_Begin(URL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
 947 }
 948 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 949 {
 950         return Curl_Begin(URL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
 951 }
 952 qboolean Curl_Begin_ToMemory_POST(const char *URL, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 953 {
 954         return Curl_Begin(URL, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
 955 }
 956
 957 /*
 958 ====================
 959 Curl_Run
 960
 961 call this regularily as this will always download as much as possible without
 962 blocking.
 963 ====================
 964 */
 965 void Curl_Run(void)
 966 {
 967         double maxspeed;
 968         downloadinfo *di;
 969
 970         noclear = FALSE;
 971
 972         if(!cl_curl_enabled.integer)
 973                 return;
 974
 975         if(!curl_dll)
 976                 return;
 977
 978         Curl_CheckCommandWhenDone();
 979
 980         if(!downloads)
 981                 return;
 982
 983         if(realtime < curltime) // throttle
 984                 return;
 985
 986         {
 987                 int remaining;
 988                 CURLMcode mc;
 989
 990                 do
 991                 {
 992                         mc = qcurl_multi_perform(curlm, &remaining);
 993                 }
 994                 while(mc == CURLM_CALL_MULTI_PERFORM);
 995
 996                 for(di = downloads; di; di = di->next)
 997                 {
 998                         double b = 0;
 999                         qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1000                         bytes_sent += (b - di->bytes_sent_curl);
1001                         di->bytes_sent_curl = b;
1002                         qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1003                         bytes_sent += (b - di->bytes_received_curl);
1004                         di->bytes_received_curl = b;
1005                 }
1006
1007                 for(;;)
1008                 {
1009                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1010                         if(!msg)
1011                                 break;
1012                         if(msg->msg == CURLMSG_DONE)
1013                         {
1014                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1015                                 CURLcode result;
1016                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1017                                 result = msg->data.result;
1018                                 if(result)
1019                                 {
1020                                         failed = CURL_DOWNLOAD_FAILED;
1021                                 }
1022                                 else
1023                                 {
1024                                         long code;
1025                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1026                                         switch(code / 100)
1027                                         {
1028                                                 case 4: // e.g. 404?
1029                                                 case 5: // e.g. 500?
1030                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1031                                                         result = (CURLcode) code;
1032                                                         break;
1033                                         }
1034                                 }
1035
1036                                 Curl_EndDownload(di, failed, result);
1037                         }
1038                 }
1039         }
1040
1041         CheckPendingDownloads();
1042
1043         // when will we curl the next time?
1044         // we will wait a bit to ensure our download rate is kept.
1045         // we now know that realtime >= curltime... so set up a new curltime
1046
1047         // use the slowest allowing download to derive the maxspeed... this CAN
1048         // be done better, but maybe later
1049         maxspeed = cl_curl_maxspeed.value;
1050         for(di = downloads; di; di = di->next)
1051                 if(di->maxspeed > 0)
1052                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1053                                 maxspeed = di->maxspeed;
1054
1055         if(maxspeed > 0)
1056         {
1057                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1058                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1059                 bytes_sent = 0;
1060                 bytes_received = 0;
1061         }
1062         else
1063                 curltime = realtime;
1064 }
1065
1066 /*
1067 ====================
1068 Curl_CancelAll
1069
1070 Stops ALL downloads.
1071 ====================
1072 */
1073 void Curl_CancelAll(void)
1074 {
1075         if(!curl_dll)
1076                 return;
1077
1078         while(downloads)
1079         {
1080                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1081                 // INVARIANT: downloads will point to the next download after that!
1082         }
1083 }
1084
1085 /*
1086 ====================
1087 Curl_Running
1088
1089 returns true iff there is a download running.
1090 ====================
1091 */
1092 qboolean Curl_Running(void)
1093 {
1094         if(!curl_dll)
1095                 return false;
1096
1097         return downloads != NULL;
1098 }
1099
1100 /*
1101 ====================
1102 Curl_GetDownloadAmount
1103
1104 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1105 for the given download.
1106 ====================
1107 */
1108 static double Curl_GetDownloadAmount(downloadinfo *di)
1109 {
1110         if(!curl_dll)
1111                 return -2;
1112         if(di->curle)
1113         {
1114                 double length;
1115                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1116                 if(length > 0)
1117                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1118                 else
1119                         return 0;
1120         }
1121         else
1122                 return -1;
1123 }
1124
1125 /*
1126 ====================
1127 Curl_GetDownloadSpeed
1128
1129 returns the speed of the given download in bytes per second
1130 ====================
1131 */
1132 static double Curl_GetDownloadSpeed(downloadinfo *di)
1133 {
1134         if(!curl_dll)
1135                 return -2;
1136         if(di->curle)
1137         {
1138                 double speed;
1139                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1140                 return speed;
1141         }
1142         else
1143                 return -1;
1144 }
1145
1146 /*
1147 ====================
1148 Curl_Info_f
1149
1150 prints the download list
1151 ====================
1152 */
1153 // TODO rewrite using Curl_GetDownloadInfo?
1154 static void Curl_Info_f(void)
1155 {
1156         downloadinfo *di;
1157         if(!curl_dll)
1158                 return;
1159         if(Curl_Running())
1160         {
1161                 Con_Print("Currently running downloads:\n");
1162                 for(di = downloads; di; di = di->next)
1163                 {
1164                         double speed, percent;
1165                         Con_Printf("  %s -> %s ",  CleanURL(di->url), di->filename);
1166                         percent = 100.0 * Curl_GetDownloadAmount(di);
1167                         speed = Curl_GetDownloadSpeed(di);
1168                         if(percent >= 0)
1169                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1170                         else
1171                                 Con_Print("(queued)\n");
1172                 }
1173         }
1174         else
1175         {
1176                 Con_Print("No downloads running.\n");
1177         }
1178 }
1179
1180 /*
1181 ====================
1182 Curl_Curl_f
1183
1184 implements the "curl" console command
1185
1186 curl --info
1187 curl --cancel
1188 curl --cancel filename
1189 curl url
1190
1191 For internal use:
1192
1193 curl [--pak] [--forthismap] [--for filename filename...] url
1194         --pak: after downloading, load the package into the virtual file system
1195         --for filename...: only download of at least one of the named files is missing
1196         --forthismap: don't reconnect on failure
1197
1198 curl --clear_autodownload
1199         clears the download success/failure counters
1200
1201 curl --finish_autodownload
1202         if at least one download has been started, disconnect and drop to the menu
1203         once the last download completes successfully, reconnect to the current server
1204 ====================
1205 */
1206 void Curl_Curl_f(void)
1207 {
1208         double maxspeed = 0;
1209         int i;
1210         int end;
1211         qboolean pak = false;
1212         qboolean forthismap = false;
1213         const char *url;
1214         const char *name = 0;
1215
1216         if(!curl_dll)
1217         {
1218                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1219                 return;
1220         }
1221
1222         if(!cl_curl_enabled.integer)
1223         {
1224                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1225                 return;
1226         }
1227
1228         if(Cmd_Argc() < 2)
1229         {
1230                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1231                 return;
1232         }
1233
1234         url = Cmd_Argv(Cmd_Argc() - 1);
1235         end = Cmd_Argc();
1236
1237         for(i = 1; i != end; ++i)
1238         {
1239                 const char *a = Cmd_Argv(i);
1240                 if(!strcmp(a, "--info"))
1241                 {
1242                         Curl_Info_f();
1243                         return;
1244                 }
1245                 else if(!strcmp(a, "--cancel"))
1246                 {
1247                         if(i == end - 1) // last argument
1248                                 Curl_CancelAll();
1249                         else
1250                         {
1251                                 downloadinfo *di = Curl_Find(url);
1252                                 if(di)
1253                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1254                                 else
1255                                         Con_Print("download not found\n");
1256                         }
1257                         return;
1258                 }
1259                 else if(!strcmp(a, "--pak"))
1260                 {
1261                         pak = true;
1262                 }
1263                 else if(!strcmp(a, "--for")) // must be last option
1264                 {
1265                         for(i = i + 1; i != end - 1; ++i)
1266                         {
1267                                 if(!FS_FileExists(Cmd_Argv(i)))
1268                                         goto needthefile; // why can't I have a "double break"?
1269                         }
1270                         // if we get here, we have all the files...
1271                         return;
1272                 }
1273                 else if(!strcmp(a, "--forthismap"))
1274                 {
1275                         forthismap = true;
1276                 }
1277                 else if(!strcmp(a, "--as"))
1278                 {
1279                         if(i < end - 1)
1280                         {
1281                                 ++i;
1282                                 name = Cmd_Argv(i);
1283                         }
1284                 }
1285                 else if(!strcmp(a, "--clear_autodownload"))
1286                 {
1287                         // mark all running downloads as "not for this map", so if they
1288                         // fail, it does not matter
1289                         Curl_Clear_forthismap();
1290                         return;
1291                 }
1292                 else if(!strcmp(a, "--finish_autodownload"))
1293                 {
1294                         if(numdownloads_added)
1295                         {
1296                                 char donecommand[256];
1297                                 if(cls.netcon)
1298                                 {
1299                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1300                                         {
1301                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1302                                                 Curl_CommandWhenDone(donecommand);
1303                                                 noclear = TRUE;
1304                                                 CL_Disconnect();
1305                                                 noclear = FALSE;
1306                                                 Curl_CheckCommandWhenDone();
1307                                         }
1308                                         else
1309                                                 Curl_Register_predownload();
1310                                 }
1311                         }
1312                         return;
1313                 }
1314                 else if(!strncmp(a, "--maxspeed=", 11))
1315                 {
1316                         maxspeed = atof(a + 11);
1317                 }
1318                 else if(*a == '-')
1319                 {
1320                         Con_Printf("curl: invalid option %s\n", a);
1321                         // but we ignore the option
1322                 }
1323         }
1324
1325 needthefile:
1326         Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1327 }
1328
1329 /*
1330 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1331 {
1332         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1333         Z_Free(buffer);
1334 }
1335
1336 void Curl_CurlCat_f(void)
1337 {
1338         unsigned char *buf;
1339         const char *url = Cmd_Argv(1);
1340         buf = Z_Malloc(16384);
1341         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1342 }
1343 */
1344
1345 /*
1346 ====================
1347 Curl_Init_Commands
1348
1349 loads the commands and cvars this library uses
1350 ====================
1351 */
1352 void Curl_Init_Commands(void)
1353 {
1354         Cvar_RegisterVariable (&cl_curl_enabled);
1355         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1356         Cvar_RegisterVariable (&cl_curl_maxspeed);
1357         Cvar_RegisterVariable (&sv_curl_defaulturl);
1358         Cvar_RegisterVariable (&sv_curl_serverpackages);
1359         Cvar_RegisterVariable (&sv_curl_maxspeed);
1360         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1361         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1362 }
1363
1364 /*
1365 ====================
1366 Curl_GetDownloadInfo
1367
1368 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1369 The number of elements in the array is returned in int *nDownloads.
1370 const char **additional_info may be set to a string of additional user
1371 information, or to NULL if no such display shall occur. The returned
1372 array must be freed later using Z_Free.
1373 ====================
1374 */
1375 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1376 {
1377         int i;
1378         downloadinfo *di;
1379         Curl_downloadinfo_t *downinfo;
1380         static char addinfo[128];
1381
1382         if(!curl_dll)
1383         {
1384                 *nDownloads = 0;
1385                 if(additional_info)
1386                         *additional_info = NULL;
1387                 return NULL;
1388         }
1389
1390         i = 0;
1391         for(di = downloads; di; di = di->next)
1392                 ++i;
1393
1394         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1395         i = 0;
1396         for(di = downloads; di; di = di->next)
1397         {
1398                 // do not show infobars for background downloads
1399                 if(developer.integer <= 0)
1400                         if(di->buffer)
1401                                 continue;
1402                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1403                 if(di->curle)
1404                 {
1405                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1406                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1407                         downinfo[i].queued = false;
1408                 }
1409                 else
1410                 {
1411                         downinfo[i].queued = true;
1412                 }
1413                 ++i;
1414         }
1415
1416         if(additional_info)
1417         {
1418                 // TODO: can I clear command_when_done as soon as the first download fails?
1419                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1420                 {
1421                         if(!strncmp(command_when_done, "connect ", 8))
1422                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1423                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1424                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1425                         else
1426                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1427                         *additional_info = addinfo;
1428                 }
1429                 else
1430                         *additional_info = NULL;
1431         }
1432
1433         *nDownloads = i;
1434         return downinfo;
1435 }
1436
1437
1438 /*
1439 ====================
1440 Curl_FindPackURL
1441
1442 finds the URL where to find a given package.
1443
1444 For this, it reads a file "curl_urls.txt" of the following format:
1445
1446         data*.pk3       -
1447         revdm*.pk3      http://revdm/downloads/are/here/
1448         *                       http://any/other/stuff/is/here/
1449
1450 The URLs should end in /. If not, downloads will still work, but the cached files
1451 can't be just put into the data directory with the same download configuration
1452 (you might want to do this if you want to tag downloaded files from your
1453 server, but you should not). "-" means "don't download".
1454
1455 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1456 location instead.
1457
1458 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1459 this file for obvious reasons.
1460 ====================
1461 */
1462 static const char *Curl_FindPackURL(const char *filename)
1463 {
1464         static char foundurl[1024];
1465         fs_offset_t filesize;
1466         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1467         if(buf && filesize)
1468         {
1469                 // read lines of format "pattern url"
1470                 char *p = buf;
1471                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1472                 qboolean eof = false;
1473
1474                 pattern = p;
1475                 while(!eof)
1476                 {
1477                         switch(*p)
1478                         {
1479                                 case 0:
1480                                         eof = true;
1481                                         // fallthrough
1482                                 case '\n':
1483                                 case '\r':
1484                                         if(pattern && url && patternend)
1485                                         {
1486                                                 if(!urlend)
1487                                                         urlend = p;
1488                                                 *patternend = 0;
1489                                                 *urlend = 0;
1490                                                 if(matchpattern(filename, pattern, true))
1491                                                 {
1492                                                         strlcpy(foundurl, url, sizeof(foundurl));
1493                                                         Z_Free(buf);
1494                                                         return foundurl;
1495                                                 }
1496                                         }
1497                                         pattern = NULL;
1498                                         patternend = NULL;
1499                                         url = NULL;
1500                                         urlend = NULL;
1501                                         break;
1502                                 case ' ':
1503                                 case '\t':
1504                                         if(pattern && !patternend)
1505                                                 patternend = p;
1506                                         else if(url && !urlend)
1507                                                 urlend = p;
1508                                         break;
1509                                 default:
1510                                         if(!pattern)
1511                                                 pattern = p;
1512                                         else if(pattern && patternend && !url)
1513                                                 url = p;
1514                                         break;
1515                         }
1516                         ++p;
1517                 }
1518         }
1519         if(buf)
1520                 Z_Free(buf);
1521         return sv_curl_defaulturl.string;
1522 }
1523
1524 typedef struct requirement_s
1525 {
1526         struct requirement_s *next;
1527         char filename[MAX_OSPATH];
1528 }
1529 requirement;
1530 static requirement *requirements = NULL;
1531
1532
1533 /*
1534 ====================
1535 Curl_RequireFile
1536
1537 Adds the given file to the list of requirements.
1538 ====================
1539 */
1540 void Curl_RequireFile(const char *filename)
1541 {
1542         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1543         req->next = requirements;
1544         strlcpy(req->filename, filename, sizeof(req->filename));
1545         requirements = req;
1546 }
1547
1548 /*
1549 ====================
1550 Curl_ClearRequirements
1551
1552 Clears the list of required files for playing on the current map.
1553 This should be called at every map change.
1554 ====================
1555 */
1556 void Curl_ClearRequirements(void)
1557 {
1558         const char *p;
1559         while(requirements)
1560         {
1561                 requirement *req = requirements;
1562                 requirements = requirements->next;
1563                 Z_Free(req);
1564         }
1565         p = sv_curl_serverpackages.string;
1566         Con_DPrintf("Require all of: %s\n", p);
1567         while(COM_ParseToken_Simple(&p, false, false))
1568         {
1569                 Con_DPrintf("Require: %s\n", com_token);
1570                 Curl_RequireFile(com_token);
1571         }
1572 }
1573
1574 /*
1575 ====================
1576 Curl_SendRequirements
1577
1578 Makes the current host_clients download all files he needs.
1579 This is done by sending him the following console commands:
1580
1581         curl --clear_autodownload
1582         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1583         curl --finish_autodownload
1584 ====================
1585 */
1586 void Curl_SendRequirements(void)
1587 {
1588         // for each requirement, find the pack name
1589         char sendbuffer[4096] = "";
1590         requirement *req;
1591         qboolean foundone = false;
1592
1593         for(req = requirements; req; req = req->next)
1594         {
1595                 const char *p;
1596                 const char *thispack = FS_WhichPack(req->filename);
1597                 const char *packurl;
1598
1599                 if(!thispack)
1600                         continue;
1601
1602                 p = strrchr(thispack, '/');
1603                 if(p)
1604                         thispack = p + 1;
1605
1606                 packurl = Curl_FindPackURL(thispack);
1607
1608                 if(packurl && *packurl && strcmp(packurl, "-"))
1609                 {
1610                         if(!foundone)
1611                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1612
1613                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1614                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1615                         if(sv_curl_maxspeed.value > 0)
1616                                 dpsnprintf(sendbuffer + strlen(sendbuffer), sizeof(sendbuffer) - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1617                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1618                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1619                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1620                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1621                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1622                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1623
1624                         foundone = true;
1625                 }
1626         }
1627
1628         if(foundone)
1629                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1630
1631         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1632                 Host_ClientCommands("%s", sendbuffer);
1633         else
1634                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1635 }