libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5 #include "com_list.h"
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CF_SERVER | CF_ARCHIVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CF_SERVER | CF_ARCHIVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CF_SERVER | CF_ARCHIVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CF_CLIENT | CF_ARCHIVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {CF_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {CF_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18 static cvar_t developer_curl = {CF_CLIENT | CF_SERVER, "developer_curl","0", "whether verbose curl output should be printed to stderr"};
  19
  20 /*
  21 =================================================================
  22
  23   Minimal set of definitions from libcurl
  24
  25   WARNING: for a matter of simplicity, several pointer types are
  26   casted to "void*", and most enumerated values are not included
  27
  28 =================================================================
  29 */
  30
  31 typedef struct CURL_s CURL;
  32 typedef struct CURLM_s CURLM;
  33 typedef struct curl_slist curl_slist;
  34 typedef enum
  35 {
  36         CURLE_OK = 0
  37 }
  38 CURLcode;
  39 typedef enum
  40 {
  41         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  42         CURLM_OK = 0
  43 }
  44 CURLMcode;
  45 #define CURL_GLOBAL_NOTHING 0
  46 #define CURL_GLOBAL_SSL 1
  47 #define CURL_GLOBAL_WIN32 2
  48 #define CURLOPTTYPE_LONG          0
  49 #define CURLOPTTYPE_OBJECTPOINT   10000
  50 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  51 #define CURLOPTTYPE_OFF_T         30000
  52 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  53 typedef enum
  54 {
  55         CINIT(WRITEDATA, OBJECTPOINT, 1),
  56         CINIT(URL,  OBJECTPOINT, 2),
  57         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  58         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  59         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  60         CINIT(REFERER, OBJECTPOINT, 16),
  61         CINIT(USERAGENT, OBJECTPOINT, 18),
  62         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  63         CINIT(LOW_SPEED_TIME, LONG, 20),
  64         CINIT(RESUME_FROM, LONG, 21),
  65         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  66         CINIT(VERBOSE, LONG, 41),
  67         CINIT(POST, LONG, 47),         /* HTTP POST method */
  68         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  69         CINIT(POSTFIELDSIZE, LONG, 60),
  70         CINIT(PRIVATE, OBJECTPOINT, 103),
  71         CINIT(PROTOCOLS, LONG, 181),
  72         CINIT(REDIR_PROTOCOLS, LONG, 182)
  73 }
  74 CURLoption;
  75 #define CURLPROTO_HTTP   (1<<0)
  76 #define CURLPROTO_HTTPS  (1<<1)
  77 #define CURLPROTO_FTP    (1<<2)
  78 typedef enum
  79 {
  80         CURLINFO_TEXT = 0,
  81         CURLINFO_HEADER_IN,    /* 1 */
  82         CURLINFO_HEADER_OUT,   /* 2 */
  83         CURLINFO_DATA_IN,      /* 3 */
  84         CURLINFO_DATA_OUT,     /* 4 */
  85         CURLINFO_SSL_DATA_IN,  /* 5 */
  86         CURLINFO_SSL_DATA_OUT, /* 6 */
  87         CURLINFO_END
  88 }
  89 curl_infotype;
  90 #define CURLINFO_STRING   0x100000
  91 #define CURLINFO_LONG     0x200000
  92 #define CURLINFO_DOUBLE   0x300000
  93 #define CURLINFO_SLIST    0x400000
  94 #define CURLINFO_MASK     0x0fffff
  95 #define CURLINFO_TYPEMASK 0xf00000
  96 typedef enum
  97 {
  98         CURLINFO_NONE, /* first, never use this */
  99         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
 100         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
 101         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 102         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 103         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 104         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 105         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 106         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 107         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 108         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 109         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 110         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 111         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 112         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 113         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 114         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 115         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 116         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 117         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 118         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 119         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 120         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 121         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 122         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 123         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 124         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 125         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 126 }
 127 CURLINFO;
 128
 129 typedef enum
 130 {
 131         CURLMSG_NONE, /* first, not used */
 132         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 133                                          the CURLcode of the transfer */
 134         CURLMSG_LAST
 135 }
 136 CURLMSG;
 137 typedef struct
 138 {
 139         CURLMSG msg;       /* what this message means */
 140         CURL *easy_handle; /* the handle it concerns */
 141         union
 142         {
 143                 void *whatever;    /* message-specific data */
 144                 CURLcode result;   /* return code for transfer */
 145         }
 146         data;
 147 }
 148 CURLMsg;
 149
 150 static void (*qcurl_global_init) (long flags);
 151 static void (*qcurl_global_cleanup) (void);
 152
 153 static CURL * (*qcurl_easy_init) (void);
 154 static void (*qcurl_easy_cleanup) (CURL *handle);
 155 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 156 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 157 static const char * (*qcurl_easy_strerror) (CURLcode);
 158
 159 static CURLM * (*qcurl_multi_init) (void);
 160 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 161 static CURLMcode (*qcurl_multi_poll) (CURLM *multi_handle, void*, unsigned int extra_nfds, int timeout_ms, int *ret);
 162 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 163 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 164 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 165 static void (*qcurl_multi_cleanup) (CURLM *);
 166 static const char * (*qcurl_multi_strerror) (CURLcode);
 167 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 168 static void (*qcurl_slist_free_all) (curl_slist *list);
 169
 170 static dllfunction_t curlfuncs[] =
 171 {
 172         {"curl_global_init",            (void **) &qcurl_global_init},
 173         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 174         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 175         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 176         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 177         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 178         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 179         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 180         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 181         {"curl_multi_poll",             (void **) &qcurl_multi_poll},
 182         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 183         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 184         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 185         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 186         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 187         {"curl_slist_append",           (void **) &qcurl_slist_append},
 188         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 189         {NULL, NULL}
 190 };
 191
 192 // Handle for CURL DLL
 193 static dllhandle_t curl_dll = NULL;
 194 // will be checked at many places to find out if qcurl calls are allowed
 195
 196 #define LOADTYPE_NONE 0
 197 #define LOADTYPE_PAK 1
 198 #define LOADTYPE_CACHEPIC 2
 199 #define LOADTYPE_SKINFRAME 3
 200
 201 void *curl_mutex = NULL;
 202
 203 typedef struct downloadinfo_s
 204 {
 205         char filename[MAX_OSPATH];
 206         char url[1024];
 207         char referer[256];
 208         qfile_t *stream;
 209         fs_offset_t startpos;
 210         CURL *curle;
 211         qbool started;
 212         int loadtype;
 213         size_t bytes_received; // for buffer
 214         double bytes_received_curl; // for throttling
 215         double bytes_sent_curl; // for throttling
 216         llist_t list;
 217         qbool forthismap;
 218         double maxspeed;
 219         curl_slist *slist; // http headers
 220
 221         unsigned char *buffer;
 222         size_t buffersize;
 223         curl_callback_t callback;
 224         void *callback_data;
 225
 226         const unsigned char *postbuf;
 227         size_t postbufsize;
 228         const char *post_content_type;
 229         const char *extraheaders;
 230 }
 231 downloadinfo;
 232 LIST_HEAD(downloads);
 233 static int numdownloads = 0;
 234
 235 static qbool noclear = false;
 236
 237 static int numdownloads_fail = 0;
 238 static int numdownloads_success = 0;
 239 static int numdownloads_added = 0;
 240 static char command_when_done[256] = "";
 241 static char command_when_error[256] = "";
 242
 243 /*
 244 ====================
 245 Curl_CommandWhenDone
 246
 247 Sets the command which is to be executed when the last download completes AND
 248 all downloads since last server connect ended with a successful status.
 249 Setting the command to NULL clears it.
 250 ====================
 251 */
 252 static void Curl_CommandWhenDone(const char *cmd)
 253 {
 254         if(!curl_dll)
 255                 return;
 256         if(cmd)
 257                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 258         else
 259                 *command_when_done = 0;
 260 }
 261
 262 /*
 263 FIXME
 264 Do not use yet. Not complete.
 265 Problem: what counts as an error?
 266 */
 267
 268 static void Curl_CommandWhenError(const char *cmd)
 269 {
 270         if(!curl_dll)
 271                 return;
 272         if(cmd)
 273                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 274         else
 275                 *command_when_error = 0;
 276 }
 277
 278 /*
 279 ====================
 280 Curl_Clear_forthismap
 281
 282 Clears the "will disconnect on failure" flags.
 283 ====================
 284 */
 285 void Curl_Clear_forthismap(void)
 286 {
 287         downloadinfo *di;
 288         if(noclear)
 289                 return;
 290         if (curl_mutex) Thread_LockMutex(curl_mutex);
 291         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 292                 di->forthismap = false;
 293         Curl_CommandWhenError(NULL);
 294         Curl_CommandWhenDone(NULL);
 295         numdownloads_fail = 0;
 296         numdownloads_success = 0;
 297         numdownloads_added = 0;
 298         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 299 }
 300
 301 /*
 302 ====================
 303 Curl_Have_forthismap
 304
 305 Returns true if a download needed for the current game is running.
 306 ====================
 307 */
 308 qbool Curl_Have_forthismap(void)
 309 {
 310         return numdownloads_added != 0;
 311 }
 312
 313 void Curl_Register_predownload(void)
 314 {
 315         if (curl_mutex) Thread_LockMutex(curl_mutex);
 316         Curl_CommandWhenDone("cl_begindownloads");
 317         Curl_CommandWhenError("cl_begindownloads");
 318         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 319 }
 320
 321 /*
 322 ====================
 323 Curl_CheckCommandWhenDone
 324
 325 Checks if a "done command" is to be executed.
 326 All downloads finished, at least one success since connect, no single failure
 327 -> execute the command.
 328 */
 329 static void Curl_CheckCommandWhenDone(void)
 330 {
 331         if(!curl_dll)
 332                 return;
 333         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 334         {
 335                 if(numdownloads_fail == 0)
 336                 {
 337                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 338                         Cbuf_AddText(cmd_local, "\n");
 339                         Cbuf_AddText(cmd_local, command_when_done);
 340                         Cbuf_AddText(cmd_local, "\n");
 341                 }
 342                 else
 343                 {
 344                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 345                         Cbuf_AddText(cmd_local, "\n");
 346                         Cbuf_AddText(cmd_local, command_when_error);
 347                         Cbuf_AddText(cmd_local, "\n");
 348                 }
 349                 Curl_Clear_forthismap();
 350         }
 351 }
 352
 353 /*
 354 ====================
 355 CURL_CloseLibrary
 356
 357 Load the cURL DLL
 358 ====================
 359 */
 360 static qbool CURL_OpenLibrary (void)
 361 {
 362         const char* dllnames [] =
 363         {
 364 #if defined(WIN32)
 365                 "libcurl-4.dll",
 366                 "libcurl-3.dll",
 367 #elif defined(MACOSX)
 368                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 369                 "libcurl.3.dylib", // Mac OS X Tiger
 370                 "libcurl.2.dylib", // Mac OS X Panther
 371 #else
 372                 "libcurl.so.4",
 373                 "libcurl.so.3",
 374                 "libcurl.so", // FreeBSD
 375 #endif
 376                 NULL
 377         };
 378
 379         // Already loaded?
 380         if (curl_dll)
 381                 return true;
 382
 383         // Load the DLL
 384         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
 385 }
 386
 387
 388 /*
 389 ====================
 390 CURL_CloseLibrary
 391
 392 Unload the cURL DLL
 393 ====================
 394 */
 395 static void CURL_CloseLibrary (void)
 396 {
 397         Sys_FreeLibrary (&curl_dll);
 398 }
 399
 400
 401 static CURLM *curlm = NULL;
 402 static double bytes_received = 0; // used for bandwidth throttling
 403 static double bytes_sent = 0; // used for bandwidth throttling
 404 static double curltime = 0;
 405
 406 /*
 407 ====================
 408 CURL_fwrite
 409
 410 fwrite-compatible function that writes the data to a file. libcurl can call
 411 this.
 412 ====================
 413 */
 414 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 415 {
 416         fs_offset_t ret = -1;
 417         size_t bytes = size * nmemb;
 418         downloadinfo *di = (downloadinfo *) vdi;
 419
 420         if(di->buffer)
 421         {
 422                 if(di->bytes_received + bytes <= di->buffersize)
 423                 {
 424                         memcpy(di->buffer + di->bytes_received, data, bytes);
 425                         ret = bytes;
 426                 }
 427                 // otherwise: buffer overrun, ret stays -1
 428         }
 429
 430         if(di->stream)
 431         {
 432                 ret = FS_Write(di->stream, data, bytes);
 433         }
 434
 435         di->bytes_received += bytes;
 436
 437         //Con_Printf("CURL_fwrite callback timestamp: %f bytes: %ld\n", host.realtime, ret);
 438
 439         return ret;
 440         // Why not ret / nmemb?
 441         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 442         // Yes, this is incompatible to fwrite(2).
 443 }
 444
 445 typedef enum
 446 {
 447         CURL_DOWNLOAD_SUCCESS = 0,
 448         CURL_DOWNLOAD_FAILED,
 449         CURL_DOWNLOAD_ABORTED,
 450         CURL_DOWNLOAD_SERVERERROR
 451 }
 452 CurlStatus;
 453
 454 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 455 {
 456         downloadinfo *di = (downloadinfo *) cbdata;
 457         switch(status)
 458         {
 459                 case CURLCBSTATUS_OK:
 460                         Con_DPrintf("Download of %s: OK\n", di->filename);
 461                         break;
 462                 case CURLCBSTATUS_FAILED:
 463                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 464                         break;
 465                 case CURLCBSTATUS_ABORTED:
 466                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 467                         break;
 468                 case CURLCBSTATUS_SERVERERROR:
 469                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 470                         break;
 471                 case CURLCBSTATUS_UNKNOWN:
 472                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 473                         break;
 474                 default:
 475                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 476                         break;
 477         }
 478 }
 479
 480 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 481 {
 482         curl_default_callback(status, length_received, buffer, cbdata);
 483 }
 484
 485 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 486 {
 487         unsigned char *pixels = NULL;
 488         fs_offset_t filesize = 0;
 489         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 490         if(data)
 491         {
 492                 int mip = 0;
 493                 if(!strcmp(content_type, "image/jpeg"))
 494                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 495                 else if(!strcmp(content_type, "image/png"))
 496                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 497                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 498                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 499                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 500                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 501                 else
 502                         Con_Printf("Did not detect content type: %s\n", content_type);
 503                 Mem_Free(data);
 504         }
 505         // do we call Image_MakeLinearColorsFromsRGB or not?
 506         return pixels;
 507 }
 508
 509 /*
 510 ====================
 511 Curl_EndDownload
 512
 513 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 514 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 515 code from libcurl, or 0, if another error has occurred.
 516 ====================
 517 */
 518 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 519 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 520 {
 521         char content_type[64];
 522         qbool ok = false;
 523         if(!curl_dll)
 524                 return;
 525         switch(status)
 526         {
 527                 case CURL_DOWNLOAD_SUCCESS:
 528                         ok = true;
 529                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 530                         break;
 531                 case CURL_DOWNLOAD_FAILED:
 532                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 533                         break;
 534                 case CURL_DOWNLOAD_ABORTED:
 535                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 536                         break;
 537                 case CURL_DOWNLOAD_SERVERERROR:
 538                         // reopen to enforce it to have zero bytes again
 539                         if(di->stream)
 540                         {
 541                                 FS_Close(di->stream);
 542                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 543                         }
 544
 545                         if(di->callback)
 546                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 547                         break;
 548                 default:
 549                         if(di->callback)
 550                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 551                         break;
 552         }
 553         if(content_type_)
 554                 strlcpy(content_type, content_type_, sizeof(content_type));
 555         else
 556                 *content_type = 0;
 557
 558         if(di->curle)
 559         {
 560                 qcurl_multi_remove_handle(curlm, di->curle);
 561                 qcurl_easy_cleanup(di->curle);
 562                 if(di->slist)
 563                         qcurl_slist_free_all(di->slist);
 564         }
 565
 566         if(!di->callback && ok && !di->bytes_received)
 567         {
 568                 Con_Printf("ERROR: empty file\n");
 569                 ok = false;
 570         }
 571
 572         if(di->stream)
 573                 FS_Close(di->stream);
 574
 575 #define CLEAR_AND_RETRY() \
 576         do \
 577         { \
 578                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 579                 FS_Close(di->stream); \
 580                 if(di->startpos && !di->callback) \
 581                 { \
 582                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 583                         di->forthismap = false; \
 584                 } \
 585         } \
 586         while(0)
 587
 588         if(ok && di->loadtype == LOADTYPE_PAK)
 589         {
 590                 ok = FS_AddPack(di->filename, NULL, true, true);
 591                 if(!ok)
 592                         CLEAR_AND_RETRY();
 593         }
 594         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 595         {
 596                 const char *p;
 597                 unsigned char *pixels = NULL;
 598
 599                 p = di->filename;
 600 #ifdef WE_ARE_EVIL
 601                 if(!strncmp(p, "dlcache/", 8))
 602                         p += 8;
 603 #endif
 604
 605                 pixels = decode_image(di, content_type);
 606                 if(pixels)
 607                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 608                 else
 609                         CLEAR_AND_RETRY();
 610         }
 611         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 612         {
 613                 const char *p;
 614                 unsigned char *pixels = NULL;
 615
 616                 p = di->filename;
 617 #ifdef WE_ARE_EVIL
 618                 if(!strncmp(p, "dlcache/", 8))
 619                         p += 8;
 620 #endif
 621
 622                 pixels = decode_image(di, content_type);
 623                 if(pixels)
 624                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 625                 else
 626                         CLEAR_AND_RETRY();
 627         }
 628
 629         List_Delete(&di->list);
 630
 631         --numdownloads;
 632         if(di->forthismap)
 633         {
 634                 if(ok)
 635                         ++numdownloads_success;
 636                 else
 637                         ++numdownloads_fail;
 638         }
 639         Z_Free(di);
 640 }
 641
 642 /*
 643 ====================
 644 CleanURL
 645
 646 Returns a "cleaned up" URL for display (to strip login data)
 647 ====================
 648 */
 649 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 650 {
 651         const char *p, *q, *r;
 652
 653         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 654         p = strstr(url, "://");
 655         if(p)
 656         {
 657                 q = strchr(p + 3, '@');
 658                 if(q)
 659                 {
 660                         r = strchr(p + 3, '/');
 661                         if(!r || q < r)
 662                         {
 663                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 664                                 return urlbuf;
 665                         }
 666                 }
 667         }
 668
 669         return url;
 670 }
 671
 672 /*
 673 ====================
 674 CheckPendingDownloads
 675
 676 checks if there are free download slots to start new downloads in.
 677 To not start too many downloads at once, only one download is added at a time,
 678 up to a maximum number of cl_curl_maxdownloads are running.
 679 ====================
 680 */
 681 static void CheckPendingDownloads(void)
 682 {
 683         const char *h;
 684         char urlbuf[1024];
 685         char vabuf[1024];
 686         if(!curl_dll)
 687                 return;
 688         if(numdownloads < cl_curl_maxdownloads.integer)
 689         {
 690                 downloadinfo *di;
 691                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
 692                 {
 693                         if(!di->started)
 694                         {
 695                                 if(!di->buffer)
 696                                 {
 697                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 698
 699                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 700                                         if(!di->stream)
 701                                         {
 702                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 703                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 704                                                 return;
 705                                         }
 706                                         FS_Seek(di->stream, 0, SEEK_END);
 707                                         di->startpos = FS_Tell(di->stream);
 708
 709                                         if(di->startpos > 0)
 710                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 711                                         Con_Print("...\n");
 712                                 }
 713                                 else
 714                                 {
 715                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 716                                         di->startpos = 0;
 717                                 }
 718
 719                                 di->curle = qcurl_easy_init();
 720                                 di->slist = NULL;
 721                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 722                                 if(cl_curl_useragent.integer)
 723                                 {
 724                                         const char *ua
 725 #ifdef HTTP_USER_AGENT
 726                                                 = HTTP_USER_AGENT;
 727 #else
 728                                                 = engineversion;
 729 #endif
 730                                         if(!ua)
 731                                                 ua = "";
 732                                         if(*cl_curl_useragent_append.string)
 733                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 734                                                         ua,
 735                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 736                                                                 ? " "
 737                                                                 : "",
 738                                                         cl_curl_useragent_append.string);
 739                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 740                                 }
 741                                 else
 742                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 743                                 if(developer_curl.integer)
 744                                         qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 746                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 750                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 751                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 752                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 753                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 754                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 755                                 {
 756                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 757                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 758                                 }
 759                                 if(di->post_content_type)
 760                                 {
 761                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 762                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 763                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 764                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 765                                 }
 766
 767                                 // parse extra headers into slist
 768                                 // \n separated list!
 769                                 h = di->extraheaders;
 770                                 while(h)
 771                                 {
 772                                         const char *hh = strchr(h, '\n');
 773                                         if(hh)
 774                                         {
 775                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 776                                                 memcpy(buf, h, hh - h);
 777                                                 buf[hh - h] = 0;
 778                                                 di->slist = qcurl_slist_append(di->slist, buf);
 779                                                 h = hh + 1;
 780                                         }
 781                                         else
 782                                         {
 783                                                 di->slist = qcurl_slist_append(di->slist, h);
 784                                                 h = NULL;
 785                                         }
 786                                 }
 787
 788                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 789
 790                                 qcurl_multi_add_handle(curlm, di->curle);
 791                                 di->started = true;
 792                                 ++numdownloads;
 793                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 794                                         break;
 795                         }
 796                 }
 797         }
 798 }
 799
 800 /*
 801 ====================
 802 Curl_Init
 803
 804 this function MUST be called before using anything else in this file.
 805 On Win32, this must be called AFTER WSAStartup has been done!
 806 ====================
 807 */
 808 void Curl_Init(void)
 809 {
 810         CURL_OpenLibrary();
 811         if(!curl_dll)
 812                 return;
 813         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 814         qcurl_global_init(CURL_GLOBAL_SSL);
 815         curlm = qcurl_multi_init();
 816 }
 817
 818 /*
 819 ====================
 820 Curl_Shutdown
 821
 822 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 823 ====================
 824 */
 825 void Curl_ClearRequirements(void);
 826 void Curl_Shutdown(void)
 827 {
 828         if(!curl_dll)
 829                 return;
 830         Curl_ClearRequirements();
 831         Curl_CancelAll();
 832         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 833         CURL_CloseLibrary();
 834         curl_dll = NULL;
 835 }
 836
 837 // for VM_checkextension()
 838 qbool Curl_Available(void)
 839 {
 840         return curl_dll ? true : false;
 841 }
 842
 843 /*
 844 ====================
 845 Curl_Find
 846
 847 Finds the internal information block for a download given by file name.
 848 ====================
 849 */
 850 static downloadinfo *Curl_Find(const char *filename)
 851 {
 852         downloadinfo *di;
 853         if(!curl_dll)
 854                 return NULL;
 855         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 856                 if(!strcasecmp(di->filename, filename))
 857                         return di;
 858         return NULL;
 859 }
 860
 861 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 862 {
 863         downloadinfo *di, *ndi;
 864         if(!curl_dll)
 865                 return;
 866         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
 867         {
 868                 if(di->callback == callback && di->callback_data == cbdata)
 869                 {
 870                         di->callback = curl_quiet_callback; // do NOT call the callback
 871                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 872                 }
 873         }
 874 }
 875
 876 /*
 877 ====================
 878 Curl_Begin
 879
 880 Starts a download of a given URL to the file name portion of this URL (or name
 881 if given) in the "dlcache/" folder.
 882 ====================
 883 */
 884 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 885 {
 886         if(buf)
 887                 if(loadtype != LOADTYPE_NONE)
 888                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 889
 890         if(!curl_dll || !cl_curl_enabled.integer)
 891         {
 892                 return false;
 893         }
 894         else
 895         {
 896                 char fn[MAX_OSPATH];
 897                 char urlbuf[1024];
 898                 const char *p, *q;
 899                 size_t length;
 900                 downloadinfo *di;
 901
 902                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 903                 p = strchr(URL, ':');
 904                 if(p)
 905                 {
 906                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 907                         {
 908                                 char addressstring[128];
 909                                 *addressstring = 0;
 910                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 911                                 q = strchr(addressstring, ':');
 912                                 if(!q)
 913                                         q = addressstring + strlen(addressstring);
 914                                 if(*addressstring)
 915                                 {
 916                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 917                                         URL = urlbuf;
 918                                 }
 919                         }
 920                 }
 921
 922                 // Note: This extraction of the file name portion is NOT entirely correct.
 923                 //
 924                 // It does the following:
 925                 //
 926                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 927                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 928                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 929                 //
 930                 // However, I'd like to keep this "buggy" behavior so that PHP script
 931                 // authors can write download scripts without having to enable
 932                 // AcceptPathInfo on Apache. They just have to ensure that their script
 933                 // can be called with such a "fake" path name like
 934                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 935                 //
 936                 // By the way, such PHP scripts should either send the file or a
 937                 // "Location:" redirect; PHP code example:
 938                 //
 939                 //   header("Location: http://www.example.com/");
 940                 //
 941                 // By the way, this will set User-Agent to something like
 942                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 943                 // dp://serverhost:serverport/ so you can filter on this; an example
 944                 // httpd log file line might be:
 945                 //
 946                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 947
 948                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 949
 950                 if(buf)
 951                 {
 952                         if(!name)
 953                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 954                 }
 955                 else
 956                 {
 957                         if(!name)
 958                         {
 959                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 960                                 p = strrchr(name, '/');
 961                                 p = p ? (p+1) : name;
 962                                 q = strchr(p, '?');
 963                                 length = q ? (size_t)(q - p) : strlen(p);
 964                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 965                         }
 966                         else
 967                         {
 968                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 969                         }
 970
 971                         name = fn; // make it point back
 972
 973                         // already downloading the file?
 974                         {
 975                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 976                                 if(existingdownloadinfo)
 977                                 {
 978                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 979
 980                                         // however, if it was not for this map yet...
 981                                         if(forthismap && !existingdownloadinfo->forthismap)
 982                                         {
 983                                                 existingdownloadinfo->forthismap = true;
 984                                                 // this "fakes" a download attempt so the client will wait for
 985                                                 // the download to finish and then reconnect
 986                                                 ++numdownloads_added;
 987                                         }
 988
 989                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 990                                         return false;
 991                                 }
 992                         }
 993
 994                         if(FS_FileExists(fn))
 995                         {
 996                                 if(loadtype == LOADTYPE_PAK)
 997                                 {
 998                                         qbool already_loaded;
 999                                         if(FS_AddPack(fn, &already_loaded, true, true))
1000                                         {
1001                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
1002                                                 if(already_loaded)
1003                                                         Con_DPrintf("(pak was already loaded)\n");
1004                                                 else
1005                                                 {
1006                                                         if(forthismap)
1007                                                         {
1008                                                                 ++numdownloads_added;
1009                                                                 ++numdownloads_success;
1010                                                         }
1011                                                 }
1012
1013                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1014                                                 return false;
1015                                         }
1016                                         else
1017                                         {
1018                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1019                                                 if(f)
1020                                                 {
1021                                                         char b[4] = {0};
1022                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1023
1024                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1025                                                         {
1026                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1027                                                                 FS_Close(f);
1028                                                                 f = FS_OpenRealFile(fn, "wb", false);
1029                                                                 if(f)
1030                                                                         FS_Close(f);
1031                                                         }
1032                                                         else
1033                                                         {
1034                                                                 // OK
1035                                                                 FS_Close(f);
1036                                                         }
1037                                                 }
1038                                         }
1039                                 }
1040                                 else
1041                                 {
1042                                         // never resume these
1043                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1044                                         if(f)
1045                                                 FS_Close(f);
1046                                 }
1047                         }
1048                 }
1049
1050                 // if we get here, we actually want to download... so first verify the
1051                 // URL scheme (so one can't read local files using file://)
1052                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1053                 {
1054                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1055                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1056                         return false;
1057                 }
1058
1059                 if(forthismap)
1060                         ++numdownloads_added;
1061                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1062                 strlcpy(di->filename, name, sizeof(di->filename));
1063                 strlcpy(di->url, URL, sizeof(di->url));
1064                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1065                 di->forthismap = forthismap;
1066                 di->stream = NULL;
1067                 di->startpos = 0;
1068                 di->curle = NULL;
1069                 di->started = false;
1070                 di->loadtype = loadtype;
1071                 di->maxspeed = maxspeed;
1072                 di->bytes_received = 0;
1073                 di->bytes_received_curl = 0;
1074                 di->bytes_sent_curl = 0;
1075                 di->extraheaders = extraheaders;
1076                 di->buffer = buf;
1077                 di->buffersize = bufsize;
1078                 if(callback == NULL)
1079                 {
1080                         di->callback = curl_default_callback;
1081                         di->callback_data = di;
1082                 }
1083                 else
1084                 {
1085                         di->callback = callback;
1086                         di->callback_data = cbdata;
1087                 }
1088
1089                 if(post_content_type)
1090                 {
1091                         di->post_content_type = post_content_type;
1092                         di->postbuf = postbuf;
1093                         di->postbufsize = postbufsize;
1094                 }
1095                 else
1096                 {
1097                         di->post_content_type = NULL;
1098                         di->postbuf = NULL;
1099                         di->postbufsize = 0;
1100                 }
1101
1102                 List_Add(&di->list, &downloads);
1103
1104                 if (curl_mutex)
1105                         Thread_UnlockMutex(curl_mutex);
1106
1107                 return true;
1108         }
1109 }
1110
1111 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1112 {
1113         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1114 }
1115 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1116 {
1117         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1118 }
1119 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1120 {
1121         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1122 }
1123
1124 /*
1125 ====================
1126 Curl_Frame
1127
1128 call this regularily as this will always download as much as possible without
1129 blocking.
1130 ====================
1131 */
1132 void Curl_Frame(void)
1133 {
1134         double maxspeed;
1135         downloadinfo *di;
1136
1137         noclear = false;
1138
1139         if(!cl_curl_enabled.integer)
1140                 return;
1141
1142         if(!curl_dll)
1143                 return;
1144
1145         if (curl_mutex) Thread_LockMutex(curl_mutex);
1146
1147         Curl_CheckCommandWhenDone();
1148
1149         if(List_Is_Empty(&downloads))
1150         {
1151                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1152                 return;
1153         }
1154
1155         if(host.realtime < curltime) // throttle
1156         {
1157                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1158                 return;
1159         }
1160
1161         {
1162                 int remaining;
1163                 CURLMcode mc;
1164
1165                 do
1166                 {
1167                         mc = qcurl_multi_perform(curlm, &remaining);
1168                 }
1169                 while(mc == CURLM_CALL_MULTI_PERFORM);
1170
1171                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1172                 {
1173                         double b = 0;
1174                         if(di->curle)
1175                         {
1176                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1177                                 bytes_sent += (b - di->bytes_sent_curl);
1178                                 di->bytes_sent_curl = b;
1179                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1180                                 bytes_sent += (b - di->bytes_received_curl);
1181                                 di->bytes_received_curl = b;
1182                         }
1183                 }
1184
1185                 for(;;)
1186                 {
1187                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1188                         if(!msg)
1189                                 break;
1190                         if(msg->msg == CURLMSG_DONE)
1191                         {
1192                                 const char *ct = NULL;
1193                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1194                                 CURLcode result;
1195                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1196                                 result = msg->data.result;
1197                                 if(result)
1198                                 {
1199                                         failed = CURL_DOWNLOAD_FAILED;
1200                                 }
1201                                 else
1202                                 {
1203                                         long code;
1204                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1205                                         switch(code / 100)
1206                                         {
1207                                                 case 4: // e.g. 404?
1208                                                 case 5: // e.g. 500?
1209                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1210                                                         result = (CURLcode) code;
1211                                                         break;
1212                                         }
1213                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1214                                 }
1215
1216                                 Curl_EndDownload(di, failed, result, ct);
1217                         }
1218                 }
1219         }
1220
1221         CheckPendingDownloads();
1222
1223         // when will we curl the next time?
1224         // we will wait a bit to ensure our download rate is kept.
1225         // we now know that realtime >= curltime... so set up a new curltime
1226
1227         // use the slowest allowing download to derive the maxspeed... this CAN
1228         // be done better, but maybe later
1229         maxspeed = cl_curl_maxspeed.value;
1230         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1231                 if(di->maxspeed > 0)
1232                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1233                                 maxspeed = di->maxspeed;
1234
1235         if(maxspeed > 0)
1236         {
1237                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1238                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1239                 bytes_sent = 0;
1240                 bytes_received = 0;
1241         }
1242         else
1243                 curltime = host.realtime;
1244
1245         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1246 }
1247
1248 /*
1249 ====================
1250 Curl_Select
1251
1252 Sleeps until there's some transfer progress or a timeout is reached,
1253 unfortunately the timeout is only in milliseconds.
1254 This allows good throughput even at very low FPS.
1255 ====================
1256 */
1257 void Curl_Select(double *microseconds)
1258 {
1259         if (List_Is_Empty(&downloads))
1260                 return;
1261         if (qcurl_multi_poll(curlm, NULL, 0, *microseconds / 1000, NULL) == CURLM_OK)
1262                 *microseconds = 0; // either we finished waiting or a transfer progressed
1263         else
1264                 Con_Print("There's an emergency going on!\nIt's still going on!\nMaybe you need to upgrade libcurl?\n");
1265 }
1266
1267 /*
1268 ====================
1269 Curl_CancelAll
1270
1271 Stops ALL downloads.
1272 ====================
1273 */
1274 void Curl_CancelAll(void)
1275 {
1276         if(!curl_dll)
1277                 return;
1278
1279         if (curl_mutex) Thread_LockMutex(curl_mutex);
1280
1281         while(!List_Is_Empty(&downloads))
1282         {
1283                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1284                 // INVARIANT: downloads will point to the next download after that!
1285         }
1286
1287         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1288 }
1289
1290 /*
1291 ====================
1292 Curl_Running
1293
1294 returns true if there is a download running.
1295 ====================
1296 */
1297 qbool Curl_Running(void)
1298 {
1299         if(!curl_dll)
1300                 return false;
1301
1302         return !List_Is_Empty(&downloads);
1303 }
1304
1305 /*
1306 ====================
1307 Curl_GetDownloadAmount
1308
1309 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1310 for the given download.
1311 ====================
1312 */
1313 static double Curl_GetDownloadAmount(downloadinfo *di)
1314 {
1315         if(!curl_dll)
1316                 return -2;
1317         if(di->curle)
1318         {
1319                 double length;
1320                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1321                 if(length > 0)
1322                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1323                 else
1324                         return 0;
1325         }
1326         else
1327                 return -1;
1328 }
1329
1330 /*
1331 ====================
1332 Curl_GetDownloadSpeed
1333
1334 returns the speed of the given download in bytes per second
1335 ====================
1336 */
1337 static double Curl_GetDownloadSpeed(downloadinfo *di)
1338 {
1339         if(!curl_dll)
1340                 return -2;
1341         if(di->curle)
1342         {
1343                 double speed;
1344                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1345                 return speed;
1346         }
1347         else
1348                 return -1;
1349 }
1350
1351 /*
1352 ====================
1353 Curl_Info_f
1354
1355 prints the download list
1356 ====================
1357 */
1358 // TODO rewrite using Curl_GetDownloadInfo?
1359 static void Curl_Info_f(cmd_state_t *cmd)
1360 {
1361         downloadinfo *di;
1362         char urlbuf[1024];
1363         if(!curl_dll)
1364                 return;
1365         if(Curl_Running())
1366         {
1367                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1368                 Con_Print("Currently running downloads:\n");
1369                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1370                 {
1371                         double speed, percent;
1372                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1373                         percent = 100.0 * Curl_GetDownloadAmount(di);
1374                         speed = Curl_GetDownloadSpeed(di);
1375                         if(percent >= 0)
1376                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1377                         else
1378                                 Con_Print("(queued)\n");
1379                 }
1380                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1381         }
1382         else
1383         {
1384                 Con_Print("No downloads running.\n");
1385         }
1386 }
1387
1388 /*
1389 ====================
1390 Curl_Curl_f
1391
1392 implements the "curl" console command
1393
1394 curl --info
1395 curl --cancel
1396 curl --cancel filename
1397 curl url
1398
1399 For internal use:
1400
1401 curl [--pak] [--forthismap] [--for filename filename...] url
1402         --pak: after downloading, load the package into the virtual file system
1403         --for filename...: only download of at least one of the named files is missing
1404         --forthismap: don't reconnect on failure
1405
1406 curl --clear_autodownload
1407         clears the download success/failure counters
1408
1409 curl --finish_autodownload
1410         if at least one download has been started, disconnect and drop to the menu
1411         once the last download completes successfully, reconnect to the current server
1412 ====================
1413 */
1414 static void Curl_Curl_f(cmd_state_t *cmd)
1415 {
1416         double maxspeed = 0;
1417         int i;
1418         int end;
1419         int loadtype = LOADTYPE_NONE;
1420         qbool forthismap = false;
1421         const char *url;
1422         const char *name = 0;
1423
1424         if(!curl_dll)
1425         {
1426                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1427                 return;
1428         }
1429
1430         if(!cl_curl_enabled.integer)
1431         {
1432                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1433                 return;
1434         }
1435
1436         if(Cmd_Argc(cmd) < 2)
1437         {
1438                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1439                 return;
1440         }
1441
1442         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1443         end = Cmd_Argc(cmd);
1444
1445         for(i = 1; i != end; ++i)
1446         {
1447                 const char *a = Cmd_Argv(cmd, i);
1448                 if(!strcmp(a, "--info"))
1449                 {
1450                         Curl_Info_f(cmd);
1451                         return;
1452                 }
1453                 else if(!strcmp(a, "--cancel"))
1454                 {
1455                         if(i == end - 1) // last argument
1456                                 Curl_CancelAll();
1457                         else
1458                         {
1459                                 downloadinfo *di = Curl_Find(url);
1460                                 if(di)
1461                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1462                                 else
1463                                         Con_Print("download not found\n");
1464                         }
1465                         return;
1466                 }
1467                 else if(!strcmp(a, "--pak"))
1468                 {
1469                         loadtype = LOADTYPE_PAK;
1470                 }
1471                 else if(!strcmp(a, "--cachepic"))
1472                 {
1473                         loadtype = LOADTYPE_CACHEPIC;
1474                 }
1475                 else if(!strcmp(a, "--skinframe"))
1476                 {
1477                         loadtype = LOADTYPE_SKINFRAME;
1478                 }
1479                 else if(!strcmp(a, "--for")) // must be last option
1480                 {
1481                         for(i = i + 1; i != end - 1; ++i)
1482                         {
1483                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1484                                         goto needthefile; // why can't I have a "double break"?
1485                         }
1486                         // if we get here, we have all the files...
1487                         return;
1488                 }
1489                 else if(!strcmp(a, "--forthismap"))
1490                 {
1491                         forthismap = true;
1492                 }
1493                 else if(!strcmp(a, "--as"))
1494                 {
1495                         if(i < end - 1)
1496                         {
1497                                 ++i;
1498                                 name = Cmd_Argv(cmd, i);
1499                         }
1500                 }
1501                 else if(!strcmp(a, "--clear_autodownload"))
1502                 {
1503                         // mark all running downloads as "not for this map", so if they
1504                         // fail, it does not matter
1505                         Curl_Clear_forthismap();
1506                         return;
1507                 }
1508                 else if(!strcmp(a, "--finish_autodownload"))
1509                 {
1510                         if(numdownloads_added)
1511                         {
1512                                 char donecommand[256];
1513                                 if(cls.netcon)
1514                                 {
1515                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1516                                         {
1517                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1518                                                 Curl_CommandWhenDone(donecommand);
1519                                                 noclear = true;
1520                                                 CL_Disconnect();
1521                                                 noclear = false;
1522                                                 Curl_CheckCommandWhenDone();
1523                                         }
1524                                         else
1525                                                 Curl_Register_predownload();
1526                                 }
1527                         }
1528                         return;
1529                 }
1530                 else if(!strncmp(a, "--maxspeed=", 11))
1531                 {
1532                         maxspeed = atof(a + 11);
1533                 }
1534                 else if(*a == '-')
1535                 {
1536                         Con_Printf("curl: invalid option %s\n", a);
1537                         // but we ignore the option
1538                 }
1539         }
1540
1541 needthefile:
1542         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1543 }
1544
1545 /*
1546 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1547 {
1548         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1549         Z_Free(buffer);
1550 }
1551
1552 void Curl_CurlCat_f(cmd_state_t *cmd)
1553 {
1554         unsigned char *buf;
1555         const char *url = Cmd_Argv(cmd, 1);
1556         buf = Z_Malloc(16384);
1557         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1558 }
1559 */
1560
1561 /*
1562 ====================
1563 Curl_Init_Commands
1564
1565 loads the commands and cvars this library uses
1566 ====================
1567 */
1568 void Curl_Init_Commands(void)
1569 {
1570         Cvar_RegisterVariable (&cl_curl_enabled);
1571         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1572         Cvar_RegisterVariable (&cl_curl_maxspeed);
1573         Cvar_RegisterVariable (&sv_curl_defaulturl);
1574         Cvar_RegisterVariable (&sv_curl_serverpackages);
1575         Cvar_RegisterVariable (&sv_curl_maxspeed);
1576         Cvar_RegisterVariable (&cl_curl_useragent);
1577         Cvar_RegisterVariable (&cl_curl_useragent_append);
1578         Cvar_RegisterVariable (&developer_curl);
1579         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1580         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1581 }
1582
1583 /*
1584 ====================
1585 Curl_GetDownloadInfo
1586
1587 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1588 The number of elements in the array is returned in int *nDownloads.
1589 const char **additional_info may be set to a string of additional user
1590 information, or to NULL if no such display shall occur. The returned
1591 array must be freed later using Z_Free.
1592 ====================
1593 */
1594 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1595 {
1596         int i;
1597         downloadinfo *di;
1598         Curl_downloadinfo_t *downinfo;
1599
1600         if(!curl_dll)
1601         {
1602                 *nDownloads = 0;
1603                 if(additional_info)
1604                         *additional_info = NULL;
1605                 return NULL;
1606         }
1607
1608         if (curl_mutex) Thread_LockMutex(curl_mutex);
1609
1610         i = 0;
1611         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1612                 ++i;
1613
1614         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1615         i = 0;
1616         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1617         {
1618                 // do not show infobars for background downloads
1619                 if(developer.integer <= 0)
1620                         if(di->buffer)
1621                                 continue;
1622                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1623                 if(di->curle)
1624                 {
1625                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1626                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1627                         downinfo[i].queued = false;
1628                 }
1629                 else
1630                 {
1631                         downinfo[i].queued = true;
1632                 }
1633                 ++i;
1634         }
1635
1636         if(additional_info)
1637         {
1638                 // TODO: can I clear command_when_done as soon as the first download fails?
1639                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1640                 {
1641                         if(!strncmp(command_when_done, "connect ", 8))
1642                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1643                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1644                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1645                         else
1646                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1647                         *additional_info = addinfo;
1648                 }
1649                 else
1650                         *additional_info = NULL;
1651         }
1652
1653         *nDownloads = i;
1654         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1655         return downinfo;
1656 }
1657
1658
1659 /*
1660 ====================
1661 Curl_FindPackURL
1662
1663 finds the URL where to find a given package.
1664
1665 For this, it reads a file "curl_urls.txt" of the following format:
1666
1667         data*.pk3       -
1668         revdm*.pk3      http://revdm/downloads/are/here/
1669         *                       http://any/other/stuff/is/here/
1670
1671 The URLs should end in /. If not, downloads will still work, but the cached files
1672 can't be just put into the data directory with the same download configuration
1673 (you might want to do this if you want to tag downloaded files from your
1674 server, but you should not). "-" means "don't download".
1675
1676 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1677 location instead.
1678
1679 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1680 this file for obvious reasons.
1681 ====================
1682 */
1683 static const char *Curl_FindPackURL(const char *filename)
1684 {
1685         static char foundurl[1024]; // invoked only by server
1686         fs_offset_t filesize;
1687         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1688         if(buf && filesize)
1689         {
1690                 // read lines of format "pattern url"
1691                 char *p = buf;
1692                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1693                 qbool eof = false;
1694
1695                 pattern = p;
1696                 while(!eof)
1697                 {
1698                         switch(*p)
1699                         {
1700                                 case 0:
1701                                         eof = true;
1702                                         // fallthrough
1703                                 case '\n':
1704                                 case '\r':
1705                                         if(pattern && url && patternend)
1706                                         {
1707                                                 if(!urlend)
1708                                                         urlend = p;
1709                                                 *patternend = 0;
1710                                                 *urlend = 0;
1711                                                 if(matchpattern(filename, pattern, true))
1712                                                 {
1713                                                         strlcpy(foundurl, url, sizeof(foundurl));
1714                                                         Z_Free(buf);
1715                                                         return foundurl;
1716                                                 }
1717                                         }
1718                                         pattern = NULL;
1719                                         patternend = NULL;
1720                                         url = NULL;
1721                                         urlend = NULL;
1722                                         break;
1723                                 case ' ':
1724                                 case '\t':
1725                                         if(pattern && !patternend)
1726                                                 patternend = p;
1727                                         else if(url && !urlend)
1728                                                 urlend = p;
1729                                         break;
1730                                 default:
1731                                         if(!pattern)
1732                                                 pattern = p;
1733                                         else if(pattern && patternend && !url)
1734                                                 url = p;
1735                                         break;
1736                         }
1737                         ++p;
1738                 }
1739         }
1740         if(buf)
1741                 Z_Free(buf);
1742         return sv_curl_defaulturl.string;
1743 }
1744
1745 typedef struct requirement_s
1746 {
1747         struct requirement_s *next;
1748         char filename[MAX_OSPATH];
1749 }
1750 requirement;
1751 static requirement *requirements = NULL;
1752
1753
1754 /*
1755 ====================
1756 Curl_RequireFile
1757
1758 Adds the given file to the list of requirements.
1759 ====================
1760 */
1761 void Curl_RequireFile(const char *filename)
1762 {
1763         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1764         req->next = requirements;
1765         strlcpy(req->filename, filename, sizeof(req->filename));
1766         requirements = req;
1767 }
1768
1769 /*
1770 ====================
1771 Curl_ClearRequirements
1772
1773 Clears the list of required files for playing on the current map.
1774 This should be called at every map change.
1775 ====================
1776 */
1777 void Curl_ClearRequirements(void)
1778 {
1779         while(requirements)
1780         {
1781                 requirement *req = requirements;
1782                 requirements = requirements->next;
1783                 Z_Free(req);
1784         }
1785 }
1786
1787 /*
1788 ====================
1789 Curl_SendRequirements
1790
1791 Makes the current host_clients download all files he needs.
1792 This is done by sending him the following console commands:
1793
1794         curl --clear_autodownload
1795         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1796         curl --finish_autodownload
1797 ====================
1798 */
1799 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1800 {
1801         const char *p;
1802         const char *thispack = FS_WhichPack(filename);
1803         const char *packurl;
1804
1805         if(!thispack || !*thispack)
1806                 return false;
1807
1808         p = strrchr(thispack, '/');
1809         if(p)
1810                 thispack = p + 1;
1811
1812         packurl = Curl_FindPackURL(thispack);
1813
1814         if(packurl && *packurl && strcmp(packurl, "-"))
1815         {
1816                 if(!foundone)
1817                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1818
1819                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1820                 strlcat(sendbuffer, thispack, sendbuffer_len);
1821                 if(sv_curl_maxspeed.value > 0)
1822                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1823                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1824                 strlcat(sendbuffer, filename, sendbuffer_len);
1825                 strlcat(sendbuffer, " ", sendbuffer_len);
1826                 strlcat(sendbuffer, packurl, sendbuffer_len);
1827                 strlcat(sendbuffer, thispack, sendbuffer_len);
1828                 strlcat(sendbuffer, "\n", sendbuffer_len);
1829
1830                 return true;
1831         }
1832
1833         return false;
1834 }
1835 void Curl_SendRequirements(void)
1836 {
1837         // for each requirement, find the pack name
1838         char sendbuffer[4096] = "";
1839         requirement *req;
1840         qbool foundone = false;
1841         const char *p;
1842
1843         for(req = requirements; req; req = req->next)
1844                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1845
1846         p = sv_curl_serverpackages.string;
1847         while(COM_ParseToken_Simple(&p, false, false, true))
1848                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1849
1850         if(foundone)
1851                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1852
1853         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1854                 SV_ClientCommands("%s", sendbuffer);
1855         else
1856                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1857 }