libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5 #include "com_list.h"
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t curl_enabled = {CF_SHARED | CF_ARCHIVE, "curl_enabled","1", "whether libcurl may be used to GET files or POST data"};
  11 static cvar_t curl_maxdownloads = {CF_SHARED | CF_ARCHIVE, "curl_maxdownloads","3", "maximum number of concurrent HTTP/FTP downloads"};
  12 static cvar_t curl_maxspeed = {CF_SHARED | CF_ARCHIVE, "curl_maxspeed","0", "maximum download speed (KiB/s)"};
  13 static cvar_t curl_useragent = {CF_SHARED, "curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  14 static cvar_t curl_useragent_append = {CF_SHARED, "curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  15
  16 static cvar_t sv_curl_defaulturl = {CF_SERVER, "sv_curl_defaulturl","", "default autodownload source URL"};
  17 static cvar_t sv_curl_serverpackages = {CF_SERVER, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  18 static cvar_t sv_curl_maxspeed = {CF_SERVER, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  19
  20 static cvar_t developer_curl = {CF_SHARED, "developer_curl","0", "whether verbose libcurl output should be printed to stderr"};
  21
  22 /*
  23 =================================================================
  24
  25   Minimal set of definitions from libcurl
  26
  27   WARNING: for a matter of simplicity, several pointer types are
  28   casted to "void*", and most enumerated values are not included
  29
  30 =================================================================
  31 */
  32
  33 typedef struct CURL_s CURL;
  34 typedef struct CURLM_s CURLM;
  35 typedef struct curl_slist curl_slist;
  36 typedef enum
  37 {
  38         CURLE_OK = 0
  39 }
  40 CURLcode;
  41 typedef enum
  42 {
  43         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  44         CURLM_OK = 0
  45 }
  46 CURLMcode;
  47 #define CURL_GLOBAL_NOTHING 0
  48 #define CURL_GLOBAL_SSL 1
  49 #define CURL_GLOBAL_WIN32 2
  50 #define CURLOPTTYPE_LONG          0
  51 #define CURLOPTTYPE_OBJECTPOINT   10000
  52 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  53 #define CURLOPTTYPE_OFF_T         30000
  54 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  55 typedef enum
  56 {
  57         CINIT(WRITEDATA, OBJECTPOINT, 1),
  58         CINIT(URL,  OBJECTPOINT, 2),
  59         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  60         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  61         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  62         CINIT(REFERER, OBJECTPOINT, 16),
  63         CINIT(USERAGENT, OBJECTPOINT, 18),
  64         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  65         CINIT(LOW_SPEED_TIME, LONG, 20),
  66         CINIT(RESUME_FROM, LONG, 21),
  67         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  68         CINIT(VERBOSE, LONG, 41),
  69         CINIT(POST, LONG, 47),         /* HTTP POST method */
  70         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  71         CINIT(POSTFIELDSIZE, LONG, 60),
  72         CINIT(PRIVATE, OBJECTPOINT, 103),
  73         CINIT(PROTOCOLS, LONG, 181),
  74         CINIT(REDIR_PROTOCOLS, LONG, 182)
  75 }
  76 CURLoption;
  77 #define CURLPROTO_HTTP   (1<<0)
  78 #define CURLPROTO_HTTPS  (1<<1)
  79 #define CURLPROTO_FTP    (1<<2)
  80 typedef enum
  81 {
  82         CURLINFO_TEXT = 0,
  83         CURLINFO_HEADER_IN,    /* 1 */
  84         CURLINFO_HEADER_OUT,   /* 2 */
  85         CURLINFO_DATA_IN,      /* 3 */
  86         CURLINFO_DATA_OUT,     /* 4 */
  87         CURLINFO_SSL_DATA_IN,  /* 5 */
  88         CURLINFO_SSL_DATA_OUT, /* 6 */
  89         CURLINFO_END
  90 }
  91 curl_infotype;
  92 #define CURLINFO_STRING   0x100000
  93 #define CURLINFO_LONG     0x200000
  94 #define CURLINFO_DOUBLE   0x300000
  95 #define CURLINFO_SLIST    0x400000
  96 #define CURLINFO_MASK     0x0fffff
  97 #define CURLINFO_TYPEMASK 0xf00000
  98 typedef enum
  99 {
 100         CURLINFO_NONE, /* first, never use this */
 101         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
 102         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
 103         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 104         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 105         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 106         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 107         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 108         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 109         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 110         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 111         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 112         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 113         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 114         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 115         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 116         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 117         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 118         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 119         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 120         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 121         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 122         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 123         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 124         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 125         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 126         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 127         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 128 }
 129 CURLINFO;
 130
 131 typedef enum
 132 {
 133         CURLMSG_NONE, /* first, not used */
 134         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 135                                          the CURLcode of the transfer */
 136         CURLMSG_LAST
 137 }
 138 CURLMSG;
 139 typedef struct
 140 {
 141         CURLMSG msg;       /* what this message means */
 142         CURL *easy_handle; /* the handle it concerns */
 143         union
 144         {
 145                 void *whatever;    /* message-specific data */
 146                 CURLcode result;   /* return code for transfer */
 147         }
 148         data;
 149 }
 150 CURLMsg;
 151
 152 static void (*qcurl_global_init) (long flags);
 153 static void (*qcurl_global_cleanup) (void);
 154
 155 static CURL * (*qcurl_easy_init) (void);
 156 static void (*qcurl_easy_cleanup) (CURL *handle);
 157 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 158 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 159 static const char * (*qcurl_easy_strerror) (CURLcode);
 160
 161 static CURLM * (*qcurl_multi_init) (void);
 162 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 163 static CURLMcode (*qcurl_multi_wait) (CURLM *multi_handle, void*, unsigned int extra_nfds, int timeout_ms, int *ret);
 164 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 165 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 166 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 167 static void (*qcurl_multi_cleanup) (CURLM *);
 168 static const char * (*qcurl_multi_strerror) (CURLcode);
 169 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 170 static void (*qcurl_slist_free_all) (curl_slist *list);
 171
 172 static dllfunction_t curlfuncs[] =
 173 {
 174         {"curl_global_init",            (void **) &qcurl_global_init},
 175         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 176         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 177         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 178         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 179         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 180         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 181         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 182         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 183         {"curl_multi_wait",             (void **) &qcurl_multi_wait},
 184         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 185         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 186         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 187         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 188         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 189         {"curl_slist_append",           (void **) &qcurl_slist_append},
 190         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 191         {NULL, NULL}
 192 };
 193
 194 // Handle for CURL DLL
 195 static dllhandle_t curl_dll = NULL;
 196 // will be checked at many places to find out if qcurl calls are allowed
 197
 198 #define LOADTYPE_NONE 0
 199 #define LOADTYPE_PAK 1
 200 #define LOADTYPE_CACHEPIC 2
 201 #define LOADTYPE_SKINFRAME 3
 202
 203 void *curl_mutex = NULL;
 204
 205 typedef struct downloadinfo_s
 206 {
 207         char filename[MAX_OSPATH];
 208         char url[1024];
 209         char referer[256];
 210         qfile_t *stream;
 211         fs_offset_t startpos;
 212         CURL *curle;
 213         qbool started;
 214         int loadtype;
 215         size_t bytes_received; // for buffer
 216         double bytes_received_curl; // for throttling
 217         double bytes_sent_curl; // for throttling
 218         llist_t list;
 219         qbool forthismap;
 220         double maxspeed;
 221         curl_slist *slist; // http headers
 222
 223         unsigned char *buffer;
 224         size_t buffersize;
 225         curl_callback_t callback;
 226         void *callback_data;
 227
 228         const unsigned char *postbuf;
 229         size_t postbufsize;
 230         const char *post_content_type;
 231         const char *extraheaders;
 232 }
 233 downloadinfo;
 234 LIST_HEAD(downloads);
 235 static int numdownloads = 0;
 236
 237 static qbool noclear = false;
 238
 239 static int numdownloads_fail = 0;
 240 static int numdownloads_success = 0;
 241 static int numdownloads_added = 0;
 242 static char command_when_done[256] = "";
 243 static char command_when_error[256] = "";
 244
 245 /*
 246 ====================
 247 Curl_CommandWhenDone
 248
 249 Sets the command which is to be executed when the last download completes AND
 250 all downloads since last server connect ended with a successful status.
 251 Setting the command to NULL clears it.
 252 ====================
 253 */
 254 static void Curl_CommandWhenDone(const char *cmd)
 255 {
 256         if(!curl_dll)
 257                 return;
 258         if(cmd)
 259                 dp_strlcpy(command_when_done, cmd, sizeof(command_when_done));
 260         else
 261                 *command_when_done = 0;
 262 }
 263
 264 /*
 265 FIXME
 266 Do not use yet. Not complete.
 267 Problem: what counts as an error?
 268 */
 269
 270 static void Curl_CommandWhenError(const char *cmd)
 271 {
 272         if(!curl_dll)
 273                 return;
 274         if(cmd)
 275                 dp_strlcpy(command_when_error, cmd, sizeof(command_when_error));
 276         else
 277                 *command_when_error = 0;
 278 }
 279
 280 /*
 281 ====================
 282 Curl_Clear_forthismap
 283
 284 Clears the "will disconnect on failure" flags.
 285 ====================
 286 */
 287 void Curl_Clear_forthismap(void)
 288 {
 289         downloadinfo *di;
 290         if(noclear)
 291                 return;
 292         if (curl_mutex) Thread_LockMutex(curl_mutex);
 293         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 294                 di->forthismap = false;
 295         Curl_CommandWhenError(NULL);
 296         Curl_CommandWhenDone(NULL);
 297         numdownloads_fail = 0;
 298         numdownloads_success = 0;
 299         numdownloads_added = 0;
 300         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 301 }
 302
 303 /*
 304 ====================
 305 Curl_Have_forthismap
 306
 307 Returns true if a download needed for the current game is running.
 308 ====================
 309 */
 310 qbool Curl_Have_forthismap(void)
 311 {
 312         return numdownloads_added != 0;
 313 }
 314
 315 void Curl_Register_predownload(void)
 316 {
 317         if (curl_mutex) Thread_LockMutex(curl_mutex);
 318         Curl_CommandWhenDone("cl_begindownloads");
 319         Curl_CommandWhenError("cl_begindownloads");
 320         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 321 }
 322
 323 /*
 324 ====================
 325 Curl_CheckCommandWhenDone
 326
 327 Checks if a "done command" is to be executed.
 328 All downloads finished, at least one success since connect, no single failure
 329 -> execute the command.
 330 */
 331 static void Curl_CheckCommandWhenDone(void)
 332 {
 333         if(!curl_dll)
 334                 return;
 335         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 336         {
 337                 if(numdownloads_fail == 0)
 338                 {
 339                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 340                         Cbuf_AddText(cmd_local, "\n");
 341                         Cbuf_AddText(cmd_local, command_when_done);
 342                         Cbuf_AddText(cmd_local, "\n");
 343                 }
 344                 else
 345                 {
 346                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 347                         Cbuf_AddText(cmd_local, "\n");
 348                         Cbuf_AddText(cmd_local, command_when_error);
 349                         Cbuf_AddText(cmd_local, "\n");
 350                 }
 351                 Curl_Clear_forthismap();
 352         }
 353 }
 354
 355 /*
 356 ====================
 357 CURL_CloseLibrary
 358
 359 Load the cURL DLL
 360 ====================
 361 */
 362 static qbool CURL_OpenLibrary (void)
 363 {
 364         const char* dllnames [] =
 365         {
 366 #if defined(WIN32)
 367                 "libcurl-4.dll",
 368                 "libcurl-3.dll",
 369 #elif defined(MACOSX)
 370                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 371                 "libcurl.3.dylib", // Mac OS X Tiger
 372                 "libcurl.2.dylib", // Mac OS X Panther
 373 #else
 374                 "libcurl.so.4",
 375                 "libcurl.so.3",
 376                 "libcurl.so", // FreeBSD
 377 #endif
 378                 NULL
 379         };
 380
 381         // Already loaded?
 382         if (curl_dll)
 383                 return true;
 384
 385         // Load the DLL
 386         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
 387 }
 388
 389
 390 /*
 391 ====================
 392 CURL_CloseLibrary
 393
 394 Unload the cURL DLL
 395 ====================
 396 */
 397 static void CURL_CloseLibrary (void)
 398 {
 399         Sys_FreeLibrary (&curl_dll);
 400 }
 401
 402
 403 static CURLM *curlm = NULL;
 404 static double bytes_received = 0; // used for bandwidth throttling
 405 static double bytes_sent = 0; // used for bandwidth throttling
 406 static double curltime = 0;
 407
 408 /*
 409 ====================
 410 CURL_fwrite
 411
 412 fwrite-compatible function that writes the data to a file. libcurl can call
 413 this.
 414 ====================
 415 */
 416 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 417 {
 418         fs_offset_t ret = -1;
 419         size_t bytes = size * nmemb;
 420         downloadinfo *di = (downloadinfo *) vdi;
 421
 422         if(di->buffer)
 423         {
 424                 if(di->bytes_received + bytes <= di->buffersize)
 425                 {
 426                         memcpy(di->buffer + di->bytes_received, data, bytes);
 427                         ret = bytes;
 428                 }
 429                 // otherwise: buffer overrun, ret stays -1
 430         }
 431
 432         if(di->stream)
 433         {
 434                 ret = FS_Write(di->stream, data, bytes);
 435         }
 436
 437         di->bytes_received += bytes;
 438
 439         //Con_Printf("CURL_fwrite callback timestamp: %f bytes: %ld\n", host.realtime, ret);
 440
 441         return ret;
 442         // Why not ret / nmemb?
 443         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 444         // Yes, this is incompatible to fwrite(2).
 445 }
 446
 447 typedef enum
 448 {
 449         CURL_DOWNLOAD_SUCCESS = 0,
 450         CURL_DOWNLOAD_FAILED,
 451         CURL_DOWNLOAD_ABORTED,
 452         CURL_DOWNLOAD_SERVERERROR
 453 }
 454 CurlStatus;
 455
 456 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 457 {
 458         downloadinfo *di = (downloadinfo *) cbdata;
 459         switch(status)
 460         {
 461                 case CURLCBSTATUS_OK:
 462                         Con_DPrintf("Download of %s: OK\n", di->filename);
 463                         break;
 464                 case CURLCBSTATUS_FAILED:
 465                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 466                         break;
 467                 case CURLCBSTATUS_ABORTED:
 468                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 469                         break;
 470                 case CURLCBSTATUS_SERVERERROR:
 471                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 472                         break;
 473                 case CURLCBSTATUS_UNKNOWN:
 474                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 475                         break;
 476                 default:
 477                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 478                         break;
 479         }
 480 }
 481
 482 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 483 {
 484         curl_default_callback(status, length_received, buffer, cbdata);
 485 }
 486
 487 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 488 {
 489         unsigned char *pixels = NULL;
 490         fs_offset_t filesize = 0;
 491         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 492         if(data)
 493         {
 494                 int mip = 0;
 495                 if(!strcmp(content_type, "image/jpeg"))
 496                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 497                 else if(!strcmp(content_type, "image/png"))
 498                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 499                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 500                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 501                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 502                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 503                 else
 504                         Con_Printf("Did not detect content type: %s\n", content_type);
 505                 Mem_Free(data);
 506         }
 507         // do we call Image_MakeLinearColorsFromsRGB or not?
 508         return pixels;
 509 }
 510
 511 /*
 512 ====================
 513 Curl_EndDownload
 514
 515 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 516 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 517 code from libcurl, or 0, if another error has occurred.
 518 ====================
 519 */
 520 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 521 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 522 {
 523         char content_type[64];
 524         qbool ok = false;
 525         if(!curl_dll)
 526                 return;
 527         switch(status)
 528         {
 529                 case CURL_DOWNLOAD_SUCCESS:
 530                         ok = true;
 531                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 532                         break;
 533                 case CURL_DOWNLOAD_FAILED:
 534                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 535                         break;
 536                 case CURL_DOWNLOAD_ABORTED:
 537                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 538                         break;
 539                 case CURL_DOWNLOAD_SERVERERROR:
 540                         // reopen to enforce it to have zero bytes again
 541                         if(di->stream)
 542                         {
 543                                 FS_Close(di->stream);
 544                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 545                         }
 546
 547                         if(di->callback)
 548                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 549                         break;
 550                 default:
 551                         if(di->callback)
 552                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 553                         break;
 554         }
 555         if(content_type_)
 556                 dp_strlcpy(content_type, content_type_, sizeof(content_type));
 557         else
 558                 *content_type = 0;
 559
 560         if(di->curle)
 561         {
 562                 qcurl_multi_remove_handle(curlm, di->curle);
 563                 qcurl_easy_cleanup(di->curle);
 564                 if(di->slist)
 565                         qcurl_slist_free_all(di->slist);
 566         }
 567
 568         if(!di->callback && ok && !di->bytes_received)
 569         {
 570                 Con_Printf("ERROR: empty file\n");
 571                 ok = false;
 572         }
 573
 574         if(di->stream)
 575                 FS_Close(di->stream);
 576
 577 #define CLEAR_AND_RETRY() \
 578         do \
 579         { \
 580                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 581                 FS_Close(di->stream); \
 582                 if(di->startpos && !di->callback) \
 583                 { \
 584                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 585                         di->forthismap = false; \
 586                 } \
 587         } \
 588         while(0)
 589
 590         if(ok && di->loadtype == LOADTYPE_PAK)
 591         {
 592                 ok = FS_AddPack(di->filename, NULL, true, true);
 593                 if(!ok)
 594                         CLEAR_AND_RETRY();
 595         }
 596         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 597         {
 598                 const char *p;
 599                 unsigned char *pixels = NULL;
 600
 601                 p = di->filename;
 602 #ifdef WE_ARE_EVIL
 603                 if(!strncmp(p, "dlcache/", 8))
 604                         p += 8;
 605 #endif
 606
 607                 pixels = decode_image(di, content_type);
 608                 if(pixels)
 609                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 610                 else
 611                         CLEAR_AND_RETRY();
 612         }
 613         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 614         {
 615                 const char *p;
 616                 unsigned char *pixels = NULL;
 617
 618                 p = di->filename;
 619 #ifdef WE_ARE_EVIL
 620                 if(!strncmp(p, "dlcache/", 8))
 621                         p += 8;
 622 #endif
 623
 624                 pixels = decode_image(di, content_type);
 625                 if(pixels)
 626                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 627                 else
 628                         CLEAR_AND_RETRY();
 629         }
 630
 631         List_Delete(&di->list);
 632
 633         --numdownloads;
 634         if(di->forthismap)
 635         {
 636                 if(ok)
 637                         ++numdownloads_success;
 638                 else
 639                         ++numdownloads_fail;
 640         }
 641         Z_Free(di);
 642 }
 643
 644 /*
 645 ====================
 646 CleanURL
 647
 648 Returns a "cleaned up" URL for display (to strip login data)
 649 ====================
 650 */
 651 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 652 {
 653         const char *p, *q, *r;
 654
 655         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 656         p = strstr(url, "://");
 657         if(p)
 658         {
 659                 q = strchr(p + 3, '@');
 660                 if(q)
 661                 {
 662                         r = strchr(p + 3, '/');
 663                         if(!r || q < r)
 664                         {
 665                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 666                                 return urlbuf;
 667                         }
 668                 }
 669         }
 670
 671         return url;
 672 }
 673
 674 /*
 675 ====================
 676 CheckPendingDownloads
 677
 678 checks if there are free download slots to start new downloads in.
 679 To not start too many downloads at once, only one download is added at a time,
 680 up to a maximum number of cl_curl_maxdownloads are running.
 681 ====================
 682 */
 683 static void CheckPendingDownloads(void)
 684 {
 685         const char *h;
 686         char urlbuf[1024];
 687         char vabuf[1024];
 688         if(!curl_dll)
 689                 return;
 690         if(numdownloads < curl_maxdownloads.integer)
 691         {
 692                 downloadinfo *di;
 693                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
 694                 {
 695                         if(!di->started)
 696                         {
 697                                 if(!di->buffer)
 698                                 {
 699                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 700
 701                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 702                                         if(!di->stream)
 703                                         {
 704                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 705                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 706                                                 return;
 707                                         }
 708                                         FS_Seek(di->stream, 0, SEEK_END);
 709                                         di->startpos = FS_Tell(di->stream);
 710
 711                                         if(di->startpos > 0)
 712                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 713                                         Con_Print("...\n");
 714                                 }
 715                                 else
 716                                 {
 717                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 718                                         di->startpos = 0;
 719                                 }
 720
 721                                 di->curle = qcurl_easy_init();
 722                                 di->slist = NULL;
 723                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 724                                 if(curl_useragent.integer)
 725                                 {
 726                                         const char *ua
 727 #ifdef HTTP_USER_AGENT
 728                                                 = HTTP_USER_AGENT;
 729 #else
 730                                                 = engineversion;
 731 #endif
 732                                         if(!ua)
 733                                                 ua = "";
 734                                         if(*curl_useragent_append.string)
 735                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 736                                                         ua,
 737                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 738                                                                 ? " "
 739                                                                 : "",
 740                                                         curl_useragent_append.string);
 741                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 742                                 }
 743                                 else
 744                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 745                                 if(developer_curl.integer)
 746                                         qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 750                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 751                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 752                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 753                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 754                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 755                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 756                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 757                                 {
 758                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 759                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 760                                 }
 761                                 if(di->post_content_type)
 762                                 {
 763                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 764                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 765                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 766                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 767                                 }
 768
 769                                 // parse extra headers into slist
 770                                 // \n separated list!
 771                                 h = di->extraheaders;
 772                                 while(h)
 773                                 {
 774                                         const char *hh = strchr(h, '\n');
 775                                         if(hh)
 776                                         {
 777                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 778                                                 memcpy(buf, h, hh - h);
 779                                                 buf[hh - h] = 0;
 780                                                 di->slist = qcurl_slist_append(di->slist, buf);
 781                                                 h = hh + 1;
 782                                         }
 783                                         else
 784                                         {
 785                                                 di->slist = qcurl_slist_append(di->slist, h);
 786                                                 h = NULL;
 787                                         }
 788                                 }
 789
 790                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 791
 792                                 qcurl_multi_add_handle(curlm, di->curle);
 793                                 di->started = true;
 794                                 ++numdownloads;
 795                                 if(numdownloads >= curl_maxdownloads.integer)
 796                                         break;
 797                         }
 798                 }
 799         }
 800 }
 801
 802 /*
 803 ====================
 804 Curl_Init
 805
 806 this function MUST be called before using anything else in this file.
 807 On Win32, this must be called AFTER WSAStartup has been done!
 808 ====================
 809 */
 810 void Curl_Init(void)
 811 {
 812         CURL_OpenLibrary();
 813         if(!curl_dll)
 814                 return;
 815         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 816         qcurl_global_init(CURL_GLOBAL_SSL);
 817         curlm = qcurl_multi_init();
 818 }
 819
 820 /*
 821 ====================
 822 Curl_Shutdown
 823
 824 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 825 ====================
 826 */
 827 void Curl_ClearRequirements(void);
 828 void Curl_Shutdown(void)
 829 {
 830         if(!curl_dll)
 831                 return;
 832         Curl_ClearRequirements();
 833         Curl_CancelAll();
 834         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 835         CURL_CloseLibrary();
 836         curl_dll = NULL;
 837 }
 838
 839 // for VM_checkextension()
 840 qbool Curl_Available(void)
 841 {
 842         return curl_dll ? true : false;
 843 }
 844
 845 /*
 846 ====================
 847 Curl_Find
 848
 849 Finds the internal information block for a download given by file name.
 850 ====================
 851 */
 852 static downloadinfo *Curl_Find(const char *filename)
 853 {
 854         downloadinfo *di;
 855         if(!curl_dll)
 856                 return NULL;
 857         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 858                 if(!strcasecmp(di->filename, filename))
 859                         return di;
 860         return NULL;
 861 }
 862
 863 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 864 {
 865         downloadinfo *di, *ndi;
 866         if(!curl_dll)
 867                 return;
 868         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
 869         {
 870                 if(di->callback == callback && di->callback_data == cbdata)
 871                 {
 872                         di->callback = curl_quiet_callback; // do NOT call the callback
 873                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 874                 }
 875         }
 876 }
 877
 878 /*
 879 ====================
 880 Curl_Begin
 881
 882 Starts a download of a given URL to the file name portion of this URL (or name
 883 if given) in the "dlcache/" folder.
 884 ====================
 885 */
 886 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 887 {
 888         if(buf)
 889                 if(loadtype != LOADTYPE_NONE)
 890                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 891
 892         if(!curl_dll || !curl_enabled.integer)
 893         {
 894                 return false;
 895         }
 896         else
 897         {
 898                 char fn[MAX_OSPATH];
 899                 char urlbuf[1024];
 900                 const char *p, *q;
 901                 size_t length;
 902                 downloadinfo *di;
 903
 904                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 905                 p = strchr(URL, ':');
 906                 if(p)
 907                 {
 908                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 909                         {
 910                                 char addressstring[128];
 911                                 *addressstring = 0;
 912                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 913                                 q = strchr(addressstring, ':');
 914                                 if(!q)
 915                                         q = addressstring + strlen(addressstring);
 916                                 if(*addressstring)
 917                                 {
 918                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 919                                         URL = urlbuf;
 920                                 }
 921                         }
 922                 }
 923
 924                 // Note: This extraction of the file name portion is NOT entirely correct.
 925                 //
 926                 // It does the following:
 927                 //
 928                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 929                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 930                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 931                 //
 932                 // However, I'd like to keep this "buggy" behavior so that PHP script
 933                 // authors can write download scripts without having to enable
 934                 // AcceptPathInfo on Apache. They just have to ensure that their script
 935                 // can be called with such a "fake" path name like
 936                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 937                 //
 938                 // By the way, such PHP scripts should either send the file or a
 939                 // "Location:" redirect; PHP code example:
 940                 //
 941                 //   header("Location: http://www.example.com/");
 942                 //
 943                 // By the way, this will set User-Agent to something like
 944                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 945                 // dp://serverhost:serverport/ so you can filter on this; an example
 946                 // httpd log file line might be:
 947                 //
 948                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 949
 950                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 951
 952                 if(buf)
 953                 {
 954                         if(!name)
 955                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 956                 }
 957                 else
 958                 {
 959                         if(!name)
 960                         {
 961                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 962                                 p = strrchr(name, '/');
 963                                 p = p ? (p+1) : name;
 964                                 q = strchr(p, '?');
 965                                 length = q ? (size_t)(q - p) : strlen(p);
 966                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 967                         }
 968                         else
 969                         {
 970                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 971                         }
 972
 973                         name = fn; // make it point back
 974
 975                         // already downloading the file?
 976                         {
 977                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 978                                 if(existingdownloadinfo)
 979                                 {
 980                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 981
 982                                         // however, if it was not for this map yet...
 983                                         if(forthismap && !existingdownloadinfo->forthismap)
 984                                         {
 985                                                 existingdownloadinfo->forthismap = true;
 986                                                 // this "fakes" a download attempt so the client will wait for
 987                                                 // the download to finish and then reconnect
 988                                                 ++numdownloads_added;
 989                                         }
 990
 991                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 992                                         return false;
 993                                 }
 994                         }
 995
 996                         if(FS_FileExists(fn))
 997                         {
 998                                 if(loadtype == LOADTYPE_PAK)
 999                                 {
1000                                         qbool already_loaded;
1001                                         if(FS_AddPack(fn, &already_loaded, true, true))
1002                                         {
1003                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
1004                                                 if(already_loaded)
1005                                                         Con_DPrintf("(pak was already loaded)\n");
1006                                                 else
1007                                                 {
1008                                                         if(forthismap)
1009                                                         {
1010                                                                 ++numdownloads_added;
1011                                                                 ++numdownloads_success;
1012                                                         }
1013                                                 }
1014
1015                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1016                                                 return false;
1017                                         }
1018                                         else
1019                                         {
1020                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1021                                                 if(f)
1022                                                 {
1023                                                         char b[4] = {0};
1024                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1025
1026                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1027                                                         {
1028                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1029                                                                 FS_Close(f);
1030                                                                 f = FS_OpenRealFile(fn, "wb", false);
1031                                                                 if(f)
1032                                                                         FS_Close(f);
1033                                                         }
1034                                                         else
1035                                                         {
1036                                                                 // OK
1037                                                                 FS_Close(f);
1038                                                         }
1039                                                 }
1040                                         }
1041                                 }
1042                                 else
1043                                 {
1044                                         // never resume these
1045                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1046                                         if(f)
1047                                                 FS_Close(f);
1048                                 }
1049                         }
1050                 }
1051
1052                 // if we get here, we actually want to download... so first verify the
1053                 // URL scheme (so one can't read local files using file://)
1054                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1055                 {
1056                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1057                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1058                         return false;
1059                 }
1060
1061                 if(forthismap)
1062                         ++numdownloads_added;
1063                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1064                 dp_strlcpy(di->filename, name, sizeof(di->filename));
1065                 dp_strlcpy(di->url, URL, sizeof(di->url));
1066                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1067                 di->forthismap = forthismap;
1068                 di->stream = NULL;
1069                 di->startpos = 0;
1070                 di->curle = NULL;
1071                 di->started = false;
1072                 di->loadtype = loadtype;
1073                 di->maxspeed = maxspeed;
1074                 di->bytes_received = 0;
1075                 di->bytes_received_curl = 0;
1076                 di->bytes_sent_curl = 0;
1077                 di->extraheaders = extraheaders;
1078                 di->buffer = buf;
1079                 di->buffersize = bufsize;
1080                 if(callback == NULL)
1081                 {
1082                         di->callback = curl_default_callback;
1083                         di->callback_data = di;
1084                 }
1085                 else
1086                 {
1087                         di->callback = callback;
1088                         di->callback_data = cbdata;
1089                 }
1090
1091                 if(post_content_type)
1092                 {
1093                         di->post_content_type = post_content_type;
1094                         di->postbuf = postbuf;
1095                         di->postbufsize = postbufsize;
1096                 }
1097                 else
1098                 {
1099                         di->post_content_type = NULL;
1100                         di->postbuf = NULL;
1101                         di->postbufsize = 0;
1102                 }
1103
1104                 List_Add(&di->list, &downloads);
1105
1106                 if (curl_mutex)
1107                         Thread_UnlockMutex(curl_mutex);
1108
1109                 return true;
1110         }
1111 }
1112
1113 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1114 {
1115         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1116 }
1117 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1118 {
1119         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1120 }
1121 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1122 {
1123         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1124 }
1125
1126 /*
1127 ====================
1128 Curl_Frame
1129
1130 call this regularily as this will always download as much as possible without
1131 blocking.
1132 ====================
1133 */
1134 void Curl_Frame(void)
1135 {
1136         double maxspeed;
1137         downloadinfo *di;
1138
1139         noclear = false;
1140
1141         if(!curl_enabled.integer && cls.state != ca_dedicated)
1142                 return;
1143
1144         if(!curl_dll)
1145                 return;
1146
1147         if (curl_mutex) Thread_LockMutex(curl_mutex);
1148
1149         Curl_CheckCommandWhenDone();
1150
1151         if(List_Is_Empty(&downloads))
1152         {
1153                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1154                 return;
1155         }
1156
1157         if(host.realtime < curltime) // throttle
1158         {
1159                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1160                 return;
1161         }
1162
1163         {
1164                 int remaining;
1165                 CURLMcode mc;
1166
1167                 do
1168                 {
1169                         mc = qcurl_multi_perform(curlm, &remaining);
1170                 }
1171                 while(mc == CURLM_CALL_MULTI_PERFORM);
1172
1173                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1174                 {
1175                         double b = 0;
1176                         if(di->curle)
1177                         {
1178                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1179                                 bytes_sent += (b - di->bytes_sent_curl);
1180                                 di->bytes_sent_curl = b;
1181                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1182                                 bytes_sent += (b - di->bytes_received_curl);
1183                                 di->bytes_received_curl = b;
1184                         }
1185                 }
1186
1187                 for(;;)
1188                 {
1189                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1190                         if(!msg)
1191                                 break;
1192                         if(msg->msg == CURLMSG_DONE)
1193                         {
1194                                 const char *ct = NULL;
1195                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1196                                 CURLcode result;
1197                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1198                                 result = msg->data.result;
1199                                 if(result)
1200                                 {
1201                                         failed = CURL_DOWNLOAD_FAILED;
1202                                 }
1203                                 else
1204                                 {
1205                                         long code;
1206                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1207                                         switch(code / 100)
1208                                         {
1209                                                 case 4: // e.g. 404?
1210                                                 case 5: // e.g. 500?
1211                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1212                                                         result = (CURLcode) code;
1213                                                         break;
1214                                         }
1215                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1216                                 }
1217
1218                                 Curl_EndDownload(di, failed, result, ct);
1219                         }
1220                 }
1221         }
1222
1223         CheckPendingDownloads();
1224
1225         // when will we curl the next time?
1226         // we will wait a bit to ensure our download rate is kept.
1227         // we now know that realtime >= curltime... so set up a new curltime
1228
1229         // use the slowest allowing download to derive the maxspeed... this CAN
1230         // be done better, but maybe later
1231         maxspeed = curl_maxspeed.value;
1232         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1233                 if(di->maxspeed > 0)
1234                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1235                                 maxspeed = di->maxspeed;
1236
1237         if(maxspeed > 0)
1238         {
1239                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1240                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1241                 bytes_sent = 0;
1242                 bytes_received = 0;
1243         }
1244         else
1245                 curltime = host.realtime;
1246
1247         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1248 }
1249
1250 /*
1251 ====================
1252 Curl_Select
1253
1254 Sleeps until there's some transfer progress or a timeout is reached,
1255 unfortunately the timeout is only in milliseconds.
1256 This allows good throughput even at very low FPS.
1257 Less important on newer libcurl versions but still helps.
1258
1259 Returns 0 immediately if there's no transfers to wait for,
1260 or > 0 if a transfer is ready or the timeout was reached.
1261 ====================
1262 */
1263 int Curl_Select(int timeout_ms)
1264 {
1265         CURLMcode err;
1266         int numfds;
1267
1268         if (List_Is_Empty(&downloads))
1269                 return 0;
1270
1271         err = qcurl_multi_wait(curlm, NULL, 0, timeout_ms, &numfds);
1272         if (err == CURLM_OK)
1273                 return numfds;
1274         Con_Printf("curl_multi_wait() failed, code %d\n", err);
1275         return 0;
1276 }
1277
1278 /*
1279 ====================
1280 Curl_CancelAll
1281
1282 Stops ALL downloads.
1283 ====================
1284 */
1285 void Curl_CancelAll(void)
1286 {
1287         if(!curl_dll)
1288                 return;
1289
1290         if (curl_mutex) Thread_LockMutex(curl_mutex);
1291
1292         while(!List_Is_Empty(&downloads))
1293         {
1294                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1295                 // INVARIANT: downloads will point to the next download after that!
1296         }
1297
1298         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1299 }
1300
1301 /*
1302 ====================
1303 Curl_Running
1304
1305 returns true if there is a download running.
1306 ====================
1307 */
1308 qbool Curl_Running(void)
1309 {
1310         if(!curl_dll)
1311                 return false;
1312
1313         return !List_Is_Empty(&downloads);
1314 }
1315
1316 /*
1317 ====================
1318 Curl_GetDownloadAmount
1319
1320 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1321 for the given download.
1322 ====================
1323 */
1324 static double Curl_GetDownloadAmount(downloadinfo *di)
1325 {
1326         if(!curl_dll)
1327                 return -2;
1328         if(di->curle)
1329         {
1330                 double length;
1331                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1332                 if(length > 0)
1333                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1334                 else
1335                         return 0;
1336         }
1337         else
1338                 return -1;
1339 }
1340
1341 /*
1342 ====================
1343 Curl_GetDownloadSpeed
1344
1345 returns the speed of the given download in bytes per second
1346 ====================
1347 */
1348 static double Curl_GetDownloadSpeed(downloadinfo *di)
1349 {
1350         if(!curl_dll)
1351                 return -2;
1352         if(di->curle)
1353         {
1354                 double speed;
1355                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1356                 return speed;
1357         }
1358         else
1359                 return -1;
1360 }
1361
1362 /*
1363 ====================
1364 Curl_Info_f
1365
1366 prints the download list
1367 ====================
1368 */
1369 // TODO rewrite using Curl_GetDownloadInfo?
1370 static void Curl_Info_f(cmd_state_t *cmd)
1371 {
1372         downloadinfo *di;
1373         char urlbuf[1024];
1374         if(!curl_dll)
1375                 return;
1376         if(Curl_Running())
1377         {
1378                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1379                 Con_Print("Currently running downloads:\n");
1380                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1381                 {
1382                         double speed, percent;
1383                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1384                         percent = 100.0 * Curl_GetDownloadAmount(di);
1385                         speed = Curl_GetDownloadSpeed(di);
1386                         if(percent >= 0)
1387                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1388                         else
1389                                 Con_Print("(queued)\n");
1390                 }
1391                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1392         }
1393         else
1394         {
1395                 Con_Print("No downloads running.\n");
1396         }
1397 }
1398
1399 /*
1400 ====================
1401 Curl_Curl_f
1402
1403 implements the "curl" console command
1404
1405 curl --info
1406 curl --cancel
1407 curl --cancel filename
1408 curl url
1409
1410 For internal use:
1411
1412 curl [--pak] [--forthismap] [--for filename filename...] url
1413         --pak: after downloading, load the package into the virtual file system
1414         --for filename...: only download of at least one of the named files is missing
1415         --forthismap: don't reconnect on failure
1416
1417 curl --clear_autodownload
1418         clears the download success/failure counters
1419
1420 curl --finish_autodownload
1421         if at least one download has been started, disconnect and drop to the menu
1422         once the last download completes successfully, reconnect to the current server
1423 ====================
1424 */
1425 static void Curl_Curl_f(cmd_state_t *cmd)
1426 {
1427         double maxspeed = 0;
1428         int i;
1429         int end;
1430         int loadtype = LOADTYPE_NONE;
1431         qbool forthismap = false;
1432         const char *url;
1433         const char *name = 0;
1434
1435         if(!curl_dll)
1436         {
1437                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1438                 return;
1439         }
1440
1441         if(!curl_enabled.integer)
1442         {
1443                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1444                 return;
1445         }
1446
1447         if(Cmd_Argc(cmd) < 2)
1448         {
1449                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1450                 return;
1451         }
1452
1453         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1454         end = Cmd_Argc(cmd);
1455
1456         for(i = 1; i != end; ++i)
1457         {
1458                 const char *a = Cmd_Argv(cmd, i);
1459                 if(!strcmp(a, "--info"))
1460                 {
1461                         Curl_Info_f(cmd);
1462                         return;
1463                 }
1464                 else if(!strcmp(a, "--cancel"))
1465                 {
1466                         if(i == end - 1) // last argument
1467                                 Curl_CancelAll();
1468                         else
1469                         {
1470                                 downloadinfo *di = Curl_Find(url);
1471                                 if(di)
1472                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1473                                 else
1474                                         Con_Print("download not found\n");
1475                         }
1476                         return;
1477                 }
1478                 else if(!strcmp(a, "--pak"))
1479                 {
1480                         loadtype = LOADTYPE_PAK;
1481                 }
1482                 else if(!strcmp(a, "--cachepic"))
1483                 {
1484                         loadtype = LOADTYPE_CACHEPIC;
1485                 }
1486                 else if(!strcmp(a, "--skinframe"))
1487                 {
1488                         loadtype = LOADTYPE_SKINFRAME;
1489                 }
1490                 else if(!strcmp(a, "--for")) // must be last option
1491                 {
1492                         for(i = i + 1; i != end - 1; ++i)
1493                         {
1494                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1495                                         goto needthefile; // why can't I have a "double break"?
1496                         }
1497                         // if we get here, we have all the files...
1498                         return;
1499                 }
1500                 else if(!strcmp(a, "--forthismap"))
1501                 {
1502                         forthismap = true;
1503                 }
1504                 else if(!strcmp(a, "--as"))
1505                 {
1506                         if(i < end - 1)
1507                         {
1508                                 ++i;
1509                                 name = Cmd_Argv(cmd, i);
1510                         }
1511                 }
1512                 else if(!strcmp(a, "--clear_autodownload"))
1513                 {
1514                         // mark all running downloads as "not for this map", so if they
1515                         // fail, it does not matter
1516                         Curl_Clear_forthismap();
1517                         return;
1518                 }
1519                 else if(!strcmp(a, "--finish_autodownload"))
1520                 {
1521                         if(numdownloads_added)
1522                         {
1523                                 char donecommand[256];
1524                                 if(cls.netcon)
1525                                 {
1526                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1527                                         {
1528                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1529                                                 Curl_CommandWhenDone(donecommand);
1530                                                 noclear = true;
1531                                                 CL_Disconnect();
1532                                                 noclear = false;
1533                                                 Curl_CheckCommandWhenDone();
1534                                         }
1535                                         else
1536                                                 Curl_Register_predownload();
1537                                 }
1538                         }
1539                         return;
1540                 }
1541                 else if(!strncmp(a, "--maxspeed=", 11))
1542                 {
1543                         maxspeed = atof(a + 11);
1544                 }
1545                 else if(*a == '-')
1546                 {
1547                         Con_Printf("curl: invalid option %s\n", a);
1548                         // but we ignore the option
1549                 }
1550         }
1551
1552 needthefile:
1553         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1554 }
1555
1556 /*
1557 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1558 {
1559         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1560         Z_Free(buffer);
1561 }
1562
1563 void Curl_CurlCat_f(cmd_state_t *cmd)
1564 {
1565         unsigned char *buf;
1566         const char *url = Cmd_Argv(cmd, 1);
1567         buf = Z_Malloc(16384);
1568         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1569 }
1570 */
1571
1572 /*
1573 ====================
1574 Curl_Init_Commands
1575
1576 loads the commands and cvars this library uses
1577 ====================
1578 */
1579 void Curl_Init_Commands(void)
1580 {
1581         Cvar_RegisterVariable (&curl_enabled);
1582         Cvar_RegisterVariable (&curl_maxdownloads);
1583         Cvar_RegisterVariable (&curl_maxspeed);
1584         Cvar_RegisterVariable (&curl_useragent);
1585         Cvar_RegisterVariable (&curl_useragent_append);
1586         Cvar_RegisterVirtual  (&curl_enabled,          "cl_curl_enabled");
1587         Cvar_RegisterVirtual  (&curl_maxdownloads,     "cl_curl_maxdownloads");
1588         Cvar_RegisterVirtual  (&curl_maxspeed,         "cl_curl_maxspeed");
1589         Cvar_RegisterVirtual  (&curl_useragent,        "cl_curl_useragent");
1590         Cvar_RegisterVirtual  (&curl_useragent_append, "cl_curl_useragent_append");
1591
1592         Cvar_RegisterVariable (&sv_curl_defaulturl);
1593         Cvar_RegisterVariable (&sv_curl_serverpackages);
1594         Cvar_RegisterVariable (&sv_curl_maxspeed);
1595
1596         Cvar_RegisterVariable (&developer_curl);
1597
1598         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1599         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1600 }
1601
1602 /*
1603 ====================
1604 Curl_GetDownloadInfo
1605
1606 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1607 The number of elements in the array is returned in int *nDownloads.
1608 const char **additional_info may be set to a string of additional user
1609 information, or to NULL if no such display shall occur. The returned
1610 array must be freed later using Z_Free.
1611 ====================
1612 */
1613 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1614 {
1615         int i;
1616         downloadinfo *di;
1617         Curl_downloadinfo_t *downinfo;
1618
1619         if(!curl_dll)
1620         {
1621                 *nDownloads = 0;
1622                 if(additional_info)
1623                         *additional_info = NULL;
1624                 return NULL;
1625         }
1626
1627         if (curl_mutex) Thread_LockMutex(curl_mutex);
1628
1629         i = 0;
1630         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1631                 ++i;
1632
1633         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1634         i = 0;
1635         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1636         {
1637                 // do not show infobars for background downloads
1638                 if(developer.integer <= 0)
1639                         if(di->buffer)
1640                                 continue;
1641                 dp_strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1642                 if(di->curle)
1643                 {
1644                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1645                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1646                         downinfo[i].queued = false;
1647                 }
1648                 else
1649                 {
1650                         downinfo[i].queued = true;
1651                 }
1652                 ++i;
1653         }
1654
1655         if(additional_info)
1656         {
1657                 // TODO: can I clear command_when_done as soon as the first download fails?
1658                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1659                 {
1660                         if(!strncmp(command_when_done, "connect ", 8))
1661                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1662                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1663                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1664                         else
1665                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1666                         *additional_info = addinfo;
1667                 }
1668                 else
1669                         *additional_info = NULL;
1670         }
1671
1672         *nDownloads = i;
1673         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1674         return downinfo;
1675 }
1676
1677
1678 /*
1679 ====================
1680 Curl_FindPackURL
1681
1682 finds the URL where to find a given package.
1683
1684 For this, it reads a file "curl_urls.txt" of the following format:
1685
1686         data*.pk3       -
1687         revdm*.pk3      http://revdm/downloads/are/here/
1688         *                       http://any/other/stuff/is/here/
1689
1690 The URLs should end in /. If not, downloads will still work, but the cached files
1691 can't be just put into the data directory with the same download configuration
1692 (you might want to do this if you want to tag downloaded files from your
1693 server, but you should not). "-" means "don't download".
1694
1695 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1696 location instead.
1697
1698 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1699 this file for obvious reasons.
1700 ====================
1701 */
1702 static const char *Curl_FindPackURL(const char *filename)
1703 {
1704         static char foundurl[1024]; // invoked only by server
1705         fs_offset_t filesize;
1706         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1707         if(buf && filesize)
1708         {
1709                 // read lines of format "pattern url"
1710                 char *p = buf;
1711                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1712                 qbool eof = false;
1713
1714                 pattern = p;
1715                 while(!eof)
1716                 {
1717                         switch(*p)
1718                         {
1719                                 case 0:
1720                                         eof = true;
1721                                         // fallthrough
1722                                 case '\n':
1723                                 case '\r':
1724                                         if(pattern && url && patternend)
1725                                         {
1726                                                 if(!urlend)
1727                                                         urlend = p;
1728                                                 *patternend = 0;
1729                                                 *urlend = 0;
1730                                                 if(matchpattern(filename, pattern, true))
1731                                                 {
1732                                                         dp_strlcpy(foundurl, url, sizeof(foundurl));
1733                                                         Z_Free(buf);
1734                                                         return foundurl;
1735                                                 }
1736                                         }
1737                                         pattern = NULL;
1738                                         patternend = NULL;
1739                                         url = NULL;
1740                                         urlend = NULL;
1741                                         break;
1742                                 case ' ':
1743                                 case '\t':
1744                                         if(pattern && !patternend)
1745                                                 patternend = p;
1746                                         else if(url && !urlend)
1747                                                 urlend = p;
1748                                         break;
1749                                 default:
1750                                         if(!pattern)
1751                                                 pattern = p;
1752                                         else if(pattern && patternend && !url)
1753                                                 url = p;
1754                                         break;
1755                         }
1756                         ++p;
1757                 }
1758         }
1759         if(buf)
1760                 Z_Free(buf);
1761         return sv_curl_defaulturl.string;
1762 }
1763
1764 typedef struct requirement_s
1765 {
1766         struct requirement_s *next;
1767         char filename[MAX_OSPATH];
1768 }
1769 requirement;
1770 static requirement *requirements = NULL;
1771
1772
1773 /*
1774 ====================
1775 Curl_RequireFile
1776
1777 Adds the given file to the list of requirements.
1778 ====================
1779 */
1780 void Curl_RequireFile(const char *filename)
1781 {
1782         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1783         req->next = requirements;
1784         dp_strlcpy(req->filename, filename, sizeof(req->filename));
1785         requirements = req;
1786 }
1787
1788 /*
1789 ====================
1790 Curl_ClearRequirements
1791
1792 Clears the list of required files for playing on the current map.
1793 This should be called at every map change.
1794 ====================
1795 */
1796 void Curl_ClearRequirements(void)
1797 {
1798         while(requirements)
1799         {
1800                 requirement *req = requirements;
1801                 requirements = requirements->next;
1802                 Z_Free(req);
1803         }
1804 }
1805
1806 /*
1807 ====================
1808 Curl_SendRequirements
1809
1810 Makes the current host_clients download all files he needs.
1811 This is done by sending him the following console commands:
1812
1813         curl --clear_autodownload
1814         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1815         curl --finish_autodownload
1816 ====================
1817 */
1818 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1819 {
1820         const char *p;
1821         const char *thispack = FS_WhichPack(filename);
1822         const char *packurl;
1823
1824         if(!thispack || !*thispack)
1825                 return false;
1826
1827         p = strrchr(thispack, '/');
1828         if(p)
1829                 thispack = p + 1;
1830
1831         packurl = Curl_FindPackURL(thispack);
1832
1833         if(packurl && *packurl && strcmp(packurl, "-"))
1834         {
1835                 if(!foundone)
1836                         dp_strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1837
1838                 dp_strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1839                 dp_strlcat(sendbuffer, thispack, sendbuffer_len);
1840                 if(sv_curl_maxspeed.value > 0)
1841                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1842                 dp_strlcat(sendbuffer, " --for ", sendbuffer_len);
1843                 dp_strlcat(sendbuffer, filename, sendbuffer_len);
1844                 dp_strlcat(sendbuffer, " ", sendbuffer_len);
1845                 dp_strlcat(sendbuffer, packurl, sendbuffer_len);
1846                 dp_strlcat(sendbuffer, thispack, sendbuffer_len);
1847                 dp_strlcat(sendbuffer, "\n", sendbuffer_len);
1848
1849                 return true;
1850         }
1851
1852         return false;
1853 }
1854 void Curl_SendRequirements(void)
1855 {
1856         // for each requirement, find the pack name
1857         char sendbuffer[4096] = "";
1858         requirement *req;
1859         qbool foundone = false;
1860         const char *p;
1861
1862         for(req = requirements; req; req = req->next)
1863                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1864
1865         p = sv_curl_serverpackages.string;
1866         while(COM_ParseToken_Simple(&p, false, false, true))
1867                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1868
1869         if(foundone)
1870                 dp_strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1871
1872         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1873                 SV_ClientCommands("%s", sendbuffer);
1874         else
1875                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1876 }