libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CVAR_CLIENT | CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CVAR_CLIENT | CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CVAR_SERVER | CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CVAR_SERVER | CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CVAR_SERVER | CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CVAR_CLIENT | CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {CVAR_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {CVAR_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18
  19 /*
  20 =================================================================
  21
  22   Minimal set of definitions from libcurl
  23
  24   WARNING: for a matter of simplicity, several pointer types are
  25   casted to "void*", and most enumerated values are not included
  26
  27 =================================================================
  28 */
  29
  30 typedef struct CURL_s CURL;
  31 typedef struct CURLM_s CURLM;
  32 typedef struct curl_slist curl_slist;
  33 typedef enum
  34 {
  35         CURLE_OK = 0
  36 }
  37 CURLcode;
  38 typedef enum
  39 {
  40         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  41         CURLM_OK = 0
  42 }
  43 CURLMcode;
  44 #define CURL_GLOBAL_NOTHING 0
  45 #define CURL_GLOBAL_SSL 1
  46 #define CURL_GLOBAL_WIN32 2
  47 #define CURLOPTTYPE_LONG          0
  48 #define CURLOPTTYPE_OBJECTPOINT   10000
  49 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  50 #define CURLOPTTYPE_OFF_T         30000
  51 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  52 typedef enum
  53 {
  54         CINIT(WRITEDATA, OBJECTPOINT, 1),
  55         CINIT(URL,  OBJECTPOINT, 2),
  56         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  57         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  58         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  59         CINIT(REFERER, OBJECTPOINT, 16),
  60         CINIT(USERAGENT, OBJECTPOINT, 18),
  61         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  62         CINIT(LOW_SPEED_TIME, LONG, 20),
  63         CINIT(RESUME_FROM, LONG, 21),
  64         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  65         CINIT(POST, LONG, 47),         /* HTTP POST method */
  66         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  67         CINIT(POSTFIELDSIZE, LONG, 60),
  68         CINIT(PRIVATE, OBJECTPOINT, 103),
  69         CINIT(PROTOCOLS, LONG, 181),
  70         CINIT(REDIR_PROTOCOLS, LONG, 182)
  71 }
  72 CURLoption;
  73 #define CURLPROTO_HTTP   (1<<0)
  74 #define CURLPROTO_HTTPS  (1<<1)
  75 #define CURLPROTO_FTP    (1<<2)
  76 typedef enum
  77 {
  78         CURLINFO_TEXT = 0,
  79         CURLINFO_HEADER_IN,    /* 1 */
  80         CURLINFO_HEADER_OUT,   /* 2 */
  81         CURLINFO_DATA_IN,      /* 3 */
  82         CURLINFO_DATA_OUT,     /* 4 */
  83         CURLINFO_SSL_DATA_IN,  /* 5 */
  84         CURLINFO_SSL_DATA_OUT, /* 6 */
  85         CURLINFO_END
  86 }
  87 curl_infotype;
  88 #define CURLINFO_STRING   0x100000
  89 #define CURLINFO_LONG     0x200000
  90 #define CURLINFO_DOUBLE   0x300000
  91 #define CURLINFO_SLIST    0x400000
  92 #define CURLINFO_MASK     0x0fffff
  93 #define CURLINFO_TYPEMASK 0xf00000
  94 typedef enum
  95 {
  96         CURLINFO_NONE, /* first, never use this */
  97         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  98         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  99         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 100         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 101         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 102         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 103         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 104         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 105         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 106         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 107         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 108         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 109         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 110         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 111         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 112         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 113         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 114         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 115         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 116         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 117         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 118         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 119         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 120         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 121         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 122         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 123         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 124 }
 125 CURLINFO;
 126
 127 typedef enum
 128 {
 129         CURLMSG_NONE, /* first, not used */
 130         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 131                                          the CURLcode of the transfer */
 132         CURLMSG_LAST
 133 }
 134 CURLMSG;
 135 typedef struct
 136 {
 137         CURLMSG msg;       /* what this message means */
 138         CURL *easy_handle; /* the handle it concerns */
 139         union
 140         {
 141                 void *whatever;    /* message-specific data */
 142                 CURLcode result;   /* return code for transfer */
 143         }
 144         data;
 145 }
 146 CURLMsg;
 147
 148 static void (*qcurl_global_init) (long flags);
 149 static void (*qcurl_global_cleanup) (void);
 150
 151 static CURL * (*qcurl_easy_init) (void);
 152 static void (*qcurl_easy_cleanup) (CURL *handle);
 153 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 154 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 155 static const char * (*qcurl_easy_strerror) (CURLcode);
 156
 157 static CURLM * (*qcurl_multi_init) (void);
 158 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 159 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 160 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 161 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 162 static void (*qcurl_multi_cleanup) (CURLM *);
 163 static const char * (*qcurl_multi_strerror) (CURLcode);
 164 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 165 static void (*qcurl_slist_free_all) (curl_slist *list);
 166
 167 static dllfunction_t curlfuncs[] =
 168 {
 169         {"curl_global_init",            (void **) &qcurl_global_init},
 170         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 171         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 172         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 173         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 174         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 175         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 176         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 177         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 178         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 179         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 180         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 181         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 182         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 183         {"curl_slist_append",           (void **) &qcurl_slist_append},
 184         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 185         {NULL, NULL}
 186 };
 187
 188 // Handle for CURL DLL
 189 static dllhandle_t curl_dll = NULL;
 190 // will be checked at many places to find out if qcurl calls are allowed
 191
 192 #define LOADTYPE_NONE 0
 193 #define LOADTYPE_PAK 1
 194 #define LOADTYPE_CACHEPIC 2
 195 #define LOADTYPE_SKINFRAME 3
 196
 197 void *curl_mutex = NULL;
 198
 199 typedef struct downloadinfo_s
 200 {
 201         char filename[MAX_OSPATH];
 202         char url[1024];
 203         char referer[256];
 204         qfile_t *stream;
 205         fs_offset_t startpos;
 206         CURL *curle;
 207         qboolean started;
 208         int loadtype;
 209         size_t bytes_received; // for buffer
 210         double bytes_received_curl; // for throttling
 211         double bytes_sent_curl; // for throttling
 212         struct downloadinfo_s *next, *prev;
 213         qboolean forthismap;
 214         double maxspeed;
 215         curl_slist *slist; // http headers
 216
 217         unsigned char *buffer;
 218         size_t buffersize;
 219         curl_callback_t callback;
 220         void *callback_data;
 221
 222         const unsigned char *postbuf;
 223         size_t postbufsize;
 224         const char *post_content_type;
 225         const char *extraheaders;
 226 }
 227 downloadinfo;
 228 static downloadinfo *downloads = NULL;
 229 static int numdownloads = 0;
 230
 231 static qboolean noclear = false;
 232
 233 static int numdownloads_fail = 0;
 234 static int numdownloads_success = 0;
 235 static int numdownloads_added = 0;
 236 static char command_when_done[256] = "";
 237 static char command_when_error[256] = "";
 238
 239 /*
 240 ====================
 241 Curl_CommandWhenDone
 242
 243 Sets the command which is to be executed when the last download completes AND
 244 all downloads since last server connect ended with a successful status.
 245 Setting the command to NULL clears it.
 246 ====================
 247 */
 248 static void Curl_CommandWhenDone(const char *cmd)
 249 {
 250         if(!curl_dll)
 251                 return;
 252         if(cmd)
 253                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 254         else
 255                 *command_when_done = 0;
 256 }
 257
 258 /*
 259 FIXME
 260 Do not use yet. Not complete.
 261 Problem: what counts as an error?
 262 */
 263
 264 static void Curl_CommandWhenError(const char *cmd)
 265 {
 266         if(!curl_dll)
 267                 return;
 268         if(cmd)
 269                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 270         else
 271                 *command_when_error = 0;
 272 }
 273
 274 /*
 275 ====================
 276 Curl_Clear_forthismap
 277
 278 Clears the "will disconnect on failure" flags.
 279 ====================
 280 */
 281 void Curl_Clear_forthismap(void)
 282 {
 283         downloadinfo *di;
 284         if(noclear)
 285                 return;
 286         if (curl_mutex) Thread_LockMutex(curl_mutex);
 287         for(di = downloads; di; di = di->next)
 288                 di->forthismap = false;
 289         Curl_CommandWhenError(NULL);
 290         Curl_CommandWhenDone(NULL);
 291         numdownloads_fail = 0;
 292         numdownloads_success = 0;
 293         numdownloads_added = 0;
 294         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 295 }
 296
 297 /*
 298 ====================
 299 Curl_Have_forthismap
 300
 301 Returns true if a download needed for the current game is running.
 302 ====================
 303 */
 304 qboolean Curl_Have_forthismap(void)
 305 {
 306         return numdownloads_added != 0;
 307 }
 308
 309 void Curl_Register_predownload(void)
 310 {
 311         if (curl_mutex) Thread_LockMutex(curl_mutex);
 312         Curl_CommandWhenDone("cl_begindownloads");
 313         Curl_CommandWhenError("cl_begindownloads");
 314         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 315 }
 316
 317 /*
 318 ====================
 319 Curl_CheckCommandWhenDone
 320
 321 Checks if a "done command" is to be executed.
 322 All downloads finished, at least one success since connect, no single failure
 323 -> execute the command.
 324 */
 325 static void Curl_CheckCommandWhenDone(void)
 326 {
 327         if(!curl_dll)
 328                 return;
 329         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 330         {
 331                 if(numdownloads_fail == 0)
 332                 {
 333                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 334                         Cbuf_AddText(&cmd_client, "\n");
 335                         Cbuf_AddText(&cmd_client, command_when_done);
 336                         Cbuf_AddText(&cmd_client, "\n");
 337                 }
 338                 else
 339                 {
 340                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 341                         Cbuf_AddText(&cmd_client, "\n");
 342                         Cbuf_AddText(&cmd_client, command_when_error);
 343                         Cbuf_AddText(&cmd_client, "\n");
 344                 }
 345                 Curl_Clear_forthismap();
 346         }
 347 }
 348
 349 /*
 350 ====================
 351 CURL_CloseLibrary
 352
 353 Load the cURL DLL
 354 ====================
 355 */
 356 static qboolean CURL_OpenLibrary (void)
 357 {
 358         const char* dllnames [] =
 359         {
 360 #if defined(WIN32)
 361                 "libcurl-4.dll",
 362                 "libcurl-3.dll",
 363 #elif defined(MACOSX)
 364                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 365                 "libcurl.3.dylib", // Mac OS X Tiger
 366                 "libcurl.2.dylib", // Mac OS X Panther
 367 #else
 368                 "libcurl.so.4",
 369                 "libcurl.so.3",
 370                 "libcurl.so", // FreeBSD
 371 #endif
 372                 NULL
 373         };
 374
 375         // Already loaded?
 376         if (curl_dll)
 377                 return true;
 378
 379         // Load the DLL
 380         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 381 }
 382
 383
 384 /*
 385 ====================
 386 CURL_CloseLibrary
 387
 388 Unload the cURL DLL
 389 ====================
 390 */
 391 static void CURL_CloseLibrary (void)
 392 {
 393         Sys_UnloadLibrary (&curl_dll);
 394 }
 395
 396
 397 static CURLM *curlm = NULL;
 398 static double bytes_received = 0; // used for bandwidth throttling
 399 static double bytes_sent = 0; // used for bandwidth throttling
 400 static double curltime = 0;
 401
 402 /*
 403 ====================
 404 CURL_fwrite
 405
 406 fwrite-compatible function that writes the data to a file. libcurl can call
 407 this.
 408 ====================
 409 */
 410 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 411 {
 412         fs_offset_t ret = -1;
 413         size_t bytes = size * nmemb;
 414         downloadinfo *di = (downloadinfo *) vdi;
 415
 416         if(di->buffer)
 417         {
 418                 if(di->bytes_received + bytes <= di->buffersize)
 419                 {
 420                         memcpy(di->buffer + di->bytes_received, data, bytes);
 421                         ret = bytes;
 422                 }
 423                 // otherwise: buffer overrun, ret stays -1
 424         }
 425
 426         if(di->stream)
 427         {
 428                 ret = FS_Write(di->stream, data, bytes);
 429         }
 430
 431         di->bytes_received += bytes;
 432
 433         return ret;
 434         // Why not ret / nmemb?
 435         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 436         // Yes, this is incompatible to fwrite(2).
 437 }
 438
 439 typedef enum
 440 {
 441         CURL_DOWNLOAD_SUCCESS = 0,
 442         CURL_DOWNLOAD_FAILED,
 443         CURL_DOWNLOAD_ABORTED,
 444         CURL_DOWNLOAD_SERVERERROR
 445 }
 446 CurlStatus;
 447
 448 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 449 {
 450         downloadinfo *di = (downloadinfo *) cbdata;
 451         switch(status)
 452         {
 453                 case CURLCBSTATUS_OK:
 454                         Con_DPrintf("Download of %s: OK\n", di->filename);
 455                         break;
 456                 case CURLCBSTATUS_FAILED:
 457                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 458                         break;
 459                 case CURLCBSTATUS_ABORTED:
 460                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 461                         break;
 462                 case CURLCBSTATUS_SERVERERROR:
 463                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 464                         break;
 465                 case CURLCBSTATUS_UNKNOWN:
 466                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 467                         break;
 468                 default:
 469                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 470                         break;
 471         }
 472 }
 473
 474 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 475 {
 476         curl_default_callback(status, length_received, buffer, cbdata);
 477 }
 478
 479 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 480 {
 481         unsigned char *pixels = NULL;
 482         fs_offset_t filesize = 0;
 483         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 484         if(data)
 485         {
 486                 int mip = 0;
 487                 if(!strcmp(content_type, "image/jpeg"))
 488                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 489                 else if(!strcmp(content_type, "image/png"))
 490                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 491                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 492                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 493                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 494                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 495                 else
 496                         Con_Printf("Did not detect content type: %s\n", content_type);
 497                 Mem_Free(data);
 498         }
 499         // do we call Image_MakeLinearColorsFromsRGB or not?
 500         return pixels;
 501 }
 502
 503 /*
 504 ====================
 505 Curl_EndDownload
 506
 507 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 508 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 509 code from libcurl, or 0, if another error has occurred.
 510 ====================
 511 */
 512 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 513 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 514 {
 515         char content_type[64];
 516         qboolean ok = false;
 517         if(!curl_dll)
 518                 return;
 519         switch(status)
 520         {
 521                 case CURL_DOWNLOAD_SUCCESS:
 522                         ok = true;
 523                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 524                         break;
 525                 case CURL_DOWNLOAD_FAILED:
 526                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 527                         break;
 528                 case CURL_DOWNLOAD_ABORTED:
 529                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 530                         break;
 531                 case CURL_DOWNLOAD_SERVERERROR:
 532                         // reopen to enforce it to have zero bytes again
 533                         if(di->stream)
 534                         {
 535                                 FS_Close(di->stream);
 536                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 537                         }
 538
 539                         if(di->callback)
 540                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 541                         break;
 542                 default:
 543                         if(di->callback)
 544                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 545                         break;
 546         }
 547         if(content_type_)
 548                 strlcpy(content_type, content_type_, sizeof(content_type));
 549         else
 550                 *content_type = 0;
 551
 552         if(di->curle)
 553         {
 554                 qcurl_multi_remove_handle(curlm, di->curle);
 555                 qcurl_easy_cleanup(di->curle);
 556                 if(di->slist)
 557                         qcurl_slist_free_all(di->slist);
 558         }
 559
 560         if(!di->callback && ok && !di->bytes_received)
 561         {
 562                 Con_Printf("ERROR: empty file\n");
 563                 ok = false;
 564         }
 565
 566         if(di->stream)
 567                 FS_Close(di->stream);
 568
 569 #define CLEAR_AND_RETRY() \
 570         do \
 571         { \
 572                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 573                 FS_Close(di->stream); \
 574                 if(di->startpos && !di->callback) \
 575                 { \
 576                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 577                         di->forthismap = false; \
 578                 } \
 579         } \
 580         while(0)
 581
 582         if(ok && di->loadtype == LOADTYPE_PAK)
 583         {
 584                 ok = FS_AddPack(di->filename, NULL, true);
 585                 if(!ok)
 586                         CLEAR_AND_RETRY();
 587         }
 588         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 589         {
 590                 const char *p;
 591                 unsigned char *pixels = NULL;
 592
 593                 p = di->filename;
 594 #ifdef WE_ARE_EVIL
 595                 if(!strncmp(p, "dlcache/", 8))
 596                         p += 8;
 597 #endif
 598
 599                 pixels = decode_image(di, content_type);
 600                 if(pixels)
 601                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 602                 else
 603                         CLEAR_AND_RETRY();
 604         }
 605         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 606         {
 607                 const char *p;
 608                 unsigned char *pixels = NULL;
 609
 610                 p = di->filename;
 611 #ifdef WE_ARE_EVIL
 612                 if(!strncmp(p, "dlcache/", 8))
 613                         p += 8;
 614 #endif
 615
 616                 pixels = decode_image(di, content_type);
 617                 if(pixels)
 618                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 619                 else
 620                         CLEAR_AND_RETRY();
 621         }
 622
 623         if(di->prev)
 624                 di->prev->next = di->next;
 625         else
 626                 downloads = di->next;
 627         if(di->next)
 628                 di->next->prev = di->prev;
 629
 630         --numdownloads;
 631         if(di->forthismap)
 632         {
 633                 if(ok)
 634                         ++numdownloads_success;
 635                 else
 636                         ++numdownloads_fail;
 637         }
 638         Z_Free(di);
 639 }
 640
 641 /*
 642 ====================
 643 CleanURL
 644
 645 Returns a "cleaned up" URL for display (to strip login data)
 646 ====================
 647 */
 648 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 649 {
 650         const char *p, *q, *r;
 651
 652         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 653         p = strstr(url, "://");
 654         if(p)
 655         {
 656                 q = strchr(p + 3, '@');
 657                 if(q)
 658                 {
 659                         r = strchr(p + 3, '/');
 660                         if(!r || q < r)
 661                         {
 662                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 663                                 return urlbuf;
 664                         }
 665                 }
 666         }
 667
 668         return url;
 669 }
 670
 671 /*
 672 ====================
 673 CheckPendingDownloads
 674
 675 checks if there are free download slots to start new downloads in.
 676 To not start too many downloads at once, only one download is added at a time,
 677 up to a maximum number of cl_curl_maxdownloads are running.
 678 ====================
 679 */
 680 static void CheckPendingDownloads(void)
 681 {
 682         const char *h;
 683         char urlbuf[1024];
 684         char vabuf[1024];
 685         if(!curl_dll)
 686                 return;
 687         if(numdownloads < cl_curl_maxdownloads.integer)
 688         {
 689                 downloadinfo *di;
 690                 for(di = downloads; di; di = di->next)
 691                 {
 692                         if(!di->started)
 693                         {
 694                                 if(!di->buffer)
 695                                 {
 696                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 697
 698                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 699                                         if(!di->stream)
 700                                         {
 701                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 702                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 703                                                 return;
 704                                         }
 705                                         FS_Seek(di->stream, 0, SEEK_END);
 706                                         di->startpos = FS_Tell(di->stream);
 707
 708                                         if(di->startpos > 0)
 709                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 710                                         Con_Print("...\n");
 711                                 }
 712                                 else
 713                                 {
 714                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 715                                         di->startpos = 0;
 716                                 }
 717
 718                                 di->curle = qcurl_easy_init();
 719                                 di->slist = NULL;
 720                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 721                                 if(cl_curl_useragent.integer)
 722                                 {
 723                                         const char *ua
 724 #ifdef HTTP_USER_AGENT
 725                                                 = HTTP_USER_AGENT;
 726 #else
 727                                                 = engineversion;
 728 #endif
 729                                         if(!ua)
 730                                                 ua = "";
 731                                         if(*cl_curl_useragent_append.string)
 732                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 733                                                         ua,
 734                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 735                                                                 ? " "
 736                                                                 : "",
 737                                                         cl_curl_useragent_append.string);
 738                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 739                                 }
 740                                 else
 741                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 742                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 743                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 744                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 746                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 750                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 751                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 752                                 {
 753                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 754                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 755                                 }
 756                                 if(di->post_content_type)
 757                                 {
 758                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 759                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 760                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 761                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 762                                 }
 763
 764                                 // parse extra headers into slist
 765                                 // \n separated list!
 766                                 h = di->extraheaders;
 767                                 while(h)
 768                                 {
 769                                         const char *hh = strchr(h, '\n');
 770                                         if(hh)
 771                                         {
 772                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 773                                                 memcpy(buf, h, hh - h);
 774                                                 buf[hh - h] = 0;
 775                                                 di->slist = qcurl_slist_append(di->slist, buf);
 776                                                 h = hh + 1;
 777                                         }
 778                                         else
 779                                         {
 780                                                 di->slist = qcurl_slist_append(di->slist, h);
 781                                                 h = NULL;
 782                                         }
 783                                 }
 784
 785                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 786
 787                                 qcurl_multi_add_handle(curlm, di->curle);
 788                                 di->started = true;
 789                                 ++numdownloads;
 790                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 791                                         break;
 792                         }
 793                 }
 794         }
 795 }
 796
 797 /*
 798 ====================
 799 Curl_Init
 800
 801 this function MUST be called before using anything else in this file.
 802 On Win32, this must be called AFTER WSAStartup has been done!
 803 ====================
 804 */
 805 void Curl_Init(void)
 806 {
 807         CURL_OpenLibrary();
 808         if(!curl_dll)
 809                 return;
 810         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 811         qcurl_global_init(CURL_GLOBAL_NOTHING);
 812         curlm = qcurl_multi_init();
 813 }
 814
 815 /*
 816 ====================
 817 Curl_Shutdown
 818
 819 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 820 ====================
 821 */
 822 void Curl_ClearRequirements(void);
 823 void Curl_Shutdown(void)
 824 {
 825         if(!curl_dll)
 826                 return;
 827         Curl_ClearRequirements();
 828         Curl_CancelAll();
 829         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 830         CURL_CloseLibrary();
 831         curl_dll = NULL;
 832 }
 833
 834 /*
 835 ====================
 836 Curl_Find
 837
 838 Finds the internal information block for a download given by file name.
 839 ====================
 840 */
 841 static downloadinfo *Curl_Find(const char *filename)
 842 {
 843         downloadinfo *di;
 844         if(!curl_dll)
 845                 return NULL;
 846         for(di = downloads; di; di = di->next)
 847                 if(!strcasecmp(di->filename, filename))
 848                         return di;
 849         return NULL;
 850 }
 851
 852 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 853 {
 854         downloadinfo *di;
 855         if(!curl_dll)
 856                 return;
 857         for(di = downloads; di; )
 858         {
 859                 if(di->callback == callback && di->callback_data == cbdata)
 860                 {
 861                         di->callback = curl_quiet_callback; // do NOT call the callback
 862                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 863                         di = downloads;
 864                 }
 865                 else
 866                         di = di->next;
 867         }
 868 }
 869
 870 /*
 871 ====================
 872 Curl_Begin
 873
 874 Starts a download of a given URL to the file name portion of this URL (or name
 875 if given) in the "dlcache/" folder.
 876 ====================
 877 */
 878 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 879 {
 880         if(buf)
 881                 if(loadtype != LOADTYPE_NONE)
 882                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 883
 884         if(!curl_dll || !cl_curl_enabled.integer)
 885         {
 886                 return false;
 887         }
 888         else
 889         {
 890                 char fn[MAX_OSPATH];
 891                 char urlbuf[1024];
 892                 const char *p, *q;
 893                 size_t length;
 894                 downloadinfo *di;
 895
 896                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 897                 p = strchr(URL, ':');
 898                 if(p)
 899                 {
 900                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 901                         {
 902                                 char addressstring[128];
 903                                 *addressstring = 0;
 904                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 905                                 q = strchr(addressstring, ':');
 906                                 if(!q)
 907                                         q = addressstring + strlen(addressstring);
 908                                 if(*addressstring)
 909                                 {
 910                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 911                                         URL = urlbuf;
 912                                 }
 913                         }
 914                 }
 915
 916                 // Note: This extraction of the file name portion is NOT entirely correct.
 917                 //
 918                 // It does the following:
 919                 //
 920                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 921                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 922                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 923                 //
 924                 // However, I'd like to keep this "buggy" behavior so that PHP script
 925                 // authors can write download scripts without having to enable
 926                 // AcceptPathInfo on Apache. They just have to ensure that their script
 927                 // can be called with such a "fake" path name like
 928                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 929                 //
 930                 // By the way, such PHP scripts should either send the file or a
 931                 // "Location:" redirect; PHP code example:
 932                 //
 933                 //   header("Location: http://www.example.com/");
 934                 //
 935                 // By the way, this will set User-Agent to something like
 936                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 937                 // dp://serverhost:serverport/ so you can filter on this; an example
 938                 // httpd log file line might be:
 939                 //
 940                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 941
 942                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 943
 944                 if(buf)
 945                 {
 946                         if(!name)
 947                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 948                 }
 949                 else
 950                 {
 951                         if(!name)
 952                         {
 953                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 954                                 p = strrchr(name, '/');
 955                                 p = p ? (p+1) : name;
 956                                 q = strchr(p, '?');
 957                                 length = q ? (size_t)(q - p) : strlen(p);
 958                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 959                         }
 960                         else
 961                         {
 962                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 963                         }
 964
 965                         name = fn; // make it point back
 966
 967                         // already downloading the file?
 968                         {
 969                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 970                                 if(existingdownloadinfo)
 971                                 {
 972                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 973
 974                                         // however, if it was not for this map yet...
 975                                         if(forthismap && !existingdownloadinfo->forthismap)
 976                                         {
 977                                                 existingdownloadinfo->forthismap = true;
 978                                                 // this "fakes" a download attempt so the client will wait for
 979                                                 // the download to finish and then reconnect
 980                                                 ++numdownloads_added;
 981                                         }
 982
 983                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 984                                         return false;
 985                                 }
 986                         }
 987
 988                         if(FS_FileExists(fn))
 989                         {
 990                                 if(loadtype == LOADTYPE_PAK)
 991                                 {
 992                                         qboolean already_loaded;
 993                                         if(FS_AddPack(fn, &already_loaded, true))
 994                                         {
 995                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 996                                                 if(already_loaded)
 997                                                         Con_DPrintf("(pak was already loaded)\n");
 998                                                 else
 999                                                 {
1000                                                         if(forthismap)
1001                                                         {
1002                                                                 ++numdownloads_added;
1003                                                                 ++numdownloads_success;
1004                                                         }
1005                                                 }
1006
1007                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1008                                                 return false;
1009                                         }
1010                                         else
1011                                         {
1012                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1013                                                 if(f)
1014                                                 {
1015                                                         char b[4] = {0};
1016                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1017
1018                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1019                                                         {
1020                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1021                                                                 FS_Close(f);
1022                                                                 f = FS_OpenRealFile(fn, "wb", false);
1023                                                                 if(f)
1024                                                                         FS_Close(f);
1025                                                         }
1026                                                         else
1027                                                         {
1028                                                                 // OK
1029                                                                 FS_Close(f);
1030                                                         }
1031                                                 }
1032                                         }
1033                                 }
1034                                 else
1035                                 {
1036                                         // never resume these
1037                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1038                                         if(f)
1039                                                 FS_Close(f);
1040                                 }
1041                         }
1042                 }
1043
1044                 // if we get here, we actually want to download... so first verify the
1045                 // URL scheme (so one can't read local files using file://)
1046                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1047                 {
1048                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1049                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1050                         return false;
1051                 }
1052
1053                 if(forthismap)
1054                         ++numdownloads_added;
1055                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1056                 strlcpy(di->filename, name, sizeof(di->filename));
1057                 strlcpy(di->url, URL, sizeof(di->url));
1058                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1059                 di->forthismap = forthismap;
1060                 di->stream = NULL;
1061                 di->startpos = 0;
1062                 di->curle = NULL;
1063                 di->started = false;
1064                 di->loadtype = loadtype;
1065                 di->maxspeed = maxspeed;
1066                 di->bytes_received = 0;
1067                 di->bytes_received_curl = 0;
1068                 di->bytes_sent_curl = 0;
1069                 di->extraheaders = extraheaders;
1070                 di->next = downloads;
1071                 di->prev = NULL;
1072                 if(di->next)
1073                         di->next->prev = di;
1074
1075                 di->buffer = buf;
1076                 di->buffersize = bufsize;
1077                 if(callback == NULL)
1078                 {
1079                         di->callback = curl_default_callback;
1080                         di->callback_data = di;
1081                 }
1082                 else
1083                 {
1084                         di->callback = callback;
1085                         di->callback_data = cbdata;
1086                 }
1087
1088                 if(post_content_type)
1089                 {
1090                         di->post_content_type = post_content_type;
1091                         di->postbuf = postbuf;
1092                         di->postbufsize = postbufsize;
1093                 }
1094                 else
1095                 {
1096                         di->post_content_type = NULL;
1097                         di->postbuf = NULL;
1098                         di->postbufsize = 0;
1099                 }
1100
1101                 downloads = di;
1102                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1103                 return true;
1104         }
1105 }
1106
1107 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qboolean forthismap)
1108 {
1109         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1110 }
1111 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1112 {
1113         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1114 }
1115 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1116 {
1117         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1118 }
1119
1120 /*
1121 ====================
1122 Curl_Run
1123
1124 call this regularily as this will always download as much as possible without
1125 blocking.
1126 ====================
1127 */
1128 void Curl_Run(void)
1129 {
1130         double maxspeed;
1131         downloadinfo *di;
1132
1133         noclear = false;
1134
1135         if(!cl_curl_enabled.integer)
1136                 return;
1137
1138         if(!curl_dll)
1139                 return;
1140
1141         if (curl_mutex) Thread_LockMutex(curl_mutex);
1142
1143         Curl_CheckCommandWhenDone();
1144
1145         if(!downloads)
1146         {
1147                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1148                 return;
1149         }
1150
1151         if(host.realtime < curltime) // throttle
1152         {
1153                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1154                 return;
1155         }
1156
1157         {
1158                 int remaining;
1159                 CURLMcode mc;
1160
1161                 do
1162                 {
1163                         mc = qcurl_multi_perform(curlm, &remaining);
1164                 }
1165                 while(mc == CURLM_CALL_MULTI_PERFORM);
1166
1167                 for(di = downloads; di; di = di->next)
1168                 {
1169                         double b = 0;
1170                         if(di->curle)
1171                         {
1172                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1173                                 bytes_sent += (b - di->bytes_sent_curl);
1174                                 di->bytes_sent_curl = b;
1175                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1176                                 bytes_sent += (b - di->bytes_received_curl);
1177                                 di->bytes_received_curl = b;
1178                         }
1179                 }
1180
1181                 for(;;)
1182                 {
1183                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1184                         if(!msg)
1185                                 break;
1186                         if(msg->msg == CURLMSG_DONE)
1187                         {
1188                                 const char *ct = NULL;
1189                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1190                                 CURLcode result;
1191                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1192                                 result = msg->data.result;
1193                                 if(result)
1194                                 {
1195                                         failed = CURL_DOWNLOAD_FAILED;
1196                                 }
1197                                 else
1198                                 {
1199                                         long code;
1200                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1201                                         switch(code / 100)
1202                                         {
1203                                                 case 4: // e.g. 404?
1204                                                 case 5: // e.g. 500?
1205                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1206                                                         result = (CURLcode) code;
1207                                                         break;
1208                                         }
1209                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1210                                 }
1211
1212                                 Curl_EndDownload(di, failed, result, ct);
1213                         }
1214                 }
1215         }
1216
1217         CheckPendingDownloads();
1218
1219         // when will we curl the next time?
1220         // we will wait a bit to ensure our download rate is kept.
1221         // we now know that realtime >= curltime... so set up a new curltime
1222
1223         // use the slowest allowing download to derive the maxspeed... this CAN
1224         // be done better, but maybe later
1225         maxspeed = cl_curl_maxspeed.value;
1226         for(di = downloads; di; di = di->next)
1227                 if(di->maxspeed > 0)
1228                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1229                                 maxspeed = di->maxspeed;
1230
1231         if(maxspeed > 0)
1232         {
1233                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1234                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1235                 bytes_sent = 0;
1236                 bytes_received = 0;
1237         }
1238         else
1239                 curltime = host.realtime;
1240
1241         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1242 }
1243
1244 /*
1245 ====================
1246 Curl_CancelAll
1247
1248 Stops ALL downloads.
1249 ====================
1250 */
1251 void Curl_CancelAll(void)
1252 {
1253         if(!curl_dll)
1254                 return;
1255
1256         if (curl_mutex) Thread_LockMutex(curl_mutex);
1257
1258         while(downloads)
1259         {
1260                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1261                 // INVARIANT: downloads will point to the next download after that!
1262         }
1263
1264         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1265 }
1266
1267 /*
1268 ====================
1269 Curl_Running
1270
1271 returns true iff there is a download running.
1272 ====================
1273 */
1274 qboolean Curl_Running(void)
1275 {
1276         if(!curl_dll)
1277                 return false;
1278
1279         return downloads != NULL;
1280 }
1281
1282 /*
1283 ====================
1284 Curl_GetDownloadAmount
1285
1286 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1287 for the given download.
1288 ====================
1289 */
1290 static double Curl_GetDownloadAmount(downloadinfo *di)
1291 {
1292         if(!curl_dll)
1293                 return -2;
1294         if(di->curle)
1295         {
1296                 double length;
1297                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1298                 if(length > 0)
1299                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1300                 else
1301                         return 0;
1302         }
1303         else
1304                 return -1;
1305 }
1306
1307 /*
1308 ====================
1309 Curl_GetDownloadSpeed
1310
1311 returns the speed of the given download in bytes per second
1312 ====================
1313 */
1314 static double Curl_GetDownloadSpeed(downloadinfo *di)
1315 {
1316         if(!curl_dll)
1317                 return -2;
1318         if(di->curle)
1319         {
1320                 double speed;
1321                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1322                 return speed;
1323         }
1324         else
1325                 return -1;
1326 }
1327
1328 /*
1329 ====================
1330 Curl_Info_f
1331
1332 prints the download list
1333 ====================
1334 */
1335 // TODO rewrite using Curl_GetDownloadInfo?
1336 static void Curl_Info_f(cmd_state_t *cmd)
1337 {
1338         downloadinfo *di;
1339         char urlbuf[1024];
1340         if(!curl_dll)
1341                 return;
1342         if(Curl_Running())
1343         {
1344                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1345                 Con_Print("Currently running downloads:\n");
1346                 for(di = downloads; di; di = di->next)
1347                 {
1348                         double speed, percent;
1349                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1350                         percent = 100.0 * Curl_GetDownloadAmount(di);
1351                         speed = Curl_GetDownloadSpeed(di);
1352                         if(percent >= 0)
1353                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1354                         else
1355                                 Con_Print("(queued)\n");
1356                 }
1357                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1358         }
1359         else
1360         {
1361                 Con_Print("No downloads running.\n");
1362         }
1363 }
1364
1365 /*
1366 ====================
1367 Curl_Curl_f
1368
1369 implements the "curl" console command
1370
1371 curl --info
1372 curl --cancel
1373 curl --cancel filename
1374 curl url
1375
1376 For internal use:
1377
1378 curl [--pak] [--forthismap] [--for filename filename...] url
1379         --pak: after downloading, load the package into the virtual file system
1380         --for filename...: only download of at least one of the named files is missing
1381         --forthismap: don't reconnect on failure
1382
1383 curl --clear_autodownload
1384         clears the download success/failure counters
1385
1386 curl --finish_autodownload
1387         if at least one download has been started, disconnect and drop to the menu
1388         once the last download completes successfully, reconnect to the current server
1389 ====================
1390 */
1391 static void Curl_Curl_f(cmd_state_t *cmd)
1392 {
1393         double maxspeed = 0;
1394         int i;
1395         int end;
1396         int loadtype = LOADTYPE_NONE;
1397         qboolean forthismap = false;
1398         const char *url;
1399         const char *name = 0;
1400
1401         if(!curl_dll)
1402         {
1403                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1404                 return;
1405         }
1406
1407         if(!cl_curl_enabled.integer)
1408         {
1409                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1410                 return;
1411         }
1412
1413         if(Cmd_Argc(cmd) < 2)
1414         {
1415                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1416                 return;
1417         }
1418
1419         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1420         end = Cmd_Argc(cmd);
1421
1422         for(i = 1; i != end; ++i)
1423         {
1424                 const char *a = Cmd_Argv(cmd, i);
1425                 if(!strcmp(a, "--info"))
1426                 {
1427                         Curl_Info_f(cmd);
1428                         return;
1429                 }
1430                 else if(!strcmp(a, "--cancel"))
1431                 {
1432                         if(i == end - 1) // last argument
1433                                 Curl_CancelAll();
1434                         else
1435                         {
1436                                 downloadinfo *di = Curl_Find(url);
1437                                 if(di)
1438                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1439                                 else
1440                                         Con_Print("download not found\n");
1441                         }
1442                         return;
1443                 }
1444                 else if(!strcmp(a, "--pak"))
1445                 {
1446                         loadtype = LOADTYPE_PAK;
1447                 }
1448                 else if(!strcmp(a, "--cachepic"))
1449                 {
1450                         loadtype = LOADTYPE_CACHEPIC;
1451                 }
1452                 else if(!strcmp(a, "--skinframe"))
1453                 {
1454                         loadtype = LOADTYPE_SKINFRAME;
1455                 }
1456                 else if(!strcmp(a, "--for")) // must be last option
1457                 {
1458                         for(i = i + 1; i != end - 1; ++i)
1459                         {
1460                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1461                                         goto needthefile; // why can't I have a "double break"?
1462                         }
1463                         // if we get here, we have all the files...
1464                         return;
1465                 }
1466                 else if(!strcmp(a, "--forthismap"))
1467                 {
1468                         forthismap = true;
1469                 }
1470                 else if(!strcmp(a, "--as"))
1471                 {
1472                         if(i < end - 1)
1473                         {
1474                                 ++i;
1475                                 name = Cmd_Argv(cmd, i);
1476                         }
1477                 }
1478                 else if(!strcmp(a, "--clear_autodownload"))
1479                 {
1480                         // mark all running downloads as "not for this map", so if they
1481                         // fail, it does not matter
1482                         Curl_Clear_forthismap();
1483                         return;
1484                 }
1485                 else if(!strcmp(a, "--finish_autodownload"))
1486                 {
1487                         if(numdownloads_added)
1488                         {
1489                                 char donecommand[256];
1490                                 if(cls.netcon)
1491                                 {
1492                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1493                                         {
1494                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1495                                                 Curl_CommandWhenDone(donecommand);
1496                                                 noclear = true;
1497                                                 CL_Disconnect();
1498                                                 noclear = false;
1499                                                 Curl_CheckCommandWhenDone();
1500                                         }
1501                                         else
1502                                                 Curl_Register_predownload();
1503                                 }
1504                         }
1505                         return;
1506                 }
1507                 else if(!strncmp(a, "--maxspeed=", 11))
1508                 {
1509                         maxspeed = atof(a + 11);
1510                 }
1511                 else if(*a == '-')
1512                 {
1513                         Con_Printf("curl: invalid option %s\n", a);
1514                         // but we ignore the option
1515                 }
1516         }
1517
1518 needthefile:
1519         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1520 }
1521
1522 /*
1523 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1524 {
1525         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1526         Z_Free(buffer);
1527 }
1528
1529 void Curl_CurlCat_f(cmd_state_t *cmd)
1530 {
1531         unsigned char *buf;
1532         const char *url = Cmd_Argv(cmd, 1);
1533         buf = Z_Malloc(16384);
1534         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1535 }
1536 */
1537
1538 /*
1539 ====================
1540 Curl_Init_Commands
1541
1542 loads the commands and cvars this library uses
1543 ====================
1544 */
1545 void Curl_Init_Commands(void)
1546 {
1547         Cvar_RegisterVariable (&cl_curl_enabled);
1548         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1549         Cvar_RegisterVariable (&cl_curl_maxspeed);
1550         Cvar_RegisterVariable (&sv_curl_defaulturl);
1551         Cvar_RegisterVariable (&sv_curl_serverpackages);
1552         Cvar_RegisterVariable (&sv_curl_maxspeed);
1553         Cvar_RegisterVariable (&cl_curl_useragent);
1554         Cvar_RegisterVariable (&cl_curl_useragent_append);
1555         Cmd_AddCommand(CMD_CLIENT | CMD_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1556         //Cmd_AddCommand(&cmd_client, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1557 }
1558
1559 /*
1560 ====================
1561 Curl_GetDownloadInfo
1562
1563 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1564 The number of elements in the array is returned in int *nDownloads.
1565 const char **additional_info may be set to a string of additional user
1566 information, or to NULL if no such display shall occur. The returned
1567 array must be freed later using Z_Free.
1568 ====================
1569 */
1570 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1571 {
1572         int i;
1573         downloadinfo *di;
1574         Curl_downloadinfo_t *downinfo;
1575
1576         if(!curl_dll)
1577         {
1578                 *nDownloads = 0;
1579                 if(additional_info)
1580                         *additional_info = NULL;
1581                 return NULL;
1582         }
1583
1584         if (curl_mutex) Thread_LockMutex(curl_mutex);
1585
1586         i = 0;
1587         for(di = downloads; di; di = di->next)
1588                 ++i;
1589
1590         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1591         i = 0;
1592         for(di = downloads; di; di = di->next)
1593         {
1594                 // do not show infobars for background downloads
1595                 if(developer.integer <= 0)
1596                         if(di->buffer)
1597                                 continue;
1598                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1599                 if(di->curle)
1600                 {
1601                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1602                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1603                         downinfo[i].queued = false;
1604                 }
1605                 else
1606                 {
1607                         downinfo[i].queued = true;
1608                 }
1609                 ++i;
1610         }
1611
1612         if(additional_info)
1613         {
1614                 // TODO: can I clear command_when_done as soon as the first download fails?
1615                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1616                 {
1617                         if(!strncmp(command_when_done, "connect ", 8))
1618                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1619                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1620                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1621                         else
1622                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1623                         *additional_info = addinfo;
1624                 }
1625                 else
1626                         *additional_info = NULL;
1627         }
1628
1629         *nDownloads = i;
1630         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1631         return downinfo;
1632 }
1633
1634
1635 /*
1636 ====================
1637 Curl_FindPackURL
1638
1639 finds the URL where to find a given package.
1640
1641 For this, it reads a file "curl_urls.txt" of the following format:
1642
1643         data*.pk3       -
1644         revdm*.pk3      http://revdm/downloads/are/here/
1645         *                       http://any/other/stuff/is/here/
1646
1647 The URLs should end in /. If not, downloads will still work, but the cached files
1648 can't be just put into the data directory with the same download configuration
1649 (you might want to do this if you want to tag downloaded files from your
1650 server, but you should not). "-" means "don't download".
1651
1652 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1653 location instead.
1654
1655 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1656 this file for obvious reasons.
1657 ====================
1658 */
1659 static const char *Curl_FindPackURL(const char *filename)
1660 {
1661         static char foundurl[1024]; // invoked only by server
1662         fs_offset_t filesize;
1663         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1664         if(buf && filesize)
1665         {
1666                 // read lines of format "pattern url"
1667                 char *p = buf;
1668                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1669                 qboolean eof = false;
1670
1671                 pattern = p;
1672                 while(!eof)
1673                 {
1674                         switch(*p)
1675                         {
1676                                 case 0:
1677                                         eof = true;
1678                                         // fallthrough
1679                                 case '\n':
1680                                 case '\r':
1681                                         if(pattern && url && patternend)
1682                                         {
1683                                                 if(!urlend)
1684                                                         urlend = p;
1685                                                 *patternend = 0;
1686                                                 *urlend = 0;
1687                                                 if(matchpattern(filename, pattern, true))
1688                                                 {
1689                                                         strlcpy(foundurl, url, sizeof(foundurl));
1690                                                         Z_Free(buf);
1691                                                         return foundurl;
1692                                                 }
1693                                         }
1694                                         pattern = NULL;
1695                                         patternend = NULL;
1696                                         url = NULL;
1697                                         urlend = NULL;
1698                                         break;
1699                                 case ' ':
1700                                 case '\t':
1701                                         if(pattern && !patternend)
1702                                                 patternend = p;
1703                                         else if(url && !urlend)
1704                                                 urlend = p;
1705                                         break;
1706                                 default:
1707                                         if(!pattern)
1708                                                 pattern = p;
1709                                         else if(pattern && patternend && !url)
1710                                                 url = p;
1711                                         break;
1712                         }
1713                         ++p;
1714                 }
1715         }
1716         if(buf)
1717                 Z_Free(buf);
1718         return sv_curl_defaulturl.string;
1719 }
1720
1721 typedef struct requirement_s
1722 {
1723         struct requirement_s *next;
1724         char filename[MAX_OSPATH];
1725 }
1726 requirement;
1727 static requirement *requirements = NULL;
1728
1729
1730 /*
1731 ====================
1732 Curl_RequireFile
1733
1734 Adds the given file to the list of requirements.
1735 ====================
1736 */
1737 void Curl_RequireFile(const char *filename)
1738 {
1739         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1740         req->next = requirements;
1741         strlcpy(req->filename, filename, sizeof(req->filename));
1742         requirements = req;
1743 }
1744
1745 /*
1746 ====================
1747 Curl_ClearRequirements
1748
1749 Clears the list of required files for playing on the current map.
1750 This should be called at every map change.
1751 ====================
1752 */
1753 void Curl_ClearRequirements(void)
1754 {
1755         while(requirements)
1756         {
1757                 requirement *req = requirements;
1758                 requirements = requirements->next;
1759                 Z_Free(req);
1760         }
1761 }
1762
1763 /*
1764 ====================
1765 Curl_SendRequirements
1766
1767 Makes the current host_clients download all files he needs.
1768 This is done by sending him the following console commands:
1769
1770         curl --clear_autodownload
1771         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1772         curl --finish_autodownload
1773 ====================
1774 */
1775 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1776 {
1777         const char *p;
1778         const char *thispack = FS_WhichPack(filename);
1779         const char *packurl;
1780
1781         if(!thispack || !*thispack)
1782                 return false;
1783
1784         p = strrchr(thispack, '/');
1785         if(p)
1786                 thispack = p + 1;
1787
1788         packurl = Curl_FindPackURL(thispack);
1789
1790         if(packurl && *packurl && strcmp(packurl, "-"))
1791         {
1792                 if(!foundone)
1793                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1794
1795                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1796                 strlcat(sendbuffer, thispack, sendbuffer_len);
1797                 if(sv_curl_maxspeed.value > 0)
1798                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1799                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1800                 strlcat(sendbuffer, filename, sendbuffer_len);
1801                 strlcat(sendbuffer, " ", sendbuffer_len);
1802                 strlcat(sendbuffer, packurl, sendbuffer_len);
1803                 strlcat(sendbuffer, thispack, sendbuffer_len);
1804                 strlcat(sendbuffer, "\n", sendbuffer_len);
1805
1806                 return true;
1807         }
1808
1809         return false;
1810 }
1811 void Curl_SendRequirements(void)
1812 {
1813         // for each requirement, find the pack name
1814         char sendbuffer[4096] = "";
1815         requirement *req;
1816         qboolean foundone = false;
1817         const char *p;
1818
1819         for(req = requirements; req; req = req->next)
1820                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1821
1822         p = sv_curl_serverpackages.string;
1823         while(COM_ParseToken_Simple(&p, false, false, true))
1824                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1825
1826         if(foundone)
1827                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1828
1829         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1830                 Host_ClientCommands("%s", sendbuffer);
1831         else
1832                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1833 }