libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5 #include "com_list.h"
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t curl_enabled = {CF_SHARED | CF_ARCHIVE, "curl_enabled","1", "whether libcurl may be used to GET files or POST data"};
  11 static cvar_t curl_maxdownloads = {CF_SHARED | CF_ARCHIVE, "curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  12 static cvar_t curl_maxspeed = {CF_SHARED | CF_ARCHIVE, "curl_maxspeed","300", "maximum download speed (KiB/s)"};
  13 static cvar_t curl_useragent = {CF_SHARED, "curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  14 static cvar_t curl_useragent_append = {CF_SHARED, "curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  15
  16 static cvar_t sv_curl_defaulturl = {CF_SERVER, "sv_curl_defaulturl","", "default autodownload source URL"};
  17 static cvar_t sv_curl_serverpackages = {CF_SERVER, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  18 static cvar_t sv_curl_maxspeed = {CF_SERVER, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  19
  20 static cvar_t developer_curl = {CF_SHARED, "developer_curl","0", "whether verbose libcurl output should be printed to stderr"};
  21
  22 /*
  23 =================================================================
  24
  25   Minimal set of definitions from libcurl
  26
  27   WARNING: for a matter of simplicity, several pointer types are
  28   casted to "void*", and most enumerated values are not included
  29
  30 =================================================================
  31 */
  32
  33 typedef struct CURL_s CURL;
  34 typedef struct CURLM_s CURLM;
  35 typedef struct curl_slist curl_slist;
  36 typedef enum
  37 {
  38         CURLE_OK = 0
  39 }
  40 CURLcode;
  41 typedef enum
  42 {
  43         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  44         CURLM_OK = 0
  45 }
  46 CURLMcode;
  47 #define CURL_GLOBAL_NOTHING 0
  48 #define CURL_GLOBAL_SSL 1
  49 #define CURL_GLOBAL_WIN32 2
  50 #define CURLOPTTYPE_LONG          0
  51 #define CURLOPTTYPE_OBJECTPOINT   10000
  52 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  53 #define CURLOPTTYPE_OFF_T         30000
  54 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  55 typedef enum
  56 {
  57         CINIT(WRITEDATA, OBJECTPOINT, 1),
  58         CINIT(URL,  OBJECTPOINT, 2),
  59         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  60         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  61         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  62         CINIT(REFERER, OBJECTPOINT, 16),
  63         CINIT(USERAGENT, OBJECTPOINT, 18),
  64         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  65         CINIT(LOW_SPEED_TIME, LONG, 20),
  66         CINIT(RESUME_FROM, LONG, 21),
  67         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  68         CINIT(VERBOSE, LONG, 41),
  69         CINIT(POST, LONG, 47),         /* HTTP POST method */
  70         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  71         CINIT(POSTFIELDSIZE, LONG, 60),
  72         CINIT(PRIVATE, OBJECTPOINT, 103),
  73         CINIT(PROTOCOLS, LONG, 181),
  74         CINIT(REDIR_PROTOCOLS, LONG, 182)
  75 }
  76 CURLoption;
  77 #define CURLPROTO_HTTP   (1<<0)
  78 #define CURLPROTO_HTTPS  (1<<1)
  79 #define CURLPROTO_FTP    (1<<2)
  80 typedef enum
  81 {
  82         CURLINFO_TEXT = 0,
  83         CURLINFO_HEADER_IN,    /* 1 */
  84         CURLINFO_HEADER_OUT,   /* 2 */
  85         CURLINFO_DATA_IN,      /* 3 */
  86         CURLINFO_DATA_OUT,     /* 4 */
  87         CURLINFO_SSL_DATA_IN,  /* 5 */
  88         CURLINFO_SSL_DATA_OUT, /* 6 */
  89         CURLINFO_END
  90 }
  91 curl_infotype;
  92 #define CURLINFO_STRING   0x100000
  93 #define CURLINFO_LONG     0x200000
  94 #define CURLINFO_DOUBLE   0x300000
  95 #define CURLINFO_SLIST    0x400000
  96 #define CURLINFO_MASK     0x0fffff
  97 #define CURLINFO_TYPEMASK 0xf00000
  98 typedef enum
  99 {
 100         CURLINFO_NONE, /* first, never use this */
 101         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
 102         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
 103         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 104         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 105         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 106         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 107         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 108         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 109         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 110         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 111         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 112         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 113         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 114         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 115         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 116         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 117         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 118         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 119         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 120         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 121         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 122         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 123         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 124         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 125         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 126         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 127         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 128 }
 129 CURLINFO;
 130
 131 typedef enum
 132 {
 133         CURLMSG_NONE, /* first, not used */
 134         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 135                                          the CURLcode of the transfer */
 136         CURLMSG_LAST
 137 }
 138 CURLMSG;
 139 typedef struct
 140 {
 141         CURLMSG msg;       /* what this message means */
 142         CURL *easy_handle; /* the handle it concerns */
 143         union
 144         {
 145                 void *whatever;    /* message-specific data */
 146                 CURLcode result;   /* return code for transfer */
 147         }
 148         data;
 149 }
 150 CURLMsg;
 151
 152 static void (*qcurl_global_init) (long flags);
 153 static void (*qcurl_global_cleanup) (void);
 154
 155 static CURL * (*qcurl_easy_init) (void);
 156 static void (*qcurl_easy_cleanup) (CURL *handle);
 157 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 158 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 159 static const char * (*qcurl_easy_strerror) (CURLcode);
 160
 161 static CURLM * (*qcurl_multi_init) (void);
 162 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 163 static CURLMcode (*qcurl_multi_poll) (CURLM *multi_handle, void*, unsigned int extra_nfds, int timeout_ms, int *ret);
 164 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 165 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 166 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 167 static void (*qcurl_multi_cleanup) (CURLM *);
 168 static const char * (*qcurl_multi_strerror) (CURLcode);
 169 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 170 static void (*qcurl_slist_free_all) (curl_slist *list);
 171
 172 static dllfunction_t curlfuncs[] =
 173 {
 174         {"curl_global_init",            (void **) &qcurl_global_init},
 175         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 176         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 177         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 178         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 179         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 180         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 181         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 182         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 183         {"curl_multi_poll",             (void **) &qcurl_multi_poll},
 184         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 185         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 186         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 187         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 188         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 189         {"curl_slist_append",           (void **) &qcurl_slist_append},
 190         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 191         {NULL, NULL}
 192 };
 193
 194 // Handle for CURL DLL
 195 static dllhandle_t curl_dll = NULL;
 196 // will be checked at many places to find out if qcurl calls are allowed
 197
 198 #define LOADTYPE_NONE 0
 199 #define LOADTYPE_PAK 1
 200 #define LOADTYPE_CACHEPIC 2
 201 #define LOADTYPE_SKINFRAME 3
 202
 203 void *curl_mutex = NULL;
 204
 205 typedef struct downloadinfo_s
 206 {
 207         char filename[MAX_OSPATH];
 208         char url[1024];
 209         char referer[256];
 210         qfile_t *stream;
 211         fs_offset_t startpos;
 212         CURL *curle;
 213         qbool started;
 214         int loadtype;
 215         size_t bytes_received; // for buffer
 216         double bytes_received_curl; // for throttling
 217         double bytes_sent_curl; // for throttling
 218         llist_t list;
 219         qbool forthismap;
 220         double maxspeed;
 221         curl_slist *slist; // http headers
 222
 223         unsigned char *buffer;
 224         size_t buffersize;
 225         curl_callback_t callback;
 226         void *callback_data;
 227
 228         const unsigned char *postbuf;
 229         size_t postbufsize;
 230         const char *post_content_type;
 231         const char *extraheaders;
 232 }
 233 downloadinfo;
 234 LIST_HEAD(downloads);
 235 static int numdownloads = 0;
 236
 237 static qbool noclear = false;
 238
 239 static int numdownloads_fail = 0;
 240 static int numdownloads_success = 0;
 241 static int numdownloads_added = 0;
 242 static char command_when_done[256] = "";
 243 static char command_when_error[256] = "";
 244
 245 /*
 246 ====================
 247 Curl_CommandWhenDone
 248
 249 Sets the command which is to be executed when the last download completes AND
 250 all downloads since last server connect ended with a successful status.
 251 Setting the command to NULL clears it.
 252 ====================
 253 */
 254 static void Curl_CommandWhenDone(const char *cmd)
 255 {
 256         if(!curl_dll)
 257                 return;
 258         if(cmd)
 259                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 260         else
 261                 *command_when_done = 0;
 262 }
 263
 264 /*
 265 FIXME
 266 Do not use yet. Not complete.
 267 Problem: what counts as an error?
 268 */
 269
 270 static void Curl_CommandWhenError(const char *cmd)
 271 {
 272         if(!curl_dll)
 273                 return;
 274         if(cmd)
 275                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 276         else
 277                 *command_when_error = 0;
 278 }
 279
 280 /*
 281 ====================
 282 Curl_Clear_forthismap
 283
 284 Clears the "will disconnect on failure" flags.
 285 ====================
 286 */
 287 void Curl_Clear_forthismap(void)
 288 {
 289         downloadinfo *di;
 290         if(noclear)
 291                 return;
 292         if (curl_mutex) Thread_LockMutex(curl_mutex);
 293         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 294                 di->forthismap = false;
 295         Curl_CommandWhenError(NULL);
 296         Curl_CommandWhenDone(NULL);
 297         numdownloads_fail = 0;
 298         numdownloads_success = 0;
 299         numdownloads_added = 0;
 300         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 301 }
 302
 303 /*
 304 ====================
 305 Curl_Have_forthismap
 306
 307 Returns true if a download needed for the current game is running.
 308 ====================
 309 */
 310 qbool Curl_Have_forthismap(void)
 311 {
 312         return numdownloads_added != 0;
 313 }
 314
 315 void Curl_Register_predownload(void)
 316 {
 317         if (curl_mutex) Thread_LockMutex(curl_mutex);
 318         Curl_CommandWhenDone("cl_begindownloads");
 319         Curl_CommandWhenError("cl_begindownloads");
 320         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 321 }
 322
 323 /*
 324 ====================
 325 Curl_CheckCommandWhenDone
 326
 327 Checks if a "done command" is to be executed.
 328 All downloads finished, at least one success since connect, no single failure
 329 -> execute the command.
 330 */
 331 static void Curl_CheckCommandWhenDone(void)
 332 {
 333         if(!curl_dll)
 334                 return;
 335         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 336         {
 337                 if(numdownloads_fail == 0)
 338                 {
 339                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 340                         Cbuf_AddText(cmd_local, "\n");
 341                         Cbuf_AddText(cmd_local, command_when_done);
 342                         Cbuf_AddText(cmd_local, "\n");
 343                 }
 344                 else
 345                 {
 346                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 347                         Cbuf_AddText(cmd_local, "\n");
 348                         Cbuf_AddText(cmd_local, command_when_error);
 349                         Cbuf_AddText(cmd_local, "\n");
 350                 }
 351                 Curl_Clear_forthismap();
 352         }
 353 }
 354
 355 /*
 356 ====================
 357 CURL_CloseLibrary
 358
 359 Load the cURL DLL
 360 ====================
 361 */
 362 static qbool CURL_OpenLibrary (void)
 363 {
 364         const char* dllnames [] =
 365         {
 366 #if defined(WIN32)
 367                 "libcurl-4.dll",
 368                 "libcurl-3.dll",
 369 #elif defined(MACOSX)
 370                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 371                 "libcurl.3.dylib", // Mac OS X Tiger
 372                 "libcurl.2.dylib", // Mac OS X Panther
 373 #else
 374                 "libcurl.so.4",
 375                 "libcurl.so.3",
 376                 "libcurl.so", // FreeBSD
 377 #endif
 378                 NULL
 379         };
 380
 381         // Already loaded?
 382         if (curl_dll)
 383                 return true;
 384
 385         // Load the DLL
 386         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
 387 }
 388
 389
 390 /*
 391 ====================
 392 CURL_CloseLibrary
 393
 394 Unload the cURL DLL
 395 ====================
 396 */
 397 static void CURL_CloseLibrary (void)
 398 {
 399         Sys_FreeLibrary (&curl_dll);
 400 }
 401
 402
 403 static CURLM *curlm = NULL;
 404 static double bytes_received = 0; // used for bandwidth throttling
 405 static double bytes_sent = 0; // used for bandwidth throttling
 406 static double curltime = 0;
 407
 408 /*
 409 ====================
 410 CURL_fwrite
 411
 412 fwrite-compatible function that writes the data to a file. libcurl can call
 413 this.
 414 ====================
 415 */
 416 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 417 {
 418         fs_offset_t ret = -1;
 419         size_t bytes = size * nmemb;
 420         downloadinfo *di = (downloadinfo *) vdi;
 421
 422         if(di->buffer)
 423         {
 424                 if(di->bytes_received + bytes <= di->buffersize)
 425                 {
 426                         memcpy(di->buffer + di->bytes_received, data, bytes);
 427                         ret = bytes;
 428                 }
 429                 // otherwise: buffer overrun, ret stays -1
 430         }
 431
 432         if(di->stream)
 433         {
 434                 ret = FS_Write(di->stream, data, bytes);
 435         }
 436
 437         di->bytes_received += bytes;
 438
 439         //Con_Printf("CURL_fwrite callback timestamp: %f bytes: %ld\n", host.realtime, ret);
 440
 441         return ret;
 442         // Why not ret / nmemb?
 443         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 444         // Yes, this is incompatible to fwrite(2).
 445 }
 446
 447 typedef enum
 448 {
 449         CURL_DOWNLOAD_SUCCESS = 0,
 450         CURL_DOWNLOAD_FAILED,
 451         CURL_DOWNLOAD_ABORTED,
 452         CURL_DOWNLOAD_SERVERERROR
 453 }
 454 CurlStatus;
 455
 456 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 457 {
 458         downloadinfo *di = (downloadinfo *) cbdata;
 459         switch(status)
 460         {
 461                 case CURLCBSTATUS_OK:
 462                         Con_DPrintf("Download of %s: OK\n", di->filename);
 463                         break;
 464                 case CURLCBSTATUS_FAILED:
 465                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 466                         break;
 467                 case CURLCBSTATUS_ABORTED:
 468                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 469                         break;
 470                 case CURLCBSTATUS_SERVERERROR:
 471                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 472                         break;
 473                 case CURLCBSTATUS_UNKNOWN:
 474                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 475                         break;
 476                 default:
 477                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 478                         break;
 479         }
 480 }
 481
 482 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 483 {
 484         curl_default_callback(status, length_received, buffer, cbdata);
 485 }
 486
 487 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 488 {
 489         unsigned char *pixels = NULL;
 490         fs_offset_t filesize = 0;
 491         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 492         if(data)
 493         {
 494                 int mip = 0;
 495                 if(!strcmp(content_type, "image/jpeg"))
 496                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 497                 else if(!strcmp(content_type, "image/png"))
 498                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 499                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 500                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 501                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 502                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 503                 else
 504                         Con_Printf("Did not detect content type: %s\n", content_type);
 505                 Mem_Free(data);
 506         }
 507         // do we call Image_MakeLinearColorsFromsRGB or not?
 508         return pixels;
 509 }
 510
 511 /*
 512 ====================
 513 Curl_EndDownload
 514
 515 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 516 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 517 code from libcurl, or 0, if another error has occurred.
 518 ====================
 519 */
 520 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 521 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 522 {
 523         char content_type[64];
 524         qbool ok = false;
 525         if(!curl_dll)
 526                 return;
 527         switch(status)
 528         {
 529                 case CURL_DOWNLOAD_SUCCESS:
 530                         ok = true;
 531                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 532                         break;
 533                 case CURL_DOWNLOAD_FAILED:
 534                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 535                         break;
 536                 case CURL_DOWNLOAD_ABORTED:
 537                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 538                         break;
 539                 case CURL_DOWNLOAD_SERVERERROR:
 540                         // reopen to enforce it to have zero bytes again
 541                         if(di->stream)
 542                         {
 543                                 FS_Close(di->stream);
 544                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 545                         }
 546
 547                         if(di->callback)
 548                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 549                         break;
 550                 default:
 551                         if(di->callback)
 552                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 553                         break;
 554         }
 555         if(content_type_)
 556                 strlcpy(content_type, content_type_, sizeof(content_type));
 557         else
 558                 *content_type = 0;
 559
 560         if(di->curle)
 561         {
 562                 qcurl_multi_remove_handle(curlm, di->curle);
 563                 qcurl_easy_cleanup(di->curle);
 564                 if(di->slist)
 565                         qcurl_slist_free_all(di->slist);
 566         }
 567
 568         if(!di->callback && ok && !di->bytes_received)
 569         {
 570                 Con_Printf("ERROR: empty file\n");
 571                 ok = false;
 572         }
 573
 574         if(di->stream)
 575                 FS_Close(di->stream);
 576
 577 #define CLEAR_AND_RETRY() \
 578         do \
 579         { \
 580                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 581                 FS_Close(di->stream); \
 582                 if(di->startpos && !di->callback) \
 583                 { \
 584                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 585                         di->forthismap = false; \
 586                 } \
 587         } \
 588         while(0)
 589
 590         if(ok && di->loadtype == LOADTYPE_PAK)
 591         {
 592                 ok = FS_AddPack(di->filename, NULL, true, true);
 593                 if(!ok)
 594                         CLEAR_AND_RETRY();
 595         }
 596         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 597         {
 598                 const char *p;
 599                 unsigned char *pixels = NULL;
 600
 601                 p = di->filename;
 602 #ifdef WE_ARE_EVIL
 603                 if(!strncmp(p, "dlcache/", 8))
 604                         p += 8;
 605 #endif
 606
 607                 pixels = decode_image(di, content_type);
 608                 if(pixels)
 609                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 610                 else
 611                         CLEAR_AND_RETRY();
 612         }
 613         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 614         {
 615                 const char *p;
 616                 unsigned char *pixels = NULL;
 617
 618                 p = di->filename;
 619 #ifdef WE_ARE_EVIL
 620                 if(!strncmp(p, "dlcache/", 8))
 621                         p += 8;
 622 #endif
 623
 624                 pixels = decode_image(di, content_type);
 625                 if(pixels)
 626                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 627                 else
 628                         CLEAR_AND_RETRY();
 629         }
 630
 631         List_Delete(&di->list);
 632
 633         --numdownloads;
 634         if(di->forthismap)
 635         {
 636                 if(ok)
 637                         ++numdownloads_success;
 638                 else
 639                         ++numdownloads_fail;
 640         }
 641         Z_Free(di);
 642 }
 643
 644 /*
 645 ====================
 646 CleanURL
 647
 648 Returns a "cleaned up" URL for display (to strip login data)
 649 ====================
 650 */
 651 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 652 {
 653         const char *p, *q, *r;
 654
 655         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 656         p = strstr(url, "://");
 657         if(p)
 658         {
 659                 q = strchr(p + 3, '@');
 660                 if(q)
 661                 {
 662                         r = strchr(p + 3, '/');
 663                         if(!r || q < r)
 664                         {
 665                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 666                                 return urlbuf;
 667                         }
 668                 }
 669         }
 670
 671         return url;
 672 }
 673
 674 /*
 675 ====================
 676 CheckPendingDownloads
 677
 678 checks if there are free download slots to start new downloads in.
 679 To not start too many downloads at once, only one download is added at a time,
 680 up to a maximum number of cl_curl_maxdownloads are running.
 681 ====================
 682 */
 683 static void CheckPendingDownloads(void)
 684 {
 685         const char *h;
 686         char urlbuf[1024];
 687         char vabuf[1024];
 688         if(!curl_dll)
 689                 return;
 690         if(numdownloads < curl_maxdownloads.integer)
 691         {
 692                 downloadinfo *di;
 693                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
 694                 {
 695                         if(!di->started)
 696                         {
 697                                 if(!di->buffer)
 698                                 {
 699                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 700
 701                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 702                                         if(!di->stream)
 703                                         {
 704                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 705                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 706                                                 return;
 707                                         }
 708                                         FS_Seek(di->stream, 0, SEEK_END);
 709                                         di->startpos = FS_Tell(di->stream);
 710
 711                                         if(di->startpos > 0)
 712                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 713                                         Con_Print("...\n");
 714                                 }
 715                                 else
 716                                 {
 717                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 718                                         di->startpos = 0;
 719                                 }
 720
 721                                 di->curle = qcurl_easy_init();
 722                                 di->slist = NULL;
 723                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 724                                 if(curl_useragent.integer)
 725                                 {
 726                                         const char *ua
 727 #ifdef HTTP_USER_AGENT
 728                                                 = HTTP_USER_AGENT;
 729 #else
 730                                                 = engineversion;
 731 #endif
 732                                         if(!ua)
 733                                                 ua = "";
 734                                         if(*curl_useragent_append.string)
 735                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 736                                                         ua,
 737                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 738                                                                 ? " "
 739                                                                 : "",
 740                                                         curl_useragent_append.string);
 741                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 742                                 }
 743                                 else
 744                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 745                                 if(developer_curl.integer)
 746                                         qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 750                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 751                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 752                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 753                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 754                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 755                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 756                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 757                                 {
 758                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 759                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 760                                 }
 761                                 if(di->post_content_type)
 762                                 {
 763                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 764                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 765                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 766                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 767                                 }
 768
 769                                 // parse extra headers into slist
 770                                 // \n separated list!
 771                                 h = di->extraheaders;
 772                                 while(h)
 773                                 {
 774                                         const char *hh = strchr(h, '\n');
 775                                         if(hh)
 776                                         {
 777                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 778                                                 memcpy(buf, h, hh - h);
 779                                                 buf[hh - h] = 0;
 780                                                 di->slist = qcurl_slist_append(di->slist, buf);
 781                                                 h = hh + 1;
 782                                         }
 783                                         else
 784                                         {
 785                                                 di->slist = qcurl_slist_append(di->slist, h);
 786                                                 h = NULL;
 787                                         }
 788                                 }
 789
 790                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 791
 792                                 qcurl_multi_add_handle(curlm, di->curle);
 793                                 di->started = true;
 794                                 ++numdownloads;
 795                                 if(numdownloads >= curl_maxdownloads.integer)
 796                                         break;
 797                         }
 798                 }
 799         }
 800 }
 801
 802 /*
 803 ====================
 804 Curl_Init
 805
 806 this function MUST be called before using anything else in this file.
 807 On Win32, this must be called AFTER WSAStartup has been done!
 808 ====================
 809 */
 810 void Curl_Init(void)
 811 {
 812         CURL_OpenLibrary();
 813         if(!curl_dll)
 814                 return;
 815         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 816         qcurl_global_init(CURL_GLOBAL_SSL);
 817         curlm = qcurl_multi_init();
 818 }
 819
 820 /*
 821 ====================
 822 Curl_Shutdown
 823
 824 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 825 ====================
 826 */
 827 void Curl_ClearRequirements(void);
 828 void Curl_Shutdown(void)
 829 {
 830         if(!curl_dll)
 831                 return;
 832         Curl_ClearRequirements();
 833         Curl_CancelAll();
 834         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 835         CURL_CloseLibrary();
 836         curl_dll = NULL;
 837 }
 838
 839 // for VM_checkextension()
 840 qbool Curl_Available(void)
 841 {
 842         return curl_dll ? true : false;
 843 }
 844
 845 /*
 846 ====================
 847 Curl_Find
 848
 849 Finds the internal information block for a download given by file name.
 850 ====================
 851 */
 852 static downloadinfo *Curl_Find(const char *filename)
 853 {
 854         downloadinfo *di;
 855         if(!curl_dll)
 856                 return NULL;
 857         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 858                 if(!strcasecmp(di->filename, filename))
 859                         return di;
 860         return NULL;
 861 }
 862
 863 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 864 {
 865         downloadinfo *di, *ndi;
 866         if(!curl_dll)
 867                 return;
 868         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
 869         {
 870                 if(di->callback == callback && di->callback_data == cbdata)
 871                 {
 872                         di->callback = curl_quiet_callback; // do NOT call the callback
 873                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 874                 }
 875         }
 876 }
 877
 878 /*
 879 ====================
 880 Curl_Begin
 881
 882 Starts a download of a given URL to the file name portion of this URL (or name
 883 if given) in the "dlcache/" folder.
 884 ====================
 885 */
 886 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 887 {
 888         if(buf)
 889                 if(loadtype != LOADTYPE_NONE)
 890                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 891
 892         if(!curl_dll || !curl_enabled.integer)
 893         {
 894                 return false;
 895         }
 896         else
 897         {
 898                 char fn[MAX_OSPATH];
 899                 char urlbuf[1024];
 900                 const char *p, *q;
 901                 size_t length;
 902                 downloadinfo *di;
 903
 904                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 905                 p = strchr(URL, ':');
 906                 if(p)
 907                 {
 908                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 909                         {
 910                                 char addressstring[128];
 911                                 *addressstring = 0;
 912                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 913                                 q = strchr(addressstring, ':');
 914                                 if(!q)
 915                                         q = addressstring + strlen(addressstring);
 916                                 if(*addressstring)
 917                                 {
 918                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 919                                         URL = urlbuf;
 920                                 }
 921                         }
 922                 }
 923
 924                 // Note: This extraction of the file name portion is NOT entirely correct.
 925                 //
 926                 // It does the following:
 927                 //
 928                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 929                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 930                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 931                 //
 932                 // However, I'd like to keep this "buggy" behavior so that PHP script
 933                 // authors can write download scripts without having to enable
 934                 // AcceptPathInfo on Apache. They just have to ensure that their script
 935                 // can be called with such a "fake" path name like
 936                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 937                 //
 938                 // By the way, such PHP scripts should either send the file or a
 939                 // "Location:" redirect; PHP code example:
 940                 //
 941                 //   header("Location: http://www.example.com/");
 942                 //
 943                 // By the way, this will set User-Agent to something like
 944                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 945                 // dp://serverhost:serverport/ so you can filter on this; an example
 946                 // httpd log file line might be:
 947                 //
 948                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 949
 950                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 951
 952                 if(buf)
 953                 {
 954                         if(!name)
 955                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 956                 }
 957                 else
 958                 {
 959                         if(!name)
 960                         {
 961                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 962                                 p = strrchr(name, '/');
 963                                 p = p ? (p+1) : name;
 964                                 q = strchr(p, '?');
 965                                 length = q ? (size_t)(q - p) : strlen(p);
 966                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 967                         }
 968                         else
 969                         {
 970                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 971                         }
 972
 973                         name = fn; // make it point back
 974
 975                         // already downloading the file?
 976                         {
 977                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 978                                 if(existingdownloadinfo)
 979                                 {
 980                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 981
 982                                         // however, if it was not for this map yet...
 983                                         if(forthismap && !existingdownloadinfo->forthismap)
 984                                         {
 985                                                 existingdownloadinfo->forthismap = true;
 986                                                 // this "fakes" a download attempt so the client will wait for
 987                                                 // the download to finish and then reconnect
 988                                                 ++numdownloads_added;
 989                                         }
 990
 991                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 992                                         return false;
 993                                 }
 994                         }
 995
 996                         if(FS_FileExists(fn))
 997                         {
 998                                 if(loadtype == LOADTYPE_PAK)
 999                                 {
1000                                         qbool already_loaded;
1001                                         if(FS_AddPack(fn, &already_loaded, true, true))
1002                                         {
1003                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
1004                                                 if(already_loaded)
1005                                                         Con_DPrintf("(pak was already loaded)\n");
1006                                                 else
1007                                                 {
1008                                                         if(forthismap)
1009                                                         {
1010                                                                 ++numdownloads_added;
1011                                                                 ++numdownloads_success;
1012                                                         }
1013                                                 }
1014
1015                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1016                                                 return false;
1017                                         }
1018                                         else
1019                                         {
1020                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1021                                                 if(f)
1022                                                 {
1023                                                         char b[4] = {0};
1024                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1025
1026                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1027                                                         {
1028                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1029                                                                 FS_Close(f);
1030                                                                 f = FS_OpenRealFile(fn, "wb", false);
1031                                                                 if(f)
1032                                                                         FS_Close(f);
1033                                                         }
1034                                                         else
1035                                                         {
1036                                                                 // OK
1037                                                                 FS_Close(f);
1038                                                         }
1039                                                 }
1040                                         }
1041                                 }
1042                                 else
1043                                 {
1044                                         // never resume these
1045                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1046                                         if(f)
1047                                                 FS_Close(f);
1048                                 }
1049                         }
1050                 }
1051
1052                 // if we get here, we actually want to download... so first verify the
1053                 // URL scheme (so one can't read local files using file://)
1054                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1055                 {
1056                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1057                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1058                         return false;
1059                 }
1060
1061                 if(forthismap)
1062                         ++numdownloads_added;
1063                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1064                 strlcpy(di->filename, name, sizeof(di->filename));
1065                 strlcpy(di->url, URL, sizeof(di->url));
1066                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1067                 di->forthismap = forthismap;
1068                 di->stream = NULL;
1069                 di->startpos = 0;
1070                 di->curle = NULL;
1071                 di->started = false;
1072                 di->loadtype = loadtype;
1073                 di->maxspeed = maxspeed;
1074                 di->bytes_received = 0;
1075                 di->bytes_received_curl = 0;
1076                 di->bytes_sent_curl = 0;
1077                 di->extraheaders = extraheaders;
1078                 di->buffer = buf;
1079                 di->buffersize = bufsize;
1080                 if(callback == NULL)
1081                 {
1082                         di->callback = curl_default_callback;
1083                         di->callback_data = di;
1084                 }
1085                 else
1086                 {
1087                         di->callback = callback;
1088                         di->callback_data = cbdata;
1089                 }
1090
1091                 if(post_content_type)
1092                 {
1093                         di->post_content_type = post_content_type;
1094                         di->postbuf = postbuf;
1095                         di->postbufsize = postbufsize;
1096                 }
1097                 else
1098                 {
1099                         di->post_content_type = NULL;
1100                         di->postbuf = NULL;
1101                         di->postbufsize = 0;
1102                 }
1103
1104                 List_Add(&di->list, &downloads);
1105
1106                 if (curl_mutex)
1107                         Thread_UnlockMutex(curl_mutex);
1108
1109                 return true;
1110         }
1111 }
1112
1113 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1114 {
1115         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1116 }
1117 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1118 {
1119         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1120 }
1121 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1122 {
1123         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1124 }
1125
1126 /*
1127 ====================
1128 Curl_Frame
1129
1130 call this regularily as this will always download as much as possible without
1131 blocking.
1132 ====================
1133 */
1134 void Curl_Frame(void)
1135 {
1136         double maxspeed;
1137         downloadinfo *di;
1138
1139         noclear = false;
1140
1141         if(!curl_enabled.integer && cls.state != ca_dedicated)
1142                 return;
1143
1144         if(!curl_dll)
1145                 return;
1146
1147         if (curl_mutex) Thread_LockMutex(curl_mutex);
1148
1149         Curl_CheckCommandWhenDone();
1150
1151         if(List_Is_Empty(&downloads))
1152         {
1153                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1154                 return;
1155         }
1156
1157         if(host.realtime < curltime) // throttle
1158         {
1159                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1160                 return;
1161         }
1162
1163         {
1164                 int remaining;
1165                 CURLMcode mc;
1166
1167                 do
1168                 {
1169                         mc = qcurl_multi_perform(curlm, &remaining);
1170                 }
1171                 while(mc == CURLM_CALL_MULTI_PERFORM);
1172
1173                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1174                 {
1175                         double b = 0;
1176                         if(di->curle)
1177                         {
1178                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1179                                 bytes_sent += (b - di->bytes_sent_curl);
1180                                 di->bytes_sent_curl = b;
1181                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1182                                 bytes_sent += (b - di->bytes_received_curl);
1183                                 di->bytes_received_curl = b;
1184                         }
1185                 }
1186
1187                 for(;;)
1188                 {
1189                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1190                         if(!msg)
1191                                 break;
1192                         if(msg->msg == CURLMSG_DONE)
1193                         {
1194                                 const char *ct = NULL;
1195                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1196                                 CURLcode result;
1197                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1198                                 result = msg->data.result;
1199                                 if(result)
1200                                 {
1201                                         failed = CURL_DOWNLOAD_FAILED;
1202                                 }
1203                                 else
1204                                 {
1205                                         long code;
1206                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1207                                         switch(code / 100)
1208                                         {
1209                                                 case 4: // e.g. 404?
1210                                                 case 5: // e.g. 500?
1211                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1212                                                         result = (CURLcode) code;
1213                                                         break;
1214                                         }
1215                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1216                                 }
1217
1218                                 Curl_EndDownload(di, failed, result, ct);
1219                         }
1220                 }
1221         }
1222
1223         CheckPendingDownloads();
1224
1225         // when will we curl the next time?
1226         // we will wait a bit to ensure our download rate is kept.
1227         // we now know that realtime >= curltime... so set up a new curltime
1228
1229         // use the slowest allowing download to derive the maxspeed... this CAN
1230         // be done better, but maybe later
1231         maxspeed = curl_maxspeed.value;
1232         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1233                 if(di->maxspeed > 0)
1234                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1235                                 maxspeed = di->maxspeed;
1236
1237         if(maxspeed > 0)
1238         {
1239                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1240                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1241                 bytes_sent = 0;
1242                 bytes_received = 0;
1243         }
1244         else
1245                 curltime = host.realtime;
1246
1247         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1248 }
1249
1250 /*
1251 ====================
1252 Curl_Select
1253
1254 Sleeps until there's some transfer progress or a timeout is reached,
1255 unfortunately the timeout is only in milliseconds.
1256 This allows good throughput even at very low FPS.
1257 ====================
1258 */
1259 void Curl_Select(double *microseconds)
1260 {
1261         if (List_Is_Empty(&downloads))
1262                 return;
1263         if (qcurl_multi_poll(curlm, NULL, 0, *microseconds / 1000, NULL) == CURLM_OK)
1264                 *microseconds = 0; // either we finished waiting or a transfer progressed
1265         else
1266                 Con_Print("There's an emergency going on!\nIt's still going on!\nMaybe you need to upgrade libcurl?\n");
1267 }
1268
1269 /*
1270 ====================
1271 Curl_CancelAll
1272
1273 Stops ALL downloads.
1274 ====================
1275 */
1276 void Curl_CancelAll(void)
1277 {
1278         if(!curl_dll)
1279                 return;
1280
1281         if (curl_mutex) Thread_LockMutex(curl_mutex);
1282
1283         while(!List_Is_Empty(&downloads))
1284         {
1285                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1286                 // INVARIANT: downloads will point to the next download after that!
1287         }
1288
1289         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1290 }
1291
1292 /*
1293 ====================
1294 Curl_Running
1295
1296 returns true if there is a download running.
1297 ====================
1298 */
1299 qbool Curl_Running(void)
1300 {
1301         if(!curl_dll)
1302                 return false;
1303
1304         return !List_Is_Empty(&downloads);
1305 }
1306
1307 /*
1308 ====================
1309 Curl_GetDownloadAmount
1310
1311 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1312 for the given download.
1313 ====================
1314 */
1315 static double Curl_GetDownloadAmount(downloadinfo *di)
1316 {
1317         if(!curl_dll)
1318                 return -2;
1319         if(di->curle)
1320         {
1321                 double length;
1322                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1323                 if(length > 0)
1324                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1325                 else
1326                         return 0;
1327         }
1328         else
1329                 return -1;
1330 }
1331
1332 /*
1333 ====================
1334 Curl_GetDownloadSpeed
1335
1336 returns the speed of the given download in bytes per second
1337 ====================
1338 */
1339 static double Curl_GetDownloadSpeed(downloadinfo *di)
1340 {
1341         if(!curl_dll)
1342                 return -2;
1343         if(di->curle)
1344         {
1345                 double speed;
1346                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1347                 return speed;
1348         }
1349         else
1350                 return -1;
1351 }
1352
1353 /*
1354 ====================
1355 Curl_Info_f
1356
1357 prints the download list
1358 ====================
1359 */
1360 // TODO rewrite using Curl_GetDownloadInfo?
1361 static void Curl_Info_f(cmd_state_t *cmd)
1362 {
1363         downloadinfo *di;
1364         char urlbuf[1024];
1365         if(!curl_dll)
1366                 return;
1367         if(Curl_Running())
1368         {
1369                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1370                 Con_Print("Currently running downloads:\n");
1371                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1372                 {
1373                         double speed, percent;
1374                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1375                         percent = 100.0 * Curl_GetDownloadAmount(di);
1376                         speed = Curl_GetDownloadSpeed(di);
1377                         if(percent >= 0)
1378                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1379                         else
1380                                 Con_Print("(queued)\n");
1381                 }
1382                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1383         }
1384         else
1385         {
1386                 Con_Print("No downloads running.\n");
1387         }
1388 }
1389
1390 /*
1391 ====================
1392 Curl_Curl_f
1393
1394 implements the "curl" console command
1395
1396 curl --info
1397 curl --cancel
1398 curl --cancel filename
1399 curl url
1400
1401 For internal use:
1402
1403 curl [--pak] [--forthismap] [--for filename filename...] url
1404         --pak: after downloading, load the package into the virtual file system
1405         --for filename...: only download of at least one of the named files is missing
1406         --forthismap: don't reconnect on failure
1407
1408 curl --clear_autodownload
1409         clears the download success/failure counters
1410
1411 curl --finish_autodownload
1412         if at least one download has been started, disconnect and drop to the menu
1413         once the last download completes successfully, reconnect to the current server
1414 ====================
1415 */
1416 static void Curl_Curl_f(cmd_state_t *cmd)
1417 {
1418         double maxspeed = 0;
1419         int i;
1420         int end;
1421         int loadtype = LOADTYPE_NONE;
1422         qbool forthismap = false;
1423         const char *url;
1424         const char *name = 0;
1425
1426         if(!curl_dll)
1427         {
1428                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1429                 return;
1430         }
1431
1432         if(!curl_enabled.integer)
1433         {
1434                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1435                 return;
1436         }
1437
1438         if(Cmd_Argc(cmd) < 2)
1439         {
1440                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1441                 return;
1442         }
1443
1444         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1445         end = Cmd_Argc(cmd);
1446
1447         for(i = 1; i != end; ++i)
1448         {
1449                 const char *a = Cmd_Argv(cmd, i);
1450                 if(!strcmp(a, "--info"))
1451                 {
1452                         Curl_Info_f(cmd);
1453                         return;
1454                 }
1455                 else if(!strcmp(a, "--cancel"))
1456                 {
1457                         if(i == end - 1) // last argument
1458                                 Curl_CancelAll();
1459                         else
1460                         {
1461                                 downloadinfo *di = Curl_Find(url);
1462                                 if(di)
1463                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1464                                 else
1465                                         Con_Print("download not found\n");
1466                         }
1467                         return;
1468                 }
1469                 else if(!strcmp(a, "--pak"))
1470                 {
1471                         loadtype = LOADTYPE_PAK;
1472                 }
1473                 else if(!strcmp(a, "--cachepic"))
1474                 {
1475                         loadtype = LOADTYPE_CACHEPIC;
1476                 }
1477                 else if(!strcmp(a, "--skinframe"))
1478                 {
1479                         loadtype = LOADTYPE_SKINFRAME;
1480                 }
1481                 else if(!strcmp(a, "--for")) // must be last option
1482                 {
1483                         for(i = i + 1; i != end - 1; ++i)
1484                         {
1485                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1486                                         goto needthefile; // why can't I have a "double break"?
1487                         }
1488                         // if we get here, we have all the files...
1489                         return;
1490                 }
1491                 else if(!strcmp(a, "--forthismap"))
1492                 {
1493                         forthismap = true;
1494                 }
1495                 else if(!strcmp(a, "--as"))
1496                 {
1497                         if(i < end - 1)
1498                         {
1499                                 ++i;
1500                                 name = Cmd_Argv(cmd, i);
1501                         }
1502                 }
1503                 else if(!strcmp(a, "--clear_autodownload"))
1504                 {
1505                         // mark all running downloads as "not for this map", so if they
1506                         // fail, it does not matter
1507                         Curl_Clear_forthismap();
1508                         return;
1509                 }
1510                 else if(!strcmp(a, "--finish_autodownload"))
1511                 {
1512                         if(numdownloads_added)
1513                         {
1514                                 char donecommand[256];
1515                                 if(cls.netcon)
1516                                 {
1517                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1518                                         {
1519                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1520                                                 Curl_CommandWhenDone(donecommand);
1521                                                 noclear = true;
1522                                                 CL_Disconnect();
1523                                                 noclear = false;
1524                                                 Curl_CheckCommandWhenDone();
1525                                         }
1526                                         else
1527                                                 Curl_Register_predownload();
1528                                 }
1529                         }
1530                         return;
1531                 }
1532                 else if(!strncmp(a, "--maxspeed=", 11))
1533                 {
1534                         maxspeed = atof(a + 11);
1535                 }
1536                 else if(*a == '-')
1537                 {
1538                         Con_Printf("curl: invalid option %s\n", a);
1539                         // but we ignore the option
1540                 }
1541         }
1542
1543 needthefile:
1544         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1545 }
1546
1547 /*
1548 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1549 {
1550         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1551         Z_Free(buffer);
1552 }
1553
1554 void Curl_CurlCat_f(cmd_state_t *cmd)
1555 {
1556         unsigned char *buf;
1557         const char *url = Cmd_Argv(cmd, 1);
1558         buf = Z_Malloc(16384);
1559         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1560 }
1561 */
1562
1563 /*
1564 ====================
1565 Curl_Init_Commands
1566
1567 loads the commands and cvars this library uses
1568 ====================
1569 */
1570 void Curl_Init_Commands(void)
1571 {
1572         Cvar_RegisterVariable (&curl_enabled);
1573         Cvar_RegisterVariable (&curl_maxdownloads);
1574         Cvar_RegisterVariable (&curl_maxspeed);
1575         Cvar_RegisterVariable (&curl_useragent);
1576         Cvar_RegisterVariable (&curl_useragent_append);
1577         Cvar_RegisterVirtual  (&curl_enabled,          "cl_curl_enabled");
1578         Cvar_RegisterVirtual  (&curl_maxdownloads,     "cl_curl_maxdownloads");
1579         Cvar_RegisterVirtual  (&curl_maxspeed,         "cl_curl_maxspeed");
1580         Cvar_RegisterVirtual  (&curl_useragent,        "cl_curl_useragent");
1581         Cvar_RegisterVirtual  (&curl_useragent_append, "cl_curl_useragent_append");
1582
1583         Cvar_RegisterVariable (&sv_curl_defaulturl);
1584         Cvar_RegisterVariable (&sv_curl_serverpackages);
1585         Cvar_RegisterVariable (&sv_curl_maxspeed);
1586
1587         Cvar_RegisterVariable (&developer_curl);
1588
1589         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1590         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1591 }
1592
1593 /*
1594 ====================
1595 Curl_GetDownloadInfo
1596
1597 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1598 The number of elements in the array is returned in int *nDownloads.
1599 const char **additional_info may be set to a string of additional user
1600 information, or to NULL if no such display shall occur. The returned
1601 array must be freed later using Z_Free.
1602 ====================
1603 */
1604 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1605 {
1606         int i;
1607         downloadinfo *di;
1608         Curl_downloadinfo_t *downinfo;
1609
1610         if(!curl_dll)
1611         {
1612                 *nDownloads = 0;
1613                 if(additional_info)
1614                         *additional_info = NULL;
1615                 return NULL;
1616         }
1617
1618         if (curl_mutex) Thread_LockMutex(curl_mutex);
1619
1620         i = 0;
1621         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1622                 ++i;
1623
1624         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1625         i = 0;
1626         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1627         {
1628                 // do not show infobars for background downloads
1629                 if(developer.integer <= 0)
1630                         if(di->buffer)
1631                                 continue;
1632                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1633                 if(di->curle)
1634                 {
1635                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1636                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1637                         downinfo[i].queued = false;
1638                 }
1639                 else
1640                 {
1641                         downinfo[i].queued = true;
1642                 }
1643                 ++i;
1644         }
1645
1646         if(additional_info)
1647         {
1648                 // TODO: can I clear command_when_done as soon as the first download fails?
1649                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1650                 {
1651                         if(!strncmp(command_when_done, "connect ", 8))
1652                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1653                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1654                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1655                         else
1656                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1657                         *additional_info = addinfo;
1658                 }
1659                 else
1660                         *additional_info = NULL;
1661         }
1662
1663         *nDownloads = i;
1664         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1665         return downinfo;
1666 }
1667
1668
1669 /*
1670 ====================
1671 Curl_FindPackURL
1672
1673 finds the URL where to find a given package.
1674
1675 For this, it reads a file "curl_urls.txt" of the following format:
1676
1677         data*.pk3       -
1678         revdm*.pk3      http://revdm/downloads/are/here/
1679         *                       http://any/other/stuff/is/here/
1680
1681 The URLs should end in /. If not, downloads will still work, but the cached files
1682 can't be just put into the data directory with the same download configuration
1683 (you might want to do this if you want to tag downloaded files from your
1684 server, but you should not). "-" means "don't download".
1685
1686 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1687 location instead.
1688
1689 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1690 this file for obvious reasons.
1691 ====================
1692 */
1693 static const char *Curl_FindPackURL(const char *filename)
1694 {
1695         static char foundurl[1024]; // invoked only by server
1696         fs_offset_t filesize;
1697         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1698         if(buf && filesize)
1699         {
1700                 // read lines of format "pattern url"
1701                 char *p = buf;
1702                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1703                 qbool eof = false;
1704
1705                 pattern = p;
1706                 while(!eof)
1707                 {
1708                         switch(*p)
1709                         {
1710                                 case 0:
1711                                         eof = true;
1712                                         // fallthrough
1713                                 case '\n':
1714                                 case '\r':
1715                                         if(pattern && url && patternend)
1716                                         {
1717                                                 if(!urlend)
1718                                                         urlend = p;
1719                                                 *patternend = 0;
1720                                                 *urlend = 0;
1721                                                 if(matchpattern(filename, pattern, true))
1722                                                 {
1723                                                         strlcpy(foundurl, url, sizeof(foundurl));
1724                                                         Z_Free(buf);
1725                                                         return foundurl;
1726                                                 }
1727                                         }
1728                                         pattern = NULL;
1729                                         patternend = NULL;
1730                                         url = NULL;
1731                                         urlend = NULL;
1732                                         break;
1733                                 case ' ':
1734                                 case '\t':
1735                                         if(pattern && !patternend)
1736                                                 patternend = p;
1737                                         else if(url && !urlend)
1738                                                 urlend = p;
1739                                         break;
1740                                 default:
1741                                         if(!pattern)
1742                                                 pattern = p;
1743                                         else if(pattern && patternend && !url)
1744                                                 url = p;
1745                                         break;
1746                         }
1747                         ++p;
1748                 }
1749         }
1750         if(buf)
1751                 Z_Free(buf);
1752         return sv_curl_defaulturl.string;
1753 }
1754
1755 typedef struct requirement_s
1756 {
1757         struct requirement_s *next;
1758         char filename[MAX_OSPATH];
1759 }
1760 requirement;
1761 static requirement *requirements = NULL;
1762
1763
1764 /*
1765 ====================
1766 Curl_RequireFile
1767
1768 Adds the given file to the list of requirements.
1769 ====================
1770 */
1771 void Curl_RequireFile(const char *filename)
1772 {
1773         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1774         req->next = requirements;
1775         strlcpy(req->filename, filename, sizeof(req->filename));
1776         requirements = req;
1777 }
1778
1779 /*
1780 ====================
1781 Curl_ClearRequirements
1782
1783 Clears the list of required files for playing on the current map.
1784 This should be called at every map change.
1785 ====================
1786 */
1787 void Curl_ClearRequirements(void)
1788 {
1789         while(requirements)
1790         {
1791                 requirement *req = requirements;
1792                 requirements = requirements->next;
1793                 Z_Free(req);
1794         }
1795 }
1796
1797 /*
1798 ====================
1799 Curl_SendRequirements
1800
1801 Makes the current host_clients download all files he needs.
1802 This is done by sending him the following console commands:
1803
1804         curl --clear_autodownload
1805         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1806         curl --finish_autodownload
1807 ====================
1808 */
1809 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1810 {
1811         const char *p;
1812         const char *thispack = FS_WhichPack(filename);
1813         const char *packurl;
1814
1815         if(!thispack || !*thispack)
1816                 return false;
1817
1818         p = strrchr(thispack, '/');
1819         if(p)
1820                 thispack = p + 1;
1821
1822         packurl = Curl_FindPackURL(thispack);
1823
1824         if(packurl && *packurl && strcmp(packurl, "-"))
1825         {
1826                 if(!foundone)
1827                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1828
1829                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1830                 strlcat(sendbuffer, thispack, sendbuffer_len);
1831                 if(sv_curl_maxspeed.value > 0)
1832                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1833                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1834                 strlcat(sendbuffer, filename, sendbuffer_len);
1835                 strlcat(sendbuffer, " ", sendbuffer_len);
1836                 strlcat(sendbuffer, packurl, sendbuffer_len);
1837                 strlcat(sendbuffer, thispack, sendbuffer_len);
1838                 strlcat(sendbuffer, "\n", sendbuffer_len);
1839
1840                 return true;
1841         }
1842
1843         return false;
1844 }
1845 void Curl_SendRequirements(void)
1846 {
1847         // for each requirement, find the pack name
1848         char sendbuffer[4096] = "";
1849         requirement *req;
1850         qbool foundone = false;
1851         const char *p;
1852
1853         for(req = requirements; req; req = req->next)
1854                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1855
1856         p = sv_curl_serverpackages.string;
1857         while(COM_ParseToken_Simple(&p, false, false, true))
1858                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1859
1860         if(foundone)
1861                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1862
1863         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1864                 SV_ClientCommands("%s", sendbuffer);
1865         else
1866                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1867 }