libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5 #include "com_list.h"
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CF_SERVER | CF_ARCHIVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CF_SERVER | CF_ARCHIVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CF_SERVER | CF_ARCHIVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CF_CLIENT | CF_ARCHIVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {CF_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {CF_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18
  19 /*
  20 =================================================================
  21
  22   Minimal set of definitions from libcurl
  23
  24   WARNING: for a matter of simplicity, several pointer types are
  25   casted to "void*", and most enumerated values are not included
  26
  27 =================================================================
  28 */
  29
  30 typedef struct CURL_s CURL;
  31 typedef struct CURLM_s CURLM;
  32 typedef struct curl_slist curl_slist;
  33 typedef enum
  34 {
  35         CURLE_OK = 0
  36 }
  37 CURLcode;
  38 typedef enum
  39 {
  40         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  41         CURLM_OK = 0
  42 }
  43 CURLMcode;
  44 #define CURL_GLOBAL_NOTHING 0
  45 #define CURL_GLOBAL_SSL 1
  46 #define CURL_GLOBAL_WIN32 2
  47 #define CURLOPTTYPE_LONG          0
  48 #define CURLOPTTYPE_OBJECTPOINT   10000
  49 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  50 #define CURLOPTTYPE_OFF_T         30000
  51 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  52 typedef enum
  53 {
  54         CINIT(WRITEDATA, OBJECTPOINT, 1),
  55         CINIT(URL,  OBJECTPOINT, 2),
  56         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  57         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  58         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  59         CINIT(REFERER, OBJECTPOINT, 16),
  60         CINIT(USERAGENT, OBJECTPOINT, 18),
  61         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  62         CINIT(LOW_SPEED_TIME, LONG, 20),
  63         CINIT(RESUME_FROM, LONG, 21),
  64         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  65         CINIT(POST, LONG, 47),         /* HTTP POST method */
  66         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  67         CINIT(POSTFIELDSIZE, LONG, 60),
  68         CINIT(PRIVATE, OBJECTPOINT, 103),
  69         CINIT(PROTOCOLS, LONG, 181),
  70         CINIT(REDIR_PROTOCOLS, LONG, 182)
  71 }
  72 CURLoption;
  73 #define CURLPROTO_HTTP   (1<<0)
  74 #define CURLPROTO_HTTPS  (1<<1)
  75 #define CURLPROTO_FTP    (1<<2)
  76 typedef enum
  77 {
  78         CURLINFO_TEXT = 0,
  79         CURLINFO_HEADER_IN,    /* 1 */
  80         CURLINFO_HEADER_OUT,   /* 2 */
  81         CURLINFO_DATA_IN,      /* 3 */
  82         CURLINFO_DATA_OUT,     /* 4 */
  83         CURLINFO_SSL_DATA_IN,  /* 5 */
  84         CURLINFO_SSL_DATA_OUT, /* 6 */
  85         CURLINFO_END
  86 }
  87 curl_infotype;
  88 #define CURLINFO_STRING   0x100000
  89 #define CURLINFO_LONG     0x200000
  90 #define CURLINFO_DOUBLE   0x300000
  91 #define CURLINFO_SLIST    0x400000
  92 #define CURLINFO_MASK     0x0fffff
  93 #define CURLINFO_TYPEMASK 0xf00000
  94 typedef enum
  95 {
  96         CURLINFO_NONE, /* first, never use this */
  97         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  98         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  99         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 100         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 101         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 102         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 103         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 104         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 105         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 106         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 107         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 108         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 109         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 110         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 111         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 112         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 113         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 114         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 115         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 116         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 117         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 118         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 119         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 120         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 121         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 122         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 123         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 124 }
 125 CURLINFO;
 126
 127 typedef enum
 128 {
 129         CURLMSG_NONE, /* first, not used */
 130         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 131                                          the CURLcode of the transfer */
 132         CURLMSG_LAST
 133 }
 134 CURLMSG;
 135 typedef struct
 136 {
 137         CURLMSG msg;       /* what this message means */
 138         CURL *easy_handle; /* the handle it concerns */
 139         union
 140         {
 141                 void *whatever;    /* message-specific data */
 142                 CURLcode result;   /* return code for transfer */
 143         }
 144         data;
 145 }
 146 CURLMsg;
 147
 148 static void (*qcurl_global_init) (long flags);
 149 static void (*qcurl_global_cleanup) (void);
 150
 151 static CURL * (*qcurl_easy_init) (void);
 152 static void (*qcurl_easy_cleanup) (CURL *handle);
 153 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 154 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 155 static const char * (*qcurl_easy_strerror) (CURLcode);
 156
 157 static CURLM * (*qcurl_multi_init) (void);
 158 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 159 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 160 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 161 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 162 static void (*qcurl_multi_cleanup) (CURLM *);
 163 static const char * (*qcurl_multi_strerror) (CURLcode);
 164 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 165 static void (*qcurl_slist_free_all) (curl_slist *list);
 166
 167 static dllfunction_t curlfuncs[] =
 168 {
 169         {"curl_global_init",            (void **) &qcurl_global_init},
 170         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 171         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 172         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 173         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 174         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 175         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 176         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 177         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 178         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 179         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 180         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 181         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 182         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 183         {"curl_slist_append",           (void **) &qcurl_slist_append},
 184         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 185         {NULL, NULL}
 186 };
 187
 188 // Handle for CURL DLL
 189 static dllhandle_t curl_dll = NULL;
 190 // will be checked at many places to find out if qcurl calls are allowed
 191
 192 #define LOADTYPE_NONE 0
 193 #define LOADTYPE_PAK 1
 194 #define LOADTYPE_CACHEPIC 2
 195 #define LOADTYPE_SKINFRAME 3
 196
 197 void *curl_mutex = NULL;
 198
 199 typedef struct downloadinfo_s
 200 {
 201         char filename[MAX_OSPATH];
 202         char url[1024];
 203         char referer[256];
 204         qfile_t *stream;
 205         fs_offset_t startpos;
 206         CURL *curle;
 207         qbool started;
 208         int loadtype;
 209         size_t bytes_received; // for buffer
 210         double bytes_received_curl; // for throttling
 211         double bytes_sent_curl; // for throttling
 212         llist_t list;
 213         qbool forthismap;
 214         double maxspeed;
 215         curl_slist *slist; // http headers
 216
 217         unsigned char *buffer;
 218         size_t buffersize;
 219         curl_callback_t callback;
 220         void *callback_data;
 221
 222         const unsigned char *postbuf;
 223         size_t postbufsize;
 224         const char *post_content_type;
 225         const char *extraheaders;
 226 }
 227 downloadinfo;
 228 LIST_HEAD(downloads);
 229 static int numdownloads = 0;
 230
 231 static qbool noclear = false;
 232
 233 static int numdownloads_fail = 0;
 234 static int numdownloads_success = 0;
 235 static int numdownloads_added = 0;
 236 static char command_when_done[256] = "";
 237 static char command_when_error[256] = "";
 238
 239 /*
 240 ====================
 241 Curl_CommandWhenDone
 242
 243 Sets the command which is to be executed when the last download completes AND
 244 all downloads since last server connect ended with a successful status.
 245 Setting the command to NULL clears it.
 246 ====================
 247 */
 248 static void Curl_CommandWhenDone(const char *cmd)
 249 {
 250         if(!curl_dll)
 251                 return;
 252         if(cmd)
 253                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 254         else
 255                 *command_when_done = 0;
 256 }
 257
 258 /*
 259 FIXME
 260 Do not use yet. Not complete.
 261 Problem: what counts as an error?
 262 */
 263
 264 static void Curl_CommandWhenError(const char *cmd)
 265 {
 266         if(!curl_dll)
 267                 return;
 268         if(cmd)
 269                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 270         else
 271                 *command_when_error = 0;
 272 }
 273
 274 /*
 275 ====================
 276 Curl_Clear_forthismap
 277
 278 Clears the "will disconnect on failure" flags.
 279 ====================
 280 */
 281 void Curl_Clear_forthismap(void)
 282 {
 283         downloadinfo *di;
 284         if(noclear)
 285                 return;
 286         if (curl_mutex) Thread_LockMutex(curl_mutex);
 287         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 288                 di->forthismap = false;
 289         Curl_CommandWhenError(NULL);
 290         Curl_CommandWhenDone(NULL);
 291         numdownloads_fail = 0;
 292         numdownloads_success = 0;
 293         numdownloads_added = 0;
 294         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 295 }
 296
 297 /*
 298 ====================
 299 Curl_Have_forthismap
 300
 301 Returns true if a download needed for the current game is running.
 302 ====================
 303 */
 304 qbool Curl_Have_forthismap(void)
 305 {
 306         return numdownloads_added != 0;
 307 }
 308
 309 void Curl_Register_predownload(void)
 310 {
 311         if (curl_mutex) Thread_LockMutex(curl_mutex);
 312         Curl_CommandWhenDone("cl_begindownloads");
 313         Curl_CommandWhenError("cl_begindownloads");
 314         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 315 }
 316
 317 /*
 318 ====================
 319 Curl_CheckCommandWhenDone
 320
 321 Checks if a "done command" is to be executed.
 322 All downloads finished, at least one success since connect, no single failure
 323 -> execute the command.
 324 */
 325 static void Curl_CheckCommandWhenDone(void)
 326 {
 327         if(!curl_dll)
 328                 return;
 329         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 330         {
 331                 if(numdownloads_fail == 0)
 332                 {
 333                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 334                         Cbuf_AddText(cmd_local, "\n");
 335                         Cbuf_AddText(cmd_local, command_when_done);
 336                         Cbuf_AddText(cmd_local, "\n");
 337                 }
 338                 else
 339                 {
 340                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 341                         Cbuf_AddText(cmd_local, "\n");
 342                         Cbuf_AddText(cmd_local, command_when_error);
 343                         Cbuf_AddText(cmd_local, "\n");
 344                 }
 345                 Curl_Clear_forthismap();
 346         }
 347 }
 348
 349 /*
 350 ====================
 351 CURL_CloseLibrary
 352
 353 Load the cURL DLL
 354 ====================
 355 */
 356 static qbool CURL_OpenLibrary (void)
 357 {
 358         const char* dllnames [] =
 359         {
 360 #if defined(WIN32)
 361                 "libcurl-4.dll",
 362                 "libcurl-3.dll",
 363 #elif defined(MACOSX)
 364                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 365                 "libcurl.3.dylib", // Mac OS X Tiger
 366                 "libcurl.2.dylib", // Mac OS X Panther
 367 #else
 368                 "libcurl.so.4",
 369                 "libcurl.so.3",
 370                 "libcurl.so", // FreeBSD
 371 #endif
 372                 NULL
 373         };
 374
 375         // Already loaded?
 376         if (curl_dll)
 377                 return true;
 378
 379         // Load the DLL
 380         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
 381 }
 382
 383
 384 /*
 385 ====================
 386 CURL_CloseLibrary
 387
 388 Unload the cURL DLL
 389 ====================
 390 */
 391 static void CURL_CloseLibrary (void)
 392 {
 393         Sys_FreeLibrary (&curl_dll);
 394 }
 395
 396
 397 static CURLM *curlm = NULL;
 398 static double bytes_received = 0; // used for bandwidth throttling
 399 static double bytes_sent = 0; // used for bandwidth throttling
 400 static double curltime = 0;
 401
 402 /*
 403 ====================
 404 CURL_fwrite
 405
 406 fwrite-compatible function that writes the data to a file. libcurl can call
 407 this.
 408 ====================
 409 */
 410 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 411 {
 412         fs_offset_t ret = -1;
 413         size_t bytes = size * nmemb;
 414         downloadinfo *di = (downloadinfo *) vdi;
 415
 416         if(di->buffer)
 417         {
 418                 if(di->bytes_received + bytes <= di->buffersize)
 419                 {
 420                         memcpy(di->buffer + di->bytes_received, data, bytes);
 421                         ret = bytes;
 422                 }
 423                 // otherwise: buffer overrun, ret stays -1
 424         }
 425
 426         if(di->stream)
 427         {
 428                 ret = FS_Write(di->stream, data, bytes);
 429         }
 430
 431         di->bytes_received += bytes;
 432
 433         return ret;
 434         // Why not ret / nmemb?
 435         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 436         // Yes, this is incompatible to fwrite(2).
 437 }
 438
 439 typedef enum
 440 {
 441         CURL_DOWNLOAD_SUCCESS = 0,
 442         CURL_DOWNLOAD_FAILED,
 443         CURL_DOWNLOAD_ABORTED,
 444         CURL_DOWNLOAD_SERVERERROR
 445 }
 446 CurlStatus;
 447
 448 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 449 {
 450         downloadinfo *di = (downloadinfo *) cbdata;
 451         switch(status)
 452         {
 453                 case CURLCBSTATUS_OK:
 454                         Con_DPrintf("Download of %s: OK\n", di->filename);
 455                         break;
 456                 case CURLCBSTATUS_FAILED:
 457                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 458                         break;
 459                 case CURLCBSTATUS_ABORTED:
 460                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 461                         break;
 462                 case CURLCBSTATUS_SERVERERROR:
 463                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 464                         break;
 465                 case CURLCBSTATUS_UNKNOWN:
 466                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 467                         break;
 468                 default:
 469                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 470                         break;
 471         }
 472 }
 473
 474 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 475 {
 476         curl_default_callback(status, length_received, buffer, cbdata);
 477 }
 478
 479 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 480 {
 481         unsigned char *pixels = NULL;
 482         fs_offset_t filesize = 0;
 483         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 484         if(data)
 485         {
 486                 int mip = 0;
 487                 if(!strcmp(content_type, "image/jpeg"))
 488                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 489                 else if(!strcmp(content_type, "image/png"))
 490                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 491                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 492                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 493                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 494                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 495                 else
 496                         Con_Printf("Did not detect content type: %s\n", content_type);
 497                 Mem_Free(data);
 498         }
 499         // do we call Image_MakeLinearColorsFromsRGB or not?
 500         return pixels;
 501 }
 502
 503 /*
 504 ====================
 505 Curl_EndDownload
 506
 507 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 508 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 509 code from libcurl, or 0, if another error has occurred.
 510 ====================
 511 */
 512 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 513 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 514 {
 515         char content_type[64];
 516         qbool ok = false;
 517         if(!curl_dll)
 518                 return;
 519         switch(status)
 520         {
 521                 case CURL_DOWNLOAD_SUCCESS:
 522                         ok = true;
 523                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 524                         break;
 525                 case CURL_DOWNLOAD_FAILED:
 526                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 527                         break;
 528                 case CURL_DOWNLOAD_ABORTED:
 529                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 530                         break;
 531                 case CURL_DOWNLOAD_SERVERERROR:
 532                         // reopen to enforce it to have zero bytes again
 533                         if(di->stream)
 534                         {
 535                                 FS_Close(di->stream);
 536                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 537                         }
 538
 539                         if(di->callback)
 540                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 541                         break;
 542                 default:
 543                         if(di->callback)
 544                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 545                         break;
 546         }
 547         if(content_type_)
 548                 strlcpy(content_type, content_type_, sizeof(content_type));
 549         else
 550                 *content_type = 0;
 551
 552         if(di->curle)
 553         {
 554                 qcurl_multi_remove_handle(curlm, di->curle);
 555                 qcurl_easy_cleanup(di->curle);
 556                 if(di->slist)
 557                         qcurl_slist_free_all(di->slist);
 558         }
 559
 560         if(!di->callback && ok && !di->bytes_received)
 561         {
 562                 Con_Printf("ERROR: empty file\n");
 563                 ok = false;
 564         }
 565
 566         if(di->stream)
 567                 FS_Close(di->stream);
 568
 569 #define CLEAR_AND_RETRY() \
 570         do \
 571         { \
 572                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 573                 FS_Close(di->stream); \
 574                 if(di->startpos && !di->callback) \
 575                 { \
 576                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 577                         di->forthismap = false; \
 578                 } \
 579         } \
 580         while(0)
 581
 582         if(ok && di->loadtype == LOADTYPE_PAK)
 583         {
 584                 ok = FS_AddPack(di->filename, NULL, true);
 585                 if(!ok)
 586                         CLEAR_AND_RETRY();
 587         }
 588         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 589         {
 590                 const char *p;
 591                 unsigned char *pixels = NULL;
 592
 593                 p = di->filename;
 594 #ifdef WE_ARE_EVIL
 595                 if(!strncmp(p, "dlcache/", 8))
 596                         p += 8;
 597 #endif
 598
 599                 pixels = decode_image(di, content_type);
 600                 if(pixels)
 601                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
 602                 else
 603                         CLEAR_AND_RETRY();
 604         }
 605         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 606         {
 607                 const char *p;
 608                 unsigned char *pixels = NULL;
 609
 610                 p = di->filename;
 611 #ifdef WE_ARE_EVIL
 612                 if(!strncmp(p, "dlcache/", 8))
 613                         p += 8;
 614 #endif
 615
 616                 pixels = decode_image(di, content_type);
 617                 if(pixels)
 618                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
 619                 else
 620                         CLEAR_AND_RETRY();
 621         }
 622
 623         List_Delete(&di->list);
 624
 625         --numdownloads;
 626         if(di->forthismap)
 627         {
 628                 if(ok)
 629                         ++numdownloads_success;
 630                 else
 631                         ++numdownloads_fail;
 632         }
 633         Z_Free(di);
 634 }
 635
 636 /*
 637 ====================
 638 CleanURL
 639
 640 Returns a "cleaned up" URL for display (to strip login data)
 641 ====================
 642 */
 643 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 644 {
 645         const char *p, *q, *r;
 646
 647         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 648         p = strstr(url, "://");
 649         if(p)
 650         {
 651                 q = strchr(p + 3, '@');
 652                 if(q)
 653                 {
 654                         r = strchr(p + 3, '/');
 655                         if(!r || q < r)
 656                         {
 657                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 658                                 return urlbuf;
 659                         }
 660                 }
 661         }
 662
 663         return url;
 664 }
 665
 666 /*
 667 ====================
 668 CheckPendingDownloads
 669
 670 checks if there are free download slots to start new downloads in.
 671 To not start too many downloads at once, only one download is added at a time,
 672 up to a maximum number of cl_curl_maxdownloads are running.
 673 ====================
 674 */
 675 static void CheckPendingDownloads(void)
 676 {
 677         const char *h;
 678         char urlbuf[1024];
 679         char vabuf[1024];
 680         if(!curl_dll)
 681                 return;
 682         if(numdownloads < cl_curl_maxdownloads.integer)
 683         {
 684                 downloadinfo *di;
 685                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
 686                 {
 687                         if(!di->started)
 688                         {
 689                                 if(!di->buffer)
 690                                 {
 691                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 692
 693                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 694                                         if(!di->stream)
 695                                         {
 696                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 697                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 698                                                 return;
 699                                         }
 700                                         FS_Seek(di->stream, 0, SEEK_END);
 701                                         di->startpos = FS_Tell(di->stream);
 702
 703                                         if(di->startpos > 0)
 704                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 705                                         Con_Print("...\n");
 706                                 }
 707                                 else
 708                                 {
 709                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 710                                         di->startpos = 0;
 711                                 }
 712
 713                                 di->curle = qcurl_easy_init();
 714                                 di->slist = NULL;
 715                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 716                                 if(cl_curl_useragent.integer)
 717                                 {
 718                                         const char *ua
 719 #ifdef HTTP_USER_AGENT
 720                                                 = HTTP_USER_AGENT;
 721 #else
 722                                                 = engineversion;
 723 #endif
 724                                         if(!ua)
 725                                                 ua = "";
 726                                         if(*cl_curl_useragent_append.string)
 727                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 728                                                         ua,
 729                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 730                                                                 ? " "
 731                                                                 : "",
 732                                                         cl_curl_useragent_append.string);
 733                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 734                                 }
 735                                 else
 736                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 737                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 738                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 739                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 740                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 741                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 742                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 743                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 744                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 746                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 747                                 {
 748                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 749                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 750                                 }
 751                                 if(di->post_content_type)
 752                                 {
 753                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 754                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 755                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 756                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 757                                 }
 758
 759                                 // parse extra headers into slist
 760                                 // \n separated list!
 761                                 h = di->extraheaders;
 762                                 while(h)
 763                                 {
 764                                         const char *hh = strchr(h, '\n');
 765                                         if(hh)
 766                                         {
 767                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 768                                                 memcpy(buf, h, hh - h);
 769                                                 buf[hh - h] = 0;
 770                                                 di->slist = qcurl_slist_append(di->slist, buf);
 771                                                 h = hh + 1;
 772                                         }
 773                                         else
 774                                         {
 775                                                 di->slist = qcurl_slist_append(di->slist, h);
 776                                                 h = NULL;
 777                                         }
 778                                 }
 779
 780                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 781
 782                                 qcurl_multi_add_handle(curlm, di->curle);
 783                                 di->started = true;
 784                                 ++numdownloads;
 785                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 786                                         break;
 787                         }
 788                 }
 789         }
 790 }
 791
 792 /*
 793 ====================
 794 Curl_Init
 795
 796 this function MUST be called before using anything else in this file.
 797 On Win32, this must be called AFTER WSAStartup has been done!
 798 ====================
 799 */
 800 void Curl_Init(void)
 801 {
 802         CURL_OpenLibrary();
 803         if(!curl_dll)
 804                 return;
 805         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 806         qcurl_global_init(CURL_GLOBAL_NOTHING);
 807         curlm = qcurl_multi_init();
 808 }
 809
 810 /*
 811 ====================
 812 Curl_Shutdown
 813
 814 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 815 ====================
 816 */
 817 void Curl_ClearRequirements(void);
 818 void Curl_Shutdown(void)
 819 {
 820         if(!curl_dll)
 821                 return;
 822         Curl_ClearRequirements();
 823         Curl_CancelAll();
 824         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 825         CURL_CloseLibrary();
 826         curl_dll = NULL;
 827 }
 828
 829 /*
 830 ====================
 831 Curl_Find
 832
 833 Finds the internal information block for a download given by file name.
 834 ====================
 835 */
 836 static downloadinfo *Curl_Find(const char *filename)
 837 {
 838         downloadinfo *di;
 839         if(!curl_dll)
 840                 return NULL;
 841         List_For_Each_Entry(di, &downloads, downloadinfo, list)
 842                 if(!strcasecmp(di->filename, filename))
 843                         return di;
 844         return NULL;
 845 }
 846
 847 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 848 {
 849         downloadinfo *di, *ndi;
 850         if(!curl_dll)
 851                 return;
 852         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
 853         {
 854                 if(di->callback == callback && di->callback_data == cbdata)
 855                 {
 856                         di->callback = curl_quiet_callback; // do NOT call the callback
 857                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 858                 }
 859         }
 860 }
 861
 862 /*
 863 ====================
 864 Curl_Begin
 865
 866 Starts a download of a given URL to the file name portion of this URL (or name
 867 if given) in the "dlcache/" folder.
 868 ====================
 869 */
 870 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 871 {
 872         if(buf)
 873                 if(loadtype != LOADTYPE_NONE)
 874                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 875
 876         if(!curl_dll || !cl_curl_enabled.integer)
 877         {
 878                 return false;
 879         }
 880         else
 881         {
 882                 char fn[MAX_OSPATH];
 883                 char urlbuf[1024];
 884                 const char *p, *q;
 885                 size_t length;
 886                 downloadinfo *di;
 887
 888                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 889                 p = strchr(URL, ':');
 890                 if(p)
 891                 {
 892                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 893                         {
 894                                 char addressstring[128];
 895                                 *addressstring = 0;
 896                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 897                                 q = strchr(addressstring, ':');
 898                                 if(!q)
 899                                         q = addressstring + strlen(addressstring);
 900                                 if(*addressstring)
 901                                 {
 902                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 903                                         URL = urlbuf;
 904                                 }
 905                         }
 906                 }
 907
 908                 // Note: This extraction of the file name portion is NOT entirely correct.
 909                 //
 910                 // It does the following:
 911                 //
 912                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 913                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 914                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 915                 //
 916                 // However, I'd like to keep this "buggy" behavior so that PHP script
 917                 // authors can write download scripts without having to enable
 918                 // AcceptPathInfo on Apache. They just have to ensure that their script
 919                 // can be called with such a "fake" path name like
 920                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 921                 //
 922                 // By the way, such PHP scripts should either send the file or a
 923                 // "Location:" redirect; PHP code example:
 924                 //
 925                 //   header("Location: http://www.example.com/");
 926                 //
 927                 // By the way, this will set User-Agent to something like
 928                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 929                 // dp://serverhost:serverport/ so you can filter on this; an example
 930                 // httpd log file line might be:
 931                 //
 932                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 933
 934                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 935
 936                 if(buf)
 937                 {
 938                         if(!name)
 939                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 940                 }
 941                 else
 942                 {
 943                         if(!name)
 944                         {
 945                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 946                                 p = strrchr(name, '/');
 947                                 p = p ? (p+1) : name;
 948                                 q = strchr(p, '?');
 949                                 length = q ? (size_t)(q - p) : strlen(p);
 950                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 951                         }
 952                         else
 953                         {
 954                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 955                         }
 956
 957                         name = fn; // make it point back
 958
 959                         // already downloading the file?
 960                         {
 961                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
 962                                 if(existingdownloadinfo)
 963                                 {
 964                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
 965
 966                                         // however, if it was not for this map yet...
 967                                         if(forthismap && !existingdownloadinfo->forthismap)
 968                                         {
 969                                                 existingdownloadinfo->forthismap = true;
 970                                                 // this "fakes" a download attempt so the client will wait for
 971                                                 // the download to finish and then reconnect
 972                                                 ++numdownloads_added;
 973                                         }
 974
 975                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 976                                         return false;
 977                                 }
 978                         }
 979
 980                         if(FS_FileExists(fn))
 981                         {
 982                                 if(loadtype == LOADTYPE_PAK)
 983                                 {
 984                                         qbool already_loaded;
 985                                         if(FS_AddPack(fn, &already_loaded, true))
 986                                         {
 987                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 988                                                 if(already_loaded)
 989                                                         Con_DPrintf("(pak was already loaded)\n");
 990                                                 else
 991                                                 {
 992                                                         if(forthismap)
 993                                                         {
 994                                                                 ++numdownloads_added;
 995                                                                 ++numdownloads_success;
 996                                                         }
 997                                                 }
 998
 999                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1000                                                 return false;
1001                                         }
1002                                         else
1003                                         {
1004                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1005                                                 if(f)
1006                                                 {
1007                                                         char b[4] = {0};
1008                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1009
1010                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1011                                                         {
1012                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1013                                                                 FS_Close(f);
1014                                                                 f = FS_OpenRealFile(fn, "wb", false);
1015                                                                 if(f)
1016                                                                         FS_Close(f);
1017                                                         }
1018                                                         else
1019                                                         {
1020                                                                 // OK
1021                                                                 FS_Close(f);
1022                                                         }
1023                                                 }
1024                                         }
1025                                 }
1026                                 else
1027                                 {
1028                                         // never resume these
1029                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1030                                         if(f)
1031                                                 FS_Close(f);
1032                                 }
1033                         }
1034                 }
1035
1036                 // if we get here, we actually want to download... so first verify the
1037                 // URL scheme (so one can't read local files using file://)
1038                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1039                 {
1040                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1041                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1042                         return false;
1043                 }
1044
1045                 if(forthismap)
1046                         ++numdownloads_added;
1047                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1048                 strlcpy(di->filename, name, sizeof(di->filename));
1049                 strlcpy(di->url, URL, sizeof(di->url));
1050                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1051                 di->forthismap = forthismap;
1052                 di->stream = NULL;
1053                 di->startpos = 0;
1054                 di->curle = NULL;
1055                 di->started = false;
1056                 di->loadtype = loadtype;
1057                 di->maxspeed = maxspeed;
1058                 di->bytes_received = 0;
1059                 di->bytes_received_curl = 0;
1060                 di->bytes_sent_curl = 0;
1061                 di->extraheaders = extraheaders;
1062                 di->buffer = buf;
1063                 di->buffersize = bufsize;
1064                 if(callback == NULL)
1065                 {
1066                         di->callback = curl_default_callback;
1067                         di->callback_data = di;
1068                 }
1069                 else
1070                 {
1071                         di->callback = callback;
1072                         di->callback_data = cbdata;
1073                 }
1074
1075                 if(post_content_type)
1076                 {
1077                         di->post_content_type = post_content_type;
1078                         di->postbuf = postbuf;
1079                         di->postbufsize = postbufsize;
1080                 }
1081                 else
1082                 {
1083                         di->post_content_type = NULL;
1084                         di->postbuf = NULL;
1085                         di->postbufsize = 0;
1086                 }
1087
1088                 List_Add(&di->list, &downloads);
1089
1090                 if (curl_mutex)
1091                         Thread_UnlockMutex(curl_mutex);
1092
1093                 return true;
1094         }
1095 }
1096
1097 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1098 {
1099         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1100 }
1101 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1102 {
1103         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1104 }
1105 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1106 {
1107         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1108 }
1109
1110 /*
1111 ====================
1112 Curl_Frame
1113
1114 call this regularily as this will always download as much as possible without
1115 blocking.
1116 ====================
1117 */
1118 void Curl_Frame(void)
1119 {
1120         double maxspeed;
1121         downloadinfo *di;
1122
1123         noclear = false;
1124
1125         if(!cl_curl_enabled.integer)
1126                 return;
1127
1128         if(!curl_dll)
1129                 return;
1130
1131         if (curl_mutex) Thread_LockMutex(curl_mutex);
1132
1133         Curl_CheckCommandWhenDone();
1134
1135         if(List_Is_Empty(&downloads))
1136         {
1137                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1138                 return;
1139         }
1140
1141         if(host.realtime < curltime) // throttle
1142         {
1143                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1144                 return;
1145         }
1146
1147         {
1148                 int remaining;
1149                 CURLMcode mc;
1150
1151                 do
1152                 {
1153                         mc = qcurl_multi_perform(curlm, &remaining);
1154                 }
1155                 while(mc == CURLM_CALL_MULTI_PERFORM);
1156
1157                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1158                 {
1159                         double b = 0;
1160                         if(di->curle)
1161                         {
1162                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1163                                 bytes_sent += (b - di->bytes_sent_curl);
1164                                 di->bytes_sent_curl = b;
1165                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1166                                 bytes_sent += (b - di->bytes_received_curl);
1167                                 di->bytes_received_curl = b;
1168                         }
1169                 }
1170
1171                 for(;;)
1172                 {
1173                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1174                         if(!msg)
1175                                 break;
1176                         if(msg->msg == CURLMSG_DONE)
1177                         {
1178                                 const char *ct = NULL;
1179                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1180                                 CURLcode result;
1181                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1182                                 result = msg->data.result;
1183                                 if(result)
1184                                 {
1185                                         failed = CURL_DOWNLOAD_FAILED;
1186                                 }
1187                                 else
1188                                 {
1189                                         long code;
1190                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1191                                         switch(code / 100)
1192                                         {
1193                                                 case 4: // e.g. 404?
1194                                                 case 5: // e.g. 500?
1195                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1196                                                         result = (CURLcode) code;
1197                                                         break;
1198                                         }
1199                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1200                                 }
1201
1202                                 Curl_EndDownload(di, failed, result, ct);
1203                         }
1204                 }
1205         }
1206
1207         CheckPendingDownloads();
1208
1209         // when will we curl the next time?
1210         // we will wait a bit to ensure our download rate is kept.
1211         // we now know that realtime >= curltime... so set up a new curltime
1212
1213         // use the slowest allowing download to derive the maxspeed... this CAN
1214         // be done better, but maybe later
1215         maxspeed = cl_curl_maxspeed.value;
1216         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1217                 if(di->maxspeed > 0)
1218                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1219                                 maxspeed = di->maxspeed;
1220
1221         if(maxspeed > 0)
1222         {
1223                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1224                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1225                 bytes_sent = 0;
1226                 bytes_received = 0;
1227         }
1228         else
1229                 curltime = host.realtime;
1230
1231         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1232 }
1233
1234 /*
1235 ====================
1236 Curl_CancelAll
1237
1238 Stops ALL downloads.
1239 ====================
1240 */
1241 void Curl_CancelAll(void)
1242 {
1243         if(!curl_dll)
1244                 return;
1245
1246         if (curl_mutex) Thread_LockMutex(curl_mutex);
1247
1248         while(!List_Is_Empty(&downloads))
1249         {
1250                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1251                 // INVARIANT: downloads will point to the next download after that!
1252         }
1253
1254         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1255 }
1256
1257 /*
1258 ====================
1259 Curl_Running
1260
1261 returns true if there is a download running.
1262 ====================
1263 */
1264 qbool Curl_Running(void)
1265 {
1266         if(!curl_dll)
1267                 return false;
1268
1269         return !List_Is_Empty(&downloads);
1270 }
1271
1272 /*
1273 ====================
1274 Curl_GetDownloadAmount
1275
1276 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1277 for the given download.
1278 ====================
1279 */
1280 static double Curl_GetDownloadAmount(downloadinfo *di)
1281 {
1282         if(!curl_dll)
1283                 return -2;
1284         if(di->curle)
1285         {
1286                 double length;
1287                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1288                 if(length > 0)
1289                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1290                 else
1291                         return 0;
1292         }
1293         else
1294                 return -1;
1295 }
1296
1297 /*
1298 ====================
1299 Curl_GetDownloadSpeed
1300
1301 returns the speed of the given download in bytes per second
1302 ====================
1303 */
1304 static double Curl_GetDownloadSpeed(downloadinfo *di)
1305 {
1306         if(!curl_dll)
1307                 return -2;
1308         if(di->curle)
1309         {
1310                 double speed;
1311                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1312                 return speed;
1313         }
1314         else
1315                 return -1;
1316 }
1317
1318 /*
1319 ====================
1320 Curl_Info_f
1321
1322 prints the download list
1323 ====================
1324 */
1325 // TODO rewrite using Curl_GetDownloadInfo?
1326 static void Curl_Info_f(cmd_state_t *cmd)
1327 {
1328         downloadinfo *di;
1329         char urlbuf[1024];
1330         if(!curl_dll)
1331                 return;
1332         if(Curl_Running())
1333         {
1334                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1335                 Con_Print("Currently running downloads:\n");
1336                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1337                 {
1338                         double speed, percent;
1339                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1340                         percent = 100.0 * Curl_GetDownloadAmount(di);
1341                         speed = Curl_GetDownloadSpeed(di);
1342                         if(percent >= 0)
1343                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1344                         else
1345                                 Con_Print("(queued)\n");
1346                 }
1347                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1348         }
1349         else
1350         {
1351                 Con_Print("No downloads running.\n");
1352         }
1353 }
1354
1355 /*
1356 ====================
1357 Curl_Curl_f
1358
1359 implements the "curl" console command
1360
1361 curl --info
1362 curl --cancel
1363 curl --cancel filename
1364 curl url
1365
1366 For internal use:
1367
1368 curl [--pak] [--forthismap] [--for filename filename...] url
1369         --pak: after downloading, load the package into the virtual file system
1370         --for filename...: only download of at least one of the named files is missing
1371         --forthismap: don't reconnect on failure
1372
1373 curl --clear_autodownload
1374         clears the download success/failure counters
1375
1376 curl --finish_autodownload
1377         if at least one download has been started, disconnect and drop to the menu
1378         once the last download completes successfully, reconnect to the current server
1379 ====================
1380 */
1381 static void Curl_Curl_f(cmd_state_t *cmd)
1382 {
1383         double maxspeed = 0;
1384         int i;
1385         int end;
1386         int loadtype = LOADTYPE_NONE;
1387         qbool forthismap = false;
1388         const char *url;
1389         const char *name = 0;
1390
1391         if(!curl_dll)
1392         {
1393                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1394                 return;
1395         }
1396
1397         if(!cl_curl_enabled.integer)
1398         {
1399                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1400                 return;
1401         }
1402
1403         if(Cmd_Argc(cmd) < 2)
1404         {
1405                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1406                 return;
1407         }
1408
1409         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1410         end = Cmd_Argc(cmd);
1411
1412         for(i = 1; i != end; ++i)
1413         {
1414                 const char *a = Cmd_Argv(cmd, i);
1415                 if(!strcmp(a, "--info"))
1416                 {
1417                         Curl_Info_f(cmd);
1418                         return;
1419                 }
1420                 else if(!strcmp(a, "--cancel"))
1421                 {
1422                         if(i == end - 1) // last argument
1423                                 Curl_CancelAll();
1424                         else
1425                         {
1426                                 downloadinfo *di = Curl_Find(url);
1427                                 if(di)
1428                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1429                                 else
1430                                         Con_Print("download not found\n");
1431                         }
1432                         return;
1433                 }
1434                 else if(!strcmp(a, "--pak"))
1435                 {
1436                         loadtype = LOADTYPE_PAK;
1437                 }
1438                 else if(!strcmp(a, "--cachepic"))
1439                 {
1440                         loadtype = LOADTYPE_CACHEPIC;
1441                 }
1442                 else if(!strcmp(a, "--skinframe"))
1443                 {
1444                         loadtype = LOADTYPE_SKINFRAME;
1445                 }
1446                 else if(!strcmp(a, "--for")) // must be last option
1447                 {
1448                         for(i = i + 1; i != end - 1; ++i)
1449                         {
1450                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1451                                         goto needthefile; // why can't I have a "double break"?
1452                         }
1453                         // if we get here, we have all the files...
1454                         return;
1455                 }
1456                 else if(!strcmp(a, "--forthismap"))
1457                 {
1458                         forthismap = true;
1459                 }
1460                 else if(!strcmp(a, "--as"))
1461                 {
1462                         if(i < end - 1)
1463                         {
1464                                 ++i;
1465                                 name = Cmd_Argv(cmd, i);
1466                         }
1467                 }
1468                 else if(!strcmp(a, "--clear_autodownload"))
1469                 {
1470                         // mark all running downloads as "not for this map", so if they
1471                         // fail, it does not matter
1472                         Curl_Clear_forthismap();
1473                         return;
1474                 }
1475                 else if(!strcmp(a, "--finish_autodownload"))
1476                 {
1477                         if(numdownloads_added)
1478                         {
1479                                 char donecommand[256];
1480                                 if(cls.netcon)
1481                                 {
1482                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1483                                         {
1484                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1485                                                 Curl_CommandWhenDone(donecommand);
1486                                                 noclear = true;
1487                                                 CL_Disconnect();
1488                                                 noclear = false;
1489                                                 Curl_CheckCommandWhenDone();
1490                                         }
1491                                         else
1492                                                 Curl_Register_predownload();
1493                                 }
1494                         }
1495                         return;
1496                 }
1497                 else if(!strncmp(a, "--maxspeed=", 11))
1498                 {
1499                         maxspeed = atof(a + 11);
1500                 }
1501                 else if(*a == '-')
1502                 {
1503                         Con_Printf("curl: invalid option %s\n", a);
1504                         // but we ignore the option
1505                 }
1506         }
1507
1508 needthefile:
1509         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1510 }
1511
1512 /*
1513 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1514 {
1515         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1516         Z_Free(buffer);
1517 }
1518
1519 void Curl_CurlCat_f(cmd_state_t *cmd)
1520 {
1521         unsigned char *buf;
1522         const char *url = Cmd_Argv(cmd, 1);
1523         buf = Z_Malloc(16384);
1524         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1525 }
1526 */
1527
1528 /*
1529 ====================
1530 Curl_Init_Commands
1531
1532 loads the commands and cvars this library uses
1533 ====================
1534 */
1535 void Curl_Init_Commands(void)
1536 {
1537         Cvar_RegisterVariable (&cl_curl_enabled);
1538         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1539         Cvar_RegisterVariable (&cl_curl_maxspeed);
1540         Cvar_RegisterVariable (&sv_curl_defaulturl);
1541         Cvar_RegisterVariable (&sv_curl_serverpackages);
1542         Cvar_RegisterVariable (&sv_curl_maxspeed);
1543         Cvar_RegisterVariable (&cl_curl_useragent);
1544         Cvar_RegisterVariable (&cl_curl_useragent_append);
1545         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1546         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1547 }
1548
1549 /*
1550 ====================
1551 Curl_GetDownloadInfo
1552
1553 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1554 The number of elements in the array is returned in int *nDownloads.
1555 const char **additional_info may be set to a string of additional user
1556 information, or to NULL if no such display shall occur. The returned
1557 array must be freed later using Z_Free.
1558 ====================
1559 */
1560 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1561 {
1562         int i;
1563         downloadinfo *di;
1564         Curl_downloadinfo_t *downinfo;
1565
1566         if(!curl_dll)
1567         {
1568                 *nDownloads = 0;
1569                 if(additional_info)
1570                         *additional_info = NULL;
1571                 return NULL;
1572         }
1573
1574         if (curl_mutex) Thread_LockMutex(curl_mutex);
1575
1576         i = 0;
1577         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1578                 ++i;
1579
1580         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1581         i = 0;
1582         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1583         {
1584                 // do not show infobars for background downloads
1585                 if(developer.integer <= 0)
1586                         if(di->buffer)
1587                                 continue;
1588                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1589                 if(di->curle)
1590                 {
1591                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1592                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1593                         downinfo[i].queued = false;
1594                 }
1595                 else
1596                 {
1597                         downinfo[i].queued = true;
1598                 }
1599                 ++i;
1600         }
1601
1602         if(additional_info)
1603         {
1604                 // TODO: can I clear command_when_done as soon as the first download fails?
1605                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1606                 {
1607                         if(!strncmp(command_when_done, "connect ", 8))
1608                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1609                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1610                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1611                         else
1612                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1613                         *additional_info = addinfo;
1614                 }
1615                 else
1616                         *additional_info = NULL;
1617         }
1618
1619         *nDownloads = i;
1620         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1621         return downinfo;
1622 }
1623
1624
1625 /*
1626 ====================
1627 Curl_FindPackURL
1628
1629 finds the URL where to find a given package.
1630
1631 For this, it reads a file "curl_urls.txt" of the following format:
1632
1633         data*.pk3       -
1634         revdm*.pk3      http://revdm/downloads/are/here/
1635         *                       http://any/other/stuff/is/here/
1636
1637 The URLs should end in /. If not, downloads will still work, but the cached files
1638 can't be just put into the data directory with the same download configuration
1639 (you might want to do this if you want to tag downloaded files from your
1640 server, but you should not). "-" means "don't download".
1641
1642 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1643 location instead.
1644
1645 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1646 this file for obvious reasons.
1647 ====================
1648 */
1649 static const char *Curl_FindPackURL(const char *filename)
1650 {
1651         static char foundurl[1024]; // invoked only by server
1652         fs_offset_t filesize;
1653         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1654         if(buf && filesize)
1655         {
1656                 // read lines of format "pattern url"
1657                 char *p = buf;
1658                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1659                 qbool eof = false;
1660
1661                 pattern = p;
1662                 while(!eof)
1663                 {
1664                         switch(*p)
1665                         {
1666                                 case 0:
1667                                         eof = true;
1668                                         // fallthrough
1669                                 case '\n':
1670                                 case '\r':
1671                                         if(pattern && url && patternend)
1672                                         {
1673                                                 if(!urlend)
1674                                                         urlend = p;
1675                                                 *patternend = 0;
1676                                                 *urlend = 0;
1677                                                 if(matchpattern(filename, pattern, true))
1678                                                 {
1679                                                         strlcpy(foundurl, url, sizeof(foundurl));
1680                                                         Z_Free(buf);
1681                                                         return foundurl;
1682                                                 }
1683                                         }
1684                                         pattern = NULL;
1685                                         patternend = NULL;
1686                                         url = NULL;
1687                                         urlend = NULL;
1688                                         break;
1689                                 case ' ':
1690                                 case '\t':
1691                                         if(pattern && !patternend)
1692                                                 patternend = p;
1693                                         else if(url && !urlend)
1694                                                 urlend = p;
1695                                         break;
1696                                 default:
1697                                         if(!pattern)
1698                                                 pattern = p;
1699                                         else if(pattern && patternend && !url)
1700                                                 url = p;
1701                                         break;
1702                         }
1703                         ++p;
1704                 }
1705         }
1706         if(buf)
1707                 Z_Free(buf);
1708         return sv_curl_defaulturl.string;
1709 }
1710
1711 typedef struct requirement_s
1712 {
1713         struct requirement_s *next;
1714         char filename[MAX_OSPATH];
1715 }
1716 requirement;
1717 static requirement *requirements = NULL;
1718
1719
1720 /*
1721 ====================
1722 Curl_RequireFile
1723
1724 Adds the given file to the list of requirements.
1725 ====================
1726 */
1727 void Curl_RequireFile(const char *filename)
1728 {
1729         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1730         req->next = requirements;
1731         strlcpy(req->filename, filename, sizeof(req->filename));
1732         requirements = req;
1733 }
1734
1735 /*
1736 ====================
1737 Curl_ClearRequirements
1738
1739 Clears the list of required files for playing on the current map.
1740 This should be called at every map change.
1741 ====================
1742 */
1743 void Curl_ClearRequirements(void)
1744 {
1745         while(requirements)
1746         {
1747                 requirement *req = requirements;
1748                 requirements = requirements->next;
1749                 Z_Free(req);
1750         }
1751 }
1752
1753 /*
1754 ====================
1755 Curl_SendRequirements
1756
1757 Makes the current host_clients download all files he needs.
1758 This is done by sending him the following console commands:
1759
1760         curl --clear_autodownload
1761         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1762         curl --finish_autodownload
1763 ====================
1764 */
1765 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1766 {
1767         const char *p;
1768         const char *thispack = FS_WhichPack(filename);
1769         const char *packurl;
1770
1771         if(!thispack || !*thispack)
1772                 return false;
1773
1774         p = strrchr(thispack, '/');
1775         if(p)
1776                 thispack = p + 1;
1777
1778         packurl = Curl_FindPackURL(thispack);
1779
1780         if(packurl && *packurl && strcmp(packurl, "-"))
1781         {
1782                 if(!foundone)
1783                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1784
1785                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1786                 strlcat(sendbuffer, thispack, sendbuffer_len);
1787                 if(sv_curl_maxspeed.value > 0)
1788                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1789                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1790                 strlcat(sendbuffer, filename, sendbuffer_len);
1791                 strlcat(sendbuffer, " ", sendbuffer_len);
1792                 strlcat(sendbuffer, packurl, sendbuffer_len);
1793                 strlcat(sendbuffer, thispack, sendbuffer_len);
1794                 strlcat(sendbuffer, "\n", sendbuffer_len);
1795
1796                 return true;
1797         }
1798
1799         return false;
1800 }
1801 void Curl_SendRequirements(void)
1802 {
1803         // for each requirement, find the pack name
1804         char sendbuffer[4096] = "";
1805         requirement *req;
1806         qbool foundone = false;
1807         const char *p;
1808
1809         for(req = requirements; req; req = req->next)
1810                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1811
1812         p = sv_curl_serverpackages.string;
1813         while(COM_ParseToken_Simple(&p, false, false, true))
1814                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1815
1816         if(foundone)
1817                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1818
1819         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1820                 SV_ClientCommands("%s", sendbuffer);
1821         else
1822                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1823 }