libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5
   6 #include "image.h"
   7 #include "jpeg.h"
   8 #include "image_png.h"
   9
  10 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
  11 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
  12 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
  13 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  14 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  15 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  16 static cvar_t cl_curl_useragent = {0, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
  17 static cvar_t cl_curl_useragent_append = {0, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
  18
  19 /*
  20 =================================================================
  21
  22   Minimal set of definitions from libcurl
  23
  24   WARNING: for a matter of simplicity, several pointer types are
  25   casted to "void*", and most enumerated values are not included
  26
  27 =================================================================
  28 */
  29
  30 typedef struct CURL_s CURL;
  31 typedef struct CURLM_s CURLM;
  32 typedef struct curl_slist curl_slist;
  33 typedef enum
  34 {
  35         CURLE_OK = 0
  36 }
  37 CURLcode;
  38 typedef enum
  39 {
  40         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  41         CURLM_OK = 0
  42 }
  43 CURLMcode;
  44 #define CURL_GLOBAL_NOTHING 0
  45 #define CURL_GLOBAL_SSL 1
  46 #define CURL_GLOBAL_WIN32 2
  47 #define CURLOPTTYPE_LONG          0
  48 #define CURLOPTTYPE_OBJECTPOINT   10000
  49 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  50 #define CURLOPTTYPE_OFF_T         30000
  51 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  52 typedef enum
  53 {
  54         CINIT(WRITEDATA, OBJECTPOINT, 1),
  55         CINIT(URL,  OBJECTPOINT, 2),
  56         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  57         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  58         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  59         CINIT(REFERER, OBJECTPOINT, 16),
  60         CINIT(USERAGENT, OBJECTPOINT, 18),
  61         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  62         CINIT(LOW_SPEED_TIME, LONG, 20),
  63         CINIT(RESUME_FROM, LONG, 21),
  64         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  65         CINIT(POST, LONG, 47),         /* HTTP POST method */
  66         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  67         CINIT(POSTFIELDSIZE, LONG, 60),
  68         CINIT(PRIVATE, OBJECTPOINT, 103),
  69         CINIT(PROTOCOLS, LONG, 181),
  70         CINIT(REDIR_PROTOCOLS, LONG, 182)
  71 }
  72 CURLoption;
  73 #define CURLPROTO_HTTP   (1<<0)
  74 #define CURLPROTO_HTTPS  (1<<1)
  75 #define CURLPROTO_FTP    (1<<2)
  76 typedef enum
  77 {
  78         CURLINFO_TEXT = 0,
  79         CURLINFO_HEADER_IN,    /* 1 */
  80         CURLINFO_HEADER_OUT,   /* 2 */
  81         CURLINFO_DATA_IN,      /* 3 */
  82         CURLINFO_DATA_OUT,     /* 4 */
  83         CURLINFO_SSL_DATA_IN,  /* 5 */
  84         CURLINFO_SSL_DATA_OUT, /* 6 */
  85         CURLINFO_END
  86 }
  87 curl_infotype;
  88 #define CURLINFO_STRING   0x100000
  89 #define CURLINFO_LONG     0x200000
  90 #define CURLINFO_DOUBLE   0x300000
  91 #define CURLINFO_SLIST    0x400000
  92 #define CURLINFO_MASK     0x0fffff
  93 #define CURLINFO_TYPEMASK 0xf00000
  94 typedef enum
  95 {
  96         CURLINFO_NONE, /* first, never use this */
  97         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  98         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  99         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
 100         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
 101         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
 102         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
 103         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
 104         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
 105         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 106         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 107         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 108         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 109         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 110         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 111         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 112         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 113         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 114         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 115         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 116         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 117         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 118         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 119         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 120         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 121         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 122         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 123         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 124 }
 125 CURLINFO;
 126
 127 typedef enum
 128 {
 129         CURLMSG_NONE, /* first, not used */
 130         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 131                                          the CURLcode of the transfer */
 132         CURLMSG_LAST
 133 }
 134 CURLMSG;
 135 typedef struct
 136 {
 137         CURLMSG msg;       /* what this message means */
 138         CURL *easy_handle; /* the handle it concerns */
 139         union
 140         {
 141                 void *whatever;    /* message-specific data */
 142                 CURLcode result;   /* return code for transfer */
 143         }
 144         data;
 145 }
 146 CURLMsg;
 147
 148 static void (*qcurl_global_init) (long flags);
 149 static void (*qcurl_global_cleanup) (void);
 150
 151 static CURL * (*qcurl_easy_init) (void);
 152 static void (*qcurl_easy_cleanup) (CURL *handle);
 153 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 154 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 155 static const char * (*qcurl_easy_strerror) (CURLcode);
 156
 157 static CURLM * (*qcurl_multi_init) (void);
 158 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 159 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 160 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 161 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 162 static void (*qcurl_multi_cleanup) (CURLM *);
 163 static const char * (*qcurl_multi_strerror) (CURLcode);
 164 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 165 static void (*qcurl_slist_free_all) (curl_slist *list);
 166
 167 static dllfunction_t curlfuncs[] =
 168 {
 169         {"curl_global_init",            (void **) &qcurl_global_init},
 170         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 171         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 172         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 173         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 174         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 175         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 176         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 177         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 178         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 179         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 180         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 181         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 182         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 183         {"curl_slist_append",           (void **) &qcurl_slist_append},
 184         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 185         {NULL, NULL}
 186 };
 187
 188 // Handle for CURL DLL
 189 static dllhandle_t curl_dll = NULL;
 190 // will be checked at many places to find out if qcurl calls are allowed
 191
 192 #define LOADTYPE_NONE 0
 193 #define LOADTYPE_PAK 1
 194 #define LOADTYPE_CACHEPIC 2
 195 #define LOADTYPE_SKINFRAME 3
 196
 197 void *curl_mutex = NULL;
 198
 199 typedef struct downloadinfo_s
 200 {
 201         char filename[MAX_OSPATH];
 202         char url[1024];
 203         char referer[256];
 204         qfile_t *stream;
 205         fs_offset_t startpos;
 206         CURL *curle;
 207         qboolean started;
 208         int loadtype;
 209         size_t bytes_received; // for buffer
 210         double bytes_received_curl; // for throttling
 211         double bytes_sent_curl; // for throttling
 212         struct downloadinfo_s *next, *prev;
 213         qboolean forthismap;
 214         double maxspeed;
 215         curl_slist *slist; // http headers
 216
 217         unsigned char *buffer;
 218         size_t buffersize;
 219         curl_callback_t callback;
 220         void *callback_data;
 221
 222         const unsigned char *postbuf;
 223         size_t postbufsize;
 224         const char *post_content_type;
 225         const char *extraheaders;
 226 }
 227 downloadinfo;
 228 static downloadinfo *downloads = NULL;
 229 static int numdownloads = 0;
 230
 231 static qboolean noclear = FALSE;
 232
 233 static int numdownloads_fail = 0;
 234 static int numdownloads_success = 0;
 235 static int numdownloads_added = 0;
 236 static char command_when_done[256] = "";
 237 static char command_when_error[256] = "";
 238
 239 /*
 240 ====================
 241 Curl_CommandWhenDone
 242
 243 Sets the command which is to be executed when the last download completes AND
 244 all downloads since last server connect ended with a successful status.
 245 Setting the command to NULL clears it.
 246 ====================
 247 */
 248 static void Curl_CommandWhenDone(const char *cmd)
 249 {
 250         if(!curl_dll)
 251                 return;
 252         if(cmd)
 253                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 254         else
 255                 *command_when_done = 0;
 256 }
 257
 258 /*
 259 FIXME
 260 Do not use yet. Not complete.
 261 Problem: what counts as an error?
 262 */
 263
 264 static void Curl_CommandWhenError(const char *cmd)
 265 {
 266         if(!curl_dll)
 267                 return;
 268         if(cmd)
 269                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 270         else
 271                 *command_when_error = 0;
 272 }
 273
 274 /*
 275 ====================
 276 Curl_Clear_forthismap
 277
 278 Clears the "will disconnect on failure" flags.
 279 ====================
 280 */
 281 void Curl_Clear_forthismap(void)
 282 {
 283         downloadinfo *di;
 284         if(noclear)
 285                 return;
 286         if (curl_mutex) Thread_LockMutex(curl_mutex);
 287         for(di = downloads; di; di = di->next)
 288                 di->forthismap = false;
 289         Curl_CommandWhenError(NULL);
 290         Curl_CommandWhenDone(NULL);
 291         numdownloads_fail = 0;
 292         numdownloads_success = 0;
 293         numdownloads_added = 0;
 294         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 295 }
 296
 297 /*
 298 ====================
 299 Curl_Have_forthismap
 300
 301 Returns true if a download needed for the current game is running.
 302 ====================
 303 */
 304 qboolean Curl_Have_forthismap(void)
 305 {
 306         return numdownloads_added != 0;
 307 }
 308
 309 void Curl_Register_predownload(void)
 310 {
 311         if (curl_mutex) Thread_LockMutex(curl_mutex);
 312         Curl_CommandWhenDone("cl_begindownloads");
 313         Curl_CommandWhenError("cl_begindownloads");
 314         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 315 }
 316
 317 /*
 318 ====================
 319 Curl_CheckCommandWhenDone
 320
 321 Checks if a "done command" is to be executed.
 322 All downloads finished, at least one success since connect, no single failure
 323 -> execute the command.
 324 */
 325 static void Curl_CheckCommandWhenDone(void)
 326 {
 327         if(!curl_dll)
 328                 return;
 329         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 330         {
 331                 if(numdownloads_fail == 0)
 332                 {
 333                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 334                         Cbuf_AddText("\n");
 335                         Cbuf_AddText(command_when_done);
 336                         Cbuf_AddText("\n");
 337                 }
 338                 else
 339                 {
 340                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 341                         Cbuf_AddText("\n");
 342                         Cbuf_AddText(command_when_error);
 343                         Cbuf_AddText("\n");
 344                 }
 345                 Curl_Clear_forthismap();
 346         }
 347 }
 348
 349 /*
 350 ====================
 351 CURL_CloseLibrary
 352
 353 Load the cURL DLL
 354 ====================
 355 */
 356 static qboolean CURL_OpenLibrary (void)
 357 {
 358         const char* dllnames [] =
 359         {
 360 #if defined(WIN32)
 361                 "libcurl-4.dll",
 362                 "libcurl-3.dll",
 363 #elif defined(MACOSX)
 364                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 365                 "libcurl.3.dylib", // Mac OS X Tiger
 366                 "libcurl.2.dylib", // Mac OS X Panther
 367 #else
 368                 "libcurl.so.4",
 369                 "libcurl.so.3",
 370                 "libcurl.so", // FreeBSD
 371 #endif
 372                 NULL
 373         };
 374
 375         // Already loaded?
 376         if (curl_dll)
 377                 return true;
 378
 379         // Load the DLL
 380         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 381 }
 382
 383
 384 /*
 385 ====================
 386 CURL_CloseLibrary
 387
 388 Unload the cURL DLL
 389 ====================
 390 */
 391 static void CURL_CloseLibrary (void)
 392 {
 393         Sys_UnloadLibrary (&curl_dll);
 394 }
 395
 396
 397 static CURLM *curlm = NULL;
 398 static double bytes_received = 0; // used for bandwidth throttling
 399 static double bytes_sent = 0; // used for bandwidth throttling
 400 static double curltime = 0;
 401
 402 /*
 403 ====================
 404 CURL_fwrite
 405
 406 fwrite-compatible function that writes the data to a file. libcurl can call
 407 this.
 408 ====================
 409 */
 410 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 411 {
 412         fs_offset_t ret = -1;
 413         size_t bytes = size * nmemb;
 414         downloadinfo *di = (downloadinfo *) vdi;
 415
 416         if(di->buffer)
 417         {
 418                 if(di->bytes_received + bytes <= di->buffersize)
 419                 {
 420                         memcpy(di->buffer + di->bytes_received, data, bytes);
 421                         ret = bytes;
 422                 }
 423                 // otherwise: buffer overrun, ret stays -1
 424         }
 425
 426         if(di->stream)
 427         {
 428                 ret = FS_Write(di->stream, data, bytes);
 429         }
 430
 431         di->bytes_received += bytes;
 432
 433         return ret;
 434         // Why not ret / nmemb?
 435         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
 436         // Yes, this is incompatible to fwrite(2).
 437 }
 438
 439 typedef enum
 440 {
 441         CURL_DOWNLOAD_SUCCESS = 0,
 442         CURL_DOWNLOAD_FAILED,
 443         CURL_DOWNLOAD_ABORTED,
 444         CURL_DOWNLOAD_SERVERERROR
 445 }
 446 CurlStatus;
 447
 448 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 449 {
 450         downloadinfo *di = (downloadinfo *) cbdata;
 451         switch(status)
 452         {
 453                 case CURLCBSTATUS_OK:
 454                         Con_DPrintf("Download of %s: OK\n", di->filename);
 455                         break;
 456                 case CURLCBSTATUS_FAILED:
 457                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 458                         break;
 459                 case CURLCBSTATUS_ABORTED:
 460                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 461                         break;
 462                 case CURLCBSTATUS_SERVERERROR:
 463                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 464                         break;
 465                 case CURLCBSTATUS_UNKNOWN:
 466                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 467                         break;
 468                 default:
 469                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 470                         break;
 471         }
 472 }
 473
 474 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 475 {
 476         curl_default_callback(status, length_received, buffer, cbdata);
 477 }
 478
 479 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
 480 {
 481         unsigned char *pixels = NULL;
 482         fs_offset_t filesize = 0;
 483         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
 484         if(data)
 485         {
 486                 int mip = 0;
 487                 if(!strcmp(content_type, "image/jpeg"))
 488                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 489                 else if(!strcmp(content_type, "image/png"))
 490                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 491                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
 492                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
 493                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
 494                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
 495                 else
 496                         Con_Printf("Did not detect content type: %s\n", content_type);
 497                 Mem_Free(data);
 498         }
 499         // do we call Image_MakeLinearColorsFromsRGB or not?
 500         return pixels;
 501 }
 502
 503 /*
 504 ====================
 505 Curl_EndDownload
 506
 507 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 508 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 509 code from libcurl, or 0, if another error has occurred.
 510 ====================
 511 */
 512 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 513 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
 514 {
 515         char content_type[64];
 516         qboolean ok = false;
 517         if(!curl_dll)
 518                 return;
 519         switch(status)
 520         {
 521                 case CURL_DOWNLOAD_SUCCESS:
 522                         ok = true;
 523                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 524                         break;
 525                 case CURL_DOWNLOAD_FAILED:
 526                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 527                         break;
 528                 case CURL_DOWNLOAD_ABORTED:
 529                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 530                         break;
 531                 case CURL_DOWNLOAD_SERVERERROR:
 532                         // reopen to enforce it to have zero bytes again
 533                         if(di->stream)
 534                         {
 535                                 FS_Close(di->stream);
 536                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 537                         }
 538
 539                         if(di->callback)
 540                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 541                         break;
 542                 default:
 543                         if(di->callback)
 544                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 545                         break;
 546         }
 547         if(content_type_)
 548                 strlcpy(content_type, content_type_, sizeof(content_type));
 549         else
 550                 *content_type = 0;
 551
 552         if(di->curle)
 553         {
 554                 qcurl_multi_remove_handle(curlm, di->curle);
 555                 qcurl_easy_cleanup(di->curle);
 556                 if(di->slist)
 557                         qcurl_slist_free_all(di->slist);
 558         }
 559
 560         if(!di->callback && ok && !di->bytes_received)
 561         {
 562                 Con_Printf("ERROR: empty file\n");
 563                 ok = false;
 564         }
 565
 566         if(di->stream)
 567                 FS_Close(di->stream);
 568
 569 #define CLEAR_AND_RETRY() \
 570         do \
 571         { \
 572                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
 573                 FS_Close(di->stream); \
 574                 if(di->startpos && !di->callback) \
 575                 { \
 576                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
 577                         di->forthismap = false; \
 578                 } \
 579         } \
 580         while(0)
 581
 582         if(ok && di->loadtype == LOADTYPE_PAK)
 583         {
 584                 ok = FS_AddPack(di->filename, NULL, true);
 585                 if(!ok)
 586                         CLEAR_AND_RETRY();
 587         }
 588         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
 589         {
 590                 const char *p;
 591                 unsigned char *pixels = NULL;
 592
 593                 p = di->filename;
 594 #ifdef WE_ARE_EVIL
 595                 if(!strncmp(p, "dlcache/", 8))
 596                         p += 8;
 597 #endif
 598
 599                 pixels = decode_image(di, content_type);
 600                 if(pixels)
 601                         Draw_NewPic(p, image_width, image_height, true, pixels);
 602                 else
 603                         CLEAR_AND_RETRY();
 604         }
 605         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
 606         {
 607                 const char *p;
 608                 unsigned char *pixels = NULL;
 609
 610                 p = di->filename;
 611 #ifdef WE_ARE_EVIL
 612                 if(!strncmp(p, "dlcache/", 8))
 613                         p += 8;
 614 #endif
 615
 616                 pixels = decode_image(di, content_type);
 617                 if(pixels)
 618                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, false); // TODO what sRGB argument to put here?
 619                 else
 620                         CLEAR_AND_RETRY();
 621         }
 622
 623         if(di->prev)
 624                 di->prev->next = di->next;
 625         else
 626                 downloads = di->next;
 627         if(di->next)
 628                 di->next->prev = di->prev;
 629
 630         --numdownloads;
 631         if(di->forthismap)
 632         {
 633                 if(ok)
 634                         ++numdownloads_success;
 635                 else
 636                         ++numdownloads_fail;
 637         }
 638         Z_Free(di);
 639 }
 640
 641 /*
 642 ====================
 643 CleanURL
 644
 645 Returns a "cleaned up" URL for display (to strip login data)
 646 ====================
 647 */
 648 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 649 {
 650         const char *p, *q, *r;
 651
 652         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 653         p = strstr(url, "://");
 654         if(p)
 655         {
 656                 q = strchr(p + 3, '@');
 657                 if(q)
 658                 {
 659                         r = strchr(p + 3, '/');
 660                         if(!r || q < r)
 661                         {
 662                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 663                                 return urlbuf;
 664                         }
 665                 }
 666         }
 667
 668         return url;
 669 }
 670
 671 /*
 672 ====================
 673 CheckPendingDownloads
 674
 675 checks if there are free download slots to start new downloads in.
 676 To not start too many downloads at once, only one download is added at a time,
 677 up to a maximum number of cl_curl_maxdownloads are running.
 678 ====================
 679 */
 680 static void CheckPendingDownloads(void)
 681 {
 682         const char *h;
 683         char urlbuf[1024];
 684         char vabuf[1024];
 685         if(!curl_dll)
 686                 return;
 687         if(numdownloads < cl_curl_maxdownloads.integer)
 688         {
 689                 downloadinfo *di;
 690                 for(di = downloads; di; di = di->next)
 691                 {
 692                         if(!di->started)
 693                         {
 694                                 if(!di->buffer)
 695                                 {
 696                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 697
 698                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 699                                         if(!di->stream)
 700                                         {
 701                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 702                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
 703                                                 return;
 704                                         }
 705                                         FS_Seek(di->stream, 0, SEEK_END);
 706                                         di->startpos = FS_Tell(di->stream);
 707
 708                                         if(di->startpos > 0)
 709                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 710                                         Con_Print("...\n");
 711                                 }
 712                                 else
 713                                 {
 714                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 715                                         di->startpos = 0;
 716                                 }
 717
 718                                 di->curle = qcurl_easy_init();
 719                                 di->slist = NULL;
 720                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 721                                 if(cl_curl_useragent.integer)
 722                                 {
 723                                         const char *ua
 724 #ifdef HTTP_USER_AGENT
 725                                                 = HTTP_USER_AGENT;
 726 #else
 727                                                 = engineversion;
 728 #endif
 729                                         if(!ua)
 730                                                 ua = "";
 731                                         if(*cl_curl_useragent_append.string)
 732                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
 733                                                         ua,
 734                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
 735                                                                 ? " "
 736                                                                 : "",
 737                                                         cl_curl_useragent_append.string);
 738                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
 739                                 }
 740                                 else
 741                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
 742                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 743                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 744                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 745                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 746                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 747                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 748                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 749                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 750                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 751                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 752                                 {
 753                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 754                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 755                                 }
 756                                 if(di->post_content_type)
 757                                 {
 758                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 759                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 760                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 761                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 762                                 }
 763
 764                                 // parse extra headers into slist
 765                                 // \n separated list!
 766                                 h = di->extraheaders;
 767                                 while(h)
 768                                 {
 769                                         const char *hh = strchr(h, '\n');
 770                                         if(hh)
 771                                         {
 772                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 773                                                 memcpy(buf, h, hh - h);
 774                                                 buf[hh - h] = 0;
 775                                                 di->slist = qcurl_slist_append(di->slist, buf);
 776                                                 h = hh + 1;
 777                                         }
 778                                         else
 779                                         {
 780                                                 di->slist = qcurl_slist_append(di->slist, h);
 781                                                 h = NULL;
 782                                         }
 783                                 }
 784
 785                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 786
 787                                 qcurl_multi_add_handle(curlm, di->curle);
 788                                 di->started = true;
 789                                 ++numdownloads;
 790                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 791                                         break;
 792                         }
 793                 }
 794         }
 795 }
 796
 797 /*
 798 ====================
 799 Curl_Init
 800
 801 this function MUST be called before using anything else in this file.
 802 On Win32, this must be called AFTER WSAStartup has been done!
 803 ====================
 804 */
 805 void Curl_Init(void)
 806 {
 807         CURL_OpenLibrary();
 808         if(!curl_dll)
 809                 return;
 810         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 811         qcurl_global_init(CURL_GLOBAL_NOTHING);
 812         curlm = qcurl_multi_init();
 813 }
 814
 815 /*
 816 ====================
 817 Curl_Shutdown
 818
 819 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 820 ====================
 821 */
 822 void Curl_ClearRequirements(void);
 823 void Curl_Shutdown(void)
 824 {
 825         if(!curl_dll)
 826                 return;
 827         Curl_ClearRequirements();
 828         Curl_CancelAll();
 829         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 830         CURL_CloseLibrary();
 831         curl_dll = NULL;
 832 }
 833
 834 /*
 835 ====================
 836 Curl_Find
 837
 838 Finds the internal information block for a download given by file name.
 839 ====================
 840 */
 841 static downloadinfo *Curl_Find(const char *filename)
 842 {
 843         downloadinfo *di;
 844         if(!curl_dll)
 845                 return NULL;
 846         for(di = downloads; di; di = di->next)
 847                 if(!strcasecmp(di->filename, filename))
 848                         return di;
 849         return NULL;
 850 }
 851
 852 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
 853 {
 854         downloadinfo *di;
 855         if(!curl_dll)
 856                 return;
 857         for(di = downloads; di; )
 858         {
 859                 if(di->callback == callback && di->callback_data == cbdata)
 860                 {
 861                         di->callback = curl_quiet_callback; // do NOT call the callback
 862                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
 863                         di = downloads;
 864                 }
 865                 else
 866                         di = di->next;
 867         }
 868 }
 869
 870 /*
 871 ====================
 872 Curl_Begin
 873
 874 Starts a download of a given URL to the file name portion of this URL (or name
 875 if given) in the "dlcache/" folder.
 876 ====================
 877 */
 878 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 879 {
 880         if(buf)
 881                 if(loadtype != LOADTYPE_NONE)
 882                         Host_Error("Curl_Begin: loadtype and buffer are both set");
 883
 884         if(!curl_dll || !cl_curl_enabled.integer)
 885         {
 886                 return false;
 887         }
 888         else
 889         {
 890                 char fn[MAX_OSPATH];
 891                 char urlbuf[1024];
 892                 const char *p, *q;
 893                 size_t length;
 894                 downloadinfo *di;
 895
 896                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 897                 p = strchr(URL, ':');
 898                 if(p)
 899                 {
 900                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 901                         {
 902                                 char addressstring[128];
 903                                 *addressstring = 0;
 904                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 905                                 q = strchr(addressstring, ':');
 906                                 if(!q)
 907                                         q = addressstring + strlen(addressstring);
 908                                 if(*addressstring)
 909                                 {
 910                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 911                                         URL = urlbuf;
 912                                 }
 913                         }
 914                 }
 915
 916                 // Note: This extraction of the file name portion is NOT entirely correct.
 917                 //
 918                 // It does the following:
 919                 //
 920                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 921                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 922                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 923                 //
 924                 // However, I'd like to keep this "buggy" behavior so that PHP script
 925                 // authors can write download scripts without having to enable
 926                 // AcceptPathInfo on Apache. They just have to ensure that their script
 927                 // can be called with such a "fake" path name like
 928                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 929                 //
 930                 // By the way, such PHP scripts should either send the file or a
 931                 // "Location:" redirect; PHP code example:
 932                 //
 933                 //   header("Location: http://www.example.com/");
 934                 //
 935                 // By the way, this will set User-Agent to something like
 936                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 937                 // dp://serverhost:serverport/ so you can filter on this; an example
 938                 // httpd log file line might be:
 939                 //
 940                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 941
 942                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 943
 944                 if(buf)
 945                 {
 946                         if(!name)
 947                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 948                 }
 949                 else
 950                 {
 951                         if(!name)
 952                         {
 953                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 954                                 p = strrchr(name, '/');
 955                                 p = p ? (p+1) : name;
 956                                 q = strchr(p, '?');
 957                                 length = q ? (size_t)(q - p) : strlen(p);
 958                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 959                         }
 960                         else
 961                         {
 962                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
 963                         }
 964
 965                         name = fn; // make it point back
 966
 967                         // already downloading the file?
 968                         {
 969                                 downloadinfo *di = Curl_Find(fn);
 970                                 if(di)
 971                                 {
 972                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 973
 974                                         // however, if it was not for this map yet...
 975                                         if(forthismap && !di->forthismap)
 976                                         {
 977                                                 di->forthismap = true;
 978                                                 // this "fakes" a download attempt so the client will wait for
 979                                                 // the download to finish and then reconnect
 980                                                 ++numdownloads_added;
 981                                         }
 982
 983                                         return false;
 984                                 }
 985                         }
 986
 987                         if(FS_FileExists(fn))
 988                         {
 989                                 if(loadtype == LOADTYPE_PAK)
 990                                 {
 991                                         qboolean already_loaded;
 992                                         if(FS_AddPack(fn, &already_loaded, true))
 993                                         {
 994                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 995                                                 if(already_loaded)
 996                                                         Con_DPrintf("(pak was already loaded)\n");
 997                                                 else
 998                                                 {
 999                                                         if(forthismap)
1000                                                         {
1001                                                                 ++numdownloads_added;
1002                                                                 ++numdownloads_success;
1003                                                         }
1004                                                 }
1005
1006                                                 return false;
1007                                         }
1008                                         else
1009                                         {
1010                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1011                                                 if(f)
1012                                                 {
1013                                                         char buf[4] = {0};
1014                                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
1015
1016                                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
1017                                                         {
1018                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1019                                                                 FS_Close(f);
1020                                                                 f = FS_OpenRealFile(fn, "wb", false);
1021                                                                 if(f)
1022                                                                         FS_Close(f);
1023                                                         }
1024                                                         else
1025                                                         {
1026                                                                 // OK
1027                                                                 FS_Close(f);
1028                                                         }
1029                                                 }
1030                                         }
1031                                 }
1032                                 else
1033                                 {
1034                                         // never resume these
1035                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1036                                         if(f)
1037                                                 FS_Close(f);
1038                                 }
1039                         }
1040                 }
1041
1042                 // if we get here, we actually want to download... so first verify the
1043                 // URL scheme (so one can't read local files using file://)
1044                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1045                 {
1046                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1047                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1048                         return false;
1049                 }
1050
1051                 if(forthismap)
1052                         ++numdownloads_added;
1053                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1054                 strlcpy(di->filename, name, sizeof(di->filename));
1055                 strlcpy(di->url, URL, sizeof(di->url));
1056                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1057                 di->forthismap = forthismap;
1058                 di->stream = NULL;
1059                 di->startpos = 0;
1060                 di->curle = NULL;
1061                 di->started = false;
1062                 di->loadtype = loadtype;
1063                 di->maxspeed = maxspeed;
1064                 di->bytes_received = 0;
1065                 di->bytes_received_curl = 0;
1066                 di->bytes_sent_curl = 0;
1067                 di->extraheaders = extraheaders;
1068                 di->next = downloads;
1069                 di->prev = NULL;
1070                 if(di->next)
1071                         di->next->prev = di;
1072
1073                 di->buffer = buf;
1074                 di->buffersize = bufsize;
1075                 if(callback == NULL)
1076                 {
1077                         di->callback = curl_default_callback;
1078                         di->callback_data = di;
1079                 }
1080                 else
1081                 {
1082                         di->callback = callback;
1083                         di->callback_data = cbdata;
1084                 }
1085
1086                 if(post_content_type)
1087                 {
1088                         di->post_content_type = post_content_type;
1089                         di->postbuf = postbuf;
1090                         di->postbufsize = postbufsize;
1091                 }
1092                 else
1093                 {
1094                         di->post_content_type = NULL;
1095                         di->postbuf = NULL;
1096                         di->postbufsize = 0;
1097                 }
1098
1099                 downloads = di;
1100                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1101                 return true;
1102         }
1103 }
1104
1105 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qboolean forthismap)
1106 {
1107         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1108 }
1109 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1110 {
1111         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1112 }
1113 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1114 {
1115         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1116 }
1117
1118 /*
1119 ====================
1120 Curl_Run
1121
1122 call this regularily as this will always download as much as possible without
1123 blocking.
1124 ====================
1125 */
1126 void Curl_Run(void)
1127 {
1128         double maxspeed;
1129         downloadinfo *di;
1130
1131         noclear = FALSE;
1132
1133         if(!cl_curl_enabled.integer)
1134                 return;
1135
1136         if(!curl_dll)
1137                 return;
1138
1139         if (curl_mutex) Thread_LockMutex(curl_mutex);
1140
1141         Curl_CheckCommandWhenDone();
1142
1143         if(!downloads)
1144         {
1145                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1146                 return;
1147         }
1148
1149         if(realtime < curltime) // throttle
1150         {
1151                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1152                 return;
1153         }
1154
1155         {
1156                 int remaining;
1157                 CURLMcode mc;
1158
1159                 do
1160                 {
1161                         mc = qcurl_multi_perform(curlm, &remaining);
1162                 }
1163                 while(mc == CURLM_CALL_MULTI_PERFORM);
1164
1165                 for(di = downloads; di; di = di->next)
1166                 {
1167                         double b = 0;
1168                         if(di->curle)
1169                         {
1170                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1171                                 bytes_sent += (b - di->bytes_sent_curl);
1172                                 di->bytes_sent_curl = b;
1173                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1174                                 bytes_sent += (b - di->bytes_received_curl);
1175                                 di->bytes_received_curl = b;
1176                         }
1177                 }
1178
1179                 for(;;)
1180                 {
1181                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1182                         if(!msg)
1183                                 break;
1184                         if(msg->msg == CURLMSG_DONE)
1185                         {
1186                                 const char *ct = NULL;
1187                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1188                                 CURLcode result;
1189                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1190                                 result = msg->data.result;
1191                                 if(result)
1192                                 {
1193                                         failed = CURL_DOWNLOAD_FAILED;
1194                                 }
1195                                 else
1196                                 {
1197                                         long code;
1198                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1199                                         switch(code / 100)
1200                                         {
1201                                                 case 4: // e.g. 404?
1202                                                 case 5: // e.g. 500?
1203                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1204                                                         result = (CURLcode) code;
1205                                                         break;
1206                                         }
1207                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1208                                 }
1209
1210                                 Curl_EndDownload(di, failed, result, ct);
1211                         }
1212                 }
1213         }
1214
1215         CheckPendingDownloads();
1216
1217         // when will we curl the next time?
1218         // we will wait a bit to ensure our download rate is kept.
1219         // we now know that realtime >= curltime... so set up a new curltime
1220
1221         // use the slowest allowing download to derive the maxspeed... this CAN
1222         // be done better, but maybe later
1223         maxspeed = cl_curl_maxspeed.value;
1224         for(di = downloads; di; di = di->next)
1225                 if(di->maxspeed > 0)
1226                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1227                                 maxspeed = di->maxspeed;
1228
1229         if(maxspeed > 0)
1230         {
1231                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1232                 curltime = realtime + bytes / (maxspeed * 1024.0);
1233                 bytes_sent = 0;
1234                 bytes_received = 0;
1235         }
1236         else
1237                 curltime = realtime;
1238
1239         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1240 }
1241
1242 /*
1243 ====================
1244 Curl_CancelAll
1245
1246 Stops ALL downloads.
1247 ====================
1248 */
1249 void Curl_CancelAll(void)
1250 {
1251         if(!curl_dll)
1252                 return;
1253
1254         if (curl_mutex) Thread_LockMutex(curl_mutex);
1255
1256         while(downloads)
1257         {
1258                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1259                 // INVARIANT: downloads will point to the next download after that!
1260         }
1261
1262         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1263 }
1264
1265 /*
1266 ====================
1267 Curl_Running
1268
1269 returns true iff there is a download running.
1270 ====================
1271 */
1272 qboolean Curl_Running(void)
1273 {
1274         if(!curl_dll)
1275                 return false;
1276
1277         return downloads != NULL;
1278 }
1279
1280 /*
1281 ====================
1282 Curl_GetDownloadAmount
1283
1284 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1285 for the given download.
1286 ====================
1287 */
1288 static double Curl_GetDownloadAmount(downloadinfo *di)
1289 {
1290         if(!curl_dll)
1291                 return -2;
1292         if(di->curle)
1293         {
1294                 double length;
1295                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1296                 if(length > 0)
1297                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1298                 else
1299                         return 0;
1300         }
1301         else
1302                 return -1;
1303 }
1304
1305 /*
1306 ====================
1307 Curl_GetDownloadSpeed
1308
1309 returns the speed of the given download in bytes per second
1310 ====================
1311 */
1312 static double Curl_GetDownloadSpeed(downloadinfo *di)
1313 {
1314         if(!curl_dll)
1315                 return -2;
1316         if(di->curle)
1317         {
1318                 double speed;
1319                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1320                 return speed;
1321         }
1322         else
1323                 return -1;
1324 }
1325
1326 /*
1327 ====================
1328 Curl_Info_f
1329
1330 prints the download list
1331 ====================
1332 */
1333 // TODO rewrite using Curl_GetDownloadInfo?
1334 static void Curl_Info_f(void)
1335 {
1336         downloadinfo *di;
1337         char urlbuf[1024];
1338         if(!curl_dll)
1339                 return;
1340         if(Curl_Running())
1341         {
1342                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1343                 Con_Print("Currently running downloads:\n");
1344                 for(di = downloads; di; di = di->next)
1345                 {
1346                         double speed, percent;
1347                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1348                         percent = 100.0 * Curl_GetDownloadAmount(di);
1349                         speed = Curl_GetDownloadSpeed(di);
1350                         if(percent >= 0)
1351                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1352                         else
1353                                 Con_Print("(queued)\n");
1354                 }
1355                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1356         }
1357         else
1358         {
1359                 Con_Print("No downloads running.\n");
1360         }
1361 }
1362
1363 /*
1364 ====================
1365 Curl_Curl_f
1366
1367 implements the "curl" console command
1368
1369 curl --info
1370 curl --cancel
1371 curl --cancel filename
1372 curl url
1373
1374 For internal use:
1375
1376 curl [--pak] [--forthismap] [--for filename filename...] url
1377         --pak: after downloading, load the package into the virtual file system
1378         --for filename...: only download of at least one of the named files is missing
1379         --forthismap: don't reconnect on failure
1380
1381 curl --clear_autodownload
1382         clears the download success/failure counters
1383
1384 curl --finish_autodownload
1385         if at least one download has been started, disconnect and drop to the menu
1386         once the last download completes successfully, reconnect to the current server
1387 ====================
1388 */
1389 static void Curl_Curl_f(void)
1390 {
1391         double maxspeed = 0;
1392         int i;
1393         int end;
1394         int loadtype = LOADTYPE_NONE;
1395         qboolean forthismap = false;
1396         const char *url;
1397         const char *name = 0;
1398
1399         if(!curl_dll)
1400         {
1401                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1402                 return;
1403         }
1404
1405         if(!cl_curl_enabled.integer)
1406         {
1407                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1408                 return;
1409         }
1410
1411         if(Cmd_Argc() < 2)
1412         {
1413                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1414                 return;
1415         }
1416
1417         url = Cmd_Argv(Cmd_Argc() - 1);
1418         end = Cmd_Argc();
1419
1420         for(i = 1; i != end; ++i)
1421         {
1422                 const char *a = Cmd_Argv(i);
1423                 if(!strcmp(a, "--info"))
1424                 {
1425                         Curl_Info_f();
1426                         return;
1427                 }
1428                 else if(!strcmp(a, "--cancel"))
1429                 {
1430                         if(i == end - 1) // last argument
1431                                 Curl_CancelAll();
1432                         else
1433                         {
1434                                 downloadinfo *di = Curl_Find(url);
1435                                 if(di)
1436                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1437                                 else
1438                                         Con_Print("download not found\n");
1439                         }
1440                         return;
1441                 }
1442                 else if(!strcmp(a, "--pak"))
1443                 {
1444                         loadtype = LOADTYPE_PAK;
1445                 }
1446                 else if(!strcmp(a, "--cachepic"))
1447                 {
1448                         loadtype = LOADTYPE_CACHEPIC;
1449                 }
1450                 else if(!strcmp(a, "--skinframe"))
1451                 {
1452                         loadtype = LOADTYPE_SKINFRAME;
1453                 }
1454                 else if(!strcmp(a, "--for")) // must be last option
1455                 {
1456                         for(i = i + 1; i != end - 1; ++i)
1457                         {
1458                                 if(!FS_FileExists(Cmd_Argv(i)))
1459                                         goto needthefile; // why can't I have a "double break"?
1460                         }
1461                         // if we get here, we have all the files...
1462                         return;
1463                 }
1464                 else if(!strcmp(a, "--forthismap"))
1465                 {
1466                         forthismap = true;
1467                 }
1468                 else if(!strcmp(a, "--as"))
1469                 {
1470                         if(i < end - 1)
1471                         {
1472                                 ++i;
1473                                 name = Cmd_Argv(i);
1474                         }
1475                 }
1476                 else if(!strcmp(a, "--clear_autodownload"))
1477                 {
1478                         // mark all running downloads as "not for this map", so if they
1479                         // fail, it does not matter
1480                         Curl_Clear_forthismap();
1481                         return;
1482                 }
1483                 else if(!strcmp(a, "--finish_autodownload"))
1484                 {
1485                         if(numdownloads_added)
1486                         {
1487                                 char donecommand[256];
1488                                 if(cls.netcon)
1489                                 {
1490                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1491                                         {
1492                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1493                                                 Curl_CommandWhenDone(donecommand);
1494                                                 noclear = TRUE;
1495                                                 CL_Disconnect();
1496                                                 noclear = FALSE;
1497                                                 Curl_CheckCommandWhenDone();
1498                                         }
1499                                         else
1500                                                 Curl_Register_predownload();
1501                                 }
1502                         }
1503                         return;
1504                 }
1505                 else if(!strncmp(a, "--maxspeed=", 11))
1506                 {
1507                         maxspeed = atof(a + 11);
1508                 }
1509                 else if(*a == '-')
1510                 {
1511                         Con_Printf("curl: invalid option %s\n", a);
1512                         // but we ignore the option
1513                 }
1514         }
1515
1516 needthefile:
1517         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1518 }
1519
1520 /*
1521 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1522 {
1523         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1524         Z_Free(buffer);
1525 }
1526
1527 void Curl_CurlCat_f(void)
1528 {
1529         unsigned char *buf;
1530         const char *url = Cmd_Argv(1);
1531         buf = Z_Malloc(16384);
1532         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1533 }
1534 */
1535
1536 /*
1537 ====================
1538 Curl_Init_Commands
1539
1540 loads the commands and cvars this library uses
1541 ====================
1542 */
1543 void Curl_Init_Commands(void)
1544 {
1545         Cvar_RegisterVariable (&cl_curl_enabled);
1546         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1547         Cvar_RegisterVariable (&cl_curl_maxspeed);
1548         Cvar_RegisterVariable (&sv_curl_defaulturl);
1549         Cvar_RegisterVariable (&sv_curl_serverpackages);
1550         Cvar_RegisterVariable (&sv_curl_maxspeed);
1551         Cvar_RegisterVariable (&cl_curl_useragent);
1552         Cvar_RegisterVariable (&cl_curl_useragent_append);
1553         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1554         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1555 }
1556
1557 /*
1558 ====================
1559 Curl_GetDownloadInfo
1560
1561 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1562 The number of elements in the array is returned in int *nDownloads.
1563 const char **additional_info may be set to a string of additional user
1564 information, or to NULL if no such display shall occur. The returned
1565 array must be freed later using Z_Free.
1566 ====================
1567 */
1568 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1569 {
1570         int i;
1571         downloadinfo *di;
1572         Curl_downloadinfo_t *downinfo;
1573
1574         if(!curl_dll)
1575         {
1576                 *nDownloads = 0;
1577                 if(additional_info)
1578                         *additional_info = NULL;
1579                 return NULL;
1580         }
1581
1582         if (curl_mutex) Thread_LockMutex(curl_mutex);
1583
1584         i = 0;
1585         for(di = downloads; di; di = di->next)
1586                 ++i;
1587
1588         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1589         i = 0;
1590         for(di = downloads; di; di = di->next)
1591         {
1592                 // do not show infobars for background downloads
1593                 if(developer.integer <= 0)
1594                         if(di->buffer)
1595                                 continue;
1596                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1597                 if(di->curle)
1598                 {
1599                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1600                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1601                         downinfo[i].queued = false;
1602                 }
1603                 else
1604                 {
1605                         downinfo[i].queued = true;
1606                 }
1607                 ++i;
1608         }
1609
1610         if(additional_info)
1611         {
1612                 // TODO: can I clear command_when_done as soon as the first download fails?
1613                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1614                 {
1615                         if(!strncmp(command_when_done, "connect ", 8))
1616                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1617                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1618                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1619                         else
1620                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1621                         *additional_info = addinfo;
1622                 }
1623                 else
1624                         *additional_info = NULL;
1625         }
1626
1627         *nDownloads = i;
1628         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1629         return downinfo;
1630 }
1631
1632
1633 /*
1634 ====================
1635 Curl_FindPackURL
1636
1637 finds the URL where to find a given package.
1638
1639 For this, it reads a file "curl_urls.txt" of the following format:
1640
1641         data*.pk3       -
1642         revdm*.pk3      http://revdm/downloads/are/here/
1643         *                       http://any/other/stuff/is/here/
1644
1645 The URLs should end in /. If not, downloads will still work, but the cached files
1646 can't be just put into the data directory with the same download configuration
1647 (you might want to do this if you want to tag downloaded files from your
1648 server, but you should not). "-" means "don't download".
1649
1650 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1651 location instead.
1652
1653 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1654 this file for obvious reasons.
1655 ====================
1656 */
1657 static const char *Curl_FindPackURL(const char *filename)
1658 {
1659         static char foundurl[1024]; // invoked only by server
1660         fs_offset_t filesize;
1661         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1662         if(buf && filesize)
1663         {
1664                 // read lines of format "pattern url"
1665                 char *p = buf;
1666                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1667                 qboolean eof = false;
1668
1669                 pattern = p;
1670                 while(!eof)
1671                 {
1672                         switch(*p)
1673                         {
1674                                 case 0:
1675                                         eof = true;
1676                                         // fallthrough
1677                                 case '\n':
1678                                 case '\r':
1679                                         if(pattern && url && patternend)
1680                                         {
1681                                                 if(!urlend)
1682                                                         urlend = p;
1683                                                 *patternend = 0;
1684                                                 *urlend = 0;
1685                                                 if(matchpattern(filename, pattern, true))
1686                                                 {
1687                                                         strlcpy(foundurl, url, sizeof(foundurl));
1688                                                         Z_Free(buf);
1689                                                         return foundurl;
1690                                                 }
1691                                         }
1692                                         pattern = NULL;
1693                                         patternend = NULL;
1694                                         url = NULL;
1695                                         urlend = NULL;
1696                                         break;
1697                                 case ' ':
1698                                 case '\t':
1699                                         if(pattern && !patternend)
1700                                                 patternend = p;
1701                                         else if(url && !urlend)
1702                                                 urlend = p;
1703                                         break;
1704                                 default:
1705                                         if(!pattern)
1706                                                 pattern = p;
1707                                         else if(pattern && patternend && !url)
1708                                                 url = p;
1709                                         break;
1710                         }
1711                         ++p;
1712                 }
1713         }
1714         if(buf)
1715                 Z_Free(buf);
1716         return sv_curl_defaulturl.string;
1717 }
1718
1719 typedef struct requirement_s
1720 {
1721         struct requirement_s *next;
1722         char filename[MAX_OSPATH];
1723 }
1724 requirement;
1725 static requirement *requirements = NULL;
1726
1727
1728 /*
1729 ====================
1730 Curl_RequireFile
1731
1732 Adds the given file to the list of requirements.
1733 ====================
1734 */
1735 void Curl_RequireFile(const char *filename)
1736 {
1737         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1738         req->next = requirements;
1739         strlcpy(req->filename, filename, sizeof(req->filename));
1740         requirements = req;
1741 }
1742
1743 /*
1744 ====================
1745 Curl_ClearRequirements
1746
1747 Clears the list of required files for playing on the current map.
1748 This should be called at every map change.
1749 ====================
1750 */
1751 void Curl_ClearRequirements(void)
1752 {
1753         while(requirements)
1754         {
1755                 requirement *req = requirements;
1756                 requirements = requirements->next;
1757                 Z_Free(req);
1758         }
1759 }
1760
1761 /*
1762 ====================
1763 Curl_SendRequirements
1764
1765 Makes the current host_clients download all files he needs.
1766 This is done by sending him the following console commands:
1767
1768         curl --clear_autodownload
1769         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1770         curl --finish_autodownload
1771 ====================
1772 */
1773 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1774 {
1775         const char *p;
1776         const char *thispack = FS_WhichPack(filename);
1777         const char *packurl;
1778
1779         if(!thispack || !*thispack)
1780                 return false;
1781
1782         p = strrchr(thispack, '/');
1783         if(p)
1784                 thispack = p + 1;
1785
1786         packurl = Curl_FindPackURL(thispack);
1787
1788         if(packurl && *packurl && strcmp(packurl, "-"))
1789         {
1790                 if(!foundone)
1791                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1792
1793                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1794                 strlcat(sendbuffer, thispack, sendbuffer_len);
1795                 if(sv_curl_maxspeed.value > 0)
1796                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1797                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1798                 strlcat(sendbuffer, filename, sendbuffer_len);
1799                 strlcat(sendbuffer, " ", sendbuffer_len);
1800                 strlcat(sendbuffer, packurl, sendbuffer_len);
1801                 strlcat(sendbuffer, thispack, sendbuffer_len);
1802                 strlcat(sendbuffer, "\n", sendbuffer_len);
1803
1804                 return true;
1805         }
1806
1807         return false;
1808 }
1809 void Curl_SendRequirements(void)
1810 {
1811         // for each requirement, find the pack name
1812         char sendbuffer[4096] = "";
1813         requirement *req;
1814         qboolean foundone = false;
1815         const char *p;
1816
1817         for(req = requirements; req; req = req->next)
1818                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1819
1820         p = sv_curl_serverpackages.string;
1821         while(COM_ParseToken_Simple(&p, false, false, true))
1822                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1823
1824         if(foundone)
1825                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1826
1827         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1828                 Host_ClientCommands("%s", sendbuffer);
1829         else
1830                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1831 }