libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4 #include "thread.h"
   5
   6 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   7 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
   8 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   9 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
  10 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
  11 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  12
  13 /*
  14 =================================================================
  15
  16   Minimal set of definitions from libcurl
  17
  18   WARNING: for a matter of simplicity, several pointer types are
  19   casted to "void*", and most enumerated values are not included
  20
  21 =================================================================
  22 */
  23
  24 typedef struct CURL_s CURL;
  25 typedef struct CURLM_s CURLM;
  26 typedef struct curl_slist curl_slist;
  27 typedef enum
  28 {
  29         CURLE_OK = 0
  30 }
  31 CURLcode;
  32 typedef enum
  33 {
  34         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  35         CURLM_OK = 0
  36 }
  37 CURLMcode;
  38 #define CURL_GLOBAL_NOTHING 0
  39 #define CURL_GLOBAL_SSL 1
  40 #define CURL_GLOBAL_WIN32 2
  41 #define CURLOPTTYPE_LONG          0
  42 #define CURLOPTTYPE_OBJECTPOINT   10000
  43 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  44 #define CURLOPTTYPE_OFF_T         30000
  45 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  46 typedef enum
  47 {
  48         CINIT(WRITEDATA, OBJECTPOINT, 1),
  49         CINIT(URL,  OBJECTPOINT, 2),
  50         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  51         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  52         CINIT(POSTFIELDS, OBJECTPOINT, 15),
  53         CINIT(REFERER, OBJECTPOINT, 16),
  54         CINIT(USERAGENT, OBJECTPOINT, 18),
  55         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  56         CINIT(LOW_SPEED_TIME, LONG, 20),
  57         CINIT(RESUME_FROM, LONG, 21),
  58         CINIT(HTTPHEADER, OBJECTPOINT, 23),
  59         CINIT(POST, LONG, 47),         /* HTTP POST method */
  60         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  61         CINIT(POSTFIELDSIZE, LONG, 60),
  62         CINIT(PRIVATE, OBJECTPOINT, 103),
  63         CINIT(PROTOCOLS, LONG, 181),
  64         CINIT(REDIR_PROTOCOLS, LONG, 182)
  65 }
  66 CURLoption;
  67 #define CURLPROTO_HTTP   (1<<0)
  68 #define CURLPROTO_HTTPS  (1<<1)
  69 #define CURLPROTO_FTP    (1<<2)
  70 typedef enum
  71 {
  72         CURLINFO_TEXT = 0,
  73         CURLINFO_HEADER_IN,    /* 1 */
  74         CURLINFO_HEADER_OUT,   /* 2 */
  75         CURLINFO_DATA_IN,      /* 3 */
  76         CURLINFO_DATA_OUT,     /* 4 */
  77         CURLINFO_SSL_DATA_IN,  /* 5 */
  78         CURLINFO_SSL_DATA_OUT, /* 6 */
  79         CURLINFO_END
  80 }
  81 curl_infotype;
  82 #define CURLINFO_STRING   0x100000
  83 #define CURLINFO_LONG     0x200000
  84 #define CURLINFO_DOUBLE   0x300000
  85 #define CURLINFO_SLIST    0x400000
  86 #define CURLINFO_MASK     0x0fffff
  87 #define CURLINFO_TYPEMASK 0xf00000
  88 typedef enum
  89 {
  90         CURLINFO_NONE, /* first, never use this */
  91         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  92         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  93         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  94         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  95         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  96         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  97         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  98         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  99         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
 100         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
 101         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
 102         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
 103         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
 104         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
 105         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
 106         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
 107         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
 108         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
 109         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
 110         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
 111         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 112         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 113         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 114         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 115         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 116         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 117         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
 118 }
 119 CURLINFO;
 120
 121 typedef enum
 122 {
 123         CURLMSG_NONE, /* first, not used */
 124         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 125                                          the CURLcode of the transfer */
 126         CURLMSG_LAST
 127 }
 128 CURLMSG;
 129 typedef struct
 130 {
 131         CURLMSG msg;       /* what this message means */
 132         CURL *easy_handle; /* the handle it concerns */
 133         union
 134         {
 135                 void *whatever;    /* message-specific data */
 136                 CURLcode result;   /* return code for transfer */
 137         }
 138         data;
 139 }
 140 CURLMsg;
 141
 142 static void (*qcurl_global_init) (long flags);
 143 static void (*qcurl_global_cleanup) (void);
 144
 145 static CURL * (*qcurl_easy_init) (void);
 146 static void (*qcurl_easy_cleanup) (CURL *handle);
 147 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 148 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 149 static const char * (*qcurl_easy_strerror) (CURLcode);
 150
 151 static CURLM * (*qcurl_multi_init) (void);
 152 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 153 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 154 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 155 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 156 static void (*qcurl_multi_cleanup) (CURLM *);
 157 static const char * (*qcurl_multi_strerror) (CURLcode);
 158 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
 159 static void (*qcurl_slist_free_all) (curl_slist *list);
 160
 161 static dllfunction_t curlfuncs[] =
 162 {
 163         {"curl_global_init",            (void **) &qcurl_global_init},
 164         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 165         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 166         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 167         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 168         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 169         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 170         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 171         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 172         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 173         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 174         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 175         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 176         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 177         {"curl_slist_append",           (void **) &qcurl_slist_append},
 178         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
 179         {NULL, NULL}
 180 };
 181
 182 // Handle for CURL DLL
 183 static dllhandle_t curl_dll = NULL;
 184 // will be checked at many places to find out if qcurl calls are allowed
 185
 186 void *curl_mutex = NULL;
 187
 188 typedef struct downloadinfo_s
 189 {
 190         char filename[MAX_OSPATH];
 191         char url[1024];
 192         char referer[256];
 193         qfile_t *stream;
 194         fs_offset_t startpos;
 195         CURL *curle;
 196         qboolean started;
 197         qboolean ispak;
 198         unsigned long bytes_received; // for buffer
 199         double bytes_received_curl; // for throttling
 200         double bytes_sent_curl; // for throttling
 201         struct downloadinfo_s *next, *prev;
 202         qboolean forthismap;
 203         double maxspeed;
 204         curl_slist *slist; // http headers
 205
 206         unsigned char *buffer;
 207         size_t buffersize;
 208         curl_callback_t callback;
 209         void *callback_data;
 210
 211         const unsigned char *postbuf;
 212         size_t postbufsize;
 213         const char *post_content_type;
 214         const char *extraheaders;
 215 }
 216 downloadinfo;
 217 static downloadinfo *downloads = NULL;
 218 static int numdownloads = 0;
 219
 220 static qboolean noclear = FALSE;
 221
 222 static int numdownloads_fail = 0;
 223 static int numdownloads_success = 0;
 224 static int numdownloads_added = 0;
 225 static char command_when_done[256] = "";
 226 static char command_when_error[256] = "";
 227
 228 /*
 229 ====================
 230 Curl_CommandWhenDone
 231
 232 Sets the command which is to be executed when the last download completes AND
 233 all downloads since last server connect ended with a successful status.
 234 Setting the command to NULL clears it.
 235 ====================
 236 */
 237 static void Curl_CommandWhenDone(const char *cmd)
 238 {
 239         if(!curl_dll)
 240                 return;
 241         if(cmd)
 242                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 243         else
 244                 *command_when_done = 0;
 245 }
 246
 247 /*
 248 FIXME
 249 Do not use yet. Not complete.
 250 Problem: what counts as an error?
 251 */
 252
 253 static void Curl_CommandWhenError(const char *cmd)
 254 {
 255         if(!curl_dll)
 256                 return;
 257         if(cmd)
 258                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 259         else
 260                 *command_when_error = 0;
 261 }
 262
 263 /*
 264 ====================
 265 Curl_Clear_forthismap
 266
 267 Clears the "will disconnect on failure" flags.
 268 ====================
 269 */
 270 void Curl_Clear_forthismap(void)
 271 {
 272         downloadinfo *di;
 273         if(noclear)
 274                 return;
 275         if (curl_mutex) Thread_LockMutex(curl_mutex);
 276         for(di = downloads; di; di = di->next)
 277                 di->forthismap = false;
 278         Curl_CommandWhenError(NULL);
 279         Curl_CommandWhenDone(NULL);
 280         numdownloads_fail = 0;
 281         numdownloads_success = 0;
 282         numdownloads_added = 0;
 283         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 284 }
 285
 286 /*
 287 ====================
 288 Curl_Have_forthismap
 289
 290 Returns true if a download needed for the current game is running.
 291 ====================
 292 */
 293 qboolean Curl_Have_forthismap(void)
 294 {
 295         return numdownloads_added != 0;
 296 }
 297
 298 void Curl_Register_predownload(void)
 299 {
 300         if (curl_mutex) Thread_LockMutex(curl_mutex);
 301         Curl_CommandWhenDone("cl_begindownloads");
 302         Curl_CommandWhenError("cl_begindownloads");
 303         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 304 }
 305
 306 /*
 307 ====================
 308 Curl_CheckCommandWhenDone
 309
 310 Checks if a "done command" is to be executed.
 311 All downloads finished, at least one success since connect, no single failure
 312 -> execute the command.
 313 */
 314 static void Curl_CheckCommandWhenDone(void)
 315 {
 316         if(!curl_dll)
 317                 return;
 318         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
 319         {
 320                 if(numdownloads_fail == 0)
 321                 {
 322                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 323                         Cbuf_AddText("\n");
 324                         Cbuf_AddText(command_when_done);
 325                         Cbuf_AddText("\n");
 326                 }
 327                 else
 328                 {
 329                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 330                         Cbuf_AddText("\n");
 331                         Cbuf_AddText(command_when_error);
 332                         Cbuf_AddText("\n");
 333                 }
 334                 Curl_Clear_forthismap();
 335         }
 336 }
 337
 338 /*
 339 ====================
 340 CURL_CloseLibrary
 341
 342 Load the cURL DLL
 343 ====================
 344 */
 345 static qboolean CURL_OpenLibrary (void)
 346 {
 347         const char* dllnames [] =
 348         {
 349 #if defined(WIN32)
 350                 "libcurl-4.dll",
 351                 "libcurl-3.dll",
 352 #elif defined(MACOSX)
 353                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 354                 "libcurl.3.dylib", // Mac OS X Tiger
 355                 "libcurl.2.dylib", // Mac OS X Panther
 356 #else
 357                 "libcurl.so.4",
 358                 "libcurl.so.3",
 359                 "libcurl.so", // FreeBSD
 360 #endif
 361                 NULL
 362         };
 363
 364         // Already loaded?
 365         if (curl_dll)
 366                 return true;
 367
 368         // Load the DLL
 369         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 370 }
 371
 372
 373 /*
 374 ====================
 375 CURL_CloseLibrary
 376
 377 Unload the cURL DLL
 378 ====================
 379 */
 380 static void CURL_CloseLibrary (void)
 381 {
 382         Sys_UnloadLibrary (&curl_dll);
 383 }
 384
 385
 386 static CURLM *curlm = NULL;
 387 static double bytes_received = 0; // used for bandwidth throttling
 388 static double bytes_sent = 0; // used for bandwidth throttling
 389 static double curltime = 0;
 390
 391 /*
 392 ====================
 393 CURL_fwrite
 394
 395 fwrite-compatible function that writes the data to a file. libcurl can call
 396 this.
 397 ====================
 398 */
 399 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 400 {
 401         fs_offset_t ret = -1;
 402         size_t bytes = size * nmemb;
 403         downloadinfo *di = (downloadinfo *) vdi;
 404
 405         if(di->buffer)
 406         {
 407                 if(di->bytes_received + bytes <= di->buffersize)
 408                 {
 409                         memcpy(di->buffer + di->bytes_received, data, bytes);
 410                         ret = bytes;
 411                 }
 412                 // otherwise: buffer overrun, ret stays -1
 413         }
 414
 415         if(di->stream)
 416         {
 417                 ret = FS_Write(di->stream, data, bytes);
 418         }
 419
 420         di->bytes_received += bytes;
 421
 422         return ret; // why not ret / nmemb?
 423 }
 424
 425 typedef enum
 426 {
 427         CURL_DOWNLOAD_SUCCESS = 0,
 428         CURL_DOWNLOAD_FAILED,
 429         CURL_DOWNLOAD_ABORTED,
 430         CURL_DOWNLOAD_SERVERERROR
 431 }
 432 CurlStatus;
 433
 434 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
 435 {
 436         downloadinfo *di = (downloadinfo *) cbdata;
 437         switch(status)
 438         {
 439                 case CURLCBSTATUS_OK:
 440                         Con_DPrintf("Download of %s: OK\n", di->filename);
 441                         break;
 442                 case CURLCBSTATUS_FAILED:
 443                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
 444                         break;
 445                 case CURLCBSTATUS_ABORTED:
 446                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
 447                         break;
 448                 case CURLCBSTATUS_SERVERERROR:
 449                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
 450                         break;
 451                 case CURLCBSTATUS_UNKNOWN:
 452                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
 453                         break;
 454                 default:
 455                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
 456                         break;
 457         }
 458 }
 459
 460 /*
 461 ====================
 462 Curl_EndDownload
 463
 464 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 465 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 466 code from libcurl, or 0, if another error has occurred.
 467 ====================
 468 */
 469 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
 470 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 471 {
 472         qboolean ok = false;
 473         if(!curl_dll)
 474                 return;
 475         switch(status)
 476         {
 477                 case CURL_DOWNLOAD_SUCCESS:
 478                         ok = true;
 479                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
 480                         break;
 481                 case CURL_DOWNLOAD_FAILED:
 482                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
 483                         break;
 484                 case CURL_DOWNLOAD_ABORTED:
 485                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
 486                         break;
 487                 case CURL_DOWNLOAD_SERVERERROR:
 488                         // reopen to enforce it to have zero bytes again
 489                         if(di->stream)
 490                         {
 491                                 FS_Close(di->stream);
 492                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
 493                         }
 494
 495                         if(di->callback)
 496                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
 497                         break;
 498                 default:
 499                         if(di->callback)
 500                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
 501                         break;
 502         }
 503
 504         if(di->curle)
 505         {
 506                 qcurl_multi_remove_handle(curlm, di->curle);
 507                 qcurl_easy_cleanup(di->curle);
 508                 if(di->slist)
 509                         qcurl_slist_free_all(di->slist);
 510         }
 511
 512         if(!di->callback && ok && !di->bytes_received)
 513         {
 514                 Con_Printf("ERROR: empty file\n");
 515                 ok = false;
 516         }
 517
 518         if(di->stream)
 519                 FS_Close(di->stream);
 520
 521         if(ok && di->ispak)
 522         {
 523                 ok = FS_AddPack(di->filename, NULL, true);
 524                 if(!ok)
 525                 {
 526                         // pack loading failed?
 527                         // this is critical
 528                         // better clear the file again...
 529                         di->stream = FS_OpenRealFile(di->filename, "wb", false);
 530                         FS_Close(di->stream);
 531
 532                         if(di->startpos && !di->callback)
 533                         {
 534                                 // this was a resume?
 535                                 // then try to redownload it without reporting the error
 536                                 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
 537                                 di->forthismap = false; // don't count the error
 538                         }
 539                 }
 540         }
 541
 542         if(di->prev)
 543                 di->prev->next = di->next;
 544         else
 545                 downloads = di->next;
 546         if(di->next)
 547                 di->next->prev = di->prev;
 548
 549         --numdownloads;
 550         if(di->forthismap)
 551         {
 552                 if(ok)
 553                         ++numdownloads_success;
 554                 else
 555                         ++numdownloads_fail;
 556         }
 557         Z_Free(di);
 558 }
 559
 560 /*
 561 ====================
 562 CleanURL
 563
 564 Returns a "cleaned up" URL for display (to strip login data)
 565 ====================
 566 */
 567 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
 568 {
 569         const char *p, *q, *r;
 570
 571         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
 572         p = strstr(url, "://");
 573         if(p)
 574         {
 575                 q = strchr(p + 3, '@');
 576                 if(q)
 577                 {
 578                         r = strchr(p + 3, '/');
 579                         if(!r || q < r)
 580                         {
 581                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
 582                                 return urlbuf;
 583                         }
 584                 }
 585         }
 586
 587         return url;
 588 }
 589
 590 /*
 591 ====================
 592 CheckPendingDownloads
 593
 594 checks if there are free download slots to start new downloads in.
 595 To not start too many downloads at once, only one download is added at a time,
 596 up to a maximum number of cl_curl_maxdownloads are running.
 597 ====================
 598 */
 599 static void CheckPendingDownloads(void)
 600 {
 601         const char *h;
 602         char urlbuf[1024];
 603         char vabuf[1024];
 604         if(!curl_dll)
 605                 return;
 606         if(numdownloads < cl_curl_maxdownloads.integer)
 607         {
 608                 downloadinfo *di;
 609                 for(di = downloads; di; di = di->next)
 610                 {
 611                         if(!di->started)
 612                         {
 613                                 if(!di->buffer)
 614                                 {
 615                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
 616
 617                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
 618                                         if(!di->stream)
 619                                         {
 620                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 621                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 622                                                 return;
 623                                         }
 624                                         FS_Seek(di->stream, 0, SEEK_END);
 625                                         di->startpos = FS_Tell(di->stream);
 626
 627                                         if(di->startpos > 0)
 628                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
 629                                         Con_Print("...\n");
 630                                 }
 631                                 else
 632                                 {
 633                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 634                                         di->startpos = 0;
 635                                 }
 636
 637                                 di->curle = qcurl_easy_init();
 638                                 di->slist = NULL;
 639                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 640                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 641                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 642                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 643                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 644                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 645                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 646                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 647                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 648                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 649                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
 650                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
 651                                 {
 652                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
 653                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
 654                                 }
 655                                 if(di->post_content_type)
 656                                 {
 657                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
 658                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
 659                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
 660                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
 661                                 }
 662
 663                                 // parse extra headers into slist
 664                                 // \n separated list!
 665                                 h = di->extraheaders;
 666                                 while(h)
 667                                 {
 668                                         const char *hh = strchr(h, '\n');
 669                                         if(hh)
 670                                         {
 671                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
 672                                                 memcpy(buf, h, hh - h);
 673                                                 buf[hh - h] = 0;
 674                                                 di->slist = qcurl_slist_append(di->slist, buf);
 675                                                 h = hh + 1;
 676                                         }
 677                                         else
 678                                         {
 679                                                 di->slist = qcurl_slist_append(di->slist, h);
 680                                                 h = NULL;
 681                                         }
 682                                 }
 683
 684                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
 685
 686                                 qcurl_multi_add_handle(curlm, di->curle);
 687                                 di->started = true;
 688                                 ++numdownloads;
 689                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 690                                         break;
 691                         }
 692                 }
 693         }
 694 }
 695
 696 /*
 697 ====================
 698 Curl_Init
 699
 700 this function MUST be called before using anything else in this file.
 701 On Win32, this must be called AFTER WSAStartup has been done!
 702 ====================
 703 */
 704 void Curl_Init(void)
 705 {
 706         CURL_OpenLibrary();
 707         if(!curl_dll)
 708                 return;
 709         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
 710         qcurl_global_init(CURL_GLOBAL_NOTHING);
 711         curlm = qcurl_multi_init();
 712 }
 713
 714 /*
 715 ====================
 716 Curl_Shutdown
 717
 718 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 719 ====================
 720 */
 721 void Curl_ClearRequirements(void);
 722 void Curl_Shutdown(void)
 723 {
 724         if(!curl_dll)
 725                 return;
 726         Curl_ClearRequirements();
 727         Curl_CancelAll();
 728         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
 729         CURL_CloseLibrary();
 730         curl_dll = NULL;
 731 }
 732
 733 /*
 734 ====================
 735 Curl_Find
 736
 737 Finds the internal information block for a download given by file name.
 738 ====================
 739 */
 740 static downloadinfo *Curl_Find(const char *filename)
 741 {
 742         downloadinfo *di;
 743         if(!curl_dll)
 744                 return NULL;
 745         for(di = downloads; di; di = di->next)
 746                 if(!strcasecmp(di->filename, filename))
 747                         return di;
 748         return NULL;
 749 }
 750
 751 /*
 752 ====================
 753 Curl_Begin
 754
 755 Starts a download of a given URL to the file name portion of this URL (or name
 756 if given) in the "dlcache/" folder.
 757 ====================
 758 */
 759 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 760 {
 761         if(!curl_dll)
 762         {
 763                 return false;
 764         }
 765         else
 766         {
 767                 char fn[MAX_OSPATH];
 768                 char urlbuf[1024];
 769                 const char *p, *q;
 770                 size_t length;
 771                 downloadinfo *di;
 772
 773                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
 774                 p = strchr(URL, ':');
 775                 if(p)
 776                 {
 777                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
 778                         {
 779                                 char addressstring[128];
 780                                 *addressstring = 0;
 781                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
 782                                 q = strchr(addressstring, ':');
 783                                 if(!q)
 784                                         q = addressstring + strlen(addressstring);
 785                                 if(*addressstring)
 786                                 {
 787                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
 788                                         URL = urlbuf;
 789                                 }
 790                         }
 791                 }
 792
 793                 // Note: This extraction of the file name portion is NOT entirely correct.
 794                 //
 795                 // It does the following:
 796                 //
 797                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 798                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 799                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 800                 //
 801                 // However, I'd like to keep this "buggy" behavior so that PHP script
 802                 // authors can write download scripts without having to enable
 803                 // AcceptPathInfo on Apache. They just have to ensure that their script
 804                 // can be called with such a "fake" path name like
 805                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 806                 //
 807                 // By the way, such PHP scripts should either send the file or a
 808                 // "Location:" redirect; PHP code example:
 809                 //
 810                 //   header("Location: http://www.example.com/");
 811                 //
 812                 // By the way, this will set User-Agent to something like
 813                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 814                 // dp://serverhost:serverport/ so you can filter on this; an example
 815                 // httpd log file line might be:
 816                 //
 817                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 818
 819                 if(!name)
 820                         name = CleanURL(URL, urlbuf, sizeof(urlbuf));
 821
 822                 if (curl_mutex) Thread_LockMutex(curl_mutex);
 823
 824                 if(!buf)
 825                 {
 826                         p = strrchr(name, '/');
 827                         p = p ? (p+1) : name;
 828                         q = strchr(p, '?');
 829                         length = q ? (size_t)(q - p) : strlen(p);
 830                         dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 831
 832                         name = fn; // make it point back
 833
 834                         // already downloading the file?
 835                         {
 836                                 downloadinfo *di = Curl_Find(fn);
 837                                 if(di)
 838                                 {
 839                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url, urlbuf, sizeof(urlbuf)));
 840
 841                                         // however, if it was not for this map yet...
 842                                         if(forthismap && !di->forthismap)
 843                                         {
 844                                                 di->forthismap = true;
 845                                                 // this "fakes" a download attempt so the client will wait for
 846                                                 // the download to finish and then reconnect
 847                                                 ++numdownloads_added;
 848                                         }
 849
 850                                         return false;
 851                                 }
 852                         }
 853
 854                         if(ispak && FS_FileExists(fn))
 855                         {
 856                                 qboolean already_loaded;
 857                                 if(FS_AddPack(fn, &already_loaded, true))
 858                                 {
 859                                         Con_DPrintf("%s already exists, not downloading!\n", fn);
 860                                         if(already_loaded)
 861                                                 Con_DPrintf("(pak was already loaded)\n");
 862                                         else
 863                                         {
 864                                                 if(forthismap)
 865                                                 {
 866                                                         ++numdownloads_added;
 867                                                         ++numdownloads_success;
 868                                                 }
 869                                         }
 870
 871                                         return false;
 872                                 }
 873                                 else
 874                                 {
 875                                         qfile_t *f = FS_OpenRealFile(fn, "rb", false);
 876                                         if(f)
 877                                         {
 878                                                 char buf[4] = {0};
 879                                                 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 880
 881                                                 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 882                                                 {
 883                                                         Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 884                                                         FS_Close(f);
 885                                                         f = FS_OpenRealFile(fn, "wb", false);
 886                                                         if(f)
 887                                                                 FS_Close(f);
 888                                                 }
 889                                                 else
 890                                                 {
 891                                                         // OK
 892                                                         FS_Close(f);
 893                                                 }
 894                                         }
 895                                 }
 896                         }
 897                 }
 898
 899                 // if we get here, we actually want to download... so first verify the
 900                 // URL scheme (so one can't read local files using file://)
 901                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
 902                 {
 903                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
 904                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 905                         return false;
 906                 }
 907
 908                 if(forthismap)
 909                         ++numdownloads_added;
 910                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 911                 strlcpy(di->filename, name, sizeof(di->filename));
 912                 strlcpy(di->url, URL, sizeof(di->url));
 913                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 914                 di->forthismap = forthismap;
 915                 di->stream = NULL;
 916                 di->startpos = 0;
 917                 di->curle = NULL;
 918                 di->started = false;
 919                 di->ispak = (ispak && !buf);
 920                 di->maxspeed = maxspeed;
 921                 di->bytes_received = 0;
 922                 di->bytes_received_curl = 0;
 923                 di->bytes_sent_curl = 0;
 924                 di->extraheaders = extraheaders;
 925                 di->next = downloads;
 926                 di->prev = NULL;
 927                 if(di->next)
 928                         di->next->prev = di;
 929
 930                 di->buffer = buf;
 931                 di->buffersize = bufsize;
 932                 if(callback == NULL)
 933                 {
 934                         di->callback = curl_default_callback;
 935                         di->callback_data = di;
 936                 }
 937                 else
 938                 {
 939                         di->callback = callback;
 940                         di->callback_data = cbdata;
 941                 }
 942
 943                 if(post_content_type)
 944                 {
 945                         di->post_content_type = post_content_type;
 946                         di->postbuf = postbuf;
 947                         di->postbufsize = postbufsize;
 948                 }
 949                 else
 950                 {
 951                         di->post_content_type = NULL;
 952                         di->postbuf = NULL;
 953                         di->postbufsize = 0;
 954                 }
 955
 956                 downloads = di;
 957                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
 958                 return true;
 959         }
 960 }
 961
 962 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
 963 {
 964         return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
 965 }
 966 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 967 {
 968         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
 969 }
 970 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
 971 {
 972         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
 973 }
 974
 975 /*
 976 ====================
 977 Curl_Run
 978
 979 call this regularily as this will always download as much as possible without
 980 blocking.
 981 ====================
 982 */
 983 void Curl_Run(void)
 984 {
 985         double maxspeed;
 986         downloadinfo *di;
 987
 988         noclear = FALSE;
 989
 990         if(!cl_curl_enabled.integer)
 991                 return;
 992
 993         if(!curl_dll)
 994                 return;
 995
 996         if (curl_mutex) Thread_LockMutex(curl_mutex);
 997
 998         Curl_CheckCommandWhenDone();
 999
1000         if(!downloads)
1001         {
1002                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1003                 return;
1004         }
1005
1006         if(realtime < curltime) // throttle
1007         {
1008                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1009                 return;
1010         }
1011
1012         {
1013                 int remaining;
1014                 CURLMcode mc;
1015
1016                 do
1017                 {
1018                         mc = qcurl_multi_perform(curlm, &remaining);
1019                 }
1020                 while(mc == CURLM_CALL_MULTI_PERFORM);
1021
1022                 for(di = downloads; di; di = di->next)
1023                 {
1024                         double b = 0;
1025                         if(di->curle)
1026                         {
1027                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1028                                 bytes_sent += (b - di->bytes_sent_curl);
1029                                 di->bytes_sent_curl = b;
1030                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1031                                 bytes_sent += (b - di->bytes_received_curl);
1032                                 di->bytes_received_curl = b;
1033                         }
1034                 }
1035
1036                 for(;;)
1037                 {
1038                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1039                         if(!msg)
1040                                 break;
1041                         if(msg->msg == CURLMSG_DONE)
1042                         {
1043                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1044                                 CURLcode result;
1045                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1046                                 result = msg->data.result;
1047                                 if(result)
1048                                 {
1049                                         failed = CURL_DOWNLOAD_FAILED;
1050                                 }
1051                                 else
1052                                 {
1053                                         long code;
1054                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1055                                         switch(code / 100)
1056                                         {
1057                                                 case 4: // e.g. 404?
1058                                                 case 5: // e.g. 500?
1059                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1060                                                         result = (CURLcode) code;
1061                                                         break;
1062                                         }
1063                                 }
1064
1065                                 Curl_EndDownload(di, failed, result);
1066                         }
1067                 }
1068         }
1069
1070         CheckPendingDownloads();
1071
1072         // when will we curl the next time?
1073         // we will wait a bit to ensure our download rate is kept.
1074         // we now know that realtime >= curltime... so set up a new curltime
1075
1076         // use the slowest allowing download to derive the maxspeed... this CAN
1077         // be done better, but maybe later
1078         maxspeed = cl_curl_maxspeed.value;
1079         for(di = downloads; di; di = di->next)
1080                 if(di->maxspeed > 0)
1081                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1082                                 maxspeed = di->maxspeed;
1083
1084         if(maxspeed > 0)
1085         {
1086                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1087                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1088                 bytes_sent = 0;
1089                 bytes_received = 0;
1090         }
1091         else
1092                 curltime = realtime;
1093
1094         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1095 }
1096
1097 /*
1098 ====================
1099 Curl_CancelAll
1100
1101 Stops ALL downloads.
1102 ====================
1103 */
1104 void Curl_CancelAll(void)
1105 {
1106         if(!curl_dll)
1107                 return;
1108
1109         if (curl_mutex) Thread_LockMutex(curl_mutex);
1110
1111         while(downloads)
1112         {
1113                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1114                 // INVARIANT: downloads will point to the next download after that!
1115         }
1116
1117         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1118 }
1119
1120 /*
1121 ====================
1122 Curl_Running
1123
1124 returns true iff there is a download running.
1125 ====================
1126 */
1127 qboolean Curl_Running(void)
1128 {
1129         if(!curl_dll)
1130                 return false;
1131
1132         return downloads != NULL;
1133 }
1134
1135 /*
1136 ====================
1137 Curl_GetDownloadAmount
1138
1139 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1140 for the given download.
1141 ====================
1142 */
1143 static double Curl_GetDownloadAmount(downloadinfo *di)
1144 {
1145         if(!curl_dll)
1146                 return -2;
1147         if(di->curle)
1148         {
1149                 double length;
1150                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1151                 if(length > 0)
1152                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1153                 else
1154                         return 0;
1155         }
1156         else
1157                 return -1;
1158 }
1159
1160 /*
1161 ====================
1162 Curl_GetDownloadSpeed
1163
1164 returns the speed of the given download in bytes per second
1165 ====================
1166 */
1167 static double Curl_GetDownloadSpeed(downloadinfo *di)
1168 {
1169         if(!curl_dll)
1170                 return -2;
1171         if(di->curle)
1172         {
1173                 double speed;
1174                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1175                 return speed;
1176         }
1177         else
1178                 return -1;
1179 }
1180
1181 /*
1182 ====================
1183 Curl_Info_f
1184
1185 prints the download list
1186 ====================
1187 */
1188 // TODO rewrite using Curl_GetDownloadInfo?
1189 static void Curl_Info_f(void)
1190 {
1191         downloadinfo *di;
1192         char urlbuf[1024];
1193         if(!curl_dll)
1194                 return;
1195         if(Curl_Running())
1196         {
1197                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1198                 Con_Print("Currently running downloads:\n");
1199                 for(di = downloads; di; di = di->next)
1200                 {
1201                         double speed, percent;
1202                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1203                         percent = 100.0 * Curl_GetDownloadAmount(di);
1204                         speed = Curl_GetDownloadSpeed(di);
1205                         if(percent >= 0)
1206                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1207                         else
1208                                 Con_Print("(queued)\n");
1209                 }
1210                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1211         }
1212         else
1213         {
1214                 Con_Print("No downloads running.\n");
1215         }
1216 }
1217
1218 /*
1219 ====================
1220 Curl_Curl_f
1221
1222 implements the "curl" console command
1223
1224 curl --info
1225 curl --cancel
1226 curl --cancel filename
1227 curl url
1228
1229 For internal use:
1230
1231 curl [--pak] [--forthismap] [--for filename filename...] url
1232         --pak: after downloading, load the package into the virtual file system
1233         --for filename...: only download of at least one of the named files is missing
1234         --forthismap: don't reconnect on failure
1235
1236 curl --clear_autodownload
1237         clears the download success/failure counters
1238
1239 curl --finish_autodownload
1240         if at least one download has been started, disconnect and drop to the menu
1241         once the last download completes successfully, reconnect to the current server
1242 ====================
1243 */
1244 static void Curl_Curl_f(void)
1245 {
1246         double maxspeed = 0;
1247         int i;
1248         int end;
1249         qboolean pak = false;
1250         qboolean forthismap = false;
1251         const char *url;
1252         const char *name = 0;
1253
1254         if(!curl_dll)
1255         {
1256                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1257                 return;
1258         }
1259
1260         if(!cl_curl_enabled.integer)
1261         {
1262                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1263                 return;
1264         }
1265
1266         if(Cmd_Argc() < 2)
1267         {
1268                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1269                 return;
1270         }
1271
1272         url = Cmd_Argv(Cmd_Argc() - 1);
1273         end = Cmd_Argc();
1274
1275         for(i = 1; i != end; ++i)
1276         {
1277                 const char *a = Cmd_Argv(i);
1278                 if(!strcmp(a, "--info"))
1279                 {
1280                         Curl_Info_f();
1281                         return;
1282                 }
1283                 else if(!strcmp(a, "--cancel"))
1284                 {
1285                         if(i == end - 1) // last argument
1286                                 Curl_CancelAll();
1287                         else
1288                         {
1289                                 downloadinfo *di = Curl_Find(url);
1290                                 if(di)
1291                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1292                                 else
1293                                         Con_Print("download not found\n");
1294                         }
1295                         return;
1296                 }
1297                 else if(!strcmp(a, "--pak"))
1298                 {
1299                         pak = true;
1300                 }
1301                 else if(!strcmp(a, "--for")) // must be last option
1302                 {
1303                         for(i = i + 1; i != end - 1; ++i)
1304                         {
1305                                 if(!FS_FileExists(Cmd_Argv(i)))
1306                                         goto needthefile; // why can't I have a "double break"?
1307                         }
1308                         // if we get here, we have all the files...
1309                         return;
1310                 }
1311                 else if(!strcmp(a, "--forthismap"))
1312                 {
1313                         forthismap = true;
1314                 }
1315                 else if(!strcmp(a, "--as"))
1316                 {
1317                         if(i < end - 1)
1318                         {
1319                                 ++i;
1320                                 name = Cmd_Argv(i);
1321                         }
1322                 }
1323                 else if(!strcmp(a, "--clear_autodownload"))
1324                 {
1325                         // mark all running downloads as "not for this map", so if they
1326                         // fail, it does not matter
1327                         Curl_Clear_forthismap();
1328                         return;
1329                 }
1330                 else if(!strcmp(a, "--finish_autodownload"))
1331                 {
1332                         if(numdownloads_added)
1333                         {
1334                                 char donecommand[256];
1335                                 if(cls.netcon)
1336                                 {
1337                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1338                                         {
1339                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1340                                                 Curl_CommandWhenDone(donecommand);
1341                                                 noclear = TRUE;
1342                                                 CL_Disconnect();
1343                                                 noclear = FALSE;
1344                                                 Curl_CheckCommandWhenDone();
1345                                         }
1346                                         else
1347                                                 Curl_Register_predownload();
1348                                 }
1349                         }
1350                         return;
1351                 }
1352                 else if(!strncmp(a, "--maxspeed=", 11))
1353                 {
1354                         maxspeed = atof(a + 11);
1355                 }
1356                 else if(*a == '-')
1357                 {
1358                         Con_Printf("curl: invalid option %s\n", a);
1359                         // but we ignore the option
1360                 }
1361         }
1362
1363 needthefile:
1364         Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1365 }
1366
1367 /*
1368 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1369 {
1370         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1371         Z_Free(buffer);
1372 }
1373
1374 void Curl_CurlCat_f(void)
1375 {
1376         unsigned char *buf;
1377         const char *url = Cmd_Argv(1);
1378         buf = Z_Malloc(16384);
1379         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1380 }
1381 */
1382
1383 /*
1384 ====================
1385 Curl_Init_Commands
1386
1387 loads the commands and cvars this library uses
1388 ====================
1389 */
1390 void Curl_Init_Commands(void)
1391 {
1392         Cvar_RegisterVariable (&cl_curl_enabled);
1393         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1394         Cvar_RegisterVariable (&cl_curl_maxspeed);
1395         Cvar_RegisterVariable (&sv_curl_defaulturl);
1396         Cvar_RegisterVariable (&sv_curl_serverpackages);
1397         Cvar_RegisterVariable (&sv_curl_maxspeed);
1398         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1399         //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1400 }
1401
1402 /*
1403 ====================
1404 Curl_GetDownloadInfo
1405
1406 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1407 The number of elements in the array is returned in int *nDownloads.
1408 const char **additional_info may be set to a string of additional user
1409 information, or to NULL if no such display shall occur. The returned
1410 array must be freed later using Z_Free.
1411 ====================
1412 */
1413 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1414 {
1415         int i;
1416         downloadinfo *di;
1417         Curl_downloadinfo_t *downinfo;
1418
1419         if(!curl_dll)
1420         {
1421                 *nDownloads = 0;
1422                 if(additional_info)
1423                         *additional_info = NULL;
1424                 return NULL;
1425         }
1426
1427         if (curl_mutex) Thread_LockMutex(curl_mutex);
1428
1429         i = 0;
1430         for(di = downloads; di; di = di->next)
1431                 ++i;
1432
1433         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1434         i = 0;
1435         for(di = downloads; di; di = di->next)
1436         {
1437                 // do not show infobars for background downloads
1438                 if(developer.integer <= 0)
1439                         if(di->buffer)
1440                                 continue;
1441                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1442                 if(di->curle)
1443                 {
1444                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1445                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1446                         downinfo[i].queued = false;
1447                 }
1448                 else
1449                 {
1450                         downinfo[i].queued = true;
1451                 }
1452                 ++i;
1453         }
1454
1455         if(additional_info)
1456         {
1457                 // TODO: can I clear command_when_done as soon as the first download fails?
1458                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1459                 {
1460                         if(!strncmp(command_when_done, "connect ", 8))
1461                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1462                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1463                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1464                         else
1465                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1466                         *additional_info = addinfo;
1467                 }
1468                 else
1469                         *additional_info = NULL;
1470         }
1471
1472         *nDownloads = i;
1473         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1474         return downinfo;
1475 }
1476
1477
1478 /*
1479 ====================
1480 Curl_FindPackURL
1481
1482 finds the URL where to find a given package.
1483
1484 For this, it reads a file "curl_urls.txt" of the following format:
1485
1486         data*.pk3       -
1487         revdm*.pk3      http://revdm/downloads/are/here/
1488         *                       http://any/other/stuff/is/here/
1489
1490 The URLs should end in /. If not, downloads will still work, but the cached files
1491 can't be just put into the data directory with the same download configuration
1492 (you might want to do this if you want to tag downloaded files from your
1493 server, but you should not). "-" means "don't download".
1494
1495 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1496 location instead.
1497
1498 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1499 this file for obvious reasons.
1500 ====================
1501 */
1502 static const char *Curl_FindPackURL(const char *filename)
1503 {
1504         static char foundurl[1024]; // invoked only by server
1505         fs_offset_t filesize;
1506         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1507         if(buf && filesize)
1508         {
1509                 // read lines of format "pattern url"
1510                 char *p = buf;
1511                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1512                 qboolean eof = false;
1513
1514                 pattern = p;
1515                 while(!eof)
1516                 {
1517                         switch(*p)
1518                         {
1519                                 case 0:
1520                                         eof = true;
1521                                         // fallthrough
1522                                 case '\n':
1523                                 case '\r':
1524                                         if(pattern && url && patternend)
1525                                         {
1526                                                 if(!urlend)
1527                                                         urlend = p;
1528                                                 *patternend = 0;
1529                                                 *urlend = 0;
1530                                                 if(matchpattern(filename, pattern, true))
1531                                                 {
1532                                                         strlcpy(foundurl, url, sizeof(foundurl));
1533                                                         Z_Free(buf);
1534                                                         return foundurl;
1535                                                 }
1536                                         }
1537                                         pattern = NULL;
1538                                         patternend = NULL;
1539                                         url = NULL;
1540                                         urlend = NULL;
1541                                         break;
1542                                 case ' ':
1543                                 case '\t':
1544                                         if(pattern && !patternend)
1545                                                 patternend = p;
1546                                         else if(url && !urlend)
1547                                                 urlend = p;
1548                                         break;
1549                                 default:
1550                                         if(!pattern)
1551                                                 pattern = p;
1552                                         else if(pattern && patternend && !url)
1553                                                 url = p;
1554                                         break;
1555                         }
1556                         ++p;
1557                 }
1558         }
1559         if(buf)
1560                 Z_Free(buf);
1561         return sv_curl_defaulturl.string;
1562 }
1563
1564 typedef struct requirement_s
1565 {
1566         struct requirement_s *next;
1567         char filename[MAX_OSPATH];
1568 }
1569 requirement;
1570 static requirement *requirements = NULL;
1571
1572
1573 /*
1574 ====================
1575 Curl_RequireFile
1576
1577 Adds the given file to the list of requirements.
1578 ====================
1579 */
1580 void Curl_RequireFile(const char *filename)
1581 {
1582         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1583         req->next = requirements;
1584         strlcpy(req->filename, filename, sizeof(req->filename));
1585         requirements = req;
1586 }
1587
1588 /*
1589 ====================
1590 Curl_ClearRequirements
1591
1592 Clears the list of required files for playing on the current map.
1593 This should be called at every map change.
1594 ====================
1595 */
1596 void Curl_ClearRequirements(void)
1597 {
1598         while(requirements)
1599         {
1600                 requirement *req = requirements;
1601                 requirements = requirements->next;
1602                 Z_Free(req);
1603         }
1604 }
1605
1606 /*
1607 ====================
1608 Curl_SendRequirements
1609
1610 Makes the current host_clients download all files he needs.
1611 This is done by sending him the following console commands:
1612
1613         curl --clear_autodownload
1614         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1615         curl --finish_autodownload
1616 ====================
1617 */
1618 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1619 {
1620         const char *p;
1621         const char *thispack = FS_WhichPack(filename);
1622         const char *packurl;
1623
1624         if(!thispack)
1625                 return false;
1626
1627         p = strrchr(thispack, '/');
1628         if(p)
1629                 thispack = p + 1;
1630
1631         packurl = Curl_FindPackURL(thispack);
1632
1633         if(packurl && *packurl && strcmp(packurl, "-"))
1634         {
1635                 if(!foundone)
1636                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1637
1638                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1639                 strlcat(sendbuffer, thispack, sendbuffer_len);
1640                 if(sv_curl_maxspeed.value > 0)
1641                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1642                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1643                 strlcat(sendbuffer, filename, sendbuffer_len);
1644                 strlcat(sendbuffer, " ", sendbuffer_len);
1645                 strlcat(sendbuffer, packurl, sendbuffer_len);
1646                 strlcat(sendbuffer, thispack, sendbuffer_len);
1647                 strlcat(sendbuffer, "\n", sendbuffer_len);
1648
1649                 return true;
1650         }
1651
1652         return false;
1653 }
1654 void Curl_SendRequirements(void)
1655 {
1656         // for each requirement, find the pack name
1657         char sendbuffer[4096] = "";
1658         requirement *req;
1659         qboolean foundone = false;
1660         const char *p;
1661
1662         for(req = requirements; req; req = req->next)
1663                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1664
1665         p = sv_curl_serverpackages.string;
1666         while(COM_ParseToken_Simple(&p, false, false))
1667                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1668
1669         if(foundone)
1670                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1671
1672         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1673                 Host_ClientCommands("%s", sendbuffer);
1674         else
1675                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1676 }