libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  10
  11 /*
  12 =================================================================
  13
  14   Minimal set of definitions from libcurl
  15
  16   WARNING: for a matter of simplicity, several pointer types are
  17   casted to "void*", and most enumerated values are not included
  18
  19 =================================================================
  20 */
  21
  22 typedef struct CURL_s CURL;
  23 typedef struct CURLM_s CURLM;
  24 typedef enum
  25 {
  26         CURLE_OK = 0
  27 }
  28 CURLcode;
  29 typedef enum
  30 {
  31         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  32         CURLM_OK = 0
  33 }
  34 CURLMcode;
  35 #define CURL_GLOBAL_NOTHING 0
  36 #define CURL_GLOBAL_SSL 1
  37 #define CURL_GLOBAL_WIN32 2
  38 #define CURLOPTTYPE_LONG          0
  39 #define CURLOPTTYPE_OBJECTPOINT   10000
  40 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  41 #define CURLOPTTYPE_OFF_T         30000
  42 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  43 typedef enum
  44 {
  45         CINIT(WRITEDATA, OBJECTPOINT, 1),
  46         CINIT(URL,  OBJECTPOINT, 2),
  47         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  48         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  49         CINIT(REFERER, OBJECTPOINT, 16),
  50         CINIT(USERAGENT, OBJECTPOINT, 18),
  51         CINIT(RESUME_FROM, LONG, 21),
  52         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  53         CINIT(PRIVATE, OBJECTPOINT, 103),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56 }
  57 CURLoption;
  58 typedef enum
  59 {
  60         CURLINFO_TEXT = 0,
  61         CURLINFO_HEADER_IN,    /* 1 */
  62         CURLINFO_HEADER_OUT,   /* 2 */
  63         CURLINFO_DATA_IN,      /* 3 */
  64         CURLINFO_DATA_OUT,     /* 4 */
  65         CURLINFO_SSL_DATA_IN,  /* 5 */
  66         CURLINFO_SSL_DATA_OUT, /* 6 */
  67         CURLINFO_END
  68 }
  69 curl_infotype;
  70 #define CURLINFO_STRING   0x100000
  71 #define CURLINFO_LONG     0x200000
  72 #define CURLINFO_DOUBLE   0x300000
  73 #define CURLINFO_SLIST    0x400000
  74 #define CURLINFO_MASK     0x0fffff
  75 #define CURLINFO_TYPEMASK 0xf00000
  76 typedef enum
  77 {
  78         CURLINFO_NONE, /* first, never use this */
  79         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  80         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  81         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  82         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  83         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  84         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  85         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  86         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  87         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  88         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  89         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  90         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  91         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  92         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  93         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  94         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  95         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  96         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  97         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  98         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  99         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 100         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 101         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 102         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 103         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 104         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 105         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 106 }
 107 CURLINFO;
 108
 109 typedef enum
 110 {
 111         CURLMSG_NONE, /* first, not used */
 112         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 113                                          the CURLcode of the transfer */
 114         CURLMSG_LAST
 115 }
 116 CURLMSG;
 117 typedef struct
 118 {
 119         CURLMSG msg;       /* what this message means */
 120         CURL *easy_handle; /* the handle it concerns */
 121         union
 122         {
 123                 void *whatever;    /* message-specific data */
 124                 CURLcode result;   /* return code for transfer */
 125         }
 126         data;
 127 }
 128 CURLMsg;
 129
 130 static void (*qcurl_global_init) (long flags);
 131 static void (*qcurl_global_cleanup) ();
 132
 133 static CURL * (*qcurl_easy_init) ();
 134 static void (*qcurl_easy_cleanup) (CURL *handle);
 135 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 136 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 137 static const char * (*qcurl_easy_strerror) (CURLcode);
 138
 139 static CURLM * (*qcurl_multi_init) ();
 140 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 141 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 142 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 143 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 144 static void (*qcurl_multi_cleanup) (CURLM *);
 145 static const char * (*qcurl_multi_strerror) (CURLcode);
 146
 147 static dllfunction_t curlfuncs[] =
 148 {
 149         {"curl_global_init",            (void **) &qcurl_global_init},
 150         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 151         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 152         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 153         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 154         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 155         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 156         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 157         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 158         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 159         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 160         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 161         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 162         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 163         {NULL, NULL}
 164 };
 165
 166 // Handle for CURL DLL
 167 static dllhandle_t curl_dll = NULL;
 168 // will be checked at many places to find out if qcurl calls are allowed
 169
 170 typedef struct downloadinfo_s
 171 {
 172         char filename[MAX_QPATH];
 173         char url[256];
 174         char referer[256];
 175         qfile_t *stream;
 176         fs_offset_t startpos;
 177         CURL *curle;
 178         qboolean started;
 179         qboolean ispak;
 180         unsigned long bytes_received;
 181         struct downloadinfo_s *next, *prev;
 182         qboolean forthismap;
 183 }
 184 downloadinfo;
 185 static downloadinfo *downloads = NULL;
 186 static int numdownloads = 0;
 187
 188 static qboolean noclear = FALSE;
 189
 190 static int numdownloads_fail = 0;
 191 static int numdownloads_success = 0;
 192 static int numdownloads_added = 0;
 193 static char command_when_done[256] = "";
 194 static char command_when_error[256] = "";
 195
 196 /*
 197 ====================
 198 Curl_CommandWhenDone
 199
 200 Sets the command which is to be executed when the last download completes AND
 201 all downloads since last server connect ended with a successful status.
 202 Setting the command to NULL clears it.
 203 ====================
 204 */
 205 void Curl_CommandWhenDone(const char *cmd)
 206 {
 207         if(!curl_dll)
 208                 return;
 209         if(cmd)
 210                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 211         else
 212                 *command_when_done = 0;
 213 }
 214
 215 /*
 216 FIXME
 217 Do not use yet. Not complete.
 218 Problem: what counts as an error?
 219 */
 220
 221 void Curl_CommandWhenError(const char *cmd)
 222 {
 223         if(!curl_dll)
 224                 return;
 225         if(cmd)
 226                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 227         else
 228                 *command_when_error = 0;
 229 }
 230
 231 /*
 232 ====================
 233 Curl_Clear_forthismap
 234
 235 Clears the "will disconnect on failure" flags.
 236 ====================
 237 */
 238 void Curl_Clear_forthismap()
 239 {
 240         downloadinfo *di;
 241         if(noclear)
 242                 return;
 243         for(di = downloads; di; di = di->next)
 244                 di->forthismap = false;
 245         Curl_CommandWhenError(NULL);
 246         Curl_CommandWhenDone(NULL);
 247         numdownloads_fail = 0;
 248         numdownloads_success = 0;
 249         numdownloads_added = 0;
 250 }
 251
 252 /*
 253 ====================
 254 Curl_Have_forthismap
 255
 256 Returns true if a download needed for the current game is running.
 257 ====================
 258 */
 259 qboolean Curl_Have_forthismap()
 260 {
 261         return numdownloads_added;
 262 }
 263
 264 void Curl_Register_predownload()
 265 {
 266         Curl_CommandWhenDone("cl_begindownloads");
 267         Curl_CommandWhenError("cl_begindownloads");
 268 }
 269
 270 /*
 271 ====================
 272 Curl_CheckCommandWhenDone
 273
 274 Checks if a "done command" is to be executed.
 275 All downloads finished, at least one success since connect, no single failure
 276 -> execute the command.
 277 */
 278 static void Curl_CheckCommandWhenDone()
 279 {
 280         if(!curl_dll)
 281                 return;
 282         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 283         {
 284                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 285                 Cbuf_AddText("\n");
 286                 Cbuf_AddText(command_when_done);
 287                 Cbuf_AddText("\n");
 288                 Curl_Clear_forthismap();
 289         }
 290         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 291         {
 292                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 293                 Cbuf_AddText("\n");
 294                 Cbuf_AddText(command_when_error);
 295                 Cbuf_AddText("\n");
 296                 Curl_Clear_forthismap();
 297         }
 298 }
 299
 300 /*
 301 ====================
 302 CURL_CloseLibrary
 303
 304 Load the cURL DLL
 305 ====================
 306 */
 307 static qboolean CURL_OpenLibrary (void)
 308 {
 309         const char* dllnames [] =
 310         {
 311 #if defined(WIN64)
 312                 "libcurl64.dll",
 313 #elif defined(WIN32)
 314                 "libcurl-4.dll",
 315                 "libcurl-3.dll",
 316 #elif defined(MACOSX)
 317                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 318                 "libcurl.3.dylib", // Mac OS X Tiger
 319                 "libcurl.2.dylib", // Mac OS X Panther
 320 #else
 321                 "libcurl.so.4",
 322                 "libcurl.so.3",
 323                 "libcurl.so", // FreeBSD
 324 #endif
 325                 NULL
 326         };
 327
 328         // Already loaded?
 329         if (curl_dll)
 330                 return true;
 331
 332         // Load the DLL
 333         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
 334 }
 335
 336
 337 /*
 338 ====================
 339 CURL_CloseLibrary
 340
 341 Unload the cURL DLL
 342 ====================
 343 */
 344 static void CURL_CloseLibrary (void)
 345 {
 346         Sys_UnloadLibrary (&curl_dll);
 347 }
 348
 349
 350 static CURLM *curlm = NULL;
 351 static unsigned long bytes_received = 0; // used for bandwidth throttling
 352 static double curltime = 0;
 353
 354 /*
 355 ====================
 356 CURL_fwrite
 357
 358 fwrite-compatible function that writes the data to a file. libcurl can call
 359 this.
 360 ====================
 361 */
 362 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 363 {
 364         fs_offset_t ret;
 365         size_t bytes = size * nmemb;
 366         downloadinfo *di = (downloadinfo *) vdi;
 367
 368         bytes_received += bytes;
 369         di->bytes_received += bytes;
 370
 371         ret = FS_Write(di->stream, data, bytes);
 372
 373         return ret; // why not ret / nmemb?
 374 }
 375
 376 typedef enum
 377 {
 378         CURL_DOWNLOAD_SUCCESS = 0,
 379         CURL_DOWNLOAD_FAILED,
 380         CURL_DOWNLOAD_ABORTED,
 381         CURL_DOWNLOAD_SERVERERROR
 382 }
 383 CurlStatus;
 384
 385 /*
 386 ====================
 387 Curl_EndDownload
 388
 389 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 390 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 391 code from libcurl, or 0, if another error has occurred.
 392 ====================
 393 */
 394 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 395 {
 396         qboolean ok = false;
 397         if(!curl_dll)
 398                 return;
 399         switch(status)
 400         {
 401                 case CURL_DOWNLOAD_SUCCESS:
 402                         Con_Printf("Download of %s: OK\n", di->filename);
 403                         ok = true;
 404                         break;
 405                 case CURL_DOWNLOAD_FAILED:
 406                         Con_Printf("Download of %s: FAILED\n", di->filename);
 407                         if(error)
 408                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 409                         break;
 410                 case CURL_DOWNLOAD_ABORTED:
 411                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 412                         break;
 413                 case CURL_DOWNLOAD_SERVERERROR:
 414                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 415
 416                         // reopen to enforce it to have zero bytes again
 417                         FS_Close(di->stream);
 418                         di->stream = FS_Open(di->filename, "w", false, false);
 419
 420                         break;
 421         }
 422
 423         if(di->curle)
 424         {
 425                 qcurl_multi_remove_handle(curlm, di->curle);
 426                 qcurl_easy_cleanup(di->curle);
 427         }
 428
 429         if(ok && !di->bytes_received)
 430         {
 431                 Con_Printf("ERROR: empty file\n");
 432                 ok = false;
 433         }
 434
 435         if(di->stream)
 436                 FS_Close(di->stream);
 437
 438         if(ok && di->ispak)
 439                 ok = FS_AddPack(di->filename, NULL, true);
 440
 441         if(di->prev)
 442                 di->prev->next = di->next;
 443         else
 444                 downloads = di->next;
 445         if(di->next)
 446                 di->next->prev = di->prev;
 447
 448         --numdownloads;
 449         if(di->forthismap)
 450         {
 451                 if(ok)
 452                         ++numdownloads_success;
 453                 else
 454                         ++numdownloads_fail;
 455         }
 456         Z_Free(di);
 457 }
 458
 459 /*
 460 ====================
 461 CheckPendingDownloads
 462
 463 checks if there are free download slots to start new downloads in.
 464 To not start too many downloads at once, only one download is added at a time,
 465 up to a maximum number of cl_curl_maxdownloads are running.
 466 ====================
 467 */
 468 static void CheckPendingDownloads()
 469 {
 470         if(!curl_dll)
 471                 return;
 472         if(numdownloads < cl_curl_maxdownloads.integer)
 473         {
 474                 downloadinfo *di;
 475                 for(di = downloads; di; di = di->next)
 476                 {
 477                         if(!di->started)
 478                         {
 479                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 480
 481                                 di->stream = FS_Open(di->filename, "ab", false, false);
 482                                 if(!di->stream)
 483                                 {
 484                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 485                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 486                                         return;
 487                                 }
 488
 489                                 FS_Seek(di->stream, 0, SEEK_END);
 490                                 di->startpos = FS_Tell(di->stream);
 491                                 if(di->startpos > 0)
 492                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 493                                 Con_Print("...\n");
 494
 495                                 di->curle = qcurl_easy_init();
 496                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 497                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 498                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 499                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 500                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 501                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 502                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 503                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 504                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 505                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 506                                 qcurl_multi_add_handle(curlm, di->curle);
 507                                 di->started = true;
 508                                 ++numdownloads;
 509                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 510                                         break;
 511                         }
 512                 }
 513         }
 514 }
 515
 516 /*
 517 ====================
 518 Curl_Init
 519
 520 this function MUST be called before using anything else in this file.
 521 On Win32, this must be called AFTER WSAStartup has been done!
 522 ====================
 523 */
 524 void Curl_Init()
 525 {
 526         CURL_OpenLibrary();
 527         if(!curl_dll)
 528                 return;
 529         qcurl_global_init(CURL_GLOBAL_NOTHING);
 530         curlm = qcurl_multi_init();
 531 }
 532
 533 /*
 534 ====================
 535 Curl_Shutdown
 536
 537 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 538 ====================
 539 */
 540 void Curl_ClearRequirements();
 541 void Curl_Shutdown()
 542 {
 543         if(!curl_dll)
 544                 return;
 545         Curl_ClearRequirements();
 546         Curl_CancelAll();
 547         CURL_CloseLibrary();
 548         curl_dll = NULL;
 549 }
 550
 551 /*
 552 ====================
 553 Curl_Find
 554
 555 Finds the internal information block for a download given by file name.
 556 ====================
 557 */
 558 static downloadinfo *Curl_Find(const char *filename)
 559 {
 560         downloadinfo *di;
 561         if(!curl_dll)
 562                 return NULL;
 563         for(di = downloads; di; di = di->next)
 564                 if(!strcasecmp(di->filename, filename))
 565                         return di;
 566         return NULL;
 567 }
 568
 569 /*
 570 ====================
 571 Curl_Begin
 572
 573 Starts a download of a given URL to the file name portion of this URL (or name
 574 if given) in the "dlcache/" folder.
 575 ====================
 576 */
 577 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 578 {
 579         if(!curl_dll)
 580                 return;
 581         else
 582         {
 583                 char fn[MAX_QPATH];
 584                 const char *p, *q;
 585                 size_t length;
 586                 downloadinfo *di;
 587
 588                 // Note: This extraction of the file name portion is NOT entirely correct.
 589                 //
 590                 // It does the following:
 591                 //
 592                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 593                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 594                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 595                 //
 596                 // However, I'd like to keep this "buggy" behavior so that PHP script
 597                 // authors can write download scripts without having to enable
 598                 // AcceptPathInfo on Apache. They just have to ensure that their script
 599                 // can be called with such a "fake" path name like
 600                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 601                 //
 602                 // By the way, such PHP scripts should either send the file or a
 603                 // "Location:" redirect; PHP code example:
 604                 //
 605                 //   header("Location: http://www.example.com/");
 606                 //
 607                 // By the way, this will set User-Agent to something like
 608                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 609                 // dp://serverhost:serverport/ so you can filter on this; an example
 610                 // httpd log file line might be:
 611                 //
 612                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 613
 614                 if(!name)
 615                         name = URL;
 616                 p = strrchr(name, '/');
 617                 p = p ? (p+1) : name;
 618                 q = strchr(p, '?');
 619                 length = q ? (size_t)(q - p) : strlen(p);
 620                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 621
 622                 // already downloading the file?
 623                 {
 624                         downloadinfo *di = Curl_Find(fn);
 625                         if(di)
 626                         {
 627                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 628
 629                                 // however, if it was not for this map yet...
 630                                 if(forthismap && !di->forthismap)
 631                                 {
 632                                         di->forthismap = true;
 633                                         // this "fakes" a download attempt so the client will wait for
 634                                         // the download to finish and then reconnect
 635                                         ++numdownloads_added;
 636                                 }
 637
 638                                 return;
 639                         }
 640                 }
 641
 642                 if(ispak && FS_FileExists(fn))
 643                 {
 644                         qboolean already_loaded;
 645                         if(FS_AddPack(fn, &already_loaded, true))
 646                         {
 647                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 648                                 if(already_loaded)
 649                                         Con_DPrintf("(pak was already loaded)\n");
 650                                 else
 651                                 {
 652                                         if(forthismap)
 653                                         {
 654                                                 ++numdownloads_added;
 655                                                 ++numdownloads_success;
 656                                         }
 657                                 }
 658                                 return;
 659                         }
 660                         else
 661                         {
 662                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 663                                 if(f)
 664                                 {
 665                                         char buf[4] = {0};
 666                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 667
 668                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 669                                         {
 670                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 671                                                 FS_Close(f);
 672                                                 f = FS_Open(fn, "w", false, false);
 673                                                 if(f)
 674                                                         FS_Close(f);
 675                                         }
 676                                         else
 677                                         {
 678                                                 // OK
 679                                                 FS_Close(f);
 680                                         }
 681                                 }
 682                         }
 683                 }
 684
 685                 if(forthismap)
 686                         ++numdownloads_added;
 687                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 688                 strlcpy(di->filename, fn, sizeof(di->filename));
 689                 strlcpy(di->url, URL, sizeof(di->url));
 690                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 691                 di->forthismap = forthismap;
 692                 di->stream = NULL;
 693                 di->startpos = 0;
 694                 di->curle = NULL;
 695                 di->started = false;
 696                 di->ispak = ispak;
 697                 di->bytes_received = 0;
 698                 di->next = downloads;
 699                 di->prev = NULL;
 700                 if(di->next)
 701                         di->next->prev = di;
 702                 downloads = di;
 703         }
 704 }
 705
 706
 707 /*
 708 ====================
 709 Curl_Run
 710
 711 call this regularily as this will always download as much as possible without
 712 blocking.
 713 ====================
 714 */
 715 void Curl_Run()
 716 {
 717         noclear = FALSE;
 718
 719         if(!cl_curl_enabled.integer)
 720                 return;
 721
 722         if(!curl_dll)
 723                 return;
 724
 725         Curl_CheckCommandWhenDone();
 726
 727         if(!downloads)
 728                 return;
 729
 730         if(realtime < curltime) // throttle
 731                 return;
 732
 733         {
 734                 int remaining;
 735                 CURLMcode mc;
 736
 737                 do
 738                 {
 739                         mc = qcurl_multi_perform(curlm, &remaining);
 740                 }
 741                 while(mc == CURLM_CALL_MULTI_PERFORM);
 742
 743                 for(;;)
 744                 {
 745                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 746                         if(!msg)
 747                                 break;
 748                         if(msg->msg == CURLMSG_DONE)
 749                         {
 750                                 downloadinfo *di;
 751                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 752                                 CURLcode result;
 753                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 754                                 result = msg->data.result;
 755                                 if(result)
 756                                 {
 757                                         failed = CURL_DOWNLOAD_FAILED;
 758                                 }
 759                                 else
 760                                 {
 761                                         long code;
 762                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 763                                         switch(code / 100)
 764                                         {
 765                                                 case 4: // e.g. 404?
 766                                                 case 5: // e.g. 500?
 767                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 768                                                         result = code;
 769                                                         break;
 770                                         }
 771                                 }
 772
 773                                 Curl_EndDownload(di, failed, result);
 774                         }
 775                 }
 776         }
 777
 778         CheckPendingDownloads();
 779
 780         // when will we curl the next time?
 781         // we will wait a bit to ensure our download rate is kept.
 782         // we now know that realtime >= curltime... so set up a new curltime
 783         if(cl_curl_maxspeed.value > 0)
 784         {
 785                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 786                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 787                 bytes_received -= bytes;
 788         }
 789         else
 790                 curltime = realtime;
 791 }
 792
 793 /*
 794 ====================
 795 Curl_CancelAll
 796
 797 Stops ALL downloads.
 798 ====================
 799 */
 800 void Curl_CancelAll()
 801 {
 802         if(!curl_dll)
 803                 return;
 804
 805         while(downloads)
 806         {
 807                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 808                 // INVARIANT: downloads will point to the next download after that!
 809         }
 810 }
 811
 812 /*
 813 ====================
 814 Curl_Running
 815
 816 returns true iff there is a download running.
 817 ====================
 818 */
 819 qboolean Curl_Running()
 820 {
 821         if(!curl_dll)
 822                 return false;
 823
 824         return downloads != NULL;
 825 }
 826
 827 /*
 828 ====================
 829 Curl_GetDownloadAmount
 830
 831 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 832 for the given download.
 833 ====================
 834 */
 835 static double Curl_GetDownloadAmount(downloadinfo *di)
 836 {
 837         if(!curl_dll)
 838                 return -2;
 839         if(di->curle)
 840         {
 841                 double length;
 842                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 843                 if(length > 0)
 844                         return di->bytes_received / length;
 845                 else
 846                         return 0;
 847         }
 848         else
 849                 return -1;
 850 }
 851
 852 /*
 853 ====================
 854 Curl_GetDownloadSpeed
 855
 856 returns the speed of the given download in bytes per second
 857 ====================
 858 */
 859 static double Curl_GetDownloadSpeed(downloadinfo *di)
 860 {
 861         if(!curl_dll)
 862                 return -2;
 863         if(di->curle)
 864         {
 865                 double speed;
 866                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 867                 return speed;
 868         }
 869         else
 870                 return -1;
 871 }
 872
 873 /*
 874 ====================
 875 Curl_Info_f
 876
 877 prints the download list
 878 ====================
 879 */
 880 // TODO rewrite using Curl_GetDownloadInfo?
 881 static void Curl_Info_f()
 882 {
 883         downloadinfo *di;
 884         if(!curl_dll)
 885                 return;
 886         if(Curl_Running())
 887         {
 888                 Con_Print("Currently running downloads:\n");
 889                 for(di = downloads; di; di = di->next)
 890                 {
 891                         double speed, percent;
 892                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 893                         percent = 100.0 * Curl_GetDownloadAmount(di);
 894                         speed = Curl_GetDownloadSpeed(di);
 895                         if(percent >= 0)
 896                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 897                         else
 898                                 Con_Print("(queued)\n");
 899                 }
 900         }
 901         else
 902         {
 903                 Con_Print("No downloads running.\n");
 904         }
 905 }
 906
 907 /*
 908 ====================
 909 Curl_Curl_f
 910
 911 implements the "curl" console command
 912
 913 curl --info
 914 curl --cancel
 915 curl --cancel filename
 916 curl url
 917
 918 For internal use:
 919
 920 curl [--pak] [--forthismap] [--for filename filename...] url
 921         --pak: after downloading, load the package into the virtual file system
 922         --for filename...: only download of at least one of the named files is missing
 923         --forthismap: don't reconnect on failure
 924
 925 curl --clear_autodownload
 926         clears the download success/failure counters
 927
 928 curl --finish_autodownload
 929         if at least one download has been started, disconnect and drop to the menu
 930         once the last download completes successfully, reconnect to the current server
 931 ====================
 932 */
 933 void Curl_Curl_f(void)
 934 {
 935         int i;
 936         int end;
 937         qboolean pak = false;
 938         qboolean forthismap = false;
 939         const char *url;
 940         const char *name = 0;
 941
 942         if(!curl_dll)
 943         {
 944                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 945                 return;
 946         }
 947
 948         if(!cl_curl_enabled.integer)
 949         {
 950                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 951                 return;
 952         }
 953
 954         for(i = 0; i != Cmd_Argc(); ++i)
 955                 Con_DPrintf("%s ", Cmd_Argv(i));
 956         Con_DPrint("\n");
 957
 958         if(Cmd_Argc() < 2)
 959         {
 960                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 961                 return;
 962         }
 963
 964         url = Cmd_Argv(Cmd_Argc() - 1);
 965         end = Cmd_Argc();
 966
 967         for(i = 1; i != end; ++i)
 968         {
 969                 const char *a = Cmd_Argv(i);
 970                 if(!strcmp(a, "--info"))
 971                 {
 972                         Curl_Info_f();
 973                         return;
 974                 }
 975                 else if(!strcmp(a, "--cancel"))
 976                 {
 977                         if(i == end - 1) // last argument
 978                                 Curl_CancelAll();
 979                         else
 980                         {
 981                                 downloadinfo *di = Curl_Find(url);
 982                                 if(di)
 983                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 984                                 else
 985                                         Con_Print("download not found\n");
 986                         }
 987                         return;
 988                 }
 989                 else if(!strcmp(a, "--pak"))
 990                 {
 991                         pak = true;
 992                 }
 993                 else if(!strcmp(a, "--for"))
 994                 {
 995                         for(i = i + 1; i != end - 1; ++i)
 996                         {
 997                                 if(!FS_FileExists(Cmd_Argv(i)))
 998                                         goto needthefile; // why can't I have a "double break"?
 999                         }
1000                         // if we get here, we have all the files...
1001                         return;
1002                 }
1003                 else if(!strcmp(a, "--forthismap"))
1004                 {
1005                         forthismap = true;
1006                 }
1007                 else if(!strcmp(a, "--as"))
1008                 {
1009                         if(i < end - 1)
1010                         {
1011                                 ++i;
1012                                 name = Cmd_Argv(i);
1013                         }
1014                 }
1015                 else if(!strcmp(a, "--clear_autodownload"))
1016                 {
1017                         // mark all running downloads as "not for this map", so if they
1018                         // fail, it does not matter
1019                         Curl_Clear_forthismap();
1020                         return;
1021                 }
1022                 else if(!strcmp(a, "--finish_autodownload"))
1023                 {
1024                         if(numdownloads_added)
1025                         {
1026                                 char donecommand[256];
1027                                 if(cls.netcon)
1028                                 {
1029                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1030                                         {
1031                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1032                                                 Curl_CommandWhenDone(donecommand);
1033                                                 noclear = TRUE;
1034                                                 CL_Disconnect();
1035                                                 noclear = FALSE;
1036                                                 Curl_CheckCommandWhenDone();
1037                                         }
1038                                         else
1039                                                 Curl_Register_predownload();
1040                                 }
1041                         }
1042                         return;
1043                 }
1044                 else if(*a == '-')
1045                 {
1046                         Con_Printf("invalid option %s\n", a);
1047                         return;
1048                 }
1049         }
1050
1051 needthefile:
1052         Curl_Begin(url, name, pak, forthismap);
1053 }
1054
1055 /*
1056 ====================
1057 Curl_Init_Commands
1058
1059 loads the commands and cvars this library uses
1060 ====================
1061 */
1062 void Curl_Init_Commands(void)
1063 {
1064         Cvar_RegisterVariable (&cl_curl_enabled);
1065         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1066         Cvar_RegisterVariable (&cl_curl_maxspeed);
1067         Cvar_RegisterVariable (&sv_curl_defaulturl);
1068         Cvar_RegisterVariable (&sv_curl_serverpackages);
1069         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1070 }
1071
1072 /*
1073 ====================
1074 Curl_GetDownloadInfo
1075
1076 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1077 The number of elements in the array is returned in int *nDownloads.
1078 const char **additional_info may be set to a string of additional user
1079 information, or to NULL if no such display shall occur. The returned
1080 array must be freed later using Z_Free.
1081 ====================
1082 */
1083 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1084 {
1085         int n, i;
1086         downloadinfo *di;
1087         Curl_downloadinfo_t *downinfo;
1088         static char addinfo[128];
1089
1090         if(!curl_dll)
1091         {
1092                 *nDownloads = 0;
1093                 if(additional_info)
1094                         *additional_info = NULL;
1095                 return NULL;
1096         }
1097
1098         n = 0;
1099         for(di = downloads; di; di = di->next)
1100                 ++n;
1101
1102         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
1103         i = 0;
1104         for(di = downloads; di; di = di->next)
1105         {
1106                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1107                 if(di->curle)
1108                 {
1109                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1110                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1111                         downinfo[i].queued = false;
1112                 }
1113                 else
1114                 {
1115                         downinfo[i].queued = true;
1116                 }
1117                 ++i;
1118         }
1119
1120         if(additional_info)
1121         {
1122                 // TODO: can I clear command_when_done as soon as the first download fails?
1123                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1124                 {
1125                         if(!strncmp(command_when_done, "connect ", 8))
1126                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1127                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1128                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1129                         else
1130                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1131                         *additional_info = addinfo;
1132                 }
1133                 else
1134                         *additional_info = NULL;
1135         }
1136
1137         *nDownloads = n;
1138         return downinfo;
1139 }
1140
1141
1142 /*
1143 ====================
1144 Curl_FindPackURL
1145
1146 finds the URL where to find a given package.
1147
1148 For this, it reads a file "curl_urls.txt" of the following format:
1149
1150         data*.pk3       -
1151         revdm*.pk3      http://revdm/downloads/are/here/
1152         *                       http://any/other/stuff/is/here/
1153
1154 The URLs should end in /. If not, downloads will still work, but the cached files
1155 can't be just put into the data directory with the same download configuration
1156 (you might want to do this if you want to tag downloaded files from your
1157 server, but you should not). "-" means "don't download".
1158
1159 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1160 location instead.
1161
1162 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1163 this file for obvious reasons.
1164 ====================
1165 */
1166 static const char *Curl_FindPackURL(const char *filename)
1167 {
1168         static char foundurl[256];
1169         fs_offset_t filesize;
1170         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1171         if(buf && filesize)
1172         {
1173                 // read lines of format "pattern url"
1174                 char *p = buf;
1175                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1176                 qboolean eof = false;
1177
1178                 pattern = p;
1179                 while(!eof)
1180                 {
1181                         switch(*p)
1182                         {
1183                                 case 0:
1184                                         eof = true;
1185                                         // fallthrough
1186                                 case '\n':
1187                                 case '\r':
1188                                         if(pattern && url && patternend)
1189                                         {
1190                                                 if(!urlend)
1191                                                         urlend = p;
1192                                                 *patternend = 0;
1193                                                 *urlend = 0;
1194                                                 if(matchpattern(filename, pattern, true))
1195                                                 {
1196                                                         strlcpy(foundurl, url, sizeof(foundurl));
1197                                                         Z_Free(buf);
1198                                                         return foundurl;
1199                                                 }
1200                                         }
1201                                         pattern = NULL;
1202                                         patternend = NULL;
1203                                         url = NULL;
1204                                         urlend = NULL;
1205                                         break;
1206                                 case ' ':
1207                                 case '\t':
1208                                         if(pattern && !patternend)
1209                                                 patternend = p;
1210                                         else if(url && !urlend)
1211                                                 urlend = p;
1212                                         break;
1213                                 default:
1214                                         if(!pattern)
1215                                                 pattern = p;
1216                                         else if(pattern && patternend && !url)
1217                                                 url = p;
1218                                         break;
1219                         }
1220                         ++p;
1221                 }
1222         }
1223         if(buf)
1224                 Z_Free(buf);
1225         return sv_curl_defaulturl.string;
1226 }
1227
1228 typedef struct requirement_s
1229 {
1230         struct requirement_s *next;
1231         char filename[MAX_QPATH];
1232 }
1233 requirement;
1234 static requirement *requirements = NULL;
1235
1236
1237 /*
1238 ====================
1239 Curl_RequireFile
1240
1241 Adds the given file to the list of requirements.
1242 ====================
1243 */
1244 void Curl_RequireFile(const char *filename)
1245 {
1246         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1247         req->next = requirements;
1248         strlcpy(req->filename, filename, sizeof(req->filename));
1249         requirements = req;
1250 }
1251
1252 /*
1253 ====================
1254 Curl_ClearRequirements
1255
1256 Clears the list of required files for playing on the current map.
1257 This should be called at every map change.
1258 ====================
1259 */
1260 void Curl_ClearRequirements()
1261 {
1262         const char *p;
1263         while(requirements)
1264         {
1265                 requirement *req = requirements;
1266                 requirements = requirements->next;
1267                 Z_Free(req);
1268         }
1269         p = sv_curl_serverpackages.string;
1270         Con_DPrintf("Require all of: %s\n", p);
1271         while(COM_ParseToken_Simple(&p, false, false))
1272         {
1273                 Con_DPrintf("Require: %s\n", com_token);
1274                 Curl_RequireFile(com_token);
1275         }
1276 }
1277
1278 /*
1279 ====================
1280 Curl_SendRequirements
1281
1282 Makes the current host_clients download all files he needs.
1283 This is done by sending him the following console commands:
1284
1285         curl --clear_autodownload
1286         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1287         curl --finish_autodownload
1288 ====================
1289 */
1290 void Curl_SendRequirements()
1291 {
1292         // for each requirement, find the pack name
1293         char sendbuffer[4096] = "";
1294         requirement *req;
1295         qboolean foundone = false;
1296
1297         for(req = requirements; req; req = req->next)
1298         {
1299                 const char *p;
1300                 const char *thispack = FS_WhichPack(req->filename);
1301                 const char *packurl;
1302
1303                 if(!thispack)
1304                         continue;
1305
1306                 p = strrchr(thispack, '/');
1307                 if(p)
1308                         thispack = p + 1;
1309
1310                 packurl = Curl_FindPackURL(thispack);
1311
1312                 if(packurl && *packurl && strcmp(packurl, "-"))
1313                 {
1314                         if(!foundone)
1315                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1316
1317                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1318                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1319                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1320                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1321                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1322                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1323                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1324                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1325
1326                         foundone = true;
1327                 }
1328         }
1329
1330         if(foundone)
1331                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1332
1333         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1334                 Host_ClientCommands("%s", sendbuffer);
1335         else
1336                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1337 }