]> git.xonotic.org Git - xonotic/darkplaces.git/blob - libcurl.c
Merge PR 'Curl improvements'
[xonotic/darkplaces.git] / libcurl.c
1 #include "quakedef.h"
2 #include "fs.h"
3 #include "libcurl.h"
4 #include "thread.h"
5 #include "com_list.h"
6 #include "image.h"
7 #include "jpeg.h"
8 #include "image_png.h"
9
10 static cvar_t cl_curl_maxdownloads = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
11 static cvar_t cl_curl_maxspeed = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
12 static cvar_t sv_curl_defaulturl = {CF_SERVER | CF_ARCHIVE, "sv_curl_defaulturl","", "default autodownload source URL"};
13 static cvar_t sv_curl_serverpackages = {CF_SERVER | CF_ARCHIVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
14 static cvar_t sv_curl_maxspeed = {CF_SERVER | CF_ARCHIVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
15 static cvar_t cl_curl_enabled = {CF_CLIENT | CF_ARCHIVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
16 static cvar_t cl_curl_useragent = {CF_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
17 static cvar_t cl_curl_useragent_append = {CF_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
18 static cvar_t developer_curl = {CF_CLIENT | CF_SERVER, "developer_curl","0", "whether verbose curl output should be printed to stderr"};
19
20 /*
21 =================================================================
22
23   Minimal set of definitions from libcurl
24
25   WARNING: for a matter of simplicity, several pointer types are
26   casted to "void*", and most enumerated values are not included
27
28 =================================================================
29 */
30
31 typedef struct CURL_s CURL;
32 typedef struct CURLM_s CURLM;
33 typedef struct curl_slist curl_slist;
34 typedef enum
35 {
36         CURLE_OK = 0
37 }
38 CURLcode;
39 typedef enum
40 {
41         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
42         CURLM_OK = 0
43 }
44 CURLMcode;
45 #define CURL_GLOBAL_NOTHING 0
46 #define CURL_GLOBAL_SSL 1
47 #define CURL_GLOBAL_WIN32 2
48 #define CURLOPTTYPE_LONG          0
49 #define CURLOPTTYPE_OBJECTPOINT   10000
50 #define CURLOPTTYPE_FUNCTIONPOINT 20000
51 #define CURLOPTTYPE_OFF_T         30000
52 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
53 typedef enum
54 {
55         CINIT(WRITEDATA, OBJECTPOINT, 1),
56         CINIT(URL,  OBJECTPOINT, 2),
57         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
58         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
59         CINIT(POSTFIELDS, OBJECTPOINT, 15),
60         CINIT(REFERER, OBJECTPOINT, 16),
61         CINIT(USERAGENT, OBJECTPOINT, 18),
62         CINIT(LOW_SPEED_LIMIT, LONG , 19),
63         CINIT(LOW_SPEED_TIME, LONG, 20),
64         CINIT(RESUME_FROM, LONG, 21),
65         CINIT(HTTPHEADER, OBJECTPOINT, 23),
66         CINIT(VERBOSE, LONG, 41),
67         CINIT(POST, LONG, 47),         /* HTTP POST method */
68         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
69         CINIT(POSTFIELDSIZE, LONG, 60),
70         CINIT(PRIVATE, OBJECTPOINT, 103),
71         CINIT(PROTOCOLS, LONG, 181),
72         CINIT(REDIR_PROTOCOLS, LONG, 182)
73 }
74 CURLoption;
75 #define CURLPROTO_HTTP   (1<<0)
76 #define CURLPROTO_HTTPS  (1<<1)
77 #define CURLPROTO_FTP    (1<<2)
78 typedef enum
79 {
80         CURLINFO_TEXT = 0,
81         CURLINFO_HEADER_IN,    /* 1 */
82         CURLINFO_HEADER_OUT,   /* 2 */
83         CURLINFO_DATA_IN,      /* 3 */
84         CURLINFO_DATA_OUT,     /* 4 */
85         CURLINFO_SSL_DATA_IN,  /* 5 */
86         CURLINFO_SSL_DATA_OUT, /* 6 */
87         CURLINFO_END
88 }
89 curl_infotype;
90 #define CURLINFO_STRING   0x100000
91 #define CURLINFO_LONG     0x200000
92 #define CURLINFO_DOUBLE   0x300000
93 #define CURLINFO_SLIST    0x400000
94 #define CURLINFO_MASK     0x0fffff
95 #define CURLINFO_TYPEMASK 0xf00000
96 typedef enum
97 {
98         CURLINFO_NONE, /* first, never use this */
99         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
100         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
101         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
102         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
103         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
104         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
105         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
106         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
107         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
108         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
109         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
110         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
111         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
112         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
113         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
114         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
115         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
116         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
117         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
118         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
119         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
120         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
121         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
122         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
123         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
124         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
125         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27
126 }
127 CURLINFO;
128
129 typedef enum
130 {
131         CURLMSG_NONE, /* first, not used */
132         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
133                                          the CURLcode of the transfer */
134         CURLMSG_LAST
135 }
136 CURLMSG;
137 typedef struct
138 {
139         CURLMSG msg;       /* what this message means */
140         CURL *easy_handle; /* the handle it concerns */
141         union
142         {
143                 void *whatever;    /* message-specific data */
144                 CURLcode result;   /* return code for transfer */
145         }
146         data;
147 }
148 CURLMsg;
149
150 static void (*qcurl_global_init) (long flags);
151 static void (*qcurl_global_cleanup) (void);
152
153 static CURL * (*qcurl_easy_init) (void);
154 static void (*qcurl_easy_cleanup) (CURL *handle);
155 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
156 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
157 static const char * (*qcurl_easy_strerror) (CURLcode);
158
159 static CURLM * (*qcurl_multi_init) (void);
160 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
161 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
162 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
163 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
164 static void (*qcurl_multi_cleanup) (CURLM *);
165 static const char * (*qcurl_multi_strerror) (CURLcode);
166 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
167 static void (*qcurl_slist_free_all) (curl_slist *list);
168
169 static dllfunction_t curlfuncs[] =
170 {
171         {"curl_global_init",            (void **) &qcurl_global_init},
172         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
173         {"curl_easy_init",                      (void **) &qcurl_easy_init},
174         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
175         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
176         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
177         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
178         {"curl_multi_init",                     (void **) &qcurl_multi_init},
179         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
180         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
181         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
182         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
183         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
184         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
185         {"curl_slist_append",           (void **) &qcurl_slist_append},
186         {"curl_slist_free_all",         (void **) &qcurl_slist_free_all},
187         {NULL, NULL}
188 };
189
190 // Handle for CURL DLL
191 static dllhandle_t curl_dll = NULL;
192 // will be checked at many places to find out if qcurl calls are allowed
193
194 #define LOADTYPE_NONE 0
195 #define LOADTYPE_PAK 1
196 #define LOADTYPE_CACHEPIC 2
197 #define LOADTYPE_SKINFRAME 3
198
199 void *curl_mutex = NULL;
200
201 typedef struct downloadinfo_s
202 {
203         char filename[MAX_OSPATH];
204         char url[1024];
205         char referer[256];
206         qfile_t *stream;
207         fs_offset_t startpos;
208         CURL *curle;
209         qbool started;
210         int loadtype;
211         size_t bytes_received; // for buffer
212         double bytes_received_curl; // for throttling
213         double bytes_sent_curl; // for throttling
214         llist_t list;
215         qbool forthismap;
216         double maxspeed;
217         curl_slist *slist; // http headers
218
219         unsigned char *buffer;
220         size_t buffersize;
221         curl_callback_t callback;
222         void *callback_data;
223
224         const unsigned char *postbuf;
225         size_t postbufsize;
226         const char *post_content_type;
227         const char *extraheaders;
228 }
229 downloadinfo;
230 LIST_HEAD(downloads);
231 static int numdownloads = 0;
232
233 static qbool noclear = false;
234
235 static int numdownloads_fail = 0;
236 static int numdownloads_success = 0;
237 static int numdownloads_added = 0;
238 static char command_when_done[256] = "";
239 static char command_when_error[256] = "";
240
241 /*
242 ====================
243 Curl_CommandWhenDone
244
245 Sets the command which is to be executed when the last download completes AND
246 all downloads since last server connect ended with a successful status.
247 Setting the command to NULL clears it.
248 ====================
249 */
250 static void Curl_CommandWhenDone(const char *cmd)
251 {
252         if(!curl_dll)
253                 return;
254         if(cmd)
255                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
256         else
257                 *command_when_done = 0;
258 }
259
260 /*
261 FIXME
262 Do not use yet. Not complete.
263 Problem: what counts as an error?
264 */
265
266 static void Curl_CommandWhenError(const char *cmd)
267 {
268         if(!curl_dll)
269                 return;
270         if(cmd)
271                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
272         else
273                 *command_when_error = 0;
274 }
275
276 /*
277 ====================
278 Curl_Clear_forthismap
279
280 Clears the "will disconnect on failure" flags.
281 ====================
282 */
283 void Curl_Clear_forthismap(void)
284 {
285         downloadinfo *di;
286         if(noclear)
287                 return;
288         if (curl_mutex) Thread_LockMutex(curl_mutex);
289         List_For_Each_Entry(di, &downloads, downloadinfo, list)
290                 di->forthismap = false;
291         Curl_CommandWhenError(NULL);
292         Curl_CommandWhenDone(NULL);
293         numdownloads_fail = 0;
294         numdownloads_success = 0;
295         numdownloads_added = 0;
296         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
297 }
298
299 /*
300 ====================
301 Curl_Have_forthismap
302
303 Returns true if a download needed for the current game is running.
304 ====================
305 */
306 qbool Curl_Have_forthismap(void)
307 {
308         return numdownloads_added != 0;
309 }
310
311 void Curl_Register_predownload(void)
312 {
313         if (curl_mutex) Thread_LockMutex(curl_mutex);
314         Curl_CommandWhenDone("cl_begindownloads");
315         Curl_CommandWhenError("cl_begindownloads");
316         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
317 }
318
319 /*
320 ====================
321 Curl_CheckCommandWhenDone
322
323 Checks if a "done command" is to be executed.
324 All downloads finished, at least one success since connect, no single failure
325 -> execute the command.
326 */
327 static void Curl_CheckCommandWhenDone(void)
328 {
329         if(!curl_dll)
330                 return;
331         if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
332         {
333                 if(numdownloads_fail == 0)
334                 {
335                         Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
336                         Cbuf_AddText(cmd_local, "\n");
337                         Cbuf_AddText(cmd_local, command_when_done);
338                         Cbuf_AddText(cmd_local, "\n");
339                 }
340                 else
341                 {
342                         Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
343                         Cbuf_AddText(cmd_local, "\n");
344                         Cbuf_AddText(cmd_local, command_when_error);
345                         Cbuf_AddText(cmd_local, "\n");
346                 }
347                 Curl_Clear_forthismap();
348         }
349 }
350
351 /*
352 ====================
353 CURL_CloseLibrary
354
355 Load the cURL DLL
356 ====================
357 */
358 static qbool CURL_OpenLibrary (void)
359 {
360         const char* dllnames [] =
361         {
362 #if defined(WIN32)
363                 "libcurl-4.dll",
364                 "libcurl-3.dll",
365 #elif defined(MACOSX)
366                 "libcurl.4.dylib", // Mac OS X Notyetreleased
367                 "libcurl.3.dylib", // Mac OS X Tiger
368                 "libcurl.2.dylib", // Mac OS X Panther
369 #else
370                 "libcurl.so.4",
371                 "libcurl.so.3",
372                 "libcurl.so", // FreeBSD
373 #endif
374                 NULL
375         };
376
377         // Already loaded?
378         if (curl_dll)
379                 return true;
380
381         // Load the DLL
382         return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
383 }
384
385
386 /*
387 ====================
388 CURL_CloseLibrary
389
390 Unload the cURL DLL
391 ====================
392 */
393 static void CURL_CloseLibrary (void)
394 {
395         Sys_FreeLibrary (&curl_dll);
396 }
397
398
399 static CURLM *curlm = NULL;
400 static double bytes_received = 0; // used for bandwidth throttling
401 static double bytes_sent = 0; // used for bandwidth throttling
402 static double curltime = 0;
403
404 /*
405 ====================
406 CURL_fwrite
407
408 fwrite-compatible function that writes the data to a file. libcurl can call
409 this.
410 ====================
411 */
412 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
413 {
414         fs_offset_t ret = -1;
415         size_t bytes = size * nmemb;
416         downloadinfo *di = (downloadinfo *) vdi;
417
418         if(di->buffer)
419         {
420                 if(di->bytes_received + bytes <= di->buffersize)
421                 {
422                         memcpy(di->buffer + di->bytes_received, data, bytes);
423                         ret = bytes;
424                 }
425                 // otherwise: buffer overrun, ret stays -1
426         }
427
428         if(di->stream)
429         {
430                 ret = FS_Write(di->stream, data, bytes);
431         }
432
433         di->bytes_received += bytes;
434
435         return ret;
436         // Why not ret / nmemb?
437         // Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
438         // Yes, this is incompatible to fwrite(2).
439 }
440
441 typedef enum
442 {
443         CURL_DOWNLOAD_SUCCESS = 0,
444         CURL_DOWNLOAD_FAILED,
445         CURL_DOWNLOAD_ABORTED,
446         CURL_DOWNLOAD_SERVERERROR
447 }
448 CurlStatus;
449
450 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
451 {
452         downloadinfo *di = (downloadinfo *) cbdata;
453         switch(status)
454         {
455                 case CURLCBSTATUS_OK:
456                         Con_DPrintf("Download of %s: OK\n", di->filename);
457                         break;
458                 case CURLCBSTATUS_FAILED:
459                         Con_DPrintf("Download of %s: FAILED\n", di->filename);
460                         break;
461                 case CURLCBSTATUS_ABORTED:
462                         Con_DPrintf("Download of %s: ABORTED\n", di->filename);
463                         break;
464                 case CURLCBSTATUS_SERVERERROR:
465                         Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
466                         break;
467                 case CURLCBSTATUS_UNKNOWN:
468                         Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
469                         break;
470                 default:
471                         Con_DPrintf("Download of %s: %d\n", di->filename, status);
472                         break;
473         }
474 }
475
476 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
477 {
478         curl_default_callback(status, length_received, buffer, cbdata);
479 }
480
481 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
482 {
483         unsigned char *pixels = NULL;
484         fs_offset_t filesize = 0;
485         unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
486         if(data)
487         {
488                 int mip = 0;
489                 if(!strcmp(content_type, "image/jpeg"))
490                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
491                 else if(!strcmp(content_type, "image/png"))
492                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
493                 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
494                         pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
495                 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
496                         pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
497                 else
498                         Con_Printf("Did not detect content type: %s\n", content_type);
499                 Mem_Free(data);
500         }
501         // do we call Image_MakeLinearColorsFromsRGB or not?
502         return pixels;
503 }
504
505 /*
506 ====================
507 Curl_EndDownload
508
509 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
510 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
511 code from libcurl, or 0, if another error has occurred.
512 ====================
513 */
514 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
515 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
516 {
517         char content_type[64];
518         qbool ok = false;
519         if(!curl_dll)
520                 return;
521         switch(status)
522         {
523                 case CURL_DOWNLOAD_SUCCESS:
524                         ok = true;
525                         di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
526                         break;
527                 case CURL_DOWNLOAD_FAILED:
528                         di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
529                         break;
530                 case CURL_DOWNLOAD_ABORTED:
531                         di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
532                         break;
533                 case CURL_DOWNLOAD_SERVERERROR:
534                         // reopen to enforce it to have zero bytes again
535                         if(di->stream)
536                         {
537                                 FS_Close(di->stream);
538                                 di->stream = FS_OpenRealFile(di->filename, "wb", false);
539                         }
540
541                         if(di->callback)
542                                 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
543                         break;
544                 default:
545                         if(di->callback)
546                                 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
547                         break;
548         }
549         if(content_type_)
550                 strlcpy(content_type, content_type_, sizeof(content_type));
551         else
552                 *content_type = 0;
553
554         if(di->curle)
555         {
556                 qcurl_multi_remove_handle(curlm, di->curle);
557                 qcurl_easy_cleanup(di->curle);
558                 if(di->slist)
559                         qcurl_slist_free_all(di->slist);
560         }
561
562         if(!di->callback && ok && !di->bytes_received)
563         {
564                 Con_Printf("ERROR: empty file\n");
565                 ok = false;
566         }
567
568         if(di->stream)
569                 FS_Close(di->stream);
570
571 #define CLEAR_AND_RETRY() \
572         do \
573         { \
574                 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
575                 FS_Close(di->stream); \
576                 if(di->startpos && !di->callback) \
577                 { \
578                         Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
579                         di->forthismap = false; \
580                 } \
581         } \
582         while(0)
583
584         if(ok && di->loadtype == LOADTYPE_PAK)
585         {
586                 ok = FS_AddPack(di->filename, NULL, true);
587                 if(!ok)
588                         CLEAR_AND_RETRY();
589         }
590         else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
591         {
592                 const char *p;
593                 unsigned char *pixels = NULL;
594
595                 p = di->filename;
596 #ifdef WE_ARE_EVIL
597                 if(!strncmp(p, "dlcache/", 8))
598                         p += 8;
599 #endif
600
601                 pixels = decode_image(di, content_type);
602                 if(pixels)
603                         Draw_NewPic(p, image_width, image_height, pixels, TEXTYPE_BGRA, TEXF_ALPHA | TEXF_CLAMP);
604                 else
605                         CLEAR_AND_RETRY();
606         }
607         else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
608         {
609                 const char *p;
610                 unsigned char *pixels = NULL;
611
612                 p = di->filename;
613 #ifdef WE_ARE_EVIL
614                 if(!strncmp(p, "dlcache/", 8))
615                         p += 8;
616 #endif
617
618                 pixels = decode_image(di, content_type);
619                 if(pixels)
620                         R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, 0, 0, 0, false); // TODO what sRGB argument to put here?
621                 else
622                         CLEAR_AND_RETRY();
623         }
624
625         List_Delete(&di->list);
626
627         --numdownloads;
628         if(di->forthismap)
629         {
630                 if(ok)
631                         ++numdownloads_success;
632                 else
633                         ++numdownloads_fail;
634         }
635         Z_Free(di);
636 }
637
638 /*
639 ====================
640 CleanURL
641
642 Returns a "cleaned up" URL for display (to strip login data)
643 ====================
644 */
645 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
646 {
647         const char *p, *q, *r;
648
649         // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
650         p = strstr(url, "://");
651         if(p)
652         {
653                 q = strchr(p + 3, '@');
654                 if(q)
655                 {
656                         r = strchr(p + 3, '/');
657                         if(!r || q < r)
658                         {
659                                 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
660                                 return urlbuf;
661                         }
662                 }
663         }
664
665         return url;
666 }
667
668 /*
669 ====================
670 CheckPendingDownloads
671
672 checks if there are free download slots to start new downloads in.
673 To not start too many downloads at once, only one download is added at a time,
674 up to a maximum number of cl_curl_maxdownloads are running.
675 ====================
676 */
677 static void CheckPendingDownloads(void)
678 {
679         const char *h;
680         char urlbuf[1024];
681         char vabuf[1024];
682         if(!curl_dll)
683                 return;
684         if(numdownloads < cl_curl_maxdownloads.integer)
685         {
686                 downloadinfo *di;
687                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
688                 {
689                         if(!di->started)
690                         {
691                                 if(!di->buffer)
692                                 {
693                                         Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
694
695                                         di->stream = FS_OpenRealFile(di->filename, "ab", false);
696                                         if(!di->stream)
697                                         {
698                                                 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
699                                                 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
700                                                 return;
701                                         }
702                                         FS_Seek(di->stream, 0, SEEK_END);
703                                         di->startpos = FS_Tell(di->stream);
704
705                                         if(di->startpos > 0)
706                                                 Con_Printf(", resuming from position %ld", (long) di->startpos);
707                                         Con_Print("...\n");
708                                 }
709                                 else
710                                 {
711                                         Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
712                                         di->startpos = 0;
713                                 }
714
715                                 di->curle = qcurl_easy_init();
716                                 di->slist = NULL;
717                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
718                                 if(cl_curl_useragent.integer)
719                                 {
720                                         const char *ua
721 #ifdef HTTP_USER_AGENT
722                                                 = HTTP_USER_AGENT;
723 #else
724                                                 = engineversion;
725 #endif
726                                         if(!ua)
727                                                 ua = "";
728                                         if(*cl_curl_useragent_append.string)
729                                                 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
730                                                         ua,
731                                                         (ua[0] && ua[strlen(ua)-1] != ' ')
732                                                                 ? " "
733                                                                 : "",
734                                                         cl_curl_useragent_append.string);
735                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
736                                 }
737                                 else
738                                         qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
739                                 if(developer_curl.integer) 
740                                         qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
741                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
742                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
743                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
744                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
745                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
746                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
747                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
748                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
749                                 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
750                                 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
751                                 {
752                                         Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
753                                         //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
754                                 }
755                                 if(di->post_content_type)
756                                 {
757                                         qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
758                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
759                                         qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
760                                         di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
761                                 }
762
763                                 // parse extra headers into slist
764                                 // \n separated list!
765                                 h = di->extraheaders;
766                                 while(h)
767                                 {
768                                         const char *hh = strchr(h, '\n');
769                                         if(hh)
770                                         {
771                                                 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
772                                                 memcpy(buf, h, hh - h);
773                                                 buf[hh - h] = 0;
774                                                 di->slist = qcurl_slist_append(di->slist, buf);
775                                                 h = hh + 1;
776                                         }
777                                         else
778                                         {
779                                                 di->slist = qcurl_slist_append(di->slist, h);
780                                                 h = NULL;
781                                         }
782                                 }
783
784                                 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
785                                 
786                                 qcurl_multi_add_handle(curlm, di->curle);
787                                 di->started = true;
788                                 ++numdownloads;
789                                 if(numdownloads >= cl_curl_maxdownloads.integer)
790                                         break;
791                         }
792                 }
793         }
794 }
795
796 /*
797 ====================
798 Curl_Init
799
800 this function MUST be called before using anything else in this file.
801 On Win32, this must be called AFTER WSAStartup has been done!
802 ====================
803 */
804 void Curl_Init(void)
805 {
806         CURL_OpenLibrary();
807         if(!curl_dll)
808                 return;
809         if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
810         qcurl_global_init(CURL_GLOBAL_SSL);
811         curlm = qcurl_multi_init();
812 }
813
814 /*
815 ====================
816 Curl_Shutdown
817
818 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
819 ====================
820 */
821 void Curl_ClearRequirements(void);
822 void Curl_Shutdown(void)
823 {
824         if(!curl_dll)
825                 return;
826         Curl_ClearRequirements();
827         Curl_CancelAll();
828         if (curl_mutex) Thread_DestroyMutex(curl_mutex);
829         CURL_CloseLibrary();
830         curl_dll = NULL;
831 }
832
833 /*
834 ====================
835 Curl_Find
836
837 Finds the internal information block for a download given by file name.
838 ====================
839 */
840 static downloadinfo *Curl_Find(const char *filename)
841 {
842         downloadinfo *di;
843         if(!curl_dll)
844                 return NULL;
845         List_For_Each_Entry(di, &downloads, downloadinfo, list)
846                 if(!strcasecmp(di->filename, filename))
847                         return di;
848         return NULL;
849 }
850
851 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
852 {
853         downloadinfo *di, *ndi;
854         if(!curl_dll)
855                 return;
856         List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
857         {
858                 if(di->callback == callback && di->callback_data == cbdata)
859                 {
860                         di->callback = curl_quiet_callback; // do NOT call the callback
861                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
862                 }
863         }
864 }
865
866 /*
867 ====================
868 Curl_Begin
869
870 Starts a download of a given URL to the file name portion of this URL (or name
871 if given) in the "dlcache/" folder.
872 ====================
873 */
874 static qbool Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qbool forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
875 {
876         if(buf)
877                 if(loadtype != LOADTYPE_NONE)
878                         Host_Error("Curl_Begin: loadtype and buffer are both set");
879
880         if(!curl_dll || !cl_curl_enabled.integer)
881         {
882                 return false;
883         }
884         else
885         {
886                 char fn[MAX_OSPATH];
887                 char urlbuf[1024];
888                 const char *p, *q;
889                 size_t length;
890                 downloadinfo *di;
891
892                 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
893                 p = strchr(URL, ':');
894                 if(p)
895                 {
896                         if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
897                         {
898                                 char addressstring[128];
899                                 *addressstring = 0;
900                                 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
901                                 q = strchr(addressstring, ':');
902                                 if(!q)
903                                         q = addressstring + strlen(addressstring);
904                                 if(*addressstring)
905                                 {
906                                         dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
907                                         URL = urlbuf;
908                                 }
909                         }
910                 }
911
912                 // Note: This extraction of the file name portion is NOT entirely correct.
913                 //
914                 // It does the following:
915                 //
916                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
917                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
918                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
919                 //
920                 // However, I'd like to keep this "buggy" behavior so that PHP script
921                 // authors can write download scripts without having to enable
922                 // AcceptPathInfo on Apache. They just have to ensure that their script
923                 // can be called with such a "fake" path name like
924                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
925                 //
926                 // By the way, such PHP scripts should either send the file or a
927                 // "Location:" redirect; PHP code example:
928                 //
929                 //   header("Location: http://www.example.com/");
930                 //
931                 // By the way, this will set User-Agent to something like
932                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
933                 // dp://serverhost:serverport/ so you can filter on this; an example
934                 // httpd log file line might be:
935                 //
936                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
937
938                 if (curl_mutex) Thread_LockMutex(curl_mutex);
939
940                 if(buf)
941                 {
942                         if(!name)
943                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
944                 }
945                 else
946                 {
947                         if(!name)
948                         {
949                                 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
950                                 p = strrchr(name, '/');
951                                 p = p ? (p+1) : name;
952                                 q = strchr(p, '?');
953                                 length = q ? (size_t)(q - p) : strlen(p);
954                                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
955                         }
956                         else
957                         {
958                                 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
959                         }
960
961                         name = fn; // make it point back
962
963                         // already downloading the file?
964                         {
965                                 downloadinfo *existingdownloadinfo = Curl_Find(fn);
966                                 if(existingdownloadinfo)
967                                 {
968                                         Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(existingdownloadinfo->url, urlbuf, sizeof(urlbuf)));
969
970                                         // however, if it was not for this map yet...
971                                         if(forthismap && !existingdownloadinfo->forthismap)
972                                         {
973                                                 existingdownloadinfo->forthismap = true;
974                                                 // this "fakes" a download attempt so the client will wait for
975                                                 // the download to finish and then reconnect
976                                                 ++numdownloads_added;
977                                         }
978
979                                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
980                                         return false;
981                                 }
982                         }
983
984                         if(FS_FileExists(fn))
985                         {
986                                 if(loadtype == LOADTYPE_PAK)
987                                 {
988                                         qbool already_loaded;
989                                         if(FS_AddPack(fn, &already_loaded, true))
990                                         {
991                                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
992                                                 if(already_loaded)
993                                                         Con_DPrintf("(pak was already loaded)\n");
994                                                 else
995                                                 {
996                                                         if(forthismap)
997                                                         {
998                                                                 ++numdownloads_added;
999                                                                 ++numdownloads_success;
1000                                                         }
1001                                                 }
1002
1003                                                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1004                                                 return false;
1005                                         }
1006                                         else
1007                                         {
1008                                                 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1009                                                 if(f)
1010                                                 {
1011                                                         char b[4] = {0};
1012                                                         FS_Read(f, b, sizeof(b)); // no "-1", I will use memcmp
1013
1014                                                         if(memcmp(b, "PK\x03\x04", 4) && memcmp(b, "PACK", 4))
1015                                                         {
1016                                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1017                                                                 FS_Close(f);
1018                                                                 f = FS_OpenRealFile(fn, "wb", false);
1019                                                                 if(f)
1020                                                                         FS_Close(f);
1021                                                         }
1022                                                         else
1023                                                         {
1024                                                                 // OK
1025                                                                 FS_Close(f);
1026                                                         }
1027                                                 }
1028                                         }
1029                                 }
1030                                 else
1031                                 {
1032                                         // never resume these
1033                                         qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1034                                         if(f)
1035                                                 FS_Close(f);
1036                                 }
1037                         }
1038                 }
1039
1040                 // if we get here, we actually want to download... so first verify the
1041                 // URL scheme (so one can't read local files using file://)
1042                 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1043                 {
1044                         Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1045                         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1046                         return false;
1047                 }
1048
1049                 if(forthismap)
1050                         ++numdownloads_added;
1051                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1052                 strlcpy(di->filename, name, sizeof(di->filename));
1053                 strlcpy(di->url, URL, sizeof(di->url));
1054                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1055                 di->forthismap = forthismap;
1056                 di->stream = NULL;
1057                 di->startpos = 0;
1058                 di->curle = NULL;
1059                 di->started = false;
1060                 di->loadtype = loadtype;
1061                 di->maxspeed = maxspeed;
1062                 di->bytes_received = 0;
1063                 di->bytes_received_curl = 0;
1064                 di->bytes_sent_curl = 0;
1065                 di->extraheaders = extraheaders;
1066                 di->buffer = buf;
1067                 di->buffersize = bufsize;
1068                 if(callback == NULL)
1069                 {
1070                         di->callback = curl_default_callback;
1071                         di->callback_data = di;
1072                 }
1073                 else
1074                 {
1075                         di->callback = callback;
1076                         di->callback_data = cbdata;
1077                 }
1078
1079                 if(post_content_type)
1080                 {
1081                         di->post_content_type = post_content_type;
1082                         di->postbuf = postbuf;
1083                         di->postbufsize = postbufsize;
1084                 }
1085                 else
1086                 {
1087                         di->post_content_type = NULL;
1088                         di->postbuf = NULL;
1089                         di->postbufsize = 0;
1090                 }
1091
1092                 List_Add(&di->list, &downloads);
1093
1094                 if (curl_mutex)
1095                         Thread_UnlockMutex(curl_mutex);
1096
1097                 return true;
1098         }
1099 }
1100
1101 qbool Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qbool forthismap)
1102 {
1103         return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1104 }
1105 qbool Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1106 {
1107         return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1108 }
1109 qbool Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1110 {
1111         return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1112 }
1113
1114 /*
1115 ====================
1116 Curl_Frame
1117
1118 call this regularily as this will always download as much as possible without
1119 blocking.
1120 ====================
1121 */
1122 void Curl_Frame(void)
1123 {
1124         double maxspeed;
1125         downloadinfo *di;
1126
1127         noclear = false;
1128
1129         if(!cl_curl_enabled.integer)
1130                 return;
1131
1132         if(!curl_dll)
1133                 return;
1134
1135         if (curl_mutex) Thread_LockMutex(curl_mutex);
1136
1137         Curl_CheckCommandWhenDone();
1138
1139         if(List_Is_Empty(&downloads))
1140         {
1141                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1142                 return;
1143         }
1144
1145         if(host.realtime < curltime) // throttle
1146         {
1147                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1148                 return;
1149         }
1150
1151         {
1152                 int remaining;
1153                 CURLMcode mc;
1154
1155                 do
1156                 {
1157                         mc = qcurl_multi_perform(curlm, &remaining);
1158                 }
1159                 while(mc == CURLM_CALL_MULTI_PERFORM);
1160
1161                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1162                 {
1163                         double b = 0;
1164                         if(di->curle)
1165                         {
1166                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1167                                 bytes_sent += (b - di->bytes_sent_curl);
1168                                 di->bytes_sent_curl = b;
1169                                 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1170                                 bytes_sent += (b - di->bytes_received_curl);
1171                                 di->bytes_received_curl = b;
1172                         }
1173                 }
1174
1175                 for(;;)
1176                 {
1177                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1178                         if(!msg)
1179                                 break;
1180                         if(msg->msg == CURLMSG_DONE)
1181                         {
1182                                 const char *ct = NULL;
1183                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1184                                 CURLcode result;
1185                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1186                                 result = msg->data.result;
1187                                 if(result)
1188                                 {
1189                                         failed = CURL_DOWNLOAD_FAILED;
1190                                 }
1191                                 else
1192                                 {
1193                                         long code;
1194                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1195                                         switch(code / 100)
1196                                         {
1197                                                 case 4: // e.g. 404?
1198                                                 case 5: // e.g. 500?
1199                                                         failed = CURL_DOWNLOAD_SERVERERROR;
1200                                                         result = (CURLcode) code;
1201                                                         break;
1202                                         }
1203                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1204                                 }
1205
1206                                 Curl_EndDownload(di, failed, result, ct);
1207                         }
1208                 }
1209         }
1210
1211         CheckPendingDownloads();
1212
1213         // when will we curl the next time?
1214         // we will wait a bit to ensure our download rate is kept.
1215         // we now know that realtime >= curltime... so set up a new curltime
1216
1217         // use the slowest allowing download to derive the maxspeed... this CAN
1218         // be done better, but maybe later
1219         maxspeed = cl_curl_maxspeed.value;
1220         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1221                 if(di->maxspeed > 0)
1222                         if(di->maxspeed < maxspeed || maxspeed <= 0)
1223                                 maxspeed = di->maxspeed;
1224
1225         if(maxspeed > 0)
1226         {
1227                 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1228                 curltime = host.realtime + bytes / (maxspeed * 1024.0);
1229                 bytes_sent = 0;
1230                 bytes_received = 0;
1231         }
1232         else
1233                 curltime = host.realtime;
1234
1235         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1236 }
1237
1238 /*
1239 ====================
1240 Curl_CancelAll
1241
1242 Stops ALL downloads.
1243 ====================
1244 */
1245 void Curl_CancelAll(void)
1246 {
1247         if(!curl_dll)
1248                 return;
1249
1250         if (curl_mutex) Thread_LockMutex(curl_mutex);
1251
1252         while(!List_Is_Empty(&downloads))
1253         {
1254                 Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1255                 // INVARIANT: downloads will point to the next download after that!
1256         }
1257
1258         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1259 }
1260
1261 /*
1262 ====================
1263 Curl_Running
1264
1265 returns true if there is a download running.
1266 ====================
1267 */
1268 qbool Curl_Running(void)
1269 {
1270         if(!curl_dll)
1271                 return false;
1272
1273         return !List_Is_Empty(&downloads);
1274 }
1275
1276 /*
1277 ====================
1278 Curl_GetDownloadAmount
1279
1280 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1281 for the given download.
1282 ====================
1283 */
1284 static double Curl_GetDownloadAmount(downloadinfo *di)
1285 {
1286         if(!curl_dll)
1287                 return -2;
1288         if(di->curle)
1289         {
1290                 double length;
1291                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1292                 if(length > 0)
1293                         return (di->startpos + di->bytes_received) / (di->startpos + length);
1294                 else
1295                         return 0;
1296         }
1297         else
1298                 return -1;
1299 }
1300
1301 /*
1302 ====================
1303 Curl_GetDownloadSpeed
1304
1305 returns the speed of the given download in bytes per second
1306 ====================
1307 */
1308 static double Curl_GetDownloadSpeed(downloadinfo *di)
1309 {
1310         if(!curl_dll)
1311                 return -2;
1312         if(di->curle)
1313         {
1314                 double speed;
1315                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1316                 return speed;
1317         }
1318         else
1319                 return -1;
1320 }
1321
1322 /*
1323 ====================
1324 Curl_Info_f
1325
1326 prints the download list
1327 ====================
1328 */
1329 // TODO rewrite using Curl_GetDownloadInfo?
1330 static void Curl_Info_f(cmd_state_t *cmd)
1331 {
1332         downloadinfo *di;
1333         char urlbuf[1024];
1334         if(!curl_dll)
1335                 return;
1336         if(Curl_Running())
1337         {
1338                 if (curl_mutex) Thread_LockMutex(curl_mutex);
1339                 Con_Print("Currently running downloads:\n");
1340                 List_For_Each_Entry(di, &downloads, downloadinfo, list)
1341                 {
1342                         double speed, percent;
1343                         Con_Printf("  %s -> %s ",  CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1344                         percent = 100.0 * Curl_GetDownloadAmount(di);
1345                         speed = Curl_GetDownloadSpeed(di);
1346                         if(percent >= 0)
1347                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1348                         else
1349                                 Con_Print("(queued)\n");
1350                 }
1351                 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1352         }
1353         else
1354         {
1355                 Con_Print("No downloads running.\n");
1356         }
1357 }
1358
1359 /*
1360 ====================
1361 Curl_Curl_f
1362
1363 implements the "curl" console command
1364
1365 curl --info
1366 curl --cancel
1367 curl --cancel filename
1368 curl url
1369
1370 For internal use:
1371
1372 curl [--pak] [--forthismap] [--for filename filename...] url
1373         --pak: after downloading, load the package into the virtual file system
1374         --for filename...: only download of at least one of the named files is missing
1375         --forthismap: don't reconnect on failure
1376
1377 curl --clear_autodownload
1378         clears the download success/failure counters
1379
1380 curl --finish_autodownload
1381         if at least one download has been started, disconnect and drop to the menu
1382         once the last download completes successfully, reconnect to the current server
1383 ====================
1384 */
1385 static void Curl_Curl_f(cmd_state_t *cmd)
1386 {
1387         double maxspeed = 0;
1388         int i;
1389         int end;
1390         int loadtype = LOADTYPE_NONE;
1391         qbool forthismap = false;
1392         const char *url;
1393         const char *name = 0;
1394
1395         if(!curl_dll)
1396         {
1397                 Con_Print("libcurl DLL not found, this command is inactive.\n");
1398                 return;
1399         }
1400
1401         if(!cl_curl_enabled.integer)
1402         {
1403                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1404                 return;
1405         }
1406
1407         if(Cmd_Argc(cmd) < 2)
1408         {
1409                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1410                 return;
1411         }
1412
1413         url = Cmd_Argv(cmd, Cmd_Argc(cmd) - 1);
1414         end = Cmd_Argc(cmd);
1415
1416         for(i = 1; i != end; ++i)
1417         {
1418                 const char *a = Cmd_Argv(cmd, i);
1419                 if(!strcmp(a, "--info"))
1420                 {
1421                         Curl_Info_f(cmd);
1422                         return;
1423                 }
1424                 else if(!strcmp(a, "--cancel"))
1425                 {
1426                         if(i == end - 1) // last argument
1427                                 Curl_CancelAll();
1428                         else
1429                         {
1430                                 downloadinfo *di = Curl_Find(url);
1431                                 if(di)
1432                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1433                                 else
1434                                         Con_Print("download not found\n");
1435                         }
1436                         return;
1437                 }
1438                 else if(!strcmp(a, "--pak"))
1439                 {
1440                         loadtype = LOADTYPE_PAK;
1441                 }
1442                 else if(!strcmp(a, "--cachepic"))
1443                 {
1444                         loadtype = LOADTYPE_CACHEPIC;
1445                 }
1446                 else if(!strcmp(a, "--skinframe"))
1447                 {
1448                         loadtype = LOADTYPE_SKINFRAME;
1449                 }
1450                 else if(!strcmp(a, "--for")) // must be last option
1451                 {
1452                         for(i = i + 1; i != end - 1; ++i)
1453                         {
1454                                 if(!FS_FileExists(Cmd_Argv(cmd, i)))
1455                                         goto needthefile; // why can't I have a "double break"?
1456                         }
1457                         // if we get here, we have all the files...
1458                         return;
1459                 }
1460                 else if(!strcmp(a, "--forthismap"))
1461                 {
1462                         forthismap = true;
1463                 }
1464                 else if(!strcmp(a, "--as"))
1465                 {
1466                         if(i < end - 1)
1467                         {
1468                                 ++i;
1469                                 name = Cmd_Argv(cmd, i);
1470                         }
1471                 }
1472                 else if(!strcmp(a, "--clear_autodownload"))
1473                 {
1474                         // mark all running downloads as "not for this map", so if they
1475                         // fail, it does not matter
1476                         Curl_Clear_forthismap();
1477                         return;
1478                 }
1479                 else if(!strcmp(a, "--finish_autodownload"))
1480                 {
1481                         if(numdownloads_added)
1482                         {
1483                                 char donecommand[256];
1484                                 if(cls.netcon)
1485                                 {
1486                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1487                                         {
1488                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1489                                                 Curl_CommandWhenDone(donecommand);
1490                                                 noclear = true;
1491                                                 CL_Disconnect();
1492                                                 noclear = false;
1493                                                 Curl_CheckCommandWhenDone();
1494                                         }
1495                                         else
1496                                                 Curl_Register_predownload();
1497                                 }
1498                         }
1499                         return;
1500                 }
1501                 else if(!strncmp(a, "--maxspeed=", 11))
1502                 {
1503                         maxspeed = atof(a + 11);
1504                 }
1505                 else if(*a == '-')
1506                 {
1507                         Con_Printf("curl: invalid option %s\n", a);
1508                         // but we ignore the option
1509                 }
1510         }
1511
1512 needthefile:
1513         Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1514 }
1515
1516 /*
1517 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1518 {
1519         Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1520         Z_Free(buffer);
1521 }
1522
1523 void Curl_CurlCat_f(cmd_state_t *cmd)
1524 {
1525         unsigned char *buf;
1526         const char *url = Cmd_Argv(cmd, 1);
1527         buf = Z_Malloc(16384);
1528         Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1529 }
1530 */
1531
1532 /*
1533 ====================
1534 Curl_Init_Commands
1535
1536 loads the commands and cvars this library uses
1537 ====================
1538 */
1539 void Curl_Init_Commands(void)
1540 {
1541         Cvar_RegisterVariable (&cl_curl_enabled);
1542         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1543         Cvar_RegisterVariable (&cl_curl_maxspeed);
1544         Cvar_RegisterVariable (&sv_curl_defaulturl);
1545         Cvar_RegisterVariable (&sv_curl_serverpackages);
1546         Cvar_RegisterVariable (&sv_curl_maxspeed);
1547         Cvar_RegisterVariable (&cl_curl_useragent);
1548         Cvar_RegisterVariable (&cl_curl_useragent_append);
1549         Cvar_RegisterVariable (&developer_curl);
1550         Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
1551         //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1552 }
1553
1554 /*
1555 ====================
1556 Curl_GetDownloadInfo
1557
1558 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1559 The number of elements in the array is returned in int *nDownloads.
1560 const char **additional_info may be set to a string of additional user
1561 information, or to NULL if no such display shall occur. The returned
1562 array must be freed later using Z_Free.
1563 ====================
1564 */
1565 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1566 {
1567         int i;
1568         downloadinfo *di;
1569         Curl_downloadinfo_t *downinfo;
1570
1571         if(!curl_dll)
1572         {
1573                 *nDownloads = 0;
1574                 if(additional_info)
1575                         *additional_info = NULL;
1576                 return NULL;
1577         }
1578
1579         if (curl_mutex) Thread_LockMutex(curl_mutex);
1580
1581         i = 0;
1582         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1583                 ++i;
1584
1585         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1586         i = 0;
1587         List_For_Each_Entry(di, &downloads, downloadinfo, list)
1588         {
1589                 // do not show infobars for background downloads
1590                 if(developer.integer <= 0)
1591                         if(di->buffer)
1592                                 continue;
1593                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1594                 if(di->curle)
1595                 {
1596                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1597                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1598                         downinfo[i].queued = false;
1599                 }
1600                 else
1601                 {
1602                         downinfo[i].queued = true;
1603                 }
1604                 ++i;
1605         }
1606
1607         if(additional_info)
1608         {
1609                 // TODO: can I clear command_when_done as soon as the first download fails?
1610                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1611                 {
1612                         if(!strncmp(command_when_done, "connect ", 8))
1613                                 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1614                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1615                                 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1616                         else
1617                                 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1618                         *additional_info = addinfo;
1619                 }
1620                 else
1621                         *additional_info = NULL;
1622         }
1623
1624         *nDownloads = i;
1625         if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1626         return downinfo;
1627 }
1628
1629
1630 /*
1631 ====================
1632 Curl_FindPackURL
1633
1634 finds the URL where to find a given package.
1635
1636 For this, it reads a file "curl_urls.txt" of the following format:
1637
1638         data*.pk3       -
1639         revdm*.pk3      http://revdm/downloads/are/here/
1640         *                       http://any/other/stuff/is/here/
1641
1642 The URLs should end in /. If not, downloads will still work, but the cached files
1643 can't be just put into the data directory with the same download configuration
1644 (you might want to do this if you want to tag downloaded files from your
1645 server, but you should not). "-" means "don't download".
1646
1647 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1648 location instead.
1649
1650 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1651 this file for obvious reasons.
1652 ====================
1653 */
1654 static const char *Curl_FindPackURL(const char *filename)
1655 {
1656         static char foundurl[1024]; // invoked only by server
1657         fs_offset_t filesize;
1658         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1659         if(buf && filesize)
1660         {
1661                 // read lines of format "pattern url"
1662                 char *p = buf;
1663                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1664                 qbool eof = false;
1665
1666                 pattern = p;
1667                 while(!eof)
1668                 {
1669                         switch(*p)
1670                         {
1671                                 case 0:
1672                                         eof = true;
1673                                         // fallthrough
1674                                 case '\n':
1675                                 case '\r':
1676                                         if(pattern && url && patternend)
1677                                         {
1678                                                 if(!urlend)
1679                                                         urlend = p;
1680                                                 *patternend = 0;
1681                                                 *urlend = 0;
1682                                                 if(matchpattern(filename, pattern, true))
1683                                                 {
1684                                                         strlcpy(foundurl, url, sizeof(foundurl));
1685                                                         Z_Free(buf);
1686                                                         return foundurl;
1687                                                 }
1688                                         }
1689                                         pattern = NULL;
1690                                         patternend = NULL;
1691                                         url = NULL;
1692                                         urlend = NULL;
1693                                         break;
1694                                 case ' ':
1695                                 case '\t':
1696                                         if(pattern && !patternend)
1697                                                 patternend = p;
1698                                         else if(url && !urlend)
1699                                                 urlend = p;
1700                                         break;
1701                                 default:
1702                                         if(!pattern)
1703                                                 pattern = p;
1704                                         else if(pattern && patternend && !url)
1705                                                 url = p;
1706                                         break;
1707                         }
1708                         ++p;
1709                 }
1710         }
1711         if(buf)
1712                 Z_Free(buf);
1713         return sv_curl_defaulturl.string;
1714 }
1715
1716 typedef struct requirement_s
1717 {
1718         struct requirement_s *next;
1719         char filename[MAX_OSPATH];
1720 }
1721 requirement;
1722 static requirement *requirements = NULL;
1723
1724
1725 /*
1726 ====================
1727 Curl_RequireFile
1728
1729 Adds the given file to the list of requirements.
1730 ====================
1731 */
1732 void Curl_RequireFile(const char *filename)
1733 {
1734         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1735         req->next = requirements;
1736         strlcpy(req->filename, filename, sizeof(req->filename));
1737         requirements = req;
1738 }
1739
1740 /*
1741 ====================
1742 Curl_ClearRequirements
1743
1744 Clears the list of required files for playing on the current map.
1745 This should be called at every map change.
1746 ====================
1747 */
1748 void Curl_ClearRequirements(void)
1749 {
1750         while(requirements)
1751         {
1752                 requirement *req = requirements;
1753                 requirements = requirements->next;
1754                 Z_Free(req);
1755         }
1756 }
1757
1758 /*
1759 ====================
1760 Curl_SendRequirements
1761
1762 Makes the current host_clients download all files he needs.
1763 This is done by sending him the following console commands:
1764
1765         curl --clear_autodownload
1766         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1767         curl --finish_autodownload
1768 ====================
1769 */
1770 static qbool Curl_SendRequirement(const char *filename, qbool foundone, char *sendbuffer, size_t sendbuffer_len)
1771 {
1772         const char *p;
1773         const char *thispack = FS_WhichPack(filename);
1774         const char *packurl;
1775
1776         if(!thispack || !*thispack)
1777                 return false;
1778
1779         p = strrchr(thispack, '/');
1780         if(p)
1781                 thispack = p + 1;
1782
1783         packurl = Curl_FindPackURL(thispack);
1784
1785         if(packurl && *packurl && strcmp(packurl, "-"))
1786         {
1787                 if(!foundone)
1788                         strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1789
1790                 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1791                 strlcat(sendbuffer, thispack, sendbuffer_len);
1792                 if(sv_curl_maxspeed.value > 0)
1793                         dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1794                 strlcat(sendbuffer, " --for ", sendbuffer_len);
1795                 strlcat(sendbuffer, filename, sendbuffer_len);
1796                 strlcat(sendbuffer, " ", sendbuffer_len);
1797                 strlcat(sendbuffer, packurl, sendbuffer_len);
1798                 strlcat(sendbuffer, thispack, sendbuffer_len);
1799                 strlcat(sendbuffer, "\n", sendbuffer_len);
1800
1801                 return true;
1802         }
1803
1804         return false;
1805 }
1806 void Curl_SendRequirements(void)
1807 {
1808         // for each requirement, find the pack name
1809         char sendbuffer[4096] = "";
1810         requirement *req;
1811         qbool foundone = false;
1812         const char *p;
1813
1814         for(req = requirements; req; req = req->next)
1815                 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1816
1817         p = sv_curl_serverpackages.string;
1818         while(COM_ParseToken_Simple(&p, false, false, true))
1819                 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1820
1821         if(foundone)
1822                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1823
1824         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1825                 SV_ClientCommands("%s", sendbuffer);
1826         else
1827                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1828 }