5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
9 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
10 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
13 =================================================================
15 Minimal set of definitions from libcurl
17 WARNING: for a matter of simplicity, several pointer types are
18 casted to "void*", and most enumerated values are not included
20 =================================================================
23 typedef struct CURL_s CURL;
24 typedef struct CURLM_s CURLM;
25 typedef struct curl_slist curl_slist;
33 CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
37 #define CURL_GLOBAL_NOTHING 0
38 #define CURL_GLOBAL_SSL 1
39 #define CURL_GLOBAL_WIN32 2
40 #define CURLOPTTYPE_LONG 0
41 #define CURLOPTTYPE_OBJECTPOINT 10000
42 #define CURLOPTTYPE_FUNCTIONPOINT 20000
43 #define CURLOPTTYPE_OFF_T 30000
44 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
47 CINIT(WRITEDATA, OBJECTPOINT, 1),
48 CINIT(URL, OBJECTPOINT, 2),
49 CINIT(ERRORBUFFER, OBJECTPOINT, 10),
50 CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
51 CINIT(POSTFIELDS, OBJECTPOINT, 15),
52 CINIT(REFERER, OBJECTPOINT, 16),
53 CINIT(USERAGENT, OBJECTPOINT, 18),
54 CINIT(LOW_SPEED_LIMIT, LONG , 19),
55 CINIT(LOW_SPEED_TIME, LONG, 20),
56 CINIT(RESUME_FROM, LONG, 21),
57 CINIT(HTTPHEADER, OBJECTPOINT, 23),
58 CINIT(POST, LONG, 47), /* HTTP POST method */
59 CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */
60 CINIT(POSTFIELDSIZE, LONG, 60),
61 CINIT(PRIVATE, OBJECTPOINT, 103),
62 CINIT(PROTOCOLS, LONG, 181),
63 CINIT(REDIR_PROTOCOLS, LONG, 182)
66 #define CURLPROTO_HTTP (1<<0)
67 #define CURLPROTO_HTTPS (1<<1)
68 #define CURLPROTO_FTP (1<<2)
72 CURLINFO_HEADER_IN, /* 1 */
73 CURLINFO_HEADER_OUT, /* 2 */
74 CURLINFO_DATA_IN, /* 3 */
75 CURLINFO_DATA_OUT, /* 4 */
76 CURLINFO_SSL_DATA_IN, /* 5 */
77 CURLINFO_SSL_DATA_OUT, /* 6 */
81 #define CURLINFO_STRING 0x100000
82 #define CURLINFO_LONG 0x200000
83 #define CURLINFO_DOUBLE 0x300000
84 #define CURLINFO_SLIST 0x400000
85 #define CURLINFO_MASK 0x0fffff
86 #define CURLINFO_TYPEMASK 0xf00000
89 CURLINFO_NONE, /* first, never use this */
90 CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1,
91 CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2,
92 CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3,
93 CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4,
94 CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5,
95 CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
96 CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7,
97 CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8,
98 CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9,
99 CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10,
100 CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11,
101 CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12,
102 CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13,
103 CURLINFO_FILETIME = CURLINFO_LONG + 14,
104 CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15,
105 CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16,
106 CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
107 CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18,
108 CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19,
109 CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20,
110 CURLINFO_PRIVATE = CURLINFO_STRING + 21,
111 CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22,
112 CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23,
113 CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24,
114 CURLINFO_OS_ERRNO = CURLINFO_LONG + 25,
115 CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26,
116 CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27
122 CURLMSG_NONE, /* first, not used */
123 CURLMSG_DONE, /* This easy handle has completed. 'result' contains
124 the CURLcode of the transfer */
130 CURLMSG msg; /* what this message means */
131 CURL *easy_handle; /* the handle it concerns */
134 void *whatever; /* message-specific data */
135 CURLcode result; /* return code for transfer */
141 static void (*qcurl_global_init) (long flags);
142 static void (*qcurl_global_cleanup) (void);
144 static CURL * (*qcurl_easy_init) (void);
145 static void (*qcurl_easy_cleanup) (CURL *handle);
146 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
147 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
148 static const char * (*qcurl_easy_strerror) (CURLcode);
150 static CURLM * (*qcurl_multi_init) (void);
151 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
152 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
153 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
154 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
155 static void (*qcurl_multi_cleanup) (CURLM *);
156 static const char * (*qcurl_multi_strerror) (CURLcode);
157 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
158 static void (*qcurl_slist_free_all) (curl_slist *list);
160 static dllfunction_t curlfuncs[] =
162 {"curl_global_init", (void **) &qcurl_global_init},
163 {"curl_global_cleanup", (void **) &qcurl_global_cleanup},
164 {"curl_easy_init", (void **) &qcurl_easy_init},
165 {"curl_easy_cleanup", (void **) &qcurl_easy_cleanup},
166 {"curl_easy_setopt", (void **) &qcurl_easy_setopt},
167 {"curl_easy_strerror", (void **) &qcurl_easy_strerror},
168 {"curl_easy_getinfo", (void **) &qcurl_easy_getinfo},
169 {"curl_multi_init", (void **) &qcurl_multi_init},
170 {"curl_multi_perform", (void **) &qcurl_multi_perform},
171 {"curl_multi_add_handle", (void **) &qcurl_multi_add_handle},
172 {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
173 {"curl_multi_info_read", (void **) &qcurl_multi_info_read},
174 {"curl_multi_cleanup", (void **) &qcurl_multi_cleanup},
175 {"curl_multi_strerror", (void **) &qcurl_multi_strerror},
176 {"curl_slist_append", (void **) &qcurl_slist_append},
177 {"curl_slist_free_all", (void **) &qcurl_slist_free_all},
181 // Handle for CURL DLL
182 static dllhandle_t curl_dll = NULL;
183 // will be checked at many places to find out if qcurl calls are allowed
185 typedef struct downloadinfo_s
187 char filename[MAX_OSPATH];
191 fs_offset_t startpos;
195 unsigned long bytes_received; // for buffer
196 double bytes_received_curl; // for throttling
197 double bytes_sent_curl; // for throttling
198 struct downloadinfo_s *next, *prev;
201 curl_slist *slist; // http headers
203 unsigned char *buffer;
205 curl_callback_t callback;
208 const unsigned char *postbuf;
210 const char *post_content_type;
213 static downloadinfo *downloads = NULL;
214 static int numdownloads = 0;
216 static qboolean noclear = FALSE;
218 static int numdownloads_fail = 0;
219 static int numdownloads_success = 0;
220 static int numdownloads_added = 0;
221 static char command_when_done[256] = "";
222 static char command_when_error[256] = "";
228 Sets the command which is to be executed when the last download completes AND
229 all downloads since last server connect ended with a successful status.
230 Setting the command to NULL clears it.
233 void Curl_CommandWhenDone(const char *cmd)
238 strlcpy(command_when_done, cmd, sizeof(command_when_done));
240 *command_when_done = 0;
245 Do not use yet. Not complete.
246 Problem: what counts as an error?
249 void Curl_CommandWhenError(const char *cmd)
254 strlcpy(command_when_error, cmd, sizeof(command_when_error));
256 *command_when_error = 0;
261 Curl_Clear_forthismap
263 Clears the "will disconnect on failure" flags.
266 void Curl_Clear_forthismap(void)
271 for(di = downloads; di; di = di->next)
272 di->forthismap = false;
273 Curl_CommandWhenError(NULL);
274 Curl_CommandWhenDone(NULL);
275 numdownloads_fail = 0;
276 numdownloads_success = 0;
277 numdownloads_added = 0;
284 Returns true if a download needed for the current game is running.
287 qboolean Curl_Have_forthismap(void)
289 return numdownloads_added != 0;
292 void Curl_Register_predownload(void)
294 Curl_CommandWhenDone("cl_begindownloads");
295 Curl_CommandWhenError("cl_begindownloads");
300 Curl_CheckCommandWhenDone
302 Checks if a "done command" is to be executed.
303 All downloads finished, at least one success since connect, no single failure
304 -> execute the command.
306 static void Curl_CheckCommandWhenDone(void)
310 if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
312 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
314 Cbuf_AddText(command_when_done);
316 Curl_Clear_forthismap();
318 else if(numdownloads_added && numdownloads_fail && *command_when_error)
320 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
322 Cbuf_AddText(command_when_error);
324 Curl_Clear_forthismap();
335 static qboolean CURL_OpenLibrary (void)
337 const char* dllnames [] =
342 #elif defined(MACOSX)
343 "libcurl.4.dylib", // Mac OS X Notyetreleased
344 "libcurl.3.dylib", // Mac OS X Tiger
345 "libcurl.2.dylib", // Mac OS X Panther
349 "libcurl.so", // FreeBSD
359 return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
370 static void CURL_CloseLibrary (void)
372 Sys_UnloadLibrary (&curl_dll);
376 static CURLM *curlm = NULL;
377 static double bytes_received = 0; // used for bandwidth throttling
378 static double bytes_sent = 0; // used for bandwidth throttling
379 static double curltime = 0;
385 fwrite-compatible function that writes the data to a file. libcurl can call
389 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
391 fs_offset_t ret = -1;
392 size_t bytes = size * nmemb;
393 downloadinfo *di = (downloadinfo *) vdi;
397 if(di->bytes_received + bytes <= di->buffersize)
399 memcpy(di->buffer + di->bytes_received, data, bytes);
402 // otherwise: buffer overrun, ret stays -1
407 ret = FS_Write(di->stream, data, bytes);
410 di->bytes_received += bytes;
412 return ret; // why not ret / nmemb?
417 CURL_DOWNLOAD_SUCCESS = 0,
418 CURL_DOWNLOAD_FAILED,
419 CURL_DOWNLOAD_ABORTED,
420 CURL_DOWNLOAD_SERVERERROR
424 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
426 downloadinfo *di = (downloadinfo *) cbdata;
429 case CURLCBSTATUS_OK:
430 Con_DPrintf("Download of %s: OK\n", di->filename);
432 case CURLCBSTATUS_FAILED:
433 Con_DPrintf("Download of %s: FAILED\n", di->filename);
435 case CURLCBSTATUS_ABORTED:
436 Con_DPrintf("Download of %s: ABORTED\n", di->filename);
438 case CURLCBSTATUS_SERVERERROR:
439 Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
441 case CURLCBSTATUS_UNKNOWN:
442 Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
445 Con_DPrintf("Download of %s: %d\n", di->filename, status);
450 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
452 curl_default_callback(status, length_received, buffer, cbdata);
459 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
460 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
461 code from libcurl, or 0, if another error has occurred.
464 static qboolean Curl_Begin(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
465 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
472 case CURL_DOWNLOAD_SUCCESS:
474 di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
476 case CURL_DOWNLOAD_FAILED:
477 di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
479 case CURL_DOWNLOAD_ABORTED:
480 di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
482 case CURL_DOWNLOAD_SERVERERROR:
483 // reopen to enforce it to have zero bytes again
486 FS_Close(di->stream);
487 di->stream = FS_OpenRealFile(di->filename, "wb", false);
491 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
495 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
501 qcurl_multi_remove_handle(curlm, di->curle);
502 qcurl_easy_cleanup(di->curle);
504 qcurl_slist_free_all(di->slist);
507 if(!di->callback && ok && !di->bytes_received)
509 Con_Printf("ERROR: empty file\n");
514 FS_Close(di->stream);
518 ok = FS_AddPack(di->filename, NULL, true);
521 // pack loading failed?
523 // better clear the file again...
524 di->stream = FS_OpenRealFile(di->filename, "wb", false);
525 FS_Close(di->stream);
527 if(di->startpos && !di->callback && !di->post_content_type)
529 // this was a resume?
530 // then try to redownload it without reporting the error
531 Curl_Begin(di->url, di->maxspeed, di->filename, di->ispak, di->forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
532 di->forthismap = false; // don't count the error
538 di->prev->next = di->next;
540 downloads = di->next;
542 di->next->prev = di->prev;
548 ++numdownloads_success;
559 Returns a "cleaned up" URL for display (to strip login data)
562 static const char *CleanURL(const char *url)
564 static char urlbuf[1024];
565 const char *p, *q, *r;
567 // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
568 p = strstr(url, "://");
571 q = strchr(p + 3, '@');
574 r = strchr(p + 3, '/');
577 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s%s", (int)(p - url + 3), url, q + 1);
588 CheckPendingDownloads
590 checks if there are free download slots to start new downloads in.
591 To not start too many downloads at once, only one download is added at a time,
592 up to a maximum number of cl_curl_maxdownloads are running.
595 static void CheckPendingDownloads(void)
599 if(numdownloads < cl_curl_maxdownloads.integer)
602 for(di = downloads; di; di = di->next)
608 Con_Printf("Downloading %s -> %s", CleanURL(di->url), di->filename);
610 di->stream = FS_OpenRealFile(di->filename, "ab", false);
613 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
614 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
617 FS_Seek(di->stream, 0, SEEK_END);
618 di->startpos = FS_Tell(di->stream);
621 Con_Printf(", resuming from position %ld", (long) di->startpos);
626 Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url));
630 di->curle = qcurl_easy_init();
632 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
633 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
634 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
635 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
636 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
637 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
638 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
639 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
640 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
641 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
642 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
643 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
645 Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
646 //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
648 if(di->post_content_type)
650 qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
651 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
652 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
653 di->slist = qcurl_slist_append(di->slist, va("Content-Type: %s", di->post_content_type));
655 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
657 qcurl_multi_add_handle(curlm, di->curle);
660 if(numdownloads >= cl_curl_maxdownloads.integer)
671 this function MUST be called before using anything else in this file.
672 On Win32, this must be called AFTER WSAStartup has been done!
680 qcurl_global_init(CURL_GLOBAL_NOTHING);
681 curlm = qcurl_multi_init();
688 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
691 void Curl_ClearRequirements(void);
692 void Curl_Shutdown(void)
696 Curl_ClearRequirements();
706 Finds the internal information block for a download given by file name.
709 static downloadinfo *Curl_Find(const char *filename)
714 for(di = downloads; di; di = di->next)
715 if(!strcasecmp(di->filename, filename))
720 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
725 for(di = downloads; di; )
727 if(di->callback == callback && di->callback_data == cbdata)
729 di->callback = curl_quiet_callback; // do NOT call the callback
730 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
742 Starts a download of a given URL to the file name portion of this URL (or name
743 if given) in the "dlcache/" folder.
746 static qboolean Curl_Begin(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
760 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
761 p = strchr(URL, ':');
764 if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
766 char addressstring[128];
768 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
769 q = strchr(addressstring, ':');
771 q = addressstring + strlen(addressstring);
774 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
780 // Note: This extraction of the file name portion is NOT entirely correct.
782 // It does the following:
784 // http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
785 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
786 // http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
788 // However, I'd like to keep this "buggy" behavior so that PHP script
789 // authors can write download scripts without having to enable
790 // AcceptPathInfo on Apache. They just have to ensure that their script
791 // can be called with such a "fake" path name like
792 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
794 // By the way, such PHP scripts should either send the file or a
795 // "Location:" redirect; PHP code example:
797 // header("Location: http://www.example.com/");
799 // By the way, this will set User-Agent to something like
800 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
801 // dp://serverhost:serverport/ so you can filter on this; an example
802 // httpd log file line might be:
804 // 141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
807 name = CleanURL(URL);
811 p = strrchr(name, '/');
812 p = p ? (p+1) : name;
814 length = q ? (size_t)(q - p) : strlen(p);
815 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
817 name = fn; // make it point back
819 // already downloading the file?
821 downloadinfo *di = Curl_Find(fn);
824 Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url));
826 // however, if it was not for this map yet...
827 if(forthismap && !di->forthismap)
829 di->forthismap = true;
830 // this "fakes" a download attempt so the client will wait for
831 // the download to finish and then reconnect
832 ++numdownloads_added;
839 if(ispak && FS_FileExists(fn))
841 qboolean already_loaded;
842 if(FS_AddPack(fn, &already_loaded, true))
844 Con_DPrintf("%s already exists, not downloading!\n", fn);
846 Con_DPrintf("(pak was already loaded)\n");
851 ++numdownloads_added;
852 ++numdownloads_success;
860 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
864 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
866 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
868 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
870 f = FS_OpenRealFile(fn, "wb", false);
884 // if we get here, we actually want to download... so first verify the
885 // URL scheme (so one can't read local files using file://)
886 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
888 Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
893 ++numdownloads_added;
894 di = (downloadinfo *) Z_Malloc(sizeof(*di));
895 strlcpy(di->filename, name, sizeof(di->filename));
896 strlcpy(di->url, URL, sizeof(di->url));
897 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
898 di->forthismap = forthismap;
903 di->ispak = (ispak && !buf);
904 di->maxspeed = maxspeed;
905 di->bytes_received = 0;
906 di->bytes_received_curl = 0;
907 di->bytes_sent_curl = 0;
908 di->next = downloads;
914 di->buffersize = bufsize;
917 di->callback = curl_default_callback;
918 di->callback_data = di;
922 di->callback = callback;
923 di->callback_data = cbdata;
926 if(post_content_type)
928 di->post_content_type = post_content_type;
929 di->postbuf = postbuf;
930 di->postbufsize = postbufsize;
934 di->post_content_type = NULL;
944 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
946 return Curl_Begin(URL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
948 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
950 return Curl_Begin(URL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
952 qboolean Curl_Begin_ToMemory_POST(const char *URL, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
954 return Curl_Begin(URL, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
961 call this regularily as this will always download as much as possible without
972 if(!cl_curl_enabled.integer)
978 Curl_CheckCommandWhenDone();
983 if(realtime < curltime) // throttle
992 mc = qcurl_multi_perform(curlm, &remaining);
994 while(mc == CURLM_CALL_MULTI_PERFORM);
996 for(di = downloads; di; di = di->next)
999 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1000 bytes_sent += (b - di->bytes_sent_curl);
1001 di->bytes_sent_curl = b;
1002 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1003 bytes_sent += (b - di->bytes_received_curl);
1004 di->bytes_received_curl = b;
1009 CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1012 if(msg->msg == CURLMSG_DONE)
1014 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1016 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1017 result = msg->data.result;
1020 failed = CURL_DOWNLOAD_FAILED;
1025 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1028 case 4: // e.g. 404?
1029 case 5: // e.g. 500?
1030 failed = CURL_DOWNLOAD_SERVERERROR;
1031 result = (CURLcode) code;
1036 Curl_EndDownload(di, failed, result);
1041 CheckPendingDownloads();
1043 // when will we curl the next time?
1044 // we will wait a bit to ensure our download rate is kept.
1045 // we now know that realtime >= curltime... so set up a new curltime
1047 // use the slowest allowing download to derive the maxspeed... this CAN
1048 // be done better, but maybe later
1049 maxspeed = cl_curl_maxspeed.value;
1050 for(di = downloads; di; di = di->next)
1051 if(di->maxspeed > 0)
1052 if(di->maxspeed < maxspeed || maxspeed <= 0)
1053 maxspeed = di->maxspeed;
1057 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1058 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1063 curltime = realtime;
1067 ====================
1070 Stops ALL downloads.
1071 ====================
1073 void Curl_CancelAll(void)
1080 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1081 // INVARIANT: downloads will point to the next download after that!
1086 ====================
1089 returns true iff there is a download running.
1090 ====================
1092 qboolean Curl_Running(void)
1097 return downloads != NULL;
1101 ====================
1102 Curl_GetDownloadAmount
1104 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1105 for the given download.
1106 ====================
1108 static double Curl_GetDownloadAmount(downloadinfo *di)
1115 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1117 return (di->startpos + di->bytes_received) / (di->startpos + length);
1126 ====================
1127 Curl_GetDownloadSpeed
1129 returns the speed of the given download in bytes per second
1130 ====================
1132 static double Curl_GetDownloadSpeed(downloadinfo *di)
1139 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1147 ====================
1150 prints the download list
1151 ====================
1153 // TODO rewrite using Curl_GetDownloadInfo?
1154 static void Curl_Info_f(void)
1161 Con_Print("Currently running downloads:\n");
1162 for(di = downloads; di; di = di->next)
1164 double speed, percent;
1165 Con_Printf(" %s -> %s ", CleanURL(di->url), di->filename);
1166 percent = 100.0 * Curl_GetDownloadAmount(di);
1167 speed = Curl_GetDownloadSpeed(di);
1169 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1171 Con_Print("(queued)\n");
1176 Con_Print("No downloads running.\n");
1181 ====================
1184 implements the "curl" console command
1188 curl --cancel filename
1193 curl [--pak] [--forthismap] [--for filename filename...] url
1194 --pak: after downloading, load the package into the virtual file system
1195 --for filename...: only download of at least one of the named files is missing
1196 --forthismap: don't reconnect on failure
1198 curl --clear_autodownload
1199 clears the download success/failure counters
1201 curl --finish_autodownload
1202 if at least one download has been started, disconnect and drop to the menu
1203 once the last download completes successfully, reconnect to the current server
1204 ====================
1206 void Curl_Curl_f(void)
1208 double maxspeed = 0;
1211 qboolean pak = false;
1212 qboolean forthismap = false;
1214 const char *name = 0;
1218 Con_Print("libcurl DLL not found, this command is inactive.\n");
1222 if(!cl_curl_enabled.integer)
1224 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1230 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1234 url = Cmd_Argv(Cmd_Argc() - 1);
1237 for(i = 1; i != end; ++i)
1239 const char *a = Cmd_Argv(i);
1240 if(!strcmp(a, "--info"))
1245 else if(!strcmp(a, "--cancel"))
1247 if(i == end - 1) // last argument
1251 downloadinfo *di = Curl_Find(url);
1253 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1255 Con_Print("download not found\n");
1259 else if(!strcmp(a, "--pak"))
1263 else if(!strcmp(a, "--for")) // must be last option
1265 for(i = i + 1; i != end - 1; ++i)
1267 if(!FS_FileExists(Cmd_Argv(i)))
1268 goto needthefile; // why can't I have a "double break"?
1270 // if we get here, we have all the files...
1273 else if(!strcmp(a, "--forthismap"))
1277 else if(!strcmp(a, "--as"))
1285 else if(!strcmp(a, "--clear_autodownload"))
1287 // mark all running downloads as "not for this map", so if they
1288 // fail, it does not matter
1289 Curl_Clear_forthismap();
1292 else if(!strcmp(a, "--finish_autodownload"))
1294 if(numdownloads_added)
1296 char donecommand[256];
1299 if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1301 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1302 Curl_CommandWhenDone(donecommand);
1306 Curl_CheckCommandWhenDone();
1309 Curl_Register_predownload();
1314 else if(!strncmp(a, "--maxspeed=", 11))
1316 maxspeed = atof(a + 11);
1320 Con_Printf("curl: invalid option %s\n", a);
1321 // but we ignore the option
1326 Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1330 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1332 Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1336 void Curl_CurlCat_f(void)
1339 const char *url = Cmd_Argv(1);
1340 buf = Z_Malloc(16384);
1341 Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1346 ====================
1349 loads the commands and cvars this library uses
1350 ====================
1352 void Curl_Init_Commands(void)
1354 Cvar_RegisterVariable (&cl_curl_enabled);
1355 Cvar_RegisterVariable (&cl_curl_maxdownloads);
1356 Cvar_RegisterVariable (&cl_curl_maxspeed);
1357 Cvar_RegisterVariable (&sv_curl_defaulturl);
1358 Cvar_RegisterVariable (&sv_curl_serverpackages);
1359 Cvar_RegisterVariable (&sv_curl_maxspeed);
1360 Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1361 //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1365 ====================
1366 Curl_GetDownloadInfo
1368 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1369 The number of elements in the array is returned in int *nDownloads.
1370 const char **additional_info may be set to a string of additional user
1371 information, or to NULL if no such display shall occur. The returned
1372 array must be freed later using Z_Free.
1373 ====================
1375 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1379 Curl_downloadinfo_t *downinfo;
1380 static char addinfo[128];
1386 *additional_info = NULL;
1391 for(di = downloads; di; di = di->next)
1394 downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1396 for(di = downloads; di; di = di->next)
1398 // do not show infobars for background downloads
1399 if(developer.integer <= 0)
1402 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1405 downinfo[i].progress = Curl_GetDownloadAmount(di);
1406 downinfo[i].speed = Curl_GetDownloadSpeed(di);
1407 downinfo[i].queued = false;
1411 downinfo[i].queued = true;
1418 // TODO: can I clear command_when_done as soon as the first download fails?
1419 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1421 if(!strncmp(command_when_done, "connect ", 8))
1422 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1423 else if(!strcmp(command_when_done, "cl_begindownloads"))
1424 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1426 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1427 *additional_info = addinfo;
1430 *additional_info = NULL;
1439 ====================
1442 finds the URL where to find a given package.
1444 For this, it reads a file "curl_urls.txt" of the following format:
1447 revdm*.pk3 http://revdm/downloads/are/here/
1448 * http://any/other/stuff/is/here/
1450 The URLs should end in /. If not, downloads will still work, but the cached files
1451 can't be just put into the data directory with the same download configuration
1452 (you might want to do this if you want to tag downloaded files from your
1453 server, but you should not). "-" means "don't download".
1455 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1458 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1459 this file for obvious reasons.
1460 ====================
1462 static const char *Curl_FindPackURL(const char *filename)
1464 static char foundurl[1024];
1465 fs_offset_t filesize;
1466 char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1469 // read lines of format "pattern url"
1471 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1472 qboolean eof = false;
1484 if(pattern && url && patternend)
1490 if(matchpattern(filename, pattern, true))
1492 strlcpy(foundurl, url, sizeof(foundurl));
1504 if(pattern && !patternend)
1506 else if(url && !urlend)
1512 else if(pattern && patternend && !url)
1521 return sv_curl_defaulturl.string;
1524 typedef struct requirement_s
1526 struct requirement_s *next;
1527 char filename[MAX_OSPATH];
1530 static requirement *requirements = NULL;
1534 ====================
1537 Adds the given file to the list of requirements.
1538 ====================
1540 void Curl_RequireFile(const char *filename)
1542 requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1543 req->next = requirements;
1544 strlcpy(req->filename, filename, sizeof(req->filename));
1549 ====================
1550 Curl_ClearRequirements
1552 Clears the list of required files for playing on the current map.
1553 This should be called at every map change.
1554 ====================
1556 void Curl_ClearRequirements(void)
1561 requirement *req = requirements;
1562 requirements = requirements->next;
1565 p = sv_curl_serverpackages.string;
1566 Con_DPrintf("Require all of: %s\n", p);
1567 while(COM_ParseToken_Simple(&p, false, false))
1569 Con_DPrintf("Require: %s\n", com_token);
1570 Curl_RequireFile(com_token);
1575 ====================
1576 Curl_SendRequirements
1578 Makes the current host_clients download all files he needs.
1579 This is done by sending him the following console commands:
1581 curl --clear_autodownload
1582 curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1583 curl --finish_autodownload
1584 ====================
1586 void Curl_SendRequirements(void)
1588 // for each requirement, find the pack name
1589 char sendbuffer[4096] = "";
1591 qboolean foundone = false;
1593 for(req = requirements; req; req = req->next)
1596 const char *thispack = FS_WhichPack(req->filename);
1597 const char *packurl;
1602 p = strrchr(thispack, '/');
1606 packurl = Curl_FindPackURL(thispack);
1608 if(packurl && *packurl && strcmp(packurl, "-"))
1611 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1613 strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1614 strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1615 if(sv_curl_maxspeed.value > 0)
1616 dpsnprintf(sendbuffer + strlen(sendbuffer), sizeof(sendbuffer) - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1617 strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1618 strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1619 strlcat(sendbuffer, " ", sizeof(sendbuffer));
1620 strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1621 strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1622 strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1629 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1631 if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1632 Host_ClientCommands("%s", sendbuffer);
1634 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");