5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
9 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
10 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
13 =================================================================
15 Minimal set of definitions from libcurl
17 WARNING: for a matter of simplicity, several pointer types are
18 casted to "void*", and most enumerated values are not included
20 =================================================================
23 typedef struct CURL_s CURL;
24 typedef struct CURLM_s CURLM;
25 typedef struct curl_slist curl_slist;
33 CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
37 #define CURL_GLOBAL_NOTHING 0
38 #define CURL_GLOBAL_SSL 1
39 #define CURL_GLOBAL_WIN32 2
40 #define CURLOPTTYPE_LONG 0
41 #define CURLOPTTYPE_OBJECTPOINT 10000
42 #define CURLOPTTYPE_FUNCTIONPOINT 20000
43 #define CURLOPTTYPE_OFF_T 30000
44 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
47 CINIT(WRITEDATA, OBJECTPOINT, 1),
48 CINIT(URL, OBJECTPOINT, 2),
49 CINIT(ERRORBUFFER, OBJECTPOINT, 10),
50 CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
51 CINIT(POSTFIELDS, OBJECTPOINT, 15),
52 CINIT(REFERER, OBJECTPOINT, 16),
53 CINIT(USERAGENT, OBJECTPOINT, 18),
54 CINIT(LOW_SPEED_LIMIT, LONG , 19),
55 CINIT(LOW_SPEED_TIME, LONG, 20),
56 CINIT(RESUME_FROM, LONG, 21),
57 CINIT(HTTPHEADER, OBJECTPOINT, 23),
58 CINIT(POST, LONG, 47), /* HTTP POST method */
59 CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */
60 CINIT(POSTFIELDSIZE, LONG, 60),
61 CINIT(PRIVATE, OBJECTPOINT, 103),
62 CINIT(PROTOCOLS, LONG, 181),
63 CINIT(REDIR_PROTOCOLS, LONG, 182)
66 #define CURLPROTO_HTTP (1<<0)
67 #define CURLPROTO_HTTPS (1<<1)
68 #define CURLPROTO_FTP (1<<2)
72 CURLINFO_HEADER_IN, /* 1 */
73 CURLINFO_HEADER_OUT, /* 2 */
74 CURLINFO_DATA_IN, /* 3 */
75 CURLINFO_DATA_OUT, /* 4 */
76 CURLINFO_SSL_DATA_IN, /* 5 */
77 CURLINFO_SSL_DATA_OUT, /* 6 */
81 #define CURLINFO_STRING 0x100000
82 #define CURLINFO_LONG 0x200000
83 #define CURLINFO_DOUBLE 0x300000
84 #define CURLINFO_SLIST 0x400000
85 #define CURLINFO_MASK 0x0fffff
86 #define CURLINFO_TYPEMASK 0xf00000
89 CURLINFO_NONE, /* first, never use this */
90 CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1,
91 CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2,
92 CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3,
93 CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4,
94 CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5,
95 CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
96 CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7,
97 CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8,
98 CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9,
99 CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10,
100 CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11,
101 CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12,
102 CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13,
103 CURLINFO_FILETIME = CURLINFO_LONG + 14,
104 CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15,
105 CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16,
106 CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
107 CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18,
108 CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19,
109 CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20,
110 CURLINFO_PRIVATE = CURLINFO_STRING + 21,
111 CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22,
112 CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23,
113 CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24,
114 CURLINFO_OS_ERRNO = CURLINFO_LONG + 25,
115 CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26,
116 CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27
122 CURLMSG_NONE, /* first, not used */
123 CURLMSG_DONE, /* This easy handle has completed. 'result' contains
124 the CURLcode of the transfer */
130 CURLMSG msg; /* what this message means */
131 CURL *easy_handle; /* the handle it concerns */
134 void *whatever; /* message-specific data */
135 CURLcode result; /* return code for transfer */
141 static void (*qcurl_global_init) (long flags);
142 static void (*qcurl_global_cleanup) (void);
144 static CURL * (*qcurl_easy_init) (void);
145 static void (*qcurl_easy_cleanup) (CURL *handle);
146 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
147 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
148 static const char * (*qcurl_easy_strerror) (CURLcode);
150 static CURLM * (*qcurl_multi_init) (void);
151 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
152 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
153 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
154 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
155 static void (*qcurl_multi_cleanup) (CURLM *);
156 static const char * (*qcurl_multi_strerror) (CURLcode);
157 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
158 static void (*qcurl_slist_free_all) (curl_slist *list);
160 static dllfunction_t curlfuncs[] =
162 {"curl_global_init", (void **) &qcurl_global_init},
163 {"curl_global_cleanup", (void **) &qcurl_global_cleanup},
164 {"curl_easy_init", (void **) &qcurl_easy_init},
165 {"curl_easy_cleanup", (void **) &qcurl_easy_cleanup},
166 {"curl_easy_setopt", (void **) &qcurl_easy_setopt},
167 {"curl_easy_strerror", (void **) &qcurl_easy_strerror},
168 {"curl_easy_getinfo", (void **) &qcurl_easy_getinfo},
169 {"curl_multi_init", (void **) &qcurl_multi_init},
170 {"curl_multi_perform", (void **) &qcurl_multi_perform},
171 {"curl_multi_add_handle", (void **) &qcurl_multi_add_handle},
172 {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
173 {"curl_multi_info_read", (void **) &qcurl_multi_info_read},
174 {"curl_multi_cleanup", (void **) &qcurl_multi_cleanup},
175 {"curl_multi_strerror", (void **) &qcurl_multi_strerror},
176 {"curl_slist_append", (void **) &qcurl_slist_append},
177 {"curl_slist_free_all", (void **) &qcurl_slist_free_all},
181 // Handle for CURL DLL
182 static dllhandle_t curl_dll = NULL;
183 // will be checked at many places to find out if qcurl calls are allowed
185 typedef struct downloadinfo_s
187 char filename[MAX_OSPATH];
191 fs_offset_t startpos;
195 unsigned long bytes_received; // for buffer
196 double bytes_received_curl; // for throttling
197 double bytes_sent_curl; // for throttling
198 struct downloadinfo_s *next, *prev;
201 curl_slist *slist; // http headers
203 unsigned char *buffer;
205 curl_callback_t callback;
208 const unsigned char *postbuf;
210 const char *post_content_type;
211 const char *extraheaders;
214 static downloadinfo *downloads = NULL;
215 static int numdownloads = 0;
217 static qboolean noclear = FALSE;
219 static int numdownloads_fail = 0;
220 static int numdownloads_success = 0;
221 static int numdownloads_added = 0;
222 static char command_when_done[256] = "";
223 static char command_when_error[256] = "";
229 Sets the command which is to be executed when the last download completes AND
230 all downloads since last server connect ended with a successful status.
231 Setting the command to NULL clears it.
234 void Curl_CommandWhenDone(const char *cmd)
239 strlcpy(command_when_done, cmd, sizeof(command_when_done));
241 *command_when_done = 0;
246 Do not use yet. Not complete.
247 Problem: what counts as an error?
250 void Curl_CommandWhenError(const char *cmd)
255 strlcpy(command_when_error, cmd, sizeof(command_when_error));
257 *command_when_error = 0;
262 Curl_Clear_forthismap
264 Clears the "will disconnect on failure" flags.
267 void Curl_Clear_forthismap(void)
272 for(di = downloads; di; di = di->next)
273 di->forthismap = false;
274 Curl_CommandWhenError(NULL);
275 Curl_CommandWhenDone(NULL);
276 numdownloads_fail = 0;
277 numdownloads_success = 0;
278 numdownloads_added = 0;
285 Returns true if a download needed for the current game is running.
288 qboolean Curl_Have_forthismap(void)
290 return numdownloads_added != 0;
293 void Curl_Register_predownload(void)
295 Curl_CommandWhenDone("cl_begindownloads");
296 Curl_CommandWhenError("cl_begindownloads");
301 Curl_CheckCommandWhenDone
303 Checks if a "done command" is to be executed.
304 All downloads finished, at least one success since connect, no single failure
305 -> execute the command.
307 static void Curl_CheckCommandWhenDone(void)
311 if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
313 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
315 Cbuf_AddText(command_when_done);
317 Curl_Clear_forthismap();
319 else if(numdownloads_added && numdownloads_fail && *command_when_error)
321 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
323 Cbuf_AddText(command_when_error);
325 Curl_Clear_forthismap();
336 static qboolean CURL_OpenLibrary (void)
338 const char* dllnames [] =
343 #elif defined(MACOSX)
344 "libcurl.4.dylib", // Mac OS X Notyetreleased
345 "libcurl.3.dylib", // Mac OS X Tiger
346 "libcurl.2.dylib", // Mac OS X Panther
350 "libcurl.so", // FreeBSD
360 return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
371 static void CURL_CloseLibrary (void)
373 Sys_UnloadLibrary (&curl_dll);
377 static CURLM *curlm = NULL;
378 static double bytes_received = 0; // used for bandwidth throttling
379 static double bytes_sent = 0; // used for bandwidth throttling
380 static double curltime = 0;
386 fwrite-compatible function that writes the data to a file. libcurl can call
390 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
392 fs_offset_t ret = -1;
393 size_t bytes = size * nmemb;
394 downloadinfo *di = (downloadinfo *) vdi;
398 if(di->bytes_received + bytes <= di->buffersize)
400 memcpy(di->buffer + di->bytes_received, data, bytes);
403 // otherwise: buffer overrun, ret stays -1
408 ret = FS_Write(di->stream, data, bytes);
411 di->bytes_received += bytes;
413 return ret; // why not ret / nmemb?
418 CURL_DOWNLOAD_SUCCESS = 0,
419 CURL_DOWNLOAD_FAILED,
420 CURL_DOWNLOAD_ABORTED,
421 CURL_DOWNLOAD_SERVERERROR
425 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
427 downloadinfo *di = (downloadinfo *) cbdata;
430 case CURLCBSTATUS_OK:
431 Con_DPrintf("Download of %s: OK\n", di->filename);
433 case CURLCBSTATUS_FAILED:
434 Con_DPrintf("Download of %s: FAILED\n", di->filename);
436 case CURLCBSTATUS_ABORTED:
437 Con_DPrintf("Download of %s: ABORTED\n", di->filename);
439 case CURLCBSTATUS_SERVERERROR:
440 Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
442 case CURLCBSTATUS_UNKNOWN:
443 Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
446 Con_DPrintf("Download of %s: %d\n", di->filename, status);
451 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
453 curl_default_callback(status, length_received, buffer, cbdata);
460 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
461 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
462 code from libcurl, or 0, if another error has occurred.
465 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
466 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
473 case CURL_DOWNLOAD_SUCCESS:
475 di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
477 case CURL_DOWNLOAD_FAILED:
478 di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
480 case CURL_DOWNLOAD_ABORTED:
481 di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
483 case CURL_DOWNLOAD_SERVERERROR:
484 // reopen to enforce it to have zero bytes again
487 FS_Close(di->stream);
488 di->stream = FS_OpenRealFile(di->filename, "wb", false);
492 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
496 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
502 qcurl_multi_remove_handle(curlm, di->curle);
503 qcurl_easy_cleanup(di->curle);
505 qcurl_slist_free_all(di->slist);
508 if(!di->callback && ok && !di->bytes_received)
510 Con_Printf("ERROR: empty file\n");
515 FS_Close(di->stream);
519 ok = FS_AddPack(di->filename, NULL, true);
522 // pack loading failed?
524 // better clear the file again...
525 di->stream = FS_OpenRealFile(di->filename, "wb", false);
526 FS_Close(di->stream);
528 if(di->startpos && !di->callback)
530 // this was a resume?
531 // then try to redownload it without reporting the error
532 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
533 di->forthismap = false; // don't count the error
539 di->prev->next = di->next;
541 downloads = di->next;
543 di->next->prev = di->prev;
549 ++numdownloads_success;
560 Returns a "cleaned up" URL for display (to strip login data)
563 static const char *CleanURL(const char *url)
565 static char urlbuf[1024];
566 const char *p, *q, *r;
568 // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
569 p = strstr(url, "://");
572 q = strchr(p + 3, '@');
575 r = strchr(p + 3, '/');
578 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s%s", (int)(p - url + 3), url, q + 1);
589 CheckPendingDownloads
591 checks if there are free download slots to start new downloads in.
592 To not start too many downloads at once, only one download is added at a time,
593 up to a maximum number of cl_curl_maxdownloads are running.
596 static void CheckPendingDownloads(void)
601 if(numdownloads < cl_curl_maxdownloads.integer)
604 for(di = downloads; di; di = di->next)
610 Con_Printf("Downloading %s -> %s", CleanURL(di->url), di->filename);
612 di->stream = FS_OpenRealFile(di->filename, "ab", false);
615 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
616 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
619 FS_Seek(di->stream, 0, SEEK_END);
620 di->startpos = FS_Tell(di->stream);
623 Con_Printf(", resuming from position %ld", (long) di->startpos);
628 Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url));
632 di->curle = qcurl_easy_init();
634 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
635 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
636 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
637 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
638 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
639 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
640 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
641 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
642 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
643 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
644 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
645 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
647 Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
648 //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
650 if(di->post_content_type)
652 qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
653 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
654 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
655 di->slist = qcurl_slist_append(di->slist, va("Content-Type: %s", di->post_content_type));
658 // parse extra headers into slist
659 // \n separated list!
660 h = di->extraheaders;
663 const char *hh = strchr(h, '\n');
666 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
667 memcpy(buf, h, hh - h);
669 di->slist = qcurl_slist_append(di->slist, buf);
674 di->slist = qcurl_slist_append(di->slist, h);
679 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
681 qcurl_multi_add_handle(curlm, di->curle);
684 if(numdownloads >= cl_curl_maxdownloads.integer)
695 this function MUST be called before using anything else in this file.
696 On Win32, this must be called AFTER WSAStartup has been done!
704 qcurl_global_init(CURL_GLOBAL_NOTHING);
705 curlm = qcurl_multi_init();
712 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
715 void Curl_ClearRequirements(void);
716 void Curl_Shutdown(void)
720 Curl_ClearRequirements();
730 Finds the internal information block for a download given by file name.
733 static downloadinfo *Curl_Find(const char *filename)
738 for(di = downloads; di; di = di->next)
739 if(!strcasecmp(di->filename, filename))
744 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
749 for(di = downloads; di; )
751 if(di->callback == callback && di->callback_data == cbdata)
753 di->callback = curl_quiet_callback; // do NOT call the callback
754 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
766 Starts a download of a given URL to the file name portion of this URL (or name
767 if given) in the "dlcache/" folder.
770 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
784 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
785 p = strchr(URL, ':');
788 if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
790 char addressstring[128];
792 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
793 q = strchr(addressstring, ':');
795 q = addressstring + strlen(addressstring);
798 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
804 // Note: This extraction of the file name portion is NOT entirely correct.
806 // It does the following:
808 // http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
809 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
810 // http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
812 // However, I'd like to keep this "buggy" behavior so that PHP script
813 // authors can write download scripts without having to enable
814 // AcceptPathInfo on Apache. They just have to ensure that their script
815 // can be called with such a "fake" path name like
816 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
818 // By the way, such PHP scripts should either send the file or a
819 // "Location:" redirect; PHP code example:
821 // header("Location: http://www.example.com/");
823 // By the way, this will set User-Agent to something like
824 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
825 // dp://serverhost:serverport/ so you can filter on this; an example
826 // httpd log file line might be:
828 // 141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
831 name = CleanURL(URL);
835 p = strrchr(name, '/');
836 p = p ? (p+1) : name;
838 length = q ? (size_t)(q - p) : strlen(p);
839 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
841 name = fn; // make it point back
843 // already downloading the file?
845 downloadinfo *di = Curl_Find(fn);
848 Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url));
850 // however, if it was not for this map yet...
851 if(forthismap && !di->forthismap)
853 di->forthismap = true;
854 // this "fakes" a download attempt so the client will wait for
855 // the download to finish and then reconnect
856 ++numdownloads_added;
863 if(ispak && FS_FileExists(fn))
865 qboolean already_loaded;
866 if(FS_AddPack(fn, &already_loaded, true))
868 Con_DPrintf("%s already exists, not downloading!\n", fn);
870 Con_DPrintf("(pak was already loaded)\n");
875 ++numdownloads_added;
876 ++numdownloads_success;
884 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
888 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
890 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
892 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
894 f = FS_OpenRealFile(fn, "wb", false);
908 // if we get here, we actually want to download... so first verify the
909 // URL scheme (so one can't read local files using file://)
910 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
912 Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
917 ++numdownloads_added;
918 di = (downloadinfo *) Z_Malloc(sizeof(*di));
919 strlcpy(di->filename, name, sizeof(di->filename));
920 strlcpy(di->url, URL, sizeof(di->url));
921 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
922 di->forthismap = forthismap;
927 di->ispak = (ispak && !buf);
928 di->maxspeed = maxspeed;
929 di->bytes_received = 0;
930 di->bytes_received_curl = 0;
931 di->bytes_sent_curl = 0;
932 di->extraheaders = extraheaders;
933 di->next = downloads;
939 di->buffersize = bufsize;
942 di->callback = curl_default_callback;
943 di->callback_data = di;
947 di->callback = callback;
948 di->callback_data = cbdata;
951 if(post_content_type)
953 di->post_content_type = post_content_type;
954 di->postbuf = postbuf;
955 di->postbufsize = postbufsize;
959 di->post_content_type = NULL;
969 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
971 return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
973 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
975 return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
977 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
979 return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
986 call this regularily as this will always download as much as possible without
997 if(!cl_curl_enabled.integer)
1003 Curl_CheckCommandWhenDone();
1008 if(realtime < curltime) // throttle
1017 mc = qcurl_multi_perform(curlm, &remaining);
1019 while(mc == CURLM_CALL_MULTI_PERFORM);
1021 for(di = downloads; di; di = di->next)
1026 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1027 bytes_sent += (b - di->bytes_sent_curl);
1028 di->bytes_sent_curl = b;
1029 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1030 bytes_sent += (b - di->bytes_received_curl);
1031 di->bytes_received_curl = b;
1037 CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1040 if(msg->msg == CURLMSG_DONE)
1042 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1044 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1045 result = msg->data.result;
1048 failed = CURL_DOWNLOAD_FAILED;
1053 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1056 case 4: // e.g. 404?
1057 case 5: // e.g. 500?
1058 failed = CURL_DOWNLOAD_SERVERERROR;
1059 result = (CURLcode) code;
1064 Curl_EndDownload(di, failed, result);
1069 CheckPendingDownloads();
1071 // when will we curl the next time?
1072 // we will wait a bit to ensure our download rate is kept.
1073 // we now know that realtime >= curltime... so set up a new curltime
1075 // use the slowest allowing download to derive the maxspeed... this CAN
1076 // be done better, but maybe later
1077 maxspeed = cl_curl_maxspeed.value;
1078 for(di = downloads; di; di = di->next)
1079 if(di->maxspeed > 0)
1080 if(di->maxspeed < maxspeed || maxspeed <= 0)
1081 maxspeed = di->maxspeed;
1085 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1086 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1091 curltime = realtime;
1095 ====================
1098 Stops ALL downloads.
1099 ====================
1101 void Curl_CancelAll(void)
1108 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1109 // INVARIANT: downloads will point to the next download after that!
1114 ====================
1117 returns true iff there is a download running.
1118 ====================
1120 qboolean Curl_Running(void)
1125 return downloads != NULL;
1129 ====================
1130 Curl_GetDownloadAmount
1132 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1133 for the given download.
1134 ====================
1136 static double Curl_GetDownloadAmount(downloadinfo *di)
1143 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1145 return (di->startpos + di->bytes_received) / (di->startpos + length);
1154 ====================
1155 Curl_GetDownloadSpeed
1157 returns the speed of the given download in bytes per second
1158 ====================
1160 static double Curl_GetDownloadSpeed(downloadinfo *di)
1167 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1175 ====================
1178 prints the download list
1179 ====================
1181 // TODO rewrite using Curl_GetDownloadInfo?
1182 static void Curl_Info_f(void)
1189 Con_Print("Currently running downloads:\n");
1190 for(di = downloads; di; di = di->next)
1192 double speed, percent;
1193 Con_Printf(" %s -> %s ", CleanURL(di->url), di->filename);
1194 percent = 100.0 * Curl_GetDownloadAmount(di);
1195 speed = Curl_GetDownloadSpeed(di);
1197 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1199 Con_Print("(queued)\n");
1204 Con_Print("No downloads running.\n");
1209 ====================
1212 implements the "curl" console command
1216 curl --cancel filename
1221 curl [--pak] [--forthismap] [--for filename filename...] url
1222 --pak: after downloading, load the package into the virtual file system
1223 --for filename...: only download of at least one of the named files is missing
1224 --forthismap: don't reconnect on failure
1226 curl --clear_autodownload
1227 clears the download success/failure counters
1229 curl --finish_autodownload
1230 if at least one download has been started, disconnect and drop to the menu
1231 once the last download completes successfully, reconnect to the current server
1232 ====================
1234 void Curl_Curl_f(void)
1236 double maxspeed = 0;
1239 qboolean pak = false;
1240 qboolean forthismap = false;
1242 const char *name = 0;
1246 Con_Print("libcurl DLL not found, this command is inactive.\n");
1250 if(!cl_curl_enabled.integer)
1252 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1258 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1262 url = Cmd_Argv(Cmd_Argc() - 1);
1265 for(i = 1; i != end; ++i)
1267 const char *a = Cmd_Argv(i);
1268 if(!strcmp(a, "--info"))
1273 else if(!strcmp(a, "--cancel"))
1275 if(i == end - 1) // last argument
1279 downloadinfo *di = Curl_Find(url);
1281 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1283 Con_Print("download not found\n");
1287 else if(!strcmp(a, "--pak"))
1291 else if(!strcmp(a, "--for")) // must be last option
1293 for(i = i + 1; i != end - 1; ++i)
1295 if(!FS_FileExists(Cmd_Argv(i)))
1296 goto needthefile; // why can't I have a "double break"?
1298 // if we get here, we have all the files...
1301 else if(!strcmp(a, "--forthismap"))
1305 else if(!strcmp(a, "--as"))
1313 else if(!strcmp(a, "--clear_autodownload"))
1315 // mark all running downloads as "not for this map", so if they
1316 // fail, it does not matter
1317 Curl_Clear_forthismap();
1320 else if(!strcmp(a, "--finish_autodownload"))
1322 if(numdownloads_added)
1324 char donecommand[256];
1327 if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1329 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1330 Curl_CommandWhenDone(donecommand);
1334 Curl_CheckCommandWhenDone();
1337 Curl_Register_predownload();
1342 else if(!strncmp(a, "--maxspeed=", 11))
1344 maxspeed = atof(a + 11);
1348 Con_Printf("curl: invalid option %s\n", a);
1349 // but we ignore the option
1354 Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1358 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1360 Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1364 void Curl_CurlCat_f(void)
1367 const char *url = Cmd_Argv(1);
1368 buf = Z_Malloc(16384);
1369 Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1374 ====================
1377 loads the commands and cvars this library uses
1378 ====================
1380 void Curl_Init_Commands(void)
1382 Cvar_RegisterVariable (&cl_curl_enabled);
1383 Cvar_RegisterVariable (&cl_curl_maxdownloads);
1384 Cvar_RegisterVariable (&cl_curl_maxspeed);
1385 Cvar_RegisterVariable (&sv_curl_defaulturl);
1386 Cvar_RegisterVariable (&sv_curl_serverpackages);
1387 Cvar_RegisterVariable (&sv_curl_maxspeed);
1388 Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1389 //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1393 ====================
1394 Curl_GetDownloadInfo
1396 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1397 The number of elements in the array is returned in int *nDownloads.
1398 const char **additional_info may be set to a string of additional user
1399 information, or to NULL if no such display shall occur. The returned
1400 array must be freed later using Z_Free.
1401 ====================
1403 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1407 Curl_downloadinfo_t *downinfo;
1408 static char addinfo[128];
1414 *additional_info = NULL;
1419 for(di = downloads; di; di = di->next)
1422 downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1424 for(di = downloads; di; di = di->next)
1426 // do not show infobars for background downloads
1427 if(developer.integer <= 0)
1430 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1433 downinfo[i].progress = Curl_GetDownloadAmount(di);
1434 downinfo[i].speed = Curl_GetDownloadSpeed(di);
1435 downinfo[i].queued = false;
1439 downinfo[i].queued = true;
1446 // TODO: can I clear command_when_done as soon as the first download fails?
1447 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1449 if(!strncmp(command_when_done, "connect ", 8))
1450 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1451 else if(!strcmp(command_when_done, "cl_begindownloads"))
1452 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1454 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1455 *additional_info = addinfo;
1458 *additional_info = NULL;
1467 ====================
1470 finds the URL where to find a given package.
1472 For this, it reads a file "curl_urls.txt" of the following format:
1475 revdm*.pk3 http://revdm/downloads/are/here/
1476 * http://any/other/stuff/is/here/
1478 The URLs should end in /. If not, downloads will still work, but the cached files
1479 can't be just put into the data directory with the same download configuration
1480 (you might want to do this if you want to tag downloaded files from your
1481 server, but you should not). "-" means "don't download".
1483 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1486 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1487 this file for obvious reasons.
1488 ====================
1490 static const char *Curl_FindPackURL(const char *filename)
1492 static char foundurl[1024];
1493 fs_offset_t filesize;
1494 char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1497 // read lines of format "pattern url"
1499 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1500 qboolean eof = false;
1512 if(pattern && url && patternend)
1518 if(matchpattern(filename, pattern, true))
1520 strlcpy(foundurl, url, sizeof(foundurl));
1532 if(pattern && !patternend)
1534 else if(url && !urlend)
1540 else if(pattern && patternend && !url)
1549 return sv_curl_defaulturl.string;
1552 typedef struct requirement_s
1554 struct requirement_s *next;
1555 char filename[MAX_OSPATH];
1558 static requirement *requirements = NULL;
1562 ====================
1565 Adds the given file to the list of requirements.
1566 ====================
1568 void Curl_RequireFile(const char *filename)
1570 requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1571 req->next = requirements;
1572 strlcpy(req->filename, filename, sizeof(req->filename));
1577 ====================
1578 Curl_ClearRequirements
1580 Clears the list of required files for playing on the current map.
1581 This should be called at every map change.
1582 ====================
1584 void Curl_ClearRequirements(void)
1588 requirement *req = requirements;
1589 requirements = requirements->next;
1595 ====================
1596 Curl_SendRequirements
1598 Makes the current host_clients download all files he needs.
1599 This is done by sending him the following console commands:
1601 curl --clear_autodownload
1602 curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1603 curl --finish_autodownload
1604 ====================
1606 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1609 const char *thispack = FS_WhichPack(filename);
1610 const char *packurl;
1615 p = strrchr(thispack, '/');
1619 packurl = Curl_FindPackURL(thispack);
1621 if(packurl && *packurl && strcmp(packurl, "-"))
1624 strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1626 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1627 strlcat(sendbuffer, thispack, sendbuffer_len);
1628 if(sv_curl_maxspeed.value > 0)
1629 dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1630 strlcat(sendbuffer, " --for ", sendbuffer_len);
1631 strlcat(sendbuffer, filename, sendbuffer_len);
1632 strlcat(sendbuffer, " ", sendbuffer_len);
1633 strlcat(sendbuffer, packurl, sendbuffer_len);
1634 strlcat(sendbuffer, thispack, sendbuffer_len);
1635 strlcat(sendbuffer, "\n", sendbuffer_len);
1642 void Curl_SendRequirements(void)
1644 // for each requirement, find the pack name
1645 char sendbuffer[4096] = "";
1647 qboolean foundone = false;
1650 for(req = requirements; req; req = req->next)
1651 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1653 p = sv_curl_serverpackages.string;
1654 while(COM_ParseToken_Simple(&p, false, false))
1655 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1658 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1660 if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1661 Host_ClientCommands("%s", sendbuffer);
1663 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");