6 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
7 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
8 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
9 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
10 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
11 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
14 =================================================================
16 Minimal set of definitions from libcurl
18 WARNING: for a matter of simplicity, several pointer types are
19 casted to "void*", and most enumerated values are not included
21 =================================================================
24 typedef struct CURL_s CURL;
25 typedef struct CURLM_s CURLM;
26 typedef struct curl_slist curl_slist;
34 CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
38 #define CURL_GLOBAL_NOTHING 0
39 #define CURL_GLOBAL_SSL 1
40 #define CURL_GLOBAL_WIN32 2
41 #define CURLOPTTYPE_LONG 0
42 #define CURLOPTTYPE_OBJECTPOINT 10000
43 #define CURLOPTTYPE_FUNCTIONPOINT 20000
44 #define CURLOPTTYPE_OFF_T 30000
45 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
48 CINIT(WRITEDATA, OBJECTPOINT, 1),
49 CINIT(URL, OBJECTPOINT, 2),
50 CINIT(ERRORBUFFER, OBJECTPOINT, 10),
51 CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
52 CINIT(POSTFIELDS, OBJECTPOINT, 15),
53 CINIT(REFERER, OBJECTPOINT, 16),
54 CINIT(USERAGENT, OBJECTPOINT, 18),
55 CINIT(LOW_SPEED_LIMIT, LONG , 19),
56 CINIT(LOW_SPEED_TIME, LONG, 20),
57 CINIT(RESUME_FROM, LONG, 21),
58 CINIT(HTTPHEADER, OBJECTPOINT, 23),
59 CINIT(POST, LONG, 47), /* HTTP POST method */
60 CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */
61 CINIT(POSTFIELDSIZE, LONG, 60),
62 CINIT(PRIVATE, OBJECTPOINT, 103),
63 CINIT(PROTOCOLS, LONG, 181),
64 CINIT(REDIR_PROTOCOLS, LONG, 182)
67 #define CURLPROTO_HTTP (1<<0)
68 #define CURLPROTO_HTTPS (1<<1)
69 #define CURLPROTO_FTP (1<<2)
73 CURLINFO_HEADER_IN, /* 1 */
74 CURLINFO_HEADER_OUT, /* 2 */
75 CURLINFO_DATA_IN, /* 3 */
76 CURLINFO_DATA_OUT, /* 4 */
77 CURLINFO_SSL_DATA_IN, /* 5 */
78 CURLINFO_SSL_DATA_OUT, /* 6 */
82 #define CURLINFO_STRING 0x100000
83 #define CURLINFO_LONG 0x200000
84 #define CURLINFO_DOUBLE 0x300000
85 #define CURLINFO_SLIST 0x400000
86 #define CURLINFO_MASK 0x0fffff
87 #define CURLINFO_TYPEMASK 0xf00000
90 CURLINFO_NONE, /* first, never use this */
91 CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1,
92 CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2,
93 CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3,
94 CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4,
95 CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5,
96 CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
97 CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7,
98 CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8,
99 CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9,
100 CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10,
101 CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11,
102 CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12,
103 CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13,
104 CURLINFO_FILETIME = CURLINFO_LONG + 14,
105 CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15,
106 CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16,
107 CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
108 CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18,
109 CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19,
110 CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20,
111 CURLINFO_PRIVATE = CURLINFO_STRING + 21,
112 CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22,
113 CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23,
114 CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24,
115 CURLINFO_OS_ERRNO = CURLINFO_LONG + 25,
116 CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26,
117 CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27
123 CURLMSG_NONE, /* first, not used */
124 CURLMSG_DONE, /* This easy handle has completed. 'result' contains
125 the CURLcode of the transfer */
131 CURLMSG msg; /* what this message means */
132 CURL *easy_handle; /* the handle it concerns */
135 void *whatever; /* message-specific data */
136 CURLcode result; /* return code for transfer */
142 static void (*qcurl_global_init) (long flags);
143 static void (*qcurl_global_cleanup) (void);
145 static CURL * (*qcurl_easy_init) (void);
146 static void (*qcurl_easy_cleanup) (CURL *handle);
147 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
148 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
149 static const char * (*qcurl_easy_strerror) (CURLcode);
151 static CURLM * (*qcurl_multi_init) (void);
152 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
153 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
154 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
155 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
156 static void (*qcurl_multi_cleanup) (CURLM *);
157 static const char * (*qcurl_multi_strerror) (CURLcode);
158 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
159 static void (*qcurl_slist_free_all) (curl_slist *list);
161 static dllfunction_t curlfuncs[] =
163 {"curl_global_init", (void **) &qcurl_global_init},
164 {"curl_global_cleanup", (void **) &qcurl_global_cleanup},
165 {"curl_easy_init", (void **) &qcurl_easy_init},
166 {"curl_easy_cleanup", (void **) &qcurl_easy_cleanup},
167 {"curl_easy_setopt", (void **) &qcurl_easy_setopt},
168 {"curl_easy_strerror", (void **) &qcurl_easy_strerror},
169 {"curl_easy_getinfo", (void **) &qcurl_easy_getinfo},
170 {"curl_multi_init", (void **) &qcurl_multi_init},
171 {"curl_multi_perform", (void **) &qcurl_multi_perform},
172 {"curl_multi_add_handle", (void **) &qcurl_multi_add_handle},
173 {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
174 {"curl_multi_info_read", (void **) &qcurl_multi_info_read},
175 {"curl_multi_cleanup", (void **) &qcurl_multi_cleanup},
176 {"curl_multi_strerror", (void **) &qcurl_multi_strerror},
177 {"curl_slist_append", (void **) &qcurl_slist_append},
178 {"curl_slist_free_all", (void **) &qcurl_slist_free_all},
182 // Handle for CURL DLL
183 static dllhandle_t curl_dll = NULL;
184 // will be checked at many places to find out if qcurl calls are allowed
186 void *curl_mutex = NULL;
188 typedef struct downloadinfo_s
190 char filename[MAX_OSPATH];
194 fs_offset_t startpos;
198 unsigned long bytes_received; // for buffer
199 double bytes_received_curl; // for throttling
200 double bytes_sent_curl; // for throttling
201 struct downloadinfo_s *next, *prev;
204 curl_slist *slist; // http headers
206 unsigned char *buffer;
208 curl_callback_t callback;
211 const unsigned char *postbuf;
213 const char *post_content_type;
214 const char *extraheaders;
217 static downloadinfo *downloads = NULL;
218 static int numdownloads = 0;
220 static qboolean noclear = FALSE;
222 static int numdownloads_fail = 0;
223 static int numdownloads_success = 0;
224 static int numdownloads_added = 0;
225 static char command_when_done[256] = "";
226 static char command_when_error[256] = "";
232 Sets the command which is to be executed when the last download completes AND
233 all downloads since last server connect ended with a successful status.
234 Setting the command to NULL clears it.
237 static void Curl_CommandWhenDone(const char *cmd)
242 strlcpy(command_when_done, cmd, sizeof(command_when_done));
244 *command_when_done = 0;
249 Do not use yet. Not complete.
250 Problem: what counts as an error?
253 static void Curl_CommandWhenError(const char *cmd)
258 strlcpy(command_when_error, cmd, sizeof(command_when_error));
260 *command_when_error = 0;
265 Curl_Clear_forthismap
267 Clears the "will disconnect on failure" flags.
270 void Curl_Clear_forthismap(void)
275 if (curl_mutex) Thread_LockMutex(curl_mutex);
276 for(di = downloads; di; di = di->next)
277 di->forthismap = false;
278 Curl_CommandWhenError(NULL);
279 Curl_CommandWhenDone(NULL);
280 numdownloads_fail = 0;
281 numdownloads_success = 0;
282 numdownloads_added = 0;
283 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
290 Returns true if a download needed for the current game is running.
293 qboolean Curl_Have_forthismap(void)
295 return numdownloads_added != 0;
298 void Curl_Register_predownload(void)
300 if (curl_mutex) Thread_LockMutex(curl_mutex);
301 Curl_CommandWhenDone("cl_begindownloads");
302 Curl_CommandWhenError("cl_begindownloads");
303 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
308 Curl_CheckCommandWhenDone
310 Checks if a "done command" is to be executed.
311 All downloads finished, at least one success since connect, no single failure
312 -> execute the command.
314 static void Curl_CheckCommandWhenDone(void)
318 if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
320 if(numdownloads_fail == 0)
322 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
324 Cbuf_AddText(command_when_done);
329 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
331 Cbuf_AddText(command_when_error);
334 Curl_Clear_forthismap();
345 static qboolean CURL_OpenLibrary (void)
347 const char* dllnames [] =
352 #elif defined(MACOSX)
353 "libcurl.4.dylib", // Mac OS X Notyetreleased
354 "libcurl.3.dylib", // Mac OS X Tiger
355 "libcurl.2.dylib", // Mac OS X Panther
359 "libcurl.so", // FreeBSD
369 return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
380 static void CURL_CloseLibrary (void)
382 Sys_UnloadLibrary (&curl_dll);
386 static CURLM *curlm = NULL;
387 static double bytes_received = 0; // used for bandwidth throttling
388 static double bytes_sent = 0; // used for bandwidth throttling
389 static double curltime = 0;
395 fwrite-compatible function that writes the data to a file. libcurl can call
399 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
401 fs_offset_t ret = -1;
402 size_t bytes = size * nmemb;
403 downloadinfo *di = (downloadinfo *) vdi;
407 if(di->bytes_received + bytes <= di->buffersize)
409 memcpy(di->buffer + di->bytes_received, data, bytes);
412 // otherwise: buffer overrun, ret stays -1
417 ret = FS_Write(di->stream, data, bytes);
420 di->bytes_received += bytes;
422 return ret; // why not ret / nmemb?
427 CURL_DOWNLOAD_SUCCESS = 0,
428 CURL_DOWNLOAD_FAILED,
429 CURL_DOWNLOAD_ABORTED,
430 CURL_DOWNLOAD_SERVERERROR
434 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
436 downloadinfo *di = (downloadinfo *) cbdata;
439 case CURLCBSTATUS_OK:
440 Con_DPrintf("Download of %s: OK\n", di->filename);
442 case CURLCBSTATUS_FAILED:
443 Con_DPrintf("Download of %s: FAILED\n", di->filename);
445 case CURLCBSTATUS_ABORTED:
446 Con_DPrintf("Download of %s: ABORTED\n", di->filename);
448 case CURLCBSTATUS_SERVERERROR:
449 Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
451 case CURLCBSTATUS_UNKNOWN:
452 Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
455 Con_DPrintf("Download of %s: %d\n", di->filename, status);
464 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
465 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
466 code from libcurl, or 0, if another error has occurred.
469 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
470 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
477 case CURL_DOWNLOAD_SUCCESS:
479 di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
481 case CURL_DOWNLOAD_FAILED:
482 di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
484 case CURL_DOWNLOAD_ABORTED:
485 di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
487 case CURL_DOWNLOAD_SERVERERROR:
488 // reopen to enforce it to have zero bytes again
491 FS_Close(di->stream);
492 di->stream = FS_OpenRealFile(di->filename, "wb", false);
496 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
500 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
506 qcurl_multi_remove_handle(curlm, di->curle);
507 qcurl_easy_cleanup(di->curle);
509 qcurl_slist_free_all(di->slist);
512 if(!di->callback && ok && !di->bytes_received)
514 Con_Printf("ERROR: empty file\n");
519 FS_Close(di->stream);
523 ok = FS_AddPack(di->filename, NULL, true);
526 // pack loading failed?
528 // better clear the file again...
529 di->stream = FS_OpenRealFile(di->filename, "wb", false);
530 FS_Close(di->stream);
532 if(di->startpos && !di->callback)
534 // this was a resume?
535 // then try to redownload it without reporting the error
536 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->ispak, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL);
537 di->forthismap = false; // don't count the error
543 di->prev->next = di->next;
545 downloads = di->next;
547 di->next->prev = di->prev;
553 ++numdownloads_success;
564 Returns a "cleaned up" URL for display (to strip login data)
567 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
569 const char *p, *q, *r;
571 // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
572 p = strstr(url, "://");
575 q = strchr(p + 3, '@');
578 r = strchr(p + 3, '/');
581 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
592 CheckPendingDownloads
594 checks if there are free download slots to start new downloads in.
595 To not start too many downloads at once, only one download is added at a time,
596 up to a maximum number of cl_curl_maxdownloads are running.
599 static void CheckPendingDownloads(void)
606 if(numdownloads < cl_curl_maxdownloads.integer)
609 for(di = downloads; di; di = di->next)
615 Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
617 di->stream = FS_OpenRealFile(di->filename, "ab", false);
620 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
621 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
624 FS_Seek(di->stream, 0, SEEK_END);
625 di->startpos = FS_Tell(di->stream);
628 Con_Printf(", resuming from position %ld", (long) di->startpos);
633 Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
637 di->curle = qcurl_easy_init();
639 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
640 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
641 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
642 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
643 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
644 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
645 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
646 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
647 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
648 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
649 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
650 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
652 Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
653 //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
655 if(di->post_content_type)
657 qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
658 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
659 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
660 di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
663 // parse extra headers into slist
664 // \n separated list!
665 h = di->extraheaders;
668 const char *hh = strchr(h, '\n');
671 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
672 memcpy(buf, h, hh - h);
674 di->slist = qcurl_slist_append(di->slist, buf);
679 di->slist = qcurl_slist_append(di->slist, h);
684 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
686 qcurl_multi_add_handle(curlm, di->curle);
689 if(numdownloads >= cl_curl_maxdownloads.integer)
700 this function MUST be called before using anything else in this file.
701 On Win32, this must be called AFTER WSAStartup has been done!
709 if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
710 qcurl_global_init(CURL_GLOBAL_NOTHING);
711 curlm = qcurl_multi_init();
718 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
721 void Curl_ClearRequirements(void);
722 void Curl_Shutdown(void)
726 Curl_ClearRequirements();
728 if (curl_mutex) Thread_DestroyMutex(curl_mutex);
737 Finds the internal information block for a download given by file name.
740 static downloadinfo *Curl_Find(const char *filename)
745 for(di = downloads; di; di = di->next)
746 if(!strcasecmp(di->filename, filename))
755 Starts a download of a given URL to the file name portion of this URL (or name
756 if given) in the "dlcache/" folder.
759 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, qboolean ispak, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
773 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
774 p = strchr(URL, ':');
777 if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
779 char addressstring[128];
781 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
782 q = strchr(addressstring, ':');
784 q = addressstring + strlen(addressstring);
787 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
793 // Note: This extraction of the file name portion is NOT entirely correct.
795 // It does the following:
797 // http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
798 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
799 // http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
801 // However, I'd like to keep this "buggy" behavior so that PHP script
802 // authors can write download scripts without having to enable
803 // AcceptPathInfo on Apache. They just have to ensure that their script
804 // can be called with such a "fake" path name like
805 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
807 // By the way, such PHP scripts should either send the file or a
808 // "Location:" redirect; PHP code example:
810 // header("Location: http://www.example.com/");
812 // By the way, this will set User-Agent to something like
813 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
814 // dp://serverhost:serverport/ so you can filter on this; an example
815 // httpd log file line might be:
817 // 141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
820 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
822 if (curl_mutex) Thread_LockMutex(curl_mutex);
826 p = strrchr(name, '/');
827 p = p ? (p+1) : name;
829 length = q ? (size_t)(q - p) : strlen(p);
830 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
832 name = fn; // make it point back
834 // already downloading the file?
836 downloadinfo *di = Curl_Find(fn);
839 Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url, urlbuf, sizeof(urlbuf)));
841 // however, if it was not for this map yet...
842 if(forthismap && !di->forthismap)
844 di->forthismap = true;
845 // this "fakes" a download attempt so the client will wait for
846 // the download to finish and then reconnect
847 ++numdownloads_added;
854 if(ispak && FS_FileExists(fn))
856 qboolean already_loaded;
857 if(FS_AddPack(fn, &already_loaded, true))
859 Con_DPrintf("%s already exists, not downloading!\n", fn);
861 Con_DPrintf("(pak was already loaded)\n");
866 ++numdownloads_added;
867 ++numdownloads_success;
875 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
879 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
881 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
883 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
885 f = FS_OpenRealFile(fn, "wb", false);
899 // if we get here, we actually want to download... so first verify the
900 // URL scheme (so one can't read local files using file://)
901 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
903 Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
904 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
909 ++numdownloads_added;
910 di = (downloadinfo *) Z_Malloc(sizeof(*di));
911 strlcpy(di->filename, name, sizeof(di->filename));
912 strlcpy(di->url, URL, sizeof(di->url));
913 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
914 di->forthismap = forthismap;
919 di->ispak = (ispak && !buf);
920 di->maxspeed = maxspeed;
921 di->bytes_received = 0;
922 di->bytes_received_curl = 0;
923 di->bytes_sent_curl = 0;
924 di->extraheaders = extraheaders;
925 di->next = downloads;
931 di->buffersize = bufsize;
934 di->callback = curl_default_callback;
935 di->callback_data = di;
939 di->callback = callback;
940 di->callback_data = cbdata;
943 if(post_content_type)
945 di->post_content_type = post_content_type;
946 di->postbuf = postbuf;
947 di->postbufsize = postbufsize;
951 di->post_content_type = NULL;
957 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
962 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, qboolean ispak, qboolean forthismap)
964 return Curl_Begin(URL, NULL, maxspeed, name, ispak, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
966 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
968 return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
970 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
972 return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
979 call this regularily as this will always download as much as possible without
990 if(!cl_curl_enabled.integer)
996 if (curl_mutex) Thread_LockMutex(curl_mutex);
998 Curl_CheckCommandWhenDone();
1002 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1006 if(realtime < curltime) // throttle
1008 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1018 mc = qcurl_multi_perform(curlm, &remaining);
1020 while(mc == CURLM_CALL_MULTI_PERFORM);
1022 for(di = downloads; di; di = di->next)
1027 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1028 bytes_sent += (b - di->bytes_sent_curl);
1029 di->bytes_sent_curl = b;
1030 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1031 bytes_sent += (b - di->bytes_received_curl);
1032 di->bytes_received_curl = b;
1038 CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1041 if(msg->msg == CURLMSG_DONE)
1043 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1045 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1046 result = msg->data.result;
1049 failed = CURL_DOWNLOAD_FAILED;
1054 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1057 case 4: // e.g. 404?
1058 case 5: // e.g. 500?
1059 failed = CURL_DOWNLOAD_SERVERERROR;
1060 result = (CURLcode) code;
1065 Curl_EndDownload(di, failed, result);
1070 CheckPendingDownloads();
1072 // when will we curl the next time?
1073 // we will wait a bit to ensure our download rate is kept.
1074 // we now know that realtime >= curltime... so set up a new curltime
1076 // use the slowest allowing download to derive the maxspeed... this CAN
1077 // be done better, but maybe later
1078 maxspeed = cl_curl_maxspeed.value;
1079 for(di = downloads; di; di = di->next)
1080 if(di->maxspeed > 0)
1081 if(di->maxspeed < maxspeed || maxspeed <= 0)
1082 maxspeed = di->maxspeed;
1086 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1087 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1092 curltime = realtime;
1094 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1098 ====================
1101 Stops ALL downloads.
1102 ====================
1104 void Curl_CancelAll(void)
1109 if (curl_mutex) Thread_LockMutex(curl_mutex);
1113 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1114 // INVARIANT: downloads will point to the next download after that!
1117 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1121 ====================
1124 returns true iff there is a download running.
1125 ====================
1127 qboolean Curl_Running(void)
1132 return downloads != NULL;
1136 ====================
1137 Curl_GetDownloadAmount
1139 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1140 for the given download.
1141 ====================
1143 static double Curl_GetDownloadAmount(downloadinfo *di)
1150 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1152 return (di->startpos + di->bytes_received) / (di->startpos + length);
1161 ====================
1162 Curl_GetDownloadSpeed
1164 returns the speed of the given download in bytes per second
1165 ====================
1167 static double Curl_GetDownloadSpeed(downloadinfo *di)
1174 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1182 ====================
1185 prints the download list
1186 ====================
1188 // TODO rewrite using Curl_GetDownloadInfo?
1189 static void Curl_Info_f(void)
1197 if (curl_mutex) Thread_LockMutex(curl_mutex);
1198 Con_Print("Currently running downloads:\n");
1199 for(di = downloads; di; di = di->next)
1201 double speed, percent;
1202 Con_Printf(" %s -> %s ", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1203 percent = 100.0 * Curl_GetDownloadAmount(di);
1204 speed = Curl_GetDownloadSpeed(di);
1206 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1208 Con_Print("(queued)\n");
1210 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1214 Con_Print("No downloads running.\n");
1219 ====================
1222 implements the "curl" console command
1226 curl --cancel filename
1231 curl [--pak] [--forthismap] [--for filename filename...] url
1232 --pak: after downloading, load the package into the virtual file system
1233 --for filename...: only download of at least one of the named files is missing
1234 --forthismap: don't reconnect on failure
1236 curl --clear_autodownload
1237 clears the download success/failure counters
1239 curl --finish_autodownload
1240 if at least one download has been started, disconnect and drop to the menu
1241 once the last download completes successfully, reconnect to the current server
1242 ====================
1244 static void Curl_Curl_f(void)
1246 double maxspeed = 0;
1249 qboolean pak = false;
1250 qboolean forthismap = false;
1252 const char *name = 0;
1256 Con_Print("libcurl DLL not found, this command is inactive.\n");
1260 if(!cl_curl_enabled.integer)
1262 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1268 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1272 url = Cmd_Argv(Cmd_Argc() - 1);
1275 for(i = 1; i != end; ++i)
1277 const char *a = Cmd_Argv(i);
1278 if(!strcmp(a, "--info"))
1283 else if(!strcmp(a, "--cancel"))
1285 if(i == end - 1) // last argument
1289 downloadinfo *di = Curl_Find(url);
1291 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1293 Con_Print("download not found\n");
1297 else if(!strcmp(a, "--pak"))
1301 else if(!strcmp(a, "--for")) // must be last option
1303 for(i = i + 1; i != end - 1; ++i)
1305 if(!FS_FileExists(Cmd_Argv(i)))
1306 goto needthefile; // why can't I have a "double break"?
1308 // if we get here, we have all the files...
1311 else if(!strcmp(a, "--forthismap"))
1315 else if(!strcmp(a, "--as"))
1323 else if(!strcmp(a, "--clear_autodownload"))
1325 // mark all running downloads as "not for this map", so if they
1326 // fail, it does not matter
1327 Curl_Clear_forthismap();
1330 else if(!strcmp(a, "--finish_autodownload"))
1332 if(numdownloads_added)
1334 char donecommand[256];
1337 if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1339 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1340 Curl_CommandWhenDone(donecommand);
1344 Curl_CheckCommandWhenDone();
1347 Curl_Register_predownload();
1352 else if(!strncmp(a, "--maxspeed=", 11))
1354 maxspeed = atof(a + 11);
1358 Con_Printf("curl: invalid option %s\n", a);
1359 // but we ignore the option
1364 Curl_Begin_ToFile(url, maxspeed, name, pak, forthismap);
1368 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1370 Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1374 void Curl_CurlCat_f(void)
1377 const char *url = Cmd_Argv(1);
1378 buf = Z_Malloc(16384);
1379 Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1384 ====================
1387 loads the commands and cvars this library uses
1388 ====================
1390 void Curl_Init_Commands(void)
1392 Cvar_RegisterVariable (&cl_curl_enabled);
1393 Cvar_RegisterVariable (&cl_curl_maxdownloads);
1394 Cvar_RegisterVariable (&cl_curl_maxspeed);
1395 Cvar_RegisterVariable (&sv_curl_defaulturl);
1396 Cvar_RegisterVariable (&sv_curl_serverpackages);
1397 Cvar_RegisterVariable (&sv_curl_maxspeed);
1398 Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1399 //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1403 ====================
1404 Curl_GetDownloadInfo
1406 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1407 The number of elements in the array is returned in int *nDownloads.
1408 const char **additional_info may be set to a string of additional user
1409 information, or to NULL if no such display shall occur. The returned
1410 array must be freed later using Z_Free.
1411 ====================
1413 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1417 Curl_downloadinfo_t *downinfo;
1423 *additional_info = NULL;
1427 if (curl_mutex) Thread_LockMutex(curl_mutex);
1430 for(di = downloads; di; di = di->next)
1433 downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1435 for(di = downloads; di; di = di->next)
1437 // do not show infobars for background downloads
1438 if(developer.integer <= 0)
1441 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1444 downinfo[i].progress = Curl_GetDownloadAmount(di);
1445 downinfo[i].speed = Curl_GetDownloadSpeed(di);
1446 downinfo[i].queued = false;
1450 downinfo[i].queued = true;
1457 // TODO: can I clear command_when_done as soon as the first download fails?
1458 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1460 if(!strncmp(command_when_done, "connect ", 8))
1461 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1462 else if(!strcmp(command_when_done, "cl_begindownloads"))
1463 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1465 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1466 *additional_info = addinfo;
1469 *additional_info = NULL;
1473 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1479 ====================
1482 finds the URL where to find a given package.
1484 For this, it reads a file "curl_urls.txt" of the following format:
1487 revdm*.pk3 http://revdm/downloads/are/here/
1488 * http://any/other/stuff/is/here/
1490 The URLs should end in /. If not, downloads will still work, but the cached files
1491 can't be just put into the data directory with the same download configuration
1492 (you might want to do this if you want to tag downloaded files from your
1493 server, but you should not). "-" means "don't download".
1495 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1498 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1499 this file for obvious reasons.
1500 ====================
1502 static const char *Curl_FindPackURL(const char *filename)
1504 static char foundurl[1024]; // invoked only by server
1505 fs_offset_t filesize;
1506 char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1509 // read lines of format "pattern url"
1511 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1512 qboolean eof = false;
1524 if(pattern && url && patternend)
1530 if(matchpattern(filename, pattern, true))
1532 strlcpy(foundurl, url, sizeof(foundurl));
1544 if(pattern && !patternend)
1546 else if(url && !urlend)
1552 else if(pattern && patternend && !url)
1561 return sv_curl_defaulturl.string;
1564 typedef struct requirement_s
1566 struct requirement_s *next;
1567 char filename[MAX_OSPATH];
1570 static requirement *requirements = NULL;
1574 ====================
1577 Adds the given file to the list of requirements.
1578 ====================
1580 void Curl_RequireFile(const char *filename)
1582 requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1583 req->next = requirements;
1584 strlcpy(req->filename, filename, sizeof(req->filename));
1589 ====================
1590 Curl_ClearRequirements
1592 Clears the list of required files for playing on the current map.
1593 This should be called at every map change.
1594 ====================
1596 void Curl_ClearRequirements(void)
1600 requirement *req = requirements;
1601 requirements = requirements->next;
1607 ====================
1608 Curl_SendRequirements
1610 Makes the current host_clients download all files he needs.
1611 This is done by sending him the following console commands:
1613 curl --clear_autodownload
1614 curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1615 curl --finish_autodownload
1616 ====================
1618 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1621 const char *thispack = FS_WhichPack(filename);
1622 const char *packurl;
1627 p = strrchr(thispack, '/');
1631 packurl = Curl_FindPackURL(thispack);
1633 if(packurl && *packurl && strcmp(packurl, "-"))
1636 strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1638 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1639 strlcat(sendbuffer, thispack, sendbuffer_len);
1640 if(sv_curl_maxspeed.value > 0)
1641 dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1642 strlcat(sendbuffer, " --for ", sendbuffer_len);
1643 strlcat(sendbuffer, filename, sendbuffer_len);
1644 strlcat(sendbuffer, " ", sendbuffer_len);
1645 strlcat(sendbuffer, packurl, sendbuffer_len);
1646 strlcat(sendbuffer, thispack, sendbuffer_len);
1647 strlcat(sendbuffer, "\n", sendbuffer_len);
1654 void Curl_SendRequirements(void)
1656 // for each requirement, find the pack name
1657 char sendbuffer[4096] = "";
1659 qboolean foundone = false;
1662 for(req = requirements; req; req = req->next)
1663 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1665 p = sv_curl_serverpackages.string;
1666 while(COM_ParseToken_Simple(&p, false, false))
1667 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1670 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1672 if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1673 Host_ClientCommands("%s", sendbuffer);
1675 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");