10 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
11 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
12 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
13 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
14 static cvar_t sv_curl_maxspeed = {CVAR_SAVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
15 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
16 static cvar_t cl_curl_useragent = {0, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
17 static cvar_t cl_curl_useragent_append = {0, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
20 =================================================================
22 Minimal set of definitions from libcurl
24 WARNING: for a matter of simplicity, several pointer types are
25 casted to "void*", and most enumerated values are not included
27 =================================================================
30 typedef struct CURL_s CURL;
31 typedef struct CURLM_s CURLM;
32 typedef struct curl_slist curl_slist;
40 CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
44 #define CURL_GLOBAL_NOTHING 0
45 #define CURL_GLOBAL_SSL 1
46 #define CURL_GLOBAL_WIN32 2
47 #define CURLOPTTYPE_LONG 0
48 #define CURLOPTTYPE_OBJECTPOINT 10000
49 #define CURLOPTTYPE_FUNCTIONPOINT 20000
50 #define CURLOPTTYPE_OFF_T 30000
51 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
54 CINIT(WRITEDATA, OBJECTPOINT, 1),
55 CINIT(URL, OBJECTPOINT, 2),
56 CINIT(ERRORBUFFER, OBJECTPOINT, 10),
57 CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
58 CINIT(POSTFIELDS, OBJECTPOINT, 15),
59 CINIT(REFERER, OBJECTPOINT, 16),
60 CINIT(USERAGENT, OBJECTPOINT, 18),
61 CINIT(LOW_SPEED_LIMIT, LONG , 19),
62 CINIT(LOW_SPEED_TIME, LONG, 20),
63 CINIT(RESUME_FROM, LONG, 21),
64 CINIT(HTTPHEADER, OBJECTPOINT, 23),
65 CINIT(POST, LONG, 47), /* HTTP POST method */
66 CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */
67 CINIT(POSTFIELDSIZE, LONG, 60),
68 CINIT(PRIVATE, OBJECTPOINT, 103),
69 CINIT(PROTOCOLS, LONG, 181),
70 CINIT(REDIR_PROTOCOLS, LONG, 182)
73 #define CURLPROTO_HTTP (1<<0)
74 #define CURLPROTO_HTTPS (1<<1)
75 #define CURLPROTO_FTP (1<<2)
79 CURLINFO_HEADER_IN, /* 1 */
80 CURLINFO_HEADER_OUT, /* 2 */
81 CURLINFO_DATA_IN, /* 3 */
82 CURLINFO_DATA_OUT, /* 4 */
83 CURLINFO_SSL_DATA_IN, /* 5 */
84 CURLINFO_SSL_DATA_OUT, /* 6 */
88 #define CURLINFO_STRING 0x100000
89 #define CURLINFO_LONG 0x200000
90 #define CURLINFO_DOUBLE 0x300000
91 #define CURLINFO_SLIST 0x400000
92 #define CURLINFO_MASK 0x0fffff
93 #define CURLINFO_TYPEMASK 0xf00000
96 CURLINFO_NONE, /* first, never use this */
97 CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1,
98 CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2,
99 CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3,
100 CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4,
101 CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5,
102 CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
103 CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7,
104 CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8,
105 CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9,
106 CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10,
107 CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11,
108 CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12,
109 CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13,
110 CURLINFO_FILETIME = CURLINFO_LONG + 14,
111 CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15,
112 CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16,
113 CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
114 CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18,
115 CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19,
116 CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20,
117 CURLINFO_PRIVATE = CURLINFO_STRING + 21,
118 CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22,
119 CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23,
120 CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24,
121 CURLINFO_OS_ERRNO = CURLINFO_LONG + 25,
122 CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26,
123 CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27
129 CURLMSG_NONE, /* first, not used */
130 CURLMSG_DONE, /* This easy handle has completed. 'result' contains
131 the CURLcode of the transfer */
137 CURLMSG msg; /* what this message means */
138 CURL *easy_handle; /* the handle it concerns */
141 void *whatever; /* message-specific data */
142 CURLcode result; /* return code for transfer */
148 static void (*qcurl_global_init) (long flags);
149 static void (*qcurl_global_cleanup) (void);
151 static CURL * (*qcurl_easy_init) (void);
152 static void (*qcurl_easy_cleanup) (CURL *handle);
153 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
154 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
155 static const char * (*qcurl_easy_strerror) (CURLcode);
157 static CURLM * (*qcurl_multi_init) (void);
158 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
159 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
160 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
161 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
162 static void (*qcurl_multi_cleanup) (CURLM *);
163 static const char * (*qcurl_multi_strerror) (CURLcode);
164 static curl_slist * (*qcurl_slist_append) (curl_slist *list, const char *string);
165 static void (*qcurl_slist_free_all) (curl_slist *list);
167 static dllfunction_t curlfuncs[] =
169 {"curl_global_init", (void **) &qcurl_global_init},
170 {"curl_global_cleanup", (void **) &qcurl_global_cleanup},
171 {"curl_easy_init", (void **) &qcurl_easy_init},
172 {"curl_easy_cleanup", (void **) &qcurl_easy_cleanup},
173 {"curl_easy_setopt", (void **) &qcurl_easy_setopt},
174 {"curl_easy_strerror", (void **) &qcurl_easy_strerror},
175 {"curl_easy_getinfo", (void **) &qcurl_easy_getinfo},
176 {"curl_multi_init", (void **) &qcurl_multi_init},
177 {"curl_multi_perform", (void **) &qcurl_multi_perform},
178 {"curl_multi_add_handle", (void **) &qcurl_multi_add_handle},
179 {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
180 {"curl_multi_info_read", (void **) &qcurl_multi_info_read},
181 {"curl_multi_cleanup", (void **) &qcurl_multi_cleanup},
182 {"curl_multi_strerror", (void **) &qcurl_multi_strerror},
183 {"curl_slist_append", (void **) &qcurl_slist_append},
184 {"curl_slist_free_all", (void **) &qcurl_slist_free_all},
188 // Handle for CURL DLL
189 static dllhandle_t curl_dll = NULL;
190 // will be checked at many places to find out if qcurl calls are allowed
192 #define LOADTYPE_NONE 0
193 #define LOADTYPE_PAK 1
194 #define LOADTYPE_CACHEPIC 2
195 #define LOADTYPE_SKINFRAME 3
197 void *curl_mutex = NULL;
199 typedef struct downloadinfo_s
201 char filename[MAX_OSPATH];
205 fs_offset_t startpos;
209 unsigned long bytes_received; // for buffer
210 double bytes_received_curl; // for throttling
211 double bytes_sent_curl; // for throttling
212 struct downloadinfo_s *next, *prev;
215 curl_slist *slist; // http headers
217 unsigned char *buffer;
219 curl_callback_t callback;
222 const unsigned char *postbuf;
224 const char *post_content_type;
225 const char *extraheaders;
228 static downloadinfo *downloads = NULL;
229 static int numdownloads = 0;
231 static qboolean noclear = FALSE;
233 static int numdownloads_fail = 0;
234 static int numdownloads_success = 0;
235 static int numdownloads_added = 0;
236 static char command_when_done[256] = "";
237 static char command_when_error[256] = "";
243 Sets the command which is to be executed when the last download completes AND
244 all downloads since last server connect ended with a successful status.
245 Setting the command to NULL clears it.
248 static void Curl_CommandWhenDone(const char *cmd)
253 strlcpy(command_when_done, cmd, sizeof(command_when_done));
255 *command_when_done = 0;
260 Do not use yet. Not complete.
261 Problem: what counts as an error?
264 static void Curl_CommandWhenError(const char *cmd)
269 strlcpy(command_when_error, cmd, sizeof(command_when_error));
271 *command_when_error = 0;
276 Curl_Clear_forthismap
278 Clears the "will disconnect on failure" flags.
281 void Curl_Clear_forthismap(void)
286 if (curl_mutex) Thread_LockMutex(curl_mutex);
287 for(di = downloads; di; di = di->next)
288 di->forthismap = false;
289 Curl_CommandWhenError(NULL);
290 Curl_CommandWhenDone(NULL);
291 numdownloads_fail = 0;
292 numdownloads_success = 0;
293 numdownloads_added = 0;
294 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
301 Returns true if a download needed for the current game is running.
304 qboolean Curl_Have_forthismap(void)
306 return numdownloads_added != 0;
309 void Curl_Register_predownload(void)
311 if (curl_mutex) Thread_LockMutex(curl_mutex);
312 Curl_CommandWhenDone("cl_begindownloads");
313 Curl_CommandWhenError("cl_begindownloads");
314 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
319 Curl_CheckCommandWhenDone
321 Checks if a "done command" is to be executed.
322 All downloads finished, at least one success since connect, no single failure
323 -> execute the command.
325 static void Curl_CheckCommandWhenDone(void)
329 if(numdownloads_added && ((numdownloads_success + numdownloads_fail) == numdownloads_added))
331 if(numdownloads_fail == 0)
333 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
335 Cbuf_AddText(command_when_done);
340 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
342 Cbuf_AddText(command_when_error);
345 Curl_Clear_forthismap();
356 static qboolean CURL_OpenLibrary (void)
358 const char* dllnames [] =
363 #elif defined(MACOSX)
364 "libcurl.4.dylib", // Mac OS X Notyetreleased
365 "libcurl.3.dylib", // Mac OS X Tiger
366 "libcurl.2.dylib", // Mac OS X Panther
370 "libcurl.so", // FreeBSD
380 return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
391 static void CURL_CloseLibrary (void)
393 Sys_UnloadLibrary (&curl_dll);
397 static CURLM *curlm = NULL;
398 static double bytes_received = 0; // used for bandwidth throttling
399 static double bytes_sent = 0; // used for bandwidth throttling
400 static double curltime = 0;
406 fwrite-compatible function that writes the data to a file. libcurl can call
410 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
412 fs_offset_t ret = -1;
413 size_t bytes = size * nmemb;
414 downloadinfo *di = (downloadinfo *) vdi;
418 if(di->bytes_received + bytes <= di->buffersize)
420 memcpy(di->buffer + di->bytes_received, data, bytes);
423 // otherwise: buffer overrun, ret stays -1
428 ret = FS_Write(di->stream, data, bytes);
431 di->bytes_received += bytes;
433 return ret; // why not ret / nmemb?
438 CURL_DOWNLOAD_SUCCESS = 0,
439 CURL_DOWNLOAD_FAILED,
440 CURL_DOWNLOAD_ABORTED,
441 CURL_DOWNLOAD_SERVERERROR
445 static void curl_default_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
447 downloadinfo *di = (downloadinfo *) cbdata;
450 case CURLCBSTATUS_OK:
451 Con_DPrintf("Download of %s: OK\n", di->filename);
453 case CURLCBSTATUS_FAILED:
454 Con_DPrintf("Download of %s: FAILED\n", di->filename);
456 case CURLCBSTATUS_ABORTED:
457 Con_DPrintf("Download of %s: ABORTED\n", di->filename);
459 case CURLCBSTATUS_SERVERERROR:
460 Con_DPrintf("Download of %s: (unknown server error)\n", di->filename);
462 case CURLCBSTATUS_UNKNOWN:
463 Con_DPrintf("Download of %s: (unknown client error)\n", di->filename);
466 Con_DPrintf("Download of %s: %d\n", di->filename, status);
471 static void curl_quiet_callback(int status, size_t length_received, unsigned char *buffer, void *cbdata)
473 curl_default_callback(status, length_received, buffer, cbdata);
476 static unsigned char *decode_image(downloadinfo *di, const char *content_type)
478 unsigned char *pixels = NULL;
479 fs_offset_t filesize = 0;
480 unsigned char *data = FS_LoadFile(di->filename, tempmempool, true, &filesize);
484 if(!strcmp(content_type, "image/jpeg"))
485 pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
486 else if(!strcmp(content_type, "image/png"))
487 pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
488 else if(filesize >= 7 && !strncmp((char *) data, "\xFF\xD8", 7))
489 pixels = JPEG_LoadImage_BGRA(data, filesize, &mip);
490 else if(filesize >= 7 && !strncmp((char *) data, "\x89PNG\x0D\x0A\x1A\x0A", 7))
491 pixels = PNG_LoadImage_BGRA(data, filesize, &mip);
493 Con_Printf("Did not detect content type: %s\n", content_type);
496 // do we call Image_MakeLinearColorsFromsRGB or not?
504 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
505 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
506 code from libcurl, or 0, if another error has occurred.
509 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata);
510 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error, const char *content_type_)
512 char content_type[64];
518 case CURL_DOWNLOAD_SUCCESS:
520 di->callback(CURLCBSTATUS_OK, di->bytes_received, di->buffer, di->callback_data);
522 case CURL_DOWNLOAD_FAILED:
523 di->callback(CURLCBSTATUS_FAILED, di->bytes_received, di->buffer, di->callback_data);
525 case CURL_DOWNLOAD_ABORTED:
526 di->callback(CURLCBSTATUS_ABORTED, di->bytes_received, di->buffer, di->callback_data);
528 case CURL_DOWNLOAD_SERVERERROR:
529 // reopen to enforce it to have zero bytes again
532 FS_Close(di->stream);
533 di->stream = FS_OpenRealFile(di->filename, "wb", false);
537 di->callback(error ? (int) error : CURLCBSTATUS_SERVERERROR, di->bytes_received, di->buffer, di->callback_data);
541 di->callback(CURLCBSTATUS_UNKNOWN, di->bytes_received, di->buffer, di->callback_data);
545 strlcpy(content_type, content_type_, sizeof(content_type));
551 qcurl_multi_remove_handle(curlm, di->curle);
552 qcurl_easy_cleanup(di->curle);
554 qcurl_slist_free_all(di->slist);
557 if(!di->callback && ok && !di->bytes_received)
559 Con_Printf("ERROR: empty file\n");
564 FS_Close(di->stream);
566 #define CLEAR_AND_RETRY() \
569 di->stream = FS_OpenRealFile(di->filename, "wb", false); \
570 FS_Close(di->stream); \
571 if(di->startpos && !di->callback) \
573 Curl_Begin(di->url, di->extraheaders, di->maxspeed, di->filename, di->loadtype, di->forthismap, di->post_content_type, di->postbuf, di->postbufsize, NULL, 0, NULL, NULL); \
574 di->forthismap = false; \
579 if(ok && di->loadtype == LOADTYPE_PAK)
581 ok = FS_AddPack(di->filename, NULL, true);
585 else if(ok && di->loadtype == LOADTYPE_CACHEPIC)
588 unsigned char *pixels = NULL;
592 if(!strncmp(p, "dlcache/", 8))
596 pixels = decode_image(di, content_type);
598 Draw_NewPic(p, image_width, image_height, true, pixels);
602 else if(ok && di->loadtype == LOADTYPE_SKINFRAME)
605 unsigned char *pixels = NULL;
609 if(!strncmp(p, "dlcache/", 8))
613 pixels = decode_image(di, content_type);
615 R_SkinFrame_LoadInternalBGRA(p, TEXF_FORCE_RELOAD | TEXF_MIPMAP | TEXF_ALPHA, pixels, image_width, image_height, false); // TODO what sRGB argument to put here?
621 di->prev->next = di->next;
623 downloads = di->next;
625 di->next->prev = di->prev;
631 ++numdownloads_success;
642 Returns a "cleaned up" URL for display (to strip login data)
645 static const char *CleanURL(const char *url, char *urlbuf, size_t urlbuflength)
647 const char *p, *q, *r;
649 // if URL is of form anything://foo-without-slash@rest, replace by anything://rest
650 p = strstr(url, "://");
653 q = strchr(p + 3, '@');
656 r = strchr(p + 3, '/');
659 dpsnprintf(urlbuf, urlbuflength, "%.*s%s", (int)(p - url + 3), url, q + 1);
670 CheckPendingDownloads
672 checks if there are free download slots to start new downloads in.
673 To not start too many downloads at once, only one download is added at a time,
674 up to a maximum number of cl_curl_maxdownloads are running.
677 static void CheckPendingDownloads(void)
684 if(numdownloads < cl_curl_maxdownloads.integer)
687 for(di = downloads; di; di = di->next)
693 Con_Printf("Downloading %s -> %s", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
695 di->stream = FS_OpenRealFile(di->filename, "ab", false);
698 Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
699 Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK, NULL);
702 FS_Seek(di->stream, 0, SEEK_END);
703 di->startpos = FS_Tell(di->stream);
706 Con_Printf(", resuming from position %ld", (long) di->startpos);
711 Con_DPrintf("Downloading %s -> memory\n", CleanURL(di->url, urlbuf, sizeof(urlbuf)));
715 di->curle = qcurl_easy_init();
717 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
718 if(cl_curl_useragent.integer)
721 #ifdef HTTP_USER_AGENT
728 if(*cl_curl_useragent_append.string)
729 ua = va(vabuf, sizeof(vabuf), "%s%s%s",
731 (ua[0] && ua[strlen(ua)-1] != ' ')
734 cl_curl_useragent_append.string);
735 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
738 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
739 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
740 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
741 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
742 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
743 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
744 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
745 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
746 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
747 qcurl_easy_setopt(di->curle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP);
748 if(qcurl_easy_setopt(di->curle, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FTP) != CURLE_OK)
750 Con_Printf("^1WARNING:^7 for security reasons, please upgrade to libcurl 7.19.4 or above. In a later version of DarkPlaces, HTTP redirect support will be disabled for this libcurl version.\n");
751 //qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 0);
753 if(di->post_content_type)
755 qcurl_easy_setopt(di->curle, CURLOPT_POST, 1);
756 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDS, di->postbuf);
757 qcurl_easy_setopt(di->curle, CURLOPT_POSTFIELDSIZE, di->postbufsize);
758 di->slist = qcurl_slist_append(di->slist, va(vabuf, sizeof(vabuf), "Content-Type: %s", di->post_content_type));
761 // parse extra headers into slist
762 // \n separated list!
763 h = di->extraheaders;
766 const char *hh = strchr(h, '\n');
769 char *buf = (char *) Mem_Alloc(tempmempool, hh - h + 1);
770 memcpy(buf, h, hh - h);
772 di->slist = qcurl_slist_append(di->slist, buf);
777 di->slist = qcurl_slist_append(di->slist, h);
782 qcurl_easy_setopt(di->curle, CURLOPT_HTTPHEADER, di->slist);
784 qcurl_multi_add_handle(curlm, di->curle);
787 if(numdownloads >= cl_curl_maxdownloads.integer)
798 this function MUST be called before using anything else in this file.
799 On Win32, this must be called AFTER WSAStartup has been done!
807 if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
808 qcurl_global_init(CURL_GLOBAL_NOTHING);
809 curlm = qcurl_multi_init();
816 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
819 void Curl_ClearRequirements(void);
820 void Curl_Shutdown(void)
824 Curl_ClearRequirements();
826 if (curl_mutex) Thread_DestroyMutex(curl_mutex);
835 Finds the internal information block for a download given by file name.
838 static downloadinfo *Curl_Find(const char *filename)
843 for(di = downloads; di; di = di->next)
844 if(!strcasecmp(di->filename, filename))
849 void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
854 for(di = downloads; di; )
856 if(di->callback == callback && di->callback_data == cbdata)
858 di->callback = curl_quiet_callback; // do NOT call the callback
859 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
871 Starts a download of a given URL to the file name portion of this URL (or name
872 if given) in the "dlcache/" folder.
875 static qboolean Curl_Begin(const char *URL, const char *extraheaders, double maxspeed, const char *name, int loadtype, qboolean forthismap, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
878 if(loadtype != LOADTYPE_NONE)
879 Host_Error("Curl_Begin: loadtype and buffer are both set");
893 // if URL is protocol:///* or protocol://:port/*, insert the IP of the current server
894 p = strchr(URL, ':');
897 if(!strncmp(p, ":///", 4) || !strncmp(p, "://:", 4))
899 char addressstring[128];
901 InfoString_GetValue(cls.userinfo, "*ip", addressstring, sizeof(addressstring));
902 q = strchr(addressstring, ':');
904 q = addressstring + strlen(addressstring);
907 dpsnprintf(urlbuf, sizeof(urlbuf), "%.*s://%.*s%s", (int) (p - URL), URL, (int) (q - addressstring), addressstring, URL + (p - URL) + 3);
913 // Note: This extraction of the file name portion is NOT entirely correct.
915 // It does the following:
917 // http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
918 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
919 // http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
921 // However, I'd like to keep this "buggy" behavior so that PHP script
922 // authors can write download scripts without having to enable
923 // AcceptPathInfo on Apache. They just have to ensure that their script
924 // can be called with such a "fake" path name like
925 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
927 // By the way, such PHP scripts should either send the file or a
928 // "Location:" redirect; PHP code example:
930 // header("Location: http://www.example.com/");
932 // By the way, this will set User-Agent to something like
933 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
934 // dp://serverhost:serverport/ so you can filter on this; an example
935 // httpd log file line might be:
937 // 141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
939 if (curl_mutex) Thread_LockMutex(curl_mutex);
944 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
950 name = CleanURL(URL, urlbuf, sizeof(urlbuf));
951 p = strrchr(name, '/');
952 p = p ? (p+1) : name;
954 length = q ? (size_t)(q - p) : strlen(p);
955 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
959 dpsnprintf(fn, sizeof(fn), "dlcache/%s", name);
962 name = fn; // make it point back
964 // already downloading the file?
966 downloadinfo *di = Curl_Find(fn);
969 Con_Printf("Can't download %s, already getting it from %s!\n", fn, CleanURL(di->url, urlbuf, sizeof(urlbuf)));
971 // however, if it was not for this map yet...
972 if(forthismap && !di->forthismap)
974 di->forthismap = true;
975 // this "fakes" a download attempt so the client will wait for
976 // the download to finish and then reconnect
977 ++numdownloads_added;
984 if(FS_FileExists(fn))
986 if(loadtype == LOADTYPE_PAK)
988 qboolean already_loaded;
989 if(FS_AddPack(fn, &already_loaded, true))
991 Con_DPrintf("%s already exists, not downloading!\n", fn);
993 Con_DPrintf("(pak was already loaded)\n");
998 ++numdownloads_added;
999 ++numdownloads_success;
1007 qfile_t *f = FS_OpenRealFile(fn, "rb", false);
1011 FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
1013 if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
1015 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
1017 f = FS_OpenRealFile(fn, "wb", false);
1031 // never resume these
1032 qfile_t *f = FS_OpenRealFile(fn, "wb", false);
1039 // if we get here, we actually want to download... so first verify the
1040 // URL scheme (so one can't read local files using file://)
1041 if(strncmp(URL, "http://", 7) && strncmp(URL, "ftp://", 6) && strncmp(URL, "https://", 8))
1043 Con_Printf("Curl_Begin(\"%s\"): nasty URL scheme rejected\n", URL);
1044 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1049 ++numdownloads_added;
1050 di = (downloadinfo *) Z_Malloc(sizeof(*di));
1051 strlcpy(di->filename, name, sizeof(di->filename));
1052 strlcpy(di->url, URL, sizeof(di->url));
1053 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
1054 di->forthismap = forthismap;
1058 di->started = false;
1059 di->loadtype = loadtype;
1060 di->maxspeed = maxspeed;
1061 di->bytes_received = 0;
1062 di->bytes_received_curl = 0;
1063 di->bytes_sent_curl = 0;
1064 di->extraheaders = extraheaders;
1065 di->next = downloads;
1068 di->next->prev = di;
1071 di->buffersize = bufsize;
1072 if(callback == NULL)
1074 di->callback = curl_default_callback;
1075 di->callback_data = di;
1079 di->callback = callback;
1080 di->callback_data = cbdata;
1083 if(post_content_type)
1085 di->post_content_type = post_content_type;
1086 di->postbuf = postbuf;
1087 di->postbufsize = postbufsize;
1091 di->post_content_type = NULL;
1093 di->postbufsize = 0;
1097 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1102 qboolean Curl_Begin_ToFile(const char *URL, double maxspeed, const char *name, int loadtype, qboolean forthismap)
1104 return Curl_Begin(URL, NULL, maxspeed, name, loadtype, forthismap, NULL, NULL, 0, NULL, 0, NULL, NULL);
1106 qboolean Curl_Begin_ToMemory(const char *URL, double maxspeed, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1108 return Curl_Begin(URL, NULL, maxspeed, NULL, false, false, NULL, NULL, 0, buf, bufsize, callback, cbdata);
1110 qboolean Curl_Begin_ToMemory_POST(const char *URL, const char *extraheaders, double maxspeed, const char *post_content_type, const unsigned char *postbuf, size_t postbufsize, unsigned char *buf, size_t bufsize, curl_callback_t callback, void *cbdata)
1112 return Curl_Begin(URL, extraheaders, maxspeed, NULL, false, false, post_content_type, postbuf, postbufsize, buf, bufsize, callback, cbdata);
1116 ====================
1119 call this regularily as this will always download as much as possible without
1121 ====================
1130 if(!cl_curl_enabled.integer)
1136 if (curl_mutex) Thread_LockMutex(curl_mutex);
1138 Curl_CheckCommandWhenDone();
1142 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1146 if(realtime < curltime) // throttle
1148 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1158 mc = qcurl_multi_perform(curlm, &remaining);
1160 while(mc == CURLM_CALL_MULTI_PERFORM);
1162 for(di = downloads; di; di = di->next)
1167 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_UPLOAD, &b);
1168 bytes_sent += (b - di->bytes_sent_curl);
1169 di->bytes_sent_curl = b;
1170 qcurl_easy_getinfo(di->curle, CURLINFO_SIZE_DOWNLOAD, &b);
1171 bytes_sent += (b - di->bytes_received_curl);
1172 di->bytes_received_curl = b;
1178 CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
1181 if(msg->msg == CURLMSG_DONE)
1183 const char *ct = NULL;
1184 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
1186 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
1187 result = msg->data.result;
1190 failed = CURL_DOWNLOAD_FAILED;
1195 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
1198 case 4: // e.g. 404?
1199 case 5: // e.g. 500?
1200 failed = CURL_DOWNLOAD_SERVERERROR;
1201 result = (CURLcode) code;
1204 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_CONTENT_TYPE, &ct);
1207 Curl_EndDownload(di, failed, result, ct);
1212 CheckPendingDownloads();
1214 // when will we curl the next time?
1215 // we will wait a bit to ensure our download rate is kept.
1216 // we now know that realtime >= curltime... so set up a new curltime
1218 // use the slowest allowing download to derive the maxspeed... this CAN
1219 // be done better, but maybe later
1220 maxspeed = cl_curl_maxspeed.value;
1221 for(di = downloads; di; di = di->next)
1222 if(di->maxspeed > 0)
1223 if(di->maxspeed < maxspeed || maxspeed <= 0)
1224 maxspeed = di->maxspeed;
1228 double bytes = bytes_sent + bytes_received; // maybe smoothen a bit?
1229 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
1234 curltime = realtime;
1236 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1240 ====================
1243 Stops ALL downloads.
1244 ====================
1246 void Curl_CancelAll(void)
1251 if (curl_mutex) Thread_LockMutex(curl_mutex);
1255 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1256 // INVARIANT: downloads will point to the next download after that!
1259 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1263 ====================
1266 returns true iff there is a download running.
1267 ====================
1269 qboolean Curl_Running(void)
1274 return downloads != NULL;
1278 ====================
1279 Curl_GetDownloadAmount
1281 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
1282 for the given download.
1283 ====================
1285 static double Curl_GetDownloadAmount(downloadinfo *di)
1292 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
1294 return (di->startpos + di->bytes_received) / (di->startpos + length);
1303 ====================
1304 Curl_GetDownloadSpeed
1306 returns the speed of the given download in bytes per second
1307 ====================
1309 static double Curl_GetDownloadSpeed(downloadinfo *di)
1316 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
1324 ====================
1327 prints the download list
1328 ====================
1330 // TODO rewrite using Curl_GetDownloadInfo?
1331 static void Curl_Info_f(void)
1339 if (curl_mutex) Thread_LockMutex(curl_mutex);
1340 Con_Print("Currently running downloads:\n");
1341 for(di = downloads; di; di = di->next)
1343 double speed, percent;
1344 Con_Printf(" %s -> %s ", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
1345 percent = 100.0 * Curl_GetDownloadAmount(di);
1346 speed = Curl_GetDownloadSpeed(di);
1348 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
1350 Con_Print("(queued)\n");
1352 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1356 Con_Print("No downloads running.\n");
1361 ====================
1364 implements the "curl" console command
1368 curl --cancel filename
1373 curl [--pak] [--forthismap] [--for filename filename...] url
1374 --pak: after downloading, load the package into the virtual file system
1375 --for filename...: only download of at least one of the named files is missing
1376 --forthismap: don't reconnect on failure
1378 curl --clear_autodownload
1379 clears the download success/failure counters
1381 curl --finish_autodownload
1382 if at least one download has been started, disconnect and drop to the menu
1383 once the last download completes successfully, reconnect to the current server
1384 ====================
1386 static void Curl_Curl_f(void)
1388 double maxspeed = 0;
1391 int loadtype = LOADTYPE_NONE;
1392 qboolean forthismap = false;
1394 const char *name = 0;
1398 Con_Print("libcurl DLL not found, this command is inactive.\n");
1402 if(!cl_curl_enabled.integer)
1404 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
1410 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
1414 url = Cmd_Argv(Cmd_Argc() - 1);
1417 for(i = 1; i != end; ++i)
1419 const char *a = Cmd_Argv(i);
1420 if(!strcmp(a, "--info"))
1425 else if(!strcmp(a, "--cancel"))
1427 if(i == end - 1) // last argument
1431 downloadinfo *di = Curl_Find(url);
1433 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
1435 Con_Print("download not found\n");
1439 else if(!strcmp(a, "--pak"))
1441 loadtype = LOADTYPE_PAK;
1443 else if(!strcmp(a, "--cachepic"))
1445 loadtype = LOADTYPE_CACHEPIC;
1447 else if(!strcmp(a, "--skinframe"))
1449 loadtype = LOADTYPE_SKINFRAME;
1451 else if(!strcmp(a, "--for")) // must be last option
1453 for(i = i + 1; i != end - 1; ++i)
1455 if(!FS_FileExists(Cmd_Argv(i)))
1456 goto needthefile; // why can't I have a "double break"?
1458 // if we get here, we have all the files...
1461 else if(!strcmp(a, "--forthismap"))
1465 else if(!strcmp(a, "--as"))
1473 else if(!strcmp(a, "--clear_autodownload"))
1475 // mark all running downloads as "not for this map", so if they
1476 // fail, it does not matter
1477 Curl_Clear_forthismap();
1480 else if(!strcmp(a, "--finish_autodownload"))
1482 if(numdownloads_added)
1484 char donecommand[256];
1487 if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1489 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1490 Curl_CommandWhenDone(donecommand);
1494 Curl_CheckCommandWhenDone();
1497 Curl_Register_predownload();
1502 else if(!strncmp(a, "--maxspeed=", 11))
1504 maxspeed = atof(a + 11);
1508 Con_Printf("curl: invalid option %s\n", a);
1509 // but we ignore the option
1514 Curl_Begin_ToFile(url, maxspeed, name, loadtype, forthismap);
1518 static void curl_curlcat_callback(int code, size_t length_received, unsigned char *buffer, void *cbdata)
1520 Con_Printf("Received %d bytes (status %d):\n%.*s\n", (int) length_received, code, (int) length_received, buffer);
1524 void Curl_CurlCat_f(void)
1527 const char *url = Cmd_Argv(1);
1528 buf = Z_Malloc(16384);
1529 Curl_Begin_ToMemory(url, buf, 16384, curl_curlcat_callback, NULL);
1534 ====================
1537 loads the commands and cvars this library uses
1538 ====================
1540 void Curl_Init_Commands(void)
1542 Cvar_RegisterVariable (&cl_curl_enabled);
1543 Cvar_RegisterVariable (&cl_curl_maxdownloads);
1544 Cvar_RegisterVariable (&cl_curl_maxspeed);
1545 Cvar_RegisterVariable (&sv_curl_defaulturl);
1546 Cvar_RegisterVariable (&sv_curl_serverpackages);
1547 Cvar_RegisterVariable (&sv_curl_maxspeed);
1548 Cvar_RegisterVariable (&cl_curl_useragent);
1549 Cvar_RegisterVariable (&cl_curl_useragent_append);
1550 Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1551 //Cmd_AddCommand ("curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
1555 ====================
1556 Curl_GetDownloadInfo
1558 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1559 The number of elements in the array is returned in int *nDownloads.
1560 const char **additional_info may be set to a string of additional user
1561 information, or to NULL if no such display shall occur. The returned
1562 array must be freed later using Z_Free.
1563 ====================
1565 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info, char *addinfo, size_t addinfolength)
1569 Curl_downloadinfo_t *downinfo;
1575 *additional_info = NULL;
1579 if (curl_mutex) Thread_LockMutex(curl_mutex);
1582 for(di = downloads; di; di = di->next)
1585 downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
1587 for(di = downloads; di; di = di->next)
1589 // do not show infobars for background downloads
1590 if(developer.integer <= 0)
1593 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1596 downinfo[i].progress = Curl_GetDownloadAmount(di);
1597 downinfo[i].speed = Curl_GetDownloadSpeed(di);
1598 downinfo[i].queued = false;
1602 downinfo[i].queued = true;
1609 // TODO: can I clear command_when_done as soon as the first download fails?
1610 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1612 if(!strncmp(command_when_done, "connect ", 8))
1613 dpsnprintf(addinfo, addinfolength, "(will join %s when done)", command_when_done + 8);
1614 else if(!strcmp(command_when_done, "cl_begindownloads"))
1615 dpsnprintf(addinfo, addinfolength, "(will enter the game when done)");
1617 dpsnprintf(addinfo, addinfolength, "(will do '%s' when done)", command_when_done);
1618 *additional_info = addinfo;
1621 *additional_info = NULL;
1625 if (curl_mutex) Thread_UnlockMutex(curl_mutex);
1631 ====================
1634 finds the URL where to find a given package.
1636 For this, it reads a file "curl_urls.txt" of the following format:
1639 revdm*.pk3 http://revdm/downloads/are/here/
1640 * http://any/other/stuff/is/here/
1642 The URLs should end in /. If not, downloads will still work, but the cached files
1643 can't be just put into the data directory with the same download configuration
1644 (you might want to do this if you want to tag downloaded files from your
1645 server, but you should not). "-" means "don't download".
1647 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1650 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1651 this file for obvious reasons.
1652 ====================
1654 static const char *Curl_FindPackURL(const char *filename)
1656 static char foundurl[1024]; // invoked only by server
1657 fs_offset_t filesize;
1658 char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1661 // read lines of format "pattern url"
1663 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1664 qboolean eof = false;
1676 if(pattern && url && patternend)
1682 if(matchpattern(filename, pattern, true))
1684 strlcpy(foundurl, url, sizeof(foundurl));
1696 if(pattern && !patternend)
1698 else if(url && !urlend)
1704 else if(pattern && patternend && !url)
1713 return sv_curl_defaulturl.string;
1716 typedef struct requirement_s
1718 struct requirement_s *next;
1719 char filename[MAX_OSPATH];
1722 static requirement *requirements = NULL;
1726 ====================
1729 Adds the given file to the list of requirements.
1730 ====================
1732 void Curl_RequireFile(const char *filename)
1734 requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1735 req->next = requirements;
1736 strlcpy(req->filename, filename, sizeof(req->filename));
1741 ====================
1742 Curl_ClearRequirements
1744 Clears the list of required files for playing on the current map.
1745 This should be called at every map change.
1746 ====================
1748 void Curl_ClearRequirements(void)
1752 requirement *req = requirements;
1753 requirements = requirements->next;
1759 ====================
1760 Curl_SendRequirements
1762 Makes the current host_clients download all files he needs.
1763 This is done by sending him the following console commands:
1765 curl --clear_autodownload
1766 curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1767 curl --finish_autodownload
1768 ====================
1770 static qboolean Curl_SendRequirement(const char *filename, qboolean foundone, char *sendbuffer, size_t sendbuffer_len)
1773 const char *thispack = FS_WhichPack(filename);
1774 const char *packurl;
1779 p = strrchr(thispack, '/');
1783 packurl = Curl_FindPackURL(thispack);
1785 if(packurl && *packurl && strcmp(packurl, "-"))
1788 strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
1790 strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
1791 strlcat(sendbuffer, thispack, sendbuffer_len);
1792 if(sv_curl_maxspeed.value > 0)
1793 dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
1794 strlcat(sendbuffer, " --for ", sendbuffer_len);
1795 strlcat(sendbuffer, filename, sendbuffer_len);
1796 strlcat(sendbuffer, " ", sendbuffer_len);
1797 strlcat(sendbuffer, packurl, sendbuffer_len);
1798 strlcat(sendbuffer, thispack, sendbuffer_len);
1799 strlcat(sendbuffer, "\n", sendbuffer_len);
1806 void Curl_SendRequirements(void)
1808 // for each requirement, find the pack name
1809 char sendbuffer[4096] = "";
1811 qboolean foundone = false;
1814 for(req = requirements; req; req = req->next)
1815 foundone = Curl_SendRequirement(req->filename, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1817 p = sv_curl_serverpackages.string;
1818 while(COM_ParseToken_Simple(&p, false, false, true))
1819 foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
1822 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1824 if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1825 Host_ClientCommands("%s", sendbuffer);
1827 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");