#include "fs.h"
#include "libcurl.h"
#include "thread.h"
-
+#include "com_list.h"
#include "image.h"
#include "jpeg.h"
#include "image_png.h"
-static cvar_t cl_curl_maxdownloads = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
-static cvar_t cl_curl_maxspeed = {CF_CLIENT | CF_ARCHIVE, "cl_curl_maxspeed","300", "maximum download speed (KiB/s)"};
-static cvar_t sv_curl_defaulturl = {CF_SERVER | CF_ARCHIVE, "sv_curl_defaulturl","", "default autodownload source URL"};
-static cvar_t sv_curl_serverpackages = {CF_SERVER | CF_ARCHIVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
-static cvar_t sv_curl_maxspeed = {CF_SERVER | CF_ARCHIVE, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
-static cvar_t cl_curl_enabled = {CF_CLIENT | CF_ARCHIVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
-static cvar_t cl_curl_useragent = {CF_CLIENT, "cl_curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
-static cvar_t cl_curl_useragent_append = {CF_CLIENT, "cl_curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
+static cvar_t curl_enabled = {CF_SHARED | CF_ARCHIVE, "curl_enabled","1", "whether libcurl may be used to GET files or POST data"};
+static cvar_t curl_maxdownloads = {CF_SHARED | CF_ARCHIVE, "curl_maxdownloads","3", "maximum number of concurrent HTTP/FTP downloads"};
+static cvar_t curl_maxspeed = {CF_SHARED | CF_ARCHIVE, "curl_maxspeed","0", "maximum download speed (KiB/s)"};
+static cvar_t curl_useragent = {CF_SHARED, "curl_useragent","1", "send the User-Agent string (note: turning this off may break stuff)"};
+static cvar_t curl_useragent_append = {CF_SHARED, "curl_useragent_append","", "a string to append to the User-Agent string (useful for name and version number of your mod)"};
+
+static cvar_t sv_curl_defaulturl = {CF_SERVER, "sv_curl_defaulturl","", "default autodownload source URL"};
+static cvar_t sv_curl_serverpackages = {CF_SERVER, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
+static cvar_t sv_curl_maxspeed = {CF_SERVER, "sv_curl_maxspeed","0", "maximum download speed for clients downloading from sv_curl_defaulturl (KiB/s)"};
+
+static cvar_t developer_curl = {CF_SHARED, "developer_curl","0", "whether verbose libcurl output should be printed to stderr"};
/*
=================================================================
CINIT(LOW_SPEED_TIME, LONG, 20),
CINIT(RESUME_FROM, LONG, 21),
CINIT(HTTPHEADER, OBJECTPOINT, 23),
+ CINIT(VERBOSE, LONG, 41),
CINIT(POST, LONG, 47), /* HTTP POST method */
CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */
CINIT(POSTFIELDSIZE, LONG, 60),
static CURLM * (*qcurl_multi_init) (void);
static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
+static CURLMcode (*qcurl_multi_wait) (CURLM *multi_handle, void*, unsigned int extra_nfds, int timeout_ms, int *ret);
static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
{"curl_easy_getinfo", (void **) &qcurl_easy_getinfo},
{"curl_multi_init", (void **) &qcurl_multi_init},
{"curl_multi_perform", (void **) &qcurl_multi_perform},
+ {"curl_multi_wait", (void **) &qcurl_multi_wait},
{"curl_multi_add_handle", (void **) &qcurl_multi_add_handle},
{"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
{"curl_multi_info_read", (void **) &qcurl_multi_info_read},
size_t bytes_received; // for buffer
double bytes_received_curl; // for throttling
double bytes_sent_curl; // for throttling
- struct downloadinfo_s *next, *prev;
+ llist_t list;
qbool forthismap;
double maxspeed;
curl_slist *slist; // http headers
const char *extraheaders;
}
downloadinfo;
-static downloadinfo *downloads = NULL;
+LIST_HEAD(downloads);
static int numdownloads = 0;
static qbool noclear = false;
if(!curl_dll)
return;
if(cmd)
- strlcpy(command_when_done, cmd, sizeof(command_when_done));
+ dp_strlcpy(command_when_done, cmd, sizeof(command_when_done));
else
*command_when_done = 0;
}
if(!curl_dll)
return;
if(cmd)
- strlcpy(command_when_error, cmd, sizeof(command_when_error));
+ dp_strlcpy(command_when_error, cmd, sizeof(command_when_error));
else
*command_when_error = 0;
}
if(noclear)
return;
if (curl_mutex) Thread_LockMutex(curl_mutex);
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
di->forthismap = false;
Curl_CommandWhenError(NULL);
Curl_CommandWhenDone(NULL);
if(numdownloads_fail == 0)
{
Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
- Cbuf_AddText(&cmd_client, "\n");
- Cbuf_AddText(&cmd_client, command_when_done);
- Cbuf_AddText(&cmd_client, "\n");
+ Cbuf_AddText(cmd_local, "\n");
+ Cbuf_AddText(cmd_local, command_when_done);
+ Cbuf_AddText(cmd_local, "\n");
}
else
{
Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
- Cbuf_AddText(&cmd_client, "\n");
- Cbuf_AddText(&cmd_client, command_when_error);
- Cbuf_AddText(&cmd_client, "\n");
+ Cbuf_AddText(cmd_local, "\n");
+ Cbuf_AddText(cmd_local, command_when_error);
+ Cbuf_AddText(cmd_local, "\n");
}
Curl_Clear_forthismap();
}
return true;
// Load the DLL
- return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
+ return Sys_LoadDependency (dllnames, &curl_dll, curlfuncs);
}
di->bytes_received += bytes;
+ //Con_Printf("CURL_fwrite callback timestamp: %f bytes: %ld\n", host.realtime, ret);
+
return ret;
// Why not ret / nmemb?
// Because CURLOPT_WRITEFUNCTION docs say to return the number of bytes.
break;
}
if(content_type_)
- strlcpy(content_type, content_type_, sizeof(content_type));
+ dp_strlcpy(content_type, content_type_, sizeof(content_type));
else
*content_type = 0;
if(ok && di->loadtype == LOADTYPE_PAK)
{
- ok = FS_AddPack(di->filename, NULL, true);
+ ok = FS_AddPack(di->filename, NULL, true, true);
if(!ok)
CLEAR_AND_RETRY();
}
CLEAR_AND_RETRY();
}
- if(di->prev)
- di->prev->next = di->next;
- else
- downloads = di->next;
- if(di->next)
- di->next->prev = di->prev;
+ List_Delete(&di->list);
--numdownloads;
if(di->forthismap)
char vabuf[1024];
if(!curl_dll)
return;
- if(numdownloads < cl_curl_maxdownloads.integer)
+ if(numdownloads < curl_maxdownloads.integer)
{
downloadinfo *di;
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
{
if(!di->started)
{
di->curle = qcurl_easy_init();
di->slist = NULL;
qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
- if(cl_curl_useragent.integer)
+ if(curl_useragent.integer)
{
const char *ua
#ifdef HTTP_USER_AGENT
#endif
if(!ua)
ua = "";
- if(*cl_curl_useragent_append.string)
+ if(*curl_useragent_append.string)
ua = va(vabuf, sizeof(vabuf), "%s%s%s",
ua,
(ua[0] && ua[strlen(ua)-1] != ' ')
? " "
: "",
- cl_curl_useragent_append.string);
+ curl_useragent_append.string);
qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, ua);
}
else
qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, "");
+ if(developer_curl.integer)
+ qcurl_easy_setopt(di->curle, CURLOPT_VERBOSE, (long) 1);
qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
qcurl_multi_add_handle(curlm, di->curle);
di->started = true;
++numdownloads;
- if(numdownloads >= cl_curl_maxdownloads.integer)
+ if(numdownloads >= curl_maxdownloads.integer)
break;
}
}
if(!curl_dll)
return;
if (Thread_HasThreads()) curl_mutex = Thread_CreateMutex();
- qcurl_global_init(CURL_GLOBAL_NOTHING);
+ qcurl_global_init(CURL_GLOBAL_SSL);
curlm = qcurl_multi_init();
}
curl_dll = NULL;
}
+// for VM_checkextension()
+qbool Curl_Available(void)
+{
+ return curl_dll ? true : false;
+}
+
/*
====================
Curl_Find
downloadinfo *di;
if(!curl_dll)
return NULL;
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
if(!strcasecmp(di->filename, filename))
return di;
return NULL;
void Curl_Cancel_ToMemory(curl_callback_t callback, void *cbdata)
{
- downloadinfo *di;
+ downloadinfo *di, *ndi;
if(!curl_dll)
return;
- for(di = downloads; di; )
+ List_For_Each_Entry_Safe(di, ndi, &downloads, downloadinfo, list)
{
if(di->callback == callback && di->callback_data == cbdata)
{
di->callback = curl_quiet_callback; // do NOT call the callback
Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
- di = downloads;
}
- else
- di = di->next;
}
}
if(loadtype != LOADTYPE_NONE)
Host_Error("Curl_Begin: loadtype and buffer are both set");
- if(!curl_dll || !cl_curl_enabled.integer)
+ if(!curl_dll || !curl_enabled.integer)
{
return false;
}
if(loadtype == LOADTYPE_PAK)
{
qbool already_loaded;
- if(FS_AddPack(fn, &already_loaded, true))
+ if(FS_AddPack(fn, &already_loaded, true, true))
{
Con_DPrintf("%s already exists, not downloading!\n", fn);
if(already_loaded)
if(forthismap)
++numdownloads_added;
di = (downloadinfo *) Z_Malloc(sizeof(*di));
- strlcpy(di->filename, name, sizeof(di->filename));
- strlcpy(di->url, URL, sizeof(di->url));
+ dp_strlcpy(di->filename, name, sizeof(di->filename));
+ dp_strlcpy(di->url, URL, sizeof(di->url));
dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
di->forthismap = forthismap;
di->stream = NULL;
di->bytes_received_curl = 0;
di->bytes_sent_curl = 0;
di->extraheaders = extraheaders;
- di->next = downloads;
- di->prev = NULL;
- if(di->next)
- di->next->prev = di;
-
di->buffer = buf;
di->buffersize = bufsize;
if(callback == NULL)
di->postbufsize = 0;
}
- downloads = di;
- if (curl_mutex) Thread_UnlockMutex(curl_mutex);
+ List_Add(&di->list, &downloads);
+
+ if (curl_mutex)
+ Thread_UnlockMutex(curl_mutex);
+
return true;
}
}
/*
====================
-Curl_Run
+Curl_Frame
call this regularily as this will always download as much as possible without
blocking.
====================
*/
-void Curl_Run(void)
+void Curl_Frame(void)
{
double maxspeed;
downloadinfo *di;
noclear = false;
- if(!cl_curl_enabled.integer)
+ if(!curl_enabled.integer && cls.state != ca_dedicated)
return;
if(!curl_dll)
Curl_CheckCommandWhenDone();
- if(!downloads)
+ if(List_Is_Empty(&downloads))
{
if (curl_mutex) Thread_UnlockMutex(curl_mutex);
return;
}
while(mc == CURLM_CALL_MULTI_PERFORM);
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
{
double b = 0;
if(di->curle)
// use the slowest allowing download to derive the maxspeed... this CAN
// be done better, but maybe later
- maxspeed = cl_curl_maxspeed.value;
- for(di = downloads; di; di = di->next)
+ maxspeed = curl_maxspeed.value;
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
if(di->maxspeed > 0)
if(di->maxspeed < maxspeed || maxspeed <= 0)
maxspeed = di->maxspeed;
if (curl_mutex) Thread_UnlockMutex(curl_mutex);
}
+/*
+====================
+Curl_Select
+
+Sleeps until there's some transfer progress or a timeout is reached,
+unfortunately the timeout is only in milliseconds.
+This allows good throughput even at very low FPS.
+Less important on newer libcurl versions but still helps.
+
+Returns 0 immediately if there's no transfers to wait for,
+or > 0 if a transfer is ready or the timeout was reached.
+====================
+*/
+int Curl_Select(int timeout_ms)
+{
+ CURLMcode err;
+ int numfds;
+
+ if (List_Is_Empty(&downloads))
+ return 0;
+
+ err = qcurl_multi_wait(curlm, NULL, 0, timeout_ms, &numfds);
+ if (err == CURLM_OK)
+ return numfds;
+ Con_Printf("curl_multi_wait() failed, code %d\n", err);
+ return 0;
+}
+
/*
====================
Curl_CancelAll
if (curl_mutex) Thread_LockMutex(curl_mutex);
- while(downloads)
+ while(!List_Is_Empty(&downloads))
{
- Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
+ Curl_EndDownload(List_First_Entry(&downloads, downloadinfo, list), CURL_DOWNLOAD_ABORTED, CURLE_OK, NULL);
// INVARIANT: downloads will point to the next download after that!
}
====================
Curl_Running
-returns true iff there is a download running.
+returns true if there is a download running.
====================
*/
qbool Curl_Running(void)
if(!curl_dll)
return false;
- return downloads != NULL;
+ return !List_Is_Empty(&downloads);
}
/*
{
if (curl_mutex) Thread_LockMutex(curl_mutex);
Con_Print("Currently running downloads:\n");
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
{
double speed, percent;
Con_Printf(" %s -> %s ", CleanURL(di->url, urlbuf, sizeof(urlbuf)), di->filename);
return;
}
- if(!cl_curl_enabled.integer)
+ if(!curl_enabled.integer)
{
Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
return;
*/
void Curl_Init_Commands(void)
{
- Cvar_RegisterVariable (&cl_curl_enabled);
- Cvar_RegisterVariable (&cl_curl_maxdownloads);
- Cvar_RegisterVariable (&cl_curl_maxspeed);
+ Cvar_RegisterVariable (&curl_enabled);
+ Cvar_RegisterVariable (&curl_maxdownloads);
+ Cvar_RegisterVariable (&curl_maxspeed);
+ Cvar_RegisterVariable (&curl_useragent);
+ Cvar_RegisterVariable (&curl_useragent_append);
+ Cvar_RegisterVirtual (&curl_enabled, "cl_curl_enabled");
+ Cvar_RegisterVirtual (&curl_maxdownloads, "cl_curl_maxdownloads");
+ Cvar_RegisterVirtual (&curl_maxspeed, "cl_curl_maxspeed");
+ Cvar_RegisterVirtual (&curl_useragent, "cl_curl_useragent");
+ Cvar_RegisterVirtual (&curl_useragent_append, "cl_curl_useragent_append");
+
Cvar_RegisterVariable (&sv_curl_defaulturl);
Cvar_RegisterVariable (&sv_curl_serverpackages);
Cvar_RegisterVariable (&sv_curl_maxspeed);
- Cvar_RegisterVariable (&cl_curl_useragent);
- Cvar_RegisterVariable (&cl_curl_useragent_append);
+
+ Cvar_RegisterVariable (&developer_curl);
+
Cmd_AddCommand(CF_CLIENT | CF_CLIENT_FROM_SERVER, "curl", Curl_Curl_f, "download data from an URL and add to search path");
- //Cmd_AddCommand(&cmd_client, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
+ //Cmd_AddCommand(cmd_local, "curlcat", Curl_CurlCat_f, "display data from an URL (debugging command)");
}
/*
if (curl_mutex) Thread_LockMutex(curl_mutex);
i = 0;
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
++i;
downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * i);
i = 0;
- for(di = downloads; di; di = di->next)
+ List_For_Each_Entry(di, &downloads, downloadinfo, list)
{
// do not show infobars for background downloads
if(developer.integer <= 0)
if(di->buffer)
continue;
- strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
+ dp_strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
if(di->curle)
{
downinfo[i].progress = Curl_GetDownloadAmount(di);
*urlend = 0;
if(matchpattern(filename, pattern, true))
{
- strlcpy(foundurl, url, sizeof(foundurl));
+ dp_strlcpy(foundurl, url, sizeof(foundurl));
Z_Free(buf);
return foundurl;
}
{
requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
req->next = requirements;
- strlcpy(req->filename, filename, sizeof(req->filename));
+ dp_strlcpy(req->filename, filename, sizeof(req->filename));
requirements = req;
}
if(packurl && *packurl && strcmp(packurl, "-"))
{
if(!foundone)
- strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
+ dp_strlcat(sendbuffer, "curl --clear_autodownload\n", sendbuffer_len);
- strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
- strlcat(sendbuffer, thispack, sendbuffer_len);
+ dp_strlcat(sendbuffer, "curl --pak --forthismap --as ", sendbuffer_len);
+ dp_strlcat(sendbuffer, thispack, sendbuffer_len);
if(sv_curl_maxspeed.value > 0)
dpsnprintf(sendbuffer + strlen(sendbuffer), sendbuffer_len - strlen(sendbuffer), " --maxspeed=%.1f", sv_curl_maxspeed.value);
- strlcat(sendbuffer, " --for ", sendbuffer_len);
- strlcat(sendbuffer, filename, sendbuffer_len);
- strlcat(sendbuffer, " ", sendbuffer_len);
- strlcat(sendbuffer, packurl, sendbuffer_len);
- strlcat(sendbuffer, thispack, sendbuffer_len);
- strlcat(sendbuffer, "\n", sendbuffer_len);
+ dp_strlcat(sendbuffer, " --for ", sendbuffer_len);
+ dp_strlcat(sendbuffer, filename, sendbuffer_len);
+ dp_strlcat(sendbuffer, " ", sendbuffer_len);
+ dp_strlcat(sendbuffer, packurl, sendbuffer_len);
+ dp_strlcat(sendbuffer, thispack, sendbuffer_len);
+ dp_strlcat(sendbuffer, "\n", sendbuffer_len);
return true;
}
foundone = Curl_SendRequirement(com_token, foundone, sendbuffer, sizeof(sendbuffer)) || foundone;
if(foundone)
- strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
+ dp_strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
SV_ClientCommands("%s", sendbuffer);
projects / xonotic / darkplaces.git / blobdiff