From: NaitLee <naitli@foxmail.com>
Date: Mon, 10 Jul 2023 11:14:00 +0000 (+0800)
Subject: Fix inability to access non-ascii path under Windows with WTF-8
X-Git-Url: http://git.xonotic.org/?p=xonotic%2Fdarkplaces.git;a=commitdiff_plain;h=5a9f4fc4e2545a0821cb0b92000a560602420a27

Fix inability to access non-ascii path under Windows with WTF-8

Signed-off-by: NaitLee <naitli@foxmail.com>
---

diff --git a/filematch.c b/filematch.c
index e0643961..d9c73bbd 100644
--- a/filematch.c
+++ b/filematch.c
@@ -7,6 +7,10 @@
 
 #include "darkplaces.h"
 
+#ifdef WIN32
+#include "utf8lib.h"
+#endif
+
 // LadyHavoc: some portable directory listing code I wrote for lmp2pcx, now used in darkplaces to load id1/*.pak and such...
 
 int matchpattern(const char *in, const char *pattern, int caseinsensitive)
@@ -164,20 +168,31 @@ static void adddirentry(stringlist_t *list, const char *path, const char *name)
 #ifdef WIN32
 void listdirectory(stringlist_t *list, const char *basepath, const char *path)
 {
-	char pattern[4096];
-	WIN32_FIND_DATA n_file;
+	#define BUFSIZE 4096
+	char pattern[BUFSIZE] = {0};
+	wchar patternw[BUFSIZE] = {0};
+	char filename[BUFSIZE] = {0};
+	wchar *filenamew;
+	int lenw = 0;
+	WIN32_FIND_DATAW n_file;
 	HANDLE hFile;
 	strlcpy (pattern, basepath, sizeof(pattern));
 	strlcat (pattern, path, sizeof (pattern));
 	strlcat (pattern, "*", sizeof (pattern));
+	fromwtf8(pattern, strlen(pattern), patternw, BUFSIZE);
 	// ask for the directory listing handle
-	hFile = FindFirstFile(pattern, &n_file);
+	hFile = FindFirstFileW(patternw, &n_file);
 	if(hFile == INVALID_HANDLE_VALUE)
 		return;
 	do {
-		adddirentry(list, path, n_file.cFileName);
-	} while (FindNextFile(hFile, &n_file) != 0);
+		filenamew = n_file.cFileName;
+		lenw = 0;
+		while(filenamew[lenw] != 0) ++lenw;
+		towtf8(filenamew, lenw, filename, BUFSIZE);
+		adddirentry(list, path, filename);
+	} while (FindNextFileW(hFile, &n_file) != 0);
 	FindClose(hFile);
+	#undef BUFSIZE
 }
 #else
 void listdirectory(stringlist_t *list, const char *basepath, const char *path)
diff --git a/fs.c b/fs.c
index 25789e02..a1a76063 100644
--- a/fs.c
+++ b/fs.c
@@ -49,6 +49,10 @@
 #include "fs.h"
 #include "wad.h"
 
+#ifdef WIN32
+#include "utf8lib.h"
+#endif
+
 // Win32 requires us to add O_BINARY, but the other OSes don't have it
 #ifndef O_BINARY
 # define O_BINARY 0
@@ -441,7 +445,7 @@ static dllhandle_t zlib_dll = NULL;
 static HRESULT (WINAPI *qSHGetFolderPath) (HWND hwndOwner, int nFolder, HANDLE hToken, DWORD dwFlags, LPTSTR pszPath);
 static dllfunction_t shfolderfuncs[] =
 {
-	{"SHGetFolderPathA", (void **) &qSHGetFolderPath},
+	{"SHGetFolderPathW", (void **) &qSHGetFolderPath},
 	{NULL, NULL}
 };
 static const char* shfolderdllnames [] =
@@ -926,14 +930,30 @@ static packfile_t* FS_AddFileToPack (const char* name, pack_t* pack,
 	return pfile;
 }
 
+#if WIN32
+#define WSTRBUF 4096
+static inline int wstrlen(wchar *wstr)
+{
+	int len = 0;
+	while (wstr[len] != 0 && len < WSTRBUF)
+		++len;
+	return len;
+}
+#define widen(str, wstr) fromwtf8(str, strlen(str), wstr, WSTRBUF)
+#define narrow(wstr, str) towtf8(wstr, wstrlen(wstr), str, WSTRBUF)
+#endif
 
 static void FS_mkdir (const char *path)
 {
+#if WIN32
+	wchar pathw[WSTRBUF] = {0};
+#endif
 	if(Sys_CheckParm("-readonly"))
 		return;
 
 #if WIN32
-	if (_mkdir (path) == -1)
+	widen(path, pathw);
+	if (_wmkdir (pathw) == -1)
 #else
 	if (mkdir (path, 0777) == -1)
 #endif
@@ -945,7 +965,6 @@ static void FS_mkdir (const char *path)
 	}
 }
 
-
 /*
 ============
 FS_CreatePath
@@ -1849,13 +1868,14 @@ static int FS_ChooseUserDir(userdirmode_t userdirmode, char *userdir, size_t use
 	return -1;
 
 #elif defined(WIN32)
-	char *homedir;
+	char homedir[WSTRBUF];
+	wchar *homedirw;
 #if _MSC_VER >= 1400
-	size_t homedirlen;
+	size_t homedirwlen;
 #endif
 	TCHAR mydocsdir[MAX_PATH + 1];
 	wchar_t *savedgamesdirw;
-	char savedgamesdir[MAX_OSPATH];
+	char savedgamesdir[WSTRBUF] = {0};
 	int fd;
 	char vabuf[1024];
 
@@ -1877,16 +1897,18 @@ static int FS_ChooseUserDir(userdirmode_t userdirmode, char *userdir, size_t use
 			break;
 		}
 #if _MSC_VER >= 1400
-		_dupenv_s(&homedir, &homedirlen, "USERPROFILE");
-		if(homedir)
+		_wdupenv_s(&homedirw, &homedirwlen, L"USERPROFILE");
+		narrow(homedirw, homedir);
+		if(homedir[0])
 		{
 			dpsnprintf(userdir, userdirsize, "%s/.%s/", homedir, gameuserdirname);
-			free(homedir);
+			free(homedirw);
 			break;
 		}
 #else
-		homedir = getenv("USERPROFILE");
-		if(homedir)
+		homedirw = _wgetenv(L"USERPROFILE");
+		narrow(homedirw, homedir);
+		if(homedir[0])
 		{
 			dpsnprintf(userdir, userdirsize, "%s/.%s/", homedir, gameuserdirname);
 			break;
@@ -1911,12 +1933,7 @@ static int FS_ChooseUserDir(userdirmode_t userdirmode, char *userdir, size_t use
 */
 			if (qSHGetKnownFolderPath(&qFOLDERID_SavedGames, qKF_FLAG_CREATE | qKF_FLAG_NO_ALIAS, NULL, &savedgamesdirw) == S_OK)
 			{
-				memset(savedgamesdir, 0, sizeof(savedgamesdir));
-#if _MSC_VER >= 1400
-				wcstombs_s(NULL, savedgamesdir, sizeof(savedgamesdir), savedgamesdirw, sizeof(savedgamesdir)-1);
-#else
-				wcstombs(savedgamesdir, savedgamesdirw, sizeof(savedgamesdir)-1);
-#endif
+				narrow(savedgamesdirw, savedgamesdir);
 				qCoTaskMemFree(savedgamesdirw);
 			}
 			qCoUninitialize();
@@ -2279,6 +2296,9 @@ static filedesc_t FS_SysOpenFiledesc(const char *filepath, const char *mode, qbo
 	int mod, opt;
 	unsigned int ind;
 	qbool dolock = false;
+	#ifdef WIN32
+	wchar filepathw[WSTRBUF] = {0};
+	#endif
 
 	// Parse the mode string
 	switch (mode[0])
@@ -2330,10 +2350,11 @@ static filedesc_t FS_SysOpenFiledesc(const char *filepath, const char *mode, qbo
 	handle = SDL_RWFromFile(filepath, mode);
 #else
 # ifdef WIN32
+	widen(filepath, filepathw);
 #  if _MSC_VER >= 1400
-	_sopen_s(&handle, filepath, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
+	_wsopen_s(&handle, filepathw, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
 #  else
-	handle = _sopen (filepath, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
+	handle = _wsopen (filepathw, mod | opt, (dolock ? ((mod == O_RDONLY) ? _SH_DENYRD : _SH_DENYRW) : _SH_DENYNO), _S_IREAD | _S_IWRITE);
 #  endif
 # else
 	handle = open (filepath, mod | opt, 0666);
@@ -3592,8 +3613,10 @@ int FS_SysFileType (const char *path)
 # ifndef INVALID_FILE_ATTRIBUTES
 #  define INVALID_FILE_ATTRIBUTES ((DWORD)-1)
 # endif
-
-	DWORD result = GetFileAttributes(path);
+	wchar pathw[WSTRBUF] = {0};
+	DWORD result;
+	widen(path, pathw);
+	result = GetFileAttributesW(pathw);
 
 	if(result == INVALID_FILE_ATTRIBUTES)
 		return FS_FILETYPE_NONE;
diff --git a/netconn.c b/netconn.c
old mode 100755
new mode 100644
diff --git a/utf8lib.c b/utf8lib.c
index 155b4aea..a15c25f6 100644
--- a/utf8lib.c
+++ b/utf8lib.c
@@ -907,6 +907,112 @@ size_t u8_strpad_colorcodes(char *out, size_t outsize, const char *in, qbool lef
 	return dpsnprintf(out, outsize, "%*s%.*s%*s", lpad, "", prec, in, rpad, "");
 }
 
+#ifdef WIN32
+
+/**
+ * Convert Windows "wide" characters to WTF-8 for internal manipulation
+ */
+int towtf8(const wchar *wstr, int wlen, char *cstr, int maxclen)
+{
+    int p = 0;
+	int i;
+    for (i = 0; i < wlen; ++i)
+	{
+        wchar point = wstr[i];
+        if (point < 0x80)
+		{
+            if (p + 1 >= maxclen) break;
+            cstr[p++] = point;
+        }
+		else if (point < 0x800)
+		{
+            if (p + 2 >= maxclen) break;
+            cstr[p++] = (0xc0 | ((point >>  6) & 0x1f));
+            cstr[p++] = (0x80 | ((point >>  0) & 0x3f));
+        }
+		else
+		#if U32
+		if (point < 0x10000)
+		#endif
+        {
+            if (p + 3 >= maxclen) break;
+            cstr[p++] = (0xe0 | ((point >> 12) & 0x0f));
+            cstr[p++] = (0x80 | ((point >>  6) & 0x3f));
+            cstr[p++] = (0x80 | ((point >>  0) & 0x3f));
+        }
+        #if U32
+        else
+        #if CHECKS
+        if (point < 0x110000)
+        #endif
+        {
+            if (p + 4 >= maxclen) break;
+            cstr[p++] = (0xf0 | ((point >> 18) & 0x07));
+            cstr[p++] = (0x80 | ((point >> 12) & 0x3f));
+            cstr[p++] = (0x80 | ((point >>  6) & 0x3f));
+            cstr[p++] = (0x80 | ((point >>  0) & 0x3f));
+        }
+        #endif
+    }
+    cstr[p] = 0x00;
+    return p;
+}
+
+/**
+ * Convert WTF-8 string to "wide" characters used by Windows
+ */
+int fromwtf8(const char *cstr, int clen, wchar *wstr, int maxwlen)
+{
+    int p = 0;
+	int i;
+    for (i = 0; i < clen;)
+	{
+        char byte = cstr[i++];
+        wchar point = byte;
+        int length = 1;
+        if (p + 1 >= maxwlen) break;
+        #if CHECKS
+        if ((byte & 0xf8) == 0xf8)
+            return -1;
+        #endif
+        if ((byte & 0xf8) == 0xf0)
+		{
+            length = 4;
+            point = byte & 0x07;
+        }
+		else if ((byte & 0xf0) == 0xe0)
+		{
+            length = 3;
+            point = byte & 0x0f;
+        }
+		else if ((byte & 0xe0) == 0xc0)
+		{
+            length = 2;
+            point = byte & 0x1f;
+        }
+        #if CHECKS
+        else if ((byte & 0xc0) == 0x80)
+		{
+            return -1;
+        }
+        #endif
+        while (--length)
+		{
+            byte = cstr[i++];
+            #if CHECKS
+            if (byte == -1) return -1;
+            else if ((byte & 0xc0) != 0x80) return -1;
+            #endif
+            point = (point << 6) | (byte & 0x3f);
+        }
+        wstr[p++] = point;
+    }
+    wstr[p] = 0x00;
+    return p;
+}
+
+#endif // WIN32
+
 
 /*
 The two following functions (u8_toupper, u8_tolower) are derived from
diff --git a/utf8lib.h b/utf8lib.h
index 4b8221f5..cb7041b9 100644
--- a/utf8lib.h
+++ b/utf8lib.h
@@ -78,4 +78,17 @@ extern Uchar u8_quake2utf8map[256];
 Uchar u8_toupper(Uchar ch);
 Uchar u8_tolower(Uchar ch);
 
+#ifdef WIN32
+
+// WTF-8 encoding to circumvent Windows encodings, be it UTF-16 or random codepages
+// https://simonsapin.github.io/wtf-8/
+#define U32 0
+#define CHECKS 1
+typedef wchar_t wchar;
+
+int towtf8(const wchar* wstr, int wlen, char* cstr, int maxclen);
+int fromwtf8(const char* cstr, int clen, wchar* wstr, int maxwlen);
+
+#endif // WIN32
+
 #endif // UTF8LIB_H__