X-Git-Url: https://git.xonotic.org/?a=blobdiff_plain;f=utf8lib.c;h=bb775b7fd91b749a4593382f52691135c58f4250;hb=e1cc58c8345fbadd75421fa7843ec19bdf0b8681;hp=a5fadae77221d878dfb0820767827a96b50322df;hpb=148732505f409f6c371cc0460d1b2d8d3cac2d28;p=xonotic%2Fdarkplaces.git

diff --git a/utf8lib.c b/utf8lib.c
index a5fadae7..bb775b7f 100644
--- a/utf8lib.c
+++ b/utf8lib.c
@@ -20,6 +20,36 @@ UTF-8 encoding and decoding functions follow.
 ================================================================================
 */
 
+unsigned char utf8_lengths[256] = { // 0 = invalid
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // ascii characters
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x80 - 0xBF are within multibyte sequences
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // they could be interpreted as 2-byte starts but
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // the codepoint would be < 127
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0 and C1 would also result in overlong encodings
+	2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+	3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+	4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+	// with F5 the codepoint is above 0x10FFFF,
+	// F8-FB would start 5-byte sequences
+	// FC-FD would start 6-byte sequences
+	// ...
+};
+Uchar utf8_range[5] = {
+	1,       // invalid - let's not allow the creation of 0-bytes :P
+	1,       // ascii minimum
+	0x80,    // 2-byte minimum
+	0x800,   // 3-byte minimum
+	0x10000, // 4-byte minimum
+};
+
 /** Analyze the next character and return various information if requested.
  * @param _s      An utf-8 string.
  * @param _start  Filled with the start byte-offset of the next valid character
@@ -32,21 +62,50 @@ UTF-8 encoding and decoding functions follow.
 static qboolean u8_analyze(const char *_s, size_t *_start, size_t *_len, Uchar *_ch, size_t _maxlen)
 {
 	const unsigned char *s = (const unsigned char*)_s;
-	unsigned char bt, bc;
-	size_t i;
-	size_t bits, j;
+	size_t i, j;
+	size_t bits = 0;
 	Uchar ch;
 
 	i = 0;
 findchar:
+	while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0)
+		++i;
+
+	if (i >= _maxlen || !s[i]) {
+		if (_start) *_start = i;
+		if (_len) *_len = 0;
+		return false;
+	}
+	
+	if (bits == 1) { // ascii
+		if (_start) *_start = i;
+		if (_len) *_len = 1;
+		if (_ch) *_ch = (Uchar)s[i];
+		return true;
+	}
 
+	ch = (s[i] & (0xFF >> bits));
+	for (j = 1; j < bits; ++j)
+	{
+		if ( (s[i+j] & 0xC0) != 0x80 )
+		{
+			i += j;
+			goto findchar;
+		}
+		ch = (ch << 6) | (s[i+j] & 0x3F);
+	}
+	if (ch < utf8_range[bits] || ch >= 0x10FFFF)
+	{
+		i += bits;
+		goto findchar;
+	}
+#if 0
 	// <0xC2 is always an overlong encoding, they're invalid, thus skipped
 	while (i < _maxlen && s[i] && s[i] >= 0x80 && s[i] < 0xC2) {
 		//fprintf(stderr, "skipping\n");
 		++i;
 	}
 
-	//fprintf(stderr, "checking\n");
 	// If we hit the end, well, we're out and invalid
 	if(i >= _maxlen || !s[i]) {
 		if (_start) *_start = i;
@@ -54,8 +113,8 @@ findchar:
 		return false;
 	}
 
-	//fprintf(stderr, "checking ascii\n");
-	// ascii characters
+	// I'll leave that in - if you remove it, also change the part below
+	// to support 1-byte chars correctly
 	if (s[i] < 0x80)
 	{
 		if (_start) *_start = i;
@@ -64,7 +123,6 @@ findchar:
 		//fprintf(stderr, "valid ascii\n");
 		return true;
 	}
-	//fprintf(stderr, "checking length\n");
 
 	// Figure out the next char's length
 	bc = s[i];
@@ -117,6 +175,7 @@ findchar:
 		//fprintf(stderr, "overlong: %i bytes for %x\n", bits, ch);
 		goto findchar;
 	}
+#endif
 
 	if (_start)
 		*_start = i;
@@ -526,7 +585,7 @@ int u8_fromchar(Uchar w, char *to, size_t maxlen)
 		to[3] = 0x80 | (w & 0x3F); w >>= 6;
 		to[2] = 0x80 | (w & 0x3F); w >>= 6;
 		to[1] = 0x80 | (w & 0x3F); w >>= 6;
-		to[0] = 0xE0 | w;
+		to[0] = 0xF0 | w;
 		return 4;
 	}
 	return 0;
@@ -623,6 +682,7 @@ u8_COM_StringLengthNoColors(const char *_s, size_t size_s, qboolean *valid)
 	const unsigned char *s = (const unsigned char*)_s;
 	const unsigned char *end;
 	size_t len = 0;
+	size_t st, ln;
 
 	if (!utf8_enable.integer)
 		return COM_StringLengthNoColors(_s, size_s, valid);
@@ -668,22 +728,46 @@ u8_COM_StringLengthNoColors(const char *_s, size_t size_s, qboolean *valid)
 						++len; // the character
 						break;
 				}
-				break;
+				++s;
+				continue;
 			default:
-				++len;
 				break;
 		}
 
-		// start of a wide character
-		if (*s & 0xC0)
+		// ascii char, skip u8_analyze
+		if (*s < 0x80)
+		{
+			++len;
+			++s;
+			continue;
+		}
+
+		// invalid, skip u8_analyze
+		if (*s < 0xC2)
 		{
-			for (++s; *s >= 0x80 && *s <= 0xC0; ++s);
+			++s;
 			continue;
 		}
-		// part of a wide character, we ignore that one
-		if (*s <= 0xBF)
-			--len;
-		++s;
+
+		if (!u8_analyze((const char*)s, &st, &ln, NULL, U8_ANALYZE_INFINITY))
+		{
+			// we CAN end up here, if an invalid char is between this one and the end of the string
+			if(valid)
+				*valid = TRUE;
+			return len;
+		}
+
+		if(end && s + st + ln > end)
+		{
+			// string length exceeded by new character
+			if(valid)
+				*valid = TRUE;
+			return len;
+		}
+
+		// valid character, skip after it
+		s += st + ln;
+		++len;
 	}
 	// never get here
 }