3 static unsigned char utf8_lengths[256] = {
4 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */
5 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
6 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
7 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
8 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
9 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
10 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
11 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
12 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences
13 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, * they could be interpreted as 2-byte starts but
14 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, * the codepoint would be < 127
15 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, *
16 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, * C0 and C1 would also result in overlong encodings
17 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, */
18 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
19 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
20 /* with F5 the codepoint is above 0x10FFFF,
21 * F8-FB would start 5-byte sequences
22 * FC-FD would start 6-byte sequences
27 static uchar_t utf8_range[5] = {
28 1, /* invalid - let's not allow the creation of 0-bytes :P
30 0x80, * 2-byte minimum
31 0x800, * 3-byte minimum
32 0x10000, * 4-byte minimum */
35 /** Analyze the next character and return various information if requested.
36 * @param _s An utf-8 string.
37 * @param _start Filled with the start byte-offset of the next valid character
38 * @param _len Filled with the length of the next valid character
39 * @param _ch Filled with the unicode value of the next character
40 * @param _maxlen Maximum number of bytes to read from _s
41 * @return Whether or not another valid character is in the string
43 bool u8_analyze(const char *_s, size_t *_start, size_t *_len, uchar_t *_ch, size_t _maxlen)
45 const unsigned char *s = (const unsigned char*)_s;
52 while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0)
55 if (i >= _maxlen || !s[i]) {
56 if (_start) *_start = i;
61 if (bits == 1) { /* ascii */
62 if (_start) *_start = i;
64 if (_ch) *_ch = (uchar_t)s[i];
68 ch = (s[i] & (0xFF >> bits));
69 for (j = 1; j < bits; ++j)
71 if ( (s[i+j] & 0xC0) != 0x80 )
74 /* in gmqcc, invalid / overlong encodings are considered an error
80 ch = (ch << 6) | (s[i+j] & 0x3F);
82 if (ch < utf8_range[bits] || ch >= 0x10FFFF)
101 /* might come in handy */
102 size_t u8_strlen(const char *_s)
106 const unsigned char *s = (const unsigned char*)_s;
110 /* ascii char, skip u8_analyze */
118 /* invalid, skip u8_analyze */
125 if (!u8_analyze((const char*)s, &st, &ln, NULL, 0x10))
127 /* valid character, skip after it */
134 size_t u8_strnlen(const char *_s, size_t n)
138 const unsigned char *s = (const unsigned char*)_s;
142 /* ascii char, skip u8_analyze */
151 /* invalid, skip u8_analyze */
159 if (!u8_analyze((const char*)s, &st, &ln, NULL, n))
161 /* valid character, see if it's still inside the range specified by n: */
171 /* Required for character constants */
172 uchar_t u8_getchar(const char *_s, const char **_end)
177 if (!u8_analyze(_s, &st, &ln, &ch, 0x10))
180 *_end = _s + st + ln;
184 uchar_t u8_getnchar(const char *_s, const char **_end, size_t _maxlen)
189 if (!u8_analyze(_s, &st, &ln, &ch, _maxlen))
192 *_end = _s + st + ln;
196 /* required for \x{asdf}-like string escape sequences */
197 int u8_fromchar(uchar_t w, char *to, size_t maxlen)
205 /* We may want an -f flag for this behaviour...
218 /* for a little speedup */
227 to[1] = 0x80 | (w & 0x3F); w >>= 6;
239 to[2] = 0x80 | (w & 0x3F); w >>= 6;
240 to[1] = 0x80 | (w & 0x3F); w >>= 6;
254 to[3] = 0x80 | (w & 0x3F); w >>= 6;
255 to[2] = 0x80 | (w & 0x3F); w >>= 6;
256 to[1] = 0x80 | (w & 0x3F); w >>= 6;