/* * Copyright (C) 2012, 2013 * Wolfgang Bumiller * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies * of the Software, and to permit persons to whom the Software is furnished to do * so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "gmqcc.h" static unsigned char utf8_lengths[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* ascii characters */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x80 - 0xBF are within multibyte sequences */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* they could be interpreted as 2-byte starts but */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* the codepoint would be < 127 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* */ 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* C0 and C1 would also result in overlong encodings */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 /* with F5 the codepoint is above 0x10FFFF, * F8-FB would start 5-byte sequences * FC-FD would start 6-byte sequences * ... */ }; static uchar_t utf8_range[5] = { 1, /* invalid - let's not allow the creation of 0-bytes :P */ 1, /* ascii minimum */ 0x80, /* 2-byte minimum */ 0x800, /* 3-byte minimum */ 0x10000, /* 4-byte minimum */ }; /** Analyze the next character and return various information if requested. * @param _s An utf-8 string. * @param _start Filled with the start byte-offset of the next valid character * @param _len Filled with the length of the next valid character * @param _ch Filled with the unicode value of the next character * @param _maxlen Maximum number of bytes to read from _s * @return Whether or not another valid character is in the string */ bool u8_analyze(const char *_s, size_t *_start, size_t *_len, uchar_t *_ch, size_t _maxlen) { const unsigned char *s = (const unsigned char*)_s; size_t i, j; size_t bits = 0; uchar_t ch; i = 0; /* findchar: */ while (i < _maxlen && s[i] && (bits = utf8_lengths[s[i]]) == 0) ++i; if (i >= _maxlen || !s[i]) { if (_start) *_start = i; if (_len) *_len = 0; return false; } if (bits == 1) { /* ascii */ if (_start) *_start = i; if (_len) *_len = 1; if (_ch) *_ch = (uchar_t)s[i]; return true; } ch = (s[i] & (0xFF >> bits)); for (j = 1; j < bits; ++j) { if ( (s[i+j] & 0xC0) != 0x80 ) { i += j; /* in gmqcc, invalid / overlong encodings are considered an error * goto findchar; */ if (!s[i]) goto done; return false; } ch = (ch << 6) | (s[i+j] & 0x3F); } if (ch < utf8_range[bits] || ch >= 0x10FFFF) { /* same: error * i += bits; * goto findchar; */ return false; } done: if (_start) *_start = i; if (_len) *_len = bits; if (_ch) *_ch = ch; return true; } /* might come in handy */ size_t u8_strlen(const char *_s) { size_t st, ln; size_t len = 0; const unsigned char *s = (const unsigned char*)_s; while (*s) { /* ascii char, skip u8_analyze */ if (*s < 0x80) { ++len; ++s; continue; } /* invalid, skip u8_analyze */ if (*s < 0xC2) { ++s; continue; } if (!u8_analyze((const char*)s, &st, &ln, NULL, 0x10)) break; /* valid character, skip after it */ s += st + ln; ++len; } return len; } size_t u8_strnlen(const char *_s, size_t n) { size_t st, ln; size_t len = 0; const unsigned char *s = (const unsigned char*)_s; while (*s && n) { /* ascii char, skip u8_analyze */ if (*s < 0x80) { ++len; ++s; --n; continue; } /* invalid, skip u8_analyze */ if (*s < 0xC2) { ++s; --n; continue; } if (!u8_analyze((const char*)s, &st, &ln, NULL, n)) break; /* valid character, see if it's still inside the range specified by n: */ if (n < st + ln) return len; ++len; n -= st + ln; s += st + ln; } return len; } /* Required for character constants */ uchar_t u8_getchar(const char *_s, const char **_end) { size_t st, ln; uchar_t ch; if (!u8_analyze(_s, &st, &ln, &ch, 0x10)) ch = 0; if (_end) *_end = _s + st + ln; return ch; } uchar_t u8_getnchar(const char *_s, const char **_end, size_t _maxlen) { size_t st, ln; uchar_t ch; if (!u8_analyze(_s, &st, &ln, &ch, _maxlen)) ch = 0; if (_end) *_end = _s + st + ln; return ch; } /* required for \x{asdf}-like string escape sequences */ int u8_fromchar(uchar_t w, char *to, size_t maxlen) { if (maxlen < 1) return 0; if (!w) return 0; /* We may want an -f flag for this behaviour... if (w >= 0xE000) w -= 0xE000; */ if (w < 0x80) { to[0] = (char)w; if (maxlen < 2) return -1; to[1] = 0; return 1; } /* for a little speedup */ if (w < 0x800) { if (maxlen < 3) { to[0] = 0; return -1; } to[2] = 0; to[1] = 0x80 | (w & 0x3F); w >>= 6; to[0] = 0xC0 | w; return 2; } if (w < 0x10000) { if (maxlen < 4) { to[0] = 0; return -1; } to[3] = 0; to[2] = 0x80 | (w & 0x3F); w >>= 6; to[1] = 0x80 | (w & 0x3F); w >>= 6; to[0] = 0xE0 | w; return 3; } /* RFC 3629 */ if (w <= 0x10FFFF) { if (maxlen < 5) { to[0] = 0; return -1; } to[4] = 0; to[3] = 0x80 | (w & 0x3F); w >>= 6; to[2] = 0x80 | (w & 0x3F); w >>= 6; to[1] = 0x80 | (w & 0x3F); w >>= 6; to[0] = 0xF0 | w; return 4; } return 0; }