X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=blobdiff_plain;f=lexer.c;h=541104127393fb9ba393f64f9f49edd62214d97d;hp=a71e50f6cfa9fcbdd19224f17c48a706ce06f6be;hb=a68f0fcb355db42acabe72da5939fbd1b04f6016;hpb=61fa54318c3aa1ce0c2099f9793d830e5896c016 diff --git a/lexer.c b/lexer.c index a71e50f..5411041 100644 --- a/lexer.c +++ b/lexer.c @@ -1,41 +1,88 @@ -#include -#include +/* + * Copyright (C) 2012, 2013 + * Wolfgang Bumiller + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ #include -#include +#include #include "gmqcc.h" #include "lexer.h" +/* + * List of Keywords + */ -char* *lex_filenames; +/* original */ +static const char *keywords_qc[] = { + "for", "do", "while", + "if", "else", + "local", + "return", + "const" +}; +static size_t num_keywords_qc = sizeof(keywords_qc) / sizeof(keywords_qc[0]); + +/* For fte/gmgqcc */ +static const char *keywords_fg[] = { + "switch", "case", "default", + "struct", "union", + "break", "continue", + "typedef", + "goto", + + "__builtin_debug_printtype" +}; +static size_t num_keywords_fg = sizeof(keywords_fg) / sizeof(keywords_fg[0]); + +/* + * Lexer code + */ -void lexerror(lex_file *lex, const char *fmt, ...) +static char* *lex_filenames; + +static void lexerror(lex_file *lex, const char *fmt, ...) { va_list ap; va_start(ap, fmt); if (lex) - con_vprintmsg(LVL_ERROR, lex->name, lex->sline, "parse error", fmt, ap); + con_vprintmsg(LVL_ERROR, lex->name, lex->sline, lex->column, "parse error", fmt, ap); else - con_vprintmsg(LVL_ERROR, "", 0, "parse error", fmt, ap); + con_vprintmsg(LVL_ERROR, "", 0, 0, "parse error", fmt, ap); va_end(ap); } -bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...) +static bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...) { - va_list ap; - int lvl = LVL_WARNING; + bool r; + lex_ctx_t ctx; + va_list ap; - if (!OPTS_WARN(warntype)) - return false; - - if (opts_werror) - lvl = LVL_ERROR; + ctx.file = lex->name; + ctx.line = lex->sline; + ctx.column = lex->column; va_start(ap, fmt); - con_vprintmsg(lvl, lex->name, lex->sline, "warning", fmt, ap); + r = vcompile_warning(ctx, warntype, fmt, ap); va_end(ap); - - return opts_werror; + return r; } @@ -125,9 +172,11 @@ static void lex_token_new(lex_file *lex) #else if (lex->tok.value) vec_shrinkto(lex->tok.value, 0); + lex->tok.constval.t = 0; - lex->tok.ctx.line = lex->sline; - lex->tok.ctx.file = lex->name; + lex->tok.ctx.line = lex->sline; + lex->tok.ctx.file = lex->name; + lex->tok.ctx.column = lex->column; #endif } #endif @@ -135,7 +184,7 @@ static void lex_token_new(lex_file *lex) lex_file* lex_open(const char *file) { lex_file *lex; - FILE *in = util_fopen(file, "rb"); + FILE *in = fs_file_open(file, "rb"); if (!in) { lexerror(NULL, "open failed: '%s'\n", file); @@ -144,19 +193,19 @@ lex_file* lex_open(const char *file) lex = (lex_file*)mem_a(sizeof(*lex)); if (!lex) { - fclose(in); + fs_file_close(in); lexerror(NULL, "out of memory\n"); return NULL; } memset(lex, 0, sizeof(*lex)); - lex->file = in; - lex->name = util_strdup(file); - lex->line = 1; /* we start counting at 1 */ - + lex->file = in; + lex->name = util_strdup(file); + lex->line = 1; /* we start counting at 1 */ + lex->column = 0; lex->peekpos = 0; - lex->eof = false; + lex->eof = false; vec_push(lex_filenames, lex->name); return lex; @@ -179,11 +228,11 @@ lex_file* lex_open_string(const char *str, size_t len, const char *name) lex->open_string_length = len; lex->open_string_pos = 0; - lex->name = util_strdup(name ? name : ""); - lex->line = 1; /* we start counting at 1 */ - + lex->name = util_strdup(name ? name : ""); + lex->line = 1; /* we start counting at 1 */ lex->peekpos = 0; - lex->eof = false; + lex->eof = false; + lex->column = 0; vec_push(lex_filenames, lex->name); @@ -209,7 +258,7 @@ void lex_close(lex_file *lex) vec_free(lex->modelname); if (lex->file) - fclose(lex->file); + fs_file_close(lex->file); #if 0 if (lex->tok) token_delete(lex->tok); @@ -222,11 +271,14 @@ void lex_close(lex_file *lex) static int lex_fgetc(lex_file *lex) { - if (lex->file) - return fgetc(lex->file); + if (lex->file) { + lex->column++; + return fs_file_getc(lex->file); + } if (lex->open_string) { if (lex->open_string_pos >= lex->open_string_length) return EOF; + lex->column++; return lex->open_string[lex->open_string_pos++]; } return EOF; @@ -242,12 +294,22 @@ static int lex_try_trigraph(lex_file *lex, int old) { int c2, c3; c2 = lex_fgetc(lex); + if (!lex->push_line && c2 == '\n') { + lex->line++; + lex->column = 0; + } + if (c2 != '?') { lex_ungetch(lex, c2); return old; } c3 = lex_fgetc(lex); + if (!lex->push_line && c3 == '\n') { + lex->line++; + lex->column = 0; + } + switch (c3) { case '=': return '#'; case '/': return '\\'; @@ -269,6 +331,11 @@ static int lex_try_digraph(lex_file *lex, int ch) { int c2; c2 = lex_fgetc(lex); + /* we just used fgetc() so count lines + * need to offset a \n the ungetch would recognize + */ + if (!lex->push_line && c2 == '\n') + lex->line++; if (ch == '<' && c2 == ':') return '['; else if (ch == ':' && c2 == '>') @@ -307,8 +374,11 @@ static int lex_getch(lex_file *lex) static void lex_ungetch(lex_file *lex, int ch) { lex->peek[lex->peekpos++] = ch; - if (!lex->push_line && ch == '\n') + lex->column--; + if (!lex->push_line && ch == '\n') { lex->line--; + lex->column = 0; + } } /* classify characters @@ -318,12 +388,12 @@ static void lex_ungetch(lex_file *lex, int ch) /* Idents are alphanumberic, but they start with alpha or _ */ static bool isident_start(int ch) { - return isalpha(ch) || ch == '_'; + return util_isalpha(ch) || ch == '_'; } static bool isident(int ch) { - return isident_start(ch) || isdigit(ch); + return isident_start(ch) || util_isdigit(ch); } /* isxdigit_only is used when we already know it's not a digit @@ -384,7 +454,7 @@ static bool lex_try_pragma(lex_file *lex) goto unroll; } - for (ch = lex_getch(lex); vec_size(param) < 32 && ch != ')' && ch != '\n'; ch = lex_getch(lex)) + for (ch = lex_getch(lex); vec_size(param) < 1024 && ch != ')' && ch != '\n'; ch = lex_getch(lex)) vec_push(param, ch); vec_push(param, 0); @@ -396,7 +466,8 @@ static bool lex_try_pragma(lex_file *lex) if (!strcmp(command, "push")) { if (!strcmp(param, "line")) { lex->push_line++; - --line; + if (lex->push_line == 1) + --line; } else goto unroll; @@ -405,7 +476,8 @@ static bool lex_try_pragma(lex_file *lex) if (!strcmp(param, "line")) { if (lex->push_line) lex->push_line--; - --line; + if (lex->push_line == 0) + --line; } else goto unroll; @@ -423,29 +495,34 @@ static bool lex_try_pragma(lex_file *lex) lex->line = line; while (ch != '\n' && ch != EOF) ch = lex_getch(lex); + vec_free(command); + vec_free(param); + vec_free(pragma); return true; unroll: if (command) { vec_pop(command); while (vec_size(command)) { - lex_ungetch(lex, vec_last(command)); + lex_ungetch(lex, (unsigned char)vec_last(command)); vec_pop(command); } vec_free(command); + lex_ungetch(lex, ' '); } - if (command) { - vec_pop(command); - while (vec_size(command)) { - lex_ungetch(lex, vec_last(command)); - vec_pop(command); + if (param) { + vec_pop(param); + while (vec_size(param)) { + lex_ungetch(lex, (unsigned char)vec_last(param)); + vec_pop(param); } - vec_free(command); + vec_free(param); + lex_ungetch(lex, ' '); } if (pragma) { vec_pop(pragma); while (vec_size(pragma)) { - lex_ungetch(lex, vec_last(pragma)); + lex_ungetch(lex, (unsigned char)vec_last(pragma)); vec_pop(pragma); } vec_free(pragma); @@ -488,15 +565,15 @@ printf( "line one\n" * here is to store the line of the first character after skipping * the initial whitespace in lex->sline, this happens in lex_do. */ -static int lex_skipwhite(lex_file *lex) +static int lex_skipwhite(lex_file *lex, bool hadwhite) { int ch = 0; - bool haswhite = false; + bool haswhite = hadwhite; do { ch = lex_getch(lex); - while (ch != EOF && isspace(ch)) { + while (ch != EOF && util_isspace(ch)) { if (ch == '\n') { if (lex_try_pragma(lex)) continue; @@ -577,9 +654,13 @@ static int lex_skipwhite(lex_file *lex) } break; } + lex_ungetch(lex, ch); } if (lex->flags.preprocessing) { - lex_tokench(lex, ' '); /* ch); */ + if (ch == '\n') + lex_tokench(lex, '\n'); + else + lex_tokench(lex, ' '); /* ch); */ } } ch = ' '; /* cause TRUE in the isspace check */ @@ -590,7 +671,7 @@ static int lex_skipwhite(lex_file *lex) ch = '/'; break; } - } while (ch != EOF && isspace(ch)); + } while (ch != EOF && util_isspace(ch)); if (haswhite) { lex_endtoken(lex); @@ -626,7 +707,7 @@ static int lex_parse_frame(lex_file *lex) lex_token_new(lex); ch = lex_getch(lex); - while (ch != EOF && ch != '\n' && isspace(ch)) + while (ch != EOF && ch != '\n' && util_isspace(ch)) ch = lex_getch(lex); if (ch == '\n') @@ -674,11 +755,18 @@ static bool lex_finish_frames(lex_file *lex) vec_shrinkto(lex->tok.value, 0); vec_push(lex->frames, m); } while (true); + + return false; } static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) { + uchar_t chr; int ch = 0; + int nextch; + bool hex; + char u8buf[8]; /* way more than enough */ + int u8len, uc; while (ch != EOF) { @@ -715,6 +803,101 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) case 't': ch = '\t'; break; case 'f': ch = '\f'; break; case 'v': ch = '\v'; break; + case 'x': + case 'X': + /* same procedure as in fteqcc */ + ch = 0; + nextch = lex_getch(lex); + if (nextch >= '0' && nextch <= '9') + ch += nextch - '0'; + else if (nextch >= 'a' && nextch <= 'f') + ch += nextch - 'a' + 10; + else if (nextch >= 'A' && nextch <= 'F') + ch += nextch - 'A' + 10; + else { + lexerror(lex, "bad character code"); + lex_ungetch(lex, nextch); + return (lex->tok.ttype = TOKEN_ERROR); + } + + ch *= 0x10; + nextch = lex_getch(lex); + if (nextch >= '0' && nextch <= '9') + ch += nextch - '0'; + else if (nextch >= 'a' && nextch <= 'f') + ch += nextch - 'a' + 10; + else if (nextch >= 'A' && nextch <= 'F') + ch += nextch - 'A' + 10; + else { + lexerror(lex, "bad character code"); + lex_ungetch(lex, nextch); + return (lex->tok.ttype = TOKEN_ERROR); + } + break; + + /* fteqcc support */ + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + case '8': case '9': + ch = 18 + ch - '0'; + break; + case '<': ch = 29; break; + case '-': ch = 30; break; + case '>': ch = 31; break; + case '[': ch = 16; break; + case ']': ch = 17; break; + case '{': + chr = 0; + nextch = lex_getch(lex); + hex = (nextch == 'x'); + if (!hex) + lex_ungetch(lex, nextch); + for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) { + if (!hex) { + if (nextch >= '0' && nextch <= '9') + chr = chr * 10 + nextch - '0'; + else { + lexerror(lex, "bad character code"); + return (lex->tok.ttype = TOKEN_ERROR); + } + } else { + if (nextch >= '0' && nextch <= '9') + chr = chr * 0x10 + nextch - '0'; + else if (nextch >= 'a' && nextch <= 'f') + chr = chr * 0x10 + nextch - 'a' + 10; + else if (nextch >= 'A' && nextch <= 'F') + chr = chr * 0x10 + nextch - 'A' + 10; + else { + lexerror(lex, "bad character code"); + return (lex->tok.ttype = TOKEN_ERROR); + } + } + if (chr > 0x10FFFF || (!OPTS_FLAG(UTF8) && chr > 255)) + { + lexerror(lex, "character code out of range"); + return (lex->tok.ttype = TOKEN_ERROR); + } + } + if (OPTS_FLAG(UTF8) && chr >= 128) { + u8len = u8_fromchar(chr, u8buf, sizeof(u8buf)); + if (!u8len) + ch = 0; + else { + --u8len; + lex->column += u8len; + for (uc = 0; uc < u8len; ++uc) + lex_tokench(lex, u8buf[uc]); + /* the last character will be inserted with the tokench() call + * below the switch + */ + ch = u8buf[uc]; + } + } + else + ch = chr; + break; + case '\n': ch = '\n'; break; + default: lexwarn(lex, WARN_UNKNOWN_CONTROL_SEQUENCE, "unrecognized control sequence: \\%c", ch); /* so we just add the character plus backslash no matter what it actually is */ @@ -738,12 +921,15 @@ static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch) int ch = lastch; /* parse a number... */ - lex->tok.ttype = TOKEN_INTCONST; + if (ch == '.') + lex->tok.ttype = TOKEN_FLOATCONST; + else + lex->tok.ttype = TOKEN_INTCONST; lex_tokench(lex, ch); ch = lex_getch(lex); - if (ch != '.' && !isdigit(ch)) + if (ch != '.' && !util_isdigit(ch)) { if (lastch != '0' || ch != 'x') { @@ -764,14 +950,14 @@ static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch) { lex_tokench(lex, ch); ch = lex_getch(lex); - while (isdigit(ch) || (ishex && isxdigit_only(ch))) + while (util_isdigit(ch) || (ishex && isxdigit_only(ch))) { lex_tokench(lex, ch); ch = lex_getch(lex); } } /* NOT else, '.' can come from above as well */ - if (ch == '.' && !ishex) + if (lex->tok.ttype != TOKEN_FLOATCONST && ch == '.' && !ishex) { /* Allow floating comma in non-hex mode */ lex->tok.ttype = TOKEN_FLOATCONST; @@ -779,7 +965,7 @@ static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch) /* continue digits-only */ ch = lex_getch(lex); - while (isdigit(ch)) + while (util_isdigit(ch)) { lex_tokench(lex, ch); ch = lex_getch(lex); @@ -807,7 +993,8 @@ static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch) int lex_do(lex_file *lex) { - int ch, nextch; + int ch, nextch, thirdch; + bool hadwhite = false; lex_token_new(lex); #if 0 @@ -816,10 +1003,13 @@ int lex_do(lex_file *lex) #endif while (true) { - ch = lex_skipwhite(lex); + ch = lex_skipwhite(lex, hadwhite); + hadwhite = true; if (!lex->flags.mergelines || ch != '\\') break; ch = lex_getch(lex); + if (ch == '\r') + ch = lex_getch(lex); if (ch != '\n') { lex_ungetch(lex, ch); ch = '\\'; @@ -830,14 +1020,14 @@ int lex_do(lex_file *lex) continue; } - lex->sline = lex->line; - lex->tok.ctx.line = lex->sline; - lex->tok.ctx.file = lex->name; - if (lex->flags.preprocessing && (ch == TOKEN_WHITE || ch == TOKEN_EOL || ch == TOKEN_FATAL)) { return (lex->tok.ttype = ch); } + lex->sline = lex->line; + lex->tok.ctx.line = lex->sline; + lex->tok.ctx.file = lex->name; + if (lex->eof) return (lex->tok.ttype = TOKEN_FATAL); @@ -847,7 +1037,7 @@ int lex_do(lex_file *lex) } /* modelgen / spiritgen commands */ - if (ch == '$') { + if (ch == '$' && !lex->flags.preprocessing) { const char *v; size_t frame; @@ -878,10 +1068,10 @@ int lex_do(lex_file *lex) if (!strcmp(v, "framevalue")) { ch = lex_getch(lex); - while (ch != EOF && isspace(ch) && ch != '\n') + while (ch != EOF && util_isspace(ch) && ch != '\n') ch = lex_getch(lex); - if (!isdigit(ch)) { + if (!util_isdigit(ch)) { lexerror(lex, "$framevalue requires an integer parameter"); return lex_do(lex); } @@ -938,7 +1128,6 @@ int lex_do(lex_file *lex) if (rc < 0) return (lex->tok.ttype = TOKEN_FATAL); - v = lex->tok.value; if (lex->modelname) { frame_macro m; m.value = lex->framevalue; @@ -953,9 +1142,9 @@ int lex_do(lex_file *lex) if (!strcmp(v, "flush")) { - size_t frame; - for (frame = 0; frame < vec_size(lex->frames); ++frame) - mem_d(lex->frames[frame].name); + size_t fi; + for (fi = 0; fi < vec_size(lex->frames); ++fi) + mem_d(lex->frames[fi].name); vec_free(lex->frames); /* skip line (fteqcc does it too) */ ch = lex_getch(lex); @@ -993,19 +1182,41 @@ int lex_do(lex_file *lex) switch (ch) { case '[': + nextch = lex_getch(lex); + if (nextch == '[') { + lex_tokench(lex, ch); + lex_tokench(lex, nextch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_ATTRIBUTE_OPEN); + } + lex_ungetch(lex, nextch); + /* FALL THROUGH */ case '(': + case ':': + case '?': lex_tokench(lex, ch); lex_endtoken(lex); if (lex->flags.noops) return (lex->tok.ttype = ch); else return (lex->tok.ttype = TOKEN_OPERATOR); + + case ']': + if (lex->flags.noops) { + nextch = lex_getch(lex); + if (nextch == ']') { + lex_tokench(lex, ch); + lex_tokench(lex, nextch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_ATTRIBUTE_CLOSE); + } + lex_ungetch(lex, nextch); + } + /* FALL THROUGH */ case ')': case ';': - case ':': case '{': case '}': - case ']': case '#': lex_tokench(lex, ch); @@ -1015,6 +1226,18 @@ int lex_do(lex_file *lex) break; } + if (ch == '.') { + nextch = lex_getch(lex); + /* digits starting with a dot */ + if (util_isdigit(nextch)) { + lex_ungetch(lex, nextch); + lex->tok.ttype = lex_finish_digit(lex, ch); + lex_endtoken(lex); + return lex->tok.ttype; + } + lex_ungetch(lex, nextch); + } + if (lex->flags.noops) { /* Detect characters early which are normally @@ -1022,8 +1245,10 @@ int lex_do(lex_file *lex) */ switch (ch) { + /* case '+': case '-': + */ case '*': case '/': case '<': @@ -1041,31 +1266,37 @@ int lex_do(lex_file *lex) default: break; } + } - if (ch == '.') - { - lex_tokench(lex, ch); - /* peak ahead once */ - nextch = lex_getch(lex); - if (nextch != '.') { - lex_ungetch(lex, nextch); - lex_endtoken(lex); - return (lex->tok.ttype = ch); - } - /* peak ahead again */ - nextch = lex_getch(lex); - if (nextch != '.') { - lex_ungetch(lex, nextch); - lex_ungetch(lex, nextch); - lex_endtoken(lex); + if (ch == '.') + { + lex_tokench(lex, ch); + /* peak ahead once */ + nextch = lex_getch(lex); + if (nextch != '.') { + lex_ungetch(lex, nextch); + lex_endtoken(lex); + if (lex->flags.noops) return (lex->tok.ttype = ch); - } - /* fill the token to be "..." */ - lex_tokench(lex, ch); - lex_tokench(lex, ch); + else + return (lex->tok.ttype = TOKEN_OPERATOR); + } + /* peak ahead again */ + nextch = lex_getch(lex); + if (nextch != '.') { + lex_ungetch(lex, nextch); + lex_ungetch(lex, '.'); lex_endtoken(lex); - return (lex->tok.ttype = TOKEN_DOTS); + if (lex->flags.noops) + return (lex->tok.ttype = ch); + else + return (lex->tok.ttype = TOKEN_OPERATOR); } + /* fill the token to be "..." */ + lex_tokench(lex, ch); + lex_tokench(lex, ch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_DOTS); } if (ch == ',' || ch == '.') { @@ -1075,19 +1306,49 @@ int lex_do(lex_file *lex) } if (ch == '+' || ch == '-' || /* ++, --, +=, -= and -> as well! */ - ch == '>' || ch == '<' || /* <<, >>, <=, >= */ - ch == '=' || ch == '!' || /* ==, != */ - ch == '&' || ch == '|') /* &&, ||, &=, |= */ - { + ch == '>' || ch == '<' || /* <<, >>, <=, >= */ + ch == '=' || ch == '!' || /* <=>, ==, != */ + ch == '&' || ch == '|' || /* &&, ||, &=, |= */ + ch == '~' || ch == '^' /* ~=, ~, ^ */ + ) { lex_tokench(lex, ch); nextch = lex_getch(lex); - if (nextch == ch || nextch == '=') { + if ((nextch == '=' && ch != '<') || (nextch == ch && ch != '!')) { + lex_tokench(lex, nextch); + } else if (ch == '<' && nextch == '=') { lex_tokench(lex, nextch); + if ((thirdch = lex_getch(lex)) == '>') + lex_tokench(lex, thirdch); + else + lex_ungetch(lex, thirdch); + } else if (ch == '-' && nextch == '>') { lex_tokench(lex, nextch); - } else + } else if (ch == '&' && nextch == '~') { + thirdch = lex_getch(lex); + if (thirdch != '=') { + lex_ungetch(lex, thirdch); + lex_ungetch(lex, nextch); + } + else { + lex_tokench(lex, nextch); + lex_tokench(lex, thirdch); + } + } + else if (lex->flags.preprocessing && + ch == '-' && util_isdigit(nextch)) + { + lex->tok.ttype = lex_finish_digit(lex, nextch); + if (lex->tok.ttype == TOKEN_INTCONST) + lex->tok.constval.i = -lex->tok.constval.i; + else + lex->tok.constval.f = -lex->tok.constval.f; + lex_endtoken(lex); + return lex->tok.ttype; + } else { lex_ungetch(lex, nextch); + } lex_endtoken(lex); return (lex->tok.ttype = TOKEN_OPERATOR); @@ -1107,7 +1368,7 @@ int lex_do(lex_file *lex) lex_tokench(lex, ch); nextch = lex_getch(lex); - if (nextch == '=') { + if (nextch == '=' || nextch == '*') { lex_tokench(lex, nextch); } else lex_ungetch(lex, nextch); @@ -1116,6 +1377,12 @@ int lex_do(lex_file *lex) return (lex->tok.ttype = TOKEN_OPERATOR); } + if (ch == '%') { + lex_tokench(lex, ch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_OPERATOR); + } + if (isident_start(ch)) { const char *v; @@ -1147,29 +1414,17 @@ int lex_do(lex_file *lex) } else if (!strcmp(v, "vector")) { lex->tok.ttype = TOKEN_TYPENAME; lex->tok.constval.t = TYPE_VECTOR; - } else if (!strcmp(v, "for") || - !strcmp(v, "while") || - !strcmp(v, "do") || - !strcmp(v, "if") || - !strcmp(v, "else") || - !strcmp(v, "local") || - !strcmp(v, "return") || - !strcmp(v, "not") || - !strcmp(v, "const")) - { - lex->tok.ttype = TOKEN_KEYWORD; - } - else if (opts_standard != COMPILER_QCC) - { - /* other standards reserve these keywords */ - if (!strcmp(v, "switch") || - !strcmp(v, "struct") || - !strcmp(v, "union") || - !strcmp(v, "break") || - !strcmp(v, "continue") || - !strcmp(v, "var")) - { - lex->tok.ttype = TOKEN_KEYWORD; + } else { + size_t kw; + for (kw = 0; kw < num_keywords_qc; ++kw) { + if (!strcmp(v, keywords_qc[kw])) + return (lex->tok.ttype = TOKEN_KEYWORD); + } + if (OPTS_OPTION_U32(OPTION_STANDARD) != COMPILER_QCC) { + for (kw = 0; kw < num_keywords_fg; ++kw) { + if (!strcmp(v, keywords_fg[kw])) + return (lex->tok.ttype = TOKEN_KEYWORD); + } } } @@ -1187,7 +1442,7 @@ int lex_do(lex_file *lex) while (!lex->flags.preprocessing && lex->tok.ttype == TOKEN_STRINGCONST) { /* Allow c style "string" "continuation" */ - ch = lex_skipwhite(lex); + ch = lex_skipwhite(lex, false); if (ch != '"') { lex_ungetch(lex, ch); break; @@ -1214,8 +1469,9 @@ int lex_do(lex_file *lex) lex_tokench(lex, ch); lex_endtoken(lex); + lex->tok.ttype = TOKEN_CHARCONST; /* It's a vector if we can successfully scan 3 floats */ -#ifdef WIN32 +#ifdef _MSC_VER if (sscanf_s(lex->tok.value, " %f %f %f ", &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3) #else @@ -1226,17 +1482,41 @@ int lex_do(lex_file *lex) { lex->tok.ttype = TOKEN_VECTORCONST; } + else + { + if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) { + uchar_t u8char; + /* check for a valid utf8 character */ + if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) { + if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, + ( OPTS_FLAG(UTF8) ? "invalid multibyte character sequence `%s`" + : "multibyte character: `%s`" ), + lex->tok.value)) + return (lex->tok.ttype = TOKEN_ERROR); + } + else + lex->tok.constval.i = u8char; + } + else + lex->tok.constval.i = lex->tok.value[0]; + } return lex->tok.ttype; } - if (isdigit(ch)) + if (util_isdigit(ch)) { lex->tok.ttype = lex_finish_digit(lex, ch); lex_endtoken(lex); return lex->tok.ttype; } - lexerror(lex, "unknown token"); + if (lex->flags.preprocessing) { + lex_tokench(lex, ch); + lex_endtoken(lex); + return (lex->tok.ttype = ch); + } + + lexerror(lex, "unknown token: `%s`", lex->tok.value); return (lex->tok.ttype = TOKEN_ERROR); }