X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=blobdiff_plain;f=lexer.c;h=3dc6981b0cf09df7682d8bd2f1040f69eda9d2e7;hp=feea8a2eb49cf5ce14873df7067d25cb4526e274;hb=dc510baf4f2c8aeec289abf4abd81596c9fe7a8c;hpb=2a7933973507dfe5d4625ca9ec7eff2aea91cf26 diff --git a/lexer.c b/lexer.c index feea8a2..3dc6981 100644 --- a/lexer.c +++ b/lexer.c @@ -1,30 +1,9 @@ -/* - * Copyright (C) 2012, 2013 - * Wolfgang Bumiller - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is furnished to do - * so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ #include #include #include "gmqcc.h" #include "lexer.h" + /* * List of Keywords */ @@ -81,91 +60,8 @@ static bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...) return r; } - -#if 0 -token* token_new() -{ - token *tok = (token*)mem_a(sizeof(token)); - if (!tok) - return NULL; - memset(tok, 0, sizeof(*tok)); - return tok; -} - -void token_delete(token *self) -{ - if (self->next && self->next->prev == self) - self->next->prev = self->prev; - if (self->prev && self->prev->next == self) - self->prev->next = self->next; - MEM_VECTOR_CLEAR(self, value); - mem_d(self); -} - -token* token_copy(const token *cp) -{ - token* self = token_new(); - if (!self) - return NULL; - /* copy the value */ - self->value_alloc = cp->value_count + 1; - self->value_count = cp->value_count; - self->value = (char*)mem_a(self->value_alloc); - if (!self->value) { - mem_d(self); - return NULL; - } - memcpy(self->value, cp->value, cp->value_count); - self->value[self->value_alloc-1] = 0; - - /* rest */ - self->ctx = cp->ctx; - self->ttype = cp->ttype; - memcpy(&self->constval, &cp->constval, sizeof(self->constval)); - return self; -} - -void token_delete_all(token *t) -{ - token *n; - - do { - n = t->next; - token_delete(t); - t = n; - } while(t); -} - -token* token_copy_all(const token *cp) -{ - token *cur; - token *out; - - out = cur = token_copy(cp); - if (!out) - return NULL; - - while (cp->next) { - cp = cp->next; - cur->next = token_copy(cp); - if (!cur->next) { - token_delete_all(out); - return NULL; - } - cur->next->prev = cur; - cur = cur->next; - } - - return out; -} -#else static void lex_token_new(lex_file *lex) { -#if 0 - if (lex->tok) - token_delete(lex->tok); - lex->tok = token_new(); -#else if (lex->tok.value) vec_shrinkto(lex->tok.value, 0); @@ -173,14 +69,16 @@ static void lex_token_new(lex_file *lex) lex->tok.ctx.line = lex->sline; lex->tok.ctx.file = lex->name; lex->tok.ctx.column = lex->column; -#endif } -#endif + +static void lex_ungetch(lex_file *lex, int ch); +static int lex_getch(lex_file *lex); lex_file* lex_open(const char *file) { - lex_file *lex; - FILE *in = fs_file_open(file, "rb"); + lex_file *lex; + FILE *in = fopen(file, "rb"); + uint32_t read; if (!in) { lexerror(NULL, "open failed: '%s'\n", file); @@ -189,7 +87,7 @@ lex_file* lex_open(const char *file) lex = (lex_file*)mem_a(sizeof(*lex)); if (!lex) { - fs_file_close(in); + fclose(in); lexerror(NULL, "out of memory\n"); return NULL; } @@ -203,6 +101,19 @@ lex_file* lex_open(const char *file) lex->peekpos = 0; lex->eof = false; + /* handle BOM */ + if ((read = (lex_getch(lex) << 16) | (lex_getch(lex) << 8) | lex_getch(lex)) != 0xEFBBBF) { + lex_ungetch(lex, (read & 0x0000FF)); + lex_ungetch(lex, (read & 0x00FF00) >> 8); + lex_ungetch(lex, (read & 0xFF0000) >> 16); + } else { + /* + * otherwise the lexer has advanced 3 bytes for the BOM, we need + * to set the column back to 0 + */ + lex->column = 0; + } + vec_push(lex_filenames, lex->name); return lex; } @@ -254,22 +165,21 @@ void lex_close(lex_file *lex) vec_free(lex->modelname); if (lex->file) - fs_file_close(lex->file); -#if 0 - if (lex->tok) - token_delete(lex->tok); -#else + fclose(lex->file); + vec_free(lex->tok.value); -#endif + /* mem_d(lex->name); collected in lex_filenames */ mem_d(lex); } + + static int lex_fgetc(lex_file *lex) { if (lex->file) { lex->column++; - return fs_file_getc(lex->file); + return fgetc(lex->file); } if (lex->open_string) { if (lex->open_string_pos >= lex->open_string_length) @@ -285,7 +195,6 @@ static int lex_fgetc(lex_file *lex) * are working on. * The are merely wrapping get/put in order to count line numbers. */ -static void lex_ungetch(lex_file *lex, int ch); static int lex_try_trigraph(lex_file *lex, int old) { int c2, c3; @@ -352,14 +261,18 @@ static int lex_getch(lex_file *lex) if (lex->peekpos) { lex->peekpos--; - if (!lex->push_line && lex->peek[lex->peekpos] == '\n') + if (!lex->push_line && lex->peek[lex->peekpos] == '\n') { lex->line++; + lex->column = 0; + } return lex->peek[lex->peekpos]; } ch = lex_fgetc(lex); - if (!lex->push_line && ch == '\n') + if (!lex->push_line && ch == '\n') { lex->line++; + lex->column = 0; + } else if (ch == '?') return lex_try_trigraph(lex, ch); else if (!lex->flags.nodigraphs && (ch == '<' || ch == ':' || ch == '%')) @@ -601,10 +514,6 @@ static int lex_skipwhite(lex_file *lex, bool hadwhite) if (lex->flags.preprocessing) { haswhite = true; - /* - lex_tokench(lex, '/'); - lex_tokench(lex, '/'); - */ lex_tokench(lex, ' '); lex_tokench(lex, ' '); } @@ -626,10 +535,6 @@ static int lex_skipwhite(lex_file *lex, bool hadwhite) /* multiline comment */ if (lex->flags.preprocessing) { haswhite = true; - /* - lex_tokench(lex, '/'); - lex_tokench(lex, '*'); - */ lex_tokench(lex, ' '); lex_tokench(lex, ' '); } @@ -641,10 +546,6 @@ static int lex_skipwhite(lex_file *lex, bool hadwhite) ch = lex_getch(lex); if (ch == '/') { if (lex->flags.preprocessing) { - /* - lex_tokench(lex, '*'); - lex_tokench(lex, '/'); - */ lex_tokench(lex, ' '); lex_tokench(lex, ' '); } @@ -656,7 +557,7 @@ static int lex_skipwhite(lex_file *lex, bool hadwhite) if (ch == '\n') lex_tokench(lex, '\n'); else - lex_tokench(lex, ' '); /* ch); */ + lex_tokench(lex, ' '); } } ch = ' '; /* cause TRUE in the isspace check */ @@ -757,10 +658,11 @@ static bool lex_finish_frames(lex_file *lex) static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) { - uchar_t chr; - int ch = 0; + utf8ch_t chr = 0; + int ch = 0, texttype = 0; int nextch; bool hex; + bool oct; char u8buf[8]; /* way more than enough */ int u8len, uc; @@ -792,13 +694,12 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) case '\\': break; case '\'': break; case '"': break; - case 'a': ch = '\a'; break; - case 'b': ch = '\b'; break; - case 'r': ch = '\r'; break; - case 'n': ch = '\n'; break; - case 't': ch = '\t'; break; - case 'f': ch = '\f'; break; - case 'v': ch = '\v'; break; + case 'a': ch = '\a'; break; + case 'r': ch = '\r'; break; + case 'n': ch = '\n'; break; + case 't': ch = '\t'; break; + case 'f': ch = '\f'; break; + case 'v': ch = '\v'; break; case 'x': case 'X': /* same procedure as in fteqcc */ @@ -846,17 +747,18 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) chr = 0; nextch = lex_getch(lex); hex = (nextch == 'x'); - if (!hex) + oct = (nextch == '0'); + if (!hex && !oct) lex_ungetch(lex, nextch); for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) { - if (!hex) { + if (!hex && !oct) { if (nextch >= '0' && nextch <= '9') chr = chr * 10 + nextch - '0'; else { lexerror(lex, "bad character code"); return (lex->tok.ttype = TOKEN_ERROR); } - } else { + } else if (!oct) { if (nextch >= '0' && nextch <= '9') chr = chr * 0x10 + nextch - '0'; else if (nextch >= 'a' && nextch <= 'f') @@ -867,6 +769,13 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) lexerror(lex, "bad character code"); return (lex->tok.ttype = TOKEN_ERROR); } + } else { + if (nextch >= '0' && nextch <= '9') + chr = chr * 8 + chr - '0'; + else { + lexerror(lex, "bad character code"); + return (lex->tok.ttype = TOKEN_ERROR); + } } if (chr > 0x10FFFF || (!OPTS_FLAG(UTF8) && chr > 255)) { @@ -875,7 +784,7 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) } } if (OPTS_FLAG(UTF8) && chr >= 128) { - u8len = u8_fromchar(chr, u8buf, sizeof(u8buf)); + u8len = utf8_from(u8buf, chr); if (!u8len) ch = 0; else { @@ -883,7 +792,8 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) lex->column += u8len; for (uc = 0; uc < u8len; ++uc) lex_tokench(lex, u8buf[uc]); - /* the last character will be inserted with the tokench() call + /* + * the last character will be inserted with the tokench() call * below the switch */ ch = u8buf[uc]; @@ -892,7 +802,15 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) else ch = chr; break; - case '\n': ch = '\n'; break; + + /* high bit text */ + case 'b': case 's': + texttype ^= 128; + continue; + + case '\n': + ch = '\n'; + break; default: lexwarn(lex, WARN_UNKNOWN_CONTROL_SEQUENCE, "unrecognized control sequence: \\%c", ch); @@ -900,7 +818,7 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) lex_tokench(lex, '\\'); } /* add the character finally */ - lex_tokench(lex, ch); + lex_tokench(lex, ch | texttype); } else lex_tokench(lex, ch); @@ -993,10 +911,6 @@ int lex_do(lex_file *lex) bool hadwhite = false; lex_token_new(lex); -#if 0 - if (!lex->tok) - return TOKEN_FATAL; -#endif while (true) { ch = lex_skipwhite(lex, hadwhite); @@ -1241,10 +1155,6 @@ int lex_do(lex_file *lex) */ switch (ch) { - /* - case '+': - case '-': - */ case '*': case '/': case '<': @@ -1302,16 +1212,22 @@ int lex_do(lex_file *lex) } if (ch == '+' || ch == '-' || /* ++, --, +=, -= and -> as well! */ - ch == '>' || ch == '<' || /* <<, >>, <=, >= */ + ch == '>' || ch == '<' || /* <<, >>, <=, >= and >< as well! */ ch == '=' || ch == '!' || /* <=>, ==, != */ ch == '&' || ch == '|' || /* &&, ||, &=, |= */ ch == '~' || ch == '^' /* ~=, ~, ^ */ ) { lex_tokench(lex, ch); - nextch = lex_getch(lex); - if ((nextch == '=' && ch != '<') || (nextch == ch && ch != '!')) { + + if ((nextch == '=' && ch != '<') || (nextch == '<' && ch == '>')) + lex_tokench(lex, nextch); + else if (nextch == ch && ch != '!') { lex_tokench(lex, nextch); + if ((thirdch = lex_getch(lex)) == '=') + lex_tokench(lex, thirdch); + else + lex_ungetch(lex, thirdch); } else if (ch == '<' && nextch == '=') { lex_tokench(lex, nextch); if ((thirdch = lex_getch(lex)) == '>') @@ -1350,15 +1266,6 @@ int lex_do(lex_file *lex) return (lex->tok.ttype = TOKEN_OPERATOR); } - /* - if (ch == '^' || ch == '~' || ch == '!') - { - lex_tokench(lex, ch); - lex_endtoken(lex); - return (lex->tok.ttype = TOKEN_OPERATOR); - } - */ - if (ch == '*' || ch == '/') /* *=, /= */ { lex_tokench(lex, ch); @@ -1410,6 +1317,8 @@ int lex_do(lex_file *lex) } else if (!strcmp(v, "vector")) { lex->tok.ttype = TOKEN_TYPENAME; lex->tok.constval.t = TYPE_VECTOR; + } else if (!strcmp(v, "_length")) { + lex->tok.ttype = TOKEN_OPERATOR; } else { size_t kw; for (kw = 0; kw < GMQCC_ARRAY_COUNT(keywords_qc); ++kw) { @@ -1466,14 +1375,10 @@ int lex_do(lex_file *lex) lex_endtoken(lex); lex->tok.ttype = TOKEN_CHARCONST; - /* It's a vector if we can successfully scan 3 floats */ -#ifdef _MSC_VER - if (sscanf_s(lex->tok.value, " %f %f %f ", - &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3) -#else - if (sscanf(lex->tok.value, " %f %f %f ", + + /* It's a vector if we can successfully scan 3 floats */ + if (util_sscanf(lex->tok.value, " %f %f %f ", &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3) -#endif { lex->tok.ttype = TOKEN_VECTORCONST; @@ -1481,9 +1386,9 @@ int lex_do(lex_file *lex) else { if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) { - uchar_t u8char; + utf8ch_t u8char; /* check for a valid utf8 character */ - if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) { + if (!OPTS_FLAG(UTF8) || !utf8_to(&u8char, (const unsigned char *)lex->tok.value, 8)) { if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, ( OPTS_FLAG(UTF8) ? "invalid multibyte character sequence `%s`" : "multibyte character: `%s`" ), @@ -1513,6 +1418,6 @@ int lex_do(lex_file *lex) return (lex->tok.ttype = ch); } - lexerror(lex, "unknown token: `%s`", lex->tok.value); + lexerror(lex, "unknown token: `%c`", ch); return (lex->tok.ttype = TOKEN_ERROR); }