X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=blobdiff_plain;f=lexer.c;h=42b21a1a6cd7de46bd45aa21b227cc412a4fbdec;hp=a349e0efacf3fc79a9587a00de2d359cd9319545;hb=e11a17b40805b93e1e2ae80129c9d235b6d6119b;hpb=fefb51c5925ca506fff374a59c7e4b396ad628c2 diff --git a/lexer.c b/lexer.c index a349e0e..42b21a1 100644 --- a/lexer.c +++ b/lexer.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 + * Copyright (C) 2012, 2013 * Wolfgang Bumiller * * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -74,20 +74,17 @@ void lexerror(lex_file *lex, const char *fmt, ...) bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...) { + bool r; + lex_ctx ctx; va_list ap; - int lvl = LVL_WARNING; - if (!OPTS_WARN(warntype)) - return false; - - if (opts_werror) - lvl = LVL_ERROR; + ctx.file = lex->name; + ctx.line = lex->sline; va_start(ap, fmt); - con_vprintmsg(lvl, lex->name, lex->sline, "warning", fmt, ap); + r = vcompile_warning(ctx, warntype, fmt, ap); va_end(ap); - - return opts_werror; + return r; } @@ -187,7 +184,7 @@ static void lex_token_new(lex_file *lex) lex_file* lex_open(const char *file) { lex_file *lex; - FILE *in = util_fopen(file, "rb"); + FILE *in = file_open(file, "rb"); if (!in) { lexerror(NULL, "open failed: '%s'\n", file); @@ -196,7 +193,7 @@ lex_file* lex_open(const char *file) lex = (lex_file*)mem_a(sizeof(*lex)); if (!lex) { - fclose(in); + file_close(in); lexerror(NULL, "out of memory\n"); return NULL; } @@ -261,7 +258,7 @@ void lex_close(lex_file *lex) vec_free(lex->modelname); if (lex->file) - fclose(lex->file); + file_close(lex->file); #if 0 if (lex->tok) token_delete(lex->tok); @@ -294,12 +291,16 @@ static int lex_try_trigraph(lex_file *lex, int old) { int c2, c3; c2 = lex_fgetc(lex); + if (!lex->push_line && c2 == '\n') + lex->line++; if (c2 != '?') { lex_ungetch(lex, c2); return old; } c3 = lex_fgetc(lex); + if (!lex->push_line && c3 == '\n') + lex->line++; switch (c3) { case '=': return '#'; case '/': return '\\'; @@ -441,7 +442,7 @@ static bool lex_try_pragma(lex_file *lex) goto unroll; } - for (ch = lex_getch(lex); vec_size(param) < 32 && ch != ')' && ch != '\n'; ch = lex_getch(lex)) + for (ch = lex_getch(lex); vec_size(param) < 1024 && ch != ')' && ch != '\n'; ch = lex_getch(lex)) vec_push(param, ch); vec_push(param, 0); @@ -488,23 +489,25 @@ unroll: if (command) { vec_pop(command); while (vec_size(command)) { - lex_ungetch(lex, vec_last(command)); + lex_ungetch(lex, (unsigned char)vec_last(command)); vec_pop(command); } vec_free(command); + lex_ungetch(lex, ' '); } if (command) { vec_pop(command); while (vec_size(command)) { - lex_ungetch(lex, vec_last(command)); + lex_ungetch(lex, (unsigned char)vec_last(command)); vec_pop(command); } vec_free(command); + lex_ungetch(lex, ' '); } if (pragma) { vec_pop(pragma); while (vec_size(pragma)) { - lex_ungetch(lex, vec_last(pragma)); + lex_ungetch(lex, (unsigned char)vec_last(pragma)); vec_pop(pragma); } vec_free(pragma); @@ -547,10 +550,10 @@ printf( "line one\n" * here is to store the line of the first character after skipping * the initial whitespace in lex->sline, this happens in lex_do. */ -static int lex_skipwhite(lex_file *lex) +static int lex_skipwhite(lex_file *lex, bool hadwhite) { int ch = 0; - bool haswhite = false; + bool haswhite = hadwhite; do { @@ -737,12 +740,18 @@ static bool lex_finish_frames(lex_file *lex) vec_shrinkto(lex->tok.value, 0); vec_push(lex->frames, m); } while (true); + + return false; } static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) { + uchar_t chr; int ch = 0; int nextch; + bool hex; + char u8buf[8]; /* way more than enough */ + int u8len, uc; while (ch != EOF) { @@ -822,6 +831,55 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote) case '>': ch = 31; break; case '[': ch = 16; break; case ']': ch = 17; break; + case '{': + chr = 0; + nextch = lex_getch(lex); + hex = (nextch == 'x'); + if (!hex) + lex_ungetch(lex, nextch); + for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) { + if (!hex) { + if (nextch >= '0' && nextch <= '9') + chr = chr * 10 + nextch - '0'; + else { + lexerror(lex, "bad character code"); + return (lex->tok.ttype = TOKEN_ERROR); + } + } else { + if (nextch >= '0' && nextch <= '9') + chr = chr * 0x10 + nextch - '0'; + else if (nextch >= 'a' && nextch <= 'f') + chr = chr * 0x10 + nextch - 'a' + 10; + else if (nextch >= 'A' && nextch <= 'F') + chr = chr * 0x10 + nextch - 'A' + 10; + else { + lexerror(lex, "bad character code"); + return (lex->tok.ttype = TOKEN_ERROR); + } + } + if (chr > 0x10FFFF || (!OPTS_FLAG(UTF8) && chr > 255)) + { + lexerror(lex, "character code out of range"); + return (lex->tok.ttype = TOKEN_ERROR); + } + } + if (OPTS_FLAG(UTF8) && chr >= 128) { + u8len = u8_fromchar(chr, u8buf, sizeof(u8buf)); + if (!u8len) + ch = 0; + else { + --u8len; + for (uc = 0; uc < u8len; ++uc) + lex_tokench(lex, u8buf[uc]); + /* the last character will be inserted with the tokench() call + * below the switch + */ + ch = u8buf[uc]; + } + } + else + ch = chr; + break; case '\n': ch = '\n'; break; default: @@ -920,6 +978,7 @@ static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch) int lex_do(lex_file *lex) { int ch, nextch, thirdch; + bool hadwhite = false; lex_token_new(lex); #if 0 @@ -928,7 +987,8 @@ int lex_do(lex_file *lex) #endif while (true) { - ch = lex_skipwhite(lex); + ch = lex_skipwhite(lex, hadwhite); + hadwhite = true; if (!lex->flags.mergelines || ch != '\\') break; ch = lex_getch(lex); @@ -959,7 +1019,7 @@ int lex_do(lex_file *lex) } /* modelgen / spiritgen commands */ - if (ch == '$') { + if (ch == '$' && !lex->flags.preprocessing) { const char *v; size_t frame; @@ -1050,7 +1110,6 @@ int lex_do(lex_file *lex) if (rc < 0) return (lex->tok.ttype = TOKEN_FATAL); - v = lex->tok.value; if (lex->modelname) { frame_macro m; m.value = lex->framevalue; @@ -1105,6 +1164,15 @@ int lex_do(lex_file *lex) switch (ch) { case '[': + nextch = lex_getch(lex); + if (nextch == '[') { + lex_tokench(lex, ch); + lex_tokench(lex, nextch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_ATTRIBUTE_OPEN); + } + lex_ungetch(lex, nextch); + /* FALL THROUGH */ case '(': case ':': case '?': @@ -1114,11 +1182,23 @@ int lex_do(lex_file *lex) return (lex->tok.ttype = ch); else return (lex->tok.ttype = TOKEN_OPERATOR); + + case ']': + if (lex->flags.noops) { + nextch = lex_getch(lex); + if (nextch == ']') { + lex_tokench(lex, ch); + lex_tokench(lex, nextch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_ATTRIBUTE_CLOSE); + } + lex_ungetch(lex, nextch); + } + /* FALL THROUGH */ case ')': case ';': case '{': case '}': - case ']': case '#': lex_tokench(lex, ch); @@ -1168,31 +1248,37 @@ int lex_do(lex_file *lex) default: break; } + } - if (ch == '.') - { - lex_tokench(lex, ch); - /* peak ahead once */ - nextch = lex_getch(lex); - if (nextch != '.') { - lex_ungetch(lex, nextch); - lex_endtoken(lex); - return (lex->tok.ttype = ch); - } - /* peak ahead again */ - nextch = lex_getch(lex); - if (nextch != '.') { - lex_ungetch(lex, nextch); - lex_ungetch(lex, '.'); - lex_endtoken(lex); + if (ch == '.') + { + lex_tokench(lex, ch); + /* peak ahead once */ + nextch = lex_getch(lex); + if (nextch != '.') { + lex_ungetch(lex, nextch); + lex_endtoken(lex); + if (lex->flags.noops) return (lex->tok.ttype = ch); - } - /* fill the token to be "..." */ - lex_tokench(lex, ch); - lex_tokench(lex, ch); + else + return (lex->tok.ttype = TOKEN_OPERATOR); + } + /* peak ahead again */ + nextch = lex_getch(lex); + if (nextch != '.') { + lex_ungetch(lex, nextch); + lex_ungetch(lex, '.'); lex_endtoken(lex); - return (lex->tok.ttype = TOKEN_DOTS); + if (lex->flags.noops) + return (lex->tok.ttype = ch); + else + return (lex->tok.ttype = TOKEN_OPERATOR); } + /* fill the token to be "..." */ + lex_tokench(lex, ch); + lex_tokench(lex, ch); + lex_endtoken(lex); + return (lex->tok.ttype = TOKEN_DOTS); } if (ch == ',' || ch == '.') { @@ -1202,10 +1288,11 @@ int lex_do(lex_file *lex) } if (ch == '+' || ch == '-' || /* ++, --, +=, -= and -> as well! */ - ch == '>' || ch == '<' || /* <<, >>, <=, >= */ - ch == '=' || ch == '!' || /* ==, != */ - ch == '&' || ch == '|') /* &&, ||, &=, |= */ - { + ch == '>' || ch == '<' || /* <<, >>, <=, >= */ + ch == '=' || ch == '!' || /* ==, != */ + ch == '&' || ch == '|' || /* &&, ||, &=, |= */ + ch == '~' /* ~=, ~ */ + ) { lex_tokench(lex, ch); nextch = lex_getch(lex); @@ -1223,6 +1310,17 @@ int lex_do(lex_file *lex) lex_tokench(lex, nextch); lex_tokench(lex, thirdch); } + } + else if (lex->flags.preprocessing && + ch == '-' && isdigit(nextch)) + { + lex->tok.ttype = lex_finish_digit(lex, nextch); + if (lex->tok.ttype == TOKEN_INTCONST) + lex->tok.constval.i = -lex->tok.constval.i; + else + lex->tok.constval.f = -lex->tok.constval.f; + lex_endtoken(lex); + return lex->tok.ttype; } else lex_ungetch(lex, nextch); @@ -1290,7 +1388,7 @@ int lex_do(lex_file *lex) if (!strcmp(v, keywords_qc[kw])) return (lex->tok.ttype = TOKEN_KEYWORD); } - if (opts_standard != COMPILER_QCC) { + if (OPTS_OPTION_U32(OPTION_STANDARD) != COMPILER_QCC) { for (kw = 0; kw < num_keywords_fg; ++kw) { if (!strcmp(v, keywords_fg[kw])) return (lex->tok.ttype = TOKEN_KEYWORD); @@ -1312,7 +1410,7 @@ int lex_do(lex_file *lex) while (!lex->flags.preprocessing && lex->tok.ttype == TOKEN_STRINGCONST) { /* Allow c style "string" "continuation" */ - ch = lex_skipwhite(lex); + ch = lex_skipwhite(lex, false); if (ch != '"') { lex_ungetch(lex, ch); break; @@ -1341,7 +1439,7 @@ int lex_do(lex_file *lex) lex->tok.ttype = TOKEN_CHARCONST; /* It's a vector if we can successfully scan 3 floats */ -#ifdef WIN32 +#ifdef _MSC_VER if (sscanf_s(lex->tok.value, " %f %f %f ", &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3) #else @@ -1355,10 +1453,20 @@ int lex_do(lex_file *lex) else { if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) { - if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, "multibyte character: `%s`", lex->tok.value)) - return (lex->tok.ttype = TOKEN_ERROR); + uchar_t u8char; + /* check for a valid utf8 character */ + if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) { + if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER, + ( OPTS_FLAG(UTF8) ? "invalid multibyte character sequence `%s`" + : "multibyte character: `%s`" ), + lex->tok.value)) + return (lex->tok.ttype = TOKEN_ERROR); + } + else + lex->tok.constval.i = u8char; } - lex->tok.constval.i = lex->tok.value[0]; + else + lex->tok.constval.i = lex->tok.value[0]; } return lex->tok.ttype; @@ -1371,6 +1479,12 @@ int lex_do(lex_file *lex) return lex->tok.ttype; } - lexerror(lex, "unknown token"); + if (lex->flags.preprocessing) { + lex_tokench(lex, ch); + lex_endtoken(lex); + return (lex->tok.ttype = ch); + } + + lexerror(lex, "unknown token: `%s`", lex->tok.value); return (lex->tok.ttype = TOKEN_ERROR); }