X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=blobdiff_plain;f=lexer.c;h=19d67207f5b0c04ccff0c8b0e2af9491e06a2717;hp=ebbe1880c01f330ecb24fdf0ec81827f00000c55;hb=160e7cf7eebd7fa173fb739aca00143097a3518b;hpb=aef49d966729acec5bd76fd72e42ff0950b7f3c8

diff --git a/lexer.c b/lexer.c
index ebbe188..db0a8f4 100644
--- a/lexer.c
+++ b/lexer.c
@@ -1,3 +1,25 @@
+/*
+ * Copyright (C) 2012, 2013
+ *     Wolfgang Bumiller
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -6,136 +28,257 @@
 #include "gmqcc.h"
 #include "lexer.h"
 
-MEM_VEC_FUNCTIONS(token, char, value)
+/*
+ * List of Keywords
+ */
+
+/* original */
+static const char *keywords_qc[] = {
+    "for", "do", "while",
+    "if", "else",
+    "local",
+    "return",
+    "const"
+};
+static size_t num_keywords_qc = sizeof(keywords_qc) / sizeof(keywords_qc[0]);
+
+/* For fte/gmgqcc */
+static const char *keywords_fg[] = {
+    "switch", "case", "default",
+    "struct", "union",
+    "break", "continue",
+    "typedef",
+    "goto",
+
+    "__builtin_debug_printtype"
+};
+static size_t num_keywords_fg = sizeof(keywords_fg) / sizeof(keywords_fg[0]);
+
+/*
+ * Lexer code
+ */
+
+char* *lex_filenames;
 
 void lexerror(lex_file *lex, const char *fmt, ...)
 {
-	va_list ap;
+    va_list ap;
+
+    va_start(ap, fmt);
+    if (lex)
+        con_vprintmsg(LVL_ERROR, lex->name, lex->sline, "parse error", fmt, ap);
+    else
+        con_vprintmsg(LVL_ERROR, "", 0, "parse error", fmt, ap);
+    va_end(ap);
+}
 
-	if (lex)
-		printf("error %s:%lu: ", lex->name, (unsigned long)lex->sline);
-	else
-		printf("error: ");
+bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...)
+{
+    bool    r;
+    lex_ctx ctx;
+    va_list ap;
 
-	va_start(ap, fmt);
-	vprintf(fmt, ap);
-	va_end(ap);
+    ctx.file = lex->name;
+    ctx.line = lex->sline;
 
-	printf("\n");
+    va_start(ap, fmt);
+    r = vcompile_warning(ctx, warntype, fmt, ap);
+    va_end(ap);
+    return r;
 }
 
+
+#if 0
 token* token_new()
 {
-	token *tok = (token*)mem_a(sizeof(token));
-	if (!tok)
-		return NULL;
-	memset(tok, 0, sizeof(*tok));
-	return tok;
+    token *tok = (token*)mem_a(sizeof(token));
+    if (!tok)
+        return NULL;
+    memset(tok, 0, sizeof(*tok));
+    return tok;
 }
 
 void token_delete(token *self)
 {
-	if (self->next && self->next->prev == self)
-		self->next->prev = self->prev;
-	if (self->prev && self->prev->next == self)
-		self->prev->next = self->next;
-	MEM_VECTOR_CLEAR(self, value);
-	mem_d(self);
+    if (self->next && self->next->prev == self)
+        self->next->prev = self->prev;
+    if (self->prev && self->prev->next == self)
+        self->prev->next = self->next;
+    MEM_VECTOR_CLEAR(self, value);
+    mem_d(self);
 }
 
 token* token_copy(const token *cp)
 {
-	token* self = token_new();
-	if (!self)
-		return NULL;
-	/* copy the value */
-	self->value_alloc = cp->value_count + 1;
-	self->value_count = cp->value_count;
-	self->value = (char*)mem_a(self->value_alloc);
-	if (!self->value) {
-		mem_d(self);
-		return NULL;
-	}
-	memcpy(self->value, cp->value, cp->value_count);
-	self->value[self->value_alloc-1] = 0;
-
-	/* rest */
-	self->ctx = cp->ctx;
-	self->ttype = cp->ttype;
-	memcpy(&self->constval, &cp->constval, sizeof(self->constval));
-	return self;
+    token* self = token_new();
+    if (!self)
+        return NULL;
+    /* copy the value */
+    self->value_alloc = cp->value_count + 1;
+    self->value_count = cp->value_count;
+    self->value = (char*)mem_a(self->value_alloc);
+    if (!self->value) {
+        mem_d(self);
+        return NULL;
+    }
+    memcpy(self->value, cp->value, cp->value_count);
+    self->value[self->value_alloc-1] = 0;
+
+    /* rest */
+    self->ctx = cp->ctx;
+    self->ttype = cp->ttype;
+    memcpy(&self->constval, &cp->constval, sizeof(self->constval));
+    return self;
 }
 
 void token_delete_all(token *t)
 {
-	token *n;
+    token *n;
 
-	do {
-		n = t->next;
-		token_delete(t);
-		t = n;
-	} while(t);
+    do {
+        n = t->next;
+        token_delete(t);
+        t = n;
+    } while(t);
 }
 
 token* token_copy_all(const token *cp)
 {
-	token *cur;
-	token *out;
-
-	out = cur = token_copy(cp);
-	if (!out)
-		return NULL;
-
-	while (cp->next) {
-		cp = cp->next;
-		cur->next = token_copy(cp);
-		if (!cur->next) {
-			token_delete_all(out);
-			return NULL;
-		}
-		cur->next->prev = cur;
-		cur = cur->next;
-	}
-
-	return out;
+    token *cur;
+    token *out;
+
+    out = cur = token_copy(cp);
+    if (!out)
+        return NULL;
+
+    while (cp->next) {
+        cp = cp->next;
+        cur->next = token_copy(cp);
+        if (!cur->next) {
+            token_delete_all(out);
+            return NULL;
+        }
+        cur->next->prev = cur;
+        cur = cur->next;
+    }
+
+    return out;
+}
+#else
+static void lex_token_new(lex_file *lex)
+{
+#if 0
+    if (lex->tok)
+        token_delete(lex->tok);
+    lex->tok = token_new();
+#else
+    if (lex->tok.value)
+        vec_shrinkto(lex->tok.value, 0);
+    lex->tok.constval.t  = 0;
+    lex->tok.ctx.line = lex->sline;
+    lex->tok.ctx.file = lex->name;
+#endif
 }
+#endif
 
 lex_file* lex_open(const char *file)
 {
-	lex_file *lex;
-	FILE *in = fopen(file, "rb");
+    lex_file *lex;
+    FILE *in = fs_file_open(file, "rb");
 
-	if (!in) {
-		lexerror(NULL, "open failed: '%s'\n", file);
-		return NULL;
-	}
+    if (!in) {
+        lexerror(NULL, "open failed: '%s'\n", file);
+        return NULL;
+    }
 
-	lex = (lex_file*)mem_a(sizeof(*lex));
-	if (!lex) {
-		fclose(in);
-		lexerror(NULL, "out of memory\n");
-		return NULL;
-	}
+    lex = (lex_file*)mem_a(sizeof(*lex));
+    if (!lex) {
+        fs_file_close(in);
+        lexerror(NULL, "out of memory\n");
+        return NULL;
+    }
 
-	memset(lex, 0, sizeof(*lex));
+    memset(lex, 0, sizeof(*lex));
 
-	lex->file = in;
-	lex->name = util_strdup(file);
-	lex->line = 1; /* we start counting at 1 */
+    lex->file = in;
+    lex->name = util_strdup(file);
+    lex->line = 1; /* we start counting at 1 */
 
-	lex->peekpos = 0;
+    lex->peekpos = 0;
+    lex->eof = false;
 
-	return lex;
+    vec_push(lex_filenames, lex->name);
+    return lex;
+}
+
+lex_file* lex_open_string(const char *str, size_t len, const char *name)
+{
+    lex_file *lex;
+
+    lex = (lex_file*)mem_a(sizeof(*lex));
+    if (!lex) {
+        lexerror(NULL, "out of memory\n");
+        return NULL;
+    }
+
+    memset(lex, 0, sizeof(*lex));
+
+    lex->file = NULL;
+    lex->open_string        = str;
+    lex->open_string_length = len;
+    lex->open_string_pos    = 0;
+
+    lex->name = util_strdup(name ? name : "<string-source>");
+    lex->line = 1; /* we start counting at 1 */
+
+    lex->peekpos = 0;
+    lex->eof = false;
+
+    vec_push(lex_filenames, lex->name);
+
+    return lex;
+}
+
+void lex_cleanup(void)
+{
+    size_t i;
+    for (i = 0; i < vec_size(lex_filenames); ++i)
+        mem_d(lex_filenames[i]);
+    vec_free(lex_filenames);
 }
 
 void lex_close(lex_file *lex)
 {
-	if (lex->file)
-		fclose(lex->file);
-	if (lex->tok)
-		token_delete(lex->tok);
-	mem_d(lex->name);
-	mem_d(lex);
+    size_t i;
+    for (i = 0; i < vec_size(lex->frames); ++i)
+        mem_d(lex->frames[i].name);
+    vec_free(lex->frames);
+
+    if (lex->modelname)
+        vec_free(lex->modelname);
+
+    if (lex->file)
+        fs_file_close(lex->file);
+#if 0
+    if (lex->tok)
+        token_delete(lex->tok);
+#else
+    vec_free(lex->tok.value);
+#endif
+    /* mem_d(lex->name); collected in lex_filenames */
+    mem_d(lex);
+}
+
+static int lex_fgetc(lex_file *lex)
+{
+    if (lex->file)
+        return fs_file_getc(lex->file);
+    if (lex->open_string) {
+        if (lex->open_string_pos >= lex->open_string_length)
+            return EOF;
+        return lex->open_string[lex->open_string_pos++];
+    }
+    return EOF;
 }
 
 /* Get or put-back data
@@ -143,28 +286,87 @@ void lex_close(lex_file *lex)
  * are working on.
  * The are merely wrapping get/put in order to count line numbers.
  */
+static void lex_ungetch(lex_file *lex, int ch);
+static int lex_try_trigraph(lex_file *lex, int old)
+{
+    int c2, c3;
+    c2 = lex_fgetc(lex);
+    if (!lex->push_line && c2 == '\n')
+        lex->line++;
+    if (c2 != '?') {
+        lex_ungetch(lex, c2);
+        return old;
+    }
+
+    c3 = lex_fgetc(lex);
+    if (!lex->push_line && c3 == '\n')
+        lex->line++;
+    switch (c3) {
+        case '=': return '#';
+        case '/': return '\\';
+        case '\'': return '^';
+        case '(': return '[';
+        case ')': return ']';
+        case '!': return '|';
+        case '<': return '{';
+        case '>': return '}';
+        case '-': return '~';
+        default:
+            lex_ungetch(lex, c3);
+            lex_ungetch(lex, c2);
+            return old;
+    }
+}
+
+static int lex_try_digraph(lex_file *lex, int ch)
+{
+    int c2;
+    c2 = lex_fgetc(lex);
+    /* we just used fgetc() so count lines
+     * need to offset a \n the ungetch would recognize
+     */
+    if (!lex->push_line && c2 == '\n')
+        lex->line++;
+    if      (ch == '<' && c2 == ':')
+        return '[';
+    else if (ch == ':' && c2 == '>')
+        return ']';
+    else if (ch == '<' && c2 == '%')
+        return '{';
+    else if (ch == '%' && c2 == '>')
+        return '}';
+    else if (ch == '%' && c2 == ':')
+        return '#';
+    lex_ungetch(lex, c2);
+    return ch;
+}
+
 static int lex_getch(lex_file *lex)
 {
-	int ch;
-
-	if (lex->peekpos) {
-		lex->peekpos--;
-		if (lex->peek[lex->peekpos] == '\n')
-			lex->line++;
-		return lex->peek[lex->peekpos];
-	}
-
-	ch = fgetc(lex->file);
-	if (ch == '\n')
-		lex->line++;
-	return ch;
+    int ch;
+
+    if (lex->peekpos) {
+        lex->peekpos--;
+        if (!lex->push_line && lex->peek[lex->peekpos] == '\n')
+            lex->line++;
+        return lex->peek[lex->peekpos];
+    }
+
+    ch = lex_fgetc(lex);
+    if (!lex->push_line && ch == '\n')
+        lex->line++;
+    else if (ch == '?')
+        return lex_try_trigraph(lex, ch);
+    else if (!lex->flags.nodigraphs && (ch == '<' || ch == ':' || ch == '%'))
+        return lex_try_digraph(lex, ch);
+    return ch;
 }
 
 static void lex_ungetch(lex_file *lex, int ch)
 {
-	lex->peek[lex->peekpos++] = ch;
-	if (ch == '\n')
-		lex->line--;
+    lex->peek[lex->peekpos++] = ch;
+    if (!lex->push_line && ch == '\n')
+        lex->line--;
 }
 
 /* classify characters
@@ -174,12 +376,12 @@ static void lex_ungetch(lex_file *lex, int ch)
 /* Idents are alphanumberic, but they start with alpha or _ */
 static bool isident_start(int ch)
 {
-	return isalpha(ch) || ch == '_';
+    return isalpha(ch) || ch == '_';
 }
 
 static bool isident(int ch)
 {
-	return isident_start(ch) || isdigit(ch);
+    return isident_start(ch) || isdigit(ch);
 }
 
 /* isxdigit_only is used when we already know it's not a digit
@@ -187,7 +389,136 @@ static bool isident(int ch)
  */
 static bool isxdigit_only(int ch)
 {
-	return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
+    return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
+}
+
+/* Append a character to the token buffer */
+static void lex_tokench(lex_file *lex, int ch)
+{
+    vec_push(lex->tok.value, ch);
+}
+
+/* Append a trailing null-byte */
+static void lex_endtoken(lex_file *lex)
+{
+    vec_push(lex->tok.value, 0);
+    vec_shrinkby(lex->tok.value, 1);
+}
+
+static bool lex_try_pragma(lex_file *lex)
+{
+    int ch;
+    char *pragma  = NULL;
+    char *command = NULL;
+    char *param   = NULL;
+    size_t line;
+
+    if (lex->flags.preprocessing)
+        return false;
+
+    line = lex->line;
+
+    ch = lex_getch(lex);
+    if (ch != '#') {
+        lex_ungetch(lex, ch);
+        return false;
+    }
+
+    for (ch = lex_getch(lex); vec_size(pragma) < 8 && ch >= 'a' && ch <= 'z'; ch = lex_getch(lex))
+        vec_push(pragma, ch);
+    vec_push(pragma, 0);
+
+    if (ch != ' ' || strcmp(pragma, "pragma")) {
+        lex_ungetch(lex, ch);
+        goto unroll;
+    }
+
+    for (ch = lex_getch(lex); vec_size(command) < 32 && ch >= 'a' && ch <= 'z'; ch = lex_getch(lex))
+        vec_push(command, ch);
+    vec_push(command, 0);
+
+    if (ch != '(') {
+        lex_ungetch(lex, ch);
+        goto unroll;
+    }
+
+    for (ch = lex_getch(lex); vec_size(param) < 1024 && ch != ')' && ch != '\n'; ch = lex_getch(lex))
+        vec_push(param, ch);
+    vec_push(param, 0);
+
+    if (ch != ')') {
+        lex_ungetch(lex, ch);
+        goto unroll;
+    }
+
+    if (!strcmp(command, "push")) {
+        if (!strcmp(param, "line")) {
+            lex->push_line++;
+            if (lex->push_line == 1)
+                --line;
+        }
+        else
+            goto unroll;
+    }
+    else if (!strcmp(command, "pop")) {
+        if (!strcmp(param, "line")) {
+            if (lex->push_line)
+                lex->push_line--;
+            if (lex->push_line == 0)
+                --line;
+        }
+        else
+            goto unroll;
+    }
+    else if (!strcmp(command, "file")) {
+        lex->name = util_strdup(param);
+        vec_push(lex_filenames, lex->name);
+    }
+    else if (!strcmp(command, "line")) {
+        line = strtol(param, NULL, 0)-1;
+    }
+    else
+        goto unroll;
+
+    lex->line = line;
+    while (ch != '\n' && ch != EOF)
+        ch = lex_getch(lex);
+    vec_free(command);
+    vec_free(param);
+    vec_free(pragma);
+    return true;
+
+unroll:
+    if (command) {
+        vec_pop(command);
+        while (vec_size(command)) {
+            lex_ungetch(lex, (unsigned char)vec_last(command));
+            vec_pop(command);
+        }
+        vec_free(command);
+        lex_ungetch(lex, ' ');
+    }
+    if (param) {
+        vec_pop(param);
+        while (vec_size(param)) {
+            lex_ungetch(lex, (unsigned char)vec_last(param));
+            vec_pop(param);
+        }
+        vec_free(param);
+        lex_ungetch(lex, ' ');
+    }
+    if (pragma) {
+        vec_pop(pragma);
+        while (vec_size(pragma)) {
+            lex_ungetch(lex, (unsigned char)vec_last(pragma));
+            vec_pop(pragma);
+        }
+        vec_free(pragma);
+    }
+    lex_ungetch(lex, '#');
+
+    lex->line = line;
+    return false;
 }
 
 /* Skip whitespace and comments and return the first
@@ -222,420 +553,957 @@ printf(   "line one\n"
  *    here is to store the line of the first character after skipping
  *    the initial whitespace in lex->sline, this happens in lex_do.
  */
-static int lex_skipwhite(lex_file *lex)
+static int lex_skipwhite(lex_file *lex, bool hadwhite)
 {
-	int ch = 0;
-
-	do
-	{
-		ch = lex_getch(lex);
-		while (ch != EOF && isspace(ch)) ch = lex_getch(lex);
-
-		if (ch == '/') {
-			ch = lex_getch(lex);
-			if (ch == '/')
-			{
-				/* one line comment */
-				ch = lex_getch(lex);
-
-				/* check for special: '/', '/', '*', '/' */
-				if (ch == '*') {
-					ch = lex_getch(lex);
-					if (ch == '/') {
-						ch = ' ';
-						continue;
-					}
-				}
-
-				while (ch != EOF && ch != '\n') {
-					ch = lex_getch(lex);
-				}
-				continue;
-			}
-			if (ch == '*')
-			{
-				/* multiline comment */
-				while (ch != EOF)
-				{
-					ch = lex_getch(lex);
-					if (ch == '*') {
-						ch = lex_getch(lex);
-						if (ch == '/') {
-							ch = lex_getch(lex);
-							break;
-						}
-					}
-				}
-				if (ch == '/') /* allow *//* direct following comment */
-				{
-					lex_ungetch(lex, ch);
-					ch = ' '; /* cause TRUE in the isspace check */
-				}
-				continue;
-			}
-			/* Otherwise roll back to the slash and break out of the loop */
-			lex_ungetch(lex, ch);
-			ch = '/';
-			break;
-		}
-	} while (ch != EOF && isspace(ch));
-
-	return ch;
+    int ch = 0;
+    bool haswhite = hadwhite;
+
+    do
+    {
+        ch = lex_getch(lex);
+        while (ch != EOF && isspace(ch)) {
+            if (ch == '\n') {
+                if (lex_try_pragma(lex))
+                    continue;
+            }
+            if (lex->flags.preprocessing) {
+                if (ch == '\n') {
+                    /* end-of-line */
+                    /* see if there was whitespace first */
+                    if (haswhite) { /* (vec_size(lex->tok.value)) { */
+                        lex_ungetch(lex, ch);
+                        lex_endtoken(lex);
+                        return TOKEN_WHITE;
+                    }
+                    /* otherwise return EOL */
+                    return TOKEN_EOL;
+                }
+                haswhite = true;
+                lex_tokench(lex, ch);
+            }
+            ch = lex_getch(lex);
+        }
+
+        if (ch == '/') {
+            ch = lex_getch(lex);
+            if (ch == '/')
+            {
+                /* one line comment */
+                ch = lex_getch(lex);
+
+                if (lex->flags.preprocessing) {
+                    haswhite = true;
+                    /*
+                    lex_tokench(lex, '/');
+                    lex_tokench(lex, '/');
+                    */
+                    lex_tokench(lex, ' ');
+                    lex_tokench(lex, ' ');
+                }
+
+                while (ch != EOF && ch != '\n') {
+                    if (lex->flags.preprocessing)
+                        lex_tokench(lex, ' '); /* ch); */
+                    ch = lex_getch(lex);
+                }
+                if (lex->flags.preprocessing) {
+                    lex_ungetch(lex, '\n');
+                    lex_endtoken(lex);
+                    return TOKEN_WHITE;
+                }
+                continue;
+            }
+            if (ch == '*')
+            {
+                /* multiline comment */
+                if (lex->flags.preprocessing) {
+                    haswhite = true;
+                    /*
+                    lex_tokench(lex, '/');
+                    lex_tokench(lex, '*');
+                    */
+                    lex_tokench(lex, ' ');
+                    lex_tokench(lex, ' ');
+                }
+
+                while (ch != EOF)
+                {
+                    ch = lex_getch(lex);
+                    if (ch == '*') {
+                        ch = lex_getch(lex);
+                        if (ch == '/') {
+                            if (lex->flags.preprocessing) {
+                                /*
+                                lex_tokench(lex, '*');
+                                lex_tokench(lex, '/');
+                                */
+                                lex_tokench(lex, ' ');
+                                lex_tokench(lex, ' ');
+                            }
+                            break;
+                        }
+                        lex_ungetch(lex, ch);
+                    }
+                    if (lex->flags.preprocessing) {
+                        if (ch == '\n')
+                            lex_tokench(lex, '\n');
+                        else
+                            lex_tokench(lex, ' '); /* ch); */
+                    }
+                }
+                ch = ' '; /* cause TRUE in the isspace check */
+                continue;
+            }
+            /* Otherwise roll back to the slash and break out of the loop */
+            lex_ungetch(lex, ch);
+            ch = '/';
+            break;
+        }
+    } while (ch != EOF && isspace(ch));
+
+    if (haswhite) {
+        lex_endtoken(lex);
+        lex_ungetch(lex, ch);
+        return TOKEN_WHITE;
+    }
+    return ch;
 }
 
-/* Append a character to the token buffer */
-static bool GMQCC_WARN lex_tokench(lex_file *lex, int ch)
+/* Get a token */
+static bool GMQCC_WARN lex_finish_ident(lex_file *lex)
 {
-	if (!token_value_add(lex->tok, ch)) {
-		lexerror(lex, "out of memory");
-		return false;
-	}
-	return true;
-}
+    int ch;
 
-/* Append a trailing null-byte */
-static bool GMQCC_WARN lex_endtoken(lex_file *lex)
-{
-	if (!token_value_add(lex->tok, 0)) {
-		lexerror(lex, "out of memory");
-		return false;
-	}
-	lex->tok->value_count--;
-	return true;
+    ch = lex_getch(lex);
+    while (ch != EOF && isident(ch))
+    {
+        lex_tokench(lex, ch);
+        ch = lex_getch(lex);
+    }
+
+    /* last ch was not an ident ch: */
+    lex_ungetch(lex, ch);
+
+    return true;
 }
 
-/* Get a token */
-static bool GMQCC_WARN lex_finish_ident(lex_file *lex)
+/* read one ident for the frame list */
+static int lex_parse_frame(lex_file *lex)
 {
-	int ch;
+    int ch;
+
+    lex_token_new(lex);
+
+    ch = lex_getch(lex);
+    while (ch != EOF && ch != '\n' && isspace(ch))
+        ch = lex_getch(lex);
 
-	ch = lex_getch(lex);
-	while (ch != EOF && isident(ch))
-	{
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		ch = lex_getch(lex);
-	}
+    if (ch == '\n')
+        return 1;
 
-	/* last ch was not an ident ch: */
-	lex_ungetch(lex, ch);
+    if (!isident_start(ch)) {
+        lexerror(lex, "invalid framename, must start with one of a-z or _, got %c", ch);
+        return -1;
+    }
 
-	return true;
+    lex_tokench(lex, ch);
+    if (!lex_finish_ident(lex))
+        return -1;
+    lex_endtoken(lex);
+    return 0;
+}
+
+/* read a list of $frames */
+static bool lex_finish_frames(lex_file *lex)
+{
+    do {
+        size_t i;
+        int    rc;
+        frame_macro m;
+
+        rc = lex_parse_frame(lex);
+        if (rc > 0) /* end of line */
+            return true;
+        if (rc < 0) /* error */
+            return false;
+
+        for (i = 0; i < vec_size(lex->frames); ++i) {
+            if (!strcmp(lex->tok.value, lex->frames[i].name)) {
+                lex->frames[i].value = lex->framevalue++;
+                if (lexwarn(lex, WARN_FRAME_MACROS, "duplicate frame macro defined: `%s`", lex->tok.value))
+                    return false;
+                break;
+            }
+        }
+        if (i < vec_size(lex->frames))
+            continue;
+
+        m.value = lex->framevalue++;
+        m.name = util_strdup(lex->tok.value);
+        vec_shrinkto(lex->tok.value, 0);
+        vec_push(lex->frames, m);
+    } while (true);
+
+    return false;
 }
 
 static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
 {
-	int ch = 0;
-
-	while (ch != EOF)
-	{
-		ch = lex_getch(lex);
-		if (ch == quote)
-			return TOKEN_STRINGCONST;
-
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-
-		/* as lexer we only care about \" to not terminate the string prematurely */
-		if (ch == '\\') {
-			ch = lex_getch(lex);
-			if (ch == EOF) {
-				lexerror(lex, "unexpected end of file");
-				lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
-				return (lex->tok->ttype = TOKEN_ERROR);
-			}
-			/* so we just add the next character no matter what it actually is */
-			if (!lex_tokench(lex, ch))
-				return (lex->tok->ttype = TOKEN_FATAL);
-		}
-	}
-	lexerror(lex, "unexpected end of file within string constant");
-	lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
-	return (lex->tok->ttype = TOKEN_ERROR);
+    uchar_t chr;
+    int ch = 0;
+    int nextch;
+    bool hex;
+    char u8buf[8]; /* way more than enough */
+    int  u8len, uc;
+
+    while (ch != EOF)
+    {
+        ch = lex_getch(lex);
+        if (ch == quote)
+            return TOKEN_STRINGCONST;
+
+        if (lex->flags.preprocessing && ch == '\\') {
+            lex_tokench(lex, ch);
+            ch = lex_getch(lex);
+            if (ch == EOF) {
+                lexerror(lex, "unexpected end of file");
+                lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
+                return (lex->tok.ttype = TOKEN_ERROR);
+            }
+            lex_tokench(lex, ch);
+        }
+        else if (ch == '\\') {
+            ch = lex_getch(lex);
+            if (ch == EOF) {
+                lexerror(lex, "unexpected end of file");
+                lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
+                return (lex->tok.ttype = TOKEN_ERROR);
+            }
+
+            switch (ch) {
+            case '\\': break;
+            case '\'': break;
+            case '"':  break;
+            case 'a':  ch = '\a'; break;
+            case 'b':  ch = '\b'; break;
+            case 'r':  ch = '\r'; break;
+            case 'n':  ch = '\n'; break;
+            case 't':  ch = '\t'; break;
+            case 'f':  ch = '\f'; break;
+            case 'v':  ch = '\v'; break;
+            case 'x':
+            case 'X':
+                /* same procedure as in fteqcc */
+                ch = 0;
+                nextch = lex_getch(lex);
+                if      (nextch >= '0' && nextch <= '9')
+                    ch += nextch - '0';
+                else if (nextch >= 'a' && nextch <= 'f')
+                    ch += nextch - 'a' + 10;
+                else if (nextch >= 'A' && nextch <= 'F')
+                    ch += nextch - 'A' + 10;
+                else {
+                    lexerror(lex, "bad character code");
+                    lex_ungetch(lex, nextch);
+                    return (lex->tok.ttype = TOKEN_ERROR);
+                }
+
+                ch *= 0x10;
+                nextch = lex_getch(lex);
+                if      (nextch >= '0' && nextch <= '9')
+                    ch += nextch - '0';
+                else if (nextch >= 'a' && nextch <= 'f')
+                    ch += nextch - 'a' + 10;
+                else if (nextch >= 'A' && nextch <= 'F')
+                    ch += nextch - 'A' + 10;
+                else {
+                    lexerror(lex, "bad character code");
+                    lex_ungetch(lex, nextch);
+                    return (lex->tok.ttype = TOKEN_ERROR);
+                }
+                break;
+
+            /* fteqcc support */
+            case '0': case '1': case '2': case '3':
+            case '4': case '5': case '6': case '7':
+            case '8': case '9':
+                ch = 18 + ch - '0';
+                break;
+            case '<':  ch = 29; break;
+            case '-':  ch = 30; break;
+            case '>':  ch = 31; break;
+            case '[':  ch = 16; break;
+            case ']':  ch = 17; break;
+            case '{':
+                chr = 0;
+                nextch = lex_getch(lex);
+                hex = (nextch == 'x');
+                if (!hex)
+                    lex_ungetch(lex, nextch);
+                for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) {
+                    if (!hex) {
+                        if (nextch >= '0' && nextch <= '9')
+                            chr = chr * 10 + nextch - '0';
+                        else {
+                            lexerror(lex, "bad character code");
+                            return (lex->tok.ttype = TOKEN_ERROR);
+                        }
+                    } else {
+                        if (nextch >= '0' && nextch <= '9')
+                            chr = chr * 0x10 + nextch - '0';
+                        else if (nextch >= 'a' && nextch <= 'f')
+                            chr = chr * 0x10 + nextch - 'a' + 10;
+                        else if (nextch >= 'A' && nextch <= 'F')
+                            chr = chr * 0x10 + nextch - 'A' + 10;
+                        else {
+                            lexerror(lex, "bad character code");
+                            return (lex->tok.ttype = TOKEN_ERROR);
+                        }
+                    }
+                    if (chr > 0x10FFFF || (!OPTS_FLAG(UTF8) && chr > 255))
+                    {
+                        lexerror(lex, "character code out of range");
+                        return (lex->tok.ttype = TOKEN_ERROR);
+                    }
+                }
+                if (OPTS_FLAG(UTF8) && chr >= 128) {
+                    u8len = u8_fromchar(chr, u8buf, sizeof(u8buf));
+                    if (!u8len)
+                        ch = 0;
+                    else {
+                        --u8len;
+                        for (uc = 0; uc < u8len; ++uc)
+                            lex_tokench(lex, u8buf[uc]);
+                        /* the last character will be inserted with the tokench() call
+                         * below the switch
+                         */
+                        ch = u8buf[uc];
+                    }
+                }
+                else
+                    ch = chr;
+                break;
+            case '\n':  ch = '\n'; break;
+
+            default:
+                lexwarn(lex, WARN_UNKNOWN_CONTROL_SEQUENCE, "unrecognized control sequence: \\%c", ch);
+                /* so we just add the character plus backslash no matter what it actually is */
+                lex_tokench(lex, '\\');
+            }
+            /* add the character finally */
+            lex_tokench(lex, ch);
+        }
+        else
+            lex_tokench(lex, ch);
+    }
+    lexerror(lex, "unexpected end of file within string constant");
+    lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
+    return (lex->tok.ttype = TOKEN_ERROR);
 }
 
 static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch)
 {
-	bool ishex = false;
-
-	int  ch = lastch;
-
-	/* parse a number... */
-	lex->tok->ttype = TOKEN_INTCONST;
-
-	if (!lex_tokench(lex, ch))
-		return (lex->tok->ttype = TOKEN_FATAL);
-
-	ch = lex_getch(lex);
-	if (ch != '.' && !isdigit(ch))
-	{
-		if (lastch != '0' || ch != 'x')
-		{
-			/* end of the number or EOF */
-			lex_ungetch(lex, ch);
-			if (!lex_endtoken(lex))
-				return (lex->tok->ttype = TOKEN_FATAL);
-
-			lex->tok->constval.i = lastch - '0';
-			return lex->tok->ttype;
-		}
-
-		ishex = true;
-	}
-
-	/* EOF would have been caught above */
-
-	if (ch != '.')
-	{
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		ch = lex_getch(lex);
-		while (isdigit(ch) || (ishex && isxdigit_only(ch)))
-		{
-			if (!lex_tokench(lex, ch))
-				return (lex->tok->ttype = TOKEN_FATAL);
-			ch = lex_getch(lex);
-		}
-	}
-	/* NOT else, '.' can come from above as well */
-	if (ch == '.' && !ishex)
-	{
-		/* Allow floating comma in non-hex mode */
-		lex->tok->ttype = TOKEN_FLOATCONST;
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-
-		/* continue digits-only */
-		ch = lex_getch(lex);
-		while (isdigit(ch))
-		{
-			if (!lex_tokench(lex, ch))
-				return (lex->tok->ttype = TOKEN_FATAL);
-			ch = lex_getch(lex);
-		}
-	}
-	/* put back the last character */
-	/* but do not put back the trailing 'f' or a float */
-	if (lex->tok->ttype == TOKEN_FLOATCONST && ch == 'f')
-		ch = lex_getch(lex);
-
-	/* generally we don't want words to follow numbers: */
-	if (isident(ch)) {
-		lexerror(lex, "unexpected trailing characters after number");
-		return (lex->tok->ttype = TOKEN_ERROR);
-	}
-	lex_ungetch(lex, ch);
-
-	if (!lex_endtoken(lex))
-		return (lex->tok->ttype = TOKEN_FATAL);
-	if (lex->tok->ttype == TOKEN_FLOATCONST)
-		lex->tok->constval.f = strtod(lex->tok->value, NULL);
-	else
-		lex->tok->constval.i = strtol(lex->tok->value, NULL, 0);
-	return lex->tok->ttype;
+    bool ishex = false;
+
+    int  ch = lastch;
+
+    /* parse a number... */
+    if (ch == '.')
+        lex->tok.ttype = TOKEN_FLOATCONST;
+    else
+        lex->tok.ttype = TOKEN_INTCONST;
+
+    lex_tokench(lex, ch);
+
+    ch = lex_getch(lex);
+    if (ch != '.' && !isdigit(ch))
+    {
+        if (lastch != '0' || ch != 'x')
+        {
+            /* end of the number or EOF */
+            lex_ungetch(lex, ch);
+            lex_endtoken(lex);
+
+            lex->tok.constval.i = lastch - '0';
+            return lex->tok.ttype;
+        }
+
+        ishex = true;
+    }
+
+    /* EOF would have been caught above */
+
+    if (ch != '.')
+    {
+        lex_tokench(lex, ch);
+        ch = lex_getch(lex);
+        while (isdigit(ch) || (ishex && isxdigit_only(ch)))
+        {
+            lex_tokench(lex, ch);
+            ch = lex_getch(lex);
+        }
+    }
+    /* NOT else, '.' can come from above as well */
+    if (lex->tok.ttype != TOKEN_FLOATCONST && ch == '.' && !ishex)
+    {
+        /* Allow floating comma in non-hex mode */
+        lex->tok.ttype = TOKEN_FLOATCONST;
+        lex_tokench(lex, ch);
+
+        /* continue digits-only */
+        ch = lex_getch(lex);
+        while (isdigit(ch))
+        {
+            lex_tokench(lex, ch);
+            ch = lex_getch(lex);
+        }
+    }
+    /* put back the last character */
+    /* but do not put back the trailing 'f' or a float */
+    if (lex->tok.ttype == TOKEN_FLOATCONST && ch == 'f')
+        ch = lex_getch(lex);
+
+    /* generally we don't want words to follow numbers: */
+    if (isident(ch)) {
+        lexerror(lex, "unexpected trailing characters after number");
+        return (lex->tok.ttype = TOKEN_ERROR);
+    }
+    lex_ungetch(lex, ch);
+
+    lex_endtoken(lex);
+    if (lex->tok.ttype == TOKEN_FLOATCONST)
+        lex->tok.constval.f = strtod(lex->tok.value, NULL);
+    else
+        lex->tok.constval.i = strtol(lex->tok.value, NULL, 0);
+    return lex->tok.ttype;
 }
 
 int lex_do(lex_file *lex)
 {
-	int ch, nextch;
-
-	if (lex->tok)
-		token_delete(lex->tok);
-	lex->tok = token_new();
-	if (!lex->tok)
-		return TOKEN_FATAL;
-
-	ch = lex_skipwhite(lex);
-	lex->sline = lex->line;
-	lex->tok->ctx.line = lex->sline;
-	lex->tok->ctx.file = lex->name;
-
-	if (ch == EOF)
-		return (lex->tok->ttype = TOKEN_EOF);
-
-	/* single-character tokens */
-	switch (ch)
-	{
-		case ';':
-		case '(':
-		case ')':
-		case '{':
-		case '}':
-		case '[':
-		case ']':
-
-		case ',':
-
-		case '#':
-
-			return (lex->tok->ttype = ch);
-		default:
-			break;
-	}
-
-	if (lex->flags.noops)
-	{
-		/* Detect characters early which are normally
-		 * operators OR PART of an operator.
-		 */
-		switch (ch)
-		{
-			case '+':
-			case '-':
-			case '*':
-			case '/':
-			case '<':
-			case '>':
-			case '=':
-			case '&':
-			case '|':
-			case '^':
-			case '~':
-				return ch;
-			default:
-				break;
-		}
-	}
-
-	if (ch == '+' || ch == '-' || /* ++, --, +=, -=  and -> as well! */
-	    ch == '>' || ch == '<' || /* <<, >>, <=, >= */
-	    ch == '=' ||              /* == */
-	    ch == '&' || ch == '|')   /* &&, ||, &=, |= */
-	{
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-
-		nextch = lex_getch(lex);
-		if (nextch == ch || nextch == '=') {
-			if (!lex_tokench(lex, nextch))
-				return (lex->tok->ttype = TOKEN_FATAL);
-		} else if (ch == '-' && nextch == '>') {
-			if (!lex_tokench(lex, nextch))
-				return (lex->tok->ttype = TOKEN_FATAL);
-		} else
-			lex_ungetch(lex, nextch);
-
-		if (!lex_endtoken(lex))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		return (lex->tok->ttype = TOKEN_OPERATOR);
-	}
-
-	if (ch == '^' || ch == '~' || ch == '!')
-	{
-		if (!lex_tokench(lex, ch) ||
-			!lex_endtoken(lex))
-		{
-			return (lex->tok->ttype = TOKEN_FATAL);
-		}
-		return (lex->tok->ttype = TOKEN_OPERATOR);
-	}
-
-	if (ch == '*' || ch == '/') /* *=, /= */
-	{
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-
-		nextch = lex_getch(lex);
-		if (nextch == '=') {
-			if (!lex_tokench(lex, nextch))
-				return (lex->tok->ttype = TOKEN_FATAL);
-		} else
-			lex_ungetch(lex, nextch);
-
-		if (!lex_endtoken(lex))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		return (lex->tok->ttype = TOKEN_OPERATOR);
-	}
-
-	if (isident_start(ch))
-	{
-		const char *v;
-		if (!lex_tokench(lex, ch))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		if (!lex_finish_ident(lex)) {
-			/* error? */
-			return (lex->tok->ttype = TOKEN_ERROR);
-		}
-		if (!lex_endtoken(lex))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		lex->tok->ttype = TOKEN_IDENT;
-
-		v = lex->tok->value;
-		if (!strcmp(v, "void")) {
-			lex->tok->ttype = TOKEN_TYPENAME;
-		    lex->tok->constval.t = TYPE_VOID;
-		} else if (!strcmp(v, "int")) {
-			lex->tok->ttype = TOKEN_TYPENAME;
-		    lex->tok->constval.t = TYPE_INTEGER;
-		} else if (!strcmp(v, "float")) {
-			lex->tok->ttype = TOKEN_TYPENAME;
-		    lex->tok->constval.t = TYPE_FLOAT;
-		} else if (!strcmp(v, "string")) {
-			lex->tok->ttype = TOKEN_TYPENAME;
-		    lex->tok->constval.t = TYPE_STRING;
-		} else if (!strcmp(v, "entity")) {
-			lex->tok->ttype = TOKEN_TYPENAME;
-		    lex->tok->constval.t = TYPE_ENTITY;
-		} else if (!strcmp(v, "vector")) {
-			lex->tok->ttype = TOKEN_TYPENAME;
-		    lex->tok->constval.t = TYPE_VECTOR;
-		} else if (!strcmp(v, "for") ||
-		         !strcmp(v, "while") ||
-		         !strcmp(v, "do")    ||
-		         !strcmp(v, "var")   ||
-		         !strcmp(v, "const"))
-			lex->tok->ttype = TOKEN_KEYWORD;
-
-		return lex->tok->ttype;
-	}
-
-	if (ch == '"')
-	{
-		lex->tok->ttype = lex_finish_string(lex, '"');
-		while (lex->tok->ttype == TOKEN_STRINGCONST)
-		{
-			/* Allow c style "string" "continuation" */
-			ch = lex_skipwhite(lex);
-			if (ch != '"') {
-				lex_ungetch(lex, ch);
-				break;
-			}
-
-			lex->tok->ttype = lex_finish_string(lex, '"');
-		}
-		if (!lex_endtoken(lex))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		return lex->tok->ttype;
-	}
-
-	if (ch == '\'')
-	{
-		/* we parse character constants like string,
-		 * but return TOKEN_CHARCONST, or a vector type if it fits...
-		 * Likewise actual unescaping has to be done by the parser.
-		 * The difference is we don't allow 'char' 'continuation'.
-		 */
-		 lex->tok->ttype = lex_finish_string(lex, '\'');
-		 if (!lex_endtoken(lex))
-		 	 return (lex->tok->ttype = TOKEN_FATAL);
-
-		 /* It's a vector if we can successfully scan 3 floats */
-		 if (sscanf(lex->tok->value, " %f %f %f ", &lex->tok->constval.v.x, &lex->tok->constval.v.y, &lex->tok->constval.v.z) == 3)
-		 {
-		 	 lex->tok->ttype = TOKEN_VECTORCONST;
-		 }
-
-		 return lex->tok->ttype;
-	}
-
-	if (isdigit(ch))
-	{
-		lex->tok->ttype = lex_finish_digit(lex, ch);
-		if (!lex_endtoken(lex))
-			return (lex->tok->ttype = TOKEN_FATAL);
-		return lex->tok->ttype;
-	}
-
-	lexerror(lex, "unknown token");
-	return (lex->tok->ttype = TOKEN_ERROR);
+    int ch, nextch, thirdch;
+    bool hadwhite = false;
+
+    lex_token_new(lex);
+#if 0
+    if (!lex->tok)
+        return TOKEN_FATAL;
+#endif
+
+    while (true) {
+        ch = lex_skipwhite(lex, hadwhite);
+        hadwhite = true;
+        if (!lex->flags.mergelines || ch != '\\')
+            break;
+        ch = lex_getch(lex);
+        if (ch == '\r')
+            ch = lex_getch(lex);
+        if (ch != '\n') {
+            lex_ungetch(lex, ch);
+            ch = '\\';
+            break;
+        }
+        /* we reached a linemerge */
+        lex_tokench(lex, '\n');
+        continue;
+    }
+
+    if (lex->flags.preprocessing && (ch == TOKEN_WHITE || ch == TOKEN_EOL || ch == TOKEN_FATAL)) {
+        return (lex->tok.ttype = ch);
+    }
+
+    lex->sline = lex->line;
+    lex->tok.ctx.line = lex->sline;
+    lex->tok.ctx.file = lex->name;
+
+    if (lex->eof)
+        return (lex->tok.ttype = TOKEN_FATAL);
+
+    if (ch == EOF) {
+        lex->eof = true;
+        return (lex->tok.ttype = TOKEN_EOF);
+    }
+
+    /* modelgen / spiritgen commands */
+    if (ch == '$' && !lex->flags.preprocessing) {
+        const char *v;
+        size_t frame;
+
+        ch = lex_getch(lex);
+        if (!isident_start(ch)) {
+            lexerror(lex, "hanging '$' modelgen/spritegen command line");
+            return lex_do(lex);
+        }
+        lex_tokench(lex, ch);
+        if (!lex_finish_ident(lex))
+            return (lex->tok.ttype = TOKEN_ERROR);
+        lex_endtoken(lex);
+        /* skip the known commands */
+        v = lex->tok.value;
+
+        if (!strcmp(v, "frame") || !strcmp(v, "framesave"))
+        {
+            /* frame/framesave command works like an enum
+             * similar to fteqcc we handle this in the lexer.
+             * The reason for this is that it is sensitive to newlines,
+             * which the parser is unaware of
+             */
+            if (!lex_finish_frames(lex))
+                 return (lex->tok.ttype = TOKEN_ERROR);
+            return lex_do(lex);
+        }
+
+        if (!strcmp(v, "framevalue"))
+        {
+            ch = lex_getch(lex);
+            while (ch != EOF && isspace(ch) && ch != '\n')
+                ch = lex_getch(lex);
+
+            if (!isdigit(ch)) {
+                lexerror(lex, "$framevalue requires an integer parameter");
+                return lex_do(lex);
+            }
+
+            lex_token_new(lex);
+            lex->tok.ttype = lex_finish_digit(lex, ch);
+            lex_endtoken(lex);
+            if (lex->tok.ttype != TOKEN_INTCONST) {
+                lexerror(lex, "$framevalue requires an integer parameter");
+                return lex_do(lex);
+            }
+            lex->framevalue = lex->tok.constval.i;
+            return lex_do(lex);
+        }
+
+        if (!strcmp(v, "framerestore"))
+        {
+            int rc;
+
+            lex_token_new(lex);
+
+            rc = lex_parse_frame(lex);
+
+            if (rc > 0) {
+                lexerror(lex, "$framerestore requires a framename parameter");
+                return lex_do(lex);
+            }
+            if (rc < 0)
+                return (lex->tok.ttype = TOKEN_FATAL);
+
+            v = lex->tok.value;
+            for (frame = 0; frame < vec_size(lex->frames); ++frame) {
+                if (!strcmp(v, lex->frames[frame].name)) {
+                    lex->framevalue = lex->frames[frame].value;
+                    return lex_do(lex);
+                }
+            }
+            lexerror(lex, "unknown framename `%s`", v);
+            return lex_do(lex);
+        }
+
+        if (!strcmp(v, "modelname"))
+        {
+            int rc;
+
+            lex_token_new(lex);
+
+            rc = lex_parse_frame(lex);
+
+            if (rc > 0) {
+                lexerror(lex, "$modelname requires a parameter");
+                return lex_do(lex);
+            }
+            if (rc < 0)
+                return (lex->tok.ttype = TOKEN_FATAL);
+
+            if (lex->modelname) {
+                frame_macro m;
+                m.value = lex->framevalue;
+                m.name = lex->modelname;
+                lex->modelname = NULL;
+                vec_push(lex->frames, m);
+            }
+            lex->modelname = lex->tok.value;
+            lex->tok.value = NULL;
+            return lex_do(lex);
+        }
+
+        if (!strcmp(v, "flush"))
+        {
+            size_t fi;
+            for (fi = 0; fi < vec_size(lex->frames); ++fi)
+                mem_d(lex->frames[fi].name);
+            vec_free(lex->frames);
+            /* skip line (fteqcc does it too) */
+            ch = lex_getch(lex);
+            while (ch != EOF && ch != '\n')
+                ch = lex_getch(lex);
+            return lex_do(lex);
+        }
+
+        if (!strcmp(v, "cd") ||
+            !strcmp(v, "origin") ||
+            !strcmp(v, "base") ||
+            !strcmp(v, "flags") ||
+            !strcmp(v, "scale") ||
+            !strcmp(v, "skin"))
+        {
+            /* skip line */
+            ch = lex_getch(lex);
+            while (ch != EOF && ch != '\n')
+                ch = lex_getch(lex);
+            return lex_do(lex);
+        }
+
+        for (frame = 0; frame < vec_size(lex->frames); ++frame) {
+            if (!strcmp(v, lex->frames[frame].name)) {
+                lex->tok.constval.i = lex->frames[frame].value;
+                return (lex->tok.ttype = TOKEN_INTCONST);
+            }
+        }
+
+        lexerror(lex, "invalid frame macro");
+        return lex_do(lex);
+    }
+
+    /* single-character tokens */
+    switch (ch)
+    {
+        case '[':
+            nextch = lex_getch(lex);
+            if (nextch == '[') {
+                lex_tokench(lex, ch);
+                lex_tokench(lex, nextch);
+                lex_endtoken(lex);
+                return (lex->tok.ttype = TOKEN_ATTRIBUTE_OPEN);
+            }
+            lex_ungetch(lex, nextch);
+            /* FALL THROUGH */
+        case '(':
+        case ':':
+        case '?':
+            lex_tokench(lex, ch);
+            lex_endtoken(lex);
+            if (lex->flags.noops)
+                return (lex->tok.ttype = ch);
+            else
+                return (lex->tok.ttype = TOKEN_OPERATOR);
+
+        case ']':
+            if (lex->flags.noops) {
+                nextch = lex_getch(lex);
+                if (nextch == ']') {
+                    lex_tokench(lex, ch);
+                    lex_tokench(lex, nextch);
+                    lex_endtoken(lex);
+                    return (lex->tok.ttype = TOKEN_ATTRIBUTE_CLOSE);
+                }
+                lex_ungetch(lex, nextch);
+            }
+            /* FALL THROUGH */
+        case ')':
+        case ';':
+        case '{':
+        case '}':
+
+        case '#':
+            lex_tokench(lex, ch);
+            lex_endtoken(lex);
+            return (lex->tok.ttype = ch);
+        default:
+            break;
+    }
+
+    if (ch == '.') {
+        nextch = lex_getch(lex);
+        /* digits starting with a dot */
+        if (isdigit(nextch)) {
+            lex_ungetch(lex, nextch);
+            lex->tok.ttype = lex_finish_digit(lex, ch);
+            lex_endtoken(lex);
+            return lex->tok.ttype;
+        }
+        lex_ungetch(lex, nextch);
+    }
+
+    if (lex->flags.noops)
+    {
+        /* Detect characters early which are normally
+         * operators OR PART of an operator.
+         */
+        switch (ch)
+        {
+            /*
+            case '+':
+            case '-':
+            */
+            case '*':
+            case '/':
+            case '<':
+            case '>':
+            case '=':
+            case '&':
+            case '|':
+            case '^':
+            case '~':
+            case ',':
+            case '!':
+                lex_tokench(lex, ch);
+                lex_endtoken(lex);
+                return (lex->tok.ttype = ch);
+            default:
+                break;
+        }
+    }
+
+    if (ch == '.')
+    {
+        lex_tokench(lex, ch);
+        /* peak ahead once */
+        nextch = lex_getch(lex);
+        if (nextch != '.') {
+            lex_ungetch(lex, nextch);
+            lex_endtoken(lex);
+            if (lex->flags.noops)
+                return (lex->tok.ttype = ch);
+            else
+                return (lex->tok.ttype = TOKEN_OPERATOR);
+        }
+        /* peak ahead again */
+        nextch = lex_getch(lex);
+        if (nextch != '.') {
+            lex_ungetch(lex, nextch);
+            lex_ungetch(lex, '.');
+            lex_endtoken(lex);
+            if (lex->flags.noops)
+                return (lex->tok.ttype = ch);
+            else
+                return (lex->tok.ttype = TOKEN_OPERATOR);
+        }
+        /* fill the token to be "..." */
+        lex_tokench(lex, ch);
+        lex_tokench(lex, ch);
+        lex_endtoken(lex);
+        return (lex->tok.ttype = TOKEN_DOTS);
+    }
+
+    if (ch == ',' || ch == '.') {
+        lex_tokench(lex, ch);
+        lex_endtoken(lex);
+        return (lex->tok.ttype = TOKEN_OPERATOR);
+    }
+
+    if (ch == '+' || ch == '-' || /* ++, --, +=, -=  and -> as well! */
+        ch == '>' || ch == '<' || /* <<, >>, <=, >=                  */
+        ch == '=' || ch == '!' || /* <=>, ==, !=                     */
+        ch == '&' || ch == '|' || /* &&, ||, &=, |=                  */
+        ch == '~'                 /* ~=, ~                           */
+    )  {
+        lex_tokench(lex, ch);
+
+        nextch = lex_getch(lex);
+        if ((nextch == '=' && ch != '<') || (nextch == ch && ch != '!')) {
+            lex_tokench(lex, nextch);
+        } else if (ch == '<' && nextch == '=') {
+            lex_tokench(lex, nextch);
+            if ((thirdch = lex_getch(lex)) == '>')
+                lex_tokench(lex, thirdch);
+            else
+                lex_ungetch(lex, thirdch);
+
+        } else if (ch == '-' && nextch == '>') {
+            lex_tokench(lex, nextch);
+        } else if (ch == '&' && nextch == '~') {
+            thirdch = lex_getch(lex);
+            if (thirdch != '=') {
+                lex_ungetch(lex, thirdch);
+                lex_ungetch(lex, nextch);
+            }
+            else {
+                lex_tokench(lex, nextch);
+                lex_tokench(lex, thirdch);
+            }
+        }
+        else if (lex->flags.preprocessing &&
+                 ch == '-' && isdigit(nextch))
+        {
+            lex->tok.ttype = lex_finish_digit(lex, nextch);
+            if (lex->tok.ttype == TOKEN_INTCONST)
+                lex->tok.constval.i = -lex->tok.constval.i;
+            else
+                lex->tok.constval.f = -lex->tok.constval.f;
+            lex_endtoken(lex);
+            return lex->tok.ttype;
+        } else {
+            lex_ungetch(lex, nextch);
+        }
+
+        lex_endtoken(lex);
+        return (lex->tok.ttype = TOKEN_OPERATOR);
+    }
+
+    /*
+    if (ch == '^' || ch == '~' || ch == '!')
+    {
+        lex_tokench(lex, ch);
+        lex_endtoken(lex);
+        return (lex->tok.ttype = TOKEN_OPERATOR);
+    }
+    */
+
+    if (ch == '*' || ch == '/') /* *=, /= */
+    {
+        lex_tokench(lex, ch);
+
+        nextch = lex_getch(lex);
+        if (nextch == '=' || nextch == '*') {
+            lex_tokench(lex, nextch);
+        } else
+            lex_ungetch(lex, nextch);
+
+        lex_endtoken(lex);
+        return (lex->tok.ttype = TOKEN_OPERATOR);
+    }
+
+    if (ch == '%') {
+        lex_tokench(lex, ch);
+        lex_endtoken(lex);
+        return (lex->tok.ttype = TOKEN_OPERATOR);
+    }
+
+    if (isident_start(ch))
+    {
+        const char *v;
+
+        lex_tokench(lex, ch);
+        if (!lex_finish_ident(lex)) {
+            /* error? */
+            return (lex->tok.ttype = TOKEN_ERROR);
+        }
+        lex_endtoken(lex);
+        lex->tok.ttype = TOKEN_IDENT;
+
+        v = lex->tok.value;
+        if (!strcmp(v, "void")) {
+            lex->tok.ttype = TOKEN_TYPENAME;
+            lex->tok.constval.t = TYPE_VOID;
+        } else if (!strcmp(v, "int")) {
+            lex->tok.ttype = TOKEN_TYPENAME;
+            lex->tok.constval.t = TYPE_INTEGER;
+        } else if (!strcmp(v, "float")) {
+            lex->tok.ttype = TOKEN_TYPENAME;
+            lex->tok.constval.t = TYPE_FLOAT;
+        } else if (!strcmp(v, "string")) {
+            lex->tok.ttype = TOKEN_TYPENAME;
+            lex->tok.constval.t = TYPE_STRING;
+        } else if (!strcmp(v, "entity")) {
+            lex->tok.ttype = TOKEN_TYPENAME;
+            lex->tok.constval.t = TYPE_ENTITY;
+        } else if (!strcmp(v, "vector")) {
+            lex->tok.ttype = TOKEN_TYPENAME;
+            lex->tok.constval.t = TYPE_VECTOR;
+        } else {
+            size_t kw;
+            for (kw = 0; kw < num_keywords_qc; ++kw) {
+                if (!strcmp(v, keywords_qc[kw]))
+                    return (lex->tok.ttype = TOKEN_KEYWORD);
+            }
+            if (OPTS_OPTION_U32(OPTION_STANDARD) != COMPILER_QCC) {
+                for (kw = 0; kw < num_keywords_fg; ++kw) {
+                    if (!strcmp(v, keywords_fg[kw]))
+                        return (lex->tok.ttype = TOKEN_KEYWORD);
+                }
+            }
+        }
+
+        return lex->tok.ttype;
+    }
+
+    if (ch == '"')
+    {
+        lex->flags.nodigraphs = true;
+        if (lex->flags.preprocessing)
+            lex_tokench(lex, ch);
+        lex->tok.ttype = lex_finish_string(lex, '"');
+        if (lex->flags.preprocessing)
+            lex_tokench(lex, ch);
+        while (!lex->flags.preprocessing && lex->tok.ttype == TOKEN_STRINGCONST)
+        {
+            /* Allow c style "string" "continuation" */
+            ch = lex_skipwhite(lex, false);
+            if (ch != '"') {
+                lex_ungetch(lex, ch);
+                break;
+            }
+
+            lex->tok.ttype = lex_finish_string(lex, '"');
+        }
+        lex->flags.nodigraphs = false;
+        lex_endtoken(lex);
+        return lex->tok.ttype;
+    }
+
+    if (ch == '\'')
+    {
+        /* we parse character constants like string,
+         * but return TOKEN_CHARCONST, or a vector type if it fits...
+         * Likewise actual unescaping has to be done by the parser.
+         * The difference is we don't allow 'char' 'continuation'.
+         */
+        if (lex->flags.preprocessing)
+            lex_tokench(lex, ch);
+        lex->tok.ttype = lex_finish_string(lex, '\'');
+        if (lex->flags.preprocessing)
+            lex_tokench(lex, ch);
+        lex_endtoken(lex);
+
+        lex->tok.ttype = TOKEN_CHARCONST;
+         /* It's a vector if we can successfully scan 3 floats */
+#ifdef _MSC_VER
+        if (sscanf_s(lex->tok.value, " %f %f %f ",
+                   &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3)
+#else
+        if (sscanf(lex->tok.value, " %f %f %f ",
+                   &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3)
+#endif
+
+        {
+             lex->tok.ttype = TOKEN_VECTORCONST;
+        }
+        else
+        {
+            if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) {
+                uchar_t u8char;
+                /* check for a valid utf8 character */
+                if (!OPTS_FLAG(UTF8) || !u8_analyze(lex->tok.value, NULL, NULL, &u8char, 8)) {
+                    if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER,
+                                ( OPTS_FLAG(UTF8) ? "invalid multibyte character sequence `%s`"
+                                                  : "multibyte character: `%s`" ),
+                                lex->tok.value))
+                        return (lex->tok.ttype = TOKEN_ERROR);
+                }
+                else
+                    lex->tok.constval.i = u8char;
+            }
+            else
+                lex->tok.constval.i = lex->tok.value[0];
+        }
+
+        return lex->tok.ttype;
+    }
+
+    if (isdigit(ch))
+    {
+        lex->tok.ttype = lex_finish_digit(lex, ch);
+        lex_endtoken(lex);
+        return lex->tok.ttype;
+    }
+
+    if (lex->flags.preprocessing) {
+        lex_tokench(lex, ch);
+        lex_endtoken(lex);
+        return (lex->tok.ttype = ch);
+    }
+
+    lexerror(lex, "unknown token: `%s`", lex->tok.value);
+    return (lex->tok.ttype = TOKEN_ERROR);
 }