From: Dale Weiler <killfieldengine@gmail.com>
Date: Fri, 26 Jul 2013 16:09:26 +0000 (+0000)
Subject: Smaller memory footprint, 4/8 bytes vs 12/24 for individual token lex_ctx's. Use... 
X-Git-Tag: v0.3.0~69
X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=commitdiff_plain;h=cddf70f46bc9c46b6adda39a83e9564bf0cf34d3

Smaller memory footprint, 4/8 bytes vs 12/24 for individual token lex_ctx's. Use to be a 'shallow' copy of the details in lex_file, now it's a pointer to the contents in lex_file.
---

diff --git a/ftepp.c b/ftepp.c
index d68eed4..723ec6a 100644
--- a/ftepp.c
+++ b/ftepp.c
@@ -132,14 +132,14 @@ static char *ftepp_predef_time(lex_file *context) {
 /* __LINE__ */
 static char *ftepp_predef_line(lex_file *context) {
     char   *value;
-    util_asprintf(&value, "%d", (int)context->line);
+    util_asprintf(&value, "%d", (int)context->ctx.line);
     return value;
 }
 /* __FILE__ */
 static char *ftepp_predef_file(lex_file *context) {
-    size_t  length = strlen(context->name) + 3; /* two quotes and a terminator */
+    size_t  length = strlen(context->ctx.file) + 3; /* two quotes and a terminator */
     char   *value  = (char*)mem_a(length);
-    util_snprintf(value, length, "\"%s\"", context->name);
+    util_snprintf(value, length, "\"%s\"", context->ctx.file);
 
     return value;
 }
@@ -183,7 +183,7 @@ static char *ftepp_predef_timestamp(lex_file *context) {
     char       *find;
     char       *value;
     size_t      size;
-    if (stat(context->name, &finfo))
+    if (stat(context->ctx.file, &finfo))
         return util_strdup("\"<failed to determine timestamp>\"");
 
     /*
@@ -239,14 +239,14 @@ static GMQCC_INLINE char *(*ftepp_predef(const char *name))(lex_file *context) {
 #define ftepp_tokval(f) ((f)->lex->tok.value)
 #define ftepp_ctx(f)    ((f)->lex->tok.ctx)
 
-static void ftepp_errorat(ftepp_t *ftepp, lex_ctx ctx, const char *fmt, ...)
+static void ftepp_errorat(ftepp_t *ftepp, lex_ctx *ctx, const char *fmt, ...)
 {
     va_list ap;
 
     ftepp->errors++;
 
     va_start(ap, fmt);
-    con_cvprintmsg((void*)&ctx, LVL_ERROR, "error", fmt, ap);
+    con_cvprintmsg((void*)ctx, LVL_ERROR, "error", fmt, ap);
     va_end(ap);
 }
 
@@ -267,7 +267,7 @@ static bool GMQCC_WARN ftepp_warn(ftepp_t *ftepp, int warntype, const char *fmt,
     va_list ap;
 
     va_start(ap, fmt);
-    r = vcompile_warning(ftepp->lex->tok.ctx, warntype, fmt, ap);
+    r = vcompile_warning(*(ftepp->lex->tok.ctx), warntype, fmt, ap);
     va_end(ap);
     return r;
 }
@@ -524,7 +524,7 @@ static bool ftepp_define_body(ftepp_t *ftepp, ppmacro *macro)
 static bool ftepp_define(ftepp_t *ftepp)
 {
     ppmacro *macro = NULL;
-    size_t l = ftepp_ctx(ftepp).line;
+    size_t l = ftepp_ctx(ftepp)->line;
 
     (void)ftepp_next(ftepp);
     if (!ftepp_skipspace(ftepp))
@@ -540,7 +540,7 @@ static bool ftepp_define(ftepp_t *ftepp)
                     return false;
                 ftepp_macro_delete(ftepp, ftepp_tokval(ftepp));
             }
-            macro = ppmacro_new(ftepp_ctx(ftepp), ftepp_tokval(ftepp));
+            macro = ppmacro_new(*ftepp_ctx(ftepp), ftepp_tokval(ftepp));
             break;
         default:
             ftepp_error(ftepp, "expected macro name");
@@ -573,7 +573,7 @@ static bool ftepp_define(ftepp_t *ftepp)
         ppmacro_delete(macro);
     }
 
-    for (; l < ftepp_ctx(ftepp).line; ++l)
+    for (; l < ftepp_ctx(ftepp)->line; ++l)
         ftepp_out(ftepp, "\n", true);
     return true;
 }
@@ -840,16 +840,16 @@ static bool ftepp_macro_expand(ftepp_t *ftepp, ppmacro *macro, macroparam *param
     /*
     printf("__________\n%s\n=========\n", ftepp->output_string);
     */
-    inlex = lex_open_string(ftepp->output_string, vec_size(ftepp->output_string)-1, ftepp->lex->name);
+    inlex = lex_open_string(ftepp->output_string, vec_size(ftepp->output_string)-1, ftepp->lex->ctx.file);
     if (!inlex) {
         ftepp_error(ftepp, "internal error: failed to instantiate lexer");
         retval = false;
         goto cleanup;
     }
 
-    inlex->line  = ftepp->lex->line;
-    inlex->sline = ftepp->lex->sline;
-    ftepp->lex   = inlex;
+    inlex->sline    = ftepp->lex->sline;
+    inlex->ctx.line = ftepp->lex->ctx.line;
+    ftepp->lex      = inlex;
 
     old_inmacro     = ftepp->in_macro;
     ftepp->in_macro = true;
@@ -1446,7 +1446,7 @@ static bool ftepp_include(ftepp_t *ftepp)
         return true;
     }
 
-    ctx = ftepp_ctx(ftepp);
+    ctx = *ftepp_ctx(ftepp);
 
     unescape(ftepp_tokval(ftepp), ftepp_tokval(ftepp));
 
@@ -1517,7 +1517,7 @@ static bool ftepp_hash(ftepp_t *ftepp)
     ppcondition cond;
     ppcondition *pc;
 
-    lex_ctx ctx = ftepp_ctx(ftepp);
+    lex_ctx *ctx = ftepp_ctx(ftepp);
 
     if (!ftepp_skipspace(ftepp))
         return false;
diff --git a/gmqcc.h b/gmqcc.h
index 202ae14..eb0fc6a 100644
--- a/gmqcc.h
+++ b/gmqcc.h
@@ -746,10 +746,6 @@ qcint     code_alloc_field   (code_t *, size_t qcsize);
 void      code_push_statement(code_t *, prog_section_statement *stmt, int linenum);
 void      code_pop_statement (code_t *);
 
-/*
- * A shallow copy of a lex_file to remember where which ast node
- * came from.
- */
 typedef struct {
     const char *file;
     size_t      line;
@@ -800,10 +796,10 @@ extern size_t compile_errors;
 extern size_t compile_Werrors;
 extern size_t compile_warnings;
 
-void /********/ compile_error   (lex_ctx ctx, /*LVL_ERROR*/ const char *msg, ...);
-void /********/ vcompile_error  (lex_ctx ctx, /*LVL_ERROR*/ const char *msg, va_list ap);
-bool GMQCC_WARN compile_warning (lex_ctx ctx, int warntype, const char *fmt, ...);
-bool GMQCC_WARN vcompile_warning(lex_ctx ctx, int warntype, const char *fmt, va_list ap);
+void /********/ compile_error   (lex_ctx  ctx, /*LVL_ERROR*/ const char *msg, ...);
+void /********/ vcompile_error  (lex_ctx  ctx, /*LVL_ERROR*/ const char *msg, va_list ap);
+bool GMQCC_WARN compile_warning (lex_ctx  ctx, int warntype, const char *fmt, ...);
+bool GMQCC_WARN vcompile_warning(lex_ctx  ctx, int warntype, const char *fmt, va_list ap);
 void            compile_show_werrors(void);
 
 /*===================================================================*/
diff --git a/lexer.c b/lexer.c
index e0dafc3..a00814d 100644
--- a/lexer.c
+++ b/lexer.c
@@ -63,7 +63,7 @@ static void lexerror(lex_file *lex, const char *fmt, ...)
 
     va_start(ap, fmt);
     if (lex)
-        con_vprintmsg(LVL_ERROR, lex->name, lex->sline, lex->column, "parse error", fmt, ap);
+        con_vprintmsg(LVL_ERROR, lex->ctx.file, lex->sline, lex->ctx.column, "parse error", fmt, ap);
     else
         con_vprintmsg(LVL_ERROR, "", 0, 0, "parse error", fmt, ap);
     va_end(ap);
@@ -72,15 +72,10 @@ static void lexerror(lex_file *lex, const char *fmt, ...)
 static bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...)
 {
     bool    r;
-    lex_ctx ctx;
     va_list ap;
 
-    ctx.file   = lex->name;
-    ctx.line   = lex->sline;
-    ctx.column = lex->column;
-
     va_start(ap, fmt);
-    r = vcompile_warning(ctx, warntype, fmt, ap);
+    r = vcompile_warning(lex->ctx, warntype, fmt, ap);
     va_end(ap);
     return r;
 }
@@ -174,9 +169,7 @@ static void lex_token_new(lex_file *lex)
         vec_shrinkto(lex->tok.value, 0);
 
     lex->tok.constval.t  = 0;
-    lex->tok.ctx.line    = lex->sline;
-    lex->tok.ctx.file    = lex->name;
-    lex->tok.ctx.column  = lex->column;
+    lex->tok.ctx         = &lex->ctx; /* this is 'shallow' */
 #endif
 }
 #endif
@@ -200,14 +193,14 @@ lex_file* lex_open(const char *file)
 
     memset(lex, 0, sizeof(*lex));
 
-    lex->file    = in;
-    lex->name    = util_strdup(file);
-    lex->line    = 1; /* we start counting at 1 */
-    lex->column  = 0;
-    lex->peekpos = 0;
-    lex->eof     = false;
+    lex->file        = in;
+    lex->ctx.file    = util_strdup(file);
+    lex->ctx.line    = 1; /* we start counting at 1 */
+    lex->ctx.column  = 0;
+    lex->peekpos     = 0;
+    lex->eof         = false;
 
-    vec_push(lex_filenames, lex->name);
+    vec_push(lex_filenames, (char *)lex->ctx.file);
     return lex;
 }
 
@@ -228,13 +221,13 @@ lex_file* lex_open_string(const char *str, size_t len, const char *name)
     lex->open_string_length = len;
     lex->open_string_pos    = 0;
 
-    lex->name    = util_strdup(name ? name : "<string-source>");
-    lex->line    = 1; /* we start counting at 1 */
-    lex->peekpos = 0;
-    lex->eof     = false;
-    lex->column  = 0;
+    lex->ctx.file    = util_strdup(name ? name : "<string-source>");
+    lex->ctx.line    = 1; /* we start counting at 1 */
+    lex->ctx.column  = 0;
+    lex->peekpos     = 0;
+    lex->eof         = false;
 
-    vec_push(lex_filenames, lex->name);
+    vec_push(lex_filenames, (char*)lex->ctx.file);
 
     return lex;
 }
@@ -272,13 +265,13 @@ void lex_close(lex_file *lex)
 static int lex_fgetc(lex_file *lex)
 {
     if (lex->file) {
-        lex->column++;
+        lex->ctx.column++;
         return fs_file_getc(lex->file);
     }
     if (lex->open_string) {
         if (lex->open_string_pos >= lex->open_string_length)
             return EOF;
-        lex->column++;
+        lex->ctx.column++;
         return lex->open_string[lex->open_string_pos++];
     }
     return EOF;
@@ -295,8 +288,8 @@ static int lex_try_trigraph(lex_file *lex, int old)
     int c2, c3;
     c2 = lex_fgetc(lex);
     if (!lex->push_line && c2 == '\n') {
-        lex->line++;
-        lex->column = 0;
+        lex->ctx.line++;
+        lex->ctx.column = 0;
     }
 
     if (c2 != '?') {
@@ -306,8 +299,8 @@ static int lex_try_trigraph(lex_file *lex, int old)
 
     c3 = lex_fgetc(lex);
     if (!lex->push_line && c3 == '\n') {
-        lex->line++;
-        lex->column = 0;
+        lex->ctx.line++;
+        lex->ctx.column = 0;
     }
 
     switch (c3) {
@@ -335,7 +328,7 @@ static int lex_try_digraph(lex_file *lex, int ch)
      * need to offset a \n the ungetch would recognize
      */
     if (!lex->push_line && c2 == '\n')
-        lex->line++;
+        lex->ctx.line++;
     if      (ch == '<' && c2 == ':')
         return '[';
     else if (ch == ':' && c2 == '>')
@@ -357,13 +350,13 @@ static int lex_getch(lex_file *lex)
     if (lex->peekpos) {
         lex->peekpos--;
         if (!lex->push_line && lex->peek[lex->peekpos] == '\n')
-            lex->line++;
+            lex->ctx.line++;
         return lex->peek[lex->peekpos];
     }
 
     ch = lex_fgetc(lex);
     if (!lex->push_line && ch == '\n')
-        lex->line++;
+        lex->ctx.line++;
     else if (ch == '?')
         return lex_try_trigraph(lex, ch);
     else if (!lex->flags.nodigraphs && (ch == '<' || ch == ':' || ch == '%'))
@@ -374,10 +367,10 @@ static int lex_getch(lex_file *lex)
 static void lex_ungetch(lex_file *lex, int ch)
 {
     lex->peek[lex->peekpos++] = ch;
-    lex->column--;
+    lex->ctx.column--;
     if (!lex->push_line && ch == '\n') {
-        lex->line--;
-        lex->column = 0;
+        lex->ctx.line--;
+        lex->ctx.column = 0;
     }
 }
 
@@ -428,7 +421,7 @@ static bool lex_try_pragma(lex_file *lex)
     if (lex->flags.preprocessing)
         return false;
 
-    line = lex->line;
+    line = lex->ctx.line;
 
     ch = lex_getch(lex);
     if (ch != '#') {
@@ -483,8 +476,8 @@ static bool lex_try_pragma(lex_file *lex)
             goto unroll;
     }
     else if (!strcmp(command, "file")) {
-        lex->name = util_strdup(param);
-        vec_push(lex_filenames, lex->name);
+        lex->ctx.file = util_strdup(param);
+        vec_push(lex_filenames, (char*)lex->ctx.file);
     }
     else if (!strcmp(command, "line")) {
         line = strtol(param, NULL, 0)-1;
@@ -492,7 +485,7 @@ static bool lex_try_pragma(lex_file *lex)
     else
         goto unroll;
 
-    lex->line = line;
+    lex->ctx.line = line;
     while (ch != '\n' && ch != EOF)
         ch = lex_getch(lex);
     vec_free(command);
@@ -529,7 +522,7 @@ unroll:
     }
     lex_ungetch(lex, '#');
 
-    lex->line = line;
+    lex->ctx.line = line;
     return false;
 }
 
@@ -884,7 +877,7 @@ static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
                         ch = 0;
                     else {
                         --u8len;
-                        lex->column += u8len;
+                        lex->ctx.column += u8len;
                         for (uc = 0; uc < u8len; ++uc)
                             lex_tokench(lex, u8buf[uc]);
                         /* the last character will be inserted with the tokench() call
@@ -1024,9 +1017,9 @@ int lex_do(lex_file *lex)
         return (lex->tok.ttype = ch);
     }
 
-    lex->sline = lex->line;
-    lex->tok.ctx.line = lex->sline;
-    lex->tok.ctx.file = lex->name;
+    lex->sline         = lex->ctx.line;
+    lex->tok.ctx->line = lex->sline;
+    lex->tok.ctx->file = lex->ctx.file;
 
     if (lex->eof)
         return (lex->tok.ttype = TOKEN_FATAL);
diff --git a/lexer.h b/lexer.h
index f51d56e..b73fa5f 100644
--- a/lexer.h
+++ b/lexer.h
@@ -41,7 +41,7 @@ struct token_s {
     struct token_s *prev;
 #endif
 
-    lex_ctx ctx;
+    lex_ctx *ctx;
 };
 
 #if 0
@@ -110,10 +110,15 @@ typedef struct lex_file_s {
     size_t      open_string_length;
     size_t      open_string_pos;
 
-    char   *name;
-    size_t  line;
     size_t  sline; /* line at the start of a token */
-    size_t  column;
+
+    /* 
+     * no more 'shallow' copies, instead all new instances
+     * of a lex_ctx will just point to this lex_file::ctx.
+     * i.e 4/8 byte pointer. Instead of every token getting
+     * a 'shallow' 12/24 byte structure.
+     */
+    lex_ctx ctx;
 
     int     peek[256];
     size_t  peekpos;
diff --git a/parser.c b/parser.c
index 159c7c9..4cfd87f 100644
--- a/parser.c
+++ b/parser.c
@@ -129,7 +129,7 @@ static void parseerror(parser_t *parser, const char *fmt, ...)
 {
     va_list ap;
     va_start(ap, fmt);
-    vcompile_error(parser->lex->tok.ctx, fmt, ap);
+    vcompile_error(*(parser->lex->tok.ctx), fmt, ap);
     va_end(ap);
 }
 
@@ -139,7 +139,7 @@ static bool GMQCC_WARN parsewarning(parser_t *parser, int warntype, const char *
     bool    r;
     va_list ap;
     va_start(ap, fmt);
-    r = vcompile_warning(parser->lex->tok.ctx, warntype, fmt, ap);
+    r = vcompile_warning(*(parser->lex->tok.ctx), warntype, fmt, ap);
     va_end(ap);
     return r;
 }
@@ -199,7 +199,7 @@ static bool parser_next(parser_t *parser)
 
 #define parser_tokval(p) ((p)->lex->tok.value)
 #define parser_token(p)  (&((p)->lex->tok))
-#define parser_ctx(p)    ((p)->lex->tok.ctx)
+#define parser_ctx(p)    (*((p)->lex->tok.ctx))
 
 static ast_value* parser_const_float(parser_t *parser, double d)
 {