X-Git-Url: https://git.xonotic.org/?a=blobdiff_plain;f=test.c;h=00080689540bdcc1744c52132aa9226052803706;hb=38f509077826ded67870f23d6b38f7c1e80cfa69;hp=535d366914e331af56156b4c6e172ceb73a1ce26;hpb=d1fd7d044ba1470dd6c66b1f265ec3aafb829b07;p=xonotic%2Fgmqcc.git

diff --git a/test.c b/test.c
index 535d366..0008068 100644
--- a/test.c
+++ b/test.c
@@ -25,13 +25,141 @@
 #include <sys/stat.h>
 #include <dirent.h>
 
-bool  opts_memchk = true;
+bool  opts_memchk = false;
 bool  opts_debug  = false;
 char *task_bins[] = {
     "./gmqcc",
     "./qcvm"
 };
 
+/*
+ * TODO: Windows version
+ * this implements a unique bi-directional popen-like function that
+ * allows reading data from both stdout and stderr. And writing to
+ * stdin :)
+ * 
+ * Example of use:
+ * FILE *handles[3] = task_popen("ls", "-l", "r");
+ * if (!handles) { perror("failed to open stdin/stdout/stderr to ls");
+ * // handles[0] = stdin
+ * // handles[1] = stdout
+ * // handles[2] = stderr
+ * 
+ * task_pclose(handles); // to close
+ */
+#ifndef _WIN32
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <unistd.h>
+typedef struct {
+    FILE *handles[3];
+    int   pipes  [3];
+    
+    int stderr_fd;
+    int stdout_fd;
+    int pid;
+} popen_t;
+
+FILE ** task_popen(const char *command, const char *mode) {
+    int     inhandle  [2];
+    int     outhandle [2];
+    int     errhandle [2];
+    int     trypipe;
+    
+    popen_t *data = mem_a(sizeof(popen_t));
+    
+    /*
+     * Parse the command now into a list for execv, this is a pain
+     * in the ass.
+     */
+    char  *line = (char*)command;
+    char **argv = NULL;
+    {
+        
+        while (*line != '\0') {
+            while (*line == ' ' || *line == '\t' || *line == '\n')
+                *line++ = '\0';
+            vec_push(argv, line);
+            
+            while (*line != '\0' && *line != ' ' &&
+                   *line != '\t' && *line != '\n') line++;
+        }
+        vec_push(argv, '\0');
+    }
+    
+    
+    if ((trypipe = pipe(inhandle))  < 0) goto task_popen_error_0;
+    if ((trypipe = pipe(outhandle)) < 0) goto task_popen_error_1;
+    if ((trypipe = pipe(errhandle)) < 0) goto task_popen_error_2;
+    
+    if ((data->pid = fork()) > 0) {
+        /* parent */
+        close(inhandle  [0]);
+        close(outhandle [1]);
+        close(errhandle [1]);
+        
+        data->pipes  [0] = inhandle [1];
+        data->pipes  [1] = outhandle[0];
+        data->pipes  [2] = errhandle[0];
+        data->handles[0] = fdopen(inhandle [1], "w");
+        data->handles[1] = fdopen(outhandle[0], mode);
+        data->handles[2] = fdopen(errhandle[0], mode);
+        
+        /* sigh */
+        if (argv)
+            vec_free(argv);
+        return data->handles;
+    } else if (data->pid == 0) {
+        /* child */
+        close(inhandle [1]);
+        close(outhandle[0]);
+        close(errhandle[0]);
+        
+        /* see piping documentation for this sillyness :P */
+        close(0), dup(inhandle [0]);
+        close(1), dup(outhandle[1]);
+        close(2), dup(errhandle[1]);
+        
+        execvp(*argv, argv);
+        exit(1);
+    } else {
+        /* fork failed */
+        goto task_popen_error_3;
+    }
+    
+    if (argv)
+        vec_free(argv);
+    return data->handles;
+        
+task_popen_error_3: close(errhandle[0]), close(errhandle[1]);
+task_popen_error_2: close(outhandle[0]), close(outhandle[1]);
+task_popen_error_1: close(inhandle [0]), close(inhandle [1]);
+task_popen_error_0:
+
+    if (argv)
+        vec_free(argv);
+    return NULL;
+}
+
+int task_pclose(FILE **handles) {
+    popen_t *data   = (popen_t*)handles;
+    int      status = 0;
+    
+    close(data->pipes[0]); /* stdin  */
+    close(data->pipes[1]); /* stdout */
+    close(data->pipes[2]); /* stderr */
+    
+    waitpid(data->pid, &status, 0);
+    
+    mem_d(data);
+    
+    return status;
+}
+#else
+
+#endif
+
 #define TASK_COMPILE 0
 #define TASK_EXECUTE 1
 
@@ -93,7 +221,8 @@ char *task_bins[] = {
  * 
  *      I:
  *          Used to specify the INPUT source file to operate on, this must be
- *          provided, this tag is NOT optional.
+ *          provided, this tag is NOT optional
+ *
  * 
  *  Notes:
  *      These tags have one-time use, using them more than once will result
@@ -110,15 +239,15 @@ char *task_bins[] = {
  *      this will result in the task failing.
  */
 typedef struct {
-    char *description;
-    char *failuremessage;
-    char *successmessage;
-    char *compileflags;
-    char *executeflags;
-    char *comparematch;
-    char *proceduretype;
-    char *sourcefile;
-    char *tempfilename;
+    char  *description;
+    char  *failuremessage;
+    char  *successmessage;
+    char  *compileflags;
+    char  *executeflags;
+    char  *proceduretype;
+    char  *sourcefile;
+    char  *tempfilename;
+    char **comparematch;
 } task_template_t;
 
 /*
@@ -138,7 +267,6 @@ bool task_template_generate(task_template_t *template, char tag, const char *fil
         case 'T': destval = &template->proceduretype;  break;
         case 'C': destval = &template->compileflags;   break;
         case 'E': destval = &template->executeflags;   break;
-        case 'M': destval = &template->comparematch;   break;
         case 'I': destval = &template->sourcefile;     break;
         default:
             con_printmsg(LVL_ERROR, __FILE__, __LINE__, "internal error",
@@ -234,7 +362,6 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp)
             case 'T':
             case 'C':
             case 'E':
-            case 'M':
             case 'I':
                 if (data[1] != ':') {
                     con_printmsg(LVL_ERROR, file, line, "template parse error",
@@ -251,6 +378,35 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp)
                 }
                 break;
             
+            /*
+             * Match requires it's own system since we allow multiple M's
+             * for multi-line matching.
+             */
+            case 'M':
+            {
+                char *value = &data[3];
+                if (data[1] != ':') {
+                    con_printmsg(LVL_ERROR, file, line, "template parse error",
+                        "expected `:` after `%c`",
+                        *data
+                    );
+                    goto failure;
+                }
+                
+                if (value && *value && (*value == ' ' || *value == '\t'))
+                    value++;
+    
+                /*
+                 * Value will contain a newline character at the end, we need to strip
+                 * this otherwise kaboom, seriously, kaboom :P
+                 */
+                *strrchr(value, '\n')='\0';
+                
+                vec_push(template->comparematch, util_strdup(value));
+                
+                break;
+            }
+            
             default:
                 con_printmsg(LVL_ERROR, file, line, "template parse error",
                     "invalid tag `%c`", *data
@@ -358,8 +514,8 @@ task_template_t *task_template_compile(const char *file, const char *dir) {
         goto success;
     } else if (!strcmp(template->proceduretype, "-execute")) {
         if (!template->executeflags) {
-            con_err("template compile error: %s missing `E:` tag (use `$null` for exclude)\n", file);
-            goto failure;
+            /* default to $null */
+            template->executeflags = util_strdup("$null");
         }
         if (!template->comparematch) {
             con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file);
@@ -396,16 +552,25 @@ void task_template_destroy(task_template_t **template) {
     if ((*template)->proceduretype)  mem_d((*template)->proceduretype);
     if ((*template)->compileflags)   mem_d((*template)->compileflags);
     if ((*template)->executeflags)   mem_d((*template)->executeflags);
-    if ((*template)->comparematch)   mem_d((*template)->comparematch);
     if ((*template)->sourcefile)     mem_d((*template)->sourcefile);
     
+    /*
+     * Delete all allocated string for task template then destroy the
+     * main vector.
+     */
+    {
+        size_t i = 0;
+        for (; i < vec_size((*template)->comparematch); i++)
+            mem_d((*template)->comparematch[i]);
+            
+        vec_free((*template)->comparematch);
+    }
+    
     /*
      * Nullify all the template members otherwise NULL comparision
      * checks will fail if template pointer is reused.
      */
     mem_d(*template);
-    task_template_nullify(*template);
-    *template = NULL;
 }
 
 /*
@@ -414,7 +579,12 @@ void task_template_destroy(task_template_t **template) {
  */
 typedef struct {
     task_template_t *template;
-    FILE            *handle;
+    FILE           **runhandles;
+    FILE            *stderrlog;
+    FILE            *stdoutlog;
+    char            *stdoutlogfile;
+    char            *stderrlogfile;
+    bool             compiled;
 } task_t;
 
 task_t *task_tasks = NULL;
@@ -449,9 +619,12 @@ bool task_propogate(const char *curdir) {
          * We made it here, which concludes the file/directory is not
          * actually a directory, so it must be a file :)
          */
-        if (strstr(files->d_name, ".tmpl")) {
-            con_out("compiling task template: %s/%s\n", curdir, files->d_name);
+        if (strstr(files->d_name, ".tmpl") == &files->d_name[strlen(files->d_name) - (sizeof(".tmpl") - 1)]) {
             task_template_t *template = task_template_compile(files->d_name, curdir);
+            char             buf[4096]; /* one page should be enough */
+            task_t           task;
+            
+            util_debug("TEST", "compiling task template: %s/%s\n", curdir, files->d_name);
             if (!template) {
                 con_err("error compiling task template: %s\n", files->d_name);
                 success = false;
@@ -463,13 +636,11 @@ bool task_propogate(const char *curdir) {
              */
             template->tempfilename = tempnam(curdir, "TMPDAT");
             
-            
             /*
              * Generate the command required to open a pipe to a process
              * which will be refered to with a handle in the task for
              * reading the data from the pipe.
              */
-            char     buf[4096]; /* one page should be enough */
             memset  (buf,0,sizeof(buf));
             snprintf(buf,  sizeof(buf), "%s %s/%s %s -o %s",
                 task_bins[TASK_COMPILE],
@@ -483,14 +654,34 @@ bool task_propogate(const char *curdir) {
              * The task template was compiled, now lets create a task from
              * the template data which has now been propogated.
              */
-            task_t task;
             task.template = template;
-            if (!(task.handle = popen(buf, "r"))) {
+            if (!(task.runhandles = task_popen(buf, "r"))) {
                 con_err("error opening pipe to process for test: %s\n", template->description);
                 success = false;
                 continue;
             }
-            con_out("executing test: `%s` [%s]\n", template->description, buf);
+            
+            util_debug("TEST", "executing test: `%s` [%s]\n", template->description, buf);
+            
+            /*
+             * Open up some file desciptors for logging the stdout/stderr
+             * to our own.
+             */
+            memset  (buf,0,sizeof(buf));
+            snprintf(buf,  sizeof(buf), "%s.stdout", template->tempfilename);
+            task.stdoutlogfile = util_strdup(buf);
+            if (!(task.stdoutlog     = fopen(buf, "w"))) {
+                con_err("error opening %s for stdout\n", buf);
+                continue;
+            }
+            
+            memset  (buf,0,sizeof(buf));
+            snprintf(buf,  sizeof(buf), "%s.stderr", template->tempfilename);
+            task.stderrlogfile = util_strdup(buf);
+            if (!(task.stderrlog     = fopen(buf, "w"))) {
+                con_err("error opening %s for stderr\n", buf);
+                continue;
+            }
             
             vec_push(task_tasks, task);
         }
@@ -518,7 +709,35 @@ void task_cleanup(const char *curdir) {
             if (remove(buffer))
                 con_err("error removing temporary file: %s\n", buffer);
             else
-                con_out("removed temporary file: %s\n", buffer);
+                util_debug("TEST", "removed temporary file: %s\n", buffer);
+        }
+    }
+    
+    closedir(dir);
+}
+
+/*
+ * Task precleanup removes any existing temporary files or log files
+ * left behind from a previous invoke of the test-suite.
+ */
+void task_precleanup(const char *curdir) {
+    DIR             *dir;
+    struct dirent   *files;
+    char             buffer[4096];
+
+    dir = opendir(curdir);
+    
+    while ((files = readdir(dir))) {
+        memset(buffer, 0, sizeof(buffer));
+        if (strstr(files->d_name, "TMP")     ||
+            strstr(files->d_name, ".stdout") ||
+            strstr(files->d_name, ".stderr"))
+        {
+            snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name);
+            if (remove(buffer))
+                con_err("error removing temporary file: %s\n", buffer);
+            else
+                util_debug("TEST", "removed temporary file: %s\n", buffer);
         }
     }
     
@@ -532,8 +751,37 @@ void task_destroy(const char *curdir) {
      * temporary files.
      */
     size_t i;
-    for (i = 0; i < vec_size(task_tasks); i++)
+    for (i = 0; i < vec_size(task_tasks); i++) {
+        /*
+         * Close any open handles to files or processes here.  It's mighty
+         * annoying to have to do all this cleanup work.
+         */
+        if (task_tasks[i].runhandles) task_pclose(task_tasks[i].runhandles);
+        if (task_tasks[i].stdoutlog)  fclose     (task_tasks[i].stdoutlog);
+        if (task_tasks[i].stderrlog)  fclose     (task_tasks[i].stderrlog);
+        
+        /*
+         * Only remove the log files if the test actually compiled otherwise
+         * forget about it.
+         */
+        if (task_tasks[i].compiled) {
+            if (remove(task_tasks[i].stdoutlogfile))
+                con_err("error removing stdout log file: %s\n", task_tasks[i].stdoutlogfile);
+            else
+                util_debug("TEST", "removed stdout log file: %s\n", task_tasks[i].stdoutlogfile);
+            
+            if (remove(task_tasks[i].stderrlogfile))
+                con_err("error removing stderr log file: %s\n", task_tasks[i].stderrlogfile);
+            else
+                util_debug("TEST", "removed stderr log file: %s\n", task_tasks[i].stderrlogfile);
+        }
+        
+        /* free util_strdup data for log files */
+        mem_d(task_tasks[i].stdoutlogfile);
+        mem_d(task_tasks[i].stderrlogfile);
+        
         task_template_destroy(&task_tasks[i].template);
+    }
     vec_free(task_tasks);
     
     /*
@@ -570,7 +818,7 @@ bool task_execute(task_template_t *template) {
         );
     }
     
-    con_out("executing qcvm: `%s` [%s]\n",
+    util_debug("TEST", "executing qcvm: `%s` [%s]\n",
         template->description,
         buffer
     );
@@ -584,37 +832,36 @@ bool task_execute(task_template_t *template) {
      * and handle accordingly.
      */
     {
-        char  *data  = NULL;
-        size_t size  = 0;
-        while (util_getline(&data, &size, execute) != EOF) {}
-        
-        if (!strcmp(data, "No main function found\n")) {
-            con_err("test failure: `%s` [%s] (No main function found)\n",
-                template->description,
-                (template->failuremessage) ?
-                template->failuremessage : "unknown"
-            );
-            pclose(execute);
-            return false;
+        char  *data    = NULL;
+        size_t size    = 0;
+        size_t compare = 0;
+        while (util_getline(&data, &size, execute) != EOF) {
+            if (!strcmp(data, "No main function found\n")) {
+                con_err("test failure: `%s` [%s] (No main function found)\n",
+                    template->description,
+                    (template->failuremessage) ?
+                    template->failuremessage : "unknown"
+                );
+                pclose(execute);
+                return false;
+            }
+            
+            /* 
+             * Trim newlines from data since they will just break our
+             * ability to properly validate matches.
+             */
+            if  (strrchr(data, '\n'))
+                *strrchr(data, '\n') = '\0';
+            
+            
+            /*
+             * We only care about the last line from the output for now
+             * implementing multi-line match is TODO.
+             */
+            success = !!!(strcmp(data, template->comparematch[compare++]));
         }
-        
-        /* 
-         * Trim newlines from data since they will just break our
-         * ability to properly validate matches.
-         */
-        if  (strrchr(data, '\n'))
-            *strrchr(data, '\n') = '\0';
-        
-        /* null list */
-        if (!strcmp(template->comparematch, "$null"))
-            success = true;
-        
-        /*
-         * We only care about the last line from the output for now
-         * implementing multi-line match is TODO.
-         */
-        if (!strcmp(data, template->comparematch))
-            success = true;
+        mem_d(data);
+        data = NULL;
     }
     pclose(execute);
     return success;
@@ -626,40 +873,51 @@ bool task_execute(task_template_t *template) {
  * execution this takes more work since a task needs to be generated
  * from thin air and executed INLINE.
  */
-void task_schedualize(const char *curdir) {
-    bool   execute = false;
-    char  *back    = NULL;
-    char  *data    = NULL;
-    size_t size    = 0;
+void task_schedualize() {
+    bool   execute  = false;
+    char  *data     = NULL;
+    size_t size     = 0;
     size_t i;
     
     for (i = 0; i < vec_size(task_tasks); i++) {
         /*
-        * Generate a task from thin air if it requires execution in
-        * the QCVM.
-        */
+         * Generate a task from thin air if it requires execution in
+         * the QCVM.
+         */
         if (!strcmp(task_tasks[i].template->proceduretype, "-execute"))
             execute = true;
             
-        while (util_getline(&data, &size, task_tasks[i].handle) != EOF) {
-            back = data;
-            /* chances are we want to print errors */
+        /*
+         * We assume it compiled before we actually compiled :).  On error
+         * we change the value
+         */
+        task_tasks[i].compiled = true;
+        
+        /*
+         * Read data from stdout first and pipe that stuff into a log file
+         * then we do the same for stderr.
+         */    
+        while (util_getline(&data, &size, task_tasks[i].runhandles[1]) != EOF) {
+            fputs(data, task_tasks[i].stdoutlog);
+            fflush(task_tasks[i].stdoutlog);
+        }
+        while (util_getline(&data, &size, task_tasks[i].runhandles[2]) != EOF) {
+            /*
+             * If a string contains an error we just dissalow execution
+             * of it in the vm.
+             * 
+             * TODO: make this more percise, e.g if we print a warning
+             * that refers to a variable named error, or something like
+             * that .. then this will blowup :P
+             */
             if (strstr(data, "error")) {
-                con_out("compile failed: %s\n",
-                    /* strip the newline from the end */
-                    (*strrchr(data, '\n')='\0')
-                );
-                
-                /*
-                 * The compilation failed which means it cannot be executed
-                 * as the file simply will not exist.
-                 */
-                execute = false;
-                break;
+                execute                = false;
+                task_tasks[i].compiled = false;
             }
+            
+            fputs(data, task_tasks[i].stderrlog);
+            fflush(task_tasks[i].stdoutlog);
         }
-        if (back)
-            mem_d(back);
         
         /*
          * If we can execute we do so after all data has been read and
@@ -673,10 +931,12 @@ void task_schedualize(const char *curdir) {
          * in the virtual machine.
          */
         if (!task_execute(task_tasks[i].template)) {
-            con_err("test failure: `%s` [%s]\n",
+            con_err("test failure: `%s` [%s] see %s.stdout and %s.stderr\n",
                 task_tasks[i].template->description,
                 (task_tasks[i].template->failuremessage) ?
-                task_tasks[i].template->failuremessage : "unknown"
+                task_tasks[i].template->failuremessage : "unknown",
+                task_tasks[i].template->tempfilename,
+                task_tasks[i].template->tempfilename
             );
             continue;
         }
@@ -687,16 +947,29 @@ void task_schedualize(const char *curdir) {
             task_tasks[i].template->successmessage  : "unknown"
         );
     }
-    if (back)
-        mem_d(back);
+    mem_d(data);
 }
 
-int main(int argc, char **argv) {
-    con_init();
-    if (!task_propogate("tests")) {
+/*
+ * This is the heart of the whole test-suite process.  This cleans up
+ * any existing temporary files left behind as well as log files left
+ * behind.  Then it propogates a list of tests from `curdir` by scaning
+ * it for template files and compiling them into tasks, in which it
+ * schedualizes them (executes them) and actually reports errors and
+ * what not.  It then proceeds to destroy the tasks and return memory
+ * it's the engine :)
+ * 
+ * It returns true of tests could be propogated, otherwise it returns
+ * false.
+ * 
+ * It expects con_init() was called before hand.
+ */
+bool test_perform(const char *curdir) {
+    task_precleanup(curdir);
+    if (!task_propogate(curdir)) {
         con_err("error: failed to propogate tasks\n");
-        task_destroy("tests");
-        return -1;
+        task_destroy(curdir);
+        return false;
     }
     /*
      * If we made it here all tasks where propogated from their resultant
@@ -705,9 +978,82 @@ int main(int argc, char **argv) {
      * it's designed to prevent lock contention, and possible syncronization
      * issues.
      */
-    task_schedualize("tests");
-    task_destroy("tests");
+    task_schedualize();
+    task_destroy(curdir);
     
+    return true;
+}
+
+/*
+ * Fancy GCC-like LONG parsing allows things like --opt=param with
+ * assignment operator.  This is used for redirecting stdout/stderr
+ * console to specific files of your choice.
+ */
+static bool parsecmd(const char *optname, int *argc_, char ***argv_, char **out, int ds, bool split) {
+    int  argc   = *argc_;
+    char **argv = *argv_;
+
+    size_t len = strlen(optname);
+
+    if (strncmp(argv[0]+ds, optname, len))
+        return false;
+
+    /* it's --optname, check how the parameter is supplied */
+    if (argv[0][ds+len] == '=') {
+        *out = argv[0]+ds+len+1;
+        return true;
+    }
+
+    if (!split || argc < ds) /* no parameter was provided, or only single-arg form accepted */
+        return false;
+
+    /* using --opt param */
+    *out = argv[1];
+    --*argc_;
+    ++*argv_;
+    return true;
+}
+
+int main(int argc, char **argv) {
+    char *redirout = (char*)stdout;
+    char *redirerr = (char*)stderr;
+    con_init();
+    
+    /*
+     * Command line option parsing commences now We only need to support
+     * a few things in the test suite.
+     */
+    while (argc > 1) {
+        ++argv;
+        --argc;
+
+        if (argv[0][0] == '-') {
+            if (parsecmd("redirout", &argc, &argv, &redirout, 1, false))
+                continue;
+            if (parsecmd("redirerr", &argc, &argv, &redirerr, 1, false))
+                continue;
+            
+            con_change(redirout, redirerr);
+
+            if (!strcmp(argv[0]+1, "debug")) {
+                opts_debug = true;
+                continue;
+            }
+            if (!strcmp(argv[0]+1, "memchk")) {
+                opts_memchk = true;
+                continue;
+            }
+            if (!strcmp(argv[0]+1, "nocolor")) {
+                con_color(0);
+                continue;
+            }
+            
+            con_err("invalid argument %s\n", argv[0]+1);
+            return -1;
+        }
+    }
+    con_change(redirout, redirerr);
+    test_perform("tests");
     util_meminfo();
     return 0;
 }