]> git.xonotic.org Git - xonotic/gmqcc.git/blobdiff - test.c
Cleanups
[xonotic/gmqcc.git] / test.c
diff --git a/test.c b/test.c
old mode 100755 (executable)
new mode 100644 (file)
index f263cea..629a10e
--- a/test.c
+++ b/test.c
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#include "gmqcc.h"
+#include <stdlib.h>
+#include <string.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 
+#include "gmqcc.h"
+
 opts_cmd_t opts;
 
-const char *task_bins[] = {
+static const char *task_bins[] = {
     "./gmqcc",
     "./qcvm"
 };
@@ -60,7 +63,7 @@ typedef struct {
     int pid;
 } popen_t;
 
-FILE ** task_popen(const char *command, const char *mode) {
+static FILE ** task_popen(const char *command, const char *mode) {
     int     inhandle  [2];
     int     outhandle [2];
     int     errhandle [2];
@@ -101,13 +104,13 @@ FILE ** task_popen(const char *command, const char *mode) {
         data->pipes  [0] = inhandle [1];
         data->pipes  [1] = outhandle[0];
         data->pipes  [2] = errhandle[0];
+
         data->handles[0] = fdopen(inhandle [1], "w");
         data->handles[1] = fdopen(outhandle[0], mode);
         data->handles[2] = fdopen(errhandle[0], mode);
 
         /* sigh */
-        if (argv)
-            vec_free(argv);
+        vec_free(argv);
         return data->handles;
     } else if (data->pid == 0) {
         /* child */
@@ -116,9 +119,9 @@ FILE ** task_popen(const char *command, const char *mode) {
         close(errhandle[0]);
 
         /* see piping documentation for this sillyness :P */
-        close(0), dup(inhandle [0]);
-        close(1), dup(outhandle[1]);
-        close(2), dup(errhandle[1]);
+        dup2(inhandle [0], 0);
+        dup2(outhandle[1], 1);
+        dup2(errhandle[1], 2);
 
         execvp(*argv, argv);
         exit(EXIT_FAILURE);
@@ -132,12 +135,11 @@ task_popen_error_2: close(outhandle[0]), close(outhandle[1]);
 task_popen_error_1: close(inhandle [0]), close(inhandle [1]);
 task_popen_error_0:
 
-    if (argv)
-        vec_free(argv);
+    vec_free(argv);
     return NULL;
 }
 
-int task_pclose(FILE **handles) {
+static int task_pclose(FILE **handles) {
     popen_t *data   = (popen_t*)handles;
     int      status = 0;
 
@@ -158,7 +160,7 @@ int task_pclose(FILE **handles) {
         char  name_out[L_tmpnam];
     } popen_t;
 
-    FILE **task_popen(const char *command, const char *mode) {
+    static FILE **task_popen(const char *command, const char *mode) {
         char    *cmd  = NULL;
         popen_t *open = (popen_t*)mem_a(sizeof(popen_t));
 
@@ -179,7 +181,7 @@ int task_pclose(FILE **handles) {
         return open->handles;
     }
 
-    void task_pclose(FILE **files) {
+    static void task_pclose(FILE **files) {
         popen_t *open = ((popen_t*)files);
         fs_file_close(files[1]);
         fs_file_close(files[2]);
@@ -281,7 +283,7 @@ typedef struct {
  * This is very much like a compiler code generator :-).  This generates
  * a value from some data observed from the compiler.
  */
-bool task_template_generate(task_template_t *tmpl, char tag, const char *file, size_t line, char *value, size_t *pad) {
+static bool task_template_generate(task_template_t *tmpl, char tag, const char *file, size_t line, char *value, size_t *pad) {
     size_t desclen = 0;
     size_t filelen = 0;
     char **destval = NULL;
@@ -297,7 +299,7 @@ bool task_template_generate(task_template_t *tmpl, char tag, const char *file, s
         case 'I': destval = &tmpl->sourcefile;     break;
         case 'F': destval = &tmpl->testflags;      break;
         default:
-            con_printmsg(LVL_ERROR, __FILE__, __LINE__, "internal error",
+            con_printmsg(LVL_ERROR, __FILE__, __LINE__, 0, "internal error",
                 "invalid tag `%c:` during code generation\n",
                 tag
             );
@@ -309,7 +311,7 @@ bool task_template_generate(task_template_t *tmpl, char tag, const char *file, s
      * assigned value.
      */
     if (*destval) {
-        con_printmsg(LVL_ERROR, file, line, "compile error",
+        con_printmsg(LVL_ERROR, file, line, 0, /*TODO: column for match*/ "compile error",
             "tag `%c:` already assigned value: %s\n",
             tag, *destval
         );
@@ -322,6 +324,8 @@ bool task_template_generate(task_template_t *tmpl, char tag, const char *file, s
      */
     if (value && *value && (*value == ' ' || *value == '\t'))
         value++;
+    else if (!value)
+        exit(EXIT_FAILURE);
 
     /*
      * Value will contain a newline character at the end, we need to strip
@@ -329,8 +333,6 @@ bool task_template_generate(task_template_t *tmpl, char tag, const char *file, s
      */
     if (strchr(value, '\n'))
         *strrchr(value, '\n')='\0';
-    else /* cppcheck: possible nullpointer dereference */
-        exit(EXIT_FAILURE);
 
     /*
      * Now allocate and set the actual value for the specific tag. Which
@@ -354,7 +356,7 @@ bool task_template_generate(task_template_t *tmpl, char tag, const char *file, s
     return true;
 }
 
-bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size_t *pad) {
+static bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size_t *pad) {
     char  *data = NULL;
     char  *back = NULL;
     size_t size = 0;
@@ -377,7 +379,7 @@ bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size
              */
             case '/':
                 if (data[1] != '/') {
-                    con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
+                    con_printmsg(LVL_ERROR, file, line, 0, /*TODO: column for match*/ "tmpl parse error",
                         "invalid character `/`, perhaps you meant `//` ?");
 
                     mem_d(back);
@@ -407,14 +409,14 @@ bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size
             case 'I':
             case 'F':
                 if (data[1] != ':') {
-                    con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
+                    con_printmsg(LVL_ERROR, file, line, 0, /*TODO: column for match*/ "tmpl parse error",
                         "expected `:` after `%c`",
                         *data
                     );
                     goto failure;
                 }
                 if (!task_template_generate(tmpl, *data, file, line, &data[3], pad)) {
-                    con_printmsg(LVL_ERROR, file, line, "tmpl compile error",
+                    con_printmsg(LVL_ERROR, file, line, 0, /*TODO: column for match*/ "tmpl compile error",
                         "failed to generate for given task\n"
                     );
                     goto failure;
@@ -429,14 +431,14 @@ bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size
             {
                 char *value = &data[3];
                 if (data[1] != ':') {
-                    con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
+                    con_printmsg(LVL_ERROR, file, line, 0, /*TODO: column for match*/ "tmpl parse error",
                         "expected `:` after `%c`",
                         *data
                     );
                     goto failure;
                 }
 
-                if (value && *value && (*value == ' ' || *value == '\t'))
+                if (value && (*value == ' ' || *value == '\t'))
                     value++;
 
                 /*
@@ -454,7 +456,7 @@ bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size
             }
 
             default:
-                con_printmsg(LVL_ERROR, file, line, "tmpl parse error",
+                con_printmsg(LVL_ERROR, file, line, 0, /*TODO: column for match*/ "tmpl parse error",
                     "invalid tag `%c`", *data
                 );
                 goto failure;
@@ -471,8 +473,7 @@ bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size
     return true;
 
 failure:
-    if (back)
-        mem_d (back);
+    mem_d (back);
     return false;
 }
 
@@ -480,7 +481,7 @@ failure:
  * Nullifies the template data: used during initialization of a new
  * template and free.
  */
-void task_template_nullify(task_template_t *tmpl) {
+static void task_template_nullify(task_template_t *tmpl) {
     if (!tmpl)
         return;
 
@@ -495,7 +496,7 @@ void task_template_nullify(task_template_t *tmpl) {
     tmpl->testflags      = NULL;
 }
 
-task_template_t *task_template_compile(const char *file, const char *dir, size_t *pad) {
+static task_template_t *task_template_compile(const char *file, const char *dir, size_t *pad) {
     /* a page should be enough */
     char             fullfile[4096];
     size_t           filepadd = 0;
@@ -581,6 +582,13 @@ task_template_t *task_template_compile(const char *file, const char *dir, size_t
             con_err("template compile warning: %s erroneous tag `E:` when only failing\n", file);
         if (tmpl->comparematch)
             con_err("template compile warning: %s erroneous tag `M:` when only failing\n", file);
+    } else if (!strcmp(tmpl->proceduretype, "-diagnostic")) {
+        if (tmpl->executeflags)
+            con_err("template compile warning: %s erroneous tag `E:` when only diagnostic\n", file);
+        if (!tmpl->comparematch) {
+            con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file);
+            goto failure;
+        }
     } else if (!strcmp(tmpl->proceduretype, "-pp")) {
         if (tmpl->executeflags)
             con_err("template compile warning: %s erroneous tag `E:` when only preprocessing\n", file);
@@ -609,7 +617,7 @@ failure:
     return NULL;
 }
 
-void task_template_destroy(task_template_t **tmpl) {
+static void task_template_destroy(task_template_t **tmpl) {
     if (!tmpl)
         return;
 
@@ -637,6 +645,7 @@ void task_template_destroy(task_template_t **tmpl) {
      * Nullify all the template members otherwise NULL comparision
      * checks will fail if tmpl pointer is reused.
      */
+    mem_d((*tmpl)->tempfilename);
     mem_d(*tmpl);
 }
 
@@ -654,185 +663,217 @@ typedef struct {
     bool             compiled;
 } task_t;
 
-task_t *task_tasks = NULL;
+static task_t *task_tasks = NULL;
 
 /*
  * Read a directory and searches for all template files in it
  * which is later used to run all tests.
  */
-bool task_propagate(const char *curdir, size_t *pad, const char *defs) {
+static bool task_propagate(const char *curdir, size_t *pad, const char *defs) {
     bool             success = true;
     DIR             *dir;
     struct dirent   *files;
     struct stat      directory;
     char             buffer[4096];
     size_t           found = 0;
+    char           **directories = NULL;
+    char            *claim = util_strdup(curdir);
+    size_t           i;
+    
+    vec_push(directories, claim);
+    dir = fs_dir_open(claim);
 
-    dir = fs_dir_open(curdir);
-
+    /*
+     * Generate a list of subdirectories since we'll be checking them too
+     * for tmpl files.
+     */
     while ((files = fs_dir_read(dir))) {
-        util_snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name);
-
-        if (stat(buffer, &directory) == -1) {
-            con_err("internal error: stat failed, aborting\n");
-            abort();
+        util_asprintf(&claim, "%s/%s", curdir, files->d_name);
+        if (stat(claim, &directory) == -1)
+            return false;
+        
+        if (S_ISDIR(directory.st_mode) && files->d_name[0] != '.') {
+            vec_push(directories, claim);
+        } else {
+            mem_d(claim);
+            claim = NULL;
         }
+    }
+    fs_dir_close(dir);
 
-        /* skip directories */
-        if (S_ISDIR(directory.st_mode))
-            continue;
+    /*
+     * Now do all the work, by touching all the directories inside
+     * test as well and compile the task templates into data we can
+     * use to run the tests.
+     */
+    for (i = 0; i < vec_size(directories); i++) {
+        dir = fs_dir_open(directories[i]);
+        
+        while ((files = fs_dir_read(dir))) {
+            util_snprintf(buffer, sizeof(buffer), "%s/%s", directories[i], files->d_name);
+            if (stat(buffer, &directory) == -1) {
+                con_err("internal error: stat failed, aborting\n");
+                abort();
+            }
 
-        /*
-         * We made it here, which concludes the file/directory is not
-         * actually a directory, so it must be a file :)
-         */
-        if (strcmp(files->d_name + strlen(files->d_name) - 5, ".tmpl") == 0) {
-            task_template_t *tmpl = task_template_compile(files->d_name, curdir, pad);
-            char             buf[4096]; /* one page should be enough */
-            char            *qcflags = NULL;
-            task_t           task;
-
-            util_debug("TEST", "compiling task template: %s/%s\n", curdir, files->d_name);
-            found ++;
-            if (!tmpl) {
-                con_err("error compiling task template: %s\n", files->d_name);
-                success = false;
+            if (S_ISDIR(directory.st_mode))
                 continue;
-            }
-            /*
-             * Generate a temportary file name for the output binary
-             * so we don't trample over an existing one.
-             */
-            tmpl->tempfilename = NULL;
-            util_asprintf(&tmpl->tempfilename, "%s/TMPDAT.%s", curdir, files->d_name);
 
             /*
-             * Additional QCFLAGS enviroment variable may be used
-             * to test compile flags for all tests.  This needs to be
-             * BEFORE other flags (so that the .tmpl can override them)
+             * We made it here, which concludes the file/directory is not
+             * actually a directory, so it must be a file :)
              */
-            #ifdef _MSC_VER
-            {
-                char   buffer[4096];
-                size_t size;
-                getenv_s(&size, buffer, sizeof(buffer), "QCFLAGS");
-                qcflags = buffer;
-            }
-            #else
-            qcflags = getenv("QCFLAGS");
-            #endif
+            if (strcmp(files->d_name + strlen(files->d_name) - 5, ".tmpl") == 0) {
+                task_template_t *tmpl = task_template_compile(files->d_name, directories[i], pad);
+                char             buf[4096]; /* one page should be enough */
+                char            *qcflags = NULL;
+                task_t           task;
+
+                util_debug("TEST", "compiling task template: %s/%s\n", directories[i], files->d_name);
+                found ++;
+                if (!tmpl) {
+                    con_err("error compiling task template: %s\n", files->d_name);
+                    success = false;
+                    continue;
+                }
+                /*
+                 * Generate a temportary file name for the output binary
+                 * so we don't trample over an existing one.
+                 */
+                tmpl->tempfilename = NULL;
+                util_asprintf(&tmpl->tempfilename, "%s/TMPDAT.%s", directories[i], files->d_name);
 
-            /*
-             * Generate the command required to open a pipe to a process
-             * which will be refered to with a handle in the task for
-             * reading the data from the pipe.
-             */
-            if (strcmp(tmpl->proceduretype, "-pp")) {
-                if (qcflags) {
-                    if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
-                        util_snprintf(buf, sizeof(buf), "%s %s/%s %s %s -o %s",
-                            task_bins[TASK_COMPILE],
-                            curdir,
-                            tmpl->sourcefile,
-                            qcflags,
-                            tmpl->compileflags,
-                            tmpl->tempfilename
-                        );
+                /*
+                 * Additional QCFLAGS enviroment variable may be used
+                 * to test compile flags for all tests.  This needs to be
+                 * BEFORE other flags (so that the .tmpl can override them)
+                 */
+                #ifdef _MSC_VER
+                {
+                    char   buffer[4096];
+                    size_t size;
+                    getenv_s(&size, buffer, sizeof(buffer), "QCFLAGS");
+                    qcflags = buffer;
+                }
+                #else
+                qcflags = getenv("QCFLAGS");
+                #endif
+
+                /*
+                 * Generate the command required to open a pipe to a process
+                 * which will be refered to with a handle in the task for
+                 * reading the data from the pipe.
+                 */
+                if (strcmp(tmpl->proceduretype, "-pp")) {
+                    if (qcflags) {
+                        if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
+                            util_snprintf(buf, sizeof(buf), "%s %s/%s %s %s -o %s",
+                                task_bins[TASK_COMPILE],
+                                directories[i],
+                                tmpl->sourcefile,
+                                qcflags,
+                                tmpl->compileflags,
+                                tmpl->tempfilename
+                            );
+                        } else {
+                            util_snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s %s -o %s",
+                                task_bins[TASK_COMPILE],
+                                curdir,
+                                defs,
+                                directories[i],
+                                tmpl->sourcefile,
+                                qcflags,
+                                tmpl->compileflags,
+                                tmpl->tempfilename
+                            );
+                        }
                     } else {
-                        util_snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s %s -o %s",
-                            task_bins[TASK_COMPILE],
-                            curdir,
-                            defs,
-                            curdir,
-                            tmpl->sourcefile,
-                            qcflags,
-                            tmpl->compileflags,
-                            tmpl->tempfilename
-                        );
+                        if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
+                            util_snprintf(buf, sizeof(buf), "%s %s/%s %s -o %s",
+                                task_bins[TASK_COMPILE],
+                                directories[i],
+                                tmpl->sourcefile,
+                                tmpl->compileflags,
+                                tmpl->tempfilename
+                            );
+                        } else {
+                            util_snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s -o %s",
+                                task_bins[TASK_COMPILE],
+                                curdir,
+                                defs,
+                                directories[i],
+                                tmpl->sourcefile,
+                                tmpl->compileflags,
+                                tmpl->tempfilename
+                            );
+                        }
                     }
                 } else {
+                    /* Preprocessing (qcflags mean shit all here we don't allow them) */
                     if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
-                        util_snprintf(buf, sizeof(buf), "%s %s/%s %s -o %s",
+                        util_snprintf(buf, sizeof(buf), "%s -E %s/%s -o %s",
                             task_bins[TASK_COMPILE],
-                            curdir,
+                            directories[i],
                             tmpl->sourcefile,
-                            tmpl->compileflags,
                             tmpl->tempfilename
                         );
                     } else {
-                        util_snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s -o %s",
+                        util_snprintf(buf, sizeof(buf), "%s -E %s/%s %s/%s -o %s",
                             task_bins[TASK_COMPILE],
                             curdir,
                             defs,
-                            curdir,
+                            directories[i],
                             tmpl->sourcefile,
-                            tmpl->compileflags,
                             tmpl->tempfilename
                         );
                     }
                 }
-            } else {
-                /* Preprocessing (qcflags mean shit all here we don't allow them) */
-                if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) {
-                    util_snprintf(buf, sizeof(buf), "%s -E %s/%s -o %s",
-                        task_bins[TASK_COMPILE],
-                        curdir,
-                        tmpl->sourcefile,
-                        tmpl->tempfilename
-                    );
-                } else {
-                    util_snprintf(buf, sizeof(buf), "%s -E %s/%s %s/%s -o %s",
-                        task_bins[TASK_COMPILE],
-                        curdir,
-                        defs,
-                        curdir,
-                        tmpl->sourcefile,
-                        tmpl->tempfilename
-                    );
+
+                /*
+                 * The task template was compiled, now lets create a task from
+                 * the template data which has now been propagated.
+                 */
+                task.tmpl = tmpl;
+                if (!(task.runhandles = task_popen(buf, "r"))) {
+                    con_err("error opening pipe to process for test: %s\n", tmpl->description);
+                    success = false;
+                    continue;
                 }
-            }
 
-            /*
-             * The task template was compiled, now lets create a task from
-             * the template data which has now been propagated.
-             */
-            task.tmpl = tmpl;
-            if (!(task.runhandles = task_popen(buf, "r"))) {
-                con_err("error opening pipe to process for test: %s\n", tmpl->description);
-                success = false;
-                continue;
-            }
+                util_debug("TEST", "executing test: `%s` [%s]\n", tmpl->description, buf);
 
-            util_debug("TEST", "executing test: `%s` [%s]\n", tmpl->description, buf);
+                /*
+                 * Open up some file desciptors for logging the stdout/stderr
+                 * to our own.
+                 */
+                util_snprintf(buf,  sizeof(buf), "%s.stdout", tmpl->tempfilename);
+                task.stdoutlogfile = util_strdup(buf);
+                if (!(task.stdoutlog     = fs_file_open(buf, "w"))) {
+                    con_err("error opening %s for stdout\n", buf);
+                    continue;
+                }
 
-            /*
-             * Open up some file desciptors for logging the stdout/stderr
-             * to our own.
-             */
-            util_snprintf(buf,  sizeof(buf), "%s.stdout", tmpl->tempfilename);
-            task.stdoutlogfile = util_strdup(buf);
-            if (!(task.stdoutlog     = fs_file_open(buf, "w"))) {
-                con_err("error opening %s for stdout\n", buf);
-                continue;
-            }
+                util_snprintf(buf,  sizeof(buf), "%s.stderr", tmpl->tempfilename);
+                task.stderrlogfile = util_strdup(buf);
+                if (!(task.stderrlog = fs_file_open(buf, "w"))) {
+                    con_err("error opening %s for stderr\n", buf);
+                    continue;
+                }
 
-            util_snprintf(buf,  sizeof(buf), "%s.stderr", tmpl->tempfilename);
-            task.stderrlogfile = util_strdup(buf);
-            if (!(task.stderrlog = fs_file_open(buf, "w"))) {
-                con_err("error opening %s for stderr\n", buf);
-                continue;
+                vec_push(task_tasks, task);
             }
-
-            vec_push(task_tasks, task);
         }
+        
+        fs_dir_close(dir);
+        mem_d(directories[i]); /* free claimed memory */
     }
+    vec_free(directories);
 
     util_debug("TEST", "compiled %d task template files out of %d\n",
         vec_size(task_tasks),
         found
     );
 
-    fs_dir_close(dir);
     return success;
 }
 
@@ -840,7 +881,7 @@ bool task_propagate(const char *curdir, size_t *pad, const char *defs) {
  * Task precleanup removes any existing temporary files or log files
  * left behind from a previous invoke of the test-suite.
  */
-void task_precleanup(const char *curdir) {
+static void task_precleanup(const char *curdir) {
     DIR             *dir;
     struct dirent   *files;
     char             buffer[4096];
@@ -863,7 +904,7 @@ void task_precleanup(const char *curdir) {
     fs_dir_close(dir);
 }
 
-void task_destroy(void) {
+static void task_destroy(void) {
     /*
      * Free all the data in the task list and finally the list itself
      * then proceed to cleanup anything else outside the program like
@@ -894,7 +935,7 @@ void task_destroy(void) {
             else
                 util_debug("TEST", "removed stderr log file: %s\n", task_tasks[i].stderrlogfile);
 
-            remove(task_tasks[i].tmpl->tempfilename);
+            (void)!remove(task_tasks[i].tmpl->tempfilename);
         }
 
         /* free util_strdup data for log files */
@@ -912,14 +953,16 @@ void task_destroy(void) {
  * messages IF the procedure type is -execute, otherwise it matches
  * the preprocessor output.
  */
-bool task_trymatch(task_template_t *tmpl, char ***line) {
-    bool     success = true;
-    bool     preprocessing = false;
-    FILE    *execute;
-    char     buffer[4096];
+static bool task_trymatch(size_t i, char ***line) {
+    bool             success = true;
+    bool             process = true;
+    FILE            *execute;
+    char             buffer[4096];
+    task_template_t *tmpl = task_tasks[i].tmpl;
+    
     memset  (buffer,0,sizeof(buffer));
 
-    if (strcmp(tmpl->proceduretype, "-pp")) {
+    if (!strcmp(tmpl->proceduretype, "-execute")) {
         /*
          * Drop the execution flags for the QCVM if none where
          * actually specified.
@@ -945,7 +988,7 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
         execute = popen(buffer, "r");
         if (!execute)
             return false;
-    } else {
+    } else if (!strcmp(tmpl->proceduretype, "-pp")) {
         /*
          * we're preprocessing, which means we need to read int
          * the produced file and do some really weird shit.
@@ -953,7 +996,17 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
         if (!(execute = fs_file_open(tmpl->tempfilename, "r")))
             return false;
 
-        preprocessing = true;
+        process = false;
+    } else {
+        /*
+         * we're testing diagnostic output, which means it will be
+         * in runhandles[2] (stderr) since that is where the compiler
+         * puts it's errors.
+         */
+        if (!(execute = fs_file_open(task_tasks[i].stderrlogfile, "r")))
+            return false;
+        
+        process = false;
     }
 
     /*
@@ -964,13 +1017,14 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
         char  *data    = NULL;
         size_t size    = 0;
         size_t compare = 0;
+        
         while (fs_file_getline(&data, &size, execute) != EOF) {
             if (!strcmp(data, "No main function found\n")) {
                 con_err("test failure: `%s` (No main function found) [%s]\n",
                     tmpl->description,
                     tmpl->rulesfile
                 );
-                if (preprocessing)
+                if (!process)
                     fs_file_close(execute);
                 else
                     pclose(execute);
@@ -983,6 +1037,20 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
              */
             if  (strrchr(data, '\n'))
                 *strrchr(data, '\n') = '\0';
+                
+            /*
+             * We remove the file/directory and stuff from the error
+             * match messages when testing diagnostics.
+             */
+            if(!strcmp(tmpl->proceduretype, "-diagnostic")) {
+                if (strstr(data, "there have been errors, bailing out"))
+                    continue; /* ignore it */
+                if (strstr(data, ": error: ")) {
+                    char *claim = util_strdup(data + (strstr(data, ": error: ") - data) + 9);
+                    mem_d(data);
+                    data = claim;
+                }
+            }
 
             /*
              * If data is just null now, that means the line was an empty
@@ -992,10 +1060,11 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
                 continue;
 
             if (vec_size(tmpl->comparematch) > compare) {
-                if (strcmp(data, tmpl->comparematch[compare++]))
+                if (strcmp(data, tmpl->comparematch[compare++])) {
                     success = false;
+                }
             } else {
-                    success = false;
+                success = false;
             }
 
             /*
@@ -1012,7 +1081,7 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
         data = NULL;
     }
 
-    if (!preprocessing)
+    if (process)
         pclose(execute);
     else
         fs_file_close(execute);
@@ -1020,13 +1089,15 @@ bool task_trymatch(task_template_t *tmpl, char ***line) {
     return success;
 }
 
-const char *task_type(task_template_t *tmpl) {
+static const char *task_type(task_template_t *tmpl) {
     if (!strcmp(tmpl->proceduretype, "-pp"))
         return "type: preprocessor";
     if (!strcmp(tmpl->proceduretype, "-execute"))
         return "type: execution";
     if (!strcmp(tmpl->proceduretype, "-compile"))
         return "type: compile";
+    if (!strcmp(tmpl->proceduretype, "-diagnostic"))
+        return "type: diagnostic";
     return "type: fail";
 }
 
@@ -1037,7 +1108,7 @@ const char *task_type(task_template_t *tmpl) {
  * from thin air and executed INLINE.
  */
 #include <math.h>
-void task_schedualize(size_t *pad) {
+static size_t task_schedualize(size_t *pad) {
     char   space[2][64];
     bool   execute  = false;
     char  *data     = NULL;
@@ -1045,6 +1116,7 @@ void task_schedualize(size_t *pad) {
     size_t size     = 0;
     size_t i        = 0;
     size_t j        = 0;
+    size_t failed   = 0;
 
     util_snprintf(space[0], sizeof(space[0]), "%d", (int)vec_size(task_tasks));
 
@@ -1059,8 +1131,11 @@ void task_schedualize(size_t *pad) {
          * Generate a task from thin air if it requires execution in
          * the QCVM.
          */
+         
+        /* diagnostic is not executed, but compare tested instead, like preproessor */
         execute = !! (!strcmp(task_tasks[i].tmpl->proceduretype, "-execute")) ||
-                     (!strcmp(task_tasks[i].tmpl->proceduretype, "-pp"));
+                     (!strcmp(task_tasks[i].tmpl->proceduretype, "-pp"))      ||
+                     (!strcmp(task_tasks[i].tmpl->proceduretype, "-diagnostic"));
 
         /*
          * We assume it compiled before we actually compiled :).  On error
@@ -1089,12 +1164,13 @@ void task_schedualize(size_t *pad) {
              * that refers to a variable named error, or something like
              * that .. then this will blowup :P
              */
-            if (strstr(data, "error")) {
+            if (strstr(data, "error") && strcmp(task_tasks[i].tmpl->proceduretype, "-diagnostic")) {
                 execute                = false;
                 task_tasks[i].compiled = false;
             }
 
             fs_file_puts (task_tasks[i].stderrlog, data);
+            fflush(task_tasks[i].stderrlog); /* fast flush for read */
         }
 
         if (!task_tasks[i].compiled && strcmp(task_tasks[i].tmpl->proceduretype, "-fail")) {
@@ -1105,6 +1181,7 @@ void task_schedualize(size_t *pad) {
                 (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen("(failed to compile)") - pad[2]),
                 "(failed to compile)"
             );
+            failed++;
             continue;
         }
 
@@ -1125,7 +1202,7 @@ void task_schedualize(size_t *pad) {
          * in the virtual machine (or the preprocessor output needs to
          * be matched).
          */
-        if (!task_trymatch(task_tasks[i].tmpl, &match)) {
+        if (!task_trymatch(i, &match)) {
             size_t d = 0;
 
             con_out("failure:   `%s` %*s %*s\n",
@@ -1135,11 +1212,15 @@ void task_schedualize(size_t *pad) {
                 (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen(
                     (strcmp(task_tasks[i].tmpl->proceduretype, "-pp"))
                         ? "(invalid results from execution)"
-                        : "(invalid results from preprocessing)"
+                        : (strcmp(task_tasks[i].tmpl->proceduretype, "-diagnostic"))
+                            ? "(invalid results from preprocessing)"
+                            : "(invalid results from compiler diagnsotics)"
                 ) - pad[2]),
                 (strcmp(task_tasks[i].tmpl->proceduretype, "-pp"))
                     ? "(invalid results from execution)"
-                    : "(invalid results from preprocessing)"
+                    : (strcmp(task_tasks[i].tmpl->proceduretype, "-diagnostic"))
+                            ? "(invalid results from preprocessing)"
+                            : "(invalid results from compiler diagnsotics)"
             );
 
             /*
@@ -1153,7 +1234,7 @@ void task_schedualize(size_t *pad) {
             );
             for (; d < vec_size(task_tasks[i].tmpl->comparematch); d++) {
                 char  *select = task_tasks[i].tmpl->comparematch[d];
-                size_t length = 40 - strlen(select);
+                size_t length = 60 - strlen(select);
 
                 con_out("        Expected: \"%s\"", select);
                 while (length --)
@@ -1168,7 +1249,7 @@ void task_schedualize(size_t *pad) {
              */
             if (vec_size(match) > vec_size(task_tasks[i].tmpl->comparematch)) {
                 for (d = 0; d < vec_size(match) - vec_size(task_tasks[i].tmpl->comparematch); d++) {
-                    con_out("        Expected: Nothing                                   | Got: \"%s\"\n",
+                    con_out("        Expected: Nothing                                                       | Got: \"%s\"\n",
                         match[d + vec_size(task_tasks[i].tmpl->comparematch)]
                     );
                 }
@@ -1178,8 +1259,10 @@ void task_schedualize(size_t *pad) {
             for (j = 0; j < vec_size(match); j++)
                 mem_d(match[j]);
             vec_free(match);
+            failed++;
             continue;
         }
+        
         for (j = 0; j < vec_size(match); j++)
             mem_d(match[j]);
         vec_free(match);
@@ -1194,6 +1277,7 @@ void task_schedualize(size_t *pad) {
         );
     }
     mem_d(data);
+    return failed;
 }
 
 /*
@@ -1210,7 +1294,8 @@ void task_schedualize(size_t *pad) {
  *
  * It expects con_init() was called before hand.
  */
-GMQCC_WARN bool test_perform(const char *curdir, const char *defs) {
+static GMQCC_WARN bool test_perform(const char *curdir, const char *defs) {
+    size_t             failed       = false;
     static const char *default_defs = "defs.qh";
 
     size_t pad[] = {
@@ -1240,10 +1325,12 @@ GMQCC_WARN bool test_perform(const char *curdir, const char *defs) {
      * it's designed to prevent lock contention, and possible syncronization
      * issues.
      */
-    task_schedualize(pad);
+    failed = task_schedualize(pad);
+    if (failed)
+        con_out("%u out of %u tests failed\n", failed, vec_size(task_tasks));
     task_destroy();
 
-    return true;
+    return (failed) ? false : true;
 }
 
 /*
@@ -1283,6 +1370,7 @@ int main(int argc, char **argv) {
     char         *defs     = NULL;
 
     con_init();
+    OPTS_OPTION_U16(OPTION_MEMDUMPCOLS) = 16;
 
     /*
      * Command line option parsing commences now We only need to support
@@ -1321,7 +1409,7 @@ int main(int argc, char **argv) {
     }
     con_change(redirout, redirerr);
     succeed = test_perform("tests", defs);
-    util_meminfo();
+    stat_info();
 
 
     return (succeed) ? EXIT_SUCCESS : EXIT_FAILURE;