More fixes

[xonotic/gmqcc.git] / test.c

diff --git a/test.c b/test.c
index ca1add20faef4cbd93296ce2e2589c2c6213866c..6ca4fec8ca92ab1a06b1dd0beba7cf99f7d0dc47 100644 (file)
--- a/test.c
+++ b/test.c
@@ -25,8 +25,8 @@
 #include <sys/stat.h>
 #include <dirent.h>
 
-bool  opts_memchk = false;
-bool  opts_debug  = false;
+opts_cmd_t opts;
+
 char *task_bins[] = {
     "./gmqcc",
     "./qcvm"
@@ -50,7 +50,6 @@ char *task_bins[] = {
 #ifndef _WIN32
 #include <sys/types.h>
 #include <sys/wait.h>
-
 #include <unistd.h>
 typedef struct {
     FILE *handles[3];
@@ -157,7 +156,7 @@ int task_pclose(FILE **handles) {
     return status;
 }
 #else
-
+#error "There is no support for windows yet ... this is not a FTBFS bug"
 #endif
 
 #define TASK_COMPILE 0
@@ -201,6 +200,9 @@ int task_pclose(FILE **handles) {
  *                  This simply performs compilation only
  *              -execute
  *                  This will perform compilation and execution
+ *              -fail
+ *                  This will perform compilation, but requires
+ *                  the compilation to fail in order to succeed.   
  *
  *          This must be provided, this tag is NOT optional.
  *
@@ -521,6 +523,12 @@ task_template_t *task_template_compile(const char *file, const char *dir) {
             con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file);
             goto failure;
         }
+    } else if (!strcmp(template->proceduretype, "-fail")) {
+        if (template->executeflags)
+            con_err("template compile warning: %s erroneous tag `E:` when only failing\n", file);
+        if (template->comparematch)
+            con_err("template compile warning: %s erroneous tag `M:` when only failing\n", file);
+        goto success;
     } else {
         con_err("template compile error: %s invalid procedure type: %s\n", file, template->proceduretype);
         goto failure;
@@ -769,9 +777,10 @@ void task_destroy(const char *curdir) {
 
         /*
          * Only remove the log files if the test actually compiled otherwise
-         * forget about it.
+         * forget about it (or if it didn't compile, and the procedure type
+         * was set to -fail (meaning it shouldn't compile) .. stil remove) 
          */
-        if (task_tasks[i].compiled) {
+        if (task_tasks[i].compiled || !strcmp(task_tasks[i].template->proceduretype, "-fail")) {
             if (remove(task_tasks[i].stdoutlogfile))
                 con_err("error removing stdout log file: %s\n", task_tasks[i].stdoutlogfile);
             else
@@ -803,7 +812,7 @@ void task_destroy(const char *curdir) {
  * messages.
  */
 bool task_execute(task_template_t *template, char ***line) {
-    bool     success = false;
+    bool     success = true;
     FILE    *execute;
     char     buffer[4096];
     memset  (buffer,0,sizeof(buffer));
@@ -860,18 +869,22 @@ bool task_execute(task_template_t *template, char ***line) {
             if  (strrchr(data, '\n'))
                 *strrchr(data, '\n') = '\0';
 
-
-            /*
-             * We only care about the last line from the output for now
-             * implementing multi-line match is TODO.
-             */
-            success = !!!(strcmp(data, template->comparematch[compare++]));
+            if (vec_size(template->comparematch) > compare) {
+                if (strcmp(data, template->comparematch[compare++]))
+                    success = false;
+            } else {
+                    success = false;
+            }
 
             /*
              * Copy to output vector for diagnostics if execution match
              * fails.
              */  
             vec_push(*line, data);
+
+            /* reset */
+            data = NULL;
+            size = 0;
         }
         mem_d(data);
         data = NULL;
@@ -892,6 +905,7 @@ void task_schedualize() {
     char **match    = NULL;
     size_t size     = 0;
     size_t i;
+    size_t j;
 
     util_debug("TEST", "found %d tasks, preparing to execute\n", vec_size(task_tasks));
 
@@ -901,8 +915,7 @@ void task_schedualize() {
          * Generate a task from thin air if it requires execution in
          * the QCVM.
          */
-        if (!strcmp(task_tasks[i].template->proceduretype, "-execute"))
-            execute = true;
+        execute = !!(!strcmp(task_tasks[i].template->proceduretype, "-execute"));
 
         /*
          * We assume it compiled before we actually compiled :).  On error
@@ -942,7 +955,7 @@ void task_schedualize() {
             fflush(task_tasks[i].stdoutlog);
         }
 
-        if (!execute) {
+        if (!task_tasks[i].compiled && strcmp(task_tasks[i].template->proceduretype, "-fail")) {
             con_err("test failure: `%s` [%s] (failed to compile) see %s.stdout and %s.stderr\n",
                 task_tasks[i].template->description,
                 (task_tasks[i].template->failuremessage) ?
@@ -952,6 +965,16 @@ void task_schedualize() {
             );
             continue;
         }
+
+        if (!execute) {
+            con_out("test succeeded: `%s` [%s]\n",
+                 task_tasks[i].template->description,
+                (task_tasks[i].template->successmessage) ?
+                 task_tasks[i].template->successmessage  : "unknown"
+            );
+            continue;
+        }
+
         /*
          * If we made it here that concludes the task is to be executed
          * in the virtual machine.
@@ -970,7 +993,10 @@ void task_schedualize() {
              * handler for the all the given matches in the template file and
              * what was actually returned from executing.
              */
-            con_err("    Expected From %u Matches:\n", vec_size(task_tasks[i].template->comparematch));
+            con_err("    Expected From %u Matches: (got %u Matches)\n",
+                vec_size(task_tasks[i].template->comparematch),
+                vec_size(match)
+            );
             for (; d < vec_size(task_tasks[i].template->comparematch); d++) {
                 char  *select = task_tasks[i].template->comparematch[d];
                 size_t length = 40 - strlen(select);
@@ -980,9 +1006,29 @@ void task_schedualize() {
                     con_err(" ");
                 con_err("| Got: \"%s\"\n", (d >= vec_size(match)) ? "<<nothing else to compare>>" : match[d]);
             }
+
+            /*
+             * Print the non-expected out (since we are simply not expecting it)
+             * This will help track down bugs in template files that fail to match
+             * something.
+             */  
+            if (vec_size(match) > vec_size(task_tasks[i].template->comparematch)) {
+                for (d = 0; d < vec_size(match) - vec_size(task_tasks[i].template->comparematch); d++) {
+                    con_err("        Expected: Nothing                                   | Got: \"%s\"\n",
+                        match[d + vec_size(task_tasks[i].template->comparematch)]
+                    );
+                }
+            }
+                    
+
+            for (j = 0; j < vec_size(match); j++)
+                mem_d(match[j]);
             vec_free(match);
             continue;
         }
+        for (j = 0; j < vec_size(match); j++)
+            mem_d(match[j]);
+        vec_free(match);
 
         con_out("test succeeded: `%s` [%s]\n",
              task_tasks[i].template->description,
@@ -1080,11 +1126,11 @@ int main(int argc, char **argv) {
             con_change(redirout, redirerr);
 
             if (!strcmp(argv[0]+1, "debug")) {
-                opts_debug = true;
+                opts.debug = true;
                 continue;
             }
             if (!strcmp(argv[0]+1, "memchk")) {
-                opts_memchk = true;
+                opts.memchk = true;
                 continue;
             }
             if (!strcmp(argv[0]+1, "nocolor")) {