X-Git-Url: https://git.xonotic.org/?p=xonotic%2Fgmqcc.git;a=blobdiff_plain;f=test.c;h=7722af26a4a5caae39c636fcfc6ee78be72e40e4;hp=b59826a4139e64c0a6370b9111322ae73040b2e9;hb=4c4aa5534c34a35560ab1936bc84d37ee6e869e6;hpb=61fa54318c3aa1ce0c2099f9793d830e5896c016 diff --git a/test.c b/test.c index b59826a..7722af2 100644 --- a/test.c +++ b/test.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 + * Copyright (C) 2012, 2013 * Dale Weiler * * Permission is hereby granted, free of charge, to any person obtaining a copy of @@ -23,11 +23,10 @@ #include "gmqcc.h" #include #include -#include -bool opts_memchk = false; -bool opts_debug = false; -char *task_bins[] = { +opts_cmd_t opts; + +static const char *task_bins[] = { "./gmqcc", "./qcvm" }; @@ -37,25 +36,25 @@ char *task_bins[] = { * this implements a unique bi-directional popen-like function that * allows reading data from both stdout and stderr. And writing to * stdin :) - * + * * Example of use: * FILE *handles[3] = task_popen("ls", "-l", "r"); * if (!handles) { perror("failed to open stdin/stdout/stderr to ls"); * // handles[0] = stdin * // handles[1] = stdout * // handles[2] = stderr - * + * * task_pclose(handles); // to close */ #ifndef _WIN32 #include #include - +#include #include typedef struct { FILE *handles[3]; int pipes [3]; - + int stderr_fd; int stdout_fd; int pid; @@ -66,9 +65,9 @@ FILE ** task_popen(const char *command, const char *mode) { int outhandle [2]; int errhandle [2]; int trypipe; - - popen_t *data = mem_a(sizeof(popen_t)); - + + popen_t *data = (popen_t*)mem_a(sizeof(popen_t)); + /* * Parse the command now into a list for execv, this is a pain * in the ass. @@ -76,36 +75,36 @@ FILE ** task_popen(const char *command, const char *mode) { char *line = (char*)command; char **argv = NULL; { - + while (*line != '\0') { while (*line == ' ' || *line == '\t' || *line == '\n') *line++ = '\0'; vec_push(argv, line); - + while (*line != '\0' && *line != ' ' && *line != '\t' && *line != '\n') line++; } vec_push(argv, '\0'); } - - + + if ((trypipe = pipe(inhandle)) < 0) goto task_popen_error_0; if ((trypipe = pipe(outhandle)) < 0) goto task_popen_error_1; if ((trypipe = pipe(errhandle)) < 0) goto task_popen_error_2; - + if ((data->pid = fork()) > 0) { /* parent */ close(inhandle [0]); close(outhandle [1]); close(errhandle [1]); - + data->pipes [0] = inhandle [1]; data->pipes [1] = outhandle[0]; data->pipes [2] = errhandle[0]; data->handles[0] = fdopen(inhandle [1], "w"); data->handles[1] = fdopen(outhandle[0], mode); data->handles[2] = fdopen(errhandle[0], mode); - + /* sigh */ if (argv) vec_free(argv); @@ -115,23 +114,19 @@ FILE ** task_popen(const char *command, const char *mode) { close(inhandle [1]); close(outhandle[0]); close(errhandle[0]); - + /* see piping documentation for this sillyness :P */ - close(0), dup(inhandle [0]); - close(1), dup(outhandle[1]); - close(2), dup(errhandle[1]); - + close(0); (void)!dup(inhandle [0]); + close(1); (void)!dup(outhandle[1]); + close(2); (void)!dup(errhandle[1]); + execvp(*argv, argv); - exit(1); + exit(EXIT_FAILURE); } else { /* fork failed */ goto task_popen_error_3; } - - if (argv) - vec_free(argv); - return data->handles; - + task_popen_error_3: close(errhandle[0]), close(errhandle[1]); task_popen_error_2: close(outhandle[0]), close(outhandle[1]); task_popen_error_1: close(inhandle [0]), close(inhandle [1]); @@ -145,24 +140,58 @@ task_popen_error_0: int task_pclose(FILE **handles) { popen_t *data = (popen_t*)handles; int status = 0; - + close(data->pipes[0]); /* stdin */ close(data->pipes[1]); /* stdout */ close(data->pipes[2]); /* stderr */ - + waitpid(data->pid, &status, 0); - + mem_d(data); - + return status; } #else + typedef struct { + FILE *handles[3]; + char name_err[L_tmpnam]; + char name_out[L_tmpnam]; + } popen_t; + + FILE **task_popen(const char *command, const char *mode) { + char *cmd = NULL; + popen_t *open = (popen_t*)mem_a(sizeof(popen_t)); -#endif + tmpnam(open->name_err); + tmpnam(open->name_out); -#define TASK_COMPILE 0 -#define TASK_EXECUTE 1 + (void)mode; /* excluded */ + + util_asprintf(&cmd, "%s -redirout=%s -redirerr=%s", command, open->name_out, open->name_err); + + system(cmd); /* HACK */ + open->handles[0] = NULL; + open->handles[1] = fs_file_open(open->name_out, "r"); + open->handles[2] = fs_file_open(open->name_err, "r"); + + mem_d(cmd); + + return open->handles; + } + + void task_pclose(FILE **files) { + popen_t *open = ((popen_t*)files); + fs_file_close(files[1]); + fs_file_close(files[2]); + remove(open->name_err); + remove(open->name_out); + + mem_d(open); + } +#endif /*! _WIN32 */ +#define TASK_COMPILE 0 +#define TASK_EXECUTE 1 /* * Task template system: * templates are rules for a specific test, used to create a "task" that @@ -171,29 +200,21 @@ int task_pclose(FILE **handles) { * the information for that test there is no way to properly "test" them. * Rules for these templates are described in a template file, using a * task template language. - * + * * The language is a basic finite statemachine, top-down single-line * description language. - * + * * The languge is composed entierly of "tags" which describe a string of * text for a task. Think of it much like a configuration file. Except * it's been designed to allow flexibility and future support for prodecual * semantics. - * + * * The following "tags" are suported by the language - * + * * D: * Used to set a description of the current test, this must be * provided, this tag is NOT optional. - * - * F: - * Used to set a failure message, this message will be displayed - * if the test fails, this tag is optional - * - * S: - * Used to set a success message, this message will be displayed - * if the test succeeds, this tag is optional. - * + * * T: * Used to set the procedure for the given task, there are two * options for this: @@ -201,74 +222,80 @@ int task_pclose(FILE **handles) { * This simply performs compilation only * -execute * This will perform compilation and execution - * + * -fail + * This will perform compilation, but requires + * the compilation to fail in order to succeed. + * * This must be provided, this tag is NOT optional. - * + * * C: * Used to set the compilation flags for the given task, this * must be provided, this tag is NOT optional. - * + * + * F: Used to set some test suite flags, currently the only option + * is -no-defs (to including of defs.qh) + * * E: * Used to set the execution flags for the given task. This tag * must be provided if T == -execute, otherwise it's erroneous - * as compilation only takes place. - * + * as compilation only takes place. + * * M: * Used to describe a string of text that should be matched from * the output of executing the task. If this doesn't match the * task fails. This tag must be provided if T == -execute, otherwise * it's erroneous as compilation only takes place. - * + * * I: * Used to specify the INPUT source file to operate on, this must be * provided, this tag is NOT optional * - * + * * Notes: * These tags have one-time use, using them more than once will result * in template compilation errors. - * + * * Lines beginning with # or // in the template file are comments and * are ignored by the template parser. - * + * * Whitespace is optional, with exception to the colon ':' between the * tag and it's assignment value/ - * + * * The template compiler will detect erronrous tags (optional tags * that need not be set), as well as missing tags, and error accordingly * this will result in the task failing. */ typedef struct { - char *description; - char *failuremessage; - char *successmessage; - char *compileflags; - char *executeflags; - char *comparematch; - char *proceduretype; - char *sourcefile; - char *tempfilename; + char *description; + char *compileflags; + char *executeflags; + char *proceduretype; + char *sourcefile; + char *tempfilename; + char **comparematch; + char *rulesfile; + char *testflags; } task_template_t; /* * This is very much like a compiler code generator :-). This generates * a value from some data observed from the compiler. */ -bool task_template_generate(task_template_t *template, char tag, const char *file, size_t line, const char *value) { +bool task_template_generate(task_template_t *tmpl, char tag, const char *file, size_t line, char *value, size_t *pad) { + size_t desclen = 0; + size_t filelen = 0; char **destval = NULL; - - if (!template) + + if (!tmpl) return false; - + switch(tag) { - case 'D': destval = &template->description; break; - case 'F': destval = &template->failuremessage; break; - case 'S': destval = &template->successmessage; break; - case 'T': destval = &template->proceduretype; break; - case 'C': destval = &template->compileflags; break; - case 'E': destval = &template->executeflags; break; - case 'M': destval = &template->comparematch; break; - case 'I': destval = &template->sourcefile; break; + case 'D': destval = &tmpl->description; break; + case 'T': destval = &tmpl->proceduretype; break; + case 'C': destval = &tmpl->compileflags; break; + case 'E': destval = &tmpl->executeflags; break; + case 'I': destval = &tmpl->sourcefile; break; + case 'F': destval = &tmpl->testflags; break; default: con_printmsg(LVL_ERROR, __FILE__, __LINE__, "internal error", "invalid tag `%c:` during code generation\n", @@ -276,7 +303,7 @@ bool task_template_generate(task_template_t *template, char tag, const char *fil ); return false; } - + /* * Ensure if for the given tag, there already exists a * assigned value. @@ -288,61 +315,77 @@ bool task_template_generate(task_template_t *template, char tag, const char *fil ); return false; } - + /* * Strip any whitespace that might exist in the value for assignments * like "D: foo" */ if (value && *value && (*value == ' ' || *value == '\t')) value++; - + /* * Value will contain a newline character at the end, we need to strip * this otherwise kaboom, seriously, kaboom :P */ - *strrchr(value, '\n')='\0'; - + if (strchr(value, '\n')) + *strrchr(value, '\n')='\0'; + else /* cppcheck: possible nullpointer dereference */ + exit(EXIT_FAILURE); + /* * Now allocate and set the actual value for the specific tag. Which * was properly selected and can be accessed with *destval. */ *destval = util_strdup(value); - + + + if (*destval == tmpl->description) { + /* + * Create some padding for the description to align the + * printing of the rules file. + */ + if ((desclen = strlen(tmpl->description)) > pad[0]) + pad[0] = desclen; + } + + if ((filelen = strlen(file)) > pad[2]) + pad[2] = filelen; + return true; } -bool task_template_parse(const char *file, task_template_t *template, FILE *fp) { +bool task_template_parse(const char *file, task_template_t *tmpl, FILE *fp, size_t *pad) { char *data = NULL; char *back = NULL; size_t size = 0; size_t line = 1; - - if (!template) + + if (!tmpl) return false; - + /* top down parsing */ - while (util_getline(&back, &size, fp) != EOF) { + while (fs_file_getline(&back, &size, fp) != EOF) { /* skip whitespace */ data = back; if (*data && (*data == ' ' || *data == '\t')) data++; - + switch (*data) { /* - * Handle comments inside task template files. We're strict + * Handle comments inside task tmpl files. We're strict * about the language for fun :-) */ case '/': if (data[1] != '/') { - con_printmsg(LVL_ERROR, file, line, "template parse error", + con_printmsg(LVL_ERROR, file, line, "tmpl parse error", "invalid character `/`, perhaps you meant `//` ?"); - + mem_d(back); return false; } case '#': break; - + /* * Empty newlines are acceptable as well, so we handle that here * despite being just odd since there should't be that many @@ -351,43 +394,73 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp) case '\r': case '\n': break; - - + + /* * Now begin the actual "tag" stuff. This works as you expect * it to. */ case 'D': - case 'F': - case 'S': case 'T': case 'C': case 'E': - case 'M': case 'I': + case 'F': if (data[1] != ':') { - con_printmsg(LVL_ERROR, file, line, "template parse error", + con_printmsg(LVL_ERROR, file, line, "tmpl parse error", "expected `:` after `%c`", *data ); goto failure; } - if (!task_template_generate(template, *data, file, line, &data[3])) { - con_printmsg(LVL_ERROR, file, line, "template compile error", + if (!task_template_generate(tmpl, *data, file, line, &data[3], pad)) { + con_printmsg(LVL_ERROR, file, line, "tmpl compile error", "failed to generate for given task\n" ); goto failure; } break; - + + /* + * Match requires it's own system since we allow multiple M's + * for multi-line matching. + */ + case 'M': + { + char *value = &data[3]; + if (data[1] != ':') { + con_printmsg(LVL_ERROR, file, line, "tmpl parse error", + "expected `:` after `%c`", + *data + ); + goto failure; + } + + if (value && *value && (*value == ' ' || *value == '\t')) + value++; + + /* + * Value will contain a newline character at the end, we need to strip + * this otherwise kaboom, seriously, kaboom :P + */ + if (strrchr(value, '\n')) + *strrchr(value, '\n')='\0'; + else /* cppcheck: possible null pointer dereference */ + exit(EXIT_FAILURE); + + vec_push(tmpl->comparematch, util_strdup(value)); + + break; + } + default: - con_printmsg(LVL_ERROR, file, line, "template parse error", + con_printmsg(LVL_ERROR, file, line, "tmpl parse error", "invalid tag `%c`", *data ); goto failure; /* no break required */ } - + /* update line and free old sata */ line++; mem_d(back); @@ -396,7 +469,7 @@ bool task_template_parse(const char *file, task_template_t *template, FILE *fp) if (back) mem_d(back); return true; - + failure: if (back) mem_d (back); @@ -407,34 +480,43 @@ failure: * Nullifies the template data: used during initialization of a new * template and free. */ -void task_template_nullify(task_template_t *template) { - if (!template) +void task_template_nullify(task_template_t *tmpl) { + if (!tmpl) return; - - template->description = NULL; - template->failuremessage = NULL; - template->successmessage = NULL; - template->proceduretype = NULL; - template->compileflags = NULL; - template->executeflags = NULL; - template->comparematch = NULL; - template->sourcefile = NULL; - template->tempfilename = NULL; + + tmpl->description = NULL; + tmpl->proceduretype = NULL; + tmpl->compileflags = NULL; + tmpl->executeflags = NULL; + tmpl->comparematch = NULL; + tmpl->sourcefile = NULL; + tmpl->tempfilename = NULL; + tmpl->rulesfile = NULL; + tmpl->testflags = NULL; } -task_template_t *task_template_compile(const char *file, const char *dir) { +task_template_t *task_template_compile(const char *file, const char *dir, size_t *pad) { /* a page should be enough */ char fullfile[4096]; + size_t filepadd = 0; FILE *tempfile = NULL; - task_template_t *template = NULL; - - memset (fullfile, 0, sizeof(fullfile)); - snprintf(fullfile, sizeof(fullfile), "%s/%s", dir, file); - - tempfile = fopen(fullfile, "r"); - template = mem_a(sizeof(task_template_t)); - task_template_nullify(template); - + task_template_t *tmpl = NULL; + + util_snprintf(fullfile, sizeof(fullfile), "%s/%s", dir, file); + + tempfile = fs_file_open(fullfile, "r"); + tmpl = (task_template_t*)mem_a(sizeof(task_template_t)); + task_template_nullify(tmpl); + + /* + * Create some padding for the printing to align the + * printing of the rules file to the console. + */ + if ((filepadd = strlen(fullfile)) > pad[1]) + pad[1] = filepadd; + + tmpl->rulesfile = util_strdup(fullfile); + /* * Esnure the file even exists for the task, this is pretty useless * to even do. @@ -445,12 +527,12 @@ task_template_t *task_template_compile(const char *file, const char *dir) { ); goto failure; } - - if (!task_template_parse(file, template, tempfile)) { + + if (!task_template_parse(file, tmpl, tempfile, pad)) { con_err("template parse error: error during parsing\n"); goto failure; } - + /* * Regardless procedure type, the following tags must exist: * D @@ -458,83 +540,104 @@ task_template_t *task_template_compile(const char *file, const char *dir) { * C * I */ - if (!template->description) { + if (!tmpl->description) { con_err("template compile error: %s missing `D:` tag\n", file); goto failure; } - if (!template->proceduretype) { + if (!tmpl->proceduretype) { con_err("template compile error: %s missing `T:` tag\n", file); goto failure; } - if (!template->compileflags) { + if (!tmpl->compileflags) { con_err("template compile error: %s missing `C:` tag\n", file); goto failure; } - if (!template->sourcefile) { + if (!tmpl->sourcefile) { con_err("template compile error: %s missing `I:` tag\n", file); goto failure; } - + /* * Now lets compile the template, compilation is really just * the process of validating the input. */ - if (!strcmp(template->proceduretype, "-compile")) { - if (template->executeflags) + if (!strcmp(tmpl->proceduretype, "-compile")) { + if (tmpl->executeflags) con_err("template compile warning: %s erroneous tag `E:` when only compiling\n", file); - if (template->comparematch) + if (tmpl->comparematch) con_err("template compile warning: %s erroneous tag `M:` when only compiling\n", file); goto success; - } else if (!strcmp(template->proceduretype, "-execute")) { - if (!template->executeflags) { - con_err("template compile error: %s missing `E:` tag (use `$null` for exclude)\n", file); + } else if (!strcmp(tmpl->proceduretype, "-execute")) { + if (!tmpl->executeflags) { + /* default to $null */ + tmpl->executeflags = util_strdup("$null"); + } + if (!tmpl->comparematch) { + con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file); goto failure; } - if (!template->comparematch) { + } else if (!strcmp(tmpl->proceduretype, "-fail")) { + if (tmpl->executeflags) + con_err("template compile warning: %s erroneous tag `E:` when only failing\n", file); + if (tmpl->comparematch) + con_err("template compile warning: %s erroneous tag `M:` when only failing\n", file); + } else if (!strcmp(tmpl->proceduretype, "-pp")) { + if (tmpl->executeflags) + con_err("template compile warning: %s erroneous tag `E:` when only preprocessing\n", file); + if (!tmpl->comparematch) { con_err("template compile error: %s missing `M:` tag (use `$null` for exclude)\n", file); goto failure; } } else { - con_err("template compile error: %s invalid procedure type: %s\n", file, template->proceduretype); + con_err("template compile error: %s invalid procedure type: %s\n", file, tmpl->proceduretype); goto failure; } - + success: - fclose(tempfile); - return template; - + fs_file_close(tempfile); + return tmpl; + failure: /* * The file might not exist and we jump here when that doesn't happen * so the check to see if it's not null here is required. */ if (tempfile) - fclose(tempfile); - mem_d (template); - + fs_file_close(tempfile); + mem_d (tmpl); + return NULL; } -void task_template_destroy(task_template_t **template) { - if (!template) +void task_template_destroy(task_template_t **tmpl) { + if (!tmpl) return; - - if ((*template)->description) mem_d((*template)->description); - if ((*template)->failuremessage) mem_d((*template)->failuremessage); - if ((*template)->successmessage) mem_d((*template)->successmessage); - if ((*template)->proceduretype) mem_d((*template)->proceduretype); - if ((*template)->compileflags) mem_d((*template)->compileflags); - if ((*template)->executeflags) mem_d((*template)->executeflags); - if ((*template)->comparematch) mem_d((*template)->comparematch); - if ((*template)->sourcefile) mem_d((*template)->sourcefile); - + + if ((*tmpl)->description) mem_d((*tmpl)->description); + if ((*tmpl)->proceduretype) mem_d((*tmpl)->proceduretype); + if ((*tmpl)->compileflags) mem_d((*tmpl)->compileflags); + if ((*tmpl)->executeflags) mem_d((*tmpl)->executeflags); + if ((*tmpl)->sourcefile) mem_d((*tmpl)->sourcefile); + if ((*tmpl)->rulesfile) mem_d((*tmpl)->rulesfile); + if ((*tmpl)->testflags) mem_d((*tmpl)->testflags); + + /* + * Delete all allocated string for task tmpl then destroy the + * main vector. + */ + { + size_t i = 0; + for (; i < vec_size((*tmpl)->comparematch); i++) + mem_d((*tmpl)->comparematch[i]); + + vec_free((*tmpl)->comparematch); + } + /* * Nullify all the template members otherwise NULL comparision - * checks will fail if template pointer is reused. + * checks will fail if tmpl pointer is reused. */ - mem_d(*template); - task_template_nullify(*template); - *template = NULL; + mem_d(*tmpl); } /* @@ -542,7 +645,7 @@ void task_template_destroy(task_template_t **template) { * of a task list. This is the executor of the tasks essentially as well. */ typedef struct { - task_template_t *template; + task_template_t *tmpl; FILE **runhandles; FILE *stderrlog; FILE *stdoutlog; @@ -551,42 +654,47 @@ typedef struct { bool compiled; } task_t; -task_t *task_tasks = NULL; +static task_t *task_tasks = NULL; /* * Read a directory and searches for all template files in it * which is later used to run all tests. */ -bool task_propogate(const char *curdir) { +bool task_propagate(const char *curdir, size_t *pad, const char *defs) { bool success = true; DIR *dir; struct dirent *files; struct stat directory; char buffer[4096]; - - dir = opendir(curdir); - - while ((files = readdir(dir))) { - memset (buffer, 0,sizeof(buffer)); - snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name); - + size_t found = 0; + + dir = fs_dir_open(curdir); + + while ((files = fs_dir_read(dir))) { + util_snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name); + if (stat(buffer, &directory) == -1) { con_err("internal error: stat failed, aborting\n"); abort(); } - + /* skip directories */ if (S_ISDIR(directory.st_mode)) continue; - + /* * We made it here, which concludes the file/directory is not * actually a directory, so it must be a file :) */ - if (strstr(files->d_name, ".tmpl")) { - con_out("compiling task template: %s/%s\n", curdir, files->d_name); - task_template_t *template = task_template_compile(files->d_name, curdir); - if (!template) { + if (strcmp(files->d_name + strlen(files->d_name) - 5, ".tmpl") == 0) { + task_template_t *tmpl = task_template_compile(files->d_name, curdir, pad); + char buf[4096]; /* one page should be enough */ + char *qcflags = NULL; + task_t task; + + util_debug("TEST", "compiling task template: %s/%s\n", curdir, files->d_name); + found ++; + if (!tmpl) { con_err("error compiling task template: %s\n", files->d_name); success = false; continue; @@ -595,82 +703,137 @@ bool task_propogate(const char *curdir) { * Generate a temportary file name for the output binary * so we don't trample over an existing one. */ - template->tempfilename = tempnam(curdir, "TMPDAT"); - + tmpl->tempfilename = NULL; + util_asprintf(&tmpl->tempfilename, "%s/TMPDAT.%s", curdir, files->d_name); + + /* + * Additional QCFLAGS enviroment variable may be used + * to test compile flags for all tests. This needs to be + * BEFORE other flags (so that the .tmpl can override them) + */ + #ifdef _MSC_VER + { + char buffer[4096]; + size_t size; + getenv_s(&size, buffer, sizeof(buffer), "QCFLAGS"); + qcflags = buffer; + } + #else + qcflags = getenv("QCFLAGS"); + #endif + /* * Generate the command required to open a pipe to a process * which will be refered to with a handle in the task for * reading the data from the pipe. */ - char buf[4096]; /* one page should be enough */ - memset (buf,0,sizeof(buf)); - snprintf(buf, sizeof(buf), "%s %s/%s %s -o %s", - task_bins[TASK_COMPILE], - curdir, - template->sourcefile, - template->compileflags, - template->tempfilename - ); - + if (strcmp(tmpl->proceduretype, "-pp")) { + if (qcflags) { + if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) { + util_snprintf(buf, sizeof(buf), "%s %s/%s %s %s -o %s", + task_bins[TASK_COMPILE], + curdir, + tmpl->sourcefile, + qcflags, + tmpl->compileflags, + tmpl->tempfilename + ); + } else { + util_snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s %s -o %s", + task_bins[TASK_COMPILE], + curdir, + defs, + curdir, + tmpl->sourcefile, + qcflags, + tmpl->compileflags, + tmpl->tempfilename + ); + } + } else { + if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) { + util_snprintf(buf, sizeof(buf), "%s %s/%s %s -o %s", + task_bins[TASK_COMPILE], + curdir, + tmpl->sourcefile, + tmpl->compileflags, + tmpl->tempfilename + ); + } else { + util_snprintf(buf, sizeof(buf), "%s %s/%s %s/%s %s -o %s", + task_bins[TASK_COMPILE], + curdir, + defs, + curdir, + tmpl->sourcefile, + tmpl->compileflags, + tmpl->tempfilename + ); + } + } + } else { + /* Preprocessing (qcflags mean shit all here we don't allow them) */ + if (tmpl->testflags && !strcmp(tmpl->testflags, "-no-defs")) { + util_snprintf(buf, sizeof(buf), "%s -E %s/%s -o %s", + task_bins[TASK_COMPILE], + curdir, + tmpl->sourcefile, + tmpl->tempfilename + ); + } else { + util_snprintf(buf, sizeof(buf), "%s -E %s/%s %s/%s -o %s", + task_bins[TASK_COMPILE], + curdir, + defs, + curdir, + tmpl->sourcefile, + tmpl->tempfilename + ); + } + } + /* * The task template was compiled, now lets create a task from - * the template data which has now been propogated. + * the template data which has now been propagated. */ - task_t task; - task.template = template; + task.tmpl = tmpl; if (!(task.runhandles = task_popen(buf, "r"))) { - con_err("error opening pipe to process for test: %s\n", template->description); + con_err("error opening pipe to process for test: %s\n", tmpl->description); success = false; continue; } - - con_out("executing test: `%s` [%s]\n", template->description, buf); - + + util_debug("TEST", "executing test: `%s` [%s]\n", tmpl->description, buf); + /* * Open up some file desciptors for logging the stdout/stderr * to our own. */ - memset (buf,0,sizeof(buf)); - snprintf(buf, sizeof(buf), "%s/%s.stdout", curdir, template->sourcefile); + util_snprintf(buf, sizeof(buf), "%s.stdout", tmpl->tempfilename); task.stdoutlogfile = util_strdup(buf); - task.stdoutlog = fopen(buf, "w"); - - memset (buf,0,sizeof(buf)); - snprintf(buf, sizeof(buf), "%s/%s.stderr", curdir, template->sourcefile); + if (!(task.stdoutlog = fs_file_open(buf, "w"))) { + con_err("error opening %s for stdout\n", buf); + continue; + } + + util_snprintf(buf, sizeof(buf), "%s.stderr", tmpl->tempfilename); task.stderrlogfile = util_strdup(buf); - task.stderrlog = fopen(buf, "w"); - + if (!(task.stderrlog = fs_file_open(buf, "w"))) { + con_err("error opening %s for stderr\n", buf); + continue; + } + vec_push(task_tasks, task); } } - - closedir(dir); - return success; -} -/* - * Removes all temporary 'progs.dat' files created during compilation - * of all tests' - */ -void task_cleanup(const char *curdir) { - DIR *dir; - struct dirent *files; - char buffer[4096]; + util_debug("TEST", "compiled %d task template files out of %d\n", + vec_size(task_tasks), + found + ); - dir = opendir(curdir); - - while ((files = readdir(dir))) { - memset(buffer, 0, sizeof(buffer)); - if (strstr(files->d_name, "TMP")) { - snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name); - if (remove(buffer)) - con_err("error removing temporary file: %s\n", buffer); - else - con_out("removed temporary file: %s\n", buffer); - } - } - - closedir(dir); + fs_dir_close(dir); + return success; } /* @@ -682,26 +845,25 @@ void task_precleanup(const char *curdir) { struct dirent *files; char buffer[4096]; - dir = opendir(curdir); - - while ((files = readdir(dir))) { - memset(buffer, 0, sizeof(buffer)); + dir = fs_dir_open(curdir); + + while ((files = fs_dir_read(dir))) { if (strstr(files->d_name, "TMP") || strstr(files->d_name, ".stdout") || strstr(files->d_name, ".stderr")) { - snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name); + util_snprintf(buffer, sizeof(buffer), "%s/%s", curdir, files->d_name); if (remove(buffer)) con_err("error removing temporary file: %s\n", buffer); else - con_out("removed temporary file: %s\n", buffer); + util_debug("TEST", "removed temporary file: %s\n", buffer); } } - - closedir(dir); + + fs_dir_close(dir); } -void task_destroy(const char *curdir) { +void task_destroy(void) { /* * Free all the data in the task list and finally the list itself * then proceed to cleanup anything else outside the program like @@ -714,159 +876,215 @@ void task_destroy(const char *curdir) { * annoying to have to do all this cleanup work. */ if (task_tasks[i].runhandles) task_pclose(task_tasks[i].runhandles); - if (task_tasks[i].stdoutlog) fclose (task_tasks[i].stdoutlog); - if (task_tasks[i].stderrlog) fclose (task_tasks[i].stderrlog); - + if (task_tasks[i].stdoutlog) fs_file_close (task_tasks[i].stdoutlog); + if (task_tasks[i].stderrlog) fs_file_close (task_tasks[i].stderrlog); + /* * Only remove the log files if the test actually compiled otherwise - * forget about it. + * forget about it (or if it didn't compile, and the procedure type + * was set to -fail (meaning it shouldn't compile) .. stil remove) */ - if (task_tasks[i].compiled) { + if (task_tasks[i].compiled || !strcmp(task_tasks[i].tmpl->proceduretype, "-fail")) { if (remove(task_tasks[i].stdoutlogfile)) con_err("error removing stdout log file: %s\n", task_tasks[i].stdoutlogfile); else - con_out("removed stdout log file: %s\n", task_tasks[i].stdoutlogfile); - + util_debug("TEST", "removed stdout log file: %s\n", task_tasks[i].stdoutlogfile); if (remove(task_tasks[i].stderrlogfile)) con_err("error removing stderr log file: %s\n", task_tasks[i].stderrlogfile); else - con_out("removed stderr log file: %s\n", task_tasks[i].stderrlogfile); + util_debug("TEST", "removed stderr log file: %s\n", task_tasks[i].stderrlogfile); + + remove(task_tasks[i].tmpl->tempfilename); } - + /* free util_strdup data for log files */ mem_d(task_tasks[i].stdoutlogfile); mem_d(task_tasks[i].stderrlogfile); - - task_template_destroy(&task_tasks[i].template); + + task_template_destroy(&task_tasks[i].tmpl); } vec_free(task_tasks); - - /* - * Cleanup outside stuff like temporary files. - */ - task_cleanup(curdir); } /* * This executes the QCVM task for a specificly compiled progs.dat * using the template passed into it for call-flags and user defined - * messages. + * messages IF the procedure type is -execute, otherwise it matches + * the preprocessor output. */ -bool task_execute(task_template_t *template) { - bool success = false; +bool task_trymatch(task_template_t *tmpl, char ***line) { + bool success = true; + bool preprocessing = false; FILE *execute; char buffer[4096]; memset (buffer,0,sizeof(buffer)); - - /* - * Drop the execution flags for the QCVM if none where - * actually specified. - */ - if (!strcmp(template->executeflags, "$null")) { - snprintf(buffer, sizeof(buffer), "%s %s", - task_bins[TASK_EXECUTE], - template->tempfilename + + if (strcmp(tmpl->proceduretype, "-pp")) { + /* + * Drop the execution flags for the QCVM if none where + * actually specified. + */ + if (!strcmp(tmpl->executeflags, "$null")) { + util_snprintf(buffer, sizeof(buffer), "%s %s", + task_bins[TASK_EXECUTE], + tmpl->tempfilename + ); + } else { + util_snprintf(buffer, sizeof(buffer), "%s %s %s", + task_bins[TASK_EXECUTE], + tmpl->executeflags, + tmpl->tempfilename + ); + } + + util_debug("TEST", "executing qcvm: `%s` [%s]\n", + tmpl->description, + buffer ); + + execute = popen(buffer, "r"); + if (!execute) + return false; } else { - snprintf(buffer, sizeof(buffer), "%s %s %s", - task_bins[TASK_EXECUTE], - template->executeflags, - template->tempfilename - ); + /* + * we're preprocessing, which means we need to read int + * the produced file and do some really weird shit. + */ + if (!(execute = fs_file_open(tmpl->tempfilename, "r"))) + return false; + + preprocessing = true; } - - con_out("executing qcvm: `%s` [%s]\n", - template->description, - buffer - ); - - execute = popen(buffer, "r"); - if (!execute) - return false; - + /* * Now lets read the lines and compare them to the matches we expect * and handle accordingly. */ { - char *data = NULL; - size_t size = 0; - while (util_getline(&data, &size, execute) != EOF) {} - - if (!strcmp(data, "No main function found\n")) { - con_err("test failure: `%s` [%s] (No main function found)\n", - template->description, - (template->failuremessage) ? - template->failuremessage : "unknown" - ); - pclose(execute); - return false; + char *data = NULL; + size_t size = 0; + size_t compare = 0; + while (fs_file_getline(&data, &size, execute) != EOF) { + if (!strcmp(data, "No main function found\n")) { + con_err("test failure: `%s` (No main function found) [%s]\n", + tmpl->description, + tmpl->rulesfile + ); + if (preprocessing) + fs_file_close(execute); + else + pclose(execute); + return false; + } + + /* + * Trim newlines from data since they will just break our + * ability to properly validate matches. + */ + if (strrchr(data, '\n')) + *strrchr(data, '\n') = '\0'; + + /* + * If data is just null now, that means the line was an empty + * one and for that, we just ignore it. + */ + if (!*data) + continue; + + if (vec_size(tmpl->comparematch) > compare) { + if (strcmp(data, tmpl->comparematch[compare++])) + success = false; + } else { + success = false; + } + + /* + * Copy to output vector for diagnostics if execution match + * fails. + */ + vec_push(*line, data); + + /* reset */ + data = NULL; + size = 0; } - - /* - * Trim newlines from data since they will just break our - * ability to properly validate matches. - */ - if (strrchr(data, '\n')) - *strrchr(data, '\n') = '\0'; - - /* null list */ - if (!strcmp(template->comparematch, "$null")) - success = true; - - /* - * We only care about the last line from the output for now - * implementing multi-line match is TODO. - */ - if (!strcmp(data, template->comparematch)) - success = true; + mem_d(data); + data = NULL; } - pclose(execute); + + if (!preprocessing) + pclose(execute); + else + fs_file_close(execute); + return success; } +const char *task_type(task_template_t *tmpl) { + if (!strcmp(tmpl->proceduretype, "-pp")) + return "type: preprocessor"; + if (!strcmp(tmpl->proceduretype, "-execute")) + return "type: execution"; + if (!strcmp(tmpl->proceduretype, "-compile")) + return "type: compile"; + return "type: fail"; +} + /* * This schedualizes all tasks and actually runs them individually * this is generally easy for just -compile variants. For compile and * execution this takes more work since a task needs to be generated * from thin air and executed INLINE. */ -void task_schedualize(const char *curdir) { +#include +void task_schedualize(size_t *pad) { + char space[2][64]; bool execute = false; - char *back = NULL; char *data = NULL; + char **match = NULL; size_t size = 0; - size_t i; - - for (i = 0; i < vec_size(task_tasks); i++) { + size_t i = 0; + size_t j = 0; + + util_snprintf(space[0], sizeof(space[0]), "%d", (int)vec_size(task_tasks)); + + for (; i < vec_size(task_tasks); i++) { + memset(space[1], 0, sizeof(space[1])); + util_snprintf(space[1], sizeof(space[1]), "%d", (int)(i + 1)); + + con_out("test #%u %*s", i + 1, strlen(space[0]) - strlen(space[1]), ""); + + util_debug("TEST", "executing task: %d: %s\n", i, task_tasks[i].tmpl->description); /* - * Generate a task from thin air if it requires execution in - * the QCVM. - */ - if (!strcmp(task_tasks[i].template->proceduretype, "-execute")) - execute = true; - + * Generate a task from thin air if it requires execution in + * the QCVM. + */ + execute = !! (!strcmp(task_tasks[i].tmpl->proceduretype, "-execute")) || + (!strcmp(task_tasks[i].tmpl->proceduretype, "-pp")); + /* * We assume it compiled before we actually compiled :). On error * we change the value */ task_tasks[i].compiled = true; - + /* * Read data from stdout first and pipe that stuff into a log file * then we do the same for stderr. - */ - while (util_getline(&data, &size, task_tasks[i].runhandles[1]) != EOF) { - back = data; - fputs(data, task_tasks[i].stdoutlog); - fflush(task_tasks[i].stdoutlog); + */ + while (fs_file_getline(&data, &size, task_tasks[i].runhandles[1]) != EOF) { + fs_file_puts(task_tasks[i].stdoutlog, data); + + if (strstr(data, "failed to open file")) { + task_tasks[i].compiled = false; + execute = false; + } } - while (util_getline(&data, &size, task_tasks[i].runhandles[2]) != EOF) { - back = data; + while (fs_file_getline(&data, &size, task_tasks[i].runhandles[2]) != EOF) { /* * If a string contains an error we just dissalow execution * of it in the vm. - * + * * TODO: make this more percise, e.g if we print a warning * that refers to a variable named error, or something like * that .. then this will blowup :P @@ -875,75 +1093,156 @@ void task_schedualize(const char *curdir) { execute = false; task_tasks[i].compiled = false; } - - fputs(data, task_tasks[i].stderrlog); - fflush(task_tasks[i].stdoutlog); + + fs_file_puts (task_tasks[i].stderrlog, data); } - - if (back) - mem_d(back); - - /* - * If we can execute we do so after all data has been read and - * this paticular task has coupled execution in its procedure type - */ - if (!execute) + + if (!task_tasks[i].compiled && strcmp(task_tasks[i].tmpl->proceduretype, "-fail")) { + con_out("failure: `%s` %*s %*s\n", + task_tasks[i].tmpl->description, + (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]), + task_tasks[i].tmpl->rulesfile, + (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen("(failed to compile)") - pad[2]), + "(failed to compile)" + ); continue; - + } + + if (!execute) { + con_out("succeeded: `%s` %*s %*s\n", + task_tasks[i].tmpl->description, + (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]), + task_tasks[i].tmpl->rulesfile, + (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen(task_type(task_tasks[i].tmpl)) - pad[2]), + task_type(task_tasks[i].tmpl) + + ); + continue; + } + /* * If we made it here that concludes the task is to be executed - * in the virtual machine. + * in the virtual machine (or the preprocessor output needs to + * be matched). */ - if (!task_execute(task_tasks[i].template)) { - con_err("test failure: `%s` [%s]\n", - task_tasks[i].template->description, - (task_tasks[i].template->failuremessage) ? - task_tasks[i].template->failuremessage : "unknown" + if (!task_trymatch(task_tasks[i].tmpl, &match)) { + size_t d = 0; + + con_out("failure: `%s` %*s %*s\n", + task_tasks[i].tmpl->description, + (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]), + task_tasks[i].tmpl->rulesfile, + (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen( + (strcmp(task_tasks[i].tmpl->proceduretype, "-pp")) + ? "(invalid results from execution)" + : "(invalid results from preprocessing)" + ) - pad[2]), + (strcmp(task_tasks[i].tmpl->proceduretype, "-pp")) + ? "(invalid results from execution)" + : "(invalid results from preprocessing)" ); + + /* + * Print nicely formatted expected match lists to console error + * handler for the all the given matches in the template file and + * what was actually returned from executing. + */ + con_out(" Expected From %u Matches: (got %u Matches)\n", + vec_size(task_tasks[i].tmpl->comparematch), + vec_size(match) + ); + for (; d < vec_size(task_tasks[i].tmpl->comparematch); d++) { + char *select = task_tasks[i].tmpl->comparematch[d]; + size_t length = 40 - strlen(select); + + con_out(" Expected: \"%s\"", select); + while (length --) + con_out(" "); + con_out("| Got: \"%s\"\n", (d >= vec_size(match)) ? "<>" : match[d]); + } + + /* + * Print the non-expected out (since we are simply not expecting it) + * This will help track down bugs in template files that fail to match + * something. + */ + if (vec_size(match) > vec_size(task_tasks[i].tmpl->comparematch)) { + for (d = 0; d < vec_size(match) - vec_size(task_tasks[i].tmpl->comparematch); d++) { + con_out(" Expected: Nothing | Got: \"%s\"\n", + match[d + vec_size(task_tasks[i].tmpl->comparematch)] + ); + } + } + + + for (j = 0; j < vec_size(match); j++) + mem_d(match[j]); + vec_free(match); continue; } - - con_out("test succeed: `%s` [%s]\n", - task_tasks[i].template->description, - (task_tasks[i].template->successmessage) ? - task_tasks[i].template->successmessage : "unknown" + for (j = 0; j < vec_size(match); j++) + mem_d(match[j]); + vec_free(match); + + con_out("succeeded: `%s` %*s %*s\n", + task_tasks[i].tmpl->description, + (pad[0] + pad[1] - strlen(task_tasks[i].tmpl->description)) + (strlen(task_tasks[i].tmpl->rulesfile) - pad[1]), + task_tasks[i].tmpl->rulesfile, + (pad[1] + pad[2] - strlen(task_tasks[i].tmpl->rulesfile)) + (strlen(task_type(task_tasks[i].tmpl))- pad[2]), + task_type(task_tasks[i].tmpl) + ); } - if (back) - mem_d(back); + mem_d(data); } /* * This is the heart of the whole test-suite process. This cleans up * any existing temporary files left behind as well as log files left - * behind. Then it propogates a list of tests from `curdir` by scaning + * behind. Then it propagates a list of tests from `curdir` by scaning * it for template files and compiling them into tasks, in which it * schedualizes them (executes them) and actually reports errors and * what not. It then proceeds to destroy the tasks and return memory * it's the engine :) - * - * It returns true of tests could be propogated, otherwise it returns + * + * It returns true of tests could be propagated, otherwise it returns * false. - * + * * It expects con_init() was called before hand. */ -bool test_perform(const char *curdir) { +GMQCC_WARN bool test_perform(const char *curdir, const char *defs) { + static const char *default_defs = "defs.qh"; + + size_t pad[] = { + /* test ### [succeed/fail]: `description` [tests/template.tmpl] [type] */ + 0, 0, 0 + }; + + /* + * If the default definition file isn't set to anything. We will + * use the default_defs here, which is "defs.qc" + */ + if (!defs) { + defs = default_defs; + } + + task_precleanup(curdir); - if (!task_propogate(curdir)) { - con_err("error: failed to propogate tasks\n"); - task_destroy(curdir); + if (!task_propagate(curdir, pad, defs)) { + con_err("error: failed to propagate tasks\n"); + task_destroy(); return false; } /* - * If we made it here all tasks where propogated from their resultant + * If we made it here all tasks where propagated from their resultant * template file. So we can start the FILO scheduler, this has been * designed in the most thread-safe way possible for future threading * it's designed to prevent lock contention, and possible syncronization * issues. */ - task_schedualize(curdir); - task_destroy(curdir); - + task_schedualize(pad); + task_destroy(); + return true; } @@ -978,10 +1277,13 @@ static bool parsecmd(const char *optname, int *argc_, char ***argv_, char **out, } int main(int argc, char **argv) { - char *redirout = (char*)stdout; - char *redirerr = (char*)stderr; + bool succeed = false; + char *redirout = (char*)stdout; + char *redirerr = (char*)stderr; + char *defs = NULL; + con_init(); - + /* * Command line option parsing commences now We only need to support * a few things in the test suite. @@ -995,28 +1297,32 @@ int main(int argc, char **argv) { continue; if (parsecmd("redirerr", &argc, &argv, &redirerr, 1, false)) continue; - + if (parsecmd("defs", &argc, &argv, &defs, 1, false)) + continue; + con_change(redirout, redirerr); if (!strcmp(argv[0]+1, "debug")) { - opts_debug = true; + OPTS_OPTION_BOOL(OPTION_DEBUG) = true; continue; } if (!strcmp(argv[0]+1, "memchk")) { - opts_memchk = true; + OPTS_OPTION_BOOL(OPTION_MEMCHK) = true; continue; } if (!strcmp(argv[0]+1, "nocolor")) { con_color(0); continue; } - + con_err("invalid argument %s\n", argv[0]+1); return -1; } } con_change(redirout, redirerr); - test_perform("tests"); + succeed = test_perform("tests", defs); util_meminfo(); - return 0; + + + return (succeed) ? EXIT_SUCCESS : EXIT_FAILURE; }