12 static const char *keywords_qc[] = {
20 static const char *keywords_fg[] = {
21 "switch", "case", "default",
27 "__builtin_debug_printtype"
33 static char* *lex_filenames;
35 static void lexerror(lex_file *lex, const char *fmt, ...)
41 con_vprintmsg(LVL_ERROR, lex->name, lex->sline, lex->column, "parse error", fmt, ap);
43 con_vprintmsg(LVL_ERROR, "", 0, 0, "parse error", fmt, ap);
47 static bool lexwarn(lex_file *lex, int warntype, const char *fmt, ...)
54 ctx.line = lex->sline;
55 ctx.column = lex->column;
58 r = vcompile_warning(ctx, warntype, fmt, ap);
63 static void lex_token_new(lex_file *lex)
66 vec_shrinkto(lex->tok.value, 0);
68 lex->tok.constval.t = TYPE_VOID;
69 lex->tok.ctx.line = lex->sline;
70 lex->tok.ctx.file = lex->name;
71 lex->tok.ctx.column = lex->column;
74 static void lex_ungetch(lex_file *lex, int ch);
75 static int lex_getch(lex_file *lex);
77 lex_file* lex_open(const char *file)
80 FILE *in = fopen(file, "rb");
84 lexerror(nullptr, "open failed: '%s'\n", file);
88 lex = (lex_file*)mem_a(sizeof(*lex));
91 lexerror(nullptr, "out of memory\n");
95 memset(lex, 0, sizeof(*lex));
98 lex->name = util_strdup(file);
99 lex->line = 1; /* we start counting at 1 */
105 if ((read = (lex_getch(lex) << 16) | (lex_getch(lex) << 8) | lex_getch(lex)) != 0xEFBBBF) {
106 lex_ungetch(lex, (read & 0x0000FF));
107 lex_ungetch(lex, (read & 0x00FF00) >> 8);
108 lex_ungetch(lex, (read & 0xFF0000) >> 16);
111 * otherwise the lexer has advanced 3 bytes for the BOM, we need
112 * to set the column back to 0
117 vec_push(lex_filenames, lex->name);
121 lex_file* lex_open_string(const char *str, size_t len, const char *name)
125 lex = (lex_file*)mem_a(sizeof(*lex));
127 lexerror(nullptr, "out of memory\n");
131 memset(lex, 0, sizeof(*lex));
134 lex->open_string = str;
135 lex->open_string_length = len;
136 lex->open_string_pos = 0;
138 lex->name = util_strdup(name ? name : "<string-source>");
139 lex->line = 1; /* we start counting at 1 */
144 vec_push(lex_filenames, lex->name);
149 void lex_cleanup(void)
152 for (i = 0; i < vec_size(lex_filenames); ++i)
153 mem_d(lex_filenames[i]);
154 vec_free(lex_filenames);
157 void lex_close(lex_file *lex)
160 for (i = 0; i < vec_size(lex->frames); ++i)
161 mem_d(lex->frames[i].name);
162 vec_free(lex->frames);
165 vec_free(lex->modelname);
170 vec_free(lex->tok.value);
172 /* mem_d(lex->name); collected in lex_filenames */
178 static int lex_fgetc(lex_file *lex)
182 return fgetc(lex->file);
184 if (lex->open_string) {
185 if (lex->open_string_pos >= lex->open_string_length)
188 return lex->open_string[lex->open_string_pos++];
193 /* Get or put-back data
194 * The following to functions do NOT understand what kind of data they
196 * The are merely wrapping get/put in order to count line numbers.
198 static int lex_try_trigraph(lex_file *lex, int old)
202 if (!lex->push_line && c2 == '\n') {
208 lex_ungetch(lex, c2);
213 if (!lex->push_line && c3 == '\n') {
219 case '=': return '#';
220 case '/': return '\\';
221 case '\'': return '^';
222 case '(': return '[';
223 case ')': return ']';
224 case '!': return '|';
225 case '<': return '{';
226 case '>': return '}';
227 case '-': return '~';
229 lex_ungetch(lex, c3);
230 lex_ungetch(lex, c2);
235 static int lex_try_digraph(lex_file *lex, int ch)
239 /* we just used fgetc() so count lines
240 * need to offset a \n the ungetch would recognize
242 if (!lex->push_line && c2 == '\n')
244 if (ch == '<' && c2 == ':')
246 else if (ch == ':' && c2 == '>')
248 else if (ch == '<' && c2 == '%')
250 else if (ch == '%' && c2 == '>')
252 else if (ch == '%' && c2 == ':')
254 lex_ungetch(lex, c2);
258 static int lex_getch(lex_file *lex)
264 if (!lex->push_line && lex->peek[lex->peekpos] == '\n') {
268 return lex->peek[lex->peekpos];
272 if (!lex->push_line && ch == '\n') {
277 return lex_try_trigraph(lex, ch);
278 else if (!lex->flags.nodigraphs && (ch == '<' || ch == ':' || ch == '%'))
279 return lex_try_digraph(lex, ch);
283 static void lex_ungetch(lex_file *lex, int ch)
285 lex->peek[lex->peekpos++] = ch;
287 if (!lex->push_line && ch == '\n') {
293 /* classify characters
294 * some additions to the is*() functions of ctype.h
297 /* Idents are alphanumberic, but they start with alpha or _ */
298 static bool isident_start(int ch)
300 return util_isalpha(ch) || ch == '_';
303 static bool isident(int ch, bool allow_dot)
305 return isident_start(ch) || util_isdigit(ch) || (allow_dot && ch == '.');
308 /* isxdigit_only is used when we already know it's not a digit
309 * and want to see if it's a hex digit anyway.
311 static bool isxdigit_only(int ch)
313 return (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
316 /* Append a character to the token buffer */
317 static void lex_tokench(lex_file *lex, int ch)
319 vec_push(lex->tok.value, ch);
322 /* Append a trailing null-byte */
323 static void lex_endtoken(lex_file *lex)
325 vec_push(lex->tok.value, 0);
326 vec_shrinkby(lex->tok.value, 1);
329 static bool lex_try_pragma(lex_file *lex)
332 char *pragma = nullptr;
333 char *command = nullptr;
334 char *param = nullptr;
337 if (lex->flags.preprocessing)
344 lex_ungetch(lex, ch);
348 for (ch = lex_getch(lex); vec_size(pragma) < 8 && ch >= 'a' && ch <= 'z'; ch = lex_getch(lex))
349 vec_push(pragma, ch);
352 if (ch != ' ' || strcmp(pragma, "pragma")) {
353 lex_ungetch(lex, ch);
357 for (ch = lex_getch(lex); vec_size(command) < 32 && ch >= 'a' && ch <= 'z'; ch = lex_getch(lex))
358 vec_push(command, ch);
359 vec_push(command, 0);
362 lex_ungetch(lex, ch);
366 for (ch = lex_getch(lex); vec_size(param) < 1024 && ch != ')' && ch != '\n'; ch = lex_getch(lex))
371 lex_ungetch(lex, ch);
375 if (!strcmp(command, "push")) {
376 if (!strcmp(param, "line")) {
378 if (lex->push_line == 1)
384 else if (!strcmp(command, "pop")) {
385 if (!strcmp(param, "line")) {
388 if (lex->push_line == 0)
394 else if (!strcmp(command, "file")) {
396 lex->name = util_strdup(param);
397 vec_push(lex_filenames, lex->name);
399 else if (!strcmp(command, "line")) {
400 line = strtol(param, nullptr, 0)-1;
406 while (ch != '\n' && ch != EOF)
416 while (vec_size(command)) {
417 lex_ungetch(lex, (unsigned char)vec_last(command));
421 lex_ungetch(lex, ' ');
425 while (vec_size(param)) {
426 lex_ungetch(lex, (unsigned char)vec_last(param));
430 lex_ungetch(lex, ' ');
434 while (vec_size(pragma)) {
435 lex_ungetch(lex, (unsigned char)vec_last(pragma));
440 lex_ungetch(lex, '#');
446 /* Skip whitespace and comments and return the first
447 * non-white character.
448 * As this makes use of the above getch() ungetch() functions,
449 * we don't need to care at all about line numbering anymore.
451 * In theory, this function should only be used at the beginning
452 * of lexing, or when we *know* the next character is part of the token.
453 * Otherwise, if the parser throws an error, the linenumber may not be
454 * the line of the error, but the line of the next token AFTER the error.
456 * This is currently only problematic when using c-like string-continuation,
457 * since comments and whitespaces are allowed between 2 such strings.
461 "A continuation of the previous string"
462 // This line is skipped
465 * In this case, if the parse decides it didn't actually want a string,
466 * and uses lex->line to print an error, it will show the ', foo);' line's
469 * On the other hand, the parser is supposed to remember the line of the next
470 * token's beginning. In this case we would want skipwhite() to be called
471 * AFTER reading a token, so that the parser, before reading the NEXT token,
472 * doesn't store teh *comment's* linenumber, but the actual token's linenumber.
475 * here is to store the line of the first character after skipping
476 * the initial whitespace in lex->sline, this happens in lex_do.
478 static int lex_skipwhite(lex_file *lex, bool hadwhite)
481 bool haswhite = hadwhite;
486 while (ch != EOF && util_isspace(ch)) {
488 if (lex_try_pragma(lex))
491 if (lex->flags.preprocessing) {
494 /* see if there was whitespace first */
495 if (haswhite) { /* (vec_size(lex->tok.value)) { */
496 lex_ungetch(lex, ch);
500 /* otherwise return EOL */
504 lex_tokench(lex, ch);
513 /* one line comment */
516 if (lex->flags.preprocessing) {
518 lex_tokench(lex, ' ');
519 lex_tokench(lex, ' ');
522 while (ch != EOF && ch != '\n') {
523 if (lex->flags.preprocessing)
524 lex_tokench(lex, ' '); /* ch); */
527 if (lex->flags.preprocessing) {
528 lex_ungetch(lex, '\n');
536 /* multiline comment */
537 if (lex->flags.preprocessing) {
539 lex_tokench(lex, ' ');
540 lex_tokench(lex, ' ');
549 if (lex->flags.preprocessing) {
550 lex_tokench(lex, ' ');
551 lex_tokench(lex, ' ');
555 lex_ungetch(lex, ch);
557 if (lex->flags.preprocessing) {
559 lex_tokench(lex, '\n');
561 lex_tokench(lex, ' ');
564 ch = ' '; /* cause TRUE in the isspace check */
567 /* Otherwise roll back to the slash and break out of the loop */
568 lex_ungetch(lex, ch);
572 } while (ch != EOF && util_isspace(ch));
576 lex_ungetch(lex, ch);
583 static bool GMQCC_WARN lex_finish_ident(lex_file *lex, bool allow_dot)
588 while (ch != EOF && isident(ch, allow_dot))
590 lex_tokench(lex, ch);
594 /* last ch was not an ident ch: */
595 lex_ungetch(lex, ch);
600 /* read one ident for the frame list */
601 static int lex_parse_frame(lex_file *lex)
608 while (ch != EOF && ch != '\n' && util_isspace(ch))
614 if (!isident_start(ch)) {
615 lexerror(lex, "invalid framename, must start with one of a-z, or _, got %c", ch);
619 lex_tokench(lex, ch);
620 if (!lex_finish_ident(lex, true))
626 /* read a list of $frames */
627 static bool lex_finish_frames(lex_file *lex)
634 rc = lex_parse_frame(lex);
635 if (rc > 0) /* end of line */
637 if (rc < 0) /* error */
640 for (i = 0; i < vec_size(lex->frames); ++i) {
641 if (!strcmp(lex->tok.value, lex->frames[i].name)) {
642 lex->frames[i].value = lex->framevalue++;
643 if (lexwarn(lex, WARN_FRAME_MACROS, "duplicate frame macro defined: `%s`", lex->tok.value))
648 if (i < vec_size(lex->frames))
651 m.value = lex->framevalue++;
652 m.name = util_strdup(lex->tok.value);
653 vec_shrinkto(lex->tok.value, 0);
654 vec_push(lex->frames, m);
660 static int GMQCC_WARN lex_finish_string(lex_file *lex, int quote)
663 int ch = 0, texttype = 0;
667 char u8buf[8]; /* way more than enough */
674 return TOKEN_STRINGCONST;
676 if (lex->flags.preprocessing && ch == '\\') {
677 lex_tokench(lex, ch);
680 lexerror(lex, "unexpected end of file");
681 lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
682 return (lex->tok.ttype = TOKEN_ERROR);
684 lex_tokench(lex, ch);
686 else if (ch == '\\') {
689 lexerror(lex, "unexpected end of file");
690 lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
691 return (lex->tok.ttype = TOKEN_ERROR);
698 case 'a': ch = '\a'; break;
699 case 'r': ch = '\r'; break;
700 case 'n': ch = '\n'; break;
701 case 't': ch = '\t'; break;
702 case 'f': ch = '\f'; break;
703 case 'v': ch = '\v'; break;
706 /* same procedure as in fteqcc */
708 nextch = lex_getch(lex);
709 if (nextch >= '0' && nextch <= '9')
711 else if (nextch >= 'a' && nextch <= 'f')
712 ch += nextch - 'a' + 10;
713 else if (nextch >= 'A' && nextch <= 'F')
714 ch += nextch - 'A' + 10;
716 lexerror(lex, "bad character code");
717 lex_ungetch(lex, nextch);
718 return (lex->tok.ttype = TOKEN_ERROR);
722 nextch = lex_getch(lex);
723 if (nextch >= '0' && nextch <= '9')
725 else if (nextch >= 'a' && nextch <= 'f')
726 ch += nextch - 'a' + 10;
727 else if (nextch >= 'A' && nextch <= 'F')
728 ch += nextch - 'A' + 10;
730 lexerror(lex, "bad character code");
731 lex_ungetch(lex, nextch);
732 return (lex->tok.ttype = TOKEN_ERROR);
737 case '0': case '1': case '2': case '3':
738 case '4': case '5': case '6': case '7':
742 case '<': ch = 29; break;
743 case '-': ch = 30; break;
744 case '>': ch = 31; break;
745 case '[': ch = 16; break;
746 case ']': ch = 17; break;
749 nextch = lex_getch(lex);
750 hex = (nextch == 'x');
751 oct = (nextch == '0');
753 lex_ungetch(lex, nextch);
754 for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) {
756 if (nextch >= '0' && nextch <= '9')
757 chr = chr * 10 + nextch - '0';
759 lexerror(lex, "bad character code");
760 return (lex->tok.ttype = TOKEN_ERROR);
763 if (nextch >= '0' && nextch <= '9')
764 chr = chr * 0x10 + nextch - '0';
765 else if (nextch >= 'a' && nextch <= 'f')
766 chr = chr * 0x10 + nextch - 'a' + 10;
767 else if (nextch >= 'A' && nextch <= 'F')
768 chr = chr * 0x10 + nextch - 'A' + 10;
770 lexerror(lex, "bad character code");
771 return (lex->tok.ttype = TOKEN_ERROR);
774 if (nextch >= '0' && nextch <= '9')
775 chr = chr * 8 + chr - '0';
777 lexerror(lex, "bad character code");
778 return (lex->tok.ttype = TOKEN_ERROR);
781 if (chr > 0x10FFFF || (!OPTS_FLAG(UTF8) && chr > 255))
783 lexerror(lex, "character code out of range");
784 return (lex->tok.ttype = TOKEN_ERROR);
787 if (OPTS_FLAG(UTF8) && chr >= 128) {
788 u8len = utf8_from(u8buf, chr);
793 lex->column += u8len;
794 for (uc = 0; uc < u8len; ++uc)
795 lex_tokench(lex, u8buf[uc]);
797 * the last character will be inserted with the tokench() call
817 lexwarn(lex, WARN_UNKNOWN_CONTROL_SEQUENCE, "unrecognized control sequence: \\%c", ch);
818 /* so we just add the character plus backslash no matter what it actually is */
819 lex_tokench(lex, '\\');
821 /* add the character finally */
822 lex_tokench(lex, ch | texttype);
825 lex_tokench(lex, ch);
827 lexerror(lex, "unexpected end of file within string constant");
828 lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
829 return (lex->tok.ttype = TOKEN_ERROR);
832 static int GMQCC_WARN lex_finish_digit(lex_file *lex, int lastch)
839 /* parse a number... */
841 lex->tok.ttype = TOKEN_FLOATCONST;
843 lex->tok.ttype = TOKEN_INTCONST;
845 lex_tokench(lex, ch);
849 if (lastch == '0' && util_isdigit(ch)) {
850 if (ch < '0' || ch > '7') {
851 lexerror(lex, "invalid octal constant");
852 return (lex->tok.ttype = TOKEN_ERROR);
857 if (!isoct && ch != '.' && !util_isdigit(ch))
859 if (lastch != '0' || ch != 'x')
861 /* end of the number or EOF */
862 lex_ungetch(lex, ch);
865 lex->tok.constval.i = lastch - '0';
866 return lex->tok.ttype;
872 /* EOF would have been caught above */
876 lex_tokench(lex, ch);
878 while (util_isdigit(ch) || (ishex && isxdigit_only(ch)))
880 lex_tokench(lex, ch);
884 /* NOT else, '.' can come from above as well */
885 if (lex->tok.ttype != TOKEN_FLOATCONST && ch == '.' && !ishex)
887 /* Allow floating comma in non-hex mode */
888 lex->tok.ttype = TOKEN_FLOATCONST;
889 lex_tokench(lex, ch);
891 /* continue digits-only */
893 while (util_isdigit(ch))
895 lex_tokench(lex, ch);
899 /* put back the last character */
900 /* but do not put back the trailing 'f' or a float */
901 if (lex->tok.ttype == TOKEN_FLOATCONST && ch == 'f')
904 /* generally we don't want words to follow numbers: */
905 if (isident(ch, false)) {
906 lexerror(lex, "unexpected trailing characters after number");
907 return (lex->tok.ttype = TOKEN_ERROR);
909 lex_ungetch(lex, ch);
912 if (lex->tok.ttype == TOKEN_FLOATCONST) {
913 lex->tok.constval.f = strtod(lex->tok.value, nullptr);
915 /* determine base for strtol */
917 if (ishex) base = 16;
919 lex->tok.constval.i = strtol(lex->tok.value, nullptr, base);
921 return lex->tok.ttype;
924 int lex_do(lex_file *lex)
926 int ch, nextch, thirdch;
927 bool hadwhite = false;
932 ch = lex_skipwhite(lex, hadwhite);
934 if (!lex->flags.mergelines || ch != '\\')
940 lex_ungetch(lex, ch);
944 /* we reached a linemerge */
945 lex_tokench(lex, '\n');
949 if (lex->flags.preprocessing && (ch == TOKEN_WHITE || ch == TOKEN_EOL || ch == TOKEN_FATAL)) {
950 return (lex->tok.ttype = ch);
953 lex->sline = lex->line;
954 lex->tok.ctx.line = lex->sline;
955 lex->tok.ctx.file = lex->name;
958 return (lex->tok.ttype = TOKEN_FATAL);
962 return (lex->tok.ttype = TOKEN_EOF);
965 /* modelgen / spiritgen commands */
966 if (ch == '$' && !lex->flags.preprocessing) {
971 if (!isident_start(ch)) {
972 lexerror(lex, "hanging '$' modelgen/spritegen command line");
975 lex_tokench(lex, ch);
976 if (!lex_finish_ident(lex, true))
977 return (lex->tok.ttype = TOKEN_ERROR);
979 /* skip the known commands */
982 if (!strcmp(v, "frame") || !strcmp(v, "framesave"))
984 /* frame/framesave command works like an enum
985 * similar to fteqcc we handle this in the lexer.
986 * The reason for this is that it is sensitive to newlines,
987 * which the parser is unaware of
989 if (!lex_finish_frames(lex))
990 return (lex->tok.ttype = TOKEN_ERROR);
994 if (!strcmp(v, "framevalue"))
997 while (ch != EOF && util_isspace(ch) && ch != '\n')
1000 if (!util_isdigit(ch)) {
1001 lexerror(lex, "$framevalue requires an integer parameter");
1006 lex->tok.ttype = lex_finish_digit(lex, ch);
1008 if (lex->tok.ttype != TOKEN_INTCONST) {
1009 lexerror(lex, "$framevalue requires an integer parameter");
1012 lex->framevalue = lex->tok.constval.i;
1016 if (!strcmp(v, "framerestore"))
1022 rc = lex_parse_frame(lex);
1025 lexerror(lex, "$framerestore requires a framename parameter");
1029 return (lex->tok.ttype = TOKEN_FATAL);
1032 for (frame = 0; frame < vec_size(lex->frames); ++frame) {
1033 if (!strcmp(v, lex->frames[frame].name)) {
1034 lex->framevalue = lex->frames[frame].value;
1038 lexerror(lex, "unknown framename `%s`", v);
1042 if (!strcmp(v, "modelname"))
1048 rc = lex_parse_frame(lex);
1051 lexerror(lex, "$modelname requires a parameter");
1055 return (lex->tok.ttype = TOKEN_FATAL);
1057 if (lex->modelname) {
1059 m.value = lex->framevalue;
1060 m.name = lex->modelname;
1061 lex->modelname = nullptr;
1062 vec_push(lex->frames, m);
1064 lex->modelname = lex->tok.value;
1065 lex->tok.value = nullptr;
1069 if (!strcmp(v, "flush"))
1072 for (fi = 0; fi < vec_size(lex->frames); ++fi)
1073 mem_d(lex->frames[fi].name);
1074 vec_free(lex->frames);
1075 /* skip line (fteqcc does it too) */
1076 ch = lex_getch(lex);
1077 while (ch != EOF && ch != '\n')
1078 ch = lex_getch(lex);
1082 if (!strcmp(v, "cd") ||
1083 !strcmp(v, "origin") ||
1084 !strcmp(v, "base") ||
1085 !strcmp(v, "flags") ||
1086 !strcmp(v, "scale") ||
1090 ch = lex_getch(lex);
1091 while (ch != EOF && ch != '\n')
1092 ch = lex_getch(lex);
1096 for (frame = 0; frame < vec_size(lex->frames); ++frame) {
1097 if (!strcmp(v, lex->frames[frame].name)) {
1098 lex->tok.constval.i = lex->frames[frame].value;
1099 return (lex->tok.ttype = TOKEN_INTCONST);
1103 lexerror(lex, "invalid frame macro");
1107 /* single-character tokens */
1111 nextch = lex_getch(lex);
1112 if (nextch == '[') {
1113 lex_tokench(lex, ch);
1114 lex_tokench(lex, nextch);
1116 return (lex->tok.ttype = TOKEN_ATTRIBUTE_OPEN);
1118 lex_ungetch(lex, nextch);
1123 lex_tokench(lex, ch);
1125 if (lex->flags.noops)
1126 return (lex->tok.ttype = ch);
1128 return (lex->tok.ttype = TOKEN_OPERATOR);
1131 if (lex->flags.noops) {
1132 nextch = lex_getch(lex);
1133 if (nextch == ']') {
1134 lex_tokench(lex, ch);
1135 lex_tokench(lex, nextch);
1137 return (lex->tok.ttype = TOKEN_ATTRIBUTE_CLOSE);
1139 lex_ungetch(lex, nextch);
1148 lex_tokench(lex, ch);
1150 return (lex->tok.ttype = ch);
1156 nextch = lex_getch(lex);
1157 /* digits starting with a dot */
1158 if (util_isdigit(nextch)) {
1159 lex_ungetch(lex, nextch);
1160 lex->tok.ttype = lex_finish_digit(lex, ch);
1162 return lex->tok.ttype;
1164 lex_ungetch(lex, nextch);
1167 if (lex->flags.noops)
1169 /* Detect characters early which are normally
1170 * operators OR PART of an operator.
1185 lex_tokench(lex, ch);
1187 return (lex->tok.ttype = ch);
1195 lex_tokench(lex, ch);
1196 /* peak ahead once */
1197 nextch = lex_getch(lex);
1198 if (nextch != '.') {
1199 lex_ungetch(lex, nextch);
1201 if (lex->flags.noops)
1202 return (lex->tok.ttype = ch);
1204 return (lex->tok.ttype = TOKEN_OPERATOR);
1206 /* peak ahead again */
1207 nextch = lex_getch(lex);
1208 if (nextch != '.') {
1209 lex_ungetch(lex, nextch);
1210 lex_ungetch(lex, '.');
1212 if (lex->flags.noops)
1213 return (lex->tok.ttype = ch);
1215 return (lex->tok.ttype = TOKEN_OPERATOR);
1217 /* fill the token to be "..." */
1218 lex_tokench(lex, ch);
1219 lex_tokench(lex, ch);
1221 return (lex->tok.ttype = TOKEN_DOTS);
1224 if (ch == ',' || ch == '.') {
1225 lex_tokench(lex, ch);
1227 return (lex->tok.ttype = TOKEN_OPERATOR);
1230 if (ch == '+' || ch == '-' || /* ++, --, +=, -= and -> as well! */
1231 ch == '>' || ch == '<' || /* <<, >>, <=, >= and >< as well! */
1232 ch == '=' || ch == '!' || /* <=>, ==, != */
1233 ch == '&' || ch == '|' || /* &&, ||, &=, |= */
1234 ch == '~' || ch == '^' /* ~=, ~, ^ */
1236 lex_tokench(lex, ch);
1237 nextch = lex_getch(lex);
1239 if ((nextch == '=' && ch != '<') || (nextch == '<' && ch == '>'))
1240 lex_tokench(lex, nextch);
1241 else if (nextch == ch && ch != '!') {
1242 lex_tokench(lex, nextch);
1243 if ((thirdch = lex_getch(lex)) == '=')
1244 lex_tokench(lex, thirdch);
1246 lex_ungetch(lex, thirdch);
1247 } else if (ch == '<' && nextch == '=') {
1248 lex_tokench(lex, nextch);
1249 if ((thirdch = lex_getch(lex)) == '>')
1250 lex_tokench(lex, thirdch);
1252 lex_ungetch(lex, thirdch);
1254 } else if (ch == '-' && nextch == '>') {
1255 lex_tokench(lex, nextch);
1256 } else if (ch == '&' && nextch == '~') {
1257 thirdch = lex_getch(lex);
1258 if (thirdch != '=') {
1259 lex_ungetch(lex, thirdch);
1260 lex_ungetch(lex, nextch);
1263 lex_tokench(lex, nextch);
1264 lex_tokench(lex, thirdch);
1267 else if (lex->flags.preprocessing &&
1268 ch == '-' && util_isdigit(nextch))
1270 lex->tok.ttype = lex_finish_digit(lex, nextch);
1271 if (lex->tok.ttype == TOKEN_INTCONST)
1272 lex->tok.constval.i = -lex->tok.constval.i;
1274 lex->tok.constval.f = -lex->tok.constval.f;
1276 return lex->tok.ttype;
1278 lex_ungetch(lex, nextch);
1282 return (lex->tok.ttype = TOKEN_OPERATOR);
1285 if (ch == '*' || ch == '/') /* *=, /= */
1287 lex_tokench(lex, ch);
1289 nextch = lex_getch(lex);
1290 if (nextch == '=' || nextch == '*') {
1291 lex_tokench(lex, nextch);
1293 lex_ungetch(lex, nextch);
1296 return (lex->tok.ttype = TOKEN_OPERATOR);
1300 lex_tokench(lex, ch);
1302 return (lex->tok.ttype = TOKEN_OPERATOR);
1305 if (isident_start(ch))
1309 lex_tokench(lex, ch);
1310 if (!lex_finish_ident(lex, false)) {
1312 return (lex->tok.ttype = TOKEN_ERROR);
1315 lex->tok.ttype = TOKEN_IDENT;
1318 if (!strcmp(v, "void")) {
1319 lex->tok.ttype = TOKEN_TYPENAME;
1320 lex->tok.constval.t = TYPE_VOID;
1321 } else if (!strcmp(v, "int")) {
1322 lex->tok.ttype = TOKEN_TYPENAME;
1323 lex->tok.constval.t = TYPE_INTEGER;
1324 } else if (!strcmp(v, "float")) {
1325 lex->tok.ttype = TOKEN_TYPENAME;
1326 lex->tok.constval.t = TYPE_FLOAT;
1327 } else if (!strcmp(v, "string")) {
1328 lex->tok.ttype = TOKEN_TYPENAME;
1329 lex->tok.constval.t = TYPE_STRING;
1330 } else if (!strcmp(v, "entity")) {
1331 lex->tok.ttype = TOKEN_TYPENAME;
1332 lex->tok.constval.t = TYPE_ENTITY;
1333 } else if (!strcmp(v, "vector")) {
1334 lex->tok.ttype = TOKEN_TYPENAME;
1335 lex->tok.constval.t = TYPE_VECTOR;
1336 } else if (!strcmp(v, "_length")) {
1337 lex->tok.ttype = TOKEN_OPERATOR;
1340 for (kw = 0; kw < GMQCC_ARRAY_COUNT(keywords_qc); ++kw) {
1341 if (!strcmp(v, keywords_qc[kw]))
1342 return (lex->tok.ttype = TOKEN_KEYWORD);
1344 if (OPTS_OPTION_U32(OPTION_STANDARD) != COMPILER_QCC) {
1345 for (kw = 0; kw < GMQCC_ARRAY_COUNT(keywords_fg); ++kw) {
1346 if (!strcmp(v, keywords_fg[kw]))
1347 return (lex->tok.ttype = TOKEN_KEYWORD);
1352 return lex->tok.ttype;
1357 lex->flags.nodigraphs = true;
1358 if (lex->flags.preprocessing)
1359 lex_tokench(lex, ch);
1360 lex->tok.ttype = lex_finish_string(lex, '"');
1361 if (lex->flags.preprocessing)
1362 lex_tokench(lex, ch);
1363 while (!lex->flags.preprocessing && lex->tok.ttype == TOKEN_STRINGCONST)
1365 /* Allow c style "string" "continuation" */
1366 ch = lex_skipwhite(lex, false);
1368 lex_ungetch(lex, ch);
1372 lex->tok.ttype = lex_finish_string(lex, '"');
1374 lex->flags.nodigraphs = false;
1376 return lex->tok.ttype;
1381 /* we parse character constants like string,
1382 * but return TOKEN_CHARCONST, or a vector type if it fits...
1383 * Likewise actual unescaping has to be done by the parser.
1384 * The difference is we don't allow 'char' 'continuation'.
1386 if (lex->flags.preprocessing)
1387 lex_tokench(lex, ch);
1388 lex->tok.ttype = lex_finish_string(lex, '\'');
1389 if (lex->flags.preprocessing)
1390 lex_tokench(lex, ch);
1393 lex->tok.ttype = TOKEN_CHARCONST;
1395 /* It's a vector if we can successfully scan 3 floats */
1396 if (util_sscanf(lex->tok.value, " %f %f %f ",
1397 &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3)
1400 lex->tok.ttype = TOKEN_VECTORCONST;
1404 if (!lex->flags.preprocessing && strlen(lex->tok.value) > 1) {
1406 /* check for a valid utf8 character */
1407 if (!OPTS_FLAG(UTF8) || !utf8_to(&u8char, (const unsigned char *)lex->tok.value, 8)) {
1408 if (lexwarn(lex, WARN_MULTIBYTE_CHARACTER,
1409 ( OPTS_FLAG(UTF8) ? "invalid multibyte character sequence `%s`"
1410 : "multibyte character: `%s`" ),
1412 return (lex->tok.ttype = TOKEN_ERROR);
1415 lex->tok.constval.i = u8char;
1418 lex->tok.constval.i = lex->tok.value[0];
1421 return lex->tok.ttype;
1424 if (util_isdigit(ch))
1426 lex->tok.ttype = lex_finish_digit(lex, ch);
1428 return lex->tok.ttype;
1431 if (lex->flags.preprocessing) {
1432 lex_tokench(lex, ch);
1434 return (lex->tok.ttype = ch);
1437 lexerror(lex, "unknown token: `%c`", ch);
1438 return (lex->tok.ttype = TOKEN_ERROR);