#include "gmqcc.h"
#include "lexer.h"
+
/*
* List of Keywords
*/
}
#endif
+static void lex_ungetch(lex_file *lex, int ch);
+static int lex_getch(lex_file *lex);
+
lex_file* lex_open(const char *file)
{
- lex_file *lex;
- FILE *in = fs_file_open(file, "rb");
+ lex_file *lex;
+ fs_file_t *in = fs_file_open(file, "rb");
+ uint32_t read;
if (!in) {
lexerror(NULL, "open failed: '%s'\n", file);
lex->peekpos = 0;
lex->eof = false;
+ /* handle BOM */
+ if ((read = (lex_getch(lex) << 16) | (lex_getch(lex) << 8) | lex_getch(lex)) != 0xEFBBBF) {
+ lex_ungetch(lex, (read & 0x0000FF));
+ lex_ungetch(lex, (read & 0x00FF00) >> 8);
+ lex_ungetch(lex, (read & 0xFF0000) >> 16);
+ } else {
+ /*
+ * otherwise the lexer has advanced 3 bytes for the BOM, we need
+ * to set the column back to 0
+ */
+ lex->column = 0;
+ }
+
vec_push(lex_filenames, lex->name);
return lex;
}
mem_d(lex);
}
+
+
static int lex_fgetc(lex_file *lex)
{
if (lex->file) {
}
if (lex->open_string) {
if (lex->open_string_pos >= lex->open_string_length)
- return EOF;
+ return FS_FILE_EOF;
lex->column++;
return lex->open_string[lex->open_string_pos++];
}
- return EOF;
+ return FS_FILE_EOF;
}
/* Get or put-back data
* are working on.
* The are merely wrapping get/put in order to count line numbers.
*/
-static void lex_ungetch(lex_file *lex, int ch);
static int lex_try_trigraph(lex_file *lex, int old)
{
int c2, c3;
goto unroll;
lex->line = line;
- while (ch != '\n' && ch != EOF)
+ while (ch != '\n' && ch != FS_FILE_EOF)
ch = lex_getch(lex);
vec_free(command);
vec_free(param);
do
{
ch = lex_getch(lex);
- while (ch != EOF && util_isspace(ch)) {
+ while (ch != FS_FILE_EOF && util_isspace(ch)) {
if (ch == '\n') {
if (lex_try_pragma(lex))
continue;
lex_tokench(lex, ' ');
}
- while (ch != EOF && ch != '\n') {
+ while (ch != FS_FILE_EOF && ch != '\n') {
if (lex->flags.preprocessing)
lex_tokench(lex, ' '); /* ch); */
ch = lex_getch(lex);
lex_tokench(lex, ' ');
}
- while (ch != EOF)
+ while (ch != FS_FILE_EOF)
{
ch = lex_getch(lex);
if (ch == '*') {
ch = '/';
break;
}
- } while (ch != EOF && util_isspace(ch));
+ } while (ch != FS_FILE_EOF && util_isspace(ch));
if (haswhite) {
lex_endtoken(lex);
int ch;
ch = lex_getch(lex);
- while (ch != EOF && isident(ch))
+ while (ch != FS_FILE_EOF && isident(ch))
{
lex_tokench(lex, ch);
ch = lex_getch(lex);
lex_token_new(lex);
ch = lex_getch(lex);
- while (ch != EOF && ch != '\n' && util_isspace(ch))
+ while (ch != FS_FILE_EOF && ch != '\n' && util_isspace(ch))
ch = lex_getch(lex);
if (ch == '\n')
int ch = 0;
int nextch;
bool hex;
+ bool oct;
char u8buf[8]; /* way more than enough */
int u8len, uc;
- while (ch != EOF)
+ while (ch != FS_FILE_EOF)
{
ch = lex_getch(lex);
if (ch == quote)
if (lex->flags.preprocessing && ch == '\\') {
lex_tokench(lex, ch);
ch = lex_getch(lex);
- if (ch == EOF) {
+ if (ch == FS_FILE_EOF) {
lexerror(lex, "unexpected end of file");
- lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
+ lex_ungetch(lex, FS_FILE_EOF); /* next token to be TOKEN_EOF */
return (lex->tok.ttype = TOKEN_ERROR);
}
lex_tokench(lex, ch);
}
else if (ch == '\\') {
ch = lex_getch(lex);
- if (ch == EOF) {
+ if (ch == FS_FILE_EOF) {
lexerror(lex, "unexpected end of file");
- lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
+ lex_ungetch(lex, FS_FILE_EOF); /* next token to be TOKEN_EOF */
return (lex->tok.ttype = TOKEN_ERROR);
}
chr = 0;
nextch = lex_getch(lex);
hex = (nextch == 'x');
- if (!hex)
+ oct = (nextch == '0');
+ if (!hex && !oct)
lex_ungetch(lex, nextch);
for (nextch = lex_getch(lex); nextch != '}'; nextch = lex_getch(lex)) {
- if (!hex) {
+ if (!hex && !oct) {
if (nextch >= '0' && nextch <= '9')
chr = chr * 10 + nextch - '0';
else {
lexerror(lex, "bad character code");
return (lex->tok.ttype = TOKEN_ERROR);
}
- } else {
+ } else if (!oct) {
if (nextch >= '0' && nextch <= '9')
chr = chr * 0x10 + nextch - '0';
else if (nextch >= 'a' && nextch <= 'f')
lexerror(lex, "bad character code");
return (lex->tok.ttype = TOKEN_ERROR);
}
+ } else {
+ if (nextch >= '0' && nextch <= '9')
+ chr = chr * 8 + chr - '0';
+ else {
+ lexerror(lex, "bad character code");
+ return (lex->tok.ttype = TOKEN_ERROR);
+ }
}
if (chr > 0x10FFFF || (!OPTS_FLAG(UTF8) && chr > 255))
{
lex_tokench(lex, ch);
}
lexerror(lex, "unexpected end of file within string constant");
- lex_ungetch(lex, EOF); /* next token to be TOKEN_EOF */
+ lex_ungetch(lex, FS_FILE_EOF); /* next token to be TOKEN_EOF */
return (lex->tok.ttype = TOKEN_ERROR);
}
if (lex->eof)
return (lex->tok.ttype = TOKEN_FATAL);
- if (ch == EOF) {
+ if (ch == FS_FILE_EOF) {
lex->eof = true;
return (lex->tok.ttype = TOKEN_EOF);
}
if (!strcmp(v, "framevalue"))
{
ch = lex_getch(lex);
- while (ch != EOF && util_isspace(ch) && ch != '\n')
+ while (ch != FS_FILE_EOF && util_isspace(ch) && ch != '\n')
ch = lex_getch(lex);
if (!util_isdigit(ch)) {
vec_free(lex->frames);
/* skip line (fteqcc does it too) */
ch = lex_getch(lex);
- while (ch != EOF && ch != '\n')
+ while (ch != FS_FILE_EOF && ch != '\n')
ch = lex_getch(lex);
return lex_do(lex);
}
{
/* skip line */
ch = lex_getch(lex);
- while (ch != EOF && ch != '\n')
+ while (ch != FS_FILE_EOF && ch != '\n')
ch = lex_getch(lex);
return lex_do(lex);
}
lex_endtoken(lex);
lex->tok.ttype = TOKEN_CHARCONST;
- /* It's a vector if we can successfully scan 3 floats */
-#ifdef _MSC_VER
- if (sscanf_s(lex->tok.value, " %f %f %f ",
- &lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3)
-#else
- if (sscanf(lex->tok.value, " %f %f %f ",
+
+ /* It's a vector if we can successfully scan 3 floats */
+ if (util_sscanf(lex->tok.value, " %f %f %f ",
&lex->tok.constval.v.x, &lex->tok.constval.v.y, &lex->tok.constval.v.z) == 3)
-#endif
{
lex->tok.ttype = TOKEN_VECTORCONST;