seeseemelk/c2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Token refactor and better logs

Browse Source
This commit is contained in:
Sebastiaan de Schaetzen
2026-04-24 20:28:08 +02:00
parent da3425ec10
commit 451a9a2a22
9 changed files with 210 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
v0/test_token.c
+39 -41
View File
@@ -11,10 +11,10 @@ static void test_tokenstream_simple_keyword(void) {
TokenStream* ts = tokenstream_open("module");
Token t = tokenstream_next(ts);
if (t != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (t.token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
Token eof = tokenstream_next(ts);
if (eof != -1) fail("expected EOF");
if (eof.token != TOKEN_EOF) fail("expected EOF");
tokenstream_close(ts);
}
@@ -22,13 +22,13 @@ static void test_tokenstream_simple_keyword(void) {
static void test_tokenstream_keywords_and_symbols(void) {
TokenStream* ts = tokenstream_open("module main; import stdio;");
if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (main)");
if (tokenstream_next(ts) != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts) != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (stdio)");
if (tokenstream_next(ts) != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts) != -1) fail("expected EOF");
if (tokenstream_next(ts).token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (main)");
if (tokenstream_next(ts).token != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts).token != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (stdio)");
if (tokenstream_next(ts).token != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts).token != TOKEN_EOF) fail("expected EOF");
tokenstream_close(ts);
}
@@ -36,11 +36,11 @@ static void test_tokenstream_keywords_and_symbols(void) {
static void test_tokenstream_parentheses_and_brackets(void) {
TokenStream* ts = tokenstream_open("()[]");
if (tokenstream_next(ts) != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts) != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
if (tokenstream_next(ts) != TOKEN_BRACKET_OPEN) fail("expected TOKEN_BRACKET_OPEN");
if (tokenstream_next(ts) != TOKEN_BRACKET_CLOSE) fail("expected TOKEN_BRACKET_CLOSE");
if (tokenstream_next(ts) != -1) fail("expected EOF");
if (tokenstream_next(ts).token != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts).token != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
if (tokenstream_next(ts).token != TOKEN_BRACKET_OPEN) fail("expected TOKEN_BRACKET_OPEN");
if (tokenstream_next(ts).token != TOKEN_BRACKET_CLOSE) fail("expected TOKEN_BRACKET_CLOSE");
if (tokenstream_next(ts).token != TOKEN_EOF) fail("expected EOF");
tokenstream_close(ts);
}
@@ -48,12 +48,12 @@ static void test_tokenstream_parentheses_and_brackets(void) {
static void test_tokenstream_comma(void) {
TokenStream* ts = tokenstream_open("a,b,c");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected a");
if (tokenstream_next(ts) != TOKEN_COMMA) fail("expected comma");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected b");
if (tokenstream_next(ts) != TOKEN_COMMA) fail("expected comma");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected c");
if (tokenstream_next(ts) != -1) fail("expected EOF");
if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected a");
if (tokenstream_next(ts).token != TOKEN_COMMA) fail("expected comma");
if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected b");
if (tokenstream_next(ts).token != TOKEN_COMMA) fail("expected comma");
if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected c");
if (tokenstream_next(ts).token != TOKEN_EOF) fail("expected EOF");
tokenstream_close(ts);
}
@@ -61,10 +61,10 @@ static void test_tokenstream_comma(void) {
static void test_tokenstream_whitespace_ignored(void) {
TokenStream* ts = tokenstream_open(" module \n\t import ; ");
if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts) != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
if (tokenstream_next(ts) != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts) != -1) fail("expected EOF");
if (tokenstream_next(ts).token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts).token != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
if (tokenstream_next(ts).token != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts).token != TOKEN_EOF) fail("expected EOF");
tokenstream_close(ts);
}
@@ -72,11 +72,11 @@ static void test_tokenstream_whitespace_ignored(void) {
static void test_tokenstream_void_function_signature(void) {
TokenStream* ts = tokenstream_open("void main()");
if (tokenstream_next(ts) != TOKEN_VOID) fail("expected TOKEN_VOID");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
if (tokenstream_next(ts) != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts) != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
if (tokenstream_next(ts) != -1) fail("expected EOF");
if (tokenstream_next(ts).token != TOKEN_VOID) fail("expected TOKEN_VOID");
if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
if (tokenstream_next(ts).token != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts).token != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
if (tokenstream_next(ts).token != TOKEN_EOF) fail("expected EOF");
tokenstream_close(ts);
}
@@ -85,26 +85,24 @@ static void test_tokenstream_info(void) {
TokenStream* ts = tokenstream_open("module main;");
Token t1 = tokenstream_next(ts);
TokenInfo info1;
tokenstream_info(ts, &info1);
if (t1 != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (info1.token != TOKEN_MODULE) fail("info: expected TOKEN_MODULE");
if (t1.token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
char buf1[32];
memcpy(buf1, info1.text, info1.text_length);
buf1[info1.text_length] = '\0';
memcpy(buf1, t1.text, t1.text_length);
buf1[t1.text_length] = '\0';
assert_str("module", buf1, "info: expected 'module'");
if (t1.line != 1) fail("expected line 1");
if (t1.column != 1) fail("expected column 1");
Token t2 = tokenstream_next(ts);
TokenInfo info2;
tokenstream_info(ts, &info2);
if (t2 != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
if (info2.token != TOKEN_IDENTIFIER) fail("info: expected TOKEN_IDENTIFIER");
if (t2.token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
char buf2[32];
memcpy(buf2, info2.text, info2.text_length);
buf2[info2.text_length] = '\0';
memcpy(buf2, t2.text, t2.text_length);
buf2[t2.text_length] = '\0';
assert_str("main", buf2, "info: expected 'main'");
if (t2.line != 1) fail("expected line 1");
if (t2.column != 8) fail("expected column 8");
tokenstream_close(ts);
}