seeseemelk/c2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement tokenstream_info and refactor TokenStream interface

Browse Source
This commit is contained in:
Sebastiaan de Schaetzen
2026-04-24 14:28:57 +02:00
parent 1406cedd82
commit b6aaa0c08f
4 changed files with 106 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
v0/test_token.c
+29 -12
View File
@@ -1,5 +1,6 @@
#include "test.h"
#include "token.h"
#include <string.h>
static void test_tokenstream_open_fail(void) {
TokenStream* ts = tokenstream_open(NULL);
@@ -7,8 +8,7 @@ static void test_tokenstream_open_fail(void) {
}
static void test_tokenstream_simple_keyword(void) {
Buffer* buf = buffer_open_string("module");
TokenStream* ts = tokenstream_open(buf);
TokenStream* ts = tokenstream_open("module");
Token t = tokenstream_next(ts);
if (t != TOKEN_MODULE) fail("expected TOKEN_MODULE");
@@ -20,8 +20,7 @@ static void test_tokenstream_simple_keyword(void) {
}
static void test_tokenstream_keywords_and_symbols(void) {
Buffer* buf = buffer_open_string("module main; import stdio;");
TokenStream* ts = tokenstream_open(buf);
TokenStream* ts = tokenstream_open("module main; import stdio;");
if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (main)");
@@ -35,8 +34,7 @@ static void test_tokenstream_keywords_and_symbols(void) {
}
static void test_tokenstream_parentheses_and_brackets(void) {
Buffer* buf = buffer_open_string("()[]");
TokenStream* ts = tokenstream_open(buf);
TokenStream* ts = tokenstream_open("()[]");
if (tokenstream_next(ts) != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts) != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
@@ -48,8 +46,7 @@ static void test_tokenstream_parentheses_and_brackets(void) {
}
static void test_tokenstream_comma(void) {
Buffer* buf = buffer_open_string("a,b,c");
TokenStream* ts = tokenstream_open(buf);
TokenStream* ts = tokenstream_open("a,b,c");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected a");
if (tokenstream_next(ts) != TOKEN_COMMA) fail("expected comma");
@@ -62,8 +59,7 @@ static void test_tokenstream_comma(void) {
}
static void test_tokenstream_whitespace_ignored(void) {
Buffer* buf = buffer_open_string(" module \n\t import ; ");
TokenStream* ts = tokenstream_open(buf);
TokenStream* ts = tokenstream_open(" module \n\t import ; ");
if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts) != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
@@ -74,8 +70,7 @@ static void test_tokenstream_whitespace_ignored(void) {
}
static void test_tokenstream_void_function_signature(void) {
Buffer* buf = buffer_open_string("void main()");
TokenStream* ts = tokenstream_open(buf);
TokenStream* ts = tokenstream_open("void main()");
if (tokenstream_next(ts) != TOKEN_VOID) fail("expected TOKEN_VOID");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
@@ -85,3 +80,25 @@ static void test_tokenstream_void_function_signature(void) {
tokenstream_close(ts);
}
static void test_tokenstream_info(void) {
TokenStream* ts = tokenstream_open("module main;");
Token t1 = tokenstream_next(ts);
TokenInfo info1;
tokenstream_info(ts, &info1);
if (t1 != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (info1.token != TOKEN_MODULE) fail("info: expected TOKEN_MODULE");
if (info1.text_length != 6) fail("info: expected length 6");
if (strncmp(info1.text, "module", 6) != 0) fail("info: expected 'module'");
Token t2 = tokenstream_next(ts);
TokenInfo info2;
tokenstream_info(ts, &info2);
if (t2 != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
if (info2.token != TOKEN_IDENTIFIER) fail("info: expected TOKEN_IDENTIFIER");
if (info2.text_length != 4) fail("info: expected length 4");
if (strncmp(info2.text, "main", 4) != 0) fail("info: expected 'main'");
tokenstream_close(ts);
}