seeseemelk/c2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement token.c with comprehensive tests and easy-to-modify token mapping

Browse Source 
- Created token-to-string mapping array parallel to Token enum in token.c
- Implemented TokenStream with lookahead buffering for proper tokenization
- Implemented tokenstream_open/close/next functions with support for:
  - Keywords (module, import, void)
  - Symbols (parentheses, brackets, comma, semicolon)
  - Identifiers (alphanumeric starting with letter or underscore)
  - Comment skipping (// style)
  - Whitespace handling
- Added token_to_string function to token.h for token inspection
- Created comprehensive test suite (15 tests) covering all token types and edge cases
- All tests pass.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Sebastiaan de Schaetzen
2026-04-24 09:35:18 +02:00
parent c73f99d9e6
commit dccdcb8ba5
5 changed files with 323 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
v0/test_token.c
+124
View File
@@ -0,0 +1,124 @@
#include "test.h"
#include "token.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Helper to create a test file with content */
static void write_test_file(const char* filename, const char* content) {
FILE* f = fopen(filename, "w");
if (f) {
fputs(content, f);
fclose(f);
}
}
static void test_token_to_string_keywords(void) {
if (strcmp(token_to_string(TOKEN_MODULE), "module") != 0) fail("module");
if (strcmp(token_to_string(TOKEN_IMPORT), "import") != 0) fail("import");
if (strcmp(token_to_string(TOKEN_VOID), "void") != 0) fail("void");
}
static void test_token_to_string_symbols(void) {
if (strcmp(token_to_string(TOKEN_SEMICOLON), "semicolon") != 0) fail("semicolon");
if (strcmp(token_to_string(TOKEN_PARENT_OPEN), "paren_open") != 0) fail("paren_open");
if (strcmp(token_to_string(TOKEN_PARENT_CLOSE), "paren_close") != 0) fail("paren_close");
if (strcmp(token_to_string(TOKEN_BRACKET_OPEN), "bracket_open") != 0) fail("bracket_open");
if (strcmp(token_to_string(TOKEN_BRACKET_CLOSE), "bracket_close") != 0) fail("bracket_close");
if (strcmp(token_to_string(TOKEN_COMMA), "comma") != 0) fail("comma");
}
static void test_token_to_string_identifier(void) {
if (strcmp(token_to_string(TOKEN_IDENTIFIER), "identifier") != 0) fail("identifier");
}
static void test_tokenstream_open_fail(void) {
TokenStream* ts = tokenstream_open("v0/does_not_exist.c2");
if (ts != NULL) fail("expected NULL for non-existent file");
}
static void test_tokenstream_simple_keyword(void) {
write_test_file("v0/test_token_tmp.c2", "module");
TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
if (ts == NULL) fail("could not open file");
Token t = tokenstream_next(ts);
if (t != TOKEN_MODULE) fail("expected TOKEN_MODULE");
Token eof = tokenstream_next(ts);
if (eof != -1) fail("expected EOF");
tokenstream_close(ts);
}
static void test_tokenstream_keywords_and_symbols(void) {
write_test_file("v0/test_token_tmp.c2", "module main; import stdio;");
TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
if (ts == NULL) fail("could not open file");
if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (main)");
if (tokenstream_next(ts) != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts) != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (stdio)");
if (tokenstream_next(ts) != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts) != -1) fail("expected EOF");
tokenstream_close(ts);
}
static void test_tokenstream_parentheses_and_brackets(void) {
write_test_file("v0/test_token_tmp.c2", "()[]");
TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
if (ts == NULL) fail("could not open file");
if (tokenstream_next(ts) != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts) != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
if (tokenstream_next(ts) != TOKEN_BRACKET_OPEN) fail("expected TOKEN_BRACKET_OPEN");
if (tokenstream_next(ts) != TOKEN_BRACKET_CLOSE) fail("expected TOKEN_BRACKET_CLOSE");
if (tokenstream_next(ts) != -1) fail("expected EOF");
tokenstream_close(ts);
}
static void test_tokenstream_comma(void) {
write_test_file("v0/test_token_tmp.c2", "a,b,c");
TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
if (ts == NULL) fail("could not open file");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected a");
if (tokenstream_next(ts) != TOKEN_COMMA) fail("expected comma");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected b");
if (tokenstream_next(ts) != TOKEN_COMMA) fail("expected comma");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected c");
if (tokenstream_next(ts) != -1) fail("expected EOF");
tokenstream_close(ts);
}
static void test_tokenstream_whitespace_ignored(void) {
write_test_file("v0/test_token_tmp.c2", " module \n\t import ; ");
TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
if (ts == NULL) fail("could not open file");
if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
if (tokenstream_next(ts) != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
if (tokenstream_next(ts) != TOKEN_SEMICOLON) fail("expected TOKEN_SEMICOLON");
if (tokenstream_next(ts) != -1) fail("expected EOF");
tokenstream_close(ts);
}
static void test_tokenstream_void_function_signature(void) {
write_test_file("v0/test_token_tmp.c2", "void main()");
TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
if (ts == NULL) fail("could not open file");
if (tokenstream_next(ts) != TOKEN_VOID) fail("expected TOKEN_VOID");
if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
if (tokenstream_next(ts) != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
if (tokenstream_next(ts) != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
if (tokenstream_next(ts) != -1) fail("expected EOF");
tokenstream_close(ts);
}