Better logging in tokenstream

v0/log.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
+#include <stdarg.h>
 
 static LogError* s_logError = NULL;
 
@@ -17,17 +18,24 @@ void log_error(const char* msg) {
     }
 }
 
-void log_on_line(const char* filename, const char* line_text, int line, int from, int to, const char* msg) {
+void log_on_line(const char* filename, const char* line_text, int line, int from, int to, const char* msg, ...) {
     char line_prefix[32];
     int prefix_len = snprintf(line_prefix, sizeof(line_prefix), "%d| ", line);
     
     int caret_len = to - from + 1;
     if (caret_len < 1) caret_len = 1;
 
+    // Format the message
+    va_list args;
+    va_start(args, msg);
+    char formatted_msg[256];
+    vsnprintf(formatted_msg, sizeof(formatted_msg), msg, args);
+    va_end(args);
+
     size_t total_size = strlen(filename) + 16 + // --- filename ---
                         prefix_len + strlen(line_text) + 2 + // line| text\n
                         prefix_len + from - 1 + caret_len + 2 + // indent + ^^\n
-                        prefix_len + strlen(msg) + 2 + // indent + msg\n
+                        prefix_len + strlen(formatted_msg) + 2 + // indent + msg\n
                         1;
 
     char* buffer = (char*)malloc(total_size);
@@ -44,7 +52,7 @@ void log_on_line(const char* filename, const char* line_text, int line, int from
 
     // Message line
     for (int i = 0; i < prefix_len; i++) *p++ = ' ';
-    p += sprintf(p, "%s\n", msg);
+    p += sprintf(p, "%s\n", formatted_msg);
     
     *p = '\0';
 

v0/log.h

@@ -27,8 +27,9 @@ void log_error(const char* msg);
  * @param line The line number where the error occurred.
  * @param from The column number where the error starts.
  * @param to The column number where the error ends.
- * @param msg The error message to log.
+ * @param msg The error message to log. This can contain format specifiers like printf, and the additional arguments will be formatted into the message.
+ * @param ... Additional arguments to format into the error message.
  */
-void log_on_line(const char* filename, const char* line_text, int line, int from, int to, const char* msg);
+void log_on_line(const char* filename, const char* line_text, int line, int from, int to, const char* msg, ...);
 
 #endif

v0/test.c

@@ -70,6 +70,7 @@ static TestCase s_tests[] = {
     {"tokenstream_comma", test_tokenstream_comma},
     {"tokenstream_whitespace_ignored", test_tokenstream_whitespace_ignored},
     {"tokenstream_void_function_signature", test_tokenstream_void_function_signature},
+    {"tokenstream_unknown_token", test_tokenstream_unknown_token},
     {"tokenstream_info", test_tokenstream_info},
     {"parser_module_name", test_parser_module_name},
     {"log_error", test_log_error},

v0/test_log.c

@@ -30,4 +30,14 @@ static void test_log_on_line(void) {
     log_on_line("test.c", "int main() []", 1, 12, 13, "unexpected token");
     
     assert_log(expected, "expected formatted error message");
+
+    log_clear();
+    const char* expected2 =
+        "--- test.c ---\n"
+        "1| int main() []\n"
+        "              ^^\n"
+        "   unexpected token 'x'\n";
+
+    log_on_line("test.c", "int main() []", 1, 12, 13, "unexpected token '%c'", 'x');
+    assert_log(expected2, "expected formatted error message with variadic args");
 }

v0/test_parser.c

@@ -3,7 +3,7 @@
 #include <string.h>
 
 static void test_parser_module_name(void) {
-	TokenStream* ts = tokenstream_open("module my_module;");
+	TokenStream* ts = tokenstream_open("test.c", "module my_module;");
 	Module* m = parser_parse(ts);
 
 	assert_not_null(m, "expected module to be parsed");

v0/test_token.c

@@ -3,12 +3,12 @@
 #include <string.h>
 
 static void test_tokenstream_open_fail(void) {
-	TokenStream* ts = tokenstream_open(NULL);
+	TokenStream* ts = tokenstream_open(NULL, NULL);
 	if (ts != NULL) fail("expected NULL for NULL buffer");
 }
 
 static void test_tokenstream_simple_keyword(void) {
-	TokenStream* ts = tokenstream_open("module");
+	TokenStream* ts = tokenstream_open("test.c", "module");
 
 	Token t = tokenstream_next(ts);
 	if (t.token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
@@ -20,7 +20,7 @@ static void test_tokenstream_simple_keyword(void) {
 }
 
 static void test_tokenstream_keywords_and_symbols(void) {
-	TokenStream* ts = tokenstream_open("module main; import stdio;");
+	TokenStream* ts = tokenstream_open("test.c", "module main; import stdio;");
 
 	if (tokenstream_next(ts).token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
 	if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (main)");
@@ -34,7 +34,7 @@ static void test_tokenstream_keywords_and_symbols(void) {
 }
 
 static void test_tokenstream_parentheses_and_brackets(void) {
-	TokenStream* ts = tokenstream_open("()[]");
+	TokenStream* ts = tokenstream_open("test.c", "()[]");
 
 	if (tokenstream_next(ts).token != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
 	if (tokenstream_next(ts).token != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
@@ -46,7 +46,7 @@ static void test_tokenstream_parentheses_and_brackets(void) {
 }
 
 static void test_tokenstream_comma(void) {
-	TokenStream* ts = tokenstream_open("a,b,c");
+	TokenStream* ts = tokenstream_open("test.c", "a,b,c");
 
 	if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected a");
 	if (tokenstream_next(ts).token != TOKEN_COMMA) fail("expected comma");
@@ -59,7 +59,7 @@ static void test_tokenstream_comma(void) {
 }
 
 static void test_tokenstream_whitespace_ignored(void) {
-	TokenStream* ts = tokenstream_open("  module  \n\t import  ;  ");
+	TokenStream* ts = tokenstream_open("test.c", "  module  \n\t import  ;  ");
 
 	if (tokenstream_next(ts).token != TOKEN_MODULE) fail("expected TOKEN_MODULE");
 	if (tokenstream_next(ts).token != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
@@ -70,7 +70,7 @@ static void test_tokenstream_whitespace_ignored(void) {
 }
 
 static void test_tokenstream_void_function_signature(void) {
-	TokenStream* ts = tokenstream_open("void main()");
+	TokenStream* ts = tokenstream_open("test.c", "void main()");
 
 	if (tokenstream_next(ts).token != TOKEN_VOID) fail("expected TOKEN_VOID");
 	if (tokenstream_next(ts).token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
@@ -81,8 +81,23 @@ static void test_tokenstream_void_function_signature(void) {
 	tokenstream_close(ts);
 }
 
+static void test_tokenstream_unknown_token(void) {
+	TokenStream* ts = tokenstream_open("test.c", "%");
+
+	if (tokenstream_next(ts).token != TOKEN_UNKNOWN) fail("expected TOKEN_UNKNOWN");
+	
+	assert_log(
+		"--- test.c ---\n"
+        "1| %\n"
+        "   ^\n"
+        "   unexpected token '%'\n",
+		"expected error message for unknown token");
+
+	tokenstream_close(ts);
+}
+
 static void test_tokenstream_info(void) {
-	TokenStream* ts = tokenstream_open("module main;");
+	TokenStream* ts = tokenstream_open("test.c", "module main;");
 
 	Token t1 = tokenstream_next(ts);
 	if (t1.token != TOKEN_MODULE) fail("expected TOKEN_MODULE");

v0/token.c

@@ -1,9 +1,11 @@
 #include "token.h"
+#include "log.h"
 #include <stdlib.h>
 #include <ctype.h>
 #include <string.h>
 
 struct TokenStream {
+	const char* filename;
 	const char* code;
 	size_t pos;
 	int line;
@@ -100,7 +102,7 @@ static Token create_token(TokenStream* ts, TokenType type, const char* text, siz
 	return t;
 }
 
-TokenStream* tokenstream_open(const char* code) {
+TokenStream* tokenstream_open(const char* filename, const char* code) {
 	if (code == NULL) return NULL;
 
 	TokenStream* ts = (TokenStream*)malloc(sizeof(struct TokenStream));
@@ -108,6 +110,7 @@ TokenStream* tokenstream_open(const char* code) {
 		return NULL;
 	}
 
+	ts->filename = filename ? filename : "unknown";
 	ts->code = code;
 	ts->pos = 0;
 	ts->line = 1;
@@ -187,5 +190,7 @@ Token tokenstream_next(TokenStream* ts) {
 	}
 
 	/* Unknown character */
-	return create_token(ts, TOKEN_UNKNOWN, start_text, 1, start_line, start_column, line_start);
+	Token t = create_token(ts, TOKEN_UNKNOWN, start_text, 1, start_line, start_column, line_start);
+	log_on_line(ts->filename, t.line_text, t.line, t.column, t.column, "unexpected token '%c'", c);
+	return t;
 }

v0/token.h

@@ -65,10 +65,11 @@ typedef struct TokenStream TokenStream;
 /**
  * Returns a TokenStream for a text.
  * 
+ * @param filename The name of the file to read. This is only used for error reporting.
  * @param code The text to read.
  * @returns A handle to the TokenStream.
  */
-TokenStream* tokenstream_open(const char* code);
+TokenStream* tokenstream_open(const char* filename, const char* code);
 
 /**
  * Closes a TokenStream.