Refactor Token to use Location struct

This commit is contained in:
2026-04-24 22:07:00 +02:00
parent a89e61eedd
commit 26a1d0285e
4 changed files with 42 additions and 24 deletions
+26
View File
@@ -0,0 +1,26 @@
/**
* Location handling for error reporting.
*/
#ifndef LOCATION_H
#define LOCATION_H
#include <stddef.h>
typedef struct {
/// @brief The name of the file where the token was found.
char* filename;
/// @brief The entire line of text where the token was found.
char* line_text;
/// @brief The length of the `line_text` string.
size_t line_text_length;
/// @brief The line number where the token was found.
int line;
/// @brief The column number where the token was found.
int column;
} Location;
#endif
+4 -4
View File
@@ -101,8 +101,8 @@ static void test_tokenstream_info(void) {
memcpy(buf1, t1.text, t1.text_length); memcpy(buf1, t1.text, t1.text_length);
buf1[t1.text_length] = '\0'; buf1[t1.text_length] = '\0';
assert_str("module", buf1, "info: expected 'module'"); assert_str("module", buf1, "info: expected 'module'");
if (t1.line != 1) fail("expected line 1"); if (t1.location.line != 1) fail("expected line 1");
if (t1.column != 1) fail("expected column 1"); if (t1.location.column != 1) fail("expected column 1");
Token t2 = tokenstream_next(ts); Token t2 = tokenstream_next(ts);
if (t2.token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER"); if (t2.token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
@@ -111,8 +111,8 @@ static void test_tokenstream_info(void) {
memcpy(buf2, t2.text, t2.text_length); memcpy(buf2, t2.text, t2.text_length);
buf2[t2.text_length] = '\0'; buf2[t2.text_length] = '\0';
assert_str("main", buf2, "info: expected 'main'"); assert_str("main", buf2, "info: expected 'main'");
if (t2.line != 1) fail("expected line 1"); if (t2.location.line != 1) fail("expected line 1");
if (t2.column != 8) fail("expected column 8"); if (t2.location.column != 8) fail("expected column 8");
tokenstream_close(ts); tokenstream_close(ts);
} }
+6 -5
View File
@@ -95,10 +95,11 @@ static Token create_token(TokenStream* ts, TokenType type, const char* text, siz
t.token = type; t.token = type;
t.text = (char*)text; t.text = (char*)text;
t.text_length = length; t.text_length = length;
t.line = line; t.location.filename = (char*)ts->filename;
t.column = column; t.location.line = line;
t.line_text = (char*)line_start; t.location.column = column;
t.line_text_length = get_line_length(line_start); t.location.line_text = (char*)line_start;
t.location.line_text_length = get_line_length(line_start);
return t; return t;
} }
@@ -191,6 +192,6 @@ Token tokenstream_next(TokenStream* ts) {
/* Unknown character */ /* Unknown character */
Token t = create_token(ts, TOKEN_UNKNOWN, start_text, 1, start_line, start_column, line_start); Token t = create_token(ts, TOKEN_UNKNOWN, start_text, 1, start_line, start_column, line_start);
log_on_line(ts->filename, t.line_text, t.line, t.column, t.column, "unexpected token '%c'", c); log_on_line(ts->filename, t.location.line_text, t.location.line, t.location.column, t.location.column, "unexpected token '%c'", c);
return t; return t;
} }
+6 -15
View File
@@ -4,7 +4,7 @@
#ifndef TOKEN_H #ifndef TOKEN_H
#define TOKEN_H #define TOKEN_H
#include <stddef.h> #include "location.h"
/** /**
* A list of all possible tokens. * A list of all possible tokens.
@@ -37,27 +37,18 @@ typedef enum {
* Holds additional information about a token. * Holds additional information about a token.
*/ */
typedef struct { typedef struct {
/// @brief The actual token.
TokenType token;
/// @brief The textual representation of a token. /// @brief The textual representation of a token.
/// Note that this is not necessarily null-terminated. /// Note that this is not necessarily null-terminated.
char* text; char* text;
/// @brief The entire line of text where the token was found.
char* line_text;
/// @brief The length of the `text` string. /// @brief The length of the `text` string.
size_t text_length; size_t text_length;
/// @brief The length of the `line_text` string. /// @brief The location of the token.
size_t line_text_length; Location location;
/// @brief The actual token.
TokenType token;
/// @brief The line number where the token was found.
int line;
/// @brief The column number where the token was found.
int column;
} Token; } Token;
typedef struct TokenStream TokenStream; typedef struct TokenStream TokenStream;