Refactor Token to use Location struct

This commit is contained in:
2026-04-24 22:07:00 +02:00
parent a89e61eedd
commit 26a1d0285e
4 changed files with 42 additions and 24 deletions
+26
View File
@@ -0,0 +1,26 @@
/**
* Location handling for error reporting.
*/
#ifndef LOCATION_H
#define LOCATION_H
#include <stddef.h>
typedef struct {
/// @brief The name of the file where the token was found.
char* filename;
/// @brief The entire line of text where the token was found.
char* line_text;
/// @brief The length of the `line_text` string.
size_t line_text_length;
/// @brief The line number where the token was found.
int line;
/// @brief The column number where the token was found.
int column;
} Location;
#endif
+4 -4
View File
@@ -101,8 +101,8 @@ static void test_tokenstream_info(void) {
memcpy(buf1, t1.text, t1.text_length);
buf1[t1.text_length] = '\0';
assert_str("module", buf1, "info: expected 'module'");
if (t1.line != 1) fail("expected line 1");
if (t1.column != 1) fail("expected column 1");
if (t1.location.line != 1) fail("expected line 1");
if (t1.location.column != 1) fail("expected column 1");
Token t2 = tokenstream_next(ts);
if (t2.token != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");
@@ -111,8 +111,8 @@ static void test_tokenstream_info(void) {
memcpy(buf2, t2.text, t2.text_length);
buf2[t2.text_length] = '\0';
assert_str("main", buf2, "info: expected 'main'");
if (t2.line != 1) fail("expected line 1");
if (t2.column != 8) fail("expected column 8");
if (t2.location.line != 1) fail("expected line 1");
if (t2.location.column != 8) fail("expected column 8");
tokenstream_close(ts);
}
+6 -5
View File
@@ -95,10 +95,11 @@ static Token create_token(TokenStream* ts, TokenType type, const char* text, siz
t.token = type;
t.text = (char*)text;
t.text_length = length;
t.line = line;
t.column = column;
t.line_text = (char*)line_start;
t.line_text_length = get_line_length(line_start);
t.location.filename = (char*)ts->filename;
t.location.line = line;
t.location.column = column;
t.location.line_text = (char*)line_start;
t.location.line_text_length = get_line_length(line_start);
return t;
}
@@ -191,6 +192,6 @@ Token tokenstream_next(TokenStream* ts) {
/* Unknown character */
Token t = create_token(ts, TOKEN_UNKNOWN, start_text, 1, start_line, start_column, line_start);
log_on_line(ts->filename, t.line_text, t.line, t.column, t.column, "unexpected token '%c'", c);
log_on_line(ts->filename, t.location.line_text, t.location.line, t.location.column, t.location.column, "unexpected token '%c'", c);
return t;
}
+6 -15
View File
@@ -4,7 +4,7 @@
#ifndef TOKEN_H
#define TOKEN_H
#include <stddef.h>
#include "location.h"
/**
* A list of all possible tokens.
@@ -37,27 +37,18 @@ typedef enum {
* Holds additional information about a token.
*/
typedef struct {
/// @brief The actual token.
TokenType token;
/// @brief The textual representation of a token.
/// Note that this is not necessarily null-terminated.
char* text;
/// @brief The entire line of text where the token was found.
char* line_text;
/// @brief The length of the `text` string.
size_t text_length;
/// @brief The length of the `line_text` string.
size_t line_text_length;
/// @brief The actual token.
TokenType token;
/// @brief The line number where the token was found.
int line;
/// @brief The column number where the token was found.
int column;
/// @brief The location of the token.
Location location;
} Token;
typedef struct TokenStream TokenStream;