Token refactor and better logs

v0/token.h

@@ -27,7 +27,11 @@ typedef enum {
 
 	// Variable
 	TOKEN_IDENTIFIER,
-} Token;
+
+	// Others
+	TOKEN_EOF,
+	TOKEN_UNKNOWN,
+} TokenType;
 
 /**
  * Holds additional information about a token.
@@ -37,12 +41,24 @@ typedef struct {
 	/// Note that this is not necessarily null-terminated.
 	char* text;
 
+	/// @brief The entire line of text where the token was found.
+	char* line_text;
+
 	/// @brief The length of the `text` string.
 	size_t text_length;
 
+	/// @brief The length of the `line_text` string.
+	size_t line_text_length;
+
 	/// @brief The actual token.
-	Token token;
-} TokenInfo;
+	TokenType token;
+
+	/// @brief The line number where the token was found.
+	int line;
+
+	/// @brief The column number where the token was found.
+	int column;
+} Token;
 
 typedef struct TokenStream TokenStream;
 
@@ -67,13 +83,4 @@ void tokenstream_close(TokenStream* ts);
  */
 Token tokenstream_next(TokenStream* ts);
 
-/**
- * Gets additional information about the last token that was returned
- * by `tokenstream_next`.
- * 
- * @param ts The TokenStream to use.
- * @param info The TokenInfo object to store the results in.
- */
-void tokenstream_info(TokenStream* ts, TokenInfo* info);
-
 #endif