Add tokeninfo

v0/token.h

@@ -6,6 +6,9 @@
 
 #include "buffer.h"
 
+/**
+ * A list of all possible tokens.
+ */
 typedef enum {
 	// Keywords
 	TOKEN_MODULE,
@@ -26,6 +29,20 @@ typedef enum {
 	TOKEN_IDENTIFIER,
 } Token;
 
+/**
+ * Holds additional information about a token.
+ */
+typedef struct {
+	/// @brief The textual representation of a token.
+	char* text;
+
+	/// @brief The length of the `text` string.
+	size_t text_length;
+
+	/// @brief The actual token.
+	Token token;
+} TokenInfo;
+
 typedef struct TokenStream TokenStream;
 
 /**