diff --git a/v0/token.h b/v0/token.h index 9885b31..9482d68 100644 --- a/v0/token.h +++ b/v0/token.h @@ -6,6 +6,9 @@ #include "buffer.h" +/** + * A list of all possible tokens. + */ typedef enum { // Keywords TOKEN_MODULE, @@ -26,6 +29,20 @@ typedef enum { TOKEN_IDENTIFIER, } Token; +/** + * Holds additional information about a token. + */ +typedef struct { + /// @brief The textual representation of a token. + char* text; + + /// @brief The length of the `text` string. + size_t text_length; + + /// @brief The actual token. + Token token; +} TokenInfo; + typedef struct TokenStream TokenStream; /**