seeseemelk
dccdcb8ba5
- Created token-to-string mapping array parallel to Token enum in token.c - Implemented TokenStream with lookahead buffering for proper tokenization - Implemented tokenstream_open/close/next functions with support for: - Keywords (module, import, void) - Symbols (parentheses, brackets, comma, semicolon) - Identifiers (alphanumeric starting with letter or underscore) - Comment skipping (// style) - Whitespace handling - Added token_to_string function to token.h for token inspection - Created comprehensive test suite (15 tests) covering all token types and edge cases - All tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
56 lines
1.0 KiB
C
56 lines
1.0 KiB
C
/**
|
|
* Contains the interface for reading tokens from a file.
|
|
*/
|
|
#ifndef TOKEN_H
|
|
#define TOKEN_H
|
|
|
|
typedef enum {
|
|
// Keywords
|
|
TOKEN_MODULE,
|
|
TOKEN_IMPORT,
|
|
TOKEN_SEMICOLON,
|
|
|
|
// Symbols
|
|
TOKEN_PARENT_OPEN,
|
|
TOKEN_PARENT_CLOSE,
|
|
TOKEN_BRACKET_OPEN,
|
|
TOKEN_BRACKET_CLOSE,
|
|
TOKEN_COMMA,
|
|
|
|
// Primitives
|
|
TOKEN_VOID,
|
|
|
|
// Variable
|
|
TOKEN_IDENTIFIER,
|
|
} Token;
|
|
|
|
typedef struct TokenStream TokenStream;
|
|
|
|
/**
|
|
* Convert a Token enum to its string representation.
|
|
* @param token The token to convert.
|
|
* @returns The string name of the token.
|
|
*/
|
|
const char* token_to_string(Token token);
|
|
|
|
/**
|
|
* Opens a file and returns a TokenStream for it.
|
|
* @param path The path to the file.
|
|
* @returns A handle to the TokenStream.
|
|
*/
|
|
TokenStream* tokenstream_open(const char* path);
|
|
|
|
/**
|
|
* Closes a TokenStream.
|
|
* @param ts The TokenStream to close.
|
|
*/
|
|
void tokenstream_close(TokenStream* ts);
|
|
|
|
/**
|
|
* Gets the next token from the TokenStream.
|
|
* @param ts The TokenStream to read from.
|
|
* @returns The next token read.
|
|
*/
|
|
Token tokenstream_next(TokenStream* ts);
|
|
|
|
#endif |