Update tokenstream_open to accept Buffer parameter instead of file path

The tokenstream_open function now takes a Buffer* parameter instead of a file
path string, making the API more flexible and allowing the caller to manage
buffer lifetime. The tokenstream_close function continues to close the underlying
buffer as documented.

- Changed tokenstream_open signature from (const char* path) to (Buffer* buffer)
- Updated implementation to accept and use the provided buffer directly
- Updated all tests to open buffers separately and pass them to tokenstream_open
- Added #include "buffer.h" to token.h for Buffer type definition
- All 15 tests pass

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

v0/test_token.c

@@ -33,14 +33,20 @@ static void test_token_to_string_identifier(void) {
 }
 
 static void test_tokenstream_open_fail(void) {
-	TokenStream* ts = tokenstream_open("v0/does_not_exist.c2");
-	if (ts != NULL) fail("expected NULL for non-existent file");
+	Buffer* buf = buffer_open_file("v0/does_not_exist.c2");
+	if (buf != NULL) fail("expected NULL for non-existent file");
+	
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts != NULL) fail("expected NULL for NULL buffer");
 }
 
 static void test_tokenstream_simple_keyword(void) {
 	write_test_file("v0/test_token_tmp.c2", "module");
-	TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
-	if (ts == NULL) fail("could not open file");
+	Buffer* buf = buffer_open_file("v0/test_token_tmp.c2");
+	if (buf == NULL) fail("could not open file");
+
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts == NULL) fail("could not create tokenstream");
 
 	Token t = tokenstream_next(ts);
 	if (t != TOKEN_MODULE) fail("expected TOKEN_MODULE");
@@ -53,8 +59,11 @@ static void test_tokenstream_simple_keyword(void) {
 
 static void test_tokenstream_keywords_and_symbols(void) {
 	write_test_file("v0/test_token_tmp.c2", "module main; import stdio;");
-	TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
-	if (ts == NULL) fail("could not open file");
+	Buffer* buf = buffer_open_file("v0/test_token_tmp.c2");
+	if (buf == NULL) fail("could not open file");
+
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts == NULL) fail("could not create tokenstream");
 
 	if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
 	if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER (main)");
@@ -69,8 +78,11 @@ static void test_tokenstream_keywords_and_symbols(void) {
 
 static void test_tokenstream_parentheses_and_brackets(void) {
 	write_test_file("v0/test_token_tmp.c2", "()[]");
-	TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
-	if (ts == NULL) fail("could not open file");
+	Buffer* buf = buffer_open_file("v0/test_token_tmp.c2");
+	if (buf == NULL) fail("could not open file");
+
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts == NULL) fail("could not create tokenstream");
 
 	if (tokenstream_next(ts) != TOKEN_PARENT_OPEN) fail("expected TOKEN_PARENT_OPEN");
 	if (tokenstream_next(ts) != TOKEN_PARENT_CLOSE) fail("expected TOKEN_PARENT_CLOSE");
@@ -83,8 +95,11 @@ static void test_tokenstream_parentheses_and_brackets(void) {
 
 static void test_tokenstream_comma(void) {
 	write_test_file("v0/test_token_tmp.c2", "a,b,c");
-	TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
-	if (ts == NULL) fail("could not open file");
+	Buffer* buf = buffer_open_file("v0/test_token_tmp.c2");
+	if (buf == NULL) fail("could not open file");
+
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts == NULL) fail("could not create tokenstream");
 
 	if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected a");
 	if (tokenstream_next(ts) != TOKEN_COMMA) fail("expected comma");
@@ -98,8 +113,11 @@ static void test_tokenstream_comma(void) {
 
 static void test_tokenstream_whitespace_ignored(void) {
 	write_test_file("v0/test_token_tmp.c2", "  module  \n\t import  ;  ");
-	TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
-	if (ts == NULL) fail("could not open file");
+	Buffer* buf = buffer_open_file("v0/test_token_tmp.c2");
+	if (buf == NULL) fail("could not open file");
+
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts == NULL) fail("could not create tokenstream");
 
 	if (tokenstream_next(ts) != TOKEN_MODULE) fail("expected TOKEN_MODULE");
 	if (tokenstream_next(ts) != TOKEN_IMPORT) fail("expected TOKEN_IMPORT");
@@ -111,8 +129,11 @@ static void test_tokenstream_whitespace_ignored(void) {
 
 static void test_tokenstream_void_function_signature(void) {
 	write_test_file("v0/test_token_tmp.c2", "void main()");
-	TokenStream* ts = tokenstream_open("v0/test_token_tmp.c2");
-	if (ts == NULL) fail("could not open file");
+	Buffer* buf = buffer_open_file("v0/test_token_tmp.c2");
+	if (buf == NULL) fail("could not open file");
+
+	TokenStream* ts = tokenstream_open(buf);
+	if (ts == NULL) fail("could not create tokenstream");
 
 	if (tokenstream_next(ts) != TOKEN_VOID) fail("expected TOKEN_VOID");
 	if (tokenstream_next(ts) != TOKEN_IDENTIFIER) fail("expected TOKEN_IDENTIFIER");

v0/token.c

@@ -103,19 +103,15 @@ static Token read_keyword_or_identifier(TokenStream* ts, char first) {
 	return TOKEN_IDENTIFIER;
 }
 
-TokenStream* tokenstream_open(const char* path) {
-	if (path == NULL) return NULL;
-
-	Buffer* buf = buffer_open_file(path);
-	if (buf == NULL) return NULL;
+TokenStream* tokenstream_open(Buffer* buffer) {
+	if (buffer == NULL) return NULL;
 
 	TokenStream* ts = (TokenStream*)malloc(sizeof(struct TokenStream));
 	if (ts == NULL) {
-		buffer_close(buf);
 		return NULL;
 	}
 
-	ts->buffer = buf;
+	ts->buffer = buffer;
 	ts->lookahead = 0;
 	ts->has_lookahead = 0;
 	return ts;

v0/token.h

@@ -4,6 +4,8 @@
 #ifndef TOKEN_H
 #define TOKEN_H
 
+#include "buffer.h"
+
 typedef enum {
 	// Keywords
 	TOKEN_MODULE,
@@ -34,11 +36,14 @@ typedef struct TokenStream TokenStream;
 const char* token_to_string(Token token);
 
 /**
- * Opens a file and returns a TokenStream for it.
- * @param path The path to the file.
+ * Returns a TokenStream for a given buffer.
+ * 
+ * When the tokenstream is closed, the underlying buffer is also closed.
+ * 
+ * @param buffer The buffer to read from.
  * @returns A handle to the TokenStream.
  */
-TokenStream* tokenstream_open(const char* path);
+TokenStream* tokenstream_open(Buffer* buffer);
 
 /**
  * Closes a TokenStream.