Implement tokenstream_info and refactor TokenStream interface

v0/token.h

@@ -4,7 +4,7 @@
 #ifndef TOKEN_H
 #define TOKEN_H
 
-#include "buffer.h"
+#include <stddef.h>
 
 /**
  * A list of all possible tokens.
@@ -34,6 +34,7 @@ typedef enum {
  */
 typedef struct {
 	/// @brief The textual representation of a token.
+	/// Note that this is not necessarily null-terminated.
 	char* text;
 
 	/// @brief The length of the `text` string.
@@ -46,14 +47,12 @@ typedef struct {
 typedef struct TokenStream TokenStream;
 
 /**
- * Returns a TokenStream for a given buffer.
+ * Returns a TokenStream for a text.
  * 
- * When the tokenstream is closed, the underlying buffer is also closed.
- * 
- * @param buffer The buffer to read from.
+ * @param code The text to read.
  * @returns A handle to the TokenStream.
  */
-TokenStream* tokenstream_open(Buffer* buffer);
+TokenStream* tokenstream_open(const char* code);
 
 /**
  * Closes a TokenStream.
@@ -68,4 +67,13 @@ void tokenstream_close(TokenStream* ts);
  */
 Token tokenstream_next(TokenStream* ts);
 
+/**
+ * Gets additional information about the last token that was returned
+ * by `tokenstream_next`.
+ * 
+ * @param ts The TokenStream to use.
+ * @param info The TokenInfo object to store the results in.
+ */
+void tokenstream_info(TokenStream* ts, TokenInfo* info);
+
 #endif