From b6c482826190f2dbb7035360287ae65d2ff63abf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= <puerdomus@gmail.com>
Date: Thu, 16 Jul 2026 20:25:09 +0300
Subject: [PATCH] Define tokens

---
 .gitignore        |  2 ++
 tests/token.odin  | 34 ++++++++++++++++++++++++++++++++++
 token/parser.odin | 40 ++++++++++++++++++++++++++++++++++++++++
 token/token.odin  | 30 ++++++++++++++++++++++++++++++
 4 files changed, 106 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 tests/token.odin
 create mode 100644 token/parser.odin
 create mode 100644 token/token.odin

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4f23461
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+*.bin
+debug_*
diff --git a/tests/token.odin b/tests/token.odin
new file mode 100644
index 0000000..8d6a9b0
--- /dev/null
+++ b/tests/token.odin
@@ -0,0 +1,34 @@
+package tests
+
+import "core:testing"
+import "../token"
+
+@(test)
+single_token :: proc(t: ^testing.T) {
+	Test :: struct {
+		source: string,
+		token: token.Token,
+	}
+
+	tests := []Test{
+		{source = "~", token = {type = .Illegal}},
+		{source = "", token = {type = .EndOfFile}},
+		{source = "x", token = {type = .Identifier}},
+		{source = "=", token = {type = .Assignment}},
+		{source = "+", token = {type = .Plus}},
+		{source = "(", token = {type = .LeftParenthesis}},
+		{source = ")", token = {type = .RightParenthesis}},
+		{source = "{", token = {type = .LeftBrace}},
+		{source = "}", token = {type = .RightBrace}},
+		{source = ",", token = {type = .Comma}},
+		{source = ";", token = {type = .Semicolon}},
+	}
+
+	for test in tests {
+		tokens := token.parse(test.source, context.temp_allocator)
+		testing.expect_value(t, len(tokens), 1)
+		testing.expect_value(t, tokens[0], test.token)
+	}
+
+	free_all(context.temp_allocator)
+}
diff --git a/token/parser.odin b/token/parser.odin
new file mode 100644
index 0000000..0093c2c
--- /dev/null
+++ b/token/parser.odin
@@ -0,0 +1,40 @@
+#+private
+package token
+
+Parser :: struct {
+	input: string,
+	left, right: int,
+	value: u8,
+}
+
+read_symbol :: proc(p: ^Parser) {
+	if p.left >= len(p.input) {
+		p.value = 0
+	} else {
+		p.value = p.input[p.right]
+	}
+
+	p.left = p.right
+	p.right += 1
+}
+
+next :: proc(p: ^Parser) -> Token {
+	token: Token
+
+	read_symbol(p)
+
+	switch p.value {
+	case 0: token.type = .EndOfFile
+	case '=': token.type = .Assignment
+	case '+': token.type = .Plus
+	case '(': token.type = .LeftParenthesis
+	case ')': token.type = .RightParenthesis
+	case '{': token.type = .LeftBrace
+	case '}': token.type = .RightBrace
+	case ',': token.type = .Comma
+	case ';': token.type = .Semicolon
+	case 'A'..='Z', 'a'..='z', '_': token.type = .Identifier
+	}
+
+	return token
+}
diff --git a/token/token.odin b/token/token.odin
new file mode 100644
index 0000000..d131bc1
--- /dev/null
+++ b/token/token.odin
@@ -0,0 +1,30 @@
+package token
+
+Type :: enum {
+	Illegal,
+	EndOfFile,
+	Identifier,
+	Assignment,
+	Integer,
+	Plus,
+	Comma,
+	Semicolon,
+	LeftParenthesis,
+	RightParenthesis,
+	LeftBrace,
+	RightBrace,
+	Function,
+	Let,
+}
+
+Token :: struct {
+	type: Type
+}
+
+parse :: proc(input: string, allocator := context.allocator) -> [dynamic]Token {
+	parser := Parser{input = input}
+	tokens := make([dynamic]Token, allocator)
+	token := next(&parser)
+	append(&tokens, token)
+	return tokens
+}
-- 
2.51.2