From 26e33280c49b82fb3e911435898e4074ea33a0cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Tue, 11 Aug 2026 19:36:41 +0300 Subject: [PATCH] Implement line tracking for tokens --- .gitignore | 2 +- lexing/lexer.odin | 61 ++++++++++++++++++++++------------------ tests/lexing/lexing.odin | 21 ++++++++------ token/token.odin | 1 + 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 5caa61d..8bd5ab1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ *.bin *.dSYM -debug_build +debug_* diff --git a/lexing/lexer.odin b/lexing/lexer.odin index deb4159..1e13e7c 100644 --- a/lexing/lexer.odin +++ b/lexing/lexer.odin @@ -9,6 +9,7 @@ Lexer :: struct { left, right: int, value: u8, pause: bool, + line: uint, } init :: proc(input: string) -> Lexer { @@ -29,6 +30,9 @@ read_symbol :: proc(l: ^Lexer) { l.pause = false return } + + if l.value == '\n' || l.value == '\r' do l.line += 1 + if l.right >= len(l.input) { l.value = 0 } else { @@ -38,6 +42,7 @@ read_symbol :: proc(l: ^Lexer) { l.left = l.right l.right += 1 } + read_whitespace :: proc(l: ^Lexer) { is_whitespace :: proc(symbol: u8) -> bool { switch symbol { @@ -57,9 +62,10 @@ read_integer :: proc(l: ^Lexer) -> token.Token { } left := l.left + line := l.line for is_digit(l.value) do read_symbol(l) l.pause = true - return {type = .Integer, literal = l.input[left:l.left]} + return {type = .Integer, literal = l.input[left:l.left], line = line} } read_keyword :: proc(l: ^Lexer) -> token.Token { @@ -73,27 +79,28 @@ read_keyword :: proc(l: ^Lexer) -> token.Token { } left := l.left + line := l.line for is_letter(l.value) do read_symbol(l) l.pause = true literal := l.input[left:l.left] switch literal { case "let": - return {type = .Let, literal = "let"} + return {type = .Let, literal = "let", line = line} case "fn": - return {type = .Function, literal = "fn"} + return {type = .Function, literal = "fn", line = line} case "true": - return {type = .True, literal = "true"} + return {type = .True, literal = "true", line = line} case "false": - return {type = .False, literal = "false"} + return {type = .False, literal = "false", line = line} case "if": - return {type = .If, literal = "if"} + return {type = .If, literal = "if", line = line} case "else": - return {type = .Else, literal = "else"} + return {type = .Else, literal = "else", line = line} case "return": - return {type = .Return, literal = "return"} + return {type = .Return, literal = "return", line = line} case: - return {type = .Identifier, literal = literal} + return {type = .Identifier, literal = literal, line = line} } } @@ -103,50 +110,50 @@ next :: proc(l: ^Lexer, allocator := context.allocator) -> token.Token { switch l.value { case 0: - return {type = .End_Of_File} + return {type = .End_Of_File, line = l.line} case '=': if peek_symbol(l) == '=' { defer read_symbol(l) - return {type = .Equal, literal = "=="} + return {type = .Equal, literal = "==", line = l.line} } - return {type = .Assignment, literal = "="} + return {type = .Assignment, literal = "=", line = l.line} case '+': - return {type = .Plus, literal = "+"} + return {type = .Plus, literal = "+", line = l.line} case '-': - return {type = .Minus, literal = "-"} + return {type = .Minus, literal = "-", line = l.line} case '!': if peek_symbol(l) == '=' { defer read_symbol(l) - return {type = .Not_Equal, literal = "!="} + return {type = .Not_Equal, literal = "!=", line = l.line} } - return {type = .Bang, literal = "!"} + return {type = .Bang, literal = "!", line = l.line} case '*': - return {type = .Asterisk, literal = "*"} + return {type = .Asterisk, literal = "*", line = l.line} case '/': - return {type = .Slash, literal = "/"} + return {type = .Slash, literal = "/", line = l.line} case '(': - return {type = .Left_Parenthesis, literal = "("} + return {type = .Left_Parenthesis, literal = "(", line = l.line} case ')': - return {type = .Right_Parenthesis, literal = ")"} + return {type = .Right_Parenthesis, literal = ")", line = l.line} case '{': - return {type = .Left_Brace, literal = "{"} + return {type = .Left_Brace, literal = "{", line = l.line} case '}': - return {type = .Right_Brace, literal = "}"} + return {type = .Right_Brace, literal = "}", line = l.line} case '<': - return {type = .Less_Than, literal = "<"} + return {type = .Less_Than, literal = "<", line = l.line} case '>': - return {type = .Greater_Than, literal = ">"} + return {type = .Greater_Than, literal = ">", line = l.line} case ',': - return {type = .Comma, literal = ","} + return {type = .Comma, literal = ",", line = l.line} case ';': - return {type = .Semicolon, literal = ";"} + return {type = .Semicolon, literal = ";", line = l.line} case '0' ..= '9': return read_integer(l) case 'A' ..= 'Z', 'a' ..= 'z', '_': return read_keyword(l) } - return {type = .Illegal, literal = symbol_to_string(l.value, allocator)} + return {type = .Illegal, literal = symbol_to_string(l.value, allocator), line = l.line} } symbol_to_string :: proc(symbol: u8, allocator := context.allocator) -> string { diff --git a/tests/lexing/lexing.odin b/tests/lexing/lexing.odin index 9e0d98f..1763845 100644 --- a/tests/lexing/lexing.odin +++ b/tests/lexing/lexing.odin @@ -7,21 +7,24 @@ import "../../token" import "core:testing" @(test) -ignores_whitespace :: proc(t: ^testing.T) { - tests := map[string]string { - "empty" = "", - "newline" = "\n", - "tab" = "\t", - "carriage return" = "\r", +whitespace :: proc(t: ^testing.T) { + tests := map[string]struct { + input: string, + output: token.Token, + } { + "empty" = {input = "", output = {type = .End_Of_File}}, + "newline" = {input = "\n", output = {type = .End_Of_File, line = 1}}, + "tab" = {input = "\t", output = {type = .End_Of_File}}, + "carriage return" = {input = "\r", output = {type = .End_Of_File, line = 1}}, } defer delete(tests) - for name, input in tests { - tokens := lexing.run(input, context.temp_allocator) + for name, data in tests { + tokens := lexing.run(data.input, context.temp_allocator) defer free_all(context.temp_allocator) expect.value(t, name, len(tokens), 1) - expect.value(t, name, tokens[0], token.Token{type = .End_Of_File}) + expect.value(t, name, tokens[0], data.output) } } diff --git a/token/token.odin b/token/token.odin index 273774d..995a498 100644 --- a/token/token.odin +++ b/token/token.odin @@ -33,4 +33,5 @@ Type :: enum { Token :: struct { type: Type, literal: string, // empty for end of file + line: uint, } -- 2.51.2