Something went wrong. Try again.
A general purpose .. programming language written in zig ijadux2.tngl.sh/Rinnescript
zig language
Something went wrong. Try again.
15 kB · 368 lines
Zig
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369const std = @import("std");
pub const TokenType = enum { // Keywords kw_const, kw_var, kw_fn, kw_pub, kw_if, kw_else, kw_while, kw_break, kw_continue, pipe, kw_for, kw_in, kw_and, kw_or, kw_not, kw_return, kw_defer, kw_struct, kw_true, kw_false, kw_null,
// Types type_int, type_float, type_str, type_bool,
// At Directives (@name) -- open-ended: any @word lexes to at_directive. // Known directives (@import, @stdout, @tensor, @vector, @import_c, ...) // are interpreted by the parser/compiler, not the lexer. at_directive,
// Identifiers & Literals identifier, int_literal, float_literal, string_literal,
// Punctuation & Operators assign, // = colon, // : semicolon, // ; comma, // , period, // . question, // ? plus, // + minus, // - star, // * slash, // / percent, // % bang, // ! l_paren, // ( r_paren, // ) l_brace, // { r_brace, // } l_bracket, // [ r_bracket, // ] equal_equal, // == not_equal, // != less, // < less_equal, // <= greater, // > greater_equal, // >= // Range operators for `for i in lo..hi` (exclusive) and lo..=hi (inclusive). dot_dot, // .. dot_dot_equal, // ..=
// Special eof, invalid,};
pub const Token = struct { tag: TokenType, lexeme: []const u8, line: usize, column: usize,
pub fn directiveName(self: Token) []const u8 { if (self.tag != .at_directive or self.lexeme.len < 2) return ""; return self.lexeme[1..]; }};
pub const Lexer = struct { source: []const u8, cursor: usize = 0, line: usize = 1, column: usize = 1,
pub fn init(source: []const u8) Lexer { return .{ .source = source }; }
pub fn nextToken(self: *Lexer) Token { self.skipWhitespaceAndComments();
if (self.cursor >= self.source.len) { return .{ .tag = .eof, .lexeme = "", .line = self.line, .column = self.column, }; }
const start_cursor = self.cursor; const start_col = self.column; const c = self.advance();
// @directives: single-pass O(1) tokenization of "@identifier". if (c == '@') { while (self.cursor < self.source.len and (std.ascii.isAlphanumeric(self.peek()) or self.peek() == '_')) { _ = self.advance(); } if (self.cursor == start_cursor + 1) { return .{ .tag = .invalid, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .at_directive, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; }
if (std.ascii.isAlphabetic(c) or c == '_') { while (self.cursor < self.source.len and (std.ascii.isAlphanumeric(self.peek()) or self.peek() == '_')) { _ = self.advance(); } const lexeme = self.source[start_cursor..self.cursor]; const tag = lookupKeyword(lexeme); return .{ .tag = tag, .lexeme = lexeme, .line = self.line, .column = start_col, }; }
if (std.ascii.isDigit(c)) { var is_float = false; while (self.cursor < self.source.len and std.ascii.isDigit(self.peek())) { _ = self.advance(); } if (self.cursor < self.source.len and self.peek() == '.' and (self.cursor + 1 < self.source.len and std.ascii.isDigit(self.source[self.cursor + 1]))) { is_float = true; _ = self.advance(); // consume '.' while (self.cursor < self.source.len and std.ascii.isDigit(self.peek())) { _ = self.advance(); } } return .{ .tag = if (is_float) .float_literal else .int_literal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col, }; }
if (c == '"') { while (self.cursor < self.source.len and self.peek() != '"') { if (self.peek() == '\\') { _ = self.advance(); } if (self.peek() == '\n') { self.line += 1; self.column = 1; } _ = self.advance(); } if (self.cursor < self.source.len) { _ = self.advance(); // consume close quote return .{ .tag = .string_literal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col, }; } else { return .{ .tag = .invalid, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col, }; } }
// Single / Multi-char operators switch (c) { '=' => { if (self.match('=')) { return .{ .tag = .equal_equal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .assign, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; }, '!' => { if (self.match('=')) { return .{ .tag = .not_equal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .bang, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; }, '<' => { if (self.match('=')) { return .{ .tag = .less_equal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .less, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; }, '>' => { if (self.match('=')) { return .{ .tag = .greater_equal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .greater, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; }, ':' => return .{ .tag = .colon, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, ';' => return .{ .tag = .semicolon, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, ',' => return .{ .tag = .comma, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '.' => { if (self.match('.')) { if (self.match('=')) { return .{ .tag = .dot_dot_equal, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .dot_dot, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; } return .{ .tag = .period, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }; }, '?' => return .{ .tag = .question, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '+' => return .{ .tag = .plus, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '-' => return .{ .tag = .minus, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '*' => return .{ .tag = .star, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '/' => return .{ .tag = .slash, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '%' => return .{ .tag = .percent, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '(' => return .{ .tag = .l_paren, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, ')' => return .{ .tag = .r_paren, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '{' => return .{ .tag = .l_brace, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '}' => return .{ .tag = .r_brace, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '|' => return .{ .tag = .pipe, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, '[' => return .{ .tag = .l_bracket, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, ']' => return .{ .tag = .r_bracket, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, else => return .{ .tag = .invalid, .lexeme = self.source[start_cursor..self.cursor], .line = self.line, .column = start_col }, } }
fn advance(self: *Lexer) u8 { const c = self.source[self.cursor]; self.cursor += 1; self.column += 1; return c; }
fn peek(self: *Lexer) u8 { if (self.cursor >= self.source.len) return 0; return self.source[self.cursor]; }
fn match(self: *Lexer, expected: u8) bool { if (self.cursor >= self.source.len) return false; if (self.source[self.cursor] != expected) return false; self.cursor += 1; self.column += 1; return true; }
fn skipWhitespaceAndComments(self: *Lexer) void { while (self.cursor < self.source.len) { const c = self.peek(); switch (c) { ' ', '\r', '\t' => { _ = self.advance(); }, '\n' => { _ = self.advance(); self.line += 1; self.column = 1; }, '/' => { if (self.cursor + 1 < self.source.len and self.source[self.cursor + 1] == '/') { while (self.cursor < self.source.len and self.peek() != '\n') { _ = self.advance(); } } else { break; } }, else => break, } } }};
fn lookupKeyword(lexeme: []const u8) TokenType { if (std.mem.eql(u8, lexeme, "const")) return .kw_const; if (std.mem.eql(u8, lexeme, "var")) return .kw_var; if (std.mem.eql(u8, lexeme, "fn")) return .kw_fn; if (std.mem.eql(u8, lexeme, "pub")) return .kw_pub; if (std.mem.eql(u8, lexeme, "if")) return .kw_if; if (std.mem.eql(u8, lexeme, "else")) return .kw_else; if (std.mem.eql(u8, lexeme, "while")) return .kw_while; if (std.mem.eql(u8, lexeme, "break")) return .kw_break; if (std.mem.eql(u8, lexeme, "continue")) return .kw_continue; if (std.mem.eql(u8, lexeme, "for")) return .kw_for; if (std.mem.eql(u8, lexeme, "in")) return .kw_in; if (std.mem.eql(u8, lexeme, "and")) return .kw_and; if (std.mem.eql(u8, lexeme, "or")) return .kw_or; if (std.mem.eql(u8, lexeme, "not")) return .kw_not; if (std.mem.eql(u8, lexeme, "return")) return .kw_return; if (std.mem.eql(u8, lexeme, "defer")) return .kw_defer; if (std.mem.eql(u8, lexeme, "struct")) return .kw_struct; if (std.mem.eql(u8, lexeme, "true")) return .kw_true; if (std.mem.eql(u8, lexeme, "false")) return .kw_false; if (std.mem.eql(u8, lexeme, "null")) return .kw_null; if (std.mem.eql(u8, lexeme, "int")) return .type_int; if (std.mem.eql(u8, lexeme, "float")) return .type_float; if (std.mem.eql(u8, lexeme, "str")) return .type_str; if (std.mem.eql(u8, lexeme, "bool")) return .type_bool; return .identifier;}
test "Lexer basic tokenization" { const code = \\const tab_size: int = 4; \\@import = .{ std, ai }; \\pub fn main(args: []str) u32 { \\ @log("hello"); \\} ; var lexer = Lexer.init(code); var tok = lexer.nextToken(); try std.testing.expectEqual(TokenType.kw_const, tok.tag); tok = lexer.nextToken(); try std.testing.expectEqual(TokenType.identifier, tok.tag); try std.testing.expectEqualSlices(u8, "tab_size", tok.lexeme); tok = lexer.nextToken(); try std.testing.expectEqual(TokenType.colon, tok.tag); tok = lexer.nextToken(); try std.testing.expectEqual(TokenType.type_int, tok.tag);}
test "Lexer generalizes directives" { var lexer = Lexer.init("@import @stdout @tensor @vector @import_c"); for ([_][]const u8{ "import", "stdout", "tensor", "vector", "import_c" }) |expected| { const tok = lexer.nextToken(); try std.testing.expectEqual(TokenType.at_directive, tok.tag); try std.testing.expectEqualSlices(u8, expected, tok.directiveName()); }}
test "Lexer modulo and bang" { var lexer = Lexer.init("a % b ! c != d"); _ = lexer.nextToken(); // a const pct = lexer.nextToken(); try std.testing.expectEqual(TokenType.percent, pct.tag); _ = lexer.nextToken(); // b const bang = lexer.nextToken(); try std.testing.expectEqual(TokenType.bang, bang.tag); _ = lexer.nextToken(); // c const neq = lexer.nextToken(); try std.testing.expectEqual(TokenType.not_equal, neq.tag);}
test "Lexer loop keywords" { var lexer = Lexer.init("while break continue defer"); try std.testing.expectEqual(TokenType.kw_while, lexer.nextToken().tag); try std.testing.expectEqual(TokenType.kw_break, lexer.nextToken().tag); try std.testing.expectEqual(TokenType.kw_continue, lexer.nextToken().tag); try std.testing.expectEqual(TokenType.kw_defer, lexer.nextToken().tag);}