Something went wrong. Try again.
🌱 A minimal programming language and compiler. git.urbach.dev/cli/q
high-performance programming-language compiler
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657package token_test
import ( "testing"
"git.urbach.dev/cli/q/src/token" "git.urbach.dev/go/assert")
func TestFunction(t *testing.T) { tokens := token.Tokenize([]byte("main(){}"))
expected := []token.Kind{ token.Identifier, token.GroupStart, token.GroupEnd, token.BlockStart, token.BlockEnd, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestKeyword(t *testing.T) { tokens := token.Tokenize([]byte("as assert const else extern global go if import loop return switch"))
expected := []token.Kind{ token.Cast, token.Assert, token.Const, token.Else, token.Extern, token.Global, token.Go, token.If, token.Import, token.Loop, token.Return, token.Switch, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestBuiltin(t *testing.T) { tokens := token.Tokenize([]byte("cas delete new syscall"))
expected := []token.Kind{ token.Cas, token.Delete, token.New, token.Syscall, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestArray(t *testing.T) { tokens := token.Tokenize([]byte("array[i]"))
expected := []token.Kind{ token.Identifier, token.ArrayStart, token.Identifier, token.ArrayEnd, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNewline(t *testing.T) { tokens := token.Tokenize([]byte("\n\n"))
expected := []token.Kind{ token.NewLine, token.NewLine, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNumber(t *testing.T) { tokens := token.Tokenize([]byte(`123 456`))
expected := []token.Kind{ token.Number, token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestOperator(t *testing.T) { tokens := token.Tokenize([]byte(`a + b - c * d / e % f << g >> h & i | j ^ k`))
expected := []token.Kind{ token.Identifier, token.Add, token.Identifier, token.Sub, token.Identifier, token.Mul, token.Identifier, token.Div, token.Identifier, token.Mod, token.Identifier, token.Shl, token.Identifier, token.Shr, token.Identifier, token.And, token.Identifier, token.Or, token.Identifier, token.Xor, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestOperatorAssign(t *testing.T) { tokens := token.Tokenize([]byte(`a = b += c -= d *= e /= f %= g &= h |= i ^= j <<= k >>= l`))
expected := []token.Kind{ token.Identifier, token.Assign, token.Identifier, token.AddAssign, token.Identifier, token.SubAssign, token.Identifier, token.MulAssign, token.Identifier, token.DivAssign, token.Identifier, token.ModAssign, token.Identifier, token.AndAssign, token.Identifier, token.OrAssign, token.Identifier, token.XorAssign, token.Identifier, token.ShlAssign, token.Identifier, token.ShrAssign, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestOperatorEquality(t *testing.T) { tokens := token.Tokenize([]byte(`a == b != c <= d >= e < f > g`))
expected := []token.Kind{ token.Identifier, token.Equal, token.Identifier, token.NotEqual, token.Identifier, token.LessEqual, token.Identifier, token.GreaterEqual, token.Identifier, token.Less, token.Identifier, token.Greater, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestOperatorLogical(t *testing.T) { tokens := token.Tokenize([]byte(`a && b || c`))
expected := []token.Kind{ token.Identifier, token.LogicalAnd, token.Identifier, token.LogicalOr, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestDefine(t *testing.T) { tokens := token.Tokenize([]byte(`a := b`))
expected := []token.Kind{ token.Identifier, token.Define, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestDot(t *testing.T) { tokens := token.Tokenize([]byte(`a.b.c`))
expected := []token.Kind{ token.Identifier, token.Dot, token.Identifier, token.Dot, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNot(t *testing.T) { tokens := token.Tokenize([]byte(`!a`))
expected := []token.Kind{ token.Not, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNegateFirstToken(t *testing.T) { tokens := token.Tokenize([]byte(`-a`))
expected := []token.Kind{ token.Negate, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNegateAfterGroupStart(t *testing.T) { tokens := token.Tokenize([]byte(`(-a)`))
expected := []token.Kind{ token.GroupStart, token.Negate, token.Identifier, token.GroupEnd, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNegateSub(t *testing.T) { tokens := token.Tokenize([]byte(`-a-b`))
expected := []token.Kind{ token.Negate, token.Identifier, token.Sub, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNegateAfterOperator(t *testing.T) { tokens := token.Tokenize([]byte(`-a + -b`))
expected := []token.Kind{ token.Negate, token.Identifier, token.Add, token.Negate, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestNegateNumber(t *testing.T) { tokens := token.Tokenize([]byte(`-1`))
expected := []token.Kind{ token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestBinaryNumber(t *testing.T) { tokens := token.Tokenize([]byte(`0b1010`))
expected := []token.Kind{ token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestOctalNumber(t *testing.T) { tokens := token.Tokenize([]byte(`0o755`))
expected := []token.Kind{ token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestHexadecimalNumber(t *testing.T) { tokens := token.Tokenize([]byte(`0xCAFE`))
expected := []token.Kind{ token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestStandaloneZero(t *testing.T) { tokens := token.Tokenize([]byte(`0`))
expected := []token.Kind{ token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestLeadingZero(t *testing.T) { tokens := token.Tokenize([]byte(`0123`))
expected := []token.Kind{ token.Number, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestRange(t *testing.T) { tokens := token.Tokenize([]byte("a..b"))
expected := []token.Kind{ token.Identifier, token.Range, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestSeparator(t *testing.T) { tokens := token.Tokenize([]byte("a,b,c"))
expected := []token.Kind{ token.Identifier, token.Separator, token.Identifier, token.Separator, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestStructField(t *testing.T) { tokens := token.Tokenize([]byte("a:b,c:d"))
expected := []token.Kind{ token.Identifier, token.StructField, token.Identifier, token.Separator, token.Identifier, token.StructField, token.Identifier, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestComment(t *testing.T) { tokens := token.Tokenize([]byte("// Hello\n// World"))
expected := []token.Kind{ token.Comment, token.NewLine, token.Comment, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }
tokens = token.Tokenize([]byte("// Hello\n"))
expected = []token.Kind{ token.Comment, token.NewLine, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }
tokens = token.Tokenize([]byte(`// Hello`))
expected = []token.Kind{ token.Comment, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }
tokens = token.Tokenize([]byte(`//`))
expected = []token.Kind{ token.Comment, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }
tokens = token.Tokenize([]byte(`/`))
expected = []token.Kind{ token.Div, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestInvalid(t *testing.T) { tokens := token.Tokenize([]byte(`@@`))
expected := []token.Kind{ token.Invalid, token.Invalid, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestInvalidScript(t *testing.T) { tokens := token.Tokenize([]byte(`##`))
expected := []token.Kind{ token.Invalid, token.Invalid, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestString(t *testing.T) { tokens := token.Tokenize([]byte(`"Hello" "World"`))
expected := []token.Kind{ token.String, token.String, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestStringMultiline(t *testing.T) { tokens := token.Tokenize([]byte("\"Hello\nWorld\""))
expected := []token.Kind{ token.String, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestStringEOF(t *testing.T) { tokens := token.Tokenize([]byte(`"EOF`))
expected := []token.Kind{ token.String, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestReturnType(t *testing.T) { tokens := token.Tokenize([]byte("()->"))
expected := []token.Kind{ token.GroupStart, token.GroupEnd, token.ReturnType, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestMinusAtEOF(t *testing.T) { tokens := token.Tokenize([]byte("1-"))
expected := []token.Kind{ token.Number, token.Sub, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestRune(t *testing.T) { tokens := token.Tokenize([]byte("'a'"))
expected := []token.Kind{ token.Rune, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestScript(t *testing.T) { tokens := token.Tokenize([]byte("#!/usr/bin/env q"))
expected := []token.Kind{ token.Script, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}
func TestDereference(t *testing.T) { tokens := token.Tokenize([]byte("a=[b]"))
expected := []token.Kind{ token.Identifier, token.Assign, token.ArrayStart, token.Identifier, token.ArrayEnd, token.EOF, }
for i, kind := range expected { assert.Equal(t, tokens[i].Kind, kind) }}