Something went wrong. Try again.
🌱 A minimal programming language and compiler. git.urbach.dev/cli/q
high-performance programming-language compiler
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647package token
import ( "unsafe")
// Position is the data type for storing file offsets.type Position = uint32
// Length is the data type for storing token lengths.type Length = uint16
// Token represents a single element in a source file.// The characters that make up an identifier are grouped into a single token.// This makes parsing easier and allows us to do better syntax checks.type Token struct { Position Position Length Length Kind Kind}
// Bytes returns the byte slice.func (t Token) Bytes(buffer []byte) []byte { return buffer[t.Position : t.Position+Position(t.Length)]}
// Start returns the start position.func (t Token) Start() Position { return t.Position}
// End returns the position after the token.func (t Token) End() Position { return t.Position + Position(t.Length)}
// Reset resets the token to default values.func (t *Token) Reset() { t.Position = 0 t.Length = 0 t.Kind = Invalid}
// StringFrom returns the token string.func (t Token) StringFrom(buffer []byte) string { return unsafe.String(unsafe.SliceData(t.Bytes(buffer)), t.Length)}