From 48501595d1d610b1447fdaa49cb3e8f05b515433 Mon Sep 17 00:00:00 2001 From: Haydn Ewers <27211+haydn@users.noreply.github.com> Date: Sat, 22 Jun 2019 20:44:10 +1000 Subject: [PATCH] Add parser --- ulm-js/.gitignore | 3 +- ulm-js/README.md | 60 ++- ulm-js/Token.ts | 28 ++ ulm-js/index.ts | 818 +--------------------------------------- ulm-js/parse.test.ts | 252 +++++++++++++ ulm-js/parse.ts | 166 ++++++++ ulm-js/test.ts | 621 +----------------------------- ulm-js/tokenize.test.ts | 619 ++++++++++++++++++++++++++++++ ulm-js/tokenize.ts | 725 +++++++++++++++++++++++++++++++++++ 9 files changed, 1852 insertions(+), 1440 deletions(-) create mode 100644 ulm-js/Token.ts create mode 100644 ulm-js/parse.test.ts create mode 100644 ulm-js/parse.ts create mode 100644 ulm-js/tokenize.test.ts create mode 100644 ulm-js/tokenize.ts diff --git a/ulm-js/.gitignore b/ulm-js/.gitignore index 6129d86..7e123ad 100644 --- a/ulm-js/.gitignore +++ b/ulm-js/.gitignore @@ -1,3 +1,2 @@ -/index.js +/*.js /node_modules -/test.js diff --git a/ulm-js/README.md b/ulm-js/README.md index 802e025..54f35df 100644 --- a/ulm-js/README.md +++ b/ulm-js/README.md @@ -5,13 +5,67 @@ A JavaScript library for working with ULM. ```typescript -import { Expression, Token, tokenize, parse, toLatex } from "ulm-js"; +import { + AST, + Token, + tokenize, + parse, + toLatex, + compile, + disambiguate +} from "ulm-js"; -const ulmScript: string = "+(2, /(12, 3))"; +const ulmScript: string = "+(2, 12/3)"; const tokens: Token[] = tokenize(ulmScript); -const ast: Expression = parse(tokens); +const ast: AST = parse(tokens); const latex: string = toLatex(ast); +// Prints: +// [ +// { +// type:"NumericAdditionOperation", +// operands: [ +// { +// type: "RationalNumberLiteral", +// numerator: 2n, +// denominator: 1n +// }, +// { +// type: "RationalNumberLiteral", +// numerator: 12n, +// denominator: 3n, +// presentation: { +// display: "fraction" +// } +// } +// ] +// } +// ] +console.log(ast); + // Prints: "2 + \frac{12}{3}" console.log(latex); + +// Prints: "2 + \frac{12}{3}" +console.log(toLatex(compile(ulmScript))); + +// Prints: +// [ +// { +// type: "NumericAdditionOperation", +// operands: [ +// { +// type: "RationalNumberLiteral", +// numerator: 1n, +// denominator: 1n, +// }, +// { +// type: "RationalNumberLiteral", +// numerator: 2n, +// denominator: 1n, +// } +// ] +// } +// ] +console.log(disambiguate(compile("AmbiguousAdditionOperation(1,2)"))); ``` diff --git a/ulm-js/Token.ts b/ulm-js/Token.ts new file mode 100644 index 0000000..7562e9c --- /dev/null +++ b/ulm-js/Token.ts @@ -0,0 +1,28 @@ +interface CodePoint { + index: number; + line: number; + column: number; +} + +interface Token { + type: + | "NaturalNumberLiteral" + | "WholeNumberLiteral" + | "IntegerNumberLiteral" + | "RationalNumberLiteral" + | "LeftParenthesis" + | "RightParenthesis" + | "LeftBracket" + | "RightBracket" + | "Colon" + | "OperatorName"; + // | "ShorthandOperator" + // | "PropertyName" + // | "Comment" + value: string; + file: string | null; + location: CodePoint; +} + +export { CodePoint }; +export default Token; diff --git a/ulm-js/index.ts b/ulm-js/index.ts index b8447f0..b800716 100644 --- a/ulm-js/index.ts +++ b/ulm-js/index.ts @@ -1,816 +1,2 @@ -interface CodePoint { - index: number; - line: number; - column: number; -} - -interface Token { - type: - | "NaturalNumberLiteral" - | "WholeNumberLiteral" - | "IntegerNumberLiteral" - | "RationalNumberLiteral" - | "LeftParenthesis" - | "RightParenthesis" - | "LeftBracket" - | "RightBracket" - | "Colon" - | "PropertyKey" - | "PropertyValue" - | "Operator" - | "Comment"; - value: string; - file: string | null; - location: CodePoint; -} - -const SIGNS = new Set(["+", "-"]); - -const LEADING_DIGITS = new Set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]); - -const TRUE_DIGITS = new Set(["0", ...LEADING_DIGITS]); - -const DIGITS = new Set([...TRUE_DIGITS, "_"]); - -// const NUM_PART = new Set(["n", "w", "i", "/", "."]); - -const WHITESPACE = new Set(["\u0009", "\u0020"]); - -const IGNORED = new Set([",", ";"]); - -// const PUNCUATION = new Set(["(", ")", "{", "}", ":"]); - -const IDENTIFIERS = new Set([ - "A", - "B", - "C", - "D", - "E", - "F", - "G", - "H", - "I", - "J", - "K", - "L", - "M", - "N", - "O", - "P", - "Q", - "R", - "S", - "T", - "U", - "V", - "W", - "X", - "Y", - "Z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "_", - "-", - "!", - "*", - "/", - "\\", - "%", - "^", - "+", - "÷", - "=", - "≠", - "|" -]); - -const errorMessage = ( - message: string, - { line, column }: CodePoint, - file: string | null -) => - [message, `${file ? `In: ${file} at ` : "At: "}${line}:${column}`].join("\n"); - -const format = (string: string): string => `'${string}' (${unicode(string)})`; - -const unicode = (string: string | undefined): string => - string - ? string - .split("") - .map( - char => - `U+${char - .charCodeAt(0) - .toString(16) - .toUpperCase() - .padStart(4, "0")}` - ) - .join(" ") - : "NULL"; - -const tokenize = (input: string, file: string | null = null): Token[] => { - let index = 0; - let line = 1; - let column = 1; - let char = input[index]; - - let tokens: Token[] = []; - - while (index < input.length) { - if (char === "\r") { - const nextChar = input[index + 1]; - - if (nextChar === "\n") { - line += 1; - - index += 2; - column = 1; - char = input[index]; - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - nextChar - )} encountered. Expected ${format("\n")}.`, - { index, line, column }, - file - ); - } - - if (char === "\n") { - line += 1; - - index += 1; - column = 1; - char = input[index]; - - continue; - } - - if (WHITESPACE.has(char) || IGNORED.has(char)) { - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "(") { - tokens.push({ - type: "LeftParenthesis", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ")") { - tokens.push({ - type: "RightParenthesis", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "{") { - tokens.push({ - type: "LeftBracket", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "}") { - tokens.push({ - type: "RightBracket", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ":") { - tokens.push({ - type: "Colon", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "0") { - let value = char; - let location = { - index, - line, - column - }; - - index += 1; - column += 1; - char = input[index]; - - if (char === "w") { - value += char; - - tokens.push({ - type: "WholeNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "i") { - value += char; - - tokens.push({ - type: "IntegerNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - if (LEADING_DIGITS.has(char)) { - let value = char; - let location = { - index, - line, - column - }; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - if (char === "n") { - value += char; - - tokens.push({ - type: "NaturalNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "w") { - value += char; - - tokens.push({ - type: "WholeNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "i") { - value += char; - - tokens.push({ - type: "IntegerNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - if (SIGNS.has(char) && TRUE_DIGITS.has(input[index + 1])) { - let value = char; - let location = { - index, - line, - column - }; - - index += 1; - column += 1; - char = input[index]; - - if (char === "0") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - } - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - if (char === "i") { - value += char; - - tokens.push({ - type: "IntegerNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - if (IDENTIFIERS.has(char)) { - let value = ""; - let location = { - index, - line, - column - }; - - while (IDENTIFIERS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "Operator", - value, - file, - location - }); - - continue; - } - - // // FIXME: Is this even needed? - // if (!/[\u0009\u000A\u000D\u0020-\uFFFF]/.test(char)) { - // throw `Unexpected ( ${char} ) encountered. ${line}:${column}`; - // } - - throw errorMessage( - `Unknown character encountered: ${format(char)}`, - { index, line, column }, - file - ); - } - - return tokens; -}; - -export { tokenize }; +export { default as parse } from "./parse"; +export { default as tokenize } from "./tokenize"; diff --git a/ulm-js/parse.test.ts b/ulm-js/parse.test.ts new file mode 100644 index 0000000..d6f60f0 --- /dev/null +++ b/ulm-js/parse.test.ts @@ -0,0 +1,252 @@ +import { strict as assert } from "assert"; +import parse from "./parse"; +import Token from "./Token"; + +const tokens: Token[] = [ + { + type: "OperatorName", + value: "NumericEqualRelation", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "OperatorName", + value: "NumericAdditionOperation", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "12312", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "0.33", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "1/4", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "0.33", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "1/3", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "-2", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "+23/5", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "5.5_8", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "NaturalNumberLiteral", + value: "1_3n", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "WholeNumberLiteral", + value: "3w", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "WholeNumberLiteral", + value: "0w", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "IntegerNumberLiteral", + value: "-34i", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "IntegerNumberLiteral", + value: "2i", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "IntegerNumberLiteral", + value: "33i", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "1", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + } +]; + +assert.deepEqual(parse(tokens), [ + { + type: "NumericEqualRelation", + operands: [ + { + type: "NumericAdditionOperation", + operands: [ + { + type: "RationalNumber", + numerator: BigInt(12312), + denominator: BigInt(1) + }, + { + type: "RationalNumber", + numerator: BigInt(33), + denominator: BigInt(100) + } + ] + }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(4) } + ] + }, + { type: "RationalNumber", numerator: BigInt(33), denominator: BigInt(100) }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(3) }, + { type: "RationalNumber", numerator: BigInt(-2), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(23), denominator: BigInt(5) }, + { type: "RationalNumber", numerator: BigInt(558), denominator: BigInt(100) }, + { type: "RationalNumber", numerator: BigInt(13), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(3), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(0), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(-34), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(2), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(33), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(1) } +]); diff --git a/ulm-js/parse.ts b/ulm-js/parse.ts new file mode 100644 index 0000000..c3e5157 --- /dev/null +++ b/ulm-js/parse.ts @@ -0,0 +1,166 @@ +import Token from "./Token"; + +interface NumericNotEqualRelation { + type: "NumericNotEqualRelation"; + operands: [NumericExpression, NumericExpression]; +} + +interface NumericEqualRelation { + type: "NumericEqualRelation"; + operands: [NumericExpression, NumericExpression]; +} + +const isNumericRelation = (e: Expression): e is NumericRelation => + e.type === "NumericEqualRelation" || e.type === "NumericNotEqualRelation"; + +type NumericRelation = NumericEqualRelation | NumericNotEqualRelation; + +interface NumericMultiplicationOperation { + type: "NumericMultiplicationOperation"; + operands: [NumericExpression, NumericExpression]; +} + +interface NumericAdditionOperation { + type: "NumericAdditionOperation"; + operands: [NumericExpression, NumericExpression]; +} + +const isNumericOperation = (e: Expression): e is NumericOperation => + e.type === "NumericAdditionOperation" || + e.type === "NumericMultiplicationOperation"; + +type NumericOperation = + | NumericAdditionOperation + | NumericMultiplicationOperation; + +type NaturalNumber = bigint; + +type IntegerNumber = bigint; + +interface RationalNumber { + type: "RationalNumber"; + numerator: IntegerNumber; + denominator: NaturalNumber; +} + +const isNumericExpression = (e: Expression): e is NumericExpression => + e.type === "RationalNumber" || isNumericOperation(e); + +type NumericExpression = RationalNumber | NumericOperation; + +type Expression = NumericExpression | NumericRelation; + +type AST = Expression[]; + +const parse = (tokens: Token[]): AST => { + let ast = []; + let index = 0; + let token = tokens[index]; + + const walk = (): Expression => { + if (token.type === "OperatorName") { + let operator = token.value; + + index += 1; + token = tokens[index]; + + let operands = walkParentheses(); + + if ( + operator === "NumericEqualRelation" || + operator === "NumericNotEqualRelation" || + operator === "NumericMultiplicationOperation" || + operator === "NumericAdditionOperation" + ) { + if (operands.length !== 2) { + throw `Wrong number of operands. Expected 2, found ${ + operands.length + }.`; + } + + let [a, b] = operands; + + if (!isNumericExpression(a)) { + throw `Unexpected operand: ${a.type} Expected: 'NumericExpression'.`; + } + + if (!isNumericExpression(b)) { + throw `Unexpected operand: ${b.type} Expected: 'NumericExpression'.`; + } + + return { + type: operator, + operands: [a, b] + }; + } + + throw `Unknown operator ${operator}`; + } + + if ( + token.type === "NaturalNumberLiteral" || + token.type === "WholeNumberLiteral" || + token.type === "IntegerNumberLiteral" || + token.type === "RationalNumberLiteral" + ) { + let value = token.value.replace(/_/g, "").replace(/[nwi]$/, ""); + let numerator, denominator; + if (value.includes(".")) { + const [n, d] = value.split("."); + numerator = BigInt(n + d); + denominator = BigInt(10 ** d.length); + } else if (value.includes("/")) { + const [n, d] = value.split("/"); + numerator = BigInt(n); + denominator = BigInt(d); + } else { + numerator = BigInt(value); + denominator = BigInt(1); + } + + index += 1; + token = tokens[index]; + + return { + type: "RationalNumber", + numerator, + denominator + }; + } + + console.log(token); + throw `What's this?`; + }; + + const walkParentheses = (): Expression[] => { + let expressions = []; + + if (token.type === "LeftParenthesis") { + index += 1; + token = tokens[index]; + } else { + throw `Unexpected token: '${token.type}' Expected: 'LeftParenthesis'`; + } + + while (token && token.type !== "RightParenthesis") { + expressions.push(walk()); + } + + if (!token) { + throw `Unexpected end of file. Expected 'RightParenthesis'`; + } + + index += 1; + token = tokens[index]; + + return expressions; + }; + + while (token) { + ast.push(walk()); + } + + return ast; +}; + +export default parse; diff --git a/ulm-js/test.ts b/ulm-js/test.ts index 687b6bd..9d58890 100644 --- a/ulm-js/test.ts +++ b/ulm-js/test.ts @@ -1,619 +1,2 @@ -import { strict as assert } from "assert"; -import { tokenize } from "./"; - -// Whitespace & Ignored - -assert.deepEqual(tokenize(""), []); - -assert.deepEqual(tokenize("\r\n\n\t ,;"), []); - -assert.throws(() => { - tokenize("\r"); -}); - -assert.throws(() => { - tokenize("\r\t"); -}); - -// Puncuation - -assert.deepEqual(tokenize("("), [ - { - type: "LeftParenthesis", - value: "(", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize(")"), [ - { - type: "RightParenthesis", - value: ")", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("{"), [ - { - type: "LeftBracket", - value: "{", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("}"), [ - { - type: "RightBracket", - value: "}", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize(":"), [ - { - type: "Colon", - value: ":", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -// Numbers - -assert.deepEqual(tokenize("0"), [ - { - type: "RationalNumberLiteral", - value: "0", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1"), [ - { - type: "RationalNumberLiteral", - value: "1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890"), [ - { - type: "RationalNumberLiteral", - value: "1234567890", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_234_567_890"), [ - { - type: "RationalNumberLiteral", - value: "1_234_567_890", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1___0_"), [ - { - type: "RationalNumberLiteral", - value: "1___0_", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0.0"), [ - { - type: "RationalNumberLiteral", - value: "0.0", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0.01"), [ - { - type: "RationalNumberLiteral", - value: "0.01", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1.1"), [ - { - type: "RationalNumberLiteral", - value: "1.1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_0.0_1"), [ - { - type: "RationalNumberLiteral", - value: "1_0.0_1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.throws(() => { - tokenize("0."); -}); - -assert.throws(() => { - tokenize("1."); -}); - -assert.throws(() => { - tokenize("_."); -}); - -assert.throws(() => { - tokenize(".1"); -}); - -assert.throws(() => { - tokenize(".0"); -}); - -assert.throws(() => { - tokenize("._"); -}); - -assert.deepEqual(tokenize("0/1"), [ - { - type: "RationalNumberLiteral", - value: "0/1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0/123"), [ - { - type: "RationalNumberLiteral", - value: "0/123", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0/1_000"), [ - { - type: "RationalNumberLiteral", - value: "0/1_000", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("10/1"), [ - { - type: "RationalNumberLiteral", - value: "10/1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2_3_4_/1"), [ - { - type: "RationalNumberLiteral", - value: "1_2_3_4_/1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.throws(() => { - tokenize("1/0"); -}); - -assert.throws(() => { - tokenize("1/+1"); -}); - -assert.throws(() => { - tokenize("1/-1"); -}); - -assert.deepEqual(tokenize("1n"), [ - { - type: "NaturalNumberLiteral", - value: "1n", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890n"), [ - { - type: "NaturalNumberLiteral", - value: "1234567890n", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2__3_n"), [ - { - type: "NaturalNumberLiteral", - value: "1_2__3_n", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0w"), [ - { - type: "WholeNumberLiteral", - value: "0w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1w"), [ - { - type: "WholeNumberLiteral", - value: "1w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890w"), [ - { - type: "WholeNumberLiteral", - value: "1234567890w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2__3_w"), [ - { - type: "WholeNumberLiteral", - value: "1_2__3_w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0i"), [ - { - type: "IntegerNumberLiteral", - value: "0i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1i"), [ - { - type: "IntegerNumberLiteral", - value: "1i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890i"), [ - { - type: "IntegerNumberLiteral", - value: "1234567890i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2__3_i"), [ - { - type: "IntegerNumberLiteral", - value: "1_2__3_i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("+0i"), [ - { - type: "IntegerNumberLiteral", - value: "+0i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("-0i"), [ - { - type: "IntegerNumberLiteral", - value: "-0i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("+1i"), [ - { - type: "IntegerNumberLiteral", - value: "+1i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("-1i"), [ - { - type: "IntegerNumberLiteral", - value: "-1i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("+9_99_i"), [ - { - type: "IntegerNumberLiteral", - value: "+9_99_i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("-9_99_i"), [ - { - type: "IntegerNumberLiteral", - value: "-9_99_i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -// Combined - -assert.deepEqual(tokenize("=(25, +(20n, 5w))"), [ - { - type: "Operator", - value: "=", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - }, - { - type: "LeftParenthesis", - value: "(", - file: null, - location: { - index: 1, - column: 2, - line: 1 - } - }, - { - type: "RationalNumberLiteral", - value: "25", - file: null, - location: { - index: 2, - column: 3, - line: 1 - } - }, - { - type: "Operator", - value: "+", - file: null, - location: { - index: 6, - column: 7, - line: 1 - } - }, - { - type: "LeftParenthesis", - value: "(", - file: null, - location: { - index: 7, - column: 8, - line: 1 - } - }, - { - type: "NaturalNumberLiteral", - value: "20n", - file: null, - location: { - index: 8, - column: 9, - line: 1 - } - }, - { - type: "WholeNumberLiteral", - value: "5w", - file: null, - location: { - index: 13, - column: 14, - line: 1 - } - }, - { - type: "RightParenthesis", - value: ")", - file: null, - location: { - index: 15, - column: 16, - line: 1 - } - }, - { - type: "RightParenthesis", - value: ")", - file: null, - location: { - index: 16, - column: 17, - line: 1 - } - } -]); +import "./tokenize.test"; +import "./parse.test"; diff --git a/ulm-js/tokenize.test.ts b/ulm-js/tokenize.test.ts new file mode 100644 index 0000000..218d4ce --- /dev/null +++ b/ulm-js/tokenize.test.ts @@ -0,0 +1,619 @@ +import { strict as assert } from "assert"; +import { tokenize } from "./"; + +// Whitespace & Ignored + +assert.deepEqual(tokenize(""), []); + +assert.deepEqual(tokenize("\r\n\n\t ,;"), []); + +assert.throws(() => { + tokenize("\r"); +}); + +assert.throws(() => { + tokenize("\r\t"); +}); + +// Puncuation + +assert.deepEqual(tokenize("("), [ + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize(")"), [ + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("{"), [ + { + type: "LeftBracket", + value: "{", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("}"), [ + { + type: "RightBracket", + value: "}", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize(":"), [ + { + type: "Colon", + value: ":", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +// Numbers + +assert.deepEqual(tokenize("0"), [ + { + type: "RationalNumberLiteral", + value: "0", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1"), [ + { + type: "RationalNumberLiteral", + value: "1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890"), [ + { + type: "RationalNumberLiteral", + value: "1234567890", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_234_567_890"), [ + { + type: "RationalNumberLiteral", + value: "1_234_567_890", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1___0_"), [ + { + type: "RationalNumberLiteral", + value: "1___0_", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0.0"), [ + { + type: "RationalNumberLiteral", + value: "0.0", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0.01"), [ + { + type: "RationalNumberLiteral", + value: "0.01", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1.1"), [ + { + type: "RationalNumberLiteral", + value: "1.1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_0.0_1"), [ + { + type: "RationalNumberLiteral", + value: "1_0.0_1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.throws(() => { + tokenize("0."); +}); + +assert.throws(() => { + tokenize("1."); +}); + +assert.throws(() => { + tokenize("_."); +}); + +assert.throws(() => { + tokenize(".1"); +}); + +assert.throws(() => { + tokenize(".0"); +}); + +assert.throws(() => { + tokenize("._"); +}); + +assert.deepEqual(tokenize("0/1"), [ + { + type: "RationalNumberLiteral", + value: "0/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0/123"), [ + { + type: "RationalNumberLiteral", + value: "0/123", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0/1_000"), [ + { + type: "RationalNumberLiteral", + value: "0/1_000", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("10/1"), [ + { + type: "RationalNumberLiteral", + value: "10/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2_3_4_/1"), [ + { + type: "RationalNumberLiteral", + value: "1_2_3_4_/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.throws(() => { + tokenize("1/0"); +}); + +assert.throws(() => { + tokenize("1/+1"); +}); + +assert.throws(() => { + tokenize("1/-1"); +}); + +assert.deepEqual(tokenize("1n"), [ + { + type: "NaturalNumberLiteral", + value: "1n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890n"), [ + { + type: "NaturalNumberLiteral", + value: "1234567890n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_n"), [ + { + type: "NaturalNumberLiteral", + value: "1_2__3_n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0w"), [ + { + type: "WholeNumberLiteral", + value: "0w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1w"), [ + { + type: "WholeNumberLiteral", + value: "1w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890w"), [ + { + type: "WholeNumberLiteral", + value: "1234567890w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_w"), [ + { + type: "WholeNumberLiteral", + value: "1_2__3_w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0i"), [ + { + type: "IntegerNumberLiteral", + value: "0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1i"), [ + { + type: "IntegerNumberLiteral", + value: "1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890i"), [ + { + type: "IntegerNumberLiteral", + value: "1234567890i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_i"), [ + { + type: "IntegerNumberLiteral", + value: "1_2__3_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+0i"), [ + { + type: "IntegerNumberLiteral", + value: "+0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-0i"), [ + { + type: "IntegerNumberLiteral", + value: "-0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+1i"), [ + { + type: "IntegerNumberLiteral", + value: "+1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-1i"), [ + { + type: "IntegerNumberLiteral", + value: "-1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+9_99_i"), [ + { + type: "IntegerNumberLiteral", + value: "+9_99_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-9_99_i"), [ + { + type: "IntegerNumberLiteral", + value: "-9_99_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +// Combined + +assert.deepEqual(tokenize("=(25, +(20n, 5w))"), [ + { + type: "OperatorName", + value: "=", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 1, + column: 2, + line: 1 + } + }, + { + type: "RationalNumberLiteral", + value: "25", + file: null, + location: { + index: 2, + column: 3, + line: 1 + } + }, + { + type: "OperatorName", + value: "+", + file: null, + location: { + index: 6, + column: 7, + line: 1 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 7, + column: 8, + line: 1 + } + }, + { + type: "NaturalNumberLiteral", + value: "20n", + file: null, + location: { + index: 8, + column: 9, + line: 1 + } + }, + { + type: "WholeNumberLiteral", + value: "5w", + file: null, + location: { + index: 13, + column: 14, + line: 1 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 15, + column: 16, + line: 1 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 16, + column: 17, + line: 1 + } + } +]); diff --git a/ulm-js/tokenize.ts b/ulm-js/tokenize.ts new file mode 100644 index 0000000..4fa7aee --- /dev/null +++ b/ulm-js/tokenize.ts @@ -0,0 +1,725 @@ +import Token, { CodePoint } from "./Token"; + +const SIGNS = new Set(["+", "-"]); + +const LEADING_DIGITS = new Set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]); + +const TRUE_DIGITS = new Set(["0", ...LEADING_DIGITS]); + +const DIGITS = new Set([...TRUE_DIGITS, "_"]); + +const WHITESPACE = new Set(["\u0009", "\u0020"]); + +const IGNORED = new Set([",", ";"]); + +const NAME = new Set([ + ..."ABCDEFGHIJKLMNOPQRSTUVWXYZ", + ..."abcdefghijklmnopqrstuvwxyz", + ..."_-!*/\\%^+÷=≠|" +]); + +const errorMessage = ( + message: string, + { line, column }: CodePoint, + file: string | null +) => + [message, `${file ? `In: ${file} at ` : "At: "}${line}:${column}`].join("\n"); + +const format = (string?: string): string => + string + ? `'${string}' (${string + .split("") + .map( + char => + `U+${char + .charCodeAt(0) + .toString(16) + .toUpperCase() + .padStart(4, "0")}` + ) + .join(" ")})` + : "NULL"; + +const tokenize = (input: string, file: string | null = null): Token[] => { + let index = 0; + let line = 1; + let column = 1; + let char = input[index]; + + let tokens: Token[] = []; + + while (index < input.length) { + if (char === "\r") { + const nextChar = input[index + 1]; + + if (nextChar === "\n") { + line += 1; + + index += 2; + column = 1; + char = input[index]; + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + nextChar + )} encountered. Expected ${format("\n")}.`, + { index, line, column }, + file + ); + } + + if (char === "\n") { + line += 1; + + index += 1; + column = 1; + char = input[index]; + + continue; + } + + if (WHITESPACE.has(char) || IGNORED.has(char)) { + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "(") { + tokens.push({ + type: "LeftParenthesis", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ")") { + tokens.push({ + type: "RightParenthesis", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "{") { + tokens.push({ + type: "LeftBracket", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "}") { + tokens.push({ + type: "RightBracket", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ":") { + tokens.push({ + type: "Colon", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "0") { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + if (char === "w") { + value += char; + + tokens.push({ + type: "WholeNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (LEADING_DIGITS.has(char)) { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + if (char === "n") { + value += char; + + tokens.push({ + type: "NaturalNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "w") { + value += char; + + tokens.push({ + type: "WholeNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (SIGNS.has(char) && TRUE_DIGITS.has(input[index + 1])) { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + if (char === "0") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + } + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (NAME.has(char)) { + let value = ""; + let location = { + index, + line, + column + }; + + while (NAME.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "OperatorName", + value, + file, + location + }); + + continue; + } + + // // FIXME: Is something like this needed? + // if (!/[\u0009\u000A\u000D\u0020-\uFFFF]/.test(char)) { + // throw `Unexpected ( ${char} ) encountered. ${line}:${column}`; + // } + + throw errorMessage( + `Unknown character encountered: ${format(char)}`, + { index, line, column }, + file + ); + } + + return tokens; +}; + +export { Token }; +export default tokenize; -- 2.51.2