diff --git a/ulm-js/.gitignore b/ulm-js/.gitignore index 07e6e47..6129d86 100644 --- a/ulm-js/.gitignore +++ b/ulm-js/.gitignore @@ -1 +1,3 @@ +/index.js /node_modules +/test.js diff --git a/ulm-js/README.md b/ulm-js/README.md index 8842b59..802e025 100644 --- a/ulm-js/README.md +++ b/ulm-js/README.md @@ -1,27 +1,17 @@ -ULM.js ------- +# ULM.js ⚠️ WIP ⚠️ -A JavaScript library for working with a ULM AST. +A JavaScript library for working with ULM. -```js -import { toLatex } from "ulm-js"; +```typescript +import { Expression, Token, tokenize, parse, toLatex } from "ulm-js"; -toLatex({ - type: 'root', - value: { - type: 'add', - left: { - type: 'int', - value: 2, - }, - right: { - type: 'int', - value: 3, - } - } -}); +const ulmScript: string = "+(2, /(12, 3))"; +const tokens: Token[] = tokenize(ulmScript); +const ast: Expression = parse(tokens); +const latex: string = toLatex(ast); -// 2 + 3 +// Prints: "2 + \frac{12}{3}" +console.log(latex); ``` diff --git a/ulm-js/index.ts b/ulm-js/index.ts new file mode 100644 index 0000000..b8447f0 --- /dev/null +++ b/ulm-js/index.ts @@ -0,0 +1,816 @@ +interface CodePoint { + index: number; + line: number; + column: number; +} + +interface Token { + type: + | "NaturalNumberLiteral" + | "WholeNumberLiteral" + | "IntegerNumberLiteral" + | "RationalNumberLiteral" + | "LeftParenthesis" + | "RightParenthesis" + | "LeftBracket" + | "RightBracket" + | "Colon" + | "PropertyKey" + | "PropertyValue" + | "Operator" + | "Comment"; + value: string; + file: string | null; + location: CodePoint; +} + +const SIGNS = new Set(["+", "-"]); + +const LEADING_DIGITS = new Set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]); + +const TRUE_DIGITS = new Set(["0", ...LEADING_DIGITS]); + +const DIGITS = new Set([...TRUE_DIGITS, "_"]); + +// const NUM_PART = new Set(["n", "w", "i", "/", "."]); + +const WHITESPACE = new Set(["\u0009", "\u0020"]); + +const IGNORED = new Set([",", ";"]); + +// const PUNCUATION = new Set(["(", ")", "{", "}", ":"]); + +const IDENTIFIERS = new Set([ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", + "_", + "-", + "!", + "*", + "/", + "\\", + "%", + "^", + "+", + "÷", + "=", + "≠", + "|" +]); + +const errorMessage = ( + message: string, + { line, column }: CodePoint, + file: string | null +) => + [message, `${file ? `In: ${file} at ` : "At: "}${line}:${column}`].join("\n"); + +const format = (string: string): string => `'${string}' (${unicode(string)})`; + +const unicode = (string: string | undefined): string => + string + ? string + .split("") + .map( + char => + `U+${char + .charCodeAt(0) + .toString(16) + .toUpperCase() + .padStart(4, "0")}` + ) + .join(" ") + : "NULL"; + +const tokenize = (input: string, file: string | null = null): Token[] => { + let index = 0; + let line = 1; + let column = 1; + let char = input[index]; + + let tokens: Token[] = []; + + while (index < input.length) { + if (char === "\r") { + const nextChar = input[index + 1]; + + if (nextChar === "\n") { + line += 1; + + index += 2; + column = 1; + char = input[index]; + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + nextChar + )} encountered. Expected ${format("\n")}.`, + { index, line, column }, + file + ); + } + + if (char === "\n") { + line += 1; + + index += 1; + column = 1; + char = input[index]; + + continue; + } + + if (WHITESPACE.has(char) || IGNORED.has(char)) { + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "(") { + tokens.push({ + type: "LeftParenthesis", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ")") { + tokens.push({ + type: "RightParenthesis", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "{") { + tokens.push({ + type: "LeftBracket", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "}") { + tokens.push({ + type: "RightBracket", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ":") { + tokens.push({ + type: "Colon", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "0") { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + if (char === "w") { + value += char; + + tokens.push({ + type: "WholeNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (LEADING_DIGITS.has(char)) { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + if (char === "n") { + value += char; + + tokens.push({ + type: "NaturalNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "w") { + value += char; + + tokens.push({ + type: "WholeNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (SIGNS.has(char) && TRUE_DIGITS.has(input[index + 1])) { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + if (char === "0") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + } + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (IDENTIFIERS.has(char)) { + let value = ""; + let location = { + index, + line, + column + }; + + while (IDENTIFIERS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "Operator", + value, + file, + location + }); + + continue; + } + + // // FIXME: Is this even needed? + // if (!/[\u0009\u000A\u000D\u0020-\uFFFF]/.test(char)) { + // throw `Unexpected ( ${char} ) encountered. ${line}:${column}`; + // } + + throw errorMessage( + `Unknown character encountered: ${format(char)}`, + { index, line, column }, + file + ); + } + + return tokens; +}; + +export { tokenize }; diff --git a/ulm-js/package.json b/ulm-js/package.json index 6314ff3..fb89126 100644 --- a/ulm-js/package.json +++ b/ulm-js/package.json @@ -1,5 +1,13 @@ { "name": "ulm-js", "version": "0.0.0", - "license": "MIT" + "license": "MIT", + "scripts": { + "build": "tsc index.ts --downlevelIteration", + "test": "tsc test.ts --downlevelIteration && node test.js" + }, + "devDependencies": { + "@types/node": "^12.0.8", + "typescript": "^3.5.2" + } } diff --git a/ulm-js/test.ts b/ulm-js/test.ts new file mode 100644 index 0000000..687b6bd --- /dev/null +++ b/ulm-js/test.ts @@ -0,0 +1,619 @@ +import { strict as assert } from "assert"; +import { tokenize } from "./"; + +// Whitespace & Ignored + +assert.deepEqual(tokenize(""), []); + +assert.deepEqual(tokenize("\r\n\n\t ,;"), []); + +assert.throws(() => { + tokenize("\r"); +}); + +assert.throws(() => { + tokenize("\r\t"); +}); + +// Puncuation + +assert.deepEqual(tokenize("("), [ + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize(")"), [ + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("{"), [ + { + type: "LeftBracket", + value: "{", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("}"), [ + { + type: "RightBracket", + value: "}", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize(":"), [ + { + type: "Colon", + value: ":", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +// Numbers + +assert.deepEqual(tokenize("0"), [ + { + type: "RationalNumberLiteral", + value: "0", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1"), [ + { + type: "RationalNumberLiteral", + value: "1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890"), [ + { + type: "RationalNumberLiteral", + value: "1234567890", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_234_567_890"), [ + { + type: "RationalNumberLiteral", + value: "1_234_567_890", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1___0_"), [ + { + type: "RationalNumberLiteral", + value: "1___0_", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0.0"), [ + { + type: "RationalNumberLiteral", + value: "0.0", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0.01"), [ + { + type: "RationalNumberLiteral", + value: "0.01", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1.1"), [ + { + type: "RationalNumberLiteral", + value: "1.1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_0.0_1"), [ + { + type: "RationalNumberLiteral", + value: "1_0.0_1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.throws(() => { + tokenize("0."); +}); + +assert.throws(() => { + tokenize("1."); +}); + +assert.throws(() => { + tokenize("_."); +}); + +assert.throws(() => { + tokenize(".1"); +}); + +assert.throws(() => { + tokenize(".0"); +}); + +assert.throws(() => { + tokenize("._"); +}); + +assert.deepEqual(tokenize("0/1"), [ + { + type: "RationalNumberLiteral", + value: "0/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0/123"), [ + { + type: "RationalNumberLiteral", + value: "0/123", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0/1_000"), [ + { + type: "RationalNumberLiteral", + value: "0/1_000", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("10/1"), [ + { + type: "RationalNumberLiteral", + value: "10/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2_3_4_/1"), [ + { + type: "RationalNumberLiteral", + value: "1_2_3_4_/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.throws(() => { + tokenize("1/0"); +}); + +assert.throws(() => { + tokenize("1/+1"); +}); + +assert.throws(() => { + tokenize("1/-1"); +}); + +assert.deepEqual(tokenize("1n"), [ + { + type: "NaturalNumberLiteral", + value: "1n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890n"), [ + { + type: "NaturalNumberLiteral", + value: "1234567890n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_n"), [ + { + type: "NaturalNumberLiteral", + value: "1_2__3_n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0w"), [ + { + type: "WholeNumberLiteral", + value: "0w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1w"), [ + { + type: "WholeNumberLiteral", + value: "1w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890w"), [ + { + type: "WholeNumberLiteral", + value: "1234567890w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_w"), [ + { + type: "WholeNumberLiteral", + value: "1_2__3_w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0i"), [ + { + type: "IntegerNumberLiteral", + value: "0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1i"), [ + { + type: "IntegerNumberLiteral", + value: "1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890i"), [ + { + type: "IntegerNumberLiteral", + value: "1234567890i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_i"), [ + { + type: "IntegerNumberLiteral", + value: "1_2__3_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+0i"), [ + { + type: "IntegerNumberLiteral", + value: "+0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-0i"), [ + { + type: "IntegerNumberLiteral", + value: "-0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+1i"), [ + { + type: "IntegerNumberLiteral", + value: "+1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-1i"), [ + { + type: "IntegerNumberLiteral", + value: "-1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+9_99_i"), [ + { + type: "IntegerNumberLiteral", + value: "+9_99_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-9_99_i"), [ + { + type: "IntegerNumberLiteral", + value: "-9_99_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +// Combined + +assert.deepEqual(tokenize("=(25, +(20n, 5w))"), [ + { + type: "Operator", + value: "=", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 1, + column: 2, + line: 1 + } + }, + { + type: "RationalNumberLiteral", + value: "25", + file: null, + location: { + index: 2, + column: 3, + line: 1 + } + }, + { + type: "Operator", + value: "+", + file: null, + location: { + index: 6, + column: 7, + line: 1 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 7, + column: 8, + line: 1 + } + }, + { + type: "NaturalNumberLiteral", + value: "20n", + file: null, + location: { + index: 8, + column: 9, + line: 1 + } + }, + { + type: "WholeNumberLiteral", + value: "5w", + file: null, + location: { + index: 13, + column: 14, + line: 1 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 15, + column: 16, + line: 1 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 16, + column: 17, + line: 1 + } + } +]); diff --git a/yarn.lock b/yarn.lock index 279f80d..340f58b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@types/node@^12.0.8": + version "12.0.8" + resolved "https://registry.yarnpkg.com/@types/node/-/node-12.0.8.tgz#551466be11b2adc3f3d47156758f610bd9f6b1d8" + integrity sha512-b8bbUOTwzIY3V5vDTY1fIJ+ePKDUBqt2hC2woVGotdQQhG/2Sh62HOKHrT7ab+VerXAcPyAiTEipPu/FsreUtg== + clipboard@^1.5.5: version "1.7.1" resolved "https://registry.yarnpkg.com/clipboard/-/clipboard-1.7.1.tgz#360d6d6946e99a7a1fef395e42ba92b5e9b5a16b" @@ -46,3 +51,8 @@ tiny-emitter@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tiny-emitter/-/tiny-emitter-2.1.0.tgz#1d1a56edfc51c43e863cbb5382a72330e3555423" integrity sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q== + +typescript@^3.5.2: + version "3.5.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.5.2.tgz#a09e1dc69bc9551cadf17dba10ee42cf55e5d56c" + integrity sha512-7KxJovlYhTX5RaRbUdkAXN1KUZ8PwWlTzQdHV6xNqvuFOs7+WBo10TQUqT19Q/Jz2hk5v9TQDIhyLhhJY4p5AA== -- 2.51.2 From 48501595d1d610b1447fdaa49cb3e8f05b515433 Mon Sep 17 00:00:00 2001 From: Haydn Ewers <27211+haydn@users.noreply.github.com> Date: Sat, 22 Jun 2019 20:44:10 +1000 Subject: [PATCH 2/4] Add parser --- ulm-js/.gitignore | 3 +- ulm-js/README.md | 60 ++- ulm-js/Token.ts | 28 ++ ulm-js/index.ts | 818 +--------------------------------------- ulm-js/parse.test.ts | 252 +++++++++++++ ulm-js/parse.ts | 166 ++++++++ ulm-js/test.ts | 621 +----------------------------- ulm-js/tokenize.test.ts | 619 ++++++++++++++++++++++++++++++ ulm-js/tokenize.ts | 725 +++++++++++++++++++++++++++++++++++ 9 files changed, 1852 insertions(+), 1440 deletions(-) create mode 100644 ulm-js/Token.ts create mode 100644 ulm-js/parse.test.ts create mode 100644 ulm-js/parse.ts create mode 100644 ulm-js/tokenize.test.ts create mode 100644 ulm-js/tokenize.ts diff --git a/ulm-js/.gitignore b/ulm-js/.gitignore index 6129d86..7e123ad 100644 --- a/ulm-js/.gitignore +++ b/ulm-js/.gitignore @@ -1,3 +1,2 @@ -/index.js +/*.js /node_modules -/test.js diff --git a/ulm-js/README.md b/ulm-js/README.md index 802e025..54f35df 100644 --- a/ulm-js/README.md +++ b/ulm-js/README.md @@ -5,13 +5,67 @@ A JavaScript library for working with ULM. ```typescript -import { Expression, Token, tokenize, parse, toLatex } from "ulm-js"; +import { + AST, + Token, + tokenize, + parse, + toLatex, + compile, + disambiguate +} from "ulm-js"; -const ulmScript: string = "+(2, /(12, 3))"; +const ulmScript: string = "+(2, 12/3)"; const tokens: Token[] = tokenize(ulmScript); -const ast: Expression = parse(tokens); +const ast: AST = parse(tokens); const latex: string = toLatex(ast); +// Prints: +// [ +// { +// type:"NumericAdditionOperation", +// operands: [ +// { +// type: "RationalNumberLiteral", +// numerator: 2n, +// denominator: 1n +// }, +// { +// type: "RationalNumberLiteral", +// numerator: 12n, +// denominator: 3n, +// presentation: { +// display: "fraction" +// } +// } +// ] +// } +// ] +console.log(ast); + // Prints: "2 + \frac{12}{3}" console.log(latex); + +// Prints: "2 + \frac{12}{3}" +console.log(toLatex(compile(ulmScript))); + +// Prints: +// [ +// { +// type: "NumericAdditionOperation", +// operands: [ +// { +// type: "RationalNumberLiteral", +// numerator: 1n, +// denominator: 1n, +// }, +// { +// type: "RationalNumberLiteral", +// numerator: 2n, +// denominator: 1n, +// } +// ] +// } +// ] +console.log(disambiguate(compile("AmbiguousAdditionOperation(1,2)"))); ``` diff --git a/ulm-js/Token.ts b/ulm-js/Token.ts new file mode 100644 index 0000000..7562e9c --- /dev/null +++ b/ulm-js/Token.ts @@ -0,0 +1,28 @@ +interface CodePoint { + index: number; + line: number; + column: number; +} + +interface Token { + type: + | "NaturalNumberLiteral" + | "WholeNumberLiteral" + | "IntegerNumberLiteral" + | "RationalNumberLiteral" + | "LeftParenthesis" + | "RightParenthesis" + | "LeftBracket" + | "RightBracket" + | "Colon" + | "OperatorName"; + // | "ShorthandOperator" + // | "PropertyName" + // | "Comment" + value: string; + file: string | null; + location: CodePoint; +} + +export { CodePoint }; +export default Token; diff --git a/ulm-js/index.ts b/ulm-js/index.ts index b8447f0..b800716 100644 --- a/ulm-js/index.ts +++ b/ulm-js/index.ts @@ -1,816 +1,2 @@ -interface CodePoint { - index: number; - line: number; - column: number; -} - -interface Token { - type: - | "NaturalNumberLiteral" - | "WholeNumberLiteral" - | "IntegerNumberLiteral" - | "RationalNumberLiteral" - | "LeftParenthesis" - | "RightParenthesis" - | "LeftBracket" - | "RightBracket" - | "Colon" - | "PropertyKey" - | "PropertyValue" - | "Operator" - | "Comment"; - value: string; - file: string | null; - location: CodePoint; -} - -const SIGNS = new Set(["+", "-"]); - -const LEADING_DIGITS = new Set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]); - -const TRUE_DIGITS = new Set(["0", ...LEADING_DIGITS]); - -const DIGITS = new Set([...TRUE_DIGITS, "_"]); - -// const NUM_PART = new Set(["n", "w", "i", "/", "."]); - -const WHITESPACE = new Set(["\u0009", "\u0020"]); - -const IGNORED = new Set([",", ";"]); - -// const PUNCUATION = new Set(["(", ")", "{", "}", ":"]); - -const IDENTIFIERS = new Set([ - "A", - "B", - "C", - "D", - "E", - "F", - "G", - "H", - "I", - "J", - "K", - "L", - "M", - "N", - "O", - "P", - "Q", - "R", - "S", - "T", - "U", - "V", - "W", - "X", - "Y", - "Z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "_", - "-", - "!", - "*", - "/", - "\\", - "%", - "^", - "+", - "÷", - "=", - "≠", - "|" -]); - -const errorMessage = ( - message: string, - { line, column }: CodePoint, - file: string | null -) => - [message, `${file ? `In: ${file} at ` : "At: "}${line}:${column}`].join("\n"); - -const format = (string: string): string => `'${string}' (${unicode(string)})`; - -const unicode = (string: string | undefined): string => - string - ? string - .split("") - .map( - char => - `U+${char - .charCodeAt(0) - .toString(16) - .toUpperCase() - .padStart(4, "0")}` - ) - .join(" ") - : "NULL"; - -const tokenize = (input: string, file: string | null = null): Token[] => { - let index = 0; - let line = 1; - let column = 1; - let char = input[index]; - - let tokens: Token[] = []; - - while (index < input.length) { - if (char === "\r") { - const nextChar = input[index + 1]; - - if (nextChar === "\n") { - line += 1; - - index += 2; - column = 1; - char = input[index]; - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - nextChar - )} encountered. Expected ${format("\n")}.`, - { index, line, column }, - file - ); - } - - if (char === "\n") { - line += 1; - - index += 1; - column = 1; - char = input[index]; - - continue; - } - - if (WHITESPACE.has(char) || IGNORED.has(char)) { - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "(") { - tokens.push({ - type: "LeftParenthesis", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ")") { - tokens.push({ - type: "RightParenthesis", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "{") { - tokens.push({ - type: "LeftBracket", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "}") { - tokens.push({ - type: "RightBracket", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ":") { - tokens.push({ - type: "Colon", - value: char, - file, - location: { - index, - line, - column - } - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "0") { - let value = char; - let location = { - index, - line, - column - }; - - index += 1; - column += 1; - char = input[index]; - - if (char === "w") { - value += char; - - tokens.push({ - type: "WholeNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "i") { - value += char; - - tokens.push({ - type: "IntegerNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - if (LEADING_DIGITS.has(char)) { - let value = char; - let location = { - index, - line, - column - }; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - if (char === "n") { - value += char; - - tokens.push({ - type: "NaturalNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "w") { - value += char; - - tokens.push({ - type: "WholeNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === "i") { - value += char; - - tokens.push({ - type: "IntegerNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - if (SIGNS.has(char) && TRUE_DIGITS.has(input[index + 1])) { - let value = char; - let location = { - index, - line, - column - }; - - index += 1; - column += 1; - char = input[index]; - - if (char === "0") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - } - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - if (char === "i") { - value += char; - - tokens.push({ - type: "IntegerNumberLiteral", - value, - file, - location - }); - - index += 1; - column += 1; - char = input[index]; - - continue; - } - - if (char === ".") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (TRUE_DIGITS.has(char)) { - do { - value += char; - - index += 1; - column += 1; - char = input[index]; - } while (DIGITS.has(char)); - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - if (char === "/") { - value += char; - - index += 1; - column += 1; - char = input[index]; - - if (LEADING_DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - - while (DIGITS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - throw errorMessage( - `Unexpected character ${format( - char - )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, - { index, line, column }, - file - ); - } - - tokens.push({ - type: "RationalNumberLiteral", - value, - file, - location - }); - - continue; - } - - if (IDENTIFIERS.has(char)) { - let value = ""; - let location = { - index, - line, - column - }; - - while (IDENTIFIERS.has(char)) { - value += char; - - index += 1; - column += 1; - char = input[index]; - } - - tokens.push({ - type: "Operator", - value, - file, - location - }); - - continue; - } - - // // FIXME: Is this even needed? - // if (!/[\u0009\u000A\u000D\u0020-\uFFFF]/.test(char)) { - // throw `Unexpected ( ${char} ) encountered. ${line}:${column}`; - // } - - throw errorMessage( - `Unknown character encountered: ${format(char)}`, - { index, line, column }, - file - ); - } - - return tokens; -}; - -export { tokenize }; +export { default as parse } from "./parse"; +export { default as tokenize } from "./tokenize"; diff --git a/ulm-js/parse.test.ts b/ulm-js/parse.test.ts new file mode 100644 index 0000000..d6f60f0 --- /dev/null +++ b/ulm-js/parse.test.ts @@ -0,0 +1,252 @@ +import { strict as assert } from "assert"; +import parse from "./parse"; +import Token from "./Token"; + +const tokens: Token[] = [ + { + type: "OperatorName", + value: "NumericEqualRelation", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "OperatorName", + value: "NumericAdditionOperation", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "12312", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "0.33", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "1/4", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "0.33", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "1/3", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "-2", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "+23/5", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "5.5_8", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "NaturalNumberLiteral", + value: "1_3n", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "WholeNumberLiteral", + value: "3w", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "WholeNumberLiteral", + value: "0w", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "IntegerNumberLiteral", + value: "-34i", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "IntegerNumberLiteral", + value: "2i", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "IntegerNumberLiteral", + value: "33i", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + }, + { + type: "RationalNumberLiteral", + value: "1", + file: null, + location: { + index: 0, + line: 0, + column: 0 + } + } +]; + +assert.deepEqual(parse(tokens), [ + { + type: "NumericEqualRelation", + operands: [ + { + type: "NumericAdditionOperation", + operands: [ + { + type: "RationalNumber", + numerator: BigInt(12312), + denominator: BigInt(1) + }, + { + type: "RationalNumber", + numerator: BigInt(33), + denominator: BigInt(100) + } + ] + }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(4) } + ] + }, + { type: "RationalNumber", numerator: BigInt(33), denominator: BigInt(100) }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(3) }, + { type: "RationalNumber", numerator: BigInt(-2), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(23), denominator: BigInt(5) }, + { type: "RationalNumber", numerator: BigInt(558), denominator: BigInt(100) }, + { type: "RationalNumber", numerator: BigInt(13), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(3), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(0), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(-34), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(2), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(33), denominator: BigInt(1) }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(1) } +]); diff --git a/ulm-js/parse.ts b/ulm-js/parse.ts new file mode 100644 index 0000000..c3e5157 --- /dev/null +++ b/ulm-js/parse.ts @@ -0,0 +1,166 @@ +import Token from "./Token"; + +interface NumericNotEqualRelation { + type: "NumericNotEqualRelation"; + operands: [NumericExpression, NumericExpression]; +} + +interface NumericEqualRelation { + type: "NumericEqualRelation"; + operands: [NumericExpression, NumericExpression]; +} + +const isNumericRelation = (e: Expression): e is NumericRelation => + e.type === "NumericEqualRelation" || e.type === "NumericNotEqualRelation"; + +type NumericRelation = NumericEqualRelation | NumericNotEqualRelation; + +interface NumericMultiplicationOperation { + type: "NumericMultiplicationOperation"; + operands: [NumericExpression, NumericExpression]; +} + +interface NumericAdditionOperation { + type: "NumericAdditionOperation"; + operands: [NumericExpression, NumericExpression]; +} + +const isNumericOperation = (e: Expression): e is NumericOperation => + e.type === "NumericAdditionOperation" || + e.type === "NumericMultiplicationOperation"; + +type NumericOperation = + | NumericAdditionOperation + | NumericMultiplicationOperation; + +type NaturalNumber = bigint; + +type IntegerNumber = bigint; + +interface RationalNumber { + type: "RationalNumber"; + numerator: IntegerNumber; + denominator: NaturalNumber; +} + +const isNumericExpression = (e: Expression): e is NumericExpression => + e.type === "RationalNumber" || isNumericOperation(e); + +type NumericExpression = RationalNumber | NumericOperation; + +type Expression = NumericExpression | NumericRelation; + +type AST = Expression[]; + +const parse = (tokens: Token[]): AST => { + let ast = []; + let index = 0; + let token = tokens[index]; + + const walk = (): Expression => { + if (token.type === "OperatorName") { + let operator = token.value; + + index += 1; + token = tokens[index]; + + let operands = walkParentheses(); + + if ( + operator === "NumericEqualRelation" || + operator === "NumericNotEqualRelation" || + operator === "NumericMultiplicationOperation" || + operator === "NumericAdditionOperation" + ) { + if (operands.length !== 2) { + throw `Wrong number of operands. Expected 2, found ${ + operands.length + }.`; + } + + let [a, b] = operands; + + if (!isNumericExpression(a)) { + throw `Unexpected operand: ${a.type} Expected: 'NumericExpression'.`; + } + + if (!isNumericExpression(b)) { + throw `Unexpected operand: ${b.type} Expected: 'NumericExpression'.`; + } + + return { + type: operator, + operands: [a, b] + }; + } + + throw `Unknown operator ${operator}`; + } + + if ( + token.type === "NaturalNumberLiteral" || + token.type === "WholeNumberLiteral" || + token.type === "IntegerNumberLiteral" || + token.type === "RationalNumberLiteral" + ) { + let value = token.value.replace(/_/g, "").replace(/[nwi]$/, ""); + let numerator, denominator; + if (value.includes(".")) { + const [n, d] = value.split("."); + numerator = BigInt(n + d); + denominator = BigInt(10 ** d.length); + } else if (value.includes("/")) { + const [n, d] = value.split("/"); + numerator = BigInt(n); + denominator = BigInt(d); + } else { + numerator = BigInt(value); + denominator = BigInt(1); + } + + index += 1; + token = tokens[index]; + + return { + type: "RationalNumber", + numerator, + denominator + }; + } + + console.log(token); + throw `What's this?`; + }; + + const walkParentheses = (): Expression[] => { + let expressions = []; + + if (token.type === "LeftParenthesis") { + index += 1; + token = tokens[index]; + } else { + throw `Unexpected token: '${token.type}' Expected: 'LeftParenthesis'`; + } + + while (token && token.type !== "RightParenthesis") { + expressions.push(walk()); + } + + if (!token) { + throw `Unexpected end of file. Expected 'RightParenthesis'`; + } + + index += 1; + token = tokens[index]; + + return expressions; + }; + + while (token) { + ast.push(walk()); + } + + return ast; +}; + +export default parse; diff --git a/ulm-js/test.ts b/ulm-js/test.ts index 687b6bd..9d58890 100644 --- a/ulm-js/test.ts +++ b/ulm-js/test.ts @@ -1,619 +1,2 @@ -import { strict as assert } from "assert"; -import { tokenize } from "./"; - -// Whitespace & Ignored - -assert.deepEqual(tokenize(""), []); - -assert.deepEqual(tokenize("\r\n\n\t ,;"), []); - -assert.throws(() => { - tokenize("\r"); -}); - -assert.throws(() => { - tokenize("\r\t"); -}); - -// Puncuation - -assert.deepEqual(tokenize("("), [ - { - type: "LeftParenthesis", - value: "(", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize(")"), [ - { - type: "RightParenthesis", - value: ")", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("{"), [ - { - type: "LeftBracket", - value: "{", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("}"), [ - { - type: "RightBracket", - value: "}", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize(":"), [ - { - type: "Colon", - value: ":", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -// Numbers - -assert.deepEqual(tokenize("0"), [ - { - type: "RationalNumberLiteral", - value: "0", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1"), [ - { - type: "RationalNumberLiteral", - value: "1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890"), [ - { - type: "RationalNumberLiteral", - value: "1234567890", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_234_567_890"), [ - { - type: "RationalNumberLiteral", - value: "1_234_567_890", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1___0_"), [ - { - type: "RationalNumberLiteral", - value: "1___0_", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0.0"), [ - { - type: "RationalNumberLiteral", - value: "0.0", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0.01"), [ - { - type: "RationalNumberLiteral", - value: "0.01", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1.1"), [ - { - type: "RationalNumberLiteral", - value: "1.1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_0.0_1"), [ - { - type: "RationalNumberLiteral", - value: "1_0.0_1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.throws(() => { - tokenize("0."); -}); - -assert.throws(() => { - tokenize("1."); -}); - -assert.throws(() => { - tokenize("_."); -}); - -assert.throws(() => { - tokenize(".1"); -}); - -assert.throws(() => { - tokenize(".0"); -}); - -assert.throws(() => { - tokenize("._"); -}); - -assert.deepEqual(tokenize("0/1"), [ - { - type: "RationalNumberLiteral", - value: "0/1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0/123"), [ - { - type: "RationalNumberLiteral", - value: "0/123", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0/1_000"), [ - { - type: "RationalNumberLiteral", - value: "0/1_000", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("10/1"), [ - { - type: "RationalNumberLiteral", - value: "10/1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2_3_4_/1"), [ - { - type: "RationalNumberLiteral", - value: "1_2_3_4_/1", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.throws(() => { - tokenize("1/0"); -}); - -assert.throws(() => { - tokenize("1/+1"); -}); - -assert.throws(() => { - tokenize("1/-1"); -}); - -assert.deepEqual(tokenize("1n"), [ - { - type: "NaturalNumberLiteral", - value: "1n", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890n"), [ - { - type: "NaturalNumberLiteral", - value: "1234567890n", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2__3_n"), [ - { - type: "NaturalNumberLiteral", - value: "1_2__3_n", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0w"), [ - { - type: "WholeNumberLiteral", - value: "0w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1w"), [ - { - type: "WholeNumberLiteral", - value: "1w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890w"), [ - { - type: "WholeNumberLiteral", - value: "1234567890w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2__3_w"), [ - { - type: "WholeNumberLiteral", - value: "1_2__3_w", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("0i"), [ - { - type: "IntegerNumberLiteral", - value: "0i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1i"), [ - { - type: "IntegerNumberLiteral", - value: "1i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1234567890i"), [ - { - type: "IntegerNumberLiteral", - value: "1234567890i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("1_2__3_i"), [ - { - type: "IntegerNumberLiteral", - value: "1_2__3_i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("+0i"), [ - { - type: "IntegerNumberLiteral", - value: "+0i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("-0i"), [ - { - type: "IntegerNumberLiteral", - value: "-0i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("+1i"), [ - { - type: "IntegerNumberLiteral", - value: "+1i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("-1i"), [ - { - type: "IntegerNumberLiteral", - value: "-1i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("+9_99_i"), [ - { - type: "IntegerNumberLiteral", - value: "+9_99_i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -assert.deepEqual(tokenize("-9_99_i"), [ - { - type: "IntegerNumberLiteral", - value: "-9_99_i", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - } -]); - -// Combined - -assert.deepEqual(tokenize("=(25, +(20n, 5w))"), [ - { - type: "Operator", - value: "=", - file: null, - location: { - index: 0, - column: 1, - line: 1 - } - }, - { - type: "LeftParenthesis", - value: "(", - file: null, - location: { - index: 1, - column: 2, - line: 1 - } - }, - { - type: "RationalNumberLiteral", - value: "25", - file: null, - location: { - index: 2, - column: 3, - line: 1 - } - }, - { - type: "Operator", - value: "+", - file: null, - location: { - index: 6, - column: 7, - line: 1 - } - }, - { - type: "LeftParenthesis", - value: "(", - file: null, - location: { - index: 7, - column: 8, - line: 1 - } - }, - { - type: "NaturalNumberLiteral", - value: "20n", - file: null, - location: { - index: 8, - column: 9, - line: 1 - } - }, - { - type: "WholeNumberLiteral", - value: "5w", - file: null, - location: { - index: 13, - column: 14, - line: 1 - } - }, - { - type: "RightParenthesis", - value: ")", - file: null, - location: { - index: 15, - column: 16, - line: 1 - } - }, - { - type: "RightParenthesis", - value: ")", - file: null, - location: { - index: 16, - column: 17, - line: 1 - } - } -]); +import "./tokenize.test"; +import "./parse.test"; diff --git a/ulm-js/tokenize.test.ts b/ulm-js/tokenize.test.ts new file mode 100644 index 0000000..218d4ce --- /dev/null +++ b/ulm-js/tokenize.test.ts @@ -0,0 +1,619 @@ +import { strict as assert } from "assert"; +import { tokenize } from "./"; + +// Whitespace & Ignored + +assert.deepEqual(tokenize(""), []); + +assert.deepEqual(tokenize("\r\n\n\t ,;"), []); + +assert.throws(() => { + tokenize("\r"); +}); + +assert.throws(() => { + tokenize("\r\t"); +}); + +// Puncuation + +assert.deepEqual(tokenize("("), [ + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize(")"), [ + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("{"), [ + { + type: "LeftBracket", + value: "{", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("}"), [ + { + type: "RightBracket", + value: "}", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize(":"), [ + { + type: "Colon", + value: ":", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +// Numbers + +assert.deepEqual(tokenize("0"), [ + { + type: "RationalNumberLiteral", + value: "0", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1"), [ + { + type: "RationalNumberLiteral", + value: "1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890"), [ + { + type: "RationalNumberLiteral", + value: "1234567890", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_234_567_890"), [ + { + type: "RationalNumberLiteral", + value: "1_234_567_890", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1___0_"), [ + { + type: "RationalNumberLiteral", + value: "1___0_", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0.0"), [ + { + type: "RationalNumberLiteral", + value: "0.0", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0.01"), [ + { + type: "RationalNumberLiteral", + value: "0.01", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1.1"), [ + { + type: "RationalNumberLiteral", + value: "1.1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_0.0_1"), [ + { + type: "RationalNumberLiteral", + value: "1_0.0_1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.throws(() => { + tokenize("0."); +}); + +assert.throws(() => { + tokenize("1."); +}); + +assert.throws(() => { + tokenize("_."); +}); + +assert.throws(() => { + tokenize(".1"); +}); + +assert.throws(() => { + tokenize(".0"); +}); + +assert.throws(() => { + tokenize("._"); +}); + +assert.deepEqual(tokenize("0/1"), [ + { + type: "RationalNumberLiteral", + value: "0/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0/123"), [ + { + type: "RationalNumberLiteral", + value: "0/123", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0/1_000"), [ + { + type: "RationalNumberLiteral", + value: "0/1_000", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("10/1"), [ + { + type: "RationalNumberLiteral", + value: "10/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2_3_4_/1"), [ + { + type: "RationalNumberLiteral", + value: "1_2_3_4_/1", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.throws(() => { + tokenize("1/0"); +}); + +assert.throws(() => { + tokenize("1/+1"); +}); + +assert.throws(() => { + tokenize("1/-1"); +}); + +assert.deepEqual(tokenize("1n"), [ + { + type: "NaturalNumberLiteral", + value: "1n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890n"), [ + { + type: "NaturalNumberLiteral", + value: "1234567890n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_n"), [ + { + type: "NaturalNumberLiteral", + value: "1_2__3_n", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0w"), [ + { + type: "WholeNumberLiteral", + value: "0w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1w"), [ + { + type: "WholeNumberLiteral", + value: "1w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890w"), [ + { + type: "WholeNumberLiteral", + value: "1234567890w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_w"), [ + { + type: "WholeNumberLiteral", + value: "1_2__3_w", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("0i"), [ + { + type: "IntegerNumberLiteral", + value: "0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1i"), [ + { + type: "IntegerNumberLiteral", + value: "1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1234567890i"), [ + { + type: "IntegerNumberLiteral", + value: "1234567890i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("1_2__3_i"), [ + { + type: "IntegerNumberLiteral", + value: "1_2__3_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+0i"), [ + { + type: "IntegerNumberLiteral", + value: "+0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-0i"), [ + { + type: "IntegerNumberLiteral", + value: "-0i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+1i"), [ + { + type: "IntegerNumberLiteral", + value: "+1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-1i"), [ + { + type: "IntegerNumberLiteral", + value: "-1i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("+9_99_i"), [ + { + type: "IntegerNumberLiteral", + value: "+9_99_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +assert.deepEqual(tokenize("-9_99_i"), [ + { + type: "IntegerNumberLiteral", + value: "-9_99_i", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + } +]); + +// Combined + +assert.deepEqual(tokenize("=(25, +(20n, 5w))"), [ + { + type: "OperatorName", + value: "=", + file: null, + location: { + index: 0, + column: 1, + line: 1 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 1, + column: 2, + line: 1 + } + }, + { + type: "RationalNumberLiteral", + value: "25", + file: null, + location: { + index: 2, + column: 3, + line: 1 + } + }, + { + type: "OperatorName", + value: "+", + file: null, + location: { + index: 6, + column: 7, + line: 1 + } + }, + { + type: "LeftParenthesis", + value: "(", + file: null, + location: { + index: 7, + column: 8, + line: 1 + } + }, + { + type: "NaturalNumberLiteral", + value: "20n", + file: null, + location: { + index: 8, + column: 9, + line: 1 + } + }, + { + type: "WholeNumberLiteral", + value: "5w", + file: null, + location: { + index: 13, + column: 14, + line: 1 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 15, + column: 16, + line: 1 + } + }, + { + type: "RightParenthesis", + value: ")", + file: null, + location: { + index: 16, + column: 17, + line: 1 + } + } +]); diff --git a/ulm-js/tokenize.ts b/ulm-js/tokenize.ts new file mode 100644 index 0000000..4fa7aee --- /dev/null +++ b/ulm-js/tokenize.ts @@ -0,0 +1,725 @@ +import Token, { CodePoint } from "./Token"; + +const SIGNS = new Set(["+", "-"]); + +const LEADING_DIGITS = new Set(["1", "2", "3", "4", "5", "6", "7", "8", "9"]); + +const TRUE_DIGITS = new Set(["0", ...LEADING_DIGITS]); + +const DIGITS = new Set([...TRUE_DIGITS, "_"]); + +const WHITESPACE = new Set(["\u0009", "\u0020"]); + +const IGNORED = new Set([",", ";"]); + +const NAME = new Set([ + ..."ABCDEFGHIJKLMNOPQRSTUVWXYZ", + ..."abcdefghijklmnopqrstuvwxyz", + ..."_-!*/\\%^+÷=≠|" +]); + +const errorMessage = ( + message: string, + { line, column }: CodePoint, + file: string | null +) => + [message, `${file ? `In: ${file} at ` : "At: "}${line}:${column}`].join("\n"); + +const format = (string?: string): string => + string + ? `'${string}' (${string + .split("") + .map( + char => + `U+${char + .charCodeAt(0) + .toString(16) + .toUpperCase() + .padStart(4, "0")}` + ) + .join(" ")})` + : "NULL"; + +const tokenize = (input: string, file: string | null = null): Token[] => { + let index = 0; + let line = 1; + let column = 1; + let char = input[index]; + + let tokens: Token[] = []; + + while (index < input.length) { + if (char === "\r") { + const nextChar = input[index + 1]; + + if (nextChar === "\n") { + line += 1; + + index += 2; + column = 1; + char = input[index]; + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + nextChar + )} encountered. Expected ${format("\n")}.`, + { index, line, column }, + file + ); + } + + if (char === "\n") { + line += 1; + + index += 1; + column = 1; + char = input[index]; + + continue; + } + + if (WHITESPACE.has(char) || IGNORED.has(char)) { + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "(") { + tokens.push({ + type: "LeftParenthesis", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ")") { + tokens.push({ + type: "RightParenthesis", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "{") { + tokens.push({ + type: "LeftBracket", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "}") { + tokens.push({ + type: "RightBracket", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ":") { + tokens.push({ + type: "Colon", + value: char, + file, + location: { + index, + line, + column + } + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "0") { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + if (char === "w") { + value += char; + + tokens.push({ + type: "WholeNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (LEADING_DIGITS.has(char)) { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + if (char === "n") { + value += char; + + tokens.push({ + type: "NaturalNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "w") { + value += char; + + tokens.push({ + type: "WholeNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (SIGNS.has(char) && TRUE_DIGITS.has(input[index + 1])) { + let value = char; + let location = { + index, + line, + column + }; + + index += 1; + column += 1; + char = input[index]; + + if (char === "0") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + } + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + if (char === "i") { + value += char; + + tokens.push({ + type: "IntegerNumberLiteral", + value, + file, + location + }); + + index += 1; + column += 1; + char = input[index]; + + continue; + } + + if (char === ".") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (TRUE_DIGITS.has(char)) { + do { + value += char; + + index += 1; + column += 1; + char = input[index]; + } while (DIGITS.has(char)); + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...TRUE_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + if (char === "/") { + value += char; + + index += 1; + column += 1; + char = input[index]; + + if (LEADING_DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + + while (DIGITS.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + throw errorMessage( + `Unexpected character ${format( + char + )} encountered. Expected one of: ${[...LEADING_DIGITS].join(" ")}`, + { index, line, column }, + file + ); + } + + tokens.push({ + type: "RationalNumberLiteral", + value, + file, + location + }); + + continue; + } + + if (NAME.has(char)) { + let value = ""; + let location = { + index, + line, + column + }; + + while (NAME.has(char)) { + value += char; + + index += 1; + column += 1; + char = input[index]; + } + + tokens.push({ + type: "OperatorName", + value, + file, + location + }); + + continue; + } + + // // FIXME: Is something like this needed? + // if (!/[\u0009\u000A\u000D\u0020-\uFFFF]/.test(char)) { + // throw `Unexpected ( ${char} ) encountered. ${line}:${column}`; + // } + + throw errorMessage( + `Unknown character encountered: ${format(char)}`, + { index, line, column }, + file + ); + } + + return tokens; +}; + +export { Token }; +export default tokenize; -- 2.51.2 From 4b2afc4a7dca9dfd672d07e431d26a5094614706 Mon Sep 17 00:00:00 2001 From: Haydn Ewers <27211+haydn@users.noreply.github.com> Date: Sun, 23 Jun 2019 16:08:43 +1000 Subject: [PATCH 3/4] Add toLatex function --- ulm-js/AST.ts | 63 +++++++++++++++++++++++++++++++++++++++++ ulm-js/index.ts | 1 + ulm-js/parse.ts | 53 +--------------------------------- ulm-js/test.ts | 20 +++++++++++++ ulm-js/toLatex.test.ts | 34 ++++++++++++++++++++++ ulm-js/toLatex.ts | 36 +++++++++++++++++++++++ ulm-js/tokenize.test.ts | 2 +- 7 files changed, 156 insertions(+), 53 deletions(-) create mode 100644 ulm-js/AST.ts create mode 100644 ulm-js/toLatex.test.ts create mode 100644 ulm-js/toLatex.ts diff --git a/ulm-js/AST.ts b/ulm-js/AST.ts new file mode 100644 index 0000000..a71d6ff --- /dev/null +++ b/ulm-js/AST.ts @@ -0,0 +1,63 @@ +interface NumericNotEqualRelation { + type: "NumericNotEqualRelation"; + operands: [NumericExpression, NumericExpression]; +} + +interface NumericEqualRelation { + type: "NumericEqualRelation"; + operands: [NumericExpression, NumericExpression]; +} + +type NumericRelation = NumericEqualRelation | NumericNotEqualRelation; + +interface NumericMultiplicationOperation { + type: "NumericMultiplicationOperation"; + operands: [NumericExpression, NumericExpression]; +} + +interface NumericAdditionOperation { + type: "NumericAdditionOperation"; + operands: [NumericExpression, NumericExpression]; +} + +type NumericOperation = + | NumericAdditionOperation + | NumericMultiplicationOperation; + +type NaturalNumber = bigint; + +type IntegerNumber = bigint; + +interface RationalNumber { + type: "RationalNumber"; + numerator: IntegerNumber; + denominator: NaturalNumber; +} + +type NumericExpression = RationalNumber | NumericOperation; + +type Expression = NumericExpression | NumericRelation; + +type AST = Expression[]; + +const isNumericRelation = (e: Expression): e is NumericRelation => + e.type === "NumericEqualRelation" || e.type === "NumericNotEqualRelation"; + +const isNumericOperation = (e: Expression): e is NumericOperation => + e.type === "NumericAdditionOperation" || + e.type === "NumericMultiplicationOperation"; + +const isNumericExpression = (e: Expression): e is NumericExpression => + e.type === "RationalNumber" || isNumericOperation(e); + +const isExpression = (e: AST | Expression): e is Expression => + e.hasOwnProperty("type"); + +export { + Expression, + isExpression, + isNumericExpression, + isNumericOperation, + isNumericRelation +}; +export default AST; diff --git a/ulm-js/index.ts b/ulm-js/index.ts index b800716..ff5f5e7 100644 --- a/ulm-js/index.ts +++ b/ulm-js/index.ts @@ -1,2 +1,3 @@ export { default as parse } from "./parse"; export { default as tokenize } from "./tokenize"; +export { default as toLatex } from "./toLatex"; diff --git a/ulm-js/parse.ts b/ulm-js/parse.ts index c3e5157..04b5e12 100644 --- a/ulm-js/parse.ts +++ b/ulm-js/parse.ts @@ -1,56 +1,5 @@ import Token from "./Token"; - -interface NumericNotEqualRelation { - type: "NumericNotEqualRelation"; - operands: [NumericExpression, NumericExpression]; -} - -interface NumericEqualRelation { - type: "NumericEqualRelation"; - operands: [NumericExpression, NumericExpression]; -} - -const isNumericRelation = (e: Expression): e is NumericRelation => - e.type === "NumericEqualRelation" || e.type === "NumericNotEqualRelation"; - -type NumericRelation = NumericEqualRelation | NumericNotEqualRelation; - -interface NumericMultiplicationOperation { - type: "NumericMultiplicationOperation"; - operands: [NumericExpression, NumericExpression]; -} - -interface NumericAdditionOperation { - type: "NumericAdditionOperation"; - operands: [NumericExpression, NumericExpression]; -} - -const isNumericOperation = (e: Expression): e is NumericOperation => - e.type === "NumericAdditionOperation" || - e.type === "NumericMultiplicationOperation"; - -type NumericOperation = - | NumericAdditionOperation - | NumericMultiplicationOperation; - -type NaturalNumber = bigint; - -type IntegerNumber = bigint; - -interface RationalNumber { - type: "RationalNumber"; - numerator: IntegerNumber; - denominator: NaturalNumber; -} - -const isNumericExpression = (e: Expression): e is NumericExpression => - e.type === "RationalNumber" || isNumericOperation(e); - -type NumericExpression = RationalNumber | NumericOperation; - -type Expression = NumericExpression | NumericRelation; - -type AST = Expression[]; +import AST, { Expression, isNumericExpression } from "./AST"; const parse = (tokens: Token[]): AST => { let ast = []; diff --git a/ulm-js/test.ts b/ulm-js/test.ts index 9d58890..7e6ddbf 100644 --- a/ulm-js/test.ts +++ b/ulm-js/test.ts @@ -1,2 +1,22 @@ +import { strict as assert } from "assert"; + import "./tokenize.test"; import "./parse.test"; +import "./toLatex.test"; + +import tokenize from "./tokenize"; +import parse from "./parse"; +import toLatex from "./toLatex"; + +// End-to-end test + +assert.equal( + toLatex( + parse( + tokenize( + "NumericEqualRelation(NumericAdditionOperation(12_312, 0.33), 1/4), 10" + ) + ) + ), + "12312 + \\frac{33}{100} = \\frac{1}{4}\\\\[16pt]10" +); diff --git a/ulm-js/toLatex.test.ts b/ulm-js/toLatex.test.ts new file mode 100644 index 0000000..8eb95db --- /dev/null +++ b/ulm-js/toLatex.test.ts @@ -0,0 +1,34 @@ +import { strict as assert } from "assert"; +import toLatex from "./toLatex"; + +assert.equal( + toLatex([ + { + type: "NumericEqualRelation", + operands: [ + { + type: "NumericAdditionOperation", + operands: [ + { + type: "RationalNumber", + numerator: BigInt(12312), + denominator: BigInt(1) + }, + { + type: "RationalNumber", + numerator: BigInt(33), + denominator: BigInt(100) + } + ] + }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(4) } + ] + }, + { + type: "RationalNumber", + numerator: BigInt(10), + denominator: BigInt(1) + } + ]), + "12312 + \\frac{33}{100} = \\frac{1}{4}\\\\[16pt]10" +); diff --git a/ulm-js/toLatex.ts b/ulm-js/toLatex.ts new file mode 100644 index 0000000..677b942 --- /dev/null +++ b/ulm-js/toLatex.ts @@ -0,0 +1,36 @@ +import AST, { Expression, isExpression } from "./AST"; + +const expressionToLatex = (expression: Expression): string => { + if (expression.type === "NumericEqualRelation") { + return `${toLatex(expression.operands[0])} = ${toLatex( + expression.operands[1] + )}`; + } + if (expression.type === "NumericNotEqualRelation") { + return `${toLatex(expression.operands[0])} ≠ ${toLatex( + expression.operands[1] + )}`; + } + if (expression.type === "NumericAdditionOperation") { + return `${toLatex(expression.operands[0])} + ${toLatex( + expression.operands[1] + )}`; + } + if (expression.type === "NumericMultiplicationOperation") { + return `${toLatex(expression.operands[0])} \\times ${toLatex( + expression.operands[1] + )}`; + } + if (expression.type === "RationalNumber") { + return expression.denominator === BigInt(1) + ? expression.numerator.toString() + : `\\frac{${expression.numerator}}{${expression.denominator}}`; + } +}; + +const toLatex = (x: Expression | AST): string => + isExpression(x) + ? expressionToLatex(x) + : x.map(e => expressionToLatex(e)).join("\\\\[16pt]"); + +export default toLatex; diff --git a/ulm-js/tokenize.test.ts b/ulm-js/tokenize.test.ts index 218d4ce..d1d966c 100644 --- a/ulm-js/tokenize.test.ts +++ b/ulm-js/tokenize.test.ts @@ -1,5 +1,5 @@ import { strict as assert } from "assert"; -import { tokenize } from "./"; +import tokenize from "./tokenize"; // Whitespace & Ignored -- 2.51.2 From 9b04fcbbd374a0741edb3c81f4eb9bcafd5093f7 Mon Sep 17 00:00:00 2001 From: Haydn Ewers <27211+haydn@users.noreply.github.com> Date: Sun, 23 Jun 2019 16:12:20 +1000 Subject: [PATCH 4/4] Add compile function --- ulm-js/compile.test.ts | 29 +++++++++++++++++++++++++++++ ulm-js/compile.ts | 8 ++++++++ ulm-js/index.ts | 1 + ulm-js/test.ts | 10 ++++------ 4 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 ulm-js/compile.test.ts create mode 100644 ulm-js/compile.ts diff --git a/ulm-js/compile.test.ts b/ulm-js/compile.test.ts new file mode 100644 index 0000000..3c8c83c --- /dev/null +++ b/ulm-js/compile.test.ts @@ -0,0 +1,29 @@ +import { strict as assert } from "assert"; +import compile from "./compile"; + +assert.deepEqual( + compile("NumericEqualRelation(NumericAdditionOperation(12312, 0.33), 1/4)"), + [ + { + type: "NumericEqualRelation", + operands: [ + { + type: "NumericAdditionOperation", + operands: [ + { + type: "RationalNumber", + numerator: BigInt(12312), + denominator: BigInt(1) + }, + { + type: "RationalNumber", + numerator: BigInt(33), + denominator: BigInt(100) + } + ] + }, + { type: "RationalNumber", numerator: BigInt(1), denominator: BigInt(4) } + ] + } + ] +); diff --git a/ulm-js/compile.ts b/ulm-js/compile.ts new file mode 100644 index 0000000..e49125c --- /dev/null +++ b/ulm-js/compile.ts @@ -0,0 +1,8 @@ +import tokenize from "./tokenize"; +import parse from "./parse"; +import AST from "./AST"; + +const compile = (script: string): AST => parse(tokenize(script)); + + +export default compile; diff --git a/ulm-js/index.ts b/ulm-js/index.ts index ff5f5e7..5f145e8 100644 --- a/ulm-js/index.ts +++ b/ulm-js/index.ts @@ -1,3 +1,4 @@ export { default as parse } from "./parse"; export { default as tokenize } from "./tokenize"; +export { default as compile } from "./compile"; export { default as toLatex } from "./toLatex"; diff --git a/ulm-js/test.ts b/ulm-js/test.ts index 7e6ddbf..b56e87d 100644 --- a/ulm-js/test.ts +++ b/ulm-js/test.ts @@ -2,20 +2,18 @@ import { strict as assert } from "assert"; import "./tokenize.test"; import "./parse.test"; +import "./compile.test"; import "./toLatex.test"; -import tokenize from "./tokenize"; -import parse from "./parse"; +import compile from "./compile"; import toLatex from "./toLatex"; // End-to-end test assert.equal( toLatex( - parse( - tokenize( - "NumericEqualRelation(NumericAdditionOperation(12_312, 0.33), 1/4), 10" - ) + compile( + "NumericEqualRelation(NumericAdditionOperation(12_312, 0.33), 1/4), 10" ) ), "12312 + \\frac{33}{100} = \\frac{1}{4}\\\\[16pt]10"