From 92065a7a15cc46c7f8f34fc04a97f169ce91fa20 Mon Sep 17 00:00:00 2001 From: Corbin Crutchley Date: Wed, 13 May 2026 20:21:55 -0400 Subject: [PATCH] chore: add shared parser package --- package.json | 3 +- packages/compiler/package.json | 1 + packages/compiler/src/index.ts | 10 +- packages/compiler/src/output-transformer.ts | 55 --- packages/compiler/src/root-lexer.ts | 65 ---- packages/compiler/src/root-parser.ts | 72 ---- packages/compiler/src/root-tokens.ts | 40 -- .../compiler/tests/output-transformer.spec.ts | 8 - packages/compiler/tests/root-lexer.spec.ts | 67 ---- packages/compiler/tests/root-parser.spec.ts | 38 -- packages/language-service/package.json | 1 + .../language-service/src/language-plugin.ts | 103 +---- .../language-service/src/service-plugin.ts | 4 +- packages/language-service/tsconfig.json | 3 +- packages/parser/.gitignore | 1 + packages/parser/package.json | 28 ++ packages/parser/src/index.ts | 353 ++++++++++++++++++ .../parser/tests/output-transformer.spec.ts | 8 + packages/parser/tests/root-lexer.spec.ts | 80 ++++ packages/parser/tests/root-parser.spec.ts | 40 ++ packages/parser/tsconfig.json | 9 + pnpm-lock.yaml | 18 + 22 files changed, 565 insertions(+), 442 deletions(-) delete mode 100644 packages/compiler/src/output-transformer.ts delete mode 100644 packages/compiler/src/root-lexer.ts delete mode 100644 packages/compiler/src/root-parser.ts delete mode 100644 packages/compiler/src/root-tokens.ts delete mode 100644 packages/compiler/tests/output-transformer.spec.ts delete mode 100644 packages/compiler/tests/root-lexer.spec.ts delete mode 100644 packages/compiler/tests/root-parser.spec.ts create mode 100644 packages/parser/.gitignore create mode 100644 packages/parser/package.json create mode 100644 packages/parser/src/index.ts create mode 100644 packages/parser/tests/output-transformer.spec.ts create mode 100644 packages/parser/tests/root-lexer.spec.ts create mode 100644 packages/parser/tests/root-parser.spec.ts create mode 100644 packages/parser/tsconfig.json diff --git a/package.json b/package.json index b69e5e4..a6d4b2d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,8 @@ { "private": true, "scripts": { - "build": "pnpm run build:language-service && pnpm run build:language-server && pnpm run build:typescript-plugin && pnpm run build:vscode", + "build": "pnpm run build:parser && pnpm run build:language-service && pnpm run build:language-server && pnpm run build:typescript-plugin && pnpm run build:vscode", + "build:parser": "cd ./packages/parser && pnpm build", "build:language-service": "cd ./packages/language-service && pnpm build", "build:language-server": "cd ./packages/language-server && pnpm build", "build:typescript-plugin": "cd ./packages/typescript-plugin && pnpm build", diff --git a/packages/compiler/package.json b/packages/compiler/package.json index d321f8e..c1c69fb 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -23,6 +23,7 @@ }, "keywords": [], "dependencies": { + "tempblot-parser": "workspace:*", "typescript": "^6.0.3" }, "devDependencies": { diff --git a/packages/compiler/src/index.ts b/packages/compiler/src/index.ts index b5669b5..f90ec3b 100644 --- a/packages/compiler/src/index.ts +++ b/packages/compiler/src/index.ts @@ -1,8 +1,10 @@ import * as fs from "node:fs/promises"; import * as path from "node:path"; -import { tokenizeRoot } from "./root-lexer.js"; -import { parseRoot } from "./root-parser.js"; -import { transformOutput } from "./output-transformer.js"; +import { + parseRoot, + tokenizeRoot, + transformOutputTemplate, +} from "tempblot-parser"; import { transformSetup } from "./setup-transformer.js"; export { TempblotInstance, useParams } from "./instance.js"; @@ -28,7 +30,7 @@ export async function compilePath( const sourceDir = path.dirname(sourcePath); const rootTokens = tokenizeRoot(source); const rootAST = parseRoot(rootTokens); - const transformedOutput = transformOutput(rootAST.output.contents); + const transformedOutput = transformOutputTemplate(rootAST.output.contents); const transformedSetup = transformSetup(rootAST.setup.contents, sourcePath); const concatenatedSetupOutput = ` ${transformedSetup} diff --git a/packages/compiler/src/output-transformer.ts b/packages/compiler/src/output-transformer.ts deleted file mode 100644 index 20546bd..0000000 --- a/packages/compiler/src/output-transformer.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* -Given: - -``` -const someStr = `
<>" : "\<<">>
`; -``` - -I want to get: - -``` -`const someStr = \`
${val ? ">>" : "<<"}
\`;` -``` - -This means that we need to handle: -- Replacing `<>` with `${someVal}` -- Replacing `"\>>"` with `">>"` -- Replacing `"\<<"` with `"<<"` -- Replacing "`" with "\`" - -All without using regex. - */ -export function transformOutput(output: string): string { - const outputArr = output.split(""); - const outputArrLength = outputArr.length; - const newOutputArr: string[] = []; - let i = 0; - while (i < outputArrLength) { - if (outputArr[i] === "<" && outputArr[i + 1] === "<") { - let j = i + 2; - while (j < outputArrLength) { - if (outputArr[j - 1] !== "\\" && outputArr[j] === ">" && outputArr[j + 1] === ">") { - newOutputArr.push("${"); - newOutputArr.push(transformOutput(output.substring(i + 2, j))); - newOutputArr.push("}"); - i = j + 2; - break; - } - j++; - } - } else if (outputArr[i] === "\\" && outputArr[i + 1] === ">") { - newOutputArr.push(">"); - i += 2; - } else if (outputArr[i] === "\\" && outputArr[i + 1] === "<") { - newOutputArr.push("<"); - i += 2; - } else if (outputArr[i] === "`") { - newOutputArr.push("\\`"); - i++; - } else { - newOutputArr.push(outputArr[i]); - i++; - } - } - return newOutputArr.join(""); -} diff --git a/packages/compiler/src/root-lexer.ts b/packages/compiler/src/root-lexer.ts deleted file mode 100644 index 65ed184..0000000 --- a/packages/compiler/src/root-lexer.ts +++ /dev/null @@ -1,65 +0,0 @@ -import {RootToken, rootKeywords, rootDefaultKeyword, rootAttributeKeyword} from "./root-tokens.js"; - -export function tokenizeRoot(source: string): RootToken[] { - const tokens: RootToken[] = []; - - let currentString = ""; - for (let i = 0; i < source.length; i++) { - currentString += source[i]; - for (const keyword of rootKeywords) { - const match = currentString.match(keyword.match); - if (match) { - const text = currentString.slice(0, currentString.length - match[0].length); - if (text) { - tokens.push({ - type: rootDefaultKeyword.type, - attributes: {value: text}, - }); - currentString = currentString.slice(text.length); - } - switch (keyword.type) { - case "TagOpenStart": { - tokens.push({ - type: keyword.type, - attributes: {name: match[1]}, - }); - const subMatch = currentString.match(rootAttributeKeyword.match); - if (subMatch) { - tokens.push({ - type: rootAttributeKeyword.type, - attributes: {name: subMatch[1], value: subMatch[2]}, - }); - } else { - // Remove the opening tag and, separately, the closing angle bracket - currentString = currentString.slice(match[1].length + 1, currentString.length - 1); - if (currentString) { - tokens.push({ - type: rootDefaultKeyword.type, - attributes: {value: currentString}, - }); - } - } - tokens.push({ - type: "TagOpenEnd", - attributes: {name: match[1]}, - }); - break; - } - case "TagClose": { - tokens.push({ - type: keyword.type, - attributes: {name: match[1]}, - }); - break; - } - default: - break; - } - currentString = ""; - break; - } - } - } - - return tokens; -} diff --git a/packages/compiler/src/root-parser.ts b/packages/compiler/src/root-parser.ts deleted file mode 100644 index d22a375..0000000 --- a/packages/compiler/src/root-parser.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { RootToken } from "./root-tokens.js"; - -interface RootNode { - setup: SetupNode; - output: OutputNode; -} - -interface SetupNode { - attributes: { [key: string]: string }; - contents: string; -} - -interface OutputNode { - attributes: { [key: string]: string }; - contents: string; -} - -export function parseRoot(tokens: RootToken[]): RootNode { - const rootNode: RootNode = { - setup: { attributes: {}, contents: "" }, - output: { attributes: {}, contents: "" }, - }; - - let currentTag: keyof RootNode | null = null; - let hasSeenCurrentTagOpeningEnd = false; - - for (const token of tokens) { - // If text token and is all whitespace, skip - if (token.type === "Text" && !token.attributes.value.trim()) { - continue; - } - if (!currentTag && token.type === "TagOpenStart") { - currentTag = token.attributes.name as keyof RootNode; - continue; - } - if (currentTag && token.type === "TagClose" && token.attributes.name === currentTag) { - currentTag = null; - hasSeenCurrentTagOpeningEnd = false; - continue; - } - if (!hasSeenCurrentTagOpeningEnd && currentTag && token.type === "TagAttribute") { - rootNode[currentTag].attributes[token.attributes.name] = token.attributes.value; - continue; - } - if (!hasSeenCurrentTagOpeningEnd && currentTag && token.type === "TagOpenEnd") { - hasSeenCurrentTagOpeningEnd = true; - continue; - } - // We ignore "Text" that occurs in the first tag opening - if (hasSeenCurrentTagOpeningEnd && currentTag) { - let value = ""; - if (token.type === "TagOpenStart") { - value = `<${token.attributes.name}`; - } - if (token.type === "TagOpenEnd") { - value = `>`; - } - if (token.type === "TagClose") { - value = ``; - } - if (token.type === "TagAttribute") { - value = ` ${token.attributes.name}="${token.attributes.value}"`; - } - if (token.type === "Text") { - value = token.attributes.value; - } - rootNode[currentTag].contents += value; - } - } - - return rootNode; -} diff --git a/packages/compiler/src/root-tokens.ts b/packages/compiler/src/root-tokens.ts deleted file mode 100644 index 0f440a2..0000000 --- a/packages/compiler/src/root-tokens.ts +++ /dev/null @@ -1,40 +0,0 @@ -interface RootAttribute { - name: string; - value: string; -} - -interface RootTag { - name: string; -} - -interface RootText { - value: string; -} - -export const rootDefaultKeyword = { match: /[^<]+/, type: "Text", attributes: {} as RootText }as const; - -export const rootAttributeKeyword = - { match: /([a-zA-Z-]+)="([^"]*)"/, type: "TagAttribute", attributes: {} as RootAttribute } as const; - -export const rootKeywords = [ - // TagOpenEnd is implicit - { match: /<([a-zA-Z][a-zA-Z0-9]*)\b[^>]*>/, type: "TagOpenStart", attributes: {} as RootTag }, - { match: /<\/([a-zA-Z][a-zA-Z0-9]*)>/, type: "TagClose", attributes: {} as RootTag } -] as const; - -export type RootToken = { - type: "TagOpenStart"; - attributes: RootTag; -} | { - type: "TagOpenEnd"; - attributes: RootTag; -} | { - type: "TagClose"; - attributes: RootTag; -} | { - type: "TagAttribute"; - attributes: RootAttribute; -} | { - type: "Text"; - attributes: RootText; -}; diff --git a/packages/compiler/tests/output-transformer.spec.ts b/packages/compiler/tests/output-transformer.spec.ts deleted file mode 100644 index 99e9914..0000000 --- a/packages/compiler/tests/output-transformer.spec.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { test, expect } from "vitest"; -import {transformOutput} from "../src/output-transformer.js"; - -test("outputTransformer", {}, () => { - const source = 'const someStr = `
<\\>" : "\\<\\<">>
`;'; - const cleaned = transformOutput(source); - expect(cleaned).toEqual('const someStr = \\`
${val ? ">>" : "<<"}
\\`;'); -}) diff --git a/packages/compiler/tests/root-lexer.spec.ts b/packages/compiler/tests/root-lexer.spec.ts deleted file mode 100644 index 56bd738..0000000 --- a/packages/compiler/tests/root-lexer.spec.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { test, expect } from "vitest"; -import { tokenizeRoot } from "../src/root-lexer.js"; - -test("tokenizeRoot basic", {}, () => { - const source = ` - -const setup = 123; - - - -{ - "test": <> -} - -`.trim(); - - const tokens = tokenizeRoot(source); - expect(tokens).toStrictEqual([ - { type: 'TagOpenStart', attributes: { name: 'setup' } }, - { type: 'TagOpenEnd', attributes: { name: 'setup' } }, - { type: 'Text', attributes: { value: '\nconst setup = 123;\n' } }, - { type: 'TagClose', attributes: { name: 'setup' } }, - { type: 'Text', attributes: { value: '\n\n' } }, - { type: 'TagOpenStart', attributes: { name: 'output' } }, - { type: 'TagAttribute', attributes: { name: 'lang', value: 'json' } }, - { type: 'TagOpenEnd', attributes: { name: 'output' } }, - { type: 'Text', attributes: { value: '\n{\n "test": <' } }, - { type: 'TagOpenStart', attributes: { name: 'setup' } }, - { type: 'TagOpenEnd', attributes: { name: 'setup' } }, - { type: 'Text', attributes: { value: '>\n}\n' } }, - { type: 'TagClose', attributes: { name: 'output' } } - ] - ); -}) - -test("tokenizeRoot with other things inside of interpolation", {}, () => { - const source = ` - -const hello = 123; - - - -{ - "test": <> -} - -`.trim(); - - const tokens = tokenizeRoot(source); - expect(tokens).toStrictEqual([ - { type: 'TagOpenStart', attributes: { name: 'setup' } }, - { type: 'TagOpenEnd', attributes: { name: 'setup' } }, - { type: 'Text', attributes: { value: '\nconst hello = 123;\n' } }, - { type: 'TagClose', attributes: { name: 'setup' } }, - { type: 'Text', attributes: { value: '\n\n' } }, - { type: 'TagOpenStart', attributes: { name: 'output' } }, - { type: 'TagAttribute', attributes: { name: 'lang', value: 'json' } }, - { type: 'TagOpenEnd', attributes: { name: 'output' } }, - { type: 'Text', attributes: { value: '\n{\n "test": <' } }, - { type: 'TagOpenStart', attributes: { name: 'hello' } }, - { type: 'Text', attributes: { value: ' ? ["one", \'two\', \'three\'] : ""' } }, - { type: 'TagOpenEnd', attributes: { name: 'hello' } }, - { type: 'Text', attributes: { value: '>\n}\n' } }, - { type: 'TagClose', attributes: { name: 'output' } } - ] - ); -}) diff --git a/packages/compiler/tests/root-parser.spec.ts b/packages/compiler/tests/root-parser.spec.ts deleted file mode 100644 index b548f90..0000000 --- a/packages/compiler/tests/root-parser.spec.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { test, expect } from "vitest"; -import { parseRoot } from "../src/root-parser.js"; -import { tokenizeRoot } from "../src/root-lexer.js"; - -const source = ` - -const setup = 123; - - - -{ - "test": <> -} - -`.trim(); - -test("tokenizeRoot", {}, () => { - const tokens = tokenizeRoot(source); - const root = parseRoot(tokens); - expect(root).toEqual({ - "output": { - "attributes": { - "lang": "json", - }, - "contents": ` -{ - "test": <> -} -`, - }, - "setup": { - "attributes": {}, - "contents": ` -const setup = 123; -`, - }, - }); -}) diff --git a/packages/language-service/package.json b/packages/language-service/package.json index 96b900f..7e7db36 100644 --- a/packages/language-service/package.json +++ b/packages/language-service/package.json @@ -26,6 +26,7 @@ "@volar/language-core": "~2.4.23", "@volar/language-service": "~2.4.23", "@volar/typescript": "~2.4.23", + "tempblot-parser": "workspace:*", "vscode-html-languageservice": "^5.5.1", "vscode-uri": "^3.1.0" }, diff --git a/packages/language-service/src/language-plugin.ts b/packages/language-service/src/language-plugin.ts index 9ebd0ed..1199747 100644 --- a/packages/language-service/src/language-plugin.ts +++ b/packages/language-service/src/language-plugin.ts @@ -1,8 +1,14 @@ /// import { CodeMapping, type VirtualCode } from "@volar/language-core"; import { type LanguagePlugin } from "@volar/language-service"; +import { + getRootBlocks, + parseTempblotRoot, + scanInterpolations, + type InterpolationData, + type ParsedRoot, +} from "tempblot-parser"; import type * as ts from "typescript"; -import * as html from "vscode-html-languageservice"; import { URI } from "vscode-uri"; export function createTempblotLanguagePlugin(): LanguagePlugin< @@ -64,8 +70,6 @@ export function createTempblotLanguagePlugin(): LanguagePlugin< }; } -const htmlLs = html.getLanguageService(); - export class TempblotVirtualCode implements VirtualCode { id = "root"; languageId = "tempblot"; @@ -73,7 +77,7 @@ export class TempblotVirtualCode implements VirtualCode { embeddedCodes: VirtualCode[] = []; // Reuse in custom language service plugin - htmlDocument: html.HTMLDocument; + rootDocument: ParsedRoot; constructor(public snapshot: ts.IScriptSnapshot) { this.mappings = [ @@ -91,26 +95,19 @@ export class TempblotVirtualCode implements VirtualCode { }, }, ]; - this.htmlDocument = htmlLs.parseHTMLDocument( - html.TextDocument.create( - "", - "html", - 0, - snapshot.getText(0, snapshot.getLength()), - ), - ); + this.rootDocument = parseTempblotRoot(snapshot.getText(0, snapshot.getLength())); this.embeddedCodes = [ - ...getTempblotEmbeddedCodes(snapshot, this.htmlDocument), + ...getTempblotEmbeddedCodes(snapshot, this.rootDocument), ]; } } function* getTempblotEmbeddedCodes( snapshot: ts.IScriptSnapshot, - htmlDocument: html.HTMLDocument, + rootDocument: ParsedRoot, ): Generator { - const setups = htmlDocument.roots.filter((root) => root.tag === "setup"); - const outputs = htmlDocument.roots.filter((root) => root.tag === "output"); + const setups = getRootBlocks(rootDocument, "setup"); + const outputs = getRootBlocks(rootDocument, "output"); // If we have both setup and output, combine them into a single TypeScript context // This allows setup variables to be accessible in output interpolations @@ -118,20 +115,11 @@ function* getTempblotEmbeddedCodes( const setup = setups[0]; // Take the first setup block const output = outputs[0]; // Take the first output block - if ( - !setup.startTagEnd || - !setup.endTagStart || - !output.startTagEnd || - !output.endTagStart - ) { - return; - } - const setupText = snapshot.getText(setup.startTagEnd, setup.endTagStart); const outputText = snapshot.getText(output.startTagEnd, output.endTagStart); // Extract interpolation expressions and their positions from output - const interpolationsData = extractInterpolationsWithPositions(outputText); + const interpolationsData = scanInterpolations(outputText); // Create a combined TypeScript context wrapped in a module // This ensures each .blot file has its own isolated scope @@ -213,69 +201,6 @@ function* getTempblotEmbeddedCodes( } } -interface InterpolationData { - expression: string; - sourceStart: number; - sourceEnd: number; - fullStart: number; // includes << - fullEnd: number; // includes >> -} - -function extractInterpolationsWithPositions(text: string): InterpolationData[] { - const interpolations: InterpolationData[] = []; - let i = 0; - const length = text.length; - - while (i < length) { - if (text[i] === "<" && text[i + 1] === "<") { - const fullStart = i; - let j = i + 2; - let depth = 1; - - // Find the matching >> - while (j < length && depth > 0) { - if (text[j] === "<" && text[j + 1] === "<") { - depth++; - j += 2; - } else if (text[j] === ">" && text[j + 1] === ">") { - depth--; - if (depth === 0) { - // Extract the interpolation content - const sourceStart = i + 2; - const sourceEnd = j; - const expression = text.substring(sourceStart, sourceEnd).trim(); - const fullEnd = j + 2; - - if (expression) { - interpolations.push({ - expression, - sourceStart, - sourceEnd, - fullStart, - fullEnd, - }); - } - i = j + 2; - break; - } - j += 2; - } else { - j++; - } - } - - if (depth > 0) { - // Unclosed interpolation, skip - i++; - } - } else { - i++; - } - } - - return interpolations; -} - function createJsonWithMappings( outputText: string, interpolationsData: InterpolationData[], diff --git a/packages/language-service/src/service-plugin.ts b/packages/language-service/src/service-plugin.ts index b752e7a..24237ae 100644 --- a/packages/language-service/src/service-plugin.ts +++ b/packages/language-service/src/service-plugin.ts @@ -19,8 +19,8 @@ export function createTempblotServicePlugin() { if (!(virtualCode instanceof TempblotVirtualCode)) { return; } - const setupNodes = virtualCode.htmlDocument.roots.filter((root: any) => root.tag === 'setup'); - const outputNodes = virtualCode.htmlDocument.roots.filter((root: any) => root.tag === 'output'); + const setupNodes = virtualCode.rootDocument.blocks.filter((root) => root.tag === 'setup'); + const outputNodes = virtualCode.rootDocument.blocks.filter((root) => root.tag === 'output'); if (setupNodes.length == 1 && outputNodes.length == 1) { return; diff --git a/packages/language-service/tsconfig.json b/packages/language-service/tsconfig.json index f7897a0..3df0bd9 100644 --- a/packages/language-service/tsconfig.json +++ b/packages/language-service/tsconfig.json @@ -1,9 +1,10 @@ { "extends": "../../tsconfig.base.json", "include": ["src"], + "references": [{ "path": "../parser/tsconfig.json" }], "compilerOptions": { "declaration": true, "outDir": "lib" } -} \ No newline at end of file +} diff --git a/packages/parser/.gitignore b/packages/parser/.gitignore new file mode 100644 index 0000000..f1ff06d --- /dev/null +++ b/packages/parser/.gitignore @@ -0,0 +1 @@ +lib/ \ No newline at end of file diff --git a/packages/parser/package.json b/packages/parser/package.json new file mode 100644 index 0000000..9b075d2 --- /dev/null +++ b/packages/parser/package.json @@ -0,0 +1,28 @@ +{ + "name": "tempblot-parser", + "version": "0.0.1", + "type": "module", + "description": "Shared parser utilities for Tempblot", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "import": "./lib/index.js", + "default": "./lib/index.js" + } + }, + "files": [ + "lib" + ], + "scripts": { + "build": "tsc --build", + "prepack": "tsc --build --clean && tsc --build", + "test:lib": "vitest", + "test:types": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "^24.5.2", + "typescript": "^6.0.3", + "vitest": "^4.1.6" + }, + "sideEffects": false +} diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts new file mode 100644 index 0000000..9f18e9e --- /dev/null +++ b/packages/parser/src/index.ts @@ -0,0 +1,353 @@ +export interface RootAttribute { + name: string; + value: string; + start: number; + end: number; +} + +export interface RootTag { + name: string; + start: number; + end: number; +} + +export interface RootText { + value: string; + start: number; + end: number; +} + +export type RootToken = + | { + type: "TagOpenStart"; + attributes: RootTag; + } + | { + type: "TagOpenEnd"; + attributes: RootTag; + } + | { + type: "TagClose"; + attributes: RootTag; + } + | { + type: "TagAttribute"; + attributes: RootAttribute; + } + | { + type: "Text"; + attributes: RootText; + }; + +export interface RootBlock { + tag: string; + attributes: Record; + contents: string; + start: number; + end: number; + startTagStart: number; + startTagEnd: number; + endTagStart: number; + endTagEnd: number; +} + +export interface ParsedRoot { + blocks: RootBlock[]; +} + +export interface RequiredRoot { + setup: RootBlock; + output: RootBlock; +} + +export interface InterpolationData { + expression: string; + rawExpression: string; + sourceStart: number; + sourceEnd: number; + fullStart: number; + fullEnd: number; +} + +const emptyBlock = (tag: string): RootBlock => ({ + tag, + attributes: {}, + contents: "", + start: 0, + end: 0, + startTagStart: 0, + startTagEnd: 0, + endTagStart: 0, + endTagEnd: 0, +}); + +export function tokenizeRoot(source: string): RootToken[] { + const tokens: RootToken[] = []; + const tagPattern = /<([a-zA-Z][a-zA-Z0-9-]*)\b[^>]*>/g; + let lastEnd = 0; + let match: RegExpExecArray | null; + + while ((match = tagPattern.exec(source))) { + const tagText = match[0]; + const tagName = match[1]; + const tagStart = match.index; + const tagEnd = match.index + tagText.length; + const closeText = ``; + const closeStart = source.indexOf(closeText, tagEnd); + + if (closeStart === -1) { + continue; + } + + if (tagStart > lastEnd) { + tokens.push({ + type: "Text", + attributes: { + value: source.slice(lastEnd, tagStart), + start: lastEnd, + end: tagStart, + }, + }); + } + + tokens.push({ + type: "TagOpenStart", + attributes: { name: tagName, start: tagStart, end: tagStart + tagName.length + 1 }, + }); + + const attributePattern = /([a-zA-Z-]+)="([^"]*)"/g; + let attributeMatch: RegExpExecArray | null; + while ((attributeMatch = attributePattern.exec(tagText))) { + tokens.push({ + type: "TagAttribute", + attributes: { + name: attributeMatch[1], + value: attributeMatch[2], + start: tagStart + attributeMatch.index, + end: tagStart + attributeMatch.index + attributeMatch[0].length, + }, + }); + } + + tokens.push({ + type: "TagOpenEnd", + attributes: { name: tagName, start: tagEnd - 1, end: tagEnd }, + }); + + if (closeStart > tagEnd) { + tokens.push({ + type: "Text", + attributes: { + value: source.slice(tagEnd, closeStart), + start: tagEnd, + end: closeStart, + }, + }); + } + + const closeEnd = closeStart + closeText.length; + tokens.push({ + type: "TagClose", + attributes: { name: tagName, start: closeStart, end: closeEnd }, + }); + + lastEnd = closeEnd; + tagPattern.lastIndex = closeEnd; + } + + if (lastEnd < source.length) { + tokens.push({ + type: "Text", + attributes: { + value: source.slice(lastEnd), + start: lastEnd, + end: source.length, + }, + }); + } + + return tokens; +} + +export function parseRootDocument(tokens: RootToken[]): ParsedRoot { + const blocks: RootBlock[] = []; + let currentBlock: RootBlock | undefined; + let hasSeenCurrentTagOpeningEnd = false; + + for (const token of tokens) { + if (!currentBlock && token.type === "TagOpenStart") { + currentBlock = { + tag: token.attributes.name, + attributes: {}, + contents: "", + start: token.attributes.start, + end: token.attributes.end, + startTagStart: token.attributes.start, + startTagEnd: token.attributes.end, + endTagStart: token.attributes.end, + endTagEnd: token.attributes.end, + }; + hasSeenCurrentTagOpeningEnd = false; + continue; + } + + if (!currentBlock) { + continue; + } + + if (!hasSeenCurrentTagOpeningEnd && token.type === "TagAttribute") { + currentBlock.attributes[token.attributes.name] = token.attributes.value; + continue; + } + + if (!hasSeenCurrentTagOpeningEnd && token.type === "TagOpenEnd") { + currentBlock.startTagEnd = token.attributes.end; + hasSeenCurrentTagOpeningEnd = true; + continue; + } + + if (token.type === "TagClose" && token.attributes.name === currentBlock.tag) { + currentBlock.endTagStart = token.attributes.start; + currentBlock.endTagEnd = token.attributes.end; + currentBlock.end = token.attributes.end; + blocks.push(currentBlock); + currentBlock = undefined; + hasSeenCurrentTagOpeningEnd = false; + continue; + } + + if (hasSeenCurrentTagOpeningEnd) { + currentBlock.contents += tokenToSource(token); + } + } + + return { blocks }; +} + +export function parseRoot(tokens: RootToken[]): RequiredRoot { + const document = parseRootDocument(tokens); + return { + setup: document.blocks.find((block) => block.tag === "setup") ?? emptyBlock("setup"), + output: + document.blocks.find((block) => block.tag === "output") ?? emptyBlock("output"), + }; +} + +export function parseTempblotRoot(source: string): ParsedRoot { + return parseRootDocument(tokenizeRoot(source)); +} + +export function getRootBlocks(document: ParsedRoot, tag: string): RootBlock[] { + return document.blocks.filter((block) => block.tag === tag); +} + +export function scanInterpolations(text: string): InterpolationData[] { + const interpolations: InterpolationData[] = []; + let i = 0; + + while (i < text.length) { + if (text[i] !== "<" || text[i + 1] !== "<") { + i++; + continue; + } + + const fullStart = i; + let j = i + 2; + let depth = 1; + + while (j < text.length && depth > 0) { + if (text[j - 1] !== "\\" && text[j] === "<" && text[j + 1] === "<") { + depth++; + j += 2; + continue; + } + + if (text[j - 1] !== "\\" && text[j] === ">" && text[j + 1] === ">") { + depth--; + if (depth === 0) { + const rawExpression = text.slice(i + 2, j); + const leadingWhitespace = rawExpression.match(/^\s*/)?.[0].length ?? 0; + const trailingWhitespace = rawExpression.match(/\s*$/)?.[0].length ?? 0; + const trimmedRawExpression = rawExpression.slice( + leadingWhitespace, + rawExpression.length - trailingWhitespace, + ); + + interpolations.push({ + expression: transformOutputTemplate(trimmedRawExpression), + rawExpression: trimmedRawExpression, + sourceStart: i + 2 + leadingWhitespace, + sourceEnd: j - trailingWhitespace, + fullStart, + fullEnd: j + 2, + }); + i = j + 2; + break; + } + j += 2; + continue; + } + + j++; + } + + if (depth > 0) { + i++; + } + } + + return interpolations; +} + +export function transformOutputTemplate(output: string): string { + const interpolations = scanInterpolations(output); + let transformed = ""; + let lastOffset = 0; + + for (const interpolation of interpolations) { + transformed += escapeOutputText(output.slice(lastOffset, interpolation.fullStart)); + transformed += "${"; + transformed += interpolation.expression; + transformed += "}"; + lastOffset = interpolation.fullEnd; + } + + transformed += escapeOutputText(output.slice(lastOffset)); + return transformed; +} + +function escapeOutputText(text: string): string { + let transformed = ""; + + for (let i = 0; i < text.length; i++) { + if (text[i] === "\\" && text[i + 1] === ">") { + transformed += ">"; + i++; + } else if (text[i] === "\\" && text[i + 1] === "<") { + transformed += "<"; + i++; + } else if (text[i] === "`") { + transformed += "\\`"; + } else { + transformed += text[i]; + } + } + + return transformed; +} + +function tokenToSource(token: RootToken): string { + if (token.type === "TagOpenStart") { + return `<${token.attributes.name}`; + } + if (token.type === "TagOpenEnd") { + return ">"; + } + if (token.type === "TagClose") { + return ``; + } + if (token.type === "TagAttribute") { + return ` ${token.attributes.name}="${token.attributes.value}"`; + } + return token.attributes.value; +} diff --git a/packages/parser/tests/output-transformer.spec.ts b/packages/parser/tests/output-transformer.spec.ts new file mode 100644 index 0000000..6b2c2f3 --- /dev/null +++ b/packages/parser/tests/output-transformer.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from "vitest"; +import { transformOutputTemplate } from "../src/index.js"; + +test("transformOutputTemplate", () => { + const source = 'const someStr = `
<\\>" : "\\<\\<">>
`;'; + const cleaned = transformOutputTemplate(source); + expect(cleaned).toEqual('const someStr = \\`
${val ? ">>" : "<<"}
\\`;'); +}); diff --git a/packages/parser/tests/root-lexer.spec.ts b/packages/parser/tests/root-lexer.spec.ts new file mode 100644 index 0000000..29d1340 --- /dev/null +++ b/packages/parser/tests/root-lexer.spec.ts @@ -0,0 +1,80 @@ +import { expect, test } from "vitest"; +import { tokenizeRoot } from "../src/index.js"; + +function withoutOffsets(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(withoutOffsets); + } + + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => key !== "start" && key !== "end") + .map(([key, entry]) => [key, withoutOffsets(entry)]), + ); + } + + return value; +} + +test("tokenizeRoot basic", () => { + const source = ` + +const setup = 123; + + + +{ + "test": <> +} + +`.trim(); + + const tokens = tokenizeRoot(source); + expect(withoutOffsets(tokens)).toStrictEqual([ + { type: "TagOpenStart", attributes: { name: "setup" } }, + { type: "TagOpenEnd", attributes: { name: "setup" } }, + { type: "Text", attributes: { value: "\nconst setup = 123;\n" } }, + { type: "TagClose", attributes: { name: "setup" } }, + { type: "Text", attributes: { value: "\n\n" } }, + { type: "TagOpenStart", attributes: { name: "output" } }, + { type: "TagAttribute", attributes: { name: "lang", value: "json" } }, + { type: "TagOpenEnd", attributes: { name: "output" } }, + { type: "Text", attributes: { value: "\n{\n \"test\": <>\n}\n" } }, + { type: "TagClose", attributes: { name: "output" } }, + ]); +}); + +test("tokenizeRoot keeps interpolation contents as text", () => { + const source = ` + +const hello = 123; + + + +{ + "test": <> +} + +`.trim(); + + const tokens = tokenizeRoot(source); + expect(withoutOffsets(tokens)).toStrictEqual([ + { type: "TagOpenStart", attributes: { name: "setup" } }, + { type: "TagOpenEnd", attributes: { name: "setup" } }, + { type: "Text", attributes: { value: "\nconst hello = 123;\n" } }, + { type: "TagClose", attributes: { name: "setup" } }, + { type: "Text", attributes: { value: "\n\n" } }, + { type: "TagOpenStart", attributes: { name: "output" } }, + { type: "TagAttribute", attributes: { name: "lang", value: "json" } }, + { type: "TagOpenEnd", attributes: { name: "output" } }, + { + type: "Text", + attributes: { + value: + "\n{\n \"test\": <>\n}\n", + }, + }, + { type: "TagClose", attributes: { name: "output" } }, + ]); +}); diff --git a/packages/parser/tests/root-parser.spec.ts b/packages/parser/tests/root-parser.spec.ts new file mode 100644 index 0000000..cd31277 --- /dev/null +++ b/packages/parser/tests/root-parser.spec.ts @@ -0,0 +1,40 @@ +import { expect, test } from "vitest"; +import { parseRoot, tokenizeRoot } from "../src/index.js"; + +const source = ` + +const setup = 123; + + + +{ + "test": <> +} + +`.trim(); + +test("parseRoot returns root blocks with source offsets", () => { + const tokens = tokenizeRoot(source); + const root = parseRoot(tokens); + + expect(root).toMatchObject({ + output: { + attributes: { + lang: "json", + }, + contents: ` +{ + "test": <> +} +`, + }, + setup: { + attributes: {}, + contents: ` +const setup = 123; +`, + }, + }); + expect(root.setup.startTagEnd).toBe(source.indexOf(">") + 1); + expect(root.output.contents).toContain("<>"); +}); diff --git a/packages/parser/tsconfig.json b/packages/parser/tsconfig.json new file mode 100644 index 0000000..8c97071 --- /dev/null +++ b/packages/parser/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"], + "compilerOptions": { + "declaration": true, + "outDir": "lib", + "rootDir": "src" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca4f5b3..7176683 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: packages/compiler: dependencies: + tempblot-parser: + specifier: workspace:* + version: link:../parser typescript: specifier: ^6.0.3 version: 6.0.3 @@ -79,6 +82,9 @@ importers: '@volar/typescript': specifier: ~2.4.23 version: 2.4.28 + tempblot-parser: + specifier: workspace:* + version: link:../parser vscode-html-languageservice: specifier: ^5.5.1 version: 5.6.2 @@ -93,6 +99,18 @@ importers: specifier: ^6.0.3 version: 6.0.3 + packages/parser: + devDependencies: + '@types/node': + specifier: ^24.5.2 + version: 24.12.4 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.6 + version: 4.1.6(@types/node@24.12.4)(vite@8.0.12(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(yaml@2.9.0)) + packages/typescript-plugin: dependencies: '@volar/typescript': -- 2.51.2