diff --git a/src/ast.ts b/src/ast.ts new file mode 100644 index 0000000..31e244f --- /dev/null +++ b/src/ast.ts @@ -0,0 +1,179 @@ +export type ShellDialect = "posix" | "bash" | "mksh" | "zsh"; + +export type ParseOptions = { + dialect?: ShellDialect; + /** If true, keep comments as nodes/tokens in the output (future). */ + keepComments?: boolean; +}; + +export type Literal = { type: "Literal"; value: string }; +export type SglQuoted = { type: "SglQuoted"; value: string }; +export type DblQuoted = { type: "DblQuoted"; parts: WordPart[] }; +export type ParamExp = { + type: "ParamExp"; + short: boolean; + param: Literal; + op?: string; + value?: Word; +}; +export type CmdSubst = { type: "CmdSubst"; stmts: Statement[] }; +export type ArithExp = { type: "ArithExp"; expr: string }; +export type ProcSubst = { + type: "ProcSubst"; + op: "<" | ">"; + stmts: Statement[]; +}; +export type WordPart = + | Literal + | SglQuoted + | DblQuoted + | ParamExp + | CmdSubst + | ArithExp + | ProcSubst; +export type Word = { type: "Word"; parts: WordPart[] }; +export type Assignment = { + type: "Assignment"; + name: string; + append?: boolean; + value?: Word; + array?: ArrayExpr; +}; +export type ArrayElem = { type: "ArrayElem"; index?: Word; value?: Word }; +export type ArrayExpr = { type: "ArrayExpr"; elems: ArrayElem[] }; +export type RedirOp = + | ">" + | "<" + | ">>" + | ">|" + | ">&" + | "<&" + | "<>" + | "&>" + | "&>>" + | "<<<" + | "<<" + | "<<-"; +export type Redirect = { + type: "Redirect"; + op: RedirOp; + fd?: string; + target: Word; + heredoc?: Word; +}; +export type SimpleCommand = { + type: "SimpleCommand"; + words?: Word[]; + assignments?: Assignment[]; + redirects?: Redirect[]; +}; +export type Subshell = { type: "Subshell"; body: Statement[] }; +export type Block = { type: "Block"; body: Statement[] }; +export type IfClause = { + type: "IfClause"; + cond: Statement[]; + then: Statement[]; + else?: Statement[]; +}; +export type WhileClause = { + type: "WhileClause"; + cond: Statement[]; + body: Statement[]; + until?: boolean; +}; +export type ForClause = { + type: "ForClause"; + name: string; + items?: Word[]; + body: Statement[]; +}; +export type SelectClause = { + type: "SelectClause"; + name: string; + items?: Word[]; + body: Statement[]; +}; +export type FunctionDecl = { + type: "FunctionDecl"; + name: string; + body: Statement[]; +}; +export type CaseItem = { + type: "CaseItem"; + patterns: Word[]; + body: Statement[]; +}; +export type CaseClause = { + type: "CaseClause"; + word: Word; + items: CaseItem[]; +}; +export type TimeClause = { type: "TimeClause"; command: Statement }; +export type TestClause = { type: "TestClause"; expr: Word[] }; +export type ArithCmd = { type: "ArithCmd"; expr: string }; +export type CoprocClause = { + type: "CoprocClause"; + name?: string; + body: Statement; +}; +export type DeclClause = { + type: "DeclClause"; + variant: "declare" | "local" | "export" | "readonly" | "typeset" | "nameref"; + args?: Word[]; + assigns?: Assignment[]; + redirects?: Redirect[]; +}; +export type LetClause = { + type: "LetClause"; + exprs: Word[]; + redirects?: Redirect[]; +}; +export type CStyleLoop = { + type: "CStyleLoop"; + init?: string; + cond?: string; + post?: string; + body: Statement[]; +}; +export type CommentNode = { type: "Comment"; text: string }; +export type Pipeline = { type: "Pipeline"; commands: Statement[] }; +export type Logical = { + type: "Logical"; + op: "and" | "or"; + left: Statement; + right: Statement; +}; +export type Command = + | SimpleCommand + | Subshell + | Block + | IfClause + | WhileClause + | ForClause + | SelectClause + | FunctionDecl + | CaseClause + | TimeClause + | TestClause + | ArithCmd + | CoprocClause + | Pipeline + | Logical + | DeclClause + | LetClause + | CStyleLoop; +export type Statement = { + type: "Statement"; + command: Command; + background?: boolean; + negated?: boolean; +}; +export type Program = { + type: "Program"; + body: Statement[]; + comments?: CommentNode[]; +}; + +export type ParseResult = { + ast: Program; +}; diff --git a/src/index.ts b/src/index.ts index 88418a6..02d704d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,1715 +1,43 @@ -export type ShellDialect = "posix" | "bash" | "mksh" | "zsh"; - -export type ParseOptions = { - dialect?: ShellDialect; - /** If true, keep comments as nodes/tokens in the output (future). */ - keepComments?: boolean; -}; - -export type Literal = { type: "Literal"; value: string }; -export type SglQuoted = { type: "SglQuoted"; value: string }; -export type DblQuoted = { type: "DblQuoted"; parts: WordPart[] }; -export type ParamExp = { - type: "ParamExp"; - short: boolean; - param: Literal; - op?: string; - value?: Word; -}; -export type CmdSubst = { type: "CmdSubst"; stmts: Statement[] }; -export type ArithExp = { type: "ArithExp"; expr: string }; -export type ProcSubst = { - type: "ProcSubst"; - op: "<" | ">"; - stmts: Statement[]; -}; -export type WordPart = - | Literal - | SglQuoted - | DblQuoted - | ParamExp - | CmdSubst - | ArithExp - | ProcSubst; -export type Word = { type: "Word"; parts: WordPart[] }; -export type Assignment = { - type: "Assignment"; - name: string; - append?: boolean; - value?: Word; - array?: ArrayExpr; -}; -export type ArrayElem = { type: "ArrayElem"; index?: Word; value?: Word }; -export type ArrayExpr = { type: "ArrayExpr"; elems: ArrayElem[] }; -export type RedirOp = - | ">" - | "<" - | ">>" - | ">|" - | ">&" - | "<&" - | "<>" - | "&>" - | "&>>" - | "<<<" - | "<<" - | "<<-"; -export type Redirect = { - type: "Redirect"; - op: RedirOp; - fd?: string; - target: Word; - heredoc?: Word; -}; -export type SimpleCommand = { - type: "SimpleCommand"; - words?: Word[]; - assignments?: Assignment[]; - redirects?: Redirect[]; -}; -export type Subshell = { type: "Subshell"; body: Statement[] }; -export type Block = { type: "Block"; body: Statement[] }; -export type IfClause = { - type: "IfClause"; - cond: Statement[]; - then: Statement[]; - else?: Statement[]; -}; -export type WhileClause = { - type: "WhileClause"; - cond: Statement[]; - body: Statement[]; - until?: boolean; -}; -export type ForClause = { - type: "ForClause"; - name: string; - items?: Word[]; - body: Statement[]; -}; -export type SelectClause = { - type: "SelectClause"; - name: string; - items?: Word[]; - body: Statement[]; -}; -export type FunctionDecl = { - type: "FunctionDecl"; - name: string; - body: Statement[]; -}; -export type CaseItem = { - type: "CaseItem"; - patterns: Word[]; - body: Statement[]; -}; -export type CaseClause = { - type: "CaseClause"; - word: Word; - items: CaseItem[]; -}; -export type TimeClause = { type: "TimeClause"; command: Statement }; -export type TestClause = { type: "TestClause"; expr: Word[] }; -export type ArithCmd = { type: "ArithCmd"; expr: string }; -export type CoprocClause = { - type: "CoprocClause"; - name?: string; - body: Statement; -}; -export type DeclClause = { - type: "DeclClause"; - variant: "declare" | "local" | "export" | "readonly" | "typeset" | "nameref"; - args?: Word[]; - assigns?: Assignment[]; - redirects?: Redirect[]; -}; -export type LetClause = { - type: "LetClause"; - exprs: Word[]; - redirects?: Redirect[]; -}; -export type CStyleLoop = { - type: "CStyleLoop"; - init?: string; - cond?: string; - post?: string; - body: Statement[]; -}; -export type CommentNode = { type: "Comment"; text: string }; -export type Pipeline = { type: "Pipeline"; commands: Statement[] }; -export type Logical = { - type: "Logical"; - op: "and" | "or"; - left: Statement; - right: Statement; -}; -export type Command = - | SimpleCommand - | Subshell - | Block - | IfClause - | WhileClause - | ForClause - | SelectClause - | FunctionDecl - | CaseClause - | TimeClause - | TestClause - | ArithCmd - | CoprocClause - | Pipeline - | Logical - | DeclClause - | LetClause - | CStyleLoop; -export type Statement = { - type: "Statement"; - command: Command; - background?: boolean; - negated?: boolean; -}; -export type Program = { - type: "Program"; - body: Statement[]; - comments?: CommentNode[]; -}; - -export type ParseResult = { - ast: Program; -}; - -type OpTokenValue = "&&" | "||" | "|" | ";" | "&" | "!"; - -type SymbolTokenValue = "(" | ")" | "{" | "}"; - -type TokenWordPart = - | { type: "lit"; value: string } - | { type: "sgl"; value: string } - | { type: "dbl"; parts: TokenWordPart[] } - | { - type: "param"; - name: string; - braced: boolean; - op?: string; - value?: string; - } - | { type: "cmd-subst"; raw: string } - | { type: "arith-exp"; raw: string } - | { type: "proc-subst"; op: "<" | ">"; raw: string } - | { type: "backtick"; raw: string }; - -type Token = - | { type: "word"; parts: TokenWordPart[] } - | { type: "op"; value: OpTokenValue } - | { type: "redir"; op: RedirOp; fd?: string } - | { type: "symbol"; value: SymbolTokenValue } - | { type: "arith-cmd"; expr: string } - | { type: "heredoc-body"; content: string } - | { type: "comment"; text: string }; - -export function parse(source: string, options: ParseOptions = {}): ParseResult { - const tokens = tokenize(source, options); - const parser = new Parser(tokens, options); - const ast = parser.parseProgram(); - parser.assertEof(); - return { ast }; -} - -const operatorChars = new Set([";", "|", "&"]); -const redirChars = new Set([">", "<"]); -const symbolChars = new Set(["(", ")", "{", "}"]); - -const isDigit = (value: string) => value >= "0" && value <= "9"; - -const isNameChar = (c: string) => - (c >= "a" && c <= "z") || - (c >= "A" && c <= "Z") || - (c >= "0" && c <= "9") || - c === "_"; - -const isNameStart = (c: string) => - (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_"; - -const specialParams = new Set(["@", "*", "#", "?", "-", "$", "!"]); - -function scanExpansion( - source: string, - pos: number, -): { part: TokenWordPart; end: number } | null { - if (source.charAt(pos) !== "$") return null; - const next = source.charAt(pos + 1); - - // $((expr)) arithmetic expansion - if (next === "(" && source.charAt(pos + 2) === "(") { - let j = pos + 3; - let depth = 0; - while (j < source.length) { - if ( - source.charAt(j) === ")" && - source.charAt(j + 1) === ")" && - depth === 0 - ) - break; - if (source.charAt(j) === "(") depth++; - if (source.charAt(j) === ")") depth--; - j++; - } - const expr = source.slice(pos + 3, j).trim(); - return { part: { type: "arith-exp", raw: expr }, end: j + 2 }; - } - - // $(cmd) command substitution - if (next === "(") { - let j = pos + 2; - let depth = 1; - while (j < source.length && depth > 0) { - if (source.charAt(j) === "(") depth++; - if (source.charAt(j) === ")") depth--; - j++; - } - const raw = source.slice(pos + 2, j - 1); - return { part: { type: "cmd-subst", raw }, end: j }; - } - - // ${...} braced parameter expansion - if (next === "{") { - let j = pos + 2; - let depth = 1; - while (j < source.length && depth > 0) { - if (source.charAt(j) === "{") depth++; - if (source.charAt(j) === "}") depth--; - j++; - } - const inner = source.slice(pos + 2, j - 1); - // Parse inner: name then optional op and value - let nameEnd = 0; - // Skip optional ! # prefix - let prefix = ""; - if (inner.charAt(0) === "!" || inner.charAt(0) === "#") { - prefix = inner.charAt(0); - nameEnd = 1; - } - while (nameEnd < inner.length && isNameChar(inner.charAt(nameEnd))) { - nameEnd++; - } - const name = inner.slice(prefix ? 1 : 0, nameEnd); - const rest = inner.slice(nameEnd); - if (rest.length > 0) { - // Try to extract operator and value - const opMatch = rest.match( - /^(:-|:=|:\+|:\?|-|\+|=|\?|##|%%|#|%|\/\/|\/)/, - ); - if (opMatch) { - const op = opMatch[0]; - const value = rest.slice(op.length); - return { - part: { type: "param", name, braced: true, op, value }, - end: j, - }; - } - // Slice or other complex syntax - store as name with rest - return { - part: { - type: "param", - name: inner, - braced: true, - }, - end: j, - }; - } - return { part: { type: "param", name, braced: true }, end: j }; - } - - // $name or $N or $@ etc - if (isNameStart(next)) { - let j = pos + 2; - while (j < source.length && isNameChar(source.charAt(j))) { - j++; - } - const name = source.slice(pos + 1, j); - return { part: { type: "param", name, braced: false }, end: j }; - } - if (isDigit(next)) { - return { - part: { type: "param", name: next, braced: false }, - end: pos + 2, - }; - } - if (specialParams.has(next)) { - return { - part: { type: "param", name: next, braced: false }, - end: pos + 2, - }; - } - - return null; -} - -function scanBacktick( - source: string, - pos: number, -): { part: TokenWordPart; end: number } { - let j = pos + 1; - while (j < source.length && source.charAt(j) !== "`") { - if (source.charAt(j) === "\\") j++; - j++; - } - const raw = source.slice(pos + 1, j); - return { part: { type: "backtick", raw }, end: j + 1 }; -} - -function tryRedirOp( - source: string, - pos: number, -): { op: RedirOp; len: number } | null { - if (source.startsWith("<<<", pos)) return { op: "<<<", len: 3 }; - if (source.startsWith("&>>", pos)) return { op: "&>>", len: 3 }; - if (source.startsWith("<<-", pos)) return { op: "<<-", len: 3 }; - if (source.startsWith(">>", pos)) return { op: ">>", len: 2 }; - if (source.startsWith(">&", pos)) return { op: ">&", len: 2 }; - if (source.startsWith(">|", pos)) return { op: ">|", len: 2 }; - if (source.startsWith("<>", pos)) return { op: "<>", len: 2 }; - if (source.startsWith("<&", pos)) return { op: "<&", len: 2 }; - if (source.startsWith("&>", pos)) return { op: "&>", len: 2 }; - if (source.startsWith("<<", pos)) return { op: "<<", len: 2 }; - if (source.charAt(pos) === ">") return { op: ">", len: 1 }; - if (source.charAt(pos) === "<") return { op: "<", len: 1 }; - return null; -} - -function tokenize(source: string, options: ParseOptions = {}): Token[] { - const tokens: Token[] = []; - let i = 0; - let atBoundary = true; - - while (i < source.length) { - const ch = source.charAt(i); - - if (ch === " " || ch === "\t" || ch === "\r") { - atBoundary = true; - i += 1; - continue; - } - - if (ch === "\\" && source.charAt(i + 1) === "\n") { - atBoundary = true; - i += 2; - continue; - } - - if (ch === "\\" && source.charAt(i + 1) === "\r") { - if (source.charAt(i + 2) === "\n") { - atBoundary = true; - i += 3; - continue; - } - } - - if (ch === "\n") { - tokens.push({ type: "op", value: ";" }); - atBoundary = true; - i += 1; - - // Check for pending heredocs by scanning recent tokens - const pendingHeredocs: { strip: boolean; delimiter: string }[] = []; - for (let ti = 0; ti < tokens.length; ti++) { - const t = tokens[ti]; - if ( - t && - t.type === "redir" && - (t.op === "<<" || t.op === "<<-") && - !Object.hasOwn(t, "_collected") - ) { - const delimTok = tokens[ti + 1]; - if (delimTok && delimTok.type === "word") { - pendingHeredocs.push({ - strip: t.op === "<<-", - delimiter: tokenPartsText(delimTok.parts), - }); - (t as Record)._collected = true; - } - } - } - - // Collect heredoc bodies - for (const hd of pendingHeredocs) { - let body = ""; - while (i < source.length) { - let lineEnd = source.indexOf("\n", i); - if (lineEnd === -1) lineEnd = source.length; - const line = source.slice(i, lineEnd); - const checkLine = hd.strip ? line.replace(/^\t+/, "") : line; - i = lineEnd < source.length ? lineEnd + 1 : lineEnd; - if (checkLine === hd.delimiter) break; - const processedLine = hd.strip ? line.replace(/^\t+/, "") : line; - body += processedLine + "\n"; - } - tokens.push({ type: "heredoc-body", content: body }); - } - - continue; - } - - if (ch === "#" && atBoundary) { - const start = i + 1; - i += 1; - while (i < source.length && source.charAt(i) !== "\n") { - i += 1; - } - if (options.keepComments) { - tokens.push({ type: "comment", text: source.slice(start, i) }); - } - continue; - } - - if (ch === "!" && atBoundary) { - tokens.push({ type: "op", value: "!" }); - atBoundary = true; - i += 1; - continue; - } - - if (isDigit(ch)) { - let j = i; - while (j < source.length && isDigit(source.charAt(j))) { - j += 1; - } - const redir = tryRedirOp(source, j); - if (redir) { - tokens.push({ type: "redir", op: redir.op, fd: source.slice(i, j) }); - i = j + redir.len; - atBoundary = true; - continue; - } - } - - if (ch === "(" && source.charAt(i + 1) === "(" && atBoundary) { - let j = i + 2; - let depth = 0; - while (j < source.length) { - const c = source.charAt(j); - if (c === ")" && source.charAt(j + 1) === ")" && depth === 0) break; - if (c === "(") depth++; - if (c === ")") depth--; - j++; - } - tokens.push({ type: "arith-cmd", expr: source.slice(i + 2, j).trim() }); - i = j + 2; - atBoundary = true; - continue; - } - - if ( - (ch === "<" || ch === ">") && - source.charAt(i + 1) === "(" && - atBoundary - ) { - const op = ch as "<" | ">"; - let j = i + 2; - let depth = 1; - while (j < source.length && depth > 0) { - if (source.charAt(j) === "(") depth++; - if (source.charAt(j) === ")") depth--; - j++; - } - const raw = source.slice(i + 2, j - 1); - tokens.push({ - type: "word", - parts: [{ type: "proc-subst", op, raw }], - }); - i = j; - atBoundary = false; - continue; - } - - { - const redir = tryRedirOp(source, i); - if (redir) { - tokens.push({ type: "redir", op: redir.op }); - i += redir.len; - atBoundary = true; - continue; - } - } - - if (symbolChars.has(ch)) { - tokens.push({ type: "symbol", value: ch as SymbolTokenValue }); - atBoundary = true; - i += 1; - continue; - } - - if (source.startsWith("&&", i)) { - tokens.push({ type: "op", value: "&&" }); - atBoundary = true; - i += 2; - continue; - } - - if (source.startsWith("||", i)) { - tokens.push({ type: "op", value: "||" }); - atBoundary = true; - i += 2; - continue; - } - - if (operatorChars.has(ch)) { - tokens.push({ type: "op", value: ch as ";" | "|" | "&" }); - atBoundary = true; - i += 1; - continue; - } - - const parts: TokenWordPart[] = []; - let current = ""; - - const flushLit = () => { - if (current.length > 0) { - parts.push({ type: "lit", value: current }); - current = ""; - } - }; - - while (i < source.length) { - const currentChar = source.charAt(i); - - if (currentChar === "\\" && source.charAt(i + 1) === "\n") { - i += 2; - continue; - } - - if (currentChar === "\\" && source.charAt(i + 1) === "\r") { - if (source.charAt(i + 2) === "\n") { - i += 3; - continue; - } - } - - if ( - currentChar === " " || - currentChar === "\t" || - currentChar === "\r" || - currentChar === "\n" || - operatorChars.has(currentChar) || - redirChars.has(currentChar) || - symbolChars.has(currentChar) - ) { - break; - } - - if (currentChar === "'") { - flushLit(); - i += 1; - const start = i; - while (i < source.length && source.charAt(i) !== "'") { - i += 1; - } - if (i >= source.length) { - throw new Error("Unclosed single quote"); - } - parts.push({ type: "sgl", value: source.slice(start, i) }); - i += 1; - continue; - } - - if (currentChar === '"') { - flushLit(); - i += 1; - const dblParts: TokenWordPart[] = []; - let dblBuf = ""; - const flushDblLit = () => { - if (dblBuf.length > 0) { - dblParts.push({ type: "lit", value: dblBuf }); - dblBuf = ""; - } - }; - let closed = false; - while (i < source.length) { - const dqChar = source.charAt(i); - if (dqChar === "\\" && source.charAt(i + 1) === "\n") { - i += 2; - continue; - } - if (dqChar === "\\" && source.charAt(i + 1) === "\r") { - if (source.charAt(i + 2) === "\n") { - i += 3; - continue; - } - } - if (dqChar === "\\" && i + 1 < source.length) { - dblBuf += dqChar + source.charAt(i + 1); - i += 2; - continue; - } - if (dqChar === "$") { - flushDblLit(); - const exp = scanExpansion(source, i); - if (exp) { - dblParts.push(exp.part); - i = exp.end; - continue; - } - dblBuf += dqChar; - i += 1; - continue; - } - if (dqChar === "`") { - flushDblLit(); - const bt = scanBacktick(source, i); - dblParts.push(bt.part); - i = bt.end; - continue; - } - if (dqChar === '"') { - i += 1; - closed = true; - break; - } - dblBuf += dqChar; - i += 1; - } - if (!closed) { - throw new Error("Unclosed double quote"); - } - flushDblLit(); - parts.push({ type: "dbl", parts: dblParts }); - continue; - } - - if (currentChar === "$") { - flushLit(); - const exp = scanExpansion(source, i); - if (exp) { - parts.push(exp.part); - i = exp.end; - continue; - } - current += currentChar; - i += 1; - continue; - } - - if (currentChar === "`") { - flushLit(); - const bt = scanBacktick(source, i); - parts.push(bt.part); - i = bt.end; - continue; - } - - current += currentChar; - i += 1; - } - - flushLit(); - - if (parts.length === 0) { - throw new Error("Unexpected character"); - } - - tokens.push({ type: "word", parts }); - atBoundary = false; - } - - return tokens; -} - -function tokenPartsText(parts: TokenWordPart[]): string { - return parts - .map((p) => { - if (p.type === "lit") return p.value; - if (p.type === "sgl") return p.value; - if (p.type === "dbl") - return p.parts - .map((dp) => (dp.type === "lit" ? dp.value : "")) - .join(""); - return ""; - }) - .join(""); -} - -const DECL_KEYWORDS = new Set([ - "declare", - "local", - "export", - "readonly", - "typeset", - "nameref", -]); - -class Parser { - private index = 0; - private comments: CommentNode[] = []; - - constructor( - private readonly tokens: Token[], - private readonly options: ParseOptions = {}, - ) {} - - parseProgram(): Program { - const body: Statement[] = []; - this.skipSeparators(); - while (!this.isEof()) { - body.push(this.parseStatement()); - this.skipSeparators(); - } - const program: Program = { type: "Program", body }; - if (this.options.keepComments && this.comments.length > 0) { - program.comments = this.comments; - } - return program; - } - - assertEof() { - if (!this.isEof()) { - const token = this.peek(); - const display = token - ? token.type === "op" - ? token.value - : token.type === "redir" - ? token.op - : token.type === "symbol" - ? token.value - : token.type === "arith-cmd" - ? "(( ... ))" - : token.type === "heredoc-body" - ? "<>" - : token.type === "comment" - ? `#${token.text}` - : tokenPartsText(token.parts) - : ""; - throw new Error(`Unexpected token: ${display}`); - } - } - - private parseStatement(): Statement { - let negated = false; - if (this.matchOp("!")) { - this.consume(); - negated = true; - } - const command = this.parseLogical(); - let background = false; - if (this.matchOp("&")) { - this.consume(); - background = true; - } - const statement: Statement = { type: "Statement", command }; - if (background) { - statement.background = true; - } - if (negated) { - statement.negated = true; - } - return statement; - } - - private parseLogical(): Command { - let leftCommand = this.parsePipeline(); - while (this.matchOp("&&") || this.matchOp("||")) { - const opToken = this.consume(); - if (opToken.type !== "op") { - throw new Error("Expected logical operator"); - } - const rightCommand = this.parsePipeline(); - leftCommand = { - type: "Logical", - op: opToken.value === "&&" ? "and" : "or", - left: { type: "Statement", command: leftCommand }, - right: { type: "Statement", command: rightCommand }, - }; - } - return leftCommand; - } - - private parsePipeline(): Command { - const first = this.parseCommandAtom(); - if (!this.matchOp("|")) { - return first; - } - - const commands: Statement[] = [{ type: "Statement", command: first }]; - while (this.matchOp("|")) { - this.consume(); - const next = this.parseCommandAtom(); - commands.push({ type: "Statement", command: next }); - } - return { type: "Pipeline", commands }; - } - - private parseCommandAtom(): Command { - if (this.matchKeyword("if")) { - return this.parseIfClause(); - } - if (this.matchKeyword("while")) { - return this.parseWhileClause(false); - } - if (this.matchKeyword("until")) { - return this.parseWhileClause(true); - } - if (this.matchKeyword("for")) { - return this.parseForOrCStyleLoop(); - } - if (this.matchKeyword("select")) { - return this.parseSelectClause(); - } - if (this.matchKeyword("case")) { - return this.parseCaseClause(); - } - if (this.matchKeyword("time")) { - return this.parseTimeClause(); - } - if (this.matchKeyword("coproc")) { - return this.parseCoprocClause(); - } - if (this.matchKeyword("[[")) { - return this.parseTestClause(); - } - if (this.matchKeyword("function") || this.looksLikeFuncDecl()) { - return this.parseFunctionDecl(); - } - if (this.matchArithCmd()) { - return this.consumeArithCmd(); - } - if (this.matchSymbol("(")) { - return this.parseSubshell(); - } - if (this.matchSymbol("{")) { - return this.parseBlock(); - } - if (this.matchDeclKeyword()) { - return this.parseDeclClause(); - } - if (this.matchKeyword("let")) { - return this.parseLetClause(); - } - return this.parseSimpleCommand(); - } - - private parseSubshell(): Subshell { - this.consumeSymbol("("); - const body = this.parseStatementList(")"); - this.consumeSymbol(")"); - return { type: "Subshell", body }; - } - - private parseBlock(): Block { - this.consumeSymbol("{"); - const body = this.parseStatementList("}"); - this.consumeSymbol("}"); - return { type: "Block", body }; - } - - private parseStatementList(endSymbol: SymbolTokenValue): Statement[] { - const body: Statement[] = []; - this.skipSeparators(); - while (!this.matchSymbol(endSymbol)) { - if (this.isEof()) { - throw new Error( - `Unexpected end of input while looking for ${endSymbol}`, - ); - } - body.push(this.parseStatement()); - this.skipSeparators(); - } - return body; - } - - private parseIfClause(): IfClause { - this.consumeKeyword("if"); - const cond = this.parseStatementsUntilKeyword(["then"]); - this.consumeKeyword("then"); - const thenBranch = this.parseStatementsUntilKeyword(["else", "elif", "fi"]); - let elseBranch: Statement[] | undefined; - if (this.matchKeyword("elif")) { - elseBranch = [ - { - type: "Statement", - command: this.parseElifClause(), - }, - ]; - } else if (this.matchKeyword("else")) { - this.consumeKeyword("else"); - elseBranch = this.parseStatementsUntilKeyword(["fi"]); - } - this.consumeKeyword("fi"); - return elseBranch - ? { - type: "IfClause", - cond, - // biome-ignore lint/suspicious/noThenProperty: shell AST field - then: thenBranch, - else: elseBranch, - } - : { - type: "IfClause", - cond, - // biome-ignore lint/suspicious/noThenProperty: shell AST field - then: thenBranch, - }; - } - - private parseElifClause(): IfClause { - this.consumeKeyword("elif"); - const cond = this.parseStatementsUntilKeyword(["then"]); - this.consumeKeyword("then"); - const thenBranch = this.parseStatementsUntilKeyword(["else", "elif", "fi"]); - let elseBranch: Statement[] | undefined; - if (this.matchKeyword("elif")) { - elseBranch = [ - { - type: "Statement", - command: this.parseElifClause(), - }, - ]; - } else if (this.matchKeyword("else")) { - this.consumeKeyword("else"); - elseBranch = this.parseStatementsUntilKeyword(["fi"]); - } - return elseBranch - ? { - type: "IfClause", - cond, - // biome-ignore lint/suspicious/noThenProperty: shell AST field - then: thenBranch, - else: elseBranch, - } - : { - type: "IfClause", - cond, - // biome-ignore lint/suspicious/noThenProperty: shell AST field - then: thenBranch, - }; - } - - private parseWhileClause(until: boolean): WhileClause { - this.consumeKeyword(until ? "until" : "while"); - const cond = this.parseStatementsUntilKeyword(["do"]); - this.consumeKeyword("do"); - const body = this.parseStatementsUntilKeyword(["done"]); - this.consumeKeyword("done"); - return until - ? { type: "WhileClause", cond, body, until: true } - : { type: "WhileClause", cond, body }; - } - - private parseForOrCStyleLoop(): ForClause | CStyleLoop { - this.consumeKeyword("for"); - - // C-style: for (( init; cond; post )) - if (this.matchArithCmd()) { - return this.parseCStyleLoop(); - } - - const nameToken = this.consume(); - if (nameToken.type !== "word") { - throw new Error("Expected loop variable name"); - } - const name = tokenPartsText(nameToken.parts); - let items: Word[] | undefined; - if (this.matchKeyword("in")) { - this.consumeKeyword("in"); - const collected: Word[] = []; - while (this.matchWord() && !this.matchKeyword("do")) { - const itemToken = this.consume(); - if (itemToken.type !== "word") { - throw new Error("Expected loop item word"); - } - collected.push(this.wordFromParts(itemToken.parts)); - } - if (collected.length > 0) { - items = collected; - } - } - if (this.matchOp(";")) { - this.consume(); - } - this.skipSeparators(); - this.consumeKeyword("do"); - const body = this.parseStatementsUntilKeyword(["done"]); - this.consumeKeyword("done"); - return items - ? { type: "ForClause", name, items, body } - : { type: "ForClause", name, body }; - } - - private parseCStyleLoop(): CStyleLoop { - const token = this.consume(); - if (token.type !== "arith-cmd") { - throw new Error("Expected (( )) in c-style for"); - } - // Split expr on ";" into init, cond, post - const parts = token.expr.split(";").map((s) => s.trim()); - const init = parts[0] || undefined; - const cond = parts[1] || undefined; - const post = parts[2] || undefined; - - if (this.matchOp(";")) { - this.consume(); - } - this.skipSeparators(); - this.consumeKeyword("do"); - const body = this.parseStatementsUntilKeyword(["done"]); - this.consumeKeyword("done"); - const loop: CStyleLoop = { type: "CStyleLoop", body }; - if (init !== undefined) loop.init = init; - if (cond !== undefined) loop.cond = cond; - if (post !== undefined) loop.post = post; - return loop; - } - - private parseSelectClause(): SelectClause { - this.consumeKeyword("select"); - const nameToken = this.consume(); - if (nameToken.type !== "word") { - throw new Error("Expected select variable name"); - } - const name = tokenPartsText(nameToken.parts); - let items: Word[] | undefined; - if (this.matchKeyword("in")) { - this.consumeKeyword("in"); - const collected: Word[] = []; - while (this.matchWord() && !this.matchKeyword("do")) { - const itemToken = this.consume(); - if (itemToken.type !== "word") { - throw new Error("Expected select item word"); - } - collected.push(this.wordFromParts(itemToken.parts)); - } - if (collected.length > 0) { - items = collected; - } - } - if (this.matchOp(";")) { - this.consume(); - } - this.skipSeparators(); - this.consumeKeyword("do"); - const body = this.parseStatementsUntilKeyword(["done"]); - this.consumeKeyword("done"); - return items - ? { type: "SelectClause", name, items, body } - : { type: "SelectClause", name, body }; - } - - private parseFunctionDecl(): FunctionDecl { - if (this.matchKeyword("function")) { - this.consumeKeyword("function"); - } - const nameToken = this.consume(); - if (nameToken.type !== "word") { - throw new Error("Expected function name"); - } - const name = tokenPartsText(nameToken.parts); - if (this.matchSymbol("(")) { - this.consumeSymbol("("); - this.consumeSymbol(")"); - } - if (this.matchSymbol("{")) { - const body = this.parseBlock().body; - return { type: "FunctionDecl", name, body }; - } - throw new Error("Expected function body block"); - } - - private parseCaseClause(): CaseClause { - this.consumeKeyword("case"); - const wordToken = this.consume(); - if (wordToken.type !== "word") { - throw new Error("Expected case word"); - } - const word = this.wordFromParts(wordToken.parts); - this.consumeKeyword("in"); - const items: CaseItem[] = []; - this.skipSeparators(); - while (!this.matchKeyword("esac")) { - const patterns: Word[] = []; - while (!this.matchSymbol(")")) { - if (this.matchWord()) { - const patternToken = this.consume(); - if (patternToken.type !== "word") { - throw new Error("Expected case pattern"); - } - patterns.push(this.wordFromParts(patternToken.parts)); - continue; - } - if (this.matchOp("|")) { - this.consume(); - continue; - } - throw new Error("Expected case pattern or )"); - } - this.consumeSymbol(")"); - const body = this.parseCaseItemBody(); - items.push({ type: "CaseItem", patterns, body }); - if (this.matchOp(";") && this.peekOp(";")) { - this.consume(); - this.consume(); - } - this.skipSeparators(); - } - this.consumeKeyword("esac"); - return { type: "CaseClause", word, items }; - } - - private parseTimeClause(): TimeClause { - this.consumeKeyword("time"); - const command = this.parseStatement(); - return { type: "TimeClause", command }; - } - - private parseTestClause(): TestClause { - this.consumeKeyword("[["); - const words: Word[] = []; - while (!this.matchKeyword("]]")) { - if (this.isEof()) throw new Error("Unclosed [["); - const token = this.consume(); - if (token.type !== "word") throw new Error("Expected word in [[ ]]"); - words.push(this.wordFromParts(token.parts)); - } - this.consumeKeyword("]]"); - return { type: "TestClause", expr: words }; - } - - private matchArithCmd(): boolean { - return this.peek()?.type === "arith-cmd"; - } - - private consumeArithCmd(): ArithCmd { - const token = this.consume(); - if (token.type !== "arith-cmd") - throw new Error("Expected arithmetic command"); - return { type: "ArithCmd", expr: token.expr }; - } - - private parseCoprocClause(): CoprocClause { - this.consumeKeyword("coproc"); - if (this.matchWord() && this.peekToken(1)?.type === "symbol") { - const nameToken = this.peek(); - if ( - nameToken?.type === "word" && - this.peekToken(1)?.type === "symbol" && - (this.peekToken(1) as { value: string }).value === "{" - ) { - const name = tokenPartsText(nameToken.parts); - this.consume(); - const body = this.parseStatement(); - return { type: "CoprocClause", name, body }; - } - } - const body = this.parseStatement(); - return { type: "CoprocClause", body }; - } - - private parseCaseItemBody(): Statement[] { - const body: Statement[] = []; - this.skipCaseSeparators(); - while (!this.matchKeyword("esac") && !this.isCaseItemEnd()) { - body.push(this.parseStatement()); - if (this.isCaseItemEnd()) { - break; - } - this.skipCaseSeparators(); - } - return body; - } - - private isCaseItemEnd(): boolean { - return this.matchOp(";") && this.peekOp(";"); - } - - private parseStatementsUntilKeyword(endKeywords: string[]): Statement[] { - const body: Statement[] = []; - this.skipSeparators(); - while (!this.matchKeywordIn(endKeywords)) { - if (this.isEof()) { - throw new Error( - `Unexpected end of input while looking for ${endKeywords.join(", ")}`, - ); - } - body.push(this.parseStatement()); - this.skipSeparators(); - } - return body; - } - - private matchDeclKeyword(): boolean { - const token = this.peek(); - if (token?.type !== "word" || token.parts.length !== 1) return false; - const part = token.parts[0]; - return part?.type === "lit" && DECL_KEYWORDS.has(part.value); - } - - private parseDeclClause(): DeclClause { - const variantToken = this.consume(); - if (variantToken.type !== "word") { - throw new Error("Expected decl keyword"); - } - const variant = tokenPartsText(variantToken.parts) as DeclClause["variant"]; - - const args: Word[] = []; - const assigns: Assignment[] = []; - const redirects: Redirect[] = []; - - while (true) { - if (this.matchRedir()) { - const token = this.consume(); - if (token.type !== "redir") { - throw new Error("Expected redirect token"); - } - const targetToken = this.consume(); - if (targetToken.type !== "word") { - throw new Error("Redirect must be followed by a word"); - } - const target = this.wordFromParts(targetToken.parts); - const redir: Redirect = token.fd - ? { type: "Redirect", op: token.op, fd: token.fd, target } - : { type: "Redirect", op: token.op, target }; - redirects.push(redir); - continue; - } - - if (this.matchWord()) { - const token = this.peek(); - if (!token || token.type !== "word") break; - - // tryParseAssignment consumes tokens itself if it matches - const assignment = this.tryParseAssignment(token.parts); - if (assignment) { - assigns.push(assignment); - continue; - } - - // Otherwise it's a plain arg (flag or name) - this.consume(); - args.push(this.wordFromParts(token.parts)); - continue; - } - - break; - } - - const decl: DeclClause = { type: "DeclClause", variant }; - if (args.length > 0) decl.args = args; - if (assigns.length > 0) decl.assigns = assigns; - if (redirects.length > 0) decl.redirects = redirects; - return decl; - } - - private parseLetClause(): LetClause { - this.consumeKeyword("let"); - const exprs: Word[] = []; - const redirects: Redirect[] = []; - - while (true) { - if (this.matchRedir()) { - const token = this.consume(); - if (token.type !== "redir") { - throw new Error("Expected redirect token"); - } - const targetToken = this.consume(); - if (targetToken.type !== "word") { - throw new Error("Redirect must be followed by a word"); - } - const target = this.wordFromParts(targetToken.parts); - const redir: Redirect = token.fd - ? { type: "Redirect", op: token.op, fd: token.fd, target } - : { type: "Redirect", op: token.op, target }; - redirects.push(redir); - continue; - } - - if (this.matchWord()) { - const token = this.consume(); - if (token.type !== "word") break; - exprs.push(this.wordFromParts(token.parts)); - continue; - } - - break; - } - - if (exprs.length === 0) { - throw new Error("let requires at least one expression"); - } - - const clause: LetClause = { type: "LetClause", exprs }; - if (redirects.length > 0) clause.redirects = redirects; - return clause; - } - - private parseSimpleCommand(): SimpleCommand { - const words: Word[] = []; - const assignments: Assignment[] = []; - const redirects: Redirect[] = []; - let sawWord = false; - - while (true) { - if (this.matchWord()) { - const token = this.peek(); - if (!token || token.type !== "word") { - throw new Error("Expected word token"); - } - - // tryParseAssignment consumes tokens itself if it matches - if (!sawWord) { - const assignment = this.tryParseAssignment(token.parts); - if (assignment) { - assignments.push(assignment); - continue; - } - } - - this.consume(); - sawWord = true; - words.push(this.wordFromParts(token.parts)); - continue; - } - - if (this.matchRedir()) { - const token = this.consume(); - if (token.type !== "redir") { - throw new Error("Expected redirect token"); - } - const targetToken = this.consume(); - if (targetToken.type !== "word") { - throw new Error("Redirect must be followed by a word"); - } - const target = this.wordFromParts(targetToken.parts); - const redirect: Redirect = token.fd - ? { type: "Redirect", op: token.op, fd: token.fd, target } - : { type: "Redirect", op: token.op, target }; - // Collect heredoc body if this is a heredoc redirect - if (token.op === "<<" || token.op === "<<-") { - // Skip separators to find the heredoc-body token - this.skipSeparators(); - if (this.peek()?.type === "heredoc-body") { - const bodyToken = this.consume(); - if (bodyToken.type === "heredoc-body") { - redirect.heredoc = { - type: "Word", - parts: [{ type: "Literal", value: bodyToken.content }], - }; - } - } - } - redirects.push(redirect); - continue; - } - - break; - } - - if ( - words.length === 0 && - assignments.length === 0 && - redirects.length === 0 - ) { - throw new Error("Expected a command word"); - } - - const command: SimpleCommand = { type: "SimpleCommand" }; - if (words.length > 0) { - command.words = words; - } - if (assignments.length > 0) { - command.assignments = assignments; - } - if (redirects.length > 0) { - command.redirects = redirects; - } - return command; - } - - private convertWordPart(part: TokenWordPart): WordPart { - switch (part.type) { - case "lit": - return { type: "Literal", value: part.value }; - case "sgl": - return { type: "SglQuoted", value: part.value }; - case "dbl": - return { - type: "DblQuoted", - parts: part.parts.map((p) => this.convertWordPart(p)), - }; - case "param": { - const paramExp: ParamExp = { - type: "ParamExp", - short: !part.braced, - param: { type: "Literal", value: part.name }, - }; - if (part.op) { - paramExp.op = part.op; - } - if (part.value !== undefined) { - paramExp.value = { - type: "Word", - parts: [{ type: "Literal", value: part.value }], - }; - } - return paramExp; - } - case "cmd-subst": { - const innerTokens = tokenize(part.raw); - const innerParser = new Parser(innerTokens); - const prog = innerParser.parseProgram(); - return { type: "CmdSubst", stmts: prog.body }; - } - case "arith-exp": - return { type: "ArithExp", expr: part.raw }; - case "proc-subst": { - const innerTokens = tokenize(part.raw); - const innerParser = new Parser(innerTokens); - const prog = innerParser.parseProgram(); - return { type: "ProcSubst", op: part.op, stmts: prog.body }; - } - case "backtick": { - const innerTokens = tokenize(part.raw); - const innerParser = new Parser(innerTokens); - const prog = innerParser.parseProgram(); - return { type: "CmdSubst", stmts: prog.body }; - } - } - } - - private wordFromParts(parts: TokenWordPart[]): Word { - return { - type: "Word", - parts: parts.map((part) => this.convertWordPart(part)), - }; - } - - /** - * Try to parse an assignment from the current token's parts. - * If it returns an assignment, it has already consumed all relevant tokens - * (the word, and optionally the array `(...)` symbols). - * If it returns undefined, nothing was consumed. - */ - private tryParseAssignment(parts: TokenWordPart[]): Assignment | undefined { - if (parts.length !== 1) return undefined; - const part = parts[0]; - if (!part || part.type !== "lit") return undefined; - const raw = part.value; - - // Detect NAME= or NAME+= - let append = false; - let eqIndex = raw.indexOf("+="); - if (eqIndex > 0) { - append = true; - } else { - eqIndex = raw.indexOf("="); - } - if (eqIndex <= 0) return undefined; - - const name = raw.slice(0, eqIndex); - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) return undefined; - - const afterEq = raw.slice(eqIndex + (append ? 2 : 1)); - - // Check for array assignment: NAME=( ... ) or NAME+=( ... ) - const nextToken = this.peekToken(1); - if ( - afterEq === "" && - nextToken?.type === "symbol" && - (nextToken as { value: string }).value === "(" - ) { - this.consume(); // consume the NAME= word - return this.parseArrayAssignment(name, append); - } - - // Consume the word token - this.consume(); - - const assignment: Assignment = { type: "Assignment", name }; - if (append) assignment.append = true; - if (afterEq.length > 0) { - assignment.value = { - type: "Word", - parts: [{ type: "Literal", value: afterEq }], - }; - } - return assignment; - } - - private parseArrayAssignment(name: string, append: boolean): Assignment { - this.consumeSymbol("("); - const elems: ArrayElem[] = []; - - while (!this.matchSymbol(")")) { - if (this.isEof()) { - throw new Error("Unclosed array expression"); - } - if (this.matchOp(";")) { - this.consume(); - continue; - } - if (this.matchComment()) { - this.consumeComment(); - continue; - } - - const token = this.consume(); - if (token.type !== "word") { - throw new Error("Expected word in array expression"); - } - - const text = tokenPartsText(token.parts); - - // Check for [index]=value pattern - const indexMatch = text.match(/^\[([^\]]+)\]=(.*)$/); - if (indexMatch) { - const indexStr = indexMatch[1] as string; - const valStr = indexMatch[2] as string; - const elem: ArrayElem = { - type: "ArrayElem", - index: { - type: "Word", - parts: [{ type: "Literal", value: indexStr }], - }, - }; - if (valStr.length > 0) { - elem.value = { - type: "Word", - parts: [{ type: "Literal", value: valStr }], - }; - } - elems.push(elem); - } else { - elems.push({ - type: "ArrayElem", - value: this.wordFromParts(token.parts), - }); - } - } - - this.consumeSymbol(")"); - - const assignment: Assignment = { - type: "Assignment", - name, - array: { type: "ArrayExpr", elems }, - }; - if (append) assignment.append = true; - return assignment; - } - - private skipSeparators() { - while (this.matchOp(";") || this.matchComment()) { - if (this.matchComment()) { - this.consumeComment(); - } else { - this.consume(); - } - } - } - - private skipCaseSeparators() { - while (this.matchOp(";") && !this.peekOp(";")) { - this.consume(); - } - } - - private matchOp(value: OpTokenValue) { - const token = this.peek(); - return token?.type === "op" && token.value === value; - } - - private matchWord() { - return this.peek()?.type === "word"; - } - - private matchRedir() { - return this.peek()?.type === "redir"; - } - - private matchKeyword(value: string) { - const token = this.peek(); - if (token?.type !== "word" || token.parts.length !== 1) return false; - const part = token.parts[0]; - return part?.type === "lit" && part.value === value; - } - - private matchKeywordIn(values: string[]) { - return values.some((value) => this.matchKeyword(value)); - } - - private looksLikeFuncDecl(): boolean { - const name = this.peek(); - const next = this.peekToken(1); - const nextNext = this.peekToken(2); - const after = this.peekToken(3); - return ( - name?.type === "word" && - next?.type === "symbol" && - next.value === "(" && - nextNext?.type === "symbol" && - nextNext.value === ")" && - after?.type === "symbol" && - after.value === "{" - ); - } - - private matchSymbol(value: SymbolTokenValue) { - const token = this.peek(); - return token?.type === "symbol" && token.value === value; - } - - private consumeSymbol(value: SymbolTokenValue) { - const token = this.consume(); - if (token.type !== "symbol" || token.value !== value) { - throw new Error(`Expected symbol ${value}`); - } - } - - private consumeKeyword(value: string) { - const token = this.consume(); - if ( - token.type !== "word" || - token.parts.length !== 1 || - token.parts[0]?.type !== "lit" || - token.parts[0].value !== value - ) { - throw new Error(`Expected keyword ${value}`); - } - } - - private consume(): Token { - if (this.isEof()) { - throw new Error("Unexpected end of input"); - } - const token = this.tokens[this.index]; - if (!token) { - throw new Error("Unexpected end of input"); - } - this.index += 1; - return token; - } - - private peek(): Token | undefined { - return this.tokens[this.index]; - } - - private peekToken(offset: number): Token | undefined { - return this.tokens[this.index + offset]; - } - - private peekOp(value: OpTokenValue): boolean { - const token = this.peekToken(1); - return token?.type === "op" && token.value === value; - } - - private matchComment(): boolean { - return this.peek()?.type === "comment"; - } - - private consumeComment() { - const token = this.consume(); - if (token.type === "comment") { - this.comments.push({ type: "Comment", text: token.text }); - } - } - - private isEof() { - return this.index >= this.tokens.length; - } -} +export type { + ArithCmd, + ArithExp, + ArrayElem, + ArrayExpr, + Assignment, + Block, + CaseClause, + CaseItem, + CmdSubst, + Command, + CommentNode, + CoprocClause, + CStyleLoop, + DblQuoted, + DeclClause, + ForClause, + FunctionDecl, + IfClause, + LetClause, + Literal, + Logical, + ParamExp, + ParseOptions, + ParseResult, + Pipeline, + ProcSubst, + Program, + Redirect, + RedirOp, + SelectClause, + SglQuoted, + ShellDialect, + SimpleCommand, + Statement, + Subshell, + TestClause, + TimeClause, + WhileClause, + Word, + WordPart, +} from "./ast"; +export { parse } from "./parse"; diff --git a/src/parse.ts b/src/parse.ts new file mode 100644 index 0000000..bd0ec62 --- /dev/null +++ b/src/parse.ts @@ -0,0 +1,11 @@ +import type { ParseOptions, ParseResult } from "./ast"; +import { Parser } from "./parser"; +import { tokenize } from "./tokenizer"; + +export function parse(source: string, options: ParseOptions = {}): ParseResult { + const tokens = tokenize(source, options); + const parser = new Parser(tokens, options); + const ast = parser.parseProgram(); + parser.assertEof(); + return { ast }; +} diff --git a/src/parser.test.ts b/src/parser.test.ts deleted file mode 100644 index 62a117f..0000000 --- a/src/parser.test.ts +++ /dev/null @@ -1,1505 +0,0 @@ -import { assert, describe, expect, it } from "vitest"; -import { parse } from "./index.js"; - -type Literal = { type: "Literal"; value: string }; -type SglQuoted = { type: "SglQuoted"; value: string }; -type DblQuoted = { type: "DblQuoted"; parts: WordPart[] }; -type ParamExp = { - type: "ParamExp"; - short: boolean; - param: Literal; - op?: string; - value?: Word; -}; -type CmdSubst = { type: "CmdSubst"; stmts: Statement[] }; -type ArithExp = { type: "ArithExp"; expr: string }; -type ProcSubst = { type: "ProcSubst"; op: "<" | ">"; stmts: Statement[] }; -type WordPart = - | Literal - | SglQuoted - | DblQuoted - | ParamExp - | CmdSubst - | ArithExp - | ProcSubst; -type Word = { type: "Word"; parts: WordPart[] }; -type Assignment = { - type: "Assignment"; - name: string; - append?: boolean; - value?: Word; - array?: ArrayExpr; -}; -type RedirOp = - | ">" - | "<" - | ">>" - | ">|" - | ">&" - | "<&" - | "<>" - | "&>" - | "&>>" - | "<<<" - | "<<" - | "<<-"; -type Redirect = { - type: "Redirect"; - op: RedirOp; - fd?: string; - target: Word; - heredoc?: Word; -}; -type SimpleCommand = { - type: "SimpleCommand"; - words?: Word[]; - assignments?: Assignment[]; - redirects?: Redirect[]; -}; -type Subshell = { type: "Subshell"; body: Statement[] }; -type Block = { type: "Block"; body: Statement[] }; -type IfClause = { - type: "IfClause"; - cond: Statement[]; - then: Statement[]; - else?: Statement[]; -}; -type WhileClause = { - type: "WhileClause"; - cond: Statement[]; - body: Statement[]; - until?: boolean; -}; -type ForClause = { - type: "ForClause"; - name: string; - items?: Word[]; - body: Statement[]; -}; -type SelectClause = { - type: "SelectClause"; - name: string; - items?: Word[]; - body: Statement[]; -}; -type FunctionDecl = { type: "FunctionDecl"; name: string; body: Statement[] }; -type CaseItem = { type: "CaseItem"; patterns: Word[]; body: Statement[] }; -type CaseClause = { type: "CaseClause"; word: Word; items: CaseItem[] }; -type TimeClause = { type: "TimeClause"; command: Statement }; -type TestClause = { type: "TestClause"; expr: Word[] }; -type ArithCmd = { type: "ArithCmd"; expr: string }; -type CoprocClause = { - type: "CoprocClause"; - name?: string; - body: Statement; -}; -type Pipeline = { type: "Pipeline"; commands: Statement[] }; -type Logical = { - type: "Logical"; - op: "and" | "or"; - left: Statement; - right: Statement; -}; -type Statement = { - type: "Statement"; - command: Command; - background?: boolean; - negated?: boolean; -}; -type Program = { - type: "Program"; - body: Statement[]; - comments?: CommentNode[]; -}; -type Command = - | SimpleCommand - | Subshell - | Block - | IfClause - | WhileClause - | ForClause - | SelectClause - | FunctionDecl - | CaseClause - | TimeClause - | TestClause - | ArithCmd - | CoprocClause - | Pipeline - | Logical - | DeclClause - | LetClause - | CStyleLoop; - -type DeclClause = { - type: "DeclClause"; - variant: "declare" | "local" | "export" | "readonly" | "typeset" | "nameref"; - args?: Word[]; - assigns?: Assignment[]; - redirects?: Redirect[]; -}; -type LetClause = { type: "LetClause"; exprs: Word[]; redirects?: Redirect[] }; -type CStyleLoop = { - type: "CStyleLoop"; - init?: string; - cond?: string; - post?: string; - body: Statement[]; -}; -type ArrayExpr = { type: "ArrayExpr"; elems: ArrayElem[] }; -type ArrayElem = { type: "ArrayElem"; index?: Word; value?: Word }; -type CommentNode = { type: "Comment"; text: string }; - -const lit = (value: string): Literal => ({ type: "Literal", value }); -const sgl = (value: string): SglQuoted => ({ type: "SglQuoted", value }); -const dbl = (...parts: WordPart[]): DblQuoted => ({ - type: "DblQuoted", - parts, -}); -const paramExp = ( - name: string, - short = true, - op?: string, - value?: string, -): ParamExp => { - const p: ParamExp = { - type: "ParamExp", - short, - param: lit(name), - }; - if (op !== undefined) p.op = op; - if (value !== undefined) p.value = { type: "Word", parts: [lit(value)] }; - return p; -}; -const cmdSubst = (...stmts: Statement[]): CmdSubst => ({ - type: "CmdSubst", - stmts, -}); -const arithExp = (expr: string): ArithExp => ({ type: "ArithExp", expr }); -const word = (value: string): Word => ({ type: "Word", parts: [lit(value)] }); -const wordParts = (...parts: WordPart[]): Word => ({ - type: "Word", - parts, -}); -const simple = (...words: string[]): SimpleCommand => ({ - type: "SimpleCommand", - words: words.map(word), -}); -const assign = ( - name: string, - value?: string, - opts?: { append?: boolean; array?: ArrayExpr }, -): Assignment => { - const a: Assignment = { type: "Assignment", name }; - if (opts?.append) a.append = true; - if (value !== undefined) a.value = word(value); - if (opts?.array) a.array = opts.array; - return a; -}; -const arrayExpr = (...elems: ArrayElem[]): ArrayExpr => ({ - type: "ArrayExpr", - elems, -}); -const arrayElem = (value?: string, index?: string): ArrayElem => { - const e: ArrayElem = { type: "ArrayElem" }; - if (value !== undefined) e.value = word(value); - if (index !== undefined) e.index = word(index); - return e; -}; -const declClause = ( - variant: DeclClause["variant"], - opts?: { - args?: Word[]; - assigns?: Assignment[]; - redirects?: Redirect[]; - }, -): DeclClause => { - const d: DeclClause = { type: "DeclClause", variant }; - if (opts?.args) d.args = opts.args; - if (opts?.assigns) d.assigns = opts.assigns; - if (opts?.redirects) d.redirects = opts.redirects; - return d; -}; -const letClause = (exprs: Word[], redirects?: Redirect[]): LetClause => { - const l: LetClause = { type: "LetClause", exprs }; - if (redirects) l.redirects = redirects; - return l; -}; -const cStyleLoop = ( - body: Statement[], - init?: string, - cond?: string, - post?: string, -): CStyleLoop => { - const c: CStyleLoop = { type: "CStyleLoop", body }; - if (init !== undefined) c.init = init; - if (cond !== undefined) c.cond = cond; - if (post !== undefined) c.post = post; - return c; -}; -const redirect = (op: RedirOp, target: string, fd?: string): Redirect => - fd === undefined - ? { type: "Redirect", op, target: word(target) } - : { type: "Redirect", op, target: word(target), fd }; -const subshell = (...body: Statement[]): Subshell => ({ - type: "Subshell", - body, -}); -const block = (...body: Statement[]): Block => ({ - type: "Block", - body, -}); -const ifClause = ( - cond: Statement[], - then: Statement[], - elseBranch?: Statement[], -): IfClause => - elseBranch - ? { type: "IfClause", cond, then, else: elseBranch } - : { type: "IfClause", cond, then }; -const whileClause = ( - cond: Statement[], - body: Statement[], - until?: boolean, -): WhileClause => - until - ? { type: "WhileClause", cond, body, until } - : { type: "WhileClause", cond, body }; -const forClause = ( - name: string, - body: Statement[], - items?: Word[], -): ForClause => - items - ? { type: "ForClause", name, items, body } - : { type: "ForClause", name, body }; -const selectClause = ( - name: string, - body: Statement[], - items?: Word[], -): SelectClause => - items - ? { type: "SelectClause", name, items, body } - : { type: "SelectClause", name, body }; -const functionDecl = (name: string, body: Statement[]): FunctionDecl => ({ - type: "FunctionDecl", - name, - body, -}); -const caseItem = (patterns: Word[], body: Statement[]): CaseItem => ({ - type: "CaseItem", - patterns, - body, -}); -const caseClause = (wordValue: string, items: CaseItem[]): CaseClause => ({ - type: "CaseClause", - word: word(wordValue), - items, -}); -const testClause = (...words: Word[]): TestClause => ({ - type: "TestClause", - expr: words, -}); -const arithCmd = (expr: string): ArithCmd => ({ - type: "ArithCmd", - expr, -}); -const coprocClause = (body: Statement, name?: string): CoprocClause => - name ? { type: "CoprocClause", name, body } : { type: "CoprocClause", body }; -const timeClause = (command: Statement): TimeClause => ({ - type: "TimeClause", - command, -}); -const stmt = ( - command: Command, - background = false, - negated = false, -): Statement => { - const value: Statement = { type: "Statement", command }; - if (background) { - value.background = true; - } - if (negated) { - value.negated = true; - } - return value; -}; -const program = (...body: Statement[]): Program => ({ type: "Program", body }); - -// Tests derived from mvdan/sh syntax/filetests_test.go command parsing cases. -describe("parse (phase 1: simple commands)", () => { - it("parses empty input", () => { - expect(parse("")).toEqual({ ast: program() }); - }); - - it("parses a single simple command", () => { - expect(parse("foo")).toEqual({ ast: program(stmt(simple("foo"))) }); - }); - - it("parses multiple statements separated by newline or semicolon", () => { - expect(parse("foo\nbar")).toEqual({ - ast: program(stmt(simple("foo")), stmt(simple("bar"))), - }); - - expect(parse("foo; bar;")).toEqual({ - ast: program(stmt(simple("foo")), stmt(simple("bar"))), - }); - }); - - it("parses pipelines", () => { - expect(parse("foo | bar")).toEqual({ - ast: program( - stmt({ - type: "Pipeline", - commands: [stmt(simple("foo")), stmt(simple("bar"))], - }), - ), - }); - - expect(parse("foo | bar | baz")).toEqual({ - ast: program( - stmt({ - type: "Pipeline", - commands: [ - stmt(simple("foo")), - stmt(simple("bar")), - stmt(simple("baz")), - ], - }), - ), - }); - }); - - it("parses logical and/or", () => { - expect(parse("foo && bar")).toEqual({ - ast: program( - stmt({ - type: "Logical", - op: "and", - left: stmt(simple("foo")), - right: stmt(simple("bar")), - }), - ), - }); - - expect(parse("foo || bar")).toEqual({ - ast: program( - stmt({ - type: "Logical", - op: "or", - left: stmt(simple("foo")), - right: stmt(simple("bar")), - }), - ), - }); - - expect(parse("foo && bar || baz")).toEqual({ - ast: program( - stmt({ - type: "Logical", - op: "or", - left: stmt({ - type: "Logical", - op: "and", - left: stmt(simple("foo")), - right: stmt(simple("bar")), - }), - right: stmt(simple("baz")), - }), - ), - }); - }); - - it("gives pipelines higher precedence than logical ops", () => { - expect(parse("foo | bar || baz")).toEqual({ - ast: program( - stmt({ - type: "Logical", - op: "or", - left: stmt({ - type: "Pipeline", - commands: [stmt(simple("foo")), stmt(simple("bar"))], - }), - right: stmt(simple("baz")), - }), - ), - }); - }); - - it("parses background commands", () => { - expect(parse("foo &\nbar")).toEqual({ - ast: program(stmt(simple("foo"), true), stmt(simple("bar"))), - }); - }); -}); - -describe("parse (phase 2: words, quotes, comments)", () => { - it("ignores full-line and trailing comments", () => { - expect(parse("# foo\nbar")).toEqual({ - ast: program(stmt(simple("bar"))), - }); - - expect(parse("foo # bar")).toEqual({ - ast: program(stmt(simple("foo"))), - }); - }); - - it("keeps # when not at a boundary", () => { - expect(parse("foo#bar")).toEqual({ - ast: program(stmt(simple("foo#bar"))), - }); - }); - - it("parses single-quoted parts", () => { - expect(parse("foo'bar'")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [wordParts(lit("foo"), sgl("bar"))], - }), - ), - }); - }); - - it("parses double-quoted parts", () => { - expect(parse('"foo bar"')).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [wordParts(dbl(lit("foo bar")))], - }), - ), - }); - }); - - it("treats backslash-newline as whitespace", () => { - expect(parse("foo \\\n bar")).toEqual({ - ast: program(stmt(simple("foo", "bar"))), - }); - }); -}); - -describe("parse (phase 3: assignments and redirects)", () => { - it("parses assignment-only commands", () => { - expect(parse("a=b")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [assign("a", "b")], - }), - ), - }); - }); - - it("parses assignments before words", () => { - expect(parse("a=b foo")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [assign("a", "b")], - words: [word("foo")], - }), - ), - }); - }); - - it("parses redirects", () => { - expect(parse("foo >out")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("foo")], - redirects: [redirect(">", "out")], - }), - ), - }); - - expect(parse(">out foo")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("foo")], - redirects: [redirect(">", "out")], - }), - ), - }); - }); - - it("parses redirects with file descriptors", () => { - expect(parse("foo 2>out")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("foo")], - redirects: [redirect(">", "out", "2")], - }), - ), - }); - }); -}); - -describe("parse (phase 4: subshells and blocks)", () => { - it("parses subshells", () => { - expect(parse("(foo)")).toEqual({ - ast: program(stmt(subshell(stmt(simple("foo"))))), - }); - - expect(parse("(foo; bar)")).toEqual({ - ast: program(stmt(subshell(stmt(simple("foo")), stmt(simple("bar"))))), - }); - }); - - it("parses blocks", () => { - expect(parse("{ foo; }")).toEqual({ - ast: program(stmt(block(stmt(simple("foo"))))), - }); - }); -}); - -describe("parse (phase 5: if clauses)", () => { - it("parses if/then/fi", () => { - expect(parse("if a; then b; fi")).toEqual({ - ast: program(stmt(ifClause([stmt(simple("a"))], [stmt(simple("b"))]))), - }); - }); - - it("parses if/then/else/fi", () => { - expect(parse("if a; then b; else c; fi")).toEqual({ - ast: program( - stmt( - ifClause( - [stmt(simple("a"))], - [stmt(simple("b"))], - [stmt(simple("c"))], - ), - ), - ), - }); - }); - - it("parses if/elif/then/fi", () => { - expect(parse("if a; then b; elif c; then d; fi")).toEqual({ - ast: program( - stmt( - ifClause( - [stmt(simple("a"))], - [stmt(simple("b"))], - [stmt(ifClause([stmt(simple("c"))], [stmt(simple("d"))]))], - ), - ), - ), - }); - }); -}); - -describe("parse (phase 6: while/until clauses)", () => { - it("parses while/do/done", () => { - expect(parse("while a; do b; done")).toEqual({ - ast: program(stmt(whileClause([stmt(simple("a"))], [stmt(simple("b"))]))), - }); - }); - - it("parses until/do/done", () => { - expect(parse("until a; do b; done")).toEqual({ - ast: program( - stmt(whileClause([stmt(simple("a"))], [stmt(simple("b"))], true)), - ), - }); - }); -}); - -describe("parse (phase 7: for clauses)", () => { - it("parses for-in loops", () => { - expect(parse("for i in a b; do c; done")).toEqual({ - ast: program( - stmt(forClause("i", [stmt(simple("c"))], [word("a"), word("b")])), - ), - }); - }); - - it("parses for loops without in list", () => { - expect(parse("for i; do c; done")).toEqual({ - ast: program(stmt(forClause("i", [stmt(simple("c"))]))), - }); - }); -}); - -describe("parse (phase 8: select clauses)", () => { - it("parses select loops", () => { - expect(parse("select i in a b; do c; done")).toEqual({ - ast: program( - stmt(selectClause("i", [stmt(simple("c"))], [word("a"), word("b")])), - ), - }); - }); -}); - -describe("parse (phase 9: functions and case)", () => { - it("parses function declarations", () => { - expect(parse("foo() { bar; }")).toEqual({ - ast: program(stmt(functionDecl("foo", [stmt(simple("bar"))]))), - }); - - expect(parse("function foo { bar; }")).toEqual({ - ast: program(stmt(functionDecl("foo", [stmt(simple("bar"))]))), - }); - }); - - it("parses case clauses", () => { - expect(parse("case x in y) z ;; esac")).toEqual({ - ast: program( - stmt(caseClause("x", [caseItem([word("y")], [stmt(simple("z"))])])), - ), - }); - - expect(parse("case x in a|b) z ;; esac")).toEqual({ - ast: program( - stmt( - caseClause("x", [ - caseItem([word("a"), word("b")], [stmt(simple("z"))]), - ]), - ), - ), - }); - - expect(parse("case x in a) y ;; b) z ;; esac")).toEqual({ - ast: program( - stmt( - caseClause("x", [ - caseItem([word("a")], [stmt(simple("y"))]), - caseItem([word("b")], [stmt(simple("z"))]), - ]), - ), - ), - }); - }); -}); - -describe("parse (phase 10: negation)", () => { - it("parses negated commands", () => { - expect(parse("! foo")).toEqual({ - ast: program(stmt(simple("foo"), false, true)), - }); - }); -}); - -describe("parse (phase 11: time)", () => { - it("parses time clauses", () => { - expect(parse("time foo")).toEqual({ - ast: program(stmt(timeClause(stmt(simple("foo"))))), - }); - }); -}); - -describe("parse (phase 12: extended redirects)", () => { - it.each([ - { input: "foo >| bar", op: ">|" as RedirOp, target: "bar" }, - { input: "foo >&2", op: ">&" as RedirOp, target: "2" }, - { input: "foo <&3", op: "<&" as RedirOp, target: "3" }, - { input: "foo &>bar", op: "&>" as RedirOp, target: "bar" }, - { input: "foo &>>bar", op: "&>>" as RedirOp, target: "bar" }, - { input: "foo <<bar", op: "<>" as RedirOp, target: "bar" }, - ])("parses $op redirect", ({ input, op, target }) => { - expect(parse(input)).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("foo")], - redirects: [redirect(op, target)], - }), - ), - }); - }); - - it("parses fd dup with explicit fd 2>&1", () => { - expect(parse("foo 2>&1")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("foo")], - redirects: [redirect(">&", "1", "2")], - }), - ), - }); - }); -}); - -describe("parse (phase 13: extended test)", () => { - it("parses [[ ]]", () => { - expect(parse("[[ -f foo ]]")).toEqual({ - ast: program(stmt(testClause(word("-f"), word("foo")))), - }); - }); - - it("parses [[ with binary op ]]", () => { - expect(parse("[[ foo == bar ]]")).toEqual({ - ast: program(stmt(testClause(word("foo"), word("=="), word("bar")))), - }); - }); -}); - -describe("parse (phase 14: arithmetic command)", () => { - it("parses (( ))", () => { - expect(parse("(( x + 1 ))")).toEqual({ - ast: program(stmt(arithCmd("x + 1"))), - }); - }); - - it("parses nested parens in (( ))", () => { - expect(parse("(( (x + 1) * 2 ))")).toEqual({ - ast: program(stmt(arithCmd("(x + 1) * 2"))), - }); - }); -}); - -describe("parse (phase 15: parameter expansion)", () => { - it("parses $var", () => { - expect(parse("echo $foo")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(paramExp("foo"))], - }), - ), - }); - }); - - it("parses ${var}", () => { - expect(parse("echo ${foo}")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(paramExp("foo", false))], - }), - ), - }); - }); - - it("parses ${var:-default}", () => { - expect(parse("echo ${foo:-bar}")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(paramExp("foo", false, ":-", "bar"))], - }), - ), - }); - }); - - it("parses $var inside double quotes", () => { - expect(parse('"hello $name"')).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [wordParts(dbl(lit("hello "), paramExp("name")))], - }), - ), - }); - }); - - it("parses special params $? $# $@", () => { - expect(parse("echo $?")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(paramExp("?"))], - }), - ), - }); - }); -}); - -describe("parse (phase 16: command substitution)", () => { - it("parses $(cmd)", () => { - expect(parse("echo $(foo)")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(cmdSubst(stmt(simple("foo"))))], - }), - ), - }); - }); - - it("parses backtick substitution", () => { - expect(parse("echo `foo`")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(cmdSubst(stmt(simple("foo"))))], - }), - ), - }); - }); - - it("parses $(cmd) inside double quotes", () => { - expect(parse('"$(foo)"')).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [wordParts(dbl(cmdSubst(stmt(simple("foo")))))], - }), - ), - }); - }); -}); - -describe("parse (phase 17: arithmetic expansion)", () => { - it("parses $((expr))", () => { - expect(parse("echo $((1 + 2))")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), wordParts(arithExp("1 + 2"))], - }), - ), - }); - }); -}); - -describe("parse (phase 18: heredoc)", () => { - it("parses << heredoc", () => { - expect(parse("cat < { - expect(parse("cat <<-EOF\n\thello\n\tEOF")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("cat")], - redirects: [ - { - type: "Redirect", - op: "<<-", - target: word("EOF"), - heredoc: word("hello\n"), - }, - ], - }), - ), - }); - }); -}); - -describe("parse (phase 19: process substitution)", () => { - it("parses <() process substitution", () => { - expect(parse("diff <(foo) <(bar)")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [ - word("diff"), - wordParts({ - type: "ProcSubst", - op: "<", - stmts: [stmt(simple("foo"))], - }), - wordParts({ - type: "ProcSubst", - op: "<", - stmts: [stmt(simple("bar"))], - }), - ], - }), - ), - }); - }); -}); - -describe("parse (phase 20: mixed expansions)", () => { - it("parses mixed literal and expansion in word", () => { - expect(parse("echo foo${bar}baz")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [ - word("echo"), - wordParts(lit("foo"), paramExp("bar", false), lit("baz")), - ], - }), - ), - }); - }); - - it("parses nested command substitution", () => { - expect(parse("echo $(echo $(foo))")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [ - word("echo"), - wordParts( - cmdSubst( - stmt({ - type: "SimpleCommand", - words: [ - word("echo"), - wordParts(cmdSubst(stmt(simple("foo")))), - ], - }), - ), - ), - ], - }), - ), - }); - }); - - it("parses function with parens in function keyword form", () => { - expect(parse("function foo() { bar; }")).toEqual({ - ast: program(stmt(functionDecl("foo", [stmt(simple("bar"))]))), - }); - }); - - it("parses if/elif/else/fi chain", () => { - expect(parse("if a; then b; elif c; then d; else e; fi")).toEqual({ - ast: program( - stmt( - ifClause( - [stmt(simple("a"))], - [stmt(simple("b"))], - [ - stmt( - ifClause( - [stmt(simple("c"))], - [stmt(simple("d"))], - [stmt(simple("e"))], - ), - ), - ], - ), - ), - ), - }); - }); -}); - -describe("parse (phase 21: coproc)", () => { - it("parses coproc with command", () => { - expect(parse("coproc foo")).toEqual({ - ast: program(stmt(coprocClause(stmt(simple("foo"))))), - }); - }); - - it("parses coproc with name and block", () => { - expect(parse("coproc NAME { foo; }")).toEqual({ - ast: program( - stmt(coprocClause(stmt(block(stmt(simple("foo")))), "NAME")), - ), - }); - }); -}); - -describe("parse (phase 22: decl clause)", () => { - it.each([ - { - input: "export FOO=bar", - variant: "export" as const, - assigns: [assign("FOO", "bar")], - }, - { - input: "local x=1", - variant: "local" as const, - assigns: [assign("x", "1")], - }, - { - input: "nameref ref=target", - variant: "nameref" as const, - assigns: [assign("ref", "target")], - }, - ])("parses $variant with assignment", ({ - input, - variant, - assigns: assignList, - }) => { - expect(parse(input)).toEqual({ - ast: program(stmt(declClause(variant, { assigns: assignList }))), - }); - }); - - it("parses export with multiple names", () => { - expect(parse("export FOO BAR")).toEqual({ - ast: program( - stmt(declClause("export", { args: [word("FOO"), word("BAR")] })), - ), - }); - }); - - it.each([ - { - input: "declare -r FOO=bar", - variant: "declare" as const, - flag: "-r", - assignVal: assign("FOO", "bar"), - }, - { - input: "typeset -i count=0", - variant: "typeset" as const, - flag: "-i", - assignVal: assign("count", "0"), - }, - ])("parses $variant with flag and assignment", ({ - input, - variant, - flag, - assignVal, - }) => { - expect(parse(input)).toEqual({ - ast: program( - stmt( - declClause(variant, { - args: [word(flag)], - assigns: [assignVal], - }), - ), - ), - }); - }); - - it("parses readonly with names", () => { - expect(parse("readonly X Y")).toEqual({ - ast: program( - stmt(declClause("readonly", { args: [word("X"), word("Y")] })), - ), - }); - }); - - it("parses export with redirect", () => { - expect(parse("export FOO=bar 2>/dev/null")).toEqual({ - ast: program( - stmt( - declClause("export", { - assigns: [assign("FOO", "bar")], - redirects: [redirect(">", "/dev/null", "2")], - }), - ), - ), - }); - }); - - it("parses declare -a with array", () => { - expect(parse("declare -a arr=(a b c)")).toEqual({ - ast: program( - stmt( - declClause("declare", { - args: [word("-a")], - assigns: [ - assign("arr", undefined, { - array: arrayExpr( - arrayElem("a"), - arrayElem("b"), - arrayElem("c"), - ), - }), - ], - }), - ), - ), - }); - }); -}); - -describe("parse (phase 23: append assignment)", () => { - it("parses append assignment", () => { - expect(parse("PATH+=/usr/local/bin echo hi")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - words: [word("echo"), word("hi")], - assignments: [assign("PATH", "/usr/local/bin", { append: true })], - }), - ), - }); - }); - - it("parses standalone append assignment", () => { - expect(parse("arr+=value")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [assign("arr", "value", { append: true })], - }), - ), - }); - }); -}); - -describe("parse (phase 24: array expressions)", () => { - it("parses simple array assignment", () => { - expect(parse("arr=(a b c)")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [ - assign("arr", undefined, { - array: arrayExpr(arrayElem("a"), arrayElem("b"), arrayElem("c")), - }), - ], - }), - ), - }); - }); - - it("parses indexed array assignment", () => { - expect(parse("arr=([0]=x [1]=y)")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [ - assign("arr", undefined, { - array: arrayExpr(arrayElem("x", "0"), arrayElem("y", "1")), - }), - ], - }), - ), - }); - }); - - it("parses associative array assignment", () => { - expect(parse("arr=([key]=val [other]=thing)")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [ - assign("arr", undefined, { - array: arrayExpr( - arrayElem("val", "key"), - arrayElem("thing", "other"), - ), - }), - ], - }), - ), - }); - }); - - it("parses empty array", () => { - expect(parse("arr=()")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [assign("arr", undefined, { array: arrayExpr() })], - }), - ), - }); - }); - - it("parses append array", () => { - expect(parse("arr+=(x y)")).toEqual({ - ast: program( - stmt({ - type: "SimpleCommand", - assignments: [ - assign("arr", undefined, { - append: true, - array: arrayExpr(arrayElem("x"), arrayElem("y")), - }), - ], - }), - ), - }); - }); -}); - -describe("parse (phase 25: c-style for loop)", () => { - it("parses c-style for loop", () => { - const result = parse("for ((i=0; i<10; i++)); do echo $i; done"); - const first = result.ast.body[0]; - assert(first, "expected at least one statement"); - const command = first.command; - assert(command.type === "CStyleLoop", "expected CStyleLoop"); - expect(command.init).toBe("i=0"); - expect(command.cond).toBe("i<10"); - expect(command.post).toBe("i++"); - expect(command.body).toHaveLength(1); - }); - - it("parses c-style for loop with empty parts", () => { - expect(parse("for ((;;)); do echo loop; done")).toEqual({ - ast: program(stmt(cStyleLoop([stmt(simple("echo", "loop"))]))), - }); - }); - - it("parses c-style for loop with only condition", () => { - const result = parse("for (( ; i<5; )); do echo $i; done"); - const first = result.ast.body[0]; - assert(first, "expected at least one statement"); - const command = first.command; - assert(command.type === "CStyleLoop", "expected CStyleLoop"); - expect(command.init).toBeUndefined(); - expect(command.cond).toBe("i<5"); - expect(command.post).toBeUndefined(); - expect(command.body).toHaveLength(1); - }); -}); - -describe("parse (phase 26: let clause)", () => { - it("parses let with single expression", () => { - expect(parse("let i++")).toEqual({ - ast: program(stmt(letClause([word("i++")]))), - }); - }); - - it("parses let with multiple expressions", () => { - expect(parse("let i++ j=2")).toEqual({ - ast: program(stmt(letClause([word("i++"), word("j=2")]))), - }); - }); - - it("parses let with redirect", () => { - expect(parse("let x=1 2>/dev/null")).toEqual({ - ast: program( - stmt(letClause([word("x=1")], [redirect(">", "/dev/null", "2")])), - ), - }); - }); -}); - -describe("parse (phase 27: comments)", () => { - it("does not include comments by default", () => { - const result = parse("echo hi # a comment"); - expect(result.ast.comments).toBeUndefined(); - }); - - it("collects comments when keepComments is true", () => { - const result = parse("echo hi # a comment", { keepComments: true }); - expect(result.ast.comments).toEqual([ - { type: "Comment", text: " a comment" }, - ]); - }); - - it("collects multiple comments", () => { - const result = parse("# first\necho hi\n# second", { - keepComments: true, - }); - expect(result.ast.comments).toEqual([ - { type: "Comment", text: " first" }, - { type: "Comment", text: " second" }, - ]); - }); - - it("collects inline comment after semicolon", () => { - const result = parse("echo hi; # trailing", { keepComments: true }); - expect(result.ast.comments).toEqual([ - { type: "Comment", text: " trailing" }, - ]); - }); -}); - -// Real-world guardrail validation tests. -// The guardrails extension blocks commands matching /\bnpm\b/ on the full -// command string. This causes false positives when "npm" appears in arguments, -// grep patterns, heredocs, or subcommand strings rather than as the actual -// command being invoked. -// -// These tests validate that the parser produces an AST where the actual -// command name (first word of a SimpleCommand) is distinguishable from -// arguments, allowing a smarter guardrail to only check command positions. -describe("guardrail validation: package manager enforcement", () => { - // Helper: extract all command names (first word of each SimpleCommand) from - // a parsed program, recursively walking the AST. - function extractCommandNames(node: unknown): string[] { - if (!node || typeof node !== "object") return []; - const n = node as Record; - const names: string[] = []; - - if ( - n.type === "SimpleCommand" && - Array.isArray(n.words) && - n.words.length > 0 - ) { - const firstWord = n.words[0] as - | { parts: Array<{ type: string; value?: string }> } - | undefined; - // Only extract if first word is a plain literal (no expansions) - if ( - firstWord && - firstWord.parts.length === 1 && - firstWord.parts[0]?.type === "Literal" - ) { - names.push(firstWord.parts[0].value as string); - } - } - - for (const val of Object.values(n)) { - if (Array.isArray(val)) { - for (const item of val) { - names.push(...extractCommandNames(item)); - } - } else if (val && typeof val === "object") { - names.push(...extractCommandNames(val)); - } - } - return names; - } - - // Tests where npm appears in arguments/patterns but is NOT a command. - it.each([ - { - desc: "grep for npm pattern", - input: String.raw`grep -rn '\bnpm\b' src/`, - expected: ["grep"], - }, - { - desc: "grep with multiple npm/npx patterns", - input: String.raw`grep -rn '\bnpx\b\|\bnpm \b\|\bnpm$' AGENTS.md`, - expected: ["grep"], - }, - { - desc: "echo containing npm", - input: 'echo "use npm install instead"', - expected: ["echo"], - }, - { - desc: "cat of package.json", - input: "cat /path/to/package.json", - expected: ["cat"], - }, - { - desc: "npx is not npm", - input: "npx wrangler --version", - expected: ["npx"], - }, - { - desc: "pnpm is not npm", - input: "pnpm --filter pi-relay-server typecheck", - expected: ["pnpm"], - }, - { - desc: "which npm is not running npm", - input: "which npm 2>/dev/null || echo 'npm not found'", - expected: ["which", "echo"], - }, - { - desc: "if condition with npm check", - input: "if command -v npm; then echo found; fi", - expected: ["command", "echo"], - }, - { - desc: "variable assignment containing npm", - input: 'PKG_MGR=npm echo "using $PKG_MGR"', - expected: ["echo"], - }, - { - desc: "herestring containing npm", - input: "grep -c npm <<< 'npm install pnpm bun'", - expected: ["grep"], - }, - { - desc: "pipeline with npm in args", - input: "find . -name '*.json' | grep npm | head -5", - expected: ["find", "grep", "head"], - }, - { - desc: "biome check via pnpm exec", - input: "pnpm exec biome check --write src/sandbox/cloudflare.test.ts", - expected: ["pnpm"], - }, - { - desc: "curl piped to jq", - input: "curl -s http://localhost:31415/health | jq .", - expected: ["curl", "jq"], - }, - ])("$desc: npm is not extracted as a command", ({ input, expected }) => { - const cmds = extractCommandNames(parse(input).ast); - expect(cmds).toEqual(expected); - expect(cmds).not.toContain("npm"); - }); - - // Tests where npm IS a real command. - it.each([ - { desc: "npm install", input: "npm install --omit=dev" }, - { desc: "npm ci", input: "npm ci" }, - { desc: "npm in command substitution", input: "echo $(npm pack)" }, - { desc: "npm in subshell", input: "(cd /tmp && npm install)" }, - ])("$desc: npm is extracted as a command", ({ input }) => { - const cmds = extractCommandNames(parse(input).ast); - expect(cmds).toContain("npm"); - }); - - it("cd && pnpm: both commands identified correctly", () => { - const cmds = extractCommandNames( - parse("cd /project && pnpm --filter pi-relay-server test").ast, - ); - expect(cmds).toEqual(["cd", "pnpm"]); - expect(cmds).not.toContain("npm"); - }); - - it("npm in || fallback: both branches identified", () => { - const cmds = extractCommandNames( - parse("npm ci --omit=dev 2>/dev/null || npm install --omit=dev").ast, - ); - expect(cmds).toEqual(["npm", "npm"]); - }); - - it("heredoc containing npm install is not an npm command", () => { - const input = `cat <<'EOF' -RUN npm install --omit=dev -npm ci -EOF`; - const cmds = extractCommandNames(parse(input).ast); - expect(cmds).toEqual(["cat"]); - expect(cmds).not.toContain("npm"); - }); - - it("docker build with heredoc Dockerfile containing npm", () => { - const input = `docker build -t myimage -f - . <<'DOCKERFILE' -FROM node:22-slim -RUN npm install --omit=dev -DOCKERFILE`; - const cmds = extractCommandNames(parse(input).ast); - expect(cmds).toEqual(["docker"]); - expect(cmds).not.toContain("npm"); - }); - - it("real session: cd && pnpm filter test piped to tail", () => { - const cmds = extractCommandNames( - parse("cd /project && pnpm --filter pi-relay-server test 2>&1 | tail -15") - .ast, - ); - expect(cmds).toContain("cd"); - expect(cmds).toContain("pnpm"); - expect(cmds).toContain("tail"); - expect(cmds).not.toContain("npm"); - }); - - it("real session: write Dockerfile via heredoc then docker build", () => { - const input = `mkdir -p /tmp/cf-sandbox-test && \\ -cp bridge.js /tmp/cf-sandbox-test/ && \\ -cat > /tmp/cf-sandbox-test/Dockerfile <<'DOCKERFILE' -FROM node:22-slim -RUN apt-get update && apt-get install -y curl tar bash git -WORKDIR /bridge -COPY package.json ./ -RUN npm install --omit=dev 2>/dev/null || true -COPY bridge.js ./ -CMD ["node", "/bridge/bridge.js"] -DOCKERFILE - -docker build --platform linux/arm64 -t pi-sandbox-cf:arm64-debian /tmp/cf-sandbox-test`; - const { ast } = parse(input); - const cmds = extractCommandNames(ast); - expect(cmds).toContain("mkdir"); - expect(cmds).toContain("cp"); - expect(cmds).toContain("cat"); - expect(cmds).toContain("docker"); - expect(cmds).not.toContain("npm"); - }); -}); diff --git a/src/parser/basic.test.ts b/src/parser/basic.test.ts new file mode 100644 index 0000000..e45970c --- /dev/null +++ b/src/parser/basic.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { program, simple, stmt } from "../test-helpers/ast-builders"; + +describe("parse (phase 1: simple commands)", () => { + it("parses empty input", () => { + expect(parse("")).toEqual({ ast: program() }); + }); + + it("parses a single simple command", () => { + expect(parse("foo")).toEqual({ ast: program(stmt(simple("foo"))) }); + }); + + it("parses multiple statements separated by newline or semicolon", () => { + expect(parse("foo\nbar")).toEqual({ + ast: program(stmt(simple("foo")), stmt(simple("bar"))), + }); + + expect(parse("foo; bar;")).toEqual({ + ast: program(stmt(simple("foo")), stmt(simple("bar"))), + }); + }); + + it("parses pipelines", () => { + expect(parse("foo | bar")).toEqual({ + ast: program( + stmt({ + type: "Pipeline", + commands: [stmt(simple("foo")), stmt(simple("bar"))], + }), + ), + }); + + expect(parse("foo | bar | baz")).toEqual({ + ast: program( + stmt({ + type: "Pipeline", + commands: [ + stmt(simple("foo")), + stmt(simple("bar")), + stmt(simple("baz")), + ], + }), + ), + }); + }); + + it("parses logical and/or", () => { + expect(parse("foo && bar")).toEqual({ + ast: program( + stmt({ + type: "Logical", + op: "and", + left: stmt(simple("foo")), + right: stmt(simple("bar")), + }), + ), + }); + + expect(parse("foo || bar")).toEqual({ + ast: program( + stmt({ + type: "Logical", + op: "or", + left: stmt(simple("foo")), + right: stmt(simple("bar")), + }), + ), + }); + + expect(parse("foo && bar || baz")).toEqual({ + ast: program( + stmt({ + type: "Logical", + op: "or", + left: stmt({ + type: "Logical", + op: "and", + left: stmt(simple("foo")), + right: stmt(simple("bar")), + }), + right: stmt(simple("baz")), + }), + ), + }); + }); + + it("gives pipelines higher precedence than logical ops", () => { + expect(parse("foo | bar || baz")).toEqual({ + ast: program( + stmt({ + type: "Logical", + op: "or", + left: stmt({ + type: "Pipeline", + commands: [stmt(simple("foo")), stmt(simple("bar"))], + }), + right: stmt(simple("baz")), + }), + ), + }); + }); + + it("parses background commands", () => { + expect(parse("foo &\nbar")).toEqual({ + ast: program(stmt(simple("foo"), true), stmt(simple("bar"))), + }); + }); +}); + +describe("parse (phase 10: negation)", () => { + it("parses negated commands", () => { + expect(parse("! foo")).toEqual({ + ast: program(stmt(simple("foo"), false, true)), + }); + }); +}); diff --git a/src/parser/clauses.test.ts b/src/parser/clauses.test.ts new file mode 100644 index 0000000..8ecf440 --- /dev/null +++ b/src/parser/clauses.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { + arithCmd, + block, + caseClause, + caseItem, + coprocClause, + forClause, + functionDecl, + ifClause, + program, + selectClause, + simple, + stmt, + subshell, + testClause, + timeClause, + whileClause, + word, +} from "../test-helpers/ast-builders"; + +describe("parse (phase 4: subshells and blocks)", () => { + it("parses subshells", () => { + expect(parse("(foo)")).toEqual({ + ast: program(stmt(subshell(stmt(simple("foo"))))), + }); + + expect(parse("(foo; bar)")).toEqual({ + ast: program(stmt(subshell(stmt(simple("foo")), stmt(simple("bar"))))), + }); + }); + + it("parses blocks", () => { + expect(parse("{ foo; }")).toEqual({ + ast: program(stmt(block(stmt(simple("foo"))))), + }); + }); +}); + +describe("parse (phase 5: if clauses)", () => { + it("parses if/then/fi", () => { + expect(parse("if a; then b; fi")).toEqual({ + ast: program(stmt(ifClause([stmt(simple("a"))], [stmt(simple("b"))]))), + }); + }); + + it("parses if/then/else/fi", () => { + expect(parse("if a; then b; else c; fi")).toEqual({ + ast: program( + stmt( + ifClause( + [stmt(simple("a"))], + [stmt(simple("b"))], + [stmt(simple("c"))], + ), + ), + ), + }); + }); + + it("parses if/elif/then/fi", () => { + expect(parse("if a; then b; elif c; then d; fi")).toEqual({ + ast: program( + stmt( + ifClause( + [stmt(simple("a"))], + [stmt(simple("b"))], + [stmt(ifClause([stmt(simple("c"))], [stmt(simple("d"))]))], + ), + ), + ), + }); + }); +}); + +describe("parse (phase 6: while/until clauses)", () => { + it("parses while/do/done", () => { + expect(parse("while a; do b; done")).toEqual({ + ast: program(stmt(whileClause([stmt(simple("a"))], [stmt(simple("b"))]))), + }); + }); + + it("parses until/do/done", () => { + expect(parse("until a; do b; done")).toEqual({ + ast: program( + stmt(whileClause([stmt(simple("a"))], [stmt(simple("b"))], true)), + ), + }); + }); +}); + +describe("parse (phase 7: for clauses)", () => { + it("parses for-in loops", () => { + expect(parse("for i in a b; do c; done")).toEqual({ + ast: program( + stmt(forClause("i", [stmt(simple("c"))], [word("a"), word("b")])), + ), + }); + }); + + it("parses for loops without in list", () => { + expect(parse("for i; do c; done")).toEqual({ + ast: program(stmt(forClause("i", [stmt(simple("c"))]))), + }); + }); +}); + +describe("parse (phase 8: select clauses)", () => { + it("parses select loops", () => { + expect(parse("select i in a b; do c; done")).toEqual({ + ast: program( + stmt(selectClause("i", [stmt(simple("c"))], [word("a"), word("b")])), + ), + }); + }); +}); + +describe("parse (phase 9: functions and case)", () => { + it("parses function declarations", () => { + expect(parse("foo() { bar; }")).toEqual({ + ast: program(stmt(functionDecl("foo", [stmt(simple("bar"))]))), + }); + + expect(parse("function foo { bar; }")).toEqual({ + ast: program(stmt(functionDecl("foo", [stmt(simple("bar"))]))), + }); + }); + + it("parses case clauses", () => { + expect(parse("case x in y) z ;; esac")).toEqual({ + ast: program( + stmt(caseClause("x", [caseItem([word("y")], [stmt(simple("z"))])])), + ), + }); + + expect(parse("case x in a|b) z ;; esac")).toEqual({ + ast: program( + stmt( + caseClause("x", [ + caseItem([word("a"), word("b")], [stmt(simple("z"))]), + ]), + ), + ), + }); + + expect(parse("case x in a) y ;; b) z ;; esac")).toEqual({ + ast: program( + stmt( + caseClause("x", [ + caseItem([word("a")], [stmt(simple("y"))]), + caseItem([word("b")], [stmt(simple("z"))]), + ]), + ), + ), + }); + }); +}); + +describe("parse (phase 11: time)", () => { + it("parses time clauses", () => { + expect(parse("time foo")).toEqual({ + ast: program(stmt(timeClause(stmt(simple("foo"))))), + }); + }); +}); + +describe("parse (phase 13: extended test)", () => { + it("parses [[ ]]", () => { + expect(parse("[[ -f foo ]]")).toEqual({ + ast: program(stmt(testClause(word("-f"), word("foo")))), + }); + }); + + it("parses [[ with binary op ]]", () => { + expect(parse("[[ foo == bar ]]")).toEqual({ + ast: program(stmt(testClause(word("foo"), word("=="), word("bar")))), + }); + }); +}); + +describe("parse (phase 14: arithmetic command)", () => { + it("parses (( ))", () => { + expect(parse("(( x + 1 ))")).toEqual({ + ast: program(stmt(arithCmd("x + 1"))), + }); + }); + + it("parses nested parens in (( ))", () => { + expect(parse("(( (x + 1) * 2 ))")).toEqual({ + ast: program(stmt(arithCmd("(x + 1) * 2"))), + }); + }); +}); + +describe("parse (phase 20: mixed clauses)", () => { + it("parses function with parens in function keyword form", () => { + expect(parse("function foo() { bar; }")).toEqual({ + ast: program(stmt(functionDecl("foo", [stmt(simple("bar"))]))), + }); + }); + + it("parses if/elif/else/fi chain", () => { + expect(parse("if a; then b; elif c; then d; else e; fi")).toEqual({ + ast: program( + stmt( + ifClause( + [stmt(simple("a"))], + [stmt(simple("b"))], + [ + stmt( + ifClause( + [stmt(simple("c"))], + [stmt(simple("d"))], + [stmt(simple("e"))], + ), + ), + ], + ), + ), + ), + }); + }); +}); + +describe("parse (phase 21: coproc)", () => { + it("parses coproc with command", () => { + expect(parse("coproc foo")).toEqual({ + ast: program(stmt(coprocClause(stmt(simple("foo"))))), + }); + }); + + it("parses coproc with name and block", () => { + expect(parse("coproc NAME { foo; }")).toEqual({ + ast: program( + stmt(coprocClause(stmt(block(stmt(simple("foo")))), "NAME")), + ), + }); + }); +}); diff --git a/src/parser/constants.ts b/src/parser/constants.ts new file mode 100644 index 0000000..19f2c20 --- /dev/null +++ b/src/parser/constants.ts @@ -0,0 +1,8 @@ +export const DECL_KEYWORDS = new Set([ + "declare", + "local", + "export", + "readonly", + "typeset", + "nameref", +]); diff --git a/src/parser/decl-let-cstyle.test.ts b/src/parser/decl-let-cstyle.test.ts new file mode 100644 index 0000000..f6970e4 --- /dev/null +++ b/src/parser/decl-let-cstyle.test.ts @@ -0,0 +1,280 @@ +import { assert, describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { + arrayElem, + arrayExpr, + assign, + cStyleLoop, + declClause, + letClause, + program, + redirect, + simple, + stmt, + word, +} from "../test-helpers/ast-builders"; + +describe("parse (phase 22: decl clause)", () => { + it.each([ + { + input: "export FOO=bar", + variant: "export" as const, + assigns: [assign("FOO", "bar")], + }, + { + input: "local x=1", + variant: "local" as const, + assigns: [assign("x", "1")], + }, + { + input: "nameref ref=target", + variant: "nameref" as const, + assigns: [assign("ref", "target")], + }, + ])("parses $variant with assignment", ({ + input, + variant, + assigns: assignList, + }) => { + expect(parse(input)).toEqual({ + ast: program(stmt(declClause(variant, { assigns: assignList }))), + }); + }); + + it("parses export with multiple names", () => { + expect(parse("export FOO BAR")).toEqual({ + ast: program( + stmt(declClause("export", { args: [word("FOO"), word("BAR")] })), + ), + }); + }); + + it.each([ + { + input: "declare -r FOO=bar", + variant: "declare" as const, + flag: "-r", + assignVal: assign("FOO", "bar"), + }, + { + input: "typeset -i count=0", + variant: "typeset" as const, + flag: "-i", + assignVal: assign("count", "0"), + }, + ])("parses $variant with flag and assignment", ({ + input, + variant, + flag, + assignVal, + }) => { + expect(parse(input)).toEqual({ + ast: program( + stmt( + declClause(variant, { + args: [word(flag)], + assigns: [assignVal], + }), + ), + ), + }); + }); + + it("parses readonly with names", () => { + expect(parse("readonly X Y")).toEqual({ + ast: program( + stmt(declClause("readonly", { args: [word("X"), word("Y")] })), + ), + }); + }); + + it("parses export with redirect", () => { + expect(parse("export FOO=bar 2>/dev/null")).toEqual({ + ast: program( + stmt( + declClause("export", { + assigns: [assign("FOO", "bar")], + redirects: [redirect(">", "/dev/null", "2")], + }), + ), + ), + }); + }); + + it("parses declare -a with array", () => { + expect(parse("declare -a arr=(a b c)")).toEqual({ + ast: program( + stmt( + declClause("declare", { + args: [word("-a")], + assigns: [ + assign("arr", undefined, { + array: arrayExpr( + arrayElem("a"), + arrayElem("b"), + arrayElem("c"), + ), + }), + ], + }), + ), + ), + }); + }); +}); + +describe("parse (phase 23: append assignment)", () => { + it("parses append assignment", () => { + expect(parse("PATH+=/usr/local/bin echo hi")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), word("hi")], + assignments: [assign("PATH", "/usr/local/bin", { append: true })], + }), + ), + }); + }); + + it("parses standalone append assignment", () => { + expect(parse("arr+=value")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [assign("arr", "value", { append: true })], + }), + ), + }); + }); +}); + +describe("parse (phase 24: array expressions)", () => { + it("parses simple array assignment", () => { + expect(parse("arr=(a b c)")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [ + assign("arr", undefined, { + array: arrayExpr(arrayElem("a"), arrayElem("b"), arrayElem("c")), + }), + ], + }), + ), + }); + }); + + it("parses indexed array assignment", () => { + expect(parse("arr=([0]=x [1]=y)")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [ + assign("arr", undefined, { + array: arrayExpr(arrayElem("x", "0"), arrayElem("y", "1")), + }), + ], + }), + ), + }); + }); + + it("parses associative array assignment", () => { + expect(parse("arr=([key]=val [other]=thing)")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [ + assign("arr", undefined, { + array: arrayExpr( + arrayElem("val", "key"), + arrayElem("thing", "other"), + ), + }), + ], + }), + ), + }); + }); + + it("parses empty array", () => { + expect(parse("arr=()")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [assign("arr", undefined, { array: arrayExpr() })], + }), + ), + }); + }); + + it("parses append array", () => { + expect(parse("arr+=(x y)")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [ + assign("arr", undefined, { + append: true, + array: arrayExpr(arrayElem("x"), arrayElem("y")), + }), + ], + }), + ), + }); + }); +}); + +describe("parse (phase 25: c-style for loop)", () => { + it("parses c-style for loop", () => { + const result = parse("for ((i=0; i<10; i++)); do echo $i; done"); + const first = result.ast.body[0]; + assert(first, "expected at least one statement"); + const command = first.command; + assert(command.type === "CStyleLoop", "expected CStyleLoop"); + expect(command.init).toBe("i=0"); + expect(command.cond).toBe("i<10"); + expect(command.post).toBe("i++"); + expect(command.body).toHaveLength(1); + }); + + it("parses c-style for loop with empty parts", () => { + expect(parse("for ((;;)); do echo loop; done")).toEqual({ + ast: program(stmt(cStyleLoop([stmt(simple("echo", "loop"))]))), + }); + }); + + it("parses c-style for loop with only condition", () => { + const result = parse("for (( ; i<5; )); do echo $i; done"); + const first = result.ast.body[0]; + assert(first, "expected at least one statement"); + const command = first.command; + assert(command.type === "CStyleLoop", "expected CStyleLoop"); + expect(command.init).toBeUndefined(); + expect(command.cond).toBe("i<5"); + expect(command.post).toBeUndefined(); + expect(command.body).toHaveLength(1); + }); +}); + +describe("parse (phase 26: let clause)", () => { + it("parses let with single expression", () => { + expect(parse("let i++")).toEqual({ + ast: program(stmt(letClause([word("i++")]))), + }); + }); + + it("parses let with multiple expressions", () => { + expect(parse("let i++ j=2")).toEqual({ + ast: program(stmt(letClause([word("i++"), word("j=2")]))), + }); + }); + + it("parses let with redirect", () => { + expect(parse("let x=1 2>/dev/null")).toEqual({ + ast: program( + stmt(letClause([word("x=1")], [redirect(">", "/dev/null", "2")])), + ), + }); + }); +}); diff --git a/src/parser/expansions.test.ts b/src/parser/expansions.test.ts new file mode 100644 index 0000000..d6bdd85 --- /dev/null +++ b/src/parser/expansions.test.ts @@ -0,0 +1,185 @@ +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: shell syntax in test strings +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { + arithExp, + cmdSubst, + dbl, + lit, + paramExp, + program, + simple, + stmt, + word, + wordParts, +} from "../test-helpers/ast-builders"; + +describe("parse (phase 15: parameter expansion)", () => { + it("parses $var", () => { + expect(parse("echo $foo")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(paramExp("foo"))], + }), + ), + }); + }); + + it("parses ${var}", () => { + expect(parse("echo ${foo}")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(paramExp("foo", false))], + }), + ), + }); + }); + + it("parses ${var:-default}", () => { + expect(parse("echo ${foo:-bar}")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(paramExp("foo", false, ":-", "bar"))], + }), + ), + }); + }); + + it("parses $var inside double quotes", () => { + expect(parse('"hello $name"')).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [wordParts(dbl(lit("hello "), paramExp("name")))], + }), + ), + }); + }); + + it("parses special params $? $# $@", () => { + expect(parse("echo $?")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(paramExp("?"))], + }), + ), + }); + }); +}); + +describe("parse (phase 16: command substitution)", () => { + it("parses $(cmd)", () => { + expect(parse("echo $(foo)")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(cmdSubst(stmt(simple("foo"))))], + }), + ), + }); + }); + + it("parses backtick substitution", () => { + expect(parse("echo `foo`")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(cmdSubst(stmt(simple("foo"))))], + }), + ), + }); + }); + + it("parses $(cmd) inside double quotes", () => { + expect(parse('"$(foo)"')).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [wordParts(dbl(cmdSubst(stmt(simple("foo")))))], + }), + ), + }); + }); +}); + +describe("parse (phase 17: arithmetic expansion)", () => { + it("parses $((expr))", () => { + expect(parse("echo $((1 + 2))")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("echo"), wordParts(arithExp("1 + 2"))], + }), + ), + }); + }); +}); + +describe("parse (phase 19: process substitution)", () => { + it("parses <() process substitution", () => { + expect(parse("diff <(foo) <(bar)")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [ + word("diff"), + wordParts({ + type: "ProcSubst", + op: "<", + stmts: [stmt(simple("foo"))], + }), + wordParts({ + type: "ProcSubst", + op: "<", + stmts: [stmt(simple("bar"))], + }), + ], + }), + ), + }); + }); +}); + +describe("parse (phase 20: mixed expansions)", () => { + it("parses mixed literal and expansion in word", () => { + expect(parse("echo foo${bar}baz")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [ + word("echo"), + wordParts(lit("foo"), paramExp("bar", false), lit("baz")), + ], + }), + ), + }); + }); + + it("parses nested command substitution", () => { + expect(parse("echo $(echo $(foo))")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [ + word("echo"), + wordParts( + cmdSubst( + stmt({ + type: "SimpleCommand", + words: [ + word("echo"), + wordParts(cmdSubst(stmt(simple("foo")))), + ], + }), + ), + ), + ], + }), + ), + }); + }); +}); diff --git a/src/parser/guardrails.test.ts b/src/parser/guardrails.test.ts new file mode 100644 index 0000000..dc116cf --- /dev/null +++ b/src/parser/guardrails.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { extractCommandNames } from "../test-helpers/extract-command-names"; + +// Real-world guardrail validation tests. +// The guardrails extension blocks commands matching /\bnpm\b/ on the full +// command string. This causes false positives when "npm" appears in arguments, +// grep patterns, heredocs, or subcommand strings rather than as the actual +// command being invoked. +// +// These tests validate that the parser produces an AST where the actual +// command name (first word of a SimpleCommand) is distinguishable from +// arguments, allowing a smarter guardrail to only check command positions. +describe("guardrail validation: package manager enforcement", () => { + // Tests where npm appears in arguments/patterns but is NOT a command. + it.each([ + { + desc: "grep for npm pattern", + input: String.raw`grep -rn '\bnpm\b' src/`, + expected: ["grep"], + }, + { + desc: "grep with multiple npm/npx patterns", + input: String.raw`grep -rn '\bnpx\b\|\bnpm \b\|\bnpm$' AGENTS.md`, + expected: ["grep"], + }, + { + desc: "echo containing npm", + input: 'echo "use npm install instead"', + expected: ["echo"], + }, + { + desc: "cat of package.json", + input: "cat /path/to/package.json", + expected: ["cat"], + }, + { + desc: "npx is not npm", + input: "npx wrangler --version", + expected: ["npx"], + }, + { + desc: "pnpm is not npm", + input: "pnpm --filter pi-relay-server typecheck", + expected: ["pnpm"], + }, + { + desc: "which npm is not running npm", + input: "which npm 2>/dev/null || echo 'npm not found'", + expected: ["which", "echo"], + }, + { + desc: "if condition with npm check", + input: "if command -v npm; then echo found; fi", + expected: ["command", "echo"], + }, + { + desc: "variable assignment containing npm", + input: 'PKG_MGR=npm echo "using $PKG_MGR"', + expected: ["echo"], + }, + { + desc: "herestring containing npm", + input: "grep -c npm <<< 'npm install pnpm bun'", + expected: ["grep"], + }, + { + desc: "pipeline with npm in args", + input: "find . -name '*.json' | grep npm | head -5", + expected: ["find", "grep", "head"], + }, + { + desc: "biome check via pnpm exec", + input: "pnpm exec biome check --write src/sandbox/cloudflare.test.ts", + expected: ["pnpm"], + }, + { + desc: "curl piped to jq", + input: "curl -s http://localhost:31415/health | jq .", + expected: ["curl", "jq"], + }, + ])("$desc: npm is not extracted as a command", ({ input, expected }) => { + const cmds = extractCommandNames(parse(input).ast); + expect(cmds).toEqual(expected); + expect(cmds).not.toContain("npm"); + }); + + // Tests where npm IS a real command. + it.each([ + { desc: "npm install", input: "npm install --omit=dev" }, + { desc: "npm ci", input: "npm ci" }, + { desc: "npm in command substitution", input: "echo $(npm pack)" }, + { desc: "npm in subshell", input: "(cd /tmp && npm install)" }, + ])("$desc: npm is extracted as a command", ({ input }) => { + const cmds = extractCommandNames(parse(input).ast); + expect(cmds).toContain("npm"); + }); + + it("cd && pnpm: both commands identified correctly", () => { + const cmds = extractCommandNames( + parse("cd /project && pnpm --filter pi-relay-server test").ast, + ); + expect(cmds).toEqual(["cd", "pnpm"]); + expect(cmds).not.toContain("npm"); + }); + + it("npm in || fallback: both branches identified", () => { + const cmds = extractCommandNames( + parse("npm ci --omit=dev 2>/dev/null || npm install --omit=dev").ast, + ); + expect(cmds).toEqual(["npm", "npm"]); + }); + + it("heredoc containing npm install is not an npm command", () => { + const input = `cat <<'EOF' +RUN npm install --omit=dev +npm ci +EOF`; + const cmds = extractCommandNames(parse(input).ast); + expect(cmds).toEqual(["cat"]); + expect(cmds).not.toContain("npm"); + }); + + it("docker build with heredoc Dockerfile containing npm", () => { + const input = `docker build -t myimage -f - . <<'DOCKERFILE' +FROM node:22-slim +RUN npm install --omit=dev +DOCKERFILE`; + const cmds = extractCommandNames(parse(input).ast); + expect(cmds).toEqual(["docker"]); + expect(cmds).not.toContain("npm"); + }); + + it("real session: cd && pnpm filter test piped to tail", () => { + const cmds = extractCommandNames( + parse("cd /project && pnpm --filter pi-relay-server test 2>&1 | tail -15") + .ast, + ); + expect(cmds).toContain("cd"); + expect(cmds).toContain("pnpm"); + expect(cmds).toContain("tail"); + expect(cmds).not.toContain("npm"); + }); + + it("real session: write Dockerfile via heredoc then docker build", () => { + const input = `mkdir -p /tmp/cf-sandbox-test && \\ +cp bridge.js /tmp/cf-sandbox-test/ && \\ +cat > /tmp/cf-sandbox-test/Dockerfile <<'DOCKERFILE' +FROM node:22-slim +RUN apt-get update && apt-get install -y curl tar bash git +WORKDIR /bridge +COPY package.json ./ +RUN npm install --omit=dev 2>/dev/null || true +COPY bridge.js ./ +CMD ["node", "/bridge/bridge.js"] +DOCKERFILE + +docker build --platform linux/arm64 -t pi-sandbox-cf:arm64-debian /tmp/cf-sandbox-test`; + const { ast } = parse(input); + const cmds = extractCommandNames(ast); + expect(cmds).toContain("mkdir"); + expect(cmds).toContain("cp"); + expect(cmds).toContain("cat"); + expect(cmds).toContain("docker"); + expect(cmds).not.toContain("npm"); + }); +}); diff --git a/src/parser/heredoc.test.ts b/src/parser/heredoc.test.ts new file mode 100644 index 0000000..0b2e1da --- /dev/null +++ b/src/parser/heredoc.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { program, stmt, word } from "../test-helpers/ast-builders"; + +describe("parse (phase 18: heredoc)", () => { + it("parses << heredoc", () => { + expect(parse("cat < { + expect(parse("cat <<-EOF\n\thello\n\tEOF")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("cat")], + redirects: [ + { + type: "Redirect", + op: "<<-", + target: word("EOF"), + heredoc: word("hello\n"), + }, + ], + }), + ), + }); + }); +}); diff --git a/src/parser/index.ts b/src/parser/index.ts new file mode 100644 index 0000000..2735706 --- /dev/null +++ b/src/parser/index.ts @@ -0,0 +1 @@ +export { Parser } from "./parser"; diff --git a/src/parser/keep-comments-option.test.ts b/src/parser/keep-comments-option.test.ts new file mode 100644 index 0000000..36bd194 --- /dev/null +++ b/src/parser/keep-comments-option.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; + +describe("parse (phase 27: comments)", () => { + it("does not include comments by default", () => { + const result = parse("echo hi # a comment"); + expect(result.ast.comments).toBeUndefined(); + }); + + it("collects comments when keepComments is true", () => { + const result = parse("echo hi # a comment", { keepComments: true }); + expect(result.ast.comments).toEqual([ + { type: "Comment", text: " a comment" }, + ]); + }); + + it("collects multiple comments", () => { + const result = parse("# first\necho hi\n# second", { + keepComments: true, + }); + expect(result.ast.comments).toEqual([ + { type: "Comment", text: " first" }, + { type: "Comment", text: " second" }, + ]); + }); + + it("collects inline comment after semicolon", () => { + const result = parse("echo hi; # trailing", { keepComments: true }); + expect(result.ast.comments).toEqual([ + { type: "Comment", text: " trailing" }, + ]); + }); +}); diff --git a/src/parser/parser.ts b/src/parser/parser.ts new file mode 100644 index 0000000..476a8fe --- /dev/null +++ b/src/parser/parser.ts @@ -0,0 +1,1016 @@ +import type { + ArithCmd, + ArrayElem, + Assignment, + Block, + CaseClause, + CaseItem, + Command, + CommentNode, + CoprocClause, + CStyleLoop, + DeclClause, + ForClause, + FunctionDecl, + IfClause, + LetClause, + ParamExp, + ParseOptions, + Program, + Redirect, + SelectClause, + SimpleCommand, + Statement, + Subshell, + TestClause, + TimeClause, + WhileClause, + Word, + WordPart, +} from "../ast"; +import type { + OpTokenValue, + SymbolTokenValue, + Token, + TokenWordPart, +} from "../tokenizer"; +import { tokenPartsText } from "../tokenizer"; +import { tokenize } from "../tokenizer/tokenize"; +import { DECL_KEYWORDS } from "./constants"; + +export class Parser { + private index = 0; + private comments: CommentNode[] = []; + + constructor( + private readonly tokens: Token[], + private readonly options: ParseOptions = {}, + ) {} + + parseProgram(): Program { + const body: Statement[] = []; + this.skipSeparators(); + while (!this.isEof()) { + body.push(this.parseStatement()); + this.skipSeparators(); + } + const program: Program = { type: "Program", body }; + if (this.options.keepComments && this.comments.length > 0) { + program.comments = this.comments; + } + return program; + } + + assertEof() { + if (!this.isEof()) { + const token = this.peek(); + const display = token + ? token.type === "op" + ? token.value + : token.type === "redir" + ? token.op + : token.type === "symbol" + ? token.value + : token.type === "arith-cmd" + ? "(( ... ))" + : token.type === "heredoc-body" + ? "<>" + : token.type === "comment" + ? `#${token.text}` + : tokenPartsText(token.parts) + : ""; + throw new Error(`Unexpected token: ${display}`); + } + } + + private parseStatement(): Statement { + let negated = false; + if (this.matchOp("!")) { + this.consume(); + negated = true; + } + const command = this.parseLogical(); + let background = false; + if (this.matchOp("&")) { + this.consume(); + background = true; + } + const statement: Statement = { type: "Statement", command }; + if (background) { + statement.background = true; + } + if (negated) { + statement.negated = true; + } + return statement; + } + + private parseLogical(): Command { + let leftCommand = this.parsePipeline(); + while (this.matchOp("&&") || this.matchOp("||")) { + const opToken = this.consume(); + if (opToken.type !== "op") { + throw new Error("Expected logical operator"); + } + const rightCommand = this.parsePipeline(); + leftCommand = { + type: "Logical", + op: opToken.value === "&&" ? "and" : "or", + left: { type: "Statement", command: leftCommand }, + right: { type: "Statement", command: rightCommand }, + }; + } + return leftCommand; + } + + private parsePipeline(): Command { + const first = this.parseCommandAtom(); + if (!this.matchOp("|")) { + return first; + } + + const commands: Statement[] = [{ type: "Statement", command: first }]; + while (this.matchOp("|")) { + this.consume(); + const next = this.parseCommandAtom(); + commands.push({ type: "Statement", command: next }); + } + return { type: "Pipeline", commands }; + } + + private parseCommandAtom(): Command { + if (this.matchKeyword("if")) { + return this.parseIfClause(); + } + if (this.matchKeyword("while")) { + return this.parseWhileClause(false); + } + if (this.matchKeyword("until")) { + return this.parseWhileClause(true); + } + if (this.matchKeyword("for")) { + return this.parseForOrCStyleLoop(); + } + if (this.matchKeyword("select")) { + return this.parseSelectClause(); + } + if (this.matchKeyword("case")) { + return this.parseCaseClause(); + } + if (this.matchKeyword("time")) { + return this.parseTimeClause(); + } + if (this.matchKeyword("coproc")) { + return this.parseCoprocClause(); + } + if (this.matchKeyword("[[")) { + return this.parseTestClause(); + } + if (this.matchKeyword("function") || this.looksLikeFuncDecl()) { + return this.parseFunctionDecl(); + } + if (this.matchArithCmd()) { + return this.consumeArithCmd(); + } + if (this.matchSymbol("(")) { + return this.parseSubshell(); + } + if (this.matchSymbol("{")) { + return this.parseBlock(); + } + if (this.matchDeclKeyword()) { + return this.parseDeclClause(); + } + if (this.matchKeyword("let")) { + return this.parseLetClause(); + } + return this.parseSimpleCommand(); + } + + private parseSubshell(): Subshell { + this.consumeSymbol("("); + const body = this.parseStatementList(")"); + this.consumeSymbol(")"); + return { type: "Subshell", body }; + } + + private parseBlock(): Block { + this.consumeSymbol("{"); + const body = this.parseStatementList("}"); + this.consumeSymbol("}"); + return { type: "Block", body }; + } + + private parseStatementList(endSymbol: SymbolTokenValue): Statement[] { + const body: Statement[] = []; + this.skipSeparators(); + while (!this.matchSymbol(endSymbol)) { + if (this.isEof()) { + throw new Error( + `Unexpected end of input while looking for ${endSymbol}`, + ); + } + body.push(this.parseStatement()); + this.skipSeparators(); + } + return body; + } + + private parseIfClause(): IfClause { + this.consumeKeyword("if"); + const cond = this.parseStatementsUntilKeyword(["then"]); + this.consumeKeyword("then"); + const thenBranch = this.parseStatementsUntilKeyword(["else", "elif", "fi"]); + let elseBranch: Statement[] | undefined; + if (this.matchKeyword("elif")) { + elseBranch = [ + { + type: "Statement", + command: this.parseElifClause(), + }, + ]; + } else if (this.matchKeyword("else")) { + this.consumeKeyword("else"); + elseBranch = this.parseStatementsUntilKeyword(["fi"]); + } + this.consumeKeyword("fi"); + return elseBranch + ? { + type: "IfClause", + cond, + // biome-ignore lint/suspicious/noThenProperty: shell AST field + then: thenBranch, + else: elseBranch, + } + : { + type: "IfClause", + cond, + // biome-ignore lint/suspicious/noThenProperty: shell AST field + then: thenBranch, + }; + } + + private parseElifClause(): IfClause { + this.consumeKeyword("elif"); + const cond = this.parseStatementsUntilKeyword(["then"]); + this.consumeKeyword("then"); + const thenBranch = this.parseStatementsUntilKeyword(["else", "elif", "fi"]); + let elseBranch: Statement[] | undefined; + if (this.matchKeyword("elif")) { + elseBranch = [ + { + type: "Statement", + command: this.parseElifClause(), + }, + ]; + } else if (this.matchKeyword("else")) { + this.consumeKeyword("else"); + elseBranch = this.parseStatementsUntilKeyword(["fi"]); + } + return elseBranch + ? { + type: "IfClause", + cond, + // biome-ignore lint/suspicious/noThenProperty: shell AST field + then: thenBranch, + else: elseBranch, + } + : { + type: "IfClause", + cond, + // biome-ignore lint/suspicious/noThenProperty: shell AST field + then: thenBranch, + }; + } + + private parseWhileClause(until: boolean): WhileClause { + this.consumeKeyword(until ? "until" : "while"); + const cond = this.parseStatementsUntilKeyword(["do"]); + this.consumeKeyword("do"); + const body = this.parseStatementsUntilKeyword(["done"]); + this.consumeKeyword("done"); + return until + ? { type: "WhileClause", cond, body, until: true } + : { type: "WhileClause", cond, body }; + } + + private parseForOrCStyleLoop(): ForClause | CStyleLoop { + this.consumeKeyword("for"); + + // C-style: for (( init; cond; post )) + if (this.matchArithCmd()) { + return this.parseCStyleLoop(); + } + + const nameToken = this.consume(); + if (nameToken.type !== "word") { + throw new Error("Expected loop variable name"); + } + const name = tokenPartsText(nameToken.parts); + let items: Word[] | undefined; + if (this.matchKeyword("in")) { + this.consumeKeyword("in"); + const collected: Word[] = []; + while (this.matchWord() && !this.matchKeyword("do")) { + const itemToken = this.consume(); + if (itemToken.type !== "word") { + throw new Error("Expected loop item word"); + } + collected.push(this.wordFromParts(itemToken.parts)); + } + if (collected.length > 0) { + items = collected; + } + } + if (this.matchOp(";")) { + this.consume(); + } + this.skipSeparators(); + this.consumeKeyword("do"); + const body = this.parseStatementsUntilKeyword(["done"]); + this.consumeKeyword("done"); + return items + ? { type: "ForClause", name, items, body } + : { type: "ForClause", name, body }; + } + + private parseCStyleLoop(): CStyleLoop { + const token = this.consume(); + if (token.type !== "arith-cmd") { + throw new Error("Expected (( )) in c-style for"); + } + // Split expr on ";" into init, cond, post + const parts = token.expr.split(";").map((s) => s.trim()); + const init = parts[0] || undefined; + const cond = parts[1] || undefined; + const post = parts[2] || undefined; + + if (this.matchOp(";")) { + this.consume(); + } + this.skipSeparators(); + this.consumeKeyword("do"); + const body = this.parseStatementsUntilKeyword(["done"]); + this.consumeKeyword("done"); + const loop: CStyleLoop = { type: "CStyleLoop", body }; + if (init !== undefined) loop.init = init; + if (cond !== undefined) loop.cond = cond; + if (post !== undefined) loop.post = post; + return loop; + } + + private parseSelectClause(): SelectClause { + this.consumeKeyword("select"); + const nameToken = this.consume(); + if (nameToken.type !== "word") { + throw new Error("Expected select variable name"); + } + const name = tokenPartsText(nameToken.parts); + let items: Word[] | undefined; + if (this.matchKeyword("in")) { + this.consumeKeyword("in"); + const collected: Word[] = []; + while (this.matchWord() && !this.matchKeyword("do")) { + const itemToken = this.consume(); + if (itemToken.type !== "word") { + throw new Error("Expected select item word"); + } + collected.push(this.wordFromParts(itemToken.parts)); + } + if (collected.length > 0) { + items = collected; + } + } + if (this.matchOp(";")) { + this.consume(); + } + this.skipSeparators(); + this.consumeKeyword("do"); + const body = this.parseStatementsUntilKeyword(["done"]); + this.consumeKeyword("done"); + return items + ? { type: "SelectClause", name, items, body } + : { type: "SelectClause", name, body }; + } + + private parseFunctionDecl(): FunctionDecl { + if (this.matchKeyword("function")) { + this.consumeKeyword("function"); + } + const nameToken = this.consume(); + if (nameToken.type !== "word") { + throw new Error("Expected function name"); + } + const name = tokenPartsText(nameToken.parts); + if (this.matchSymbol("(")) { + this.consumeSymbol("("); + this.consumeSymbol(")"); + } + if (this.matchSymbol("{")) { + const body = this.parseBlock().body; + return { type: "FunctionDecl", name, body }; + } + throw new Error("Expected function body block"); + } + + private parseCaseClause(): CaseClause { + this.consumeKeyword("case"); + const wordToken = this.consume(); + if (wordToken.type !== "word") { + throw new Error("Expected case word"); + } + const word = this.wordFromParts(wordToken.parts); + this.consumeKeyword("in"); + const items: CaseItem[] = []; + this.skipSeparators(); + while (!this.matchKeyword("esac")) { + const patterns: Word[] = []; + while (!this.matchSymbol(")")) { + if (this.matchWord()) { + const patternToken = this.consume(); + if (patternToken.type !== "word") { + throw new Error("Expected case pattern"); + } + patterns.push(this.wordFromParts(patternToken.parts)); + continue; + } + if (this.matchOp("|")) { + this.consume(); + continue; + } + throw new Error("Expected case pattern or )"); + } + this.consumeSymbol(")"); + const body = this.parseCaseItemBody(); + items.push({ type: "CaseItem", patterns, body }); + if (this.matchOp(";") && this.peekOp(";")) { + this.consume(); + this.consume(); + } + this.skipSeparators(); + } + this.consumeKeyword("esac"); + return { type: "CaseClause", word, items }; + } + + private parseTimeClause(): TimeClause { + this.consumeKeyword("time"); + const command = this.parseStatement(); + return { type: "TimeClause", command }; + } + + private parseTestClause(): TestClause { + this.consumeKeyword("[["); + const words: Word[] = []; + while (!this.matchKeyword("]]")) { + if (this.isEof()) throw new Error("Unclosed [["); + const token = this.consume(); + if (token.type !== "word") throw new Error("Expected word in [[ ]]"); + words.push(this.wordFromParts(token.parts)); + } + this.consumeKeyword("]]"); + return { type: "TestClause", expr: words }; + } + + private matchArithCmd(): boolean { + return this.peek()?.type === "arith-cmd"; + } + + private consumeArithCmd(): ArithCmd { + const token = this.consume(); + if (token.type !== "arith-cmd") + throw new Error("Expected arithmetic command"); + return { type: "ArithCmd", expr: token.expr }; + } + + private parseCoprocClause(): CoprocClause { + this.consumeKeyword("coproc"); + if (this.matchWord() && this.peekToken(1)?.type === "symbol") { + const nameToken = this.peek(); + if ( + nameToken?.type === "word" && + this.peekToken(1)?.type === "symbol" && + (this.peekToken(1) as { value: string }).value === "{" + ) { + const name = tokenPartsText(nameToken.parts); + this.consume(); + const body = this.parseStatement(); + return { type: "CoprocClause", name, body }; + } + } + const body = this.parseStatement(); + return { type: "CoprocClause", body }; + } + + private parseCaseItemBody(): Statement[] { + const body: Statement[] = []; + this.skipCaseSeparators(); + while (!this.matchKeyword("esac") && !this.isCaseItemEnd()) { + body.push(this.parseStatement()); + if (this.isCaseItemEnd()) { + break; + } + this.skipCaseSeparators(); + } + return body; + } + + private isCaseItemEnd(): boolean { + return this.matchOp(";") && this.peekOp(";"); + } + + private parseStatementsUntilKeyword(endKeywords: string[]): Statement[] { + const body: Statement[] = []; + this.skipSeparators(); + while (!this.matchKeywordIn(endKeywords)) { + if (this.isEof()) { + throw new Error( + `Unexpected end of input while looking for ${endKeywords.join(", ")}`, + ); + } + body.push(this.parseStatement()); + this.skipSeparators(); + } + return body; + } + + private matchDeclKeyword(): boolean { + const token = this.peek(); + if (token?.type !== "word" || token.parts.length !== 1) return false; + const part = token.parts[0]; + return part?.type === "lit" && DECL_KEYWORDS.has(part.value); + } + + private parseDeclClause(): DeclClause { + const variantToken = this.consume(); + if (variantToken.type !== "word") { + throw new Error("Expected decl keyword"); + } + const variant = tokenPartsText(variantToken.parts) as DeclClause["variant"]; + + const args: Word[] = []; + const assigns: Assignment[] = []; + const redirects: Redirect[] = []; + + while (true) { + if (this.matchRedir()) { + const token = this.consume(); + if (token.type !== "redir") { + throw new Error("Expected redirect token"); + } + const targetToken = this.consume(); + if (targetToken.type !== "word") { + throw new Error("Redirect must be followed by a word"); + } + const target = this.wordFromParts(targetToken.parts); + const redir: Redirect = token.fd + ? { type: "Redirect", op: token.op, fd: token.fd, target } + : { type: "Redirect", op: token.op, target }; + redirects.push(redir); + continue; + } + + if (this.matchWord()) { + const token = this.peek(); + if (!token || token.type !== "word") break; + + // tryParseAssignment consumes tokens itself if it matches + const assignment = this.tryParseAssignment(token.parts); + if (assignment) { + assigns.push(assignment); + continue; + } + + // Otherwise it's a plain arg (flag or name) + this.consume(); + args.push(this.wordFromParts(token.parts)); + continue; + } + + break; + } + + const decl: DeclClause = { type: "DeclClause", variant }; + if (args.length > 0) decl.args = args; + if (assigns.length > 0) decl.assigns = assigns; + if (redirects.length > 0) decl.redirects = redirects; + return decl; + } + + private parseLetClause(): LetClause { + this.consumeKeyword("let"); + const exprs: Word[] = []; + const redirects: Redirect[] = []; + + while (true) { + if (this.matchRedir()) { + const token = this.consume(); + if (token.type !== "redir") { + throw new Error("Expected redirect token"); + } + const targetToken = this.consume(); + if (targetToken.type !== "word") { + throw new Error("Redirect must be followed by a word"); + } + const target = this.wordFromParts(targetToken.parts); + const redir: Redirect = token.fd + ? { type: "Redirect", op: token.op, fd: token.fd, target } + : { type: "Redirect", op: token.op, target }; + redirects.push(redir); + continue; + } + + if (this.matchWord()) { + const token = this.consume(); + if (token.type !== "word") break; + exprs.push(this.wordFromParts(token.parts)); + continue; + } + + break; + } + + if (exprs.length === 0) { + throw new Error("let requires at least one expression"); + } + + const clause: LetClause = { type: "LetClause", exprs }; + if (redirects.length > 0) clause.redirects = redirects; + return clause; + } + + private parseSimpleCommand(): SimpleCommand { + const words: Word[] = []; + const assignments: Assignment[] = []; + const redirects: Redirect[] = []; + let sawWord = false; + + while (true) { + if (this.matchWord()) { + const token = this.peek(); + if (!token || token.type !== "word") { + throw new Error("Expected word token"); + } + + // tryParseAssignment consumes tokens itself if it matches + if (!sawWord) { + const assignment = this.tryParseAssignment(token.parts); + if (assignment) { + assignments.push(assignment); + continue; + } + } + + this.consume(); + sawWord = true; + words.push(this.wordFromParts(token.parts)); + continue; + } + + if (this.matchRedir()) { + const token = this.consume(); + if (token.type !== "redir") { + throw new Error("Expected redirect token"); + } + const targetToken = this.consume(); + if (targetToken.type !== "word") { + throw new Error("Redirect must be followed by a word"); + } + const target = this.wordFromParts(targetToken.parts); + const redirect: Redirect = token.fd + ? { type: "Redirect", op: token.op, fd: token.fd, target } + : { type: "Redirect", op: token.op, target }; + // Collect heredoc body if this is a heredoc redirect + if (token.op === "<<" || token.op === "<<-") { + // Skip separators to find the heredoc-body token + this.skipSeparators(); + if (this.peek()?.type === "heredoc-body") { + const bodyToken = this.consume(); + if (bodyToken.type === "heredoc-body") { + redirect.heredoc = { + type: "Word", + parts: [{ type: "Literal", value: bodyToken.content }], + }; + } + } + } + redirects.push(redirect); + continue; + } + + break; + } + + if ( + words.length === 0 && + assignments.length === 0 && + redirects.length === 0 + ) { + throw new Error("Expected a command word"); + } + + const command: SimpleCommand = { type: "SimpleCommand" }; + if (words.length > 0) { + command.words = words; + } + if (assignments.length > 0) { + command.assignments = assignments; + } + if (redirects.length > 0) { + command.redirects = redirects; + } + return command; + } + + private convertWordPart(part: TokenWordPart): WordPart { + switch (part.type) { + case "lit": + return { type: "Literal", value: part.value }; + case "sgl": + return { type: "SglQuoted", value: part.value }; + case "dbl": + return { + type: "DblQuoted", + parts: part.parts.map((p) => this.convertWordPart(p)), + }; + case "param": { + const paramExp: ParamExp = { + type: "ParamExp", + short: !part.braced, + param: { type: "Literal", value: part.name }, + }; + if (part.op) { + paramExp.op = part.op; + } + if (part.value !== undefined) { + paramExp.value = { + type: "Word", + parts: [{ type: "Literal", value: part.value }], + }; + } + return paramExp; + } + case "cmd-subst": { + const innerTokens = tokenize(part.raw); + const innerParser = new Parser(innerTokens); + const prog = innerParser.parseProgram(); + return { type: "CmdSubst", stmts: prog.body }; + } + case "arith-exp": + return { type: "ArithExp", expr: part.raw }; + case "proc-subst": { + const innerTokens = tokenize(part.raw); + const innerParser = new Parser(innerTokens); + const prog = innerParser.parseProgram(); + return { type: "ProcSubst", op: part.op, stmts: prog.body }; + } + case "backtick": { + const innerTokens = tokenize(part.raw); + const innerParser = new Parser(innerTokens); + const prog = innerParser.parseProgram(); + return { type: "CmdSubst", stmts: prog.body }; + } + } + } + + private wordFromParts(parts: TokenWordPart[]): Word { + return { + type: "Word", + parts: parts.map((part) => this.convertWordPart(part)), + }; + } + + /** + * Try to parse an assignment from the current token's parts. + * If it returns an assignment, it has already consumed all relevant tokens + * (the word, and optionally the array `(...)` symbols). + * If it returns undefined, nothing was consumed. + */ + private tryParseAssignment(parts: TokenWordPart[]): Assignment | undefined { + if (parts.length !== 1) return undefined; + const part = parts[0]; + if (!part || part.type !== "lit") return undefined; + const raw = part.value; + + // Detect NAME= or NAME+= + let append = false; + let eqIndex = raw.indexOf("+="); + if (eqIndex > 0) { + append = true; + } else { + eqIndex = raw.indexOf("="); + } + if (eqIndex <= 0) return undefined; + + const name = raw.slice(0, eqIndex); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) return undefined; + + const afterEq = raw.slice(eqIndex + (append ? 2 : 1)); + + // Check for array assignment: NAME=( ... ) or NAME+=( ... ) + const nextToken = this.peekToken(1); + if ( + afterEq === "" && + nextToken?.type === "symbol" && + (nextToken as { value: string }).value === "(" + ) { + this.consume(); // consume the NAME= word + return this.parseArrayAssignment(name, append); + } + + // Consume the word token + this.consume(); + + const assignment: Assignment = { type: "Assignment", name }; + if (append) assignment.append = true; + if (afterEq.length > 0) { + assignment.value = { + type: "Word", + parts: [{ type: "Literal", value: afterEq }], + }; + } + return assignment; + } + + private parseArrayAssignment(name: string, append: boolean): Assignment { + this.consumeSymbol("("); + const elems: ArrayElem[] = []; + + while (!this.matchSymbol(")")) { + if (this.isEof()) { + throw new Error("Unclosed array expression"); + } + if (this.matchOp(";")) { + this.consume(); + continue; + } + if (this.matchComment()) { + this.consumeComment(); + continue; + } + + const token = this.consume(); + if (token.type !== "word") { + throw new Error("Expected word in array expression"); + } + + const text = tokenPartsText(token.parts); + + // Check for [index]=value pattern + const indexMatch = text.match(/^\[([^\]]+)\]=(.*)$/); + if (indexMatch) { + const indexStr = indexMatch[1] as string; + const valStr = indexMatch[2] as string; + const elem: ArrayElem = { + type: "ArrayElem", + index: { + type: "Word", + parts: [{ type: "Literal", value: indexStr }], + }, + }; + if (valStr.length > 0) { + elem.value = { + type: "Word", + parts: [{ type: "Literal", value: valStr }], + }; + } + elems.push(elem); + } else { + elems.push({ + type: "ArrayElem", + value: this.wordFromParts(token.parts), + }); + } + } + + this.consumeSymbol(")"); + + const assignment: Assignment = { + type: "Assignment", + name, + array: { type: "ArrayExpr", elems }, + }; + if (append) assignment.append = true; + return assignment; + } + + private skipSeparators() { + while (this.matchOp(";") || this.matchComment()) { + if (this.matchComment()) { + this.consumeComment(); + } else { + this.consume(); + } + } + } + + private skipCaseSeparators() { + while (this.matchOp(";") && !this.peekOp(";")) { + this.consume(); + } + } + + private matchOp(value: OpTokenValue) { + const token = this.peek(); + return token?.type === "op" && token.value === value; + } + + private matchWord() { + return this.peek()?.type === "word"; + } + + private matchRedir() { + return this.peek()?.type === "redir"; + } + + private matchKeyword(value: string) { + const token = this.peek(); + if (token?.type !== "word" || token.parts.length !== 1) return false; + const part = token.parts[0]; + return part?.type === "lit" && part.value === value; + } + + private matchKeywordIn(values: string[]) { + return values.some((value) => this.matchKeyword(value)); + } + + private looksLikeFuncDecl(): boolean { + const name = this.peek(); + const next = this.peekToken(1); + const nextNext = this.peekToken(2); + const after = this.peekToken(3); + return ( + name?.type === "word" && + next?.type === "symbol" && + next.value === "(" && + nextNext?.type === "symbol" && + nextNext.value === ")" && + after?.type === "symbol" && + after.value === "{" + ); + } + + private matchSymbol(value: SymbolTokenValue) { + const token = this.peek(); + return token?.type === "symbol" && token.value === value; + } + + private consumeSymbol(value: SymbolTokenValue) { + const token = this.consume(); + if (token.type !== "symbol" || token.value !== value) { + throw new Error(`Expected symbol ${value}`); + } + } + + private consumeKeyword(value: string) { + const token = this.consume(); + if ( + token.type !== "word" || + token.parts.length !== 1 || + token.parts[0]?.type !== "lit" || + token.parts[0].value !== value + ) { + throw new Error(`Expected keyword ${value}`); + } + } + + private consume(): Token { + if (this.isEof()) { + throw new Error("Unexpected end of input"); + } + const token = this.tokens[this.index]; + if (!token) { + throw new Error("Unexpected end of input"); + } + this.index += 1; + return token; + } + + private peek(): Token | undefined { + return this.tokens[this.index]; + } + + private peekToken(offset: number): Token | undefined { + return this.tokens[this.index + offset]; + } + + private peekOp(value: OpTokenValue): boolean { + const token = this.peekToken(1); + return token?.type === "op" && token.value === value; + } + + private matchComment(): boolean { + return this.peek()?.type === "comment"; + } + + private consumeComment() { + const token = this.consume(); + if (token.type === "comment") { + this.comments.push({ type: "Comment", text: token.text }); + } + } + + private isEof() { + return this.index >= this.tokens.length; + } +} diff --git a/src/parser/redirects.test.ts b/src/parser/redirects.test.ts new file mode 100644 index 0000000..82c8463 --- /dev/null +++ b/src/parser/redirects.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import type { RedirOp } from "../ast"; +import { parse } from "../parse"; +import { + assign, + program, + redirect, + stmt, + word, +} from "../test-helpers/ast-builders"; + +describe("parse (phase 3: assignments and redirects)", () => { + it("parses assignment-only commands", () => { + expect(parse("a=b")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [assign("a", "b")], + }), + ), + }); + }); + + it("parses assignments before words", () => { + expect(parse("a=b foo")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + assignments: [assign("a", "b")], + words: [word("foo")], + }), + ), + }); + }); + + it("parses redirects", () => { + expect(parse("foo >out")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("foo")], + redirects: [redirect(">", "out")], + }), + ), + }); + + expect(parse(">out foo")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("foo")], + redirects: [redirect(">", "out")], + }), + ), + }); + }); + + it("parses redirects with file descriptors", () => { + expect(parse("foo 2>out")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("foo")], + redirects: [redirect(">", "out", "2")], + }), + ), + }); + }); +}); + +describe("parse (phase 12: extended redirects)", () => { + it.each([ + { input: "foo >| bar", op: ">|" as RedirOp, target: "bar" }, + { input: "foo >&2", op: ">&" as RedirOp, target: "2" }, + { input: "foo <&3", op: "<&" as RedirOp, target: "3" }, + { input: "foo &>bar", op: "&>" as RedirOp, target: "bar" }, + { input: "foo &>>bar", op: "&>>" as RedirOp, target: "bar" }, + { input: "foo <<bar", op: "<>" as RedirOp, target: "bar" }, + ])("parses $op redirect", ({ input, op, target }) => { + expect(parse(input)).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("foo")], + redirects: [redirect(op, target)], + }), + ), + }); + }); + + it("parses fd dup with explicit fd 2>&1", () => { + expect(parse("foo 2>&1")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [word("foo")], + redirects: [redirect(">&", "1", "2")], + }), + ), + }); + }); +}); diff --git a/src/parser/words-quotes-comments.test.ts b/src/parser/words-quotes-comments.test.ts new file mode 100644 index 0000000..da321ea --- /dev/null +++ b/src/parser/words-quotes-comments.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "../parse"; +import { + dbl, + lit, + program, + sgl, + simple, + stmt, + wordParts, +} from "../test-helpers/ast-builders"; + +describe("parse (phase 2: words, quotes, comments)", () => { + it("ignores full-line and trailing comments", () => { + expect(parse("# foo\nbar")).toEqual({ + ast: program(stmt(simple("bar"))), + }); + + expect(parse("foo # bar")).toEqual({ + ast: program(stmt(simple("foo"))), + }); + }); + + it("keeps # when not at a boundary", () => { + expect(parse("foo#bar")).toEqual({ + ast: program(stmt(simple("foo#bar"))), + }); + }); + + it("parses single-quoted parts", () => { + expect(parse("foo'bar'")).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [wordParts(lit("foo"), sgl("bar"))], + }), + ), + }); + }); + + it("parses double-quoted parts", () => { + expect(parse('"foo bar"')).toEqual({ + ast: program( + stmt({ + type: "SimpleCommand", + words: [wordParts(dbl(lit("foo bar")))], + }), + ), + }); + }); + + it("treats backslash-newline as whitespace", () => { + expect(parse("foo \\\n bar")).toEqual({ + ast: program(stmt(simple("foo", "bar"))), + }); + }); +}); diff --git a/src/test-helpers/ast-builders.ts b/src/test-helpers/ast-builders.ts new file mode 100644 index 0000000..fc7ff00 --- /dev/null +++ b/src/test-helpers/ast-builders.ts @@ -0,0 +1,226 @@ +import type { + ArithCmd, + ArithExp, + ArrayElem, + ArrayExpr, + Assignment, + Block, + CaseClause, + CaseItem, + CmdSubst, + Command, + CoprocClause, + CStyleLoop, + DblQuoted, + DeclClause, + ForClause, + FunctionDecl, + IfClause, + LetClause, + Literal, + ParamExp, + Program, + Redirect, + RedirOp, + SelectClause, + SglQuoted, + SimpleCommand, + Statement, + Subshell, + TestClause, + TimeClause, + WhileClause, + Word, + WordPart, +} from "../ast"; + +export const lit = (value: string): Literal => ({ type: "Literal", value }); +export const sgl = (value: string): SglQuoted => ({ type: "SglQuoted", value }); +export const dbl = (...parts: WordPart[]): DblQuoted => ({ + type: "DblQuoted", + parts, +}); +export const paramExp = ( + name: string, + short = true, + op?: string, + value?: string, +): ParamExp => { + const p: ParamExp = { + type: "ParamExp", + short, + param: lit(name), + }; + if (op !== undefined) p.op = op; + if (value !== undefined) p.value = { type: "Word", parts: [lit(value)] }; + return p; +}; +export const cmdSubst = (...stmts: Statement[]): CmdSubst => ({ + type: "CmdSubst", + stmts, +}); +export const arithExp = (expr: string): ArithExp => ({ + type: "ArithExp", + expr, +}); +export const word = (value: string): Word => ({ + type: "Word", + parts: [lit(value)], +}); +export const wordParts = (...parts: WordPart[]): Word => ({ + type: "Word", + parts, +}); +export const simple = (...words: string[]): SimpleCommand => ({ + type: "SimpleCommand", + words: words.map(word), +}); +export const assign = ( + name: string, + value?: string, + opts?: { append?: boolean; array?: ArrayExpr }, +): Assignment => { + const a: Assignment = { type: "Assignment", name }; + if (opts?.append) a.append = true; + if (value !== undefined) a.value = word(value); + if (opts?.array) a.array = opts.array; + return a; +}; +export const arrayExpr = (...elems: ArrayElem[]): ArrayExpr => ({ + type: "ArrayExpr", + elems, +}); +export const arrayElem = (value?: string, index?: string): ArrayElem => { + const e: ArrayElem = { type: "ArrayElem" }; + if (value !== undefined) e.value = word(value); + if (index !== undefined) e.index = word(index); + return e; +}; +export const declClause = ( + variant: DeclClause["variant"], + opts?: { + args?: Word[]; + assigns?: Assignment[]; + redirects?: Redirect[]; + }, +): DeclClause => { + const d: DeclClause = { type: "DeclClause", variant }; + if (opts?.args) d.args = opts.args; + if (opts?.assigns) d.assigns = opts.assigns; + if (opts?.redirects) d.redirects = opts.redirects; + return d; +}; +export const letClause = (exprs: Word[], redirects?: Redirect[]): LetClause => { + const l: LetClause = { type: "LetClause", exprs }; + if (redirects) l.redirects = redirects; + return l; +}; +export const cStyleLoop = ( + body: Statement[], + init?: string, + cond?: string, + post?: string, +): CStyleLoop => { + const c: CStyleLoop = { type: "CStyleLoop", body }; + if (init !== undefined) c.init = init; + if (cond !== undefined) c.cond = cond; + if (post !== undefined) c.post = post; + return c; +}; +export const redirect = (op: RedirOp, target: string, fd?: string): Redirect => + fd === undefined + ? { type: "Redirect", op, target: word(target) } + : { type: "Redirect", op, target: word(target), fd }; +export const subshell = (...body: Statement[]): Subshell => ({ + type: "Subshell", + body, +}); +export const block = (...body: Statement[]): Block => ({ + type: "Block", + body, +}); +export const ifClause = ( + cond: Statement[], + then: Statement[], + elseBranch?: Statement[], +): IfClause => + elseBranch + ? { type: "IfClause", cond, then, else: elseBranch } + : { type: "IfClause", cond, then }; +export const whileClause = ( + cond: Statement[], + body: Statement[], + until?: boolean, +): WhileClause => + until + ? { type: "WhileClause", cond, body, until } + : { type: "WhileClause", cond, body }; +export const forClause = ( + name: string, + body: Statement[], + items?: Word[], +): ForClause => + items + ? { type: "ForClause", name, items, body } + : { type: "ForClause", name, body }; +export const selectClause = ( + name: string, + body: Statement[], + items?: Word[], +): SelectClause => + items + ? { type: "SelectClause", name, items, body } + : { type: "SelectClause", name, body }; +export const functionDecl = ( + name: string, + body: Statement[], +): FunctionDecl => ({ + type: "FunctionDecl", + name, + body, +}); +export const caseItem = (patterns: Word[], body: Statement[]): CaseItem => ({ + type: "CaseItem", + patterns, + body, +}); +export const caseClause = ( + wordValue: string, + items: CaseItem[], +): CaseClause => ({ + type: "CaseClause", + word: word(wordValue), + items, +}); +export const testClause = (...words: Word[]): TestClause => ({ + type: "TestClause", + expr: words, +}); +export const arithCmd = (expr: string): ArithCmd => ({ + type: "ArithCmd", + expr, +}); +export const coprocClause = (body: Statement, name?: string): CoprocClause => + name ? { type: "CoprocClause", name, body } : { type: "CoprocClause", body }; +export const timeClause = (command: Statement): TimeClause => ({ + type: "TimeClause", + command, +}); +export const stmt = ( + command: Command, + background = false, + negated = false, +): Statement => { + const value: Statement = { type: "Statement", command }; + if (background) { + value.background = true; + } + if (negated) { + value.negated = true; + } + return value; +}; +export const program = (...body: Statement[]): Program => ({ + type: "Program", + body, +}); diff --git a/src/test-helpers/extract-command-names.ts b/src/test-helpers/extract-command-names.ts new file mode 100644 index 0000000..9e6d982 --- /dev/null +++ b/src/test-helpers/extract-command-names.ts @@ -0,0 +1,38 @@ +/** + * Extract all command names (first word of each SimpleCommand) from + * a parsed program, recursively walking the AST. + */ +export function extractCommandNames(node: unknown): string[] { + if (!node || typeof node !== "object") return []; + const n = node as Record; + const names: string[] = []; + + if ( + n.type === "SimpleCommand" && + Array.isArray(n.words) && + n.words.length > 0 + ) { + const firstWord = n.words[0] as + | { parts: Array<{ type: string; value?: string }> } + | undefined; + // Only extract if first word is a plain literal (no expansions) + if ( + firstWord && + firstWord.parts.length === 1 && + firstWord.parts[0]?.type === "Literal" + ) { + names.push(firstWord.parts[0].value as string); + } + } + + for (const val of Object.values(n)) { + if (Array.isArray(val)) { + for (const item of val) { + names.push(...extractCommandNames(item)); + } + } else if (val && typeof val === "object") { + names.push(...extractCommandNames(val)); + } + } + return names; +} diff --git a/src/tokenizer/charsets.ts b/src/tokenizer/charsets.ts new file mode 100644 index 0000000..f3e76ba --- /dev/null +++ b/src/tokenizer/charsets.ts @@ -0,0 +1,16 @@ +export const operatorChars = new Set([";", "|", "&"]); +export const redirChars = new Set([">", "<"]); +export const symbolChars = new Set(["(", ")", "{", "}"]); + +export const isDigit = (value: string) => value >= "0" && value <= "9"; + +export const isNameChar = (c: string) => + (c >= "a" && c <= "z") || + (c >= "A" && c <= "Z") || + (c >= "0" && c <= "9") || + c === "_"; + +export const isNameStart = (c: string) => + (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_"; + +export const specialParams = new Set(["@", "*", "#", "?", "-", "$", "!"]); diff --git a/src/tokenizer/index.ts b/src/tokenizer/index.ts new file mode 100644 index 0000000..7040ea8 --- /dev/null +++ b/src/tokenizer/index.ts @@ -0,0 +1,8 @@ +export { tokenize } from "./tokenize"; +export type { + OpTokenValue, + SymbolTokenValue, + Token, + TokenWordPart, +} from "./types"; +export { tokenPartsText } from "./utils"; diff --git a/src/tokenizer/scan-backtick.ts b/src/tokenizer/scan-backtick.ts new file mode 100644 index 0000000..ea864cc --- /dev/null +++ b/src/tokenizer/scan-backtick.ts @@ -0,0 +1,14 @@ +import type { TokenWordPart } from "./types"; + +export function scanBacktick( + source: string, + pos: number, +): { part: TokenWordPart; end: number } { + let j = pos + 1; + while (j < source.length && source.charAt(j) !== "`") { + if (source.charAt(j) === "\\") j++; + j++; + } + const raw = source.slice(pos + 1, j); + return { part: { type: "backtick", raw }, end: j + 1 }; +} diff --git a/src/tokenizer/scan-expansion.ts b/src/tokenizer/scan-expansion.ts new file mode 100644 index 0000000..0b07234 --- /dev/null +++ b/src/tokenizer/scan-expansion.ts @@ -0,0 +1,115 @@ +import { isDigit, isNameChar, isNameStart, specialParams } from "./charsets"; +import type { TokenWordPart } from "./types"; + +export function scanExpansion( + source: string, + pos: number, +): { part: TokenWordPart; end: number } | null { + if (source.charAt(pos) !== "$") return null; + const next = source.charAt(pos + 1); + + // $((expr)) arithmetic expansion + if (next === "(" && source.charAt(pos + 2) === "(") { + let j = pos + 3; + let depth = 0; + while (j < source.length) { + if ( + source.charAt(j) === ")" && + source.charAt(j + 1) === ")" && + depth === 0 + ) + break; + if (source.charAt(j) === "(") depth++; + if (source.charAt(j) === ")") depth--; + j++; + } + const expr = source.slice(pos + 3, j).trim(); + return { part: { type: "arith-exp", raw: expr }, end: j + 2 }; + } + + // $(cmd) command substitution + if (next === "(") { + let j = pos + 2; + let depth = 1; + while (j < source.length && depth > 0) { + if (source.charAt(j) === "(") depth++; + if (source.charAt(j) === ")") depth--; + j++; + } + const raw = source.slice(pos + 2, j - 1); + return { part: { type: "cmd-subst", raw }, end: j }; + } + + // ${...} braced parameter expansion + if (next === "{") { + let j = pos + 2; + let depth = 1; + while (j < source.length && depth > 0) { + if (source.charAt(j) === "{") depth++; + if (source.charAt(j) === "}") depth--; + j++; + } + const inner = source.slice(pos + 2, j - 1); + // Parse inner: name then optional op and value + let nameEnd = 0; + // Skip optional ! # prefix + let prefix = ""; + if (inner.charAt(0) === "!" || inner.charAt(0) === "#") { + prefix = inner.charAt(0); + nameEnd = 1; + } + while (nameEnd < inner.length && isNameChar(inner.charAt(nameEnd))) { + nameEnd++; + } + const name = inner.slice(prefix ? 1 : 0, nameEnd); + const rest = inner.slice(nameEnd); + if (rest.length > 0) { + // Try to extract operator and value + const opMatch = rest.match( + /^(:-|:=|:\+|:\?|-|\+|=|\?|##|%%|#|%|\/\/|\/)/, + ); + if (opMatch) { + const op = opMatch[0]; + const value = rest.slice(op.length); + return { + part: { type: "param", name, braced: true, op, value }, + end: j, + }; + } + // Slice or other complex syntax - store as name with rest + return { + part: { + type: "param", + name: inner, + braced: true, + }, + end: j, + }; + } + return { part: { type: "param", name, braced: true }, end: j }; + } + + // $name or $N or $@ etc + if (isNameStart(next)) { + let j = pos + 2; + while (j < source.length && isNameChar(source.charAt(j))) { + j++; + } + const name = source.slice(pos + 1, j); + return { part: { type: "param", name, braced: false }, end: j }; + } + if (isDigit(next)) { + return { + part: { type: "param", name: next, braced: false }, + end: pos + 2, + }; + } + if (specialParams.has(next)) { + return { + part: { type: "param", name: next, braced: false }, + end: pos + 2, + }; + } + + return null; +} diff --git a/src/tokenizer/scan-redir.ts b/src/tokenizer/scan-redir.ts new file mode 100644 index 0000000..305de39 --- /dev/null +++ b/src/tokenizer/scan-redir.ts @@ -0,0 +1,20 @@ +import type { RedirOp } from "../ast"; + +export function tryRedirOp( + source: string, + pos: number, +): { op: RedirOp; len: number } | null { + if (source.startsWith("<<<", pos)) return { op: "<<<", len: 3 }; + if (source.startsWith("&>>", pos)) return { op: "&>>", len: 3 }; + if (source.startsWith("<<-", pos)) return { op: "<<-", len: 3 }; + if (source.startsWith(">>", pos)) return { op: ">>", len: 2 }; + if (source.startsWith(">&", pos)) return { op: ">&", len: 2 }; + if (source.startsWith(">|", pos)) return { op: ">|", len: 2 }; + if (source.startsWith("<>", pos)) return { op: "<>", len: 2 }; + if (source.startsWith("<&", pos)) return { op: "<&", len: 2 }; + if (source.startsWith("&>", pos)) return { op: "&>", len: 2 }; + if (source.startsWith("<<", pos)) return { op: "<<", len: 2 }; + if (source.charAt(pos) === ">") return { op: ">", len: 1 }; + if (source.charAt(pos) === "<") return { op: "<", len: 1 }; + return null; +} diff --git a/src/tokenizer/tokenize.ts b/src/tokenizer/tokenize.ts new file mode 100644 index 0000000..64aadf1 --- /dev/null +++ b/src/tokenizer/tokenize.ts @@ -0,0 +1,344 @@ +import type { ParseOptions } from "../ast"; +import { isDigit, operatorChars, redirChars, symbolChars } from "./charsets"; +import { scanBacktick } from "./scan-backtick"; +import { scanExpansion } from "./scan-expansion"; +import { tryRedirOp } from "./scan-redir"; +import type { SymbolTokenValue, Token, TokenWordPart } from "./types"; +import { tokenPartsText } from "./utils"; + +export function tokenize(source: string, options: ParseOptions = {}): Token[] { + const tokens: Token[] = []; + let i = 0; + let atBoundary = true; + + while (i < source.length) { + const ch = source.charAt(i); + + if (ch === " " || ch === "\t" || ch === "\r") { + atBoundary = true; + i += 1; + continue; + } + + if (ch === "\\" && source.charAt(i + 1) === "\n") { + atBoundary = true; + i += 2; + continue; + } + + if (ch === "\\" && source.charAt(i + 1) === "\r") { + if (source.charAt(i + 2) === "\n") { + atBoundary = true; + i += 3; + continue; + } + } + + if (ch === "\n") { + tokens.push({ type: "op", value: ";" }); + atBoundary = true; + i += 1; + + // Check for pending heredocs by scanning recent tokens + const pendingHeredocs: { strip: boolean; delimiter: string }[] = []; + for (let ti = 0; ti < tokens.length; ti++) { + const t = tokens[ti]; + if ( + t && + t.type === "redir" && + (t.op === "<<" || t.op === "<<-") && + !Object.hasOwn(t, "_collected") + ) { + const delimTok = tokens[ti + 1]; + if (delimTok && delimTok.type === "word") { + pendingHeredocs.push({ + strip: t.op === "<<-", + delimiter: tokenPartsText(delimTok.parts), + }); + (t as Record)._collected = true; + } + } + } + + // Collect heredoc bodies + for (const hd of pendingHeredocs) { + let body = ""; + while (i < source.length) { + let lineEnd = source.indexOf("\n", i); + if (lineEnd === -1) lineEnd = source.length; + const line = source.slice(i, lineEnd); + const checkLine = hd.strip ? line.replace(/^\t+/, "") : line; + i = lineEnd < source.length ? lineEnd + 1 : lineEnd; + if (checkLine === hd.delimiter) break; + const processedLine = hd.strip ? line.replace(/^\t+/, "") : line; + body += `${processedLine}\n`; + } + tokens.push({ type: "heredoc-body", content: body }); + } + + continue; + } + + if (ch === "#" && atBoundary) { + const start = i + 1; + i += 1; + while (i < source.length && source.charAt(i) !== "\n") { + i += 1; + } + if (options.keepComments) { + tokens.push({ type: "comment", text: source.slice(start, i) }); + } + continue; + } + + if (ch === "!" && atBoundary) { + tokens.push({ type: "op", value: "!" }); + atBoundary = true; + i += 1; + continue; + } + + if (isDigit(ch)) { + let j = i; + while (j < source.length && isDigit(source.charAt(j))) { + j += 1; + } + const redir = tryRedirOp(source, j); + if (redir) { + tokens.push({ type: "redir", op: redir.op, fd: source.slice(i, j) }); + i = j + redir.len; + atBoundary = true; + continue; + } + } + + if (ch === "(" && source.charAt(i + 1) === "(" && atBoundary) { + let j = i + 2; + let depth = 0; + while (j < source.length) { + const c = source.charAt(j); + if (c === ")" && source.charAt(j + 1) === ")" && depth === 0) break; + if (c === "(") depth++; + if (c === ")") depth--; + j++; + } + tokens.push({ type: "arith-cmd", expr: source.slice(i + 2, j).trim() }); + i = j + 2; + atBoundary = true; + continue; + } + + if ( + (ch === "<" || ch === ">") && + source.charAt(i + 1) === "(" && + atBoundary + ) { + const op = ch as "<" | ">"; + let j = i + 2; + let depth = 1; + while (j < source.length && depth > 0) { + if (source.charAt(j) === "(") depth++; + if (source.charAt(j) === ")") depth--; + j++; + } + const raw = source.slice(i + 2, j - 1); + tokens.push({ + type: "word", + parts: [{ type: "proc-subst", op, raw }], + }); + i = j; + atBoundary = false; + continue; + } + + { + const redir = tryRedirOp(source, i); + if (redir) { + tokens.push({ type: "redir", op: redir.op }); + i += redir.len; + atBoundary = true; + continue; + } + } + + if (symbolChars.has(ch)) { + tokens.push({ type: "symbol", value: ch as SymbolTokenValue }); + atBoundary = true; + i += 1; + continue; + } + + if (source.startsWith("&&", i)) { + tokens.push({ type: "op", value: "&&" }); + atBoundary = true; + i += 2; + continue; + } + + if (source.startsWith("||", i)) { + tokens.push({ type: "op", value: "||" }); + atBoundary = true; + i += 2; + continue; + } + + if (operatorChars.has(ch)) { + tokens.push({ type: "op", value: ch as ";" | "|" | "&" }); + atBoundary = true; + i += 1; + continue; + } + + const parts: TokenWordPart[] = []; + let current = ""; + + const flushLit = () => { + if (current.length > 0) { + parts.push({ type: "lit", value: current }); + current = ""; + } + }; + + while (i < source.length) { + const currentChar = source.charAt(i); + + if (currentChar === "\\" && source.charAt(i + 1) === "\n") { + i += 2; + continue; + } + + if (currentChar === "\\" && source.charAt(i + 1) === "\r") { + if (source.charAt(i + 2) === "\n") { + i += 3; + continue; + } + } + + if ( + currentChar === " " || + currentChar === "\t" || + currentChar === "\r" || + currentChar === "\n" || + operatorChars.has(currentChar) || + redirChars.has(currentChar) || + symbolChars.has(currentChar) + ) { + break; + } + + if (currentChar === "'") { + flushLit(); + i += 1; + const start = i; + while (i < source.length && source.charAt(i) !== "'") { + i += 1; + } + if (i >= source.length) { + throw new Error("Unclosed single quote"); + } + parts.push({ type: "sgl", value: source.slice(start, i) }); + i += 1; + continue; + } + + if (currentChar === '"') { + flushLit(); + i += 1; + const dblParts: TokenWordPart[] = []; + let dblBuf = ""; + const flushDblLit = () => { + if (dblBuf.length > 0) { + dblParts.push({ type: "lit", value: dblBuf }); + dblBuf = ""; + } + }; + let closed = false; + while (i < source.length) { + const dqChar = source.charAt(i); + if (dqChar === "\\" && source.charAt(i + 1) === "\n") { + i += 2; + continue; + } + if (dqChar === "\\" && source.charAt(i + 1) === "\r") { + if (source.charAt(i + 2) === "\n") { + i += 3; + continue; + } + } + if (dqChar === "\\" && i + 1 < source.length) { + dblBuf += dqChar + source.charAt(i + 1); + i += 2; + continue; + } + if (dqChar === "$") { + flushDblLit(); + const exp = scanExpansion(source, i); + if (exp) { + dblParts.push(exp.part); + i = exp.end; + continue; + } + dblBuf += dqChar; + i += 1; + continue; + } + if (dqChar === "`") { + flushDblLit(); + const bt = scanBacktick(source, i); + dblParts.push(bt.part); + i = bt.end; + continue; + } + if (dqChar === '"') { + i += 1; + closed = true; + break; + } + dblBuf += dqChar; + i += 1; + } + if (!closed) { + throw new Error("Unclosed double quote"); + } + flushDblLit(); + parts.push({ type: "dbl", parts: dblParts }); + continue; + } + + if (currentChar === "$") { + flushLit(); + const exp = scanExpansion(source, i); + if (exp) { + parts.push(exp.part); + i = exp.end; + continue; + } + current += currentChar; + i += 1; + continue; + } + + if (currentChar === "`") { + flushLit(); + const bt = scanBacktick(source, i); + parts.push(bt.part); + i = bt.end; + continue; + } + + current += currentChar; + i += 1; + } + + flushLit(); + + if (parts.length === 0) { + throw new Error("Unexpected character"); + } + + tokens.push({ type: "word", parts }); + atBoundary = false; + } + + return tokens; +} diff --git a/src/tokenizer/types.ts b/src/tokenizer/types.ts new file mode 100644 index 0000000..e2bdc64 --- /dev/null +++ b/src/tokenizer/types.ts @@ -0,0 +1,30 @@ +import type { RedirOp } from "../ast"; + +export type OpTokenValue = "&&" | "||" | "|" | ";" | "&" | "!"; + +export type SymbolTokenValue = "(" | ")" | "{" | "}"; + +export type TokenWordPart = + | { type: "lit"; value: string } + | { type: "sgl"; value: string } + | { type: "dbl"; parts: TokenWordPart[] } + | { + type: "param"; + name: string; + braced: boolean; + op?: string; + value?: string; + } + | { type: "cmd-subst"; raw: string } + | { type: "arith-exp"; raw: string } + | { type: "proc-subst"; op: "<" | ">"; raw: string } + | { type: "backtick"; raw: string }; + +export type Token = + | { type: "word"; parts: TokenWordPart[] } + | { type: "op"; value: OpTokenValue } + | { type: "redir"; op: RedirOp; fd?: string } + | { type: "symbol"; value: SymbolTokenValue } + | { type: "arith-cmd"; expr: string } + | { type: "heredoc-body"; content: string } + | { type: "comment"; text: string }; diff --git a/src/tokenizer/utils.ts b/src/tokenizer/utils.ts new file mode 100644 index 0000000..2513de5 --- /dev/null +++ b/src/tokenizer/utils.ts @@ -0,0 +1,15 @@ +import type { TokenWordPart } from "./types"; + +export function tokenPartsText(parts: TokenWordPart[]): string { + return parts + .map((p) => { + if (p.type === "lit") return p.value; + if (p.type === "sgl") return p.value; + if (p.type === "dbl") + return p.parts + .map((dp) => (dp.type === "lit" ? dp.value : "")) + .join(""); + return ""; + }) + .join(""); +} diff --git a/tsconfig.json b/tsconfig.json index 8b202aa..9c901ca 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,8 +3,8 @@ "target": "ES2022", "lib": ["ES2022"], - "module": "NodeNext", - "moduleResolution": "NodeNext", + "module": "Preserve", + "moduleResolution": "Bundler", "verbatimModuleSyntax": true, "strict": true,