From ea31e4bc642f64fedf46ed8e3bc64af4f51ac474 Mon Sep 17 00:00:00 2001 From: Goldfinch Date: Fri, 17 Jul 2026 21:47:31 -0600 Subject: [PATCH] Parser: statement framework, error recovery, full SELECT coverage Recursive descent with keyword dispatch and Pratt expressions on the Postgres precedence table (::, subscripts, COLLATE, AT TIME ZONE, BETWEEN [SYMMETRIC], IS [NOT] DISTINCT FROM, custom operators, CASE, ARRAY[...], row constructors, EXISTS/IN/ANY/ALL, CAST, typed literals). SELECT grammar covers WITH [RECURSIVE] with MATERIALIZED and SEARCH/CYCLE, all join forms with LATERAL/USING/NATURAL, DISTINCT [ON], GROUP BY with GROUPING SETS/ROLLUP/CUBE, FILTER/WITHIN GROUP, named windows with full frame specs, set ops, and ORDER/LIMIT/OFFSET/FETCH/ FOR locking tails (LIMIT and OFFSET accepted in either order, as PG does). Statement framework: failed statements roll back into ErrorStatement nodes holding their raw tokens verbatim plus a spanned diagnostic; recovery resumes at top-level semicolons (SQLite trigger BEGIN...END bodies are not split; dollar-quoted bodies are already single tokens). Ambiguous '((SELECT' is resolved by speculative parse + backtrack. corpus-report now tracks statement-level coverage: 410/410 SELECT-ish statements in the corpus parse; parse round-trip is asserted corpus-wide in CI; new parse_roundtrip fuzz target. TREE-95 Co-Authored-By: Claude Fable 5 --- crates/cli/src/bin/corpus_report.rs | 169 +++- crates/parser/fuzz/Cargo.toml | 7 + .../fuzz/fuzz_targets/parse_roundtrip.rs | 17 + crates/parser/src/parser.rs | 327 +++++++- crates/parser/src/parser/expr.rs | 473 +++++++++++ crates/parser/src/parser/grammar.rs | 768 ++++++++++++++++++ crates/parser/syntax.def | 83 +- crates/parser/tests/corpus.rs | 88 ++ crates/parser/tests/parser.rs | 267 ++++++ 9 files changed, 2154 insertions(+), 45 deletions(-) create mode 100644 crates/parser/fuzz/fuzz_targets/parse_roundtrip.rs create mode 100644 crates/parser/src/parser/expr.rs create mode 100644 crates/parser/src/parser/grammar.rs create mode 100644 crates/parser/tests/parser.rs diff --git a/crates/cli/src/bin/corpus_report.rs b/crates/cli/src/bin/corpus_report.rs index f14c8f5..2e1da0c 100644 --- a/crates/cli/src/bin/corpus_report.rs +++ b/crates/cli/src/bin/corpus_report.rs @@ -1,8 +1,9 @@ //! Corpus coverage harness. //! //! Runs every `.sql` file under `corpus/` through lex -> parse -> emit and -//! reports pass/fail per file and per pipeline stage. Later issues use this -//! report to measure grammar coverage. +//! reports pass/fail per file and per pipeline stage, plus statement-level +//! parse coverage (SELECT-ish statements tracked separately until DML/DDL +//! land with TREE-98). //! //! Usage: `corpus-report [--summary] [CORPUS_DIR]` //! @@ -17,35 +18,131 @@ use parser::syntax::SyntaxKind; /// Pipeline stages, in order. const STAGES: [&str; 3] = ["lex", "parse", "emit"]; -/// Run one file through the pipeline. Returns the number of stages passed -/// (0..=3) and the error message of the first failing stage, if any. +#[derive(Default)] +struct StmtStats { + total: usize, + ok: usize, + select_total: usize, + select_ok: usize, +} + +struct FileResult { + stages_passed: usize, + error: Option, + stmts: StmtStats, +} + +/// Run one file through the pipeline. /// -/// The lex stage passes when the token stream round-trips byte-for-byte -/// (the TREE-93 lossless property) and contains no error tokens. -fn run_file(source: &str) -> (usize, Option) { +/// Lex passes when the token stream round-trips byte-for-byte with no +/// error tokens; parse passes when no statement lands as ErrorStatement; +/// emit passes when the formatter renders the tree. +fn run_file(source: &str, show_diagnostics: bool) -> FileResult { + let fail = |stages_passed: usize, error: String| FileResult { + stages_passed, + error: Some(error), + stmts: StmtStats::default(), + }; + let tokens = parser::lexer::lex(source, Dialect::Postgres); let rebuilt: String = tokens.iter().map(|t| t.text).collect(); if rebuilt != source { - return (0, Some("token texts do not round-trip to the input".into())); + return fail(0, "token texts do not round-trip to the input".into()); } - let error_count = tokens - .iter() - .filter(|t| t.kind == SyntaxKind::Error) - .count(); if let Some(first) = tokens.iter().find(|t| t.kind == SyntaxKind::Error) { let snippet: String = first.text.chars().take(20).collect(); - return ( - 0, - Some(format!("{error_count} error token(s), first: {snippet:?}")), - ); + return fail(0, format!("error token: {snippet:?}")); } - let cst = match parser::parser::parse(&tokens) { - Ok(cst) => cst, - Err(err) => return (1, Some(err.to_string())), - }; - match formatter::emit(&cst) { - Ok(_) => (3, None), - Err(err) => (2, Some(err.to_string())), + + let parse = parser::parser::parse(&tokens, Dialect::Postgres); + if parse.cst.text() != source { + return fail(1, "parse tree does not round-trip to the input".into()); + } + let mut stmts = StmtStats::default(); + for child in parse.cst.root().children() { + match child.kind() { + SyntaxKind::ErrorStatement => { + stmts.total += 1; + let tokens: Vec<_> = child + .children_with_tokens() + .filter_map(|element| element.into_token()) + .filter(|token| !token.kind().is_trivia()) + .collect(); + let starts_selectish = tokens.first().is_some_and(|token| { + token.kind() == SyntaxKind::LParen + || ["select", "with", "values", "table"] + .iter() + .any(|kw| token.text().eq_ignore_ascii_case(kw)) + }); + // Statements that involve DML — top-level (`WITH ... + // UPDATE`) or in a data-modifying CTE (`AS (UPDATE ...)`) + // — are TREE-98's problem, not SELECT failures. Careful + // not to trip on `FOR [NO KEY] UPDATE` locking clauses. + let mut depth = 0i32; + let mut prev = String::new(); + let mut has_dml = false; + for token in &tokens { + if token.kind() == SyntaxKind::LParen { + depth += 1; + } else if token.kind() == SyntaxKind::RParen { + depth -= 1; + } else if token.kind() == SyntaxKind::Ident { + let text = token.text().to_ascii_lowercase(); + let is_dml_kw = + ["insert", "update", "delete", "merge"].contains(&text.as_str()); + let after_lock_kws = ["for", "key", "no"].contains(&prev.as_str()); + if is_dml_kw && (prev == "(" || (depth == 0 && !after_lock_kws)) { + has_dml = true; + break; + } + } + prev = token.text().to_ascii_lowercase(); + } + if starts_selectish && !has_dml { + stmts.select_total += 1; + if show_diagnostics { + let snippet: String = child + .to_string() + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .take(120) + .collect(); + println!(" select-ish failure: {snippet}"); + } + } + } + SyntaxKind::EmptyStmt => {} + _ => { + stmts.total += 1; + stmts.ok += 1; + stmts.select_total += 1; + stmts.select_ok += 1; + } + } + } + if stmts.ok < stmts.total { + let first = parse.diagnostics.first().expect("diagnostic per error"); + return FileResult { + stages_passed: 1, + error: Some(format!( + "{}/{} statements failed; first at {}..{}: {}", + stmts.total - stmts.ok, + stmts.total, + first.start, + first.end, + first.message + )), + stmts, + }; + } + + let emit_error = formatter::emit(&parse.cst).err(); + FileResult { + stages_passed: if emit_error.is_none() { 3 } else { 2 }, + error: emit_error.map(|e| e.to_string()), + stmts, } } @@ -64,10 +161,12 @@ fn collect_sql_files(dir: &Path, out: &mut Vec) -> std::io::Result<()> fn main() -> ExitCode { let mut summary_only = false; + let mut show_diagnostics = false; let mut root = PathBuf::from("corpus"); for arg in std::env::args().skip(1) { match arg.as_str() { "--summary" => summary_only = true, + "--diagnostics" => show_diagnostics = true, other => root = PathBuf::from(other), } } @@ -88,6 +187,7 @@ fn main() -> ExitCode { // stage_passes[i] counts files that passed stage i. let mut stage_passes = [0usize; STAGES.len()]; + let mut totals = StmtStats::default(); let mut unreadable = 0usize; for path in &files { let source = match std::fs::read_to_string(path) { @@ -100,16 +200,22 @@ fn main() -> ExitCode { continue; } }; - let (passed, error) = run_file(&source); - for count in stage_passes.iter_mut().take(passed) { + let result = run_file(&source, show_diagnostics); + for count in stage_passes.iter_mut().take(result.stages_passed) { *count += 1; } + totals.total += result.stmts.total; + totals.ok += result.stmts.ok; + totals.select_total += result.stmts.select_total; + totals.select_ok += result.stmts.select_ok; if !summary_only { - match error { + match result.error { None => println!("ok {}", path.display()), - Some(msg) => { - println!("fail:{:<6} {} ({msg})", STAGES[passed], path.display()); - } + Some(msg) => println!( + "fail:{:<6} {} ({msg})", + STAGES[result.stages_passed], + path.display() + ), } } } @@ -120,6 +226,11 @@ fn main() -> ExitCode { let pct = 100.0 * passes as f64 / total as f64; println!("{stage:<6} {passes:>5}/{total} ({pct:.1}%)"); } + println!("statements {:>5}/{} parsed", totals.ok, totals.total); + println!( + "select-ish {:>5}/{} parsed", + totals.select_ok, totals.select_total + ); if unreadable > 0 { println!("unreadable: {unreadable}"); } diff --git a/crates/parser/fuzz/Cargo.toml b/crates/parser/fuzz/Cargo.toml index ccd3862..7025ed0 100644 --- a/crates/parser/fuzz/Cargo.toml +++ b/crates/parser/fuzz/Cargo.toml @@ -18,5 +18,12 @@ test = false doc = false bench = false +[[bin]] +name = "parse_roundtrip" +path = "fuzz_targets/parse_roundtrip.rs" +test = false +doc = false +bench = false + [workspace] members = ["."] diff --git a/crates/parser/fuzz/fuzz_targets/parse_roundtrip.rs b/crates/parser/fuzz/fuzz_targets/parse_roundtrip.rs new file mode 100644 index 0000000..b7cb09c --- /dev/null +++ b/crates/parser/fuzz/fuzz_targets/parse_roundtrip.rs @@ -0,0 +1,17 @@ +//! TREE-95 acceptance: for arbitrary input, parsing never panics and the +//! resulting tree reproduces the input byte-for-byte, in both dialects. + +#![no_main] + +use libfuzzer_sys::fuzz_target; +use parser::Dialect; + +fuzz_target!(|data: &[u8]| { + if let Ok(input) = std::str::from_utf8(data) { + for dialect in [Dialect::Postgres, Dialect::Sqlite] { + let tokens = parser::lexer::lex(input, dialect); + let parse = parser::parser::parse(&tokens, dialect); + assert_eq!(parse.cst.text(), input, "{dialect:?} round-trip failed"); + } + } +}); diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 98002ed..f8bb0eb 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -1,28 +1,327 @@ -//! Parser: tokens to a concrete syntax tree. +//! Recursive descent parser: tokens to a concrete syntax tree. +//! +//! Statements are keyword-dispatched; expressions use Pratt parsing with a +//! dialect-parameterized precedence table (see `parser::expr`). The parser +//! emits [`Event`]s over the non-trivia tokens; trivia attachment happens +//! in the tree sink (TREE-94). +//! +//! Error recovery: a statement that fails to parse becomes an +//! `ErrorStatement` node containing its raw tokens verbatim plus a +//! [`Diagnostic`], and parsing resumes at the next top-level `;`. The tree +//! never drops tokens. -use std::fmt; +mod expr; +mod grammar; +use crate::dialect::Dialect; use crate::lexer::Token; +use crate::syntax::SyntaxKind; pub use crate::tree::Cst; +use crate::tree::{Event, build_tree}; -/// Errors produced while parsing. +/// A parse problem, with byte offsets into the source. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum ParseError { - /// The parser is not implemented yet. - Unimplemented, +pub struct Diagnostic { + pub message: String, + pub start: usize, + pub end: usize, } -impl fmt::Display for ParseError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ParseError::Unimplemented => f.write_str("parser not implemented"), +/// The result of parsing: always a full, lossless tree, plus diagnostics +/// for every statement that landed as an `ErrorStatement`. +#[derive(Debug)] +pub struct Parse { + pub cst: Cst, + pub diagnostics: Vec, +} + +/// Parse a lexed token stream (trivia included) into a CST. +pub fn parse(tokens: &[Token<'_>], dialect: Dialect) -> Parse { + let mut toks = Vec::new(); + let mut offset = 0; + for token in tokens { + let end = offset + token.text.len(); + if !token.kind.is_trivia() { + toks.push(Tok { + kind: token.kind, + text: token.text, + start: offset, + end, + }); } + offset = end; + } + let eof = offset; + let mut parser = Parser { + toks, + pos: 0, + events: Vec::new(), + diagnostics: Vec::new(), + dialect, + eof, + }; + parser.events.push(Event::StartNode(SyntaxKind::Root)); + while !parser.at_eof() { + parser.statement(); } + parser.events.push(Event::FinishNode); + Parse { + cst: build_tree(tokens, &parser.events), + diagnostics: parser.diagnostics, + } +} + +/// A non-trivia token with its byte span. +struct Tok<'src> { + kind: SyntaxKind, + text: &'src str, + start: usize, + end: usize, +} + +/// An error inside one statement; triggers rollback to `ErrorStatement`. +pub(crate) struct StmtError { + message: String, + /// Index into `toks` where the error occurred. + at: usize, +} + +pub(crate) type PResult = Result<(), StmtError>; + +pub(crate) struct Parser<'src> { + toks: Vec>, + pos: usize, + events: Vec, + diagnostics: Vec, + dialect: Dialect, + eof: usize, } -impl std::error::Error for ParseError {} +impl Parser<'_> { + // ---- cursor ---- + + pub(crate) fn at_eof(&self) -> bool { + self.pos >= self.toks.len() + } + + pub(crate) fn kind(&self) -> Option { + self.toks.get(self.pos).map(|t| t.kind) + } + + pub(crate) fn nth_kind(&self, n: usize) -> Option { + self.toks.get(self.pos + n).map(|t| t.kind) + } + + pub(crate) fn text(&self) -> &str { + self.toks.get(self.pos).map_or("", |t| t.text) + } + + pub(crate) fn at(&self, kind: SyntaxKind) -> bool { + self.kind() == Some(kind) + } + + pub(crate) fn nth_at(&self, n: usize, kind: SyntaxKind) -> bool { + self.nth_kind(n) == Some(kind) + } + + /// Is the current token the given keyword? Keywords are bare `Ident` + /// tokens compared case-insensitively; quoted identifiers never match. + pub(crate) fn at_kw(&self, kw: &str) -> bool { + self.nth_at_kw(0, kw) + } + + pub(crate) fn nth_at_kw(&self, n: usize, kw: &str) -> bool { + self.toks + .get(self.pos + n) + .is_some_and(|t| t.kind == SyntaxKind::Ident && t.text.eq_ignore_ascii_case(kw)) + } + + pub(crate) fn at_any_kw(&self, kws: &[&str]) -> bool { + kws.iter().any(|kw| self.at_kw(kw)) + } + + /// Is the current token an operator with exactly this text? + pub(crate) fn at_op(&self, op: &str) -> bool { + self.toks + .get(self.pos) + .is_some_and(|t| t.kind == SyntaxKind::Operator && t.text == op) + } + + // ---- events ---- + + pub(crate) fn bump(&mut self) { + debug_assert!(!self.at_eof()); + self.events.push(Event::Token); + self.pos += 1; + } + + pub(crate) fn start(&mut self, kind: SyntaxKind) { + self.events.push(Event::StartNode(kind)); + } + + pub(crate) fn finish(&mut self) { + self.events.push(Event::FinishNode); + } + + /// A position in the event stream that a later `open_at` can + /// retroactively enclose in a new node (Pratt-style left recursion). + pub(crate) fn checkpoint(&self) -> usize { + self.events.len() + } + + /// Retroactively open a node at `checkpoint`, so everything emitted + /// since becomes its first children. The caller parses the rest of the + /// node and closes it with `finish`. Wrapping repeatedly at the same + /// checkpoint nests left-associatively. + pub(crate) fn open_at(&mut self, checkpoint: usize, kind: SyntaxKind) { + self.events.insert(checkpoint, Event::StartNode(kind)); + } + + /// Change the kind of a `StartNode` event already emitted at + /// `checkpoint` (e.g. a `ParenExpr` that turned out to be a `RowExpr`). + pub(crate) fn rewrite_start(&mut self, checkpoint: usize, kind: SyntaxKind) { + debug_assert!(matches!(self.events[checkpoint], Event::StartNode(_))); + self.events[checkpoint] = Event::StartNode(kind); + } + + /// Snapshot for speculative parsing; pair with `backtrack` on failure. + pub(crate) fn state(&self) -> (usize, usize) { + (self.events.len(), self.pos) + } + + pub(crate) fn backtrack(&mut self, state: (usize, usize)) { + self.events.truncate(state.0); + self.pos = state.1; + } + + // ---- eating ---- + + pub(crate) fn eat(&mut self, kind: SyntaxKind) -> bool { + if self.at(kind) { + self.bump(); + true + } else { + false + } + } -/// Parse a token stream into a CST. -pub fn parse(_tokens: &[Token<'_>]) -> Result { - Err(ParseError::Unimplemented) + pub(crate) fn eat_kw(&mut self, kw: &str) -> bool { + if self.at_kw(kw) { + self.bump(); + true + } else { + false + } + } + + /// Eat each keyword in order; all-or-nothing is not checked — callers + /// use this for fixed keyword runs after peeking the first word. + pub(crate) fn expect_kws(&mut self, kws: &[&str]) -> PResult { + for kw in kws { + self.expect_kw(kw)?; + } + Ok(()) + } + + pub(crate) fn expect(&mut self, kind: SyntaxKind, what: &str) -> PResult { + if self.eat(kind) { + Ok(()) + } else { + Err(self.error(&format!("expected {what}"))) + } + } + + pub(crate) fn expect_kw(&mut self, kw: &str) -> PResult { + if self.eat_kw(kw) { + Ok(()) + } else { + Err(self.error(&format!("expected `{}`", kw.to_uppercase()))) + } + } + + pub(crate) fn error(&self, message: &str) -> StmtError { + let found = match self.toks.get(self.pos) { + Some(t) => format!("`{}`", t.text), + None => "end of input".to_string(), + }; + StmtError { + message: format!("{message}, found {found}"), + at: self.pos, + } + } + + // ---- statements & recovery ---- + + fn statement(&mut self) { + if self.at(SyntaxKind::Semicolon) { + self.start(SyntaxKind::EmptyStmt); + self.bump(); + self.finish(); + return; + } + let events_checkpoint = self.events.len(); + let pos_checkpoint = self.pos; + if let Err(error) = self.statement_inner() { + self.events.truncate(events_checkpoint); + self.pos = pos_checkpoint; + self.error_statement(error); + } + } + + fn statement_inner(&mut self) -> PResult { + if self.at_any_kw(&["select", "with", "values", "table"]) || self.at(SyntaxKind::LParen) { + grammar::select_stmt(self) + } else { + Err(self.error("expected a statement")) + } + } + + /// Emit an `ErrorStatement` holding every token up to and including the + /// next top-level `;`, and record the diagnostic. Top-level means: + /// dollar-quoted bodies are already single tokens, and in SQLite a + /// `BEGIN ... END` trigger body does not end the statement. + fn error_statement(&mut self, error: StmtError) { + let (start, end) = match self.toks.get(error.at) { + Some(t) => (t.start, t.end), + None => (self.eof, self.eof), + }; + self.diagnostics.push(Diagnostic { + message: error.message, + start, + end, + }); + + self.start(SyntaxKind::ErrorStatement); + let mut begin_depth = 0u32; + while !self.at_eof() { + if self.dialect == Dialect::Sqlite { + if self.at_kw("begin") && !self.begin_is_transaction() { + begin_depth += 1; + } else if self.at_kw("end") { + begin_depth = begin_depth.saturating_sub(1); + } + } + let at_semicolon = self.at(SyntaxKind::Semicolon); + self.bump(); + if at_semicolon && begin_depth == 0 { + break; + } + } + self.finish(); + } + + /// Distinguish SQLite `BEGIN [DEFERRED|IMMEDIATE|EXCLUSIVE] + /// [TRANSACTION]` from a trigger body's `BEGIN stmt; ... END`. + fn begin_is_transaction(&self) -> bool { + self.nth_at(1, SyntaxKind::Semicolon) + || self.nth_at_kw(1, "transaction") + || self.nth_at_kw(1, "deferred") + || self.nth_at_kw(1, "immediate") + || self.nth_at_kw(1, "exclusive") + || self.nth_kind(1).is_none() + } + + pub(crate) fn dialect(&self) -> Dialect { + self.dialect + } } diff --git a/crates/parser/src/parser/expr.rs b/crates/parser/src/parser/expr.rs new file mode 100644 index 0000000..0de4e30 --- /dev/null +++ b/crates/parser/src/parser/expr.rs @@ -0,0 +1,473 @@ +//! Pratt expression parsing with the Postgres precedence table +//! (dialect-parameterized where SQLite differs). + +use crate::dialect::Dialect; +use crate::syntax::SyntaxKind; + +use super::grammar::{arg_list, at_subquery_start, ident, paren_expr_list, qualified_name}; +use super::{PResult, Parser}; + +// Binding powers, mirroring Postgres's operator precedence (high binds +// tighter). Left-associative throughout: an operator binds when +// `bp > min_bp`, and its right operand is parsed with `min_bp = bp`. +const BP_CAST: u8 = 200; // :: +const BP_SUBSCRIPT: u8 = 190; // [ ] +const BP_UNARY: u8 = 170; // prefix + - and custom prefix operators +const BP_COLLATE: u8 = 160; +const BP_AT: u8 = 150; // AT TIME ZONE +const BP_EXP: u8 = 140; // ^ +const BP_MUL: u8 = 130; // * / % +const BP_ADD: u8 = 120; // + - +const BP_OTHER: u8 = 110; // any other operator (||, @>, ->, ...) +const BP_RANGE: u8 = 100; // BETWEEN, IN, LIKE, ILIKE, SIMILAR +const BP_CMP: u8 = 90; // < > = <= >= <> != +const BP_IS: u8 = 80; +const BP_NOT: u8 = 70; // prefix NOT +const BP_AND: u8 = 60; +const BP_OR: u8 = 50; + +pub(crate) fn expr(p: &mut Parser<'_>, min_bp: u8) -> PResult { + let checkpoint = p.checkpoint(); + prefix(p)?; + infix_loop(p, checkpoint, min_bp) +} + +fn infix_loop(p: &mut Parser<'_>, checkpoint: usize, min_bp: u8) -> PResult { + loop { + // Postfix: casts and subscripts. + if p.at(SyntaxKind::ColonColon) && BP_CAST > min_bp { + p.open_at(checkpoint, SyntaxKind::CastExpr); + p.bump(); + type_name(p)?; + p.finish(); + continue; + } + if p.at(SyntaxKind::LBracket) && BP_SUBSCRIPT > min_bp { + p.open_at(checkpoint, SyntaxKind::SubscriptExpr); + p.bump(); + if !p.at(SyntaxKind::Colon) && !p.at(SyntaxKind::RBracket) { + expr(p, 0)?; + } + if p.eat(SyntaxKind::Colon) && !p.at(SyntaxKind::RBracket) { + expr(p, 0)?; + } + p.expect(SyntaxKind::RBracket, "`]`")?; + p.finish(); + continue; + } + + // Operator tokens. + if p.at(SyntaxKind::Operator) { + let bp = operator_bp(p.text(), p.dialect()); + if bp > min_bp { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + p.bump(); + expr(p, bp)?; + p.finish(); + continue; + } + break; + } + + // Keyword operators. + if p.at_kw("collate") && BP_COLLATE > min_bp { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + p.bump(); + qualified_name(p)?; + p.finish(); + continue; + } + if p.at_kw("at") && BP_AT > min_bp { + if p.nth_at_kw(1, "time") && p.nth_at_kw(2, "zone") { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + p.bump(); + p.bump(); + p.bump(); + expr(p, BP_AT)?; + p.finish(); + continue; + } + if p.nth_at_kw(1, "local") { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + p.bump(); + p.bump(); + p.finish(); + continue; + } + } + if p.at_kw("is") && BP_IS > min_bp { + p.open_at(checkpoint, SyntaxKind::IsExpr); + p.bump(); + p.eat_kw("not"); + if p.eat_kw("distinct") { + p.expect_kw("from")?; + expr(p, BP_IS)?; + } else if !(p.eat_kw("null") + || p.eat_kw("true") + || p.eat_kw("false") + || p.eat_kw("unknown")) + { + return Err( + p.error("expected `NULL`, `TRUE`, `FALSE`, `UNKNOWN`, or `DISTINCT FROM`") + ); + } + p.finish(); + continue; + } + if (p.at_kw("isnull") || p.at_kw("notnull")) && BP_IS > min_bp { + p.open_at(checkpoint, SyntaxKind::IsExpr); + p.bump(); + p.finish(); + continue; + } + if p.at_kw("and") && BP_AND > min_bp { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + p.bump(); + expr(p, BP_AND)?; + p.finish(); + continue; + } + if p.at_kw("or") && BP_OR > min_bp { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + p.bump(); + expr(p, BP_OR)?; + p.finish(); + continue; + } + + // [NOT] BETWEEN / IN / LIKE / ILIKE / SIMILAR TO + let (negated, range_kw) = if p.at_kw("not") { + (true, 1) + } else { + (false, 0) + }; + let is_range = ["between", "in", "like", "ilike", "similar"] + .iter() + .any(|kw| p.nth_at_kw(range_kw, kw)); + if is_range && BP_RANGE > min_bp { + if p.nth_at_kw(range_kw, "between") { + p.open_at(checkpoint, SyntaxKind::BetweenExpr); + if negated { + p.bump(); + } + p.bump(); // BETWEEN + p.eat_kw("symmetric"); + expr(p, BP_RANGE)?; + p.expect_kw("and")?; + expr(p, BP_RANGE)?; + p.finish(); + } else if p.nth_at_kw(range_kw, "in") { + p.open_at(checkpoint, SyntaxKind::InExpr); + if negated { + p.bump(); + } + p.bump(); // IN + if p.at(SyntaxKind::LParen) && at_subquery_start(p, 1) { + subquery(p)?; + } else { + paren_expr_list(p)?; + } + p.finish(); + } else { + p.open_at(checkpoint, SyntaxKind::BinaryExpr); + if negated { + p.bump(); + } + p.bump(); // LIKE | ILIKE | SIMILAR + p.eat_kw("to"); + expr(p, BP_RANGE)?; + if p.eat_kw("escape") { + expr(p, BP_RANGE)?; + } + p.finish(); + } + continue; + } + + break; + } + Ok(()) +} + +fn operator_bp(op: &str, _dialect: Dialect) -> u8 { + match op { + "^" => BP_EXP, + "*" | "/" | "%" => BP_MUL, + "+" | "-" => BP_ADD, + "<" | ">" | "=" | "<=" | ">=" | "<>" | "!=" | "==" => BP_CMP, + _ => BP_OTHER, + } +} + +fn prefix(p: &mut Parser<'_>) -> PResult { + match p.kind() { + Some( + SyntaxKind::Number + | SyntaxKind::String + | SyntaxKind::EscapeString + | SyntaxKind::UnicodeString + | SyntaxKind::BitString + | SyntaxKind::HexString + | SyntaxKind::DollarString + | SyntaxKind::Param, + ) => { + p.start(SyntaxKind::Literal); + p.bump(); + p.finish(); + Ok(()) + } + Some(SyntaxKind::Operator) => { + if p.text() == "*" { + return Err(p.error("expected an expression")); + } + p.start(SyntaxKind::PrefixExpr); + p.bump(); + expr(p, BP_UNARY)?; + p.finish(); + Ok(()) + } + Some(SyntaxKind::LParen) => { + if !at_subquery_start(p, 1) { + paren_or_row(p) + } else if p.nth_at(1, SyntaxKind::LParen) { + // `((SELECT ...` is ambiguous: a nested subquery, or a + // row/paren whose first element is a subquery. Try the + // subquery reading and back off if it doesn't close. + let state = p.state(); + if subquery(p).is_ok() { + Ok(()) + } else { + p.backtrack(state); + paren_or_row(p) + } + } else { + subquery(p) + } + } + Some(SyntaxKind::Ident) => ident_prefix(p), + Some(SyntaxKind::QuotedIdent) => column_or_call(p), + _ => Err(p.error("expected an expression")), + } +} + +/// `( query )` wrapped as a subquery expression. +fn subquery(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::SubqueryExpr); + p.expect(SyntaxKind::LParen, "`(`")?; + super::grammar::query_body(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) +} + +fn paren_or_row(p: &mut Parser<'_>) -> PResult { + let checkpoint = p.checkpoint(); + p.start(SyntaxKind::ParenExpr); + p.bump(); // ( + expr(p, 0)?; + let mut is_row = false; + while p.eat(SyntaxKind::Comma) { + is_row = true; + expr(p, 0)?; + } + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + if is_row { + // Rewrite the node kind: reopen as RowExpr. + p.rewrite_start(checkpoint, SyntaxKind::RowExpr); + } + Ok(()) +} + +fn ident_prefix(p: &mut Parser<'_>) -> PResult { + // Keyword-introduced expression forms first. + if p.at_kw("case") { + return case_expr(p); + } + if p.at_kw("cast") && p.nth_at(1, SyntaxKind::LParen) { + p.start(SyntaxKind::CastExpr); + p.bump(); + p.bump(); + expr(p, 0)?; + p.expect_kw("as")?; + type_name(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + return Ok(()); + } + if p.at_kw("exists") && p.nth_at(1, SyntaxKind::LParen) { + p.start(SyntaxKind::SubqueryExpr); + p.bump(); + p.bump(); + super::grammar::query_body(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + return Ok(()); + } + if p.at_kw("array") { + return array_expr(p); + } + if p.at_kw("row") && p.nth_at(1, SyntaxKind::LParen) { + p.start(SyntaxKind::RowExpr); + p.bump(); + paren_expr_list(p)?; + p.finish(); + return Ok(()); + } + if p.at_kw("not") { + p.start(SyntaxKind::PrefixExpr); + p.bump(); + expr(p, BP_NOT)?; + p.finish(); + return Ok(()); + } + if (p.at_kw("any") || p.at_kw("some") || p.at_kw("all")) && p.nth_at(1, SyntaxKind::LParen) { + p.start(SyntaxKind::QuantifiedExpr); + p.bump(); + if at_subquery_start(p, 1) { + subquery(p)?; + } else { + paren_expr_list(p)?; + } + p.finish(); + return Ok(()); + } + if p.at_kw("true") || p.at_kw("false") || p.at_kw("null") || p.at_kw("default") { + p.start(SyntaxKind::Literal); + p.bump(); + p.finish(); + return Ok(()); + } + // `interval '...'` / `date '...'` style typed literals. + if matches!( + p.nth_kind(1), + Some(SyntaxKind::String | SyntaxKind::EscapeString | SyntaxKind::UnicodeString) + ) { + p.start(SyntaxKind::Literal); + p.bump(); + p.bump(); + p.finish(); + return Ok(()); + } + column_or_call(p) +} + +/// A qualified name that is either a column reference or, when followed by +/// `(`, a function call with its clauses. +fn column_or_call(p: &mut Parser<'_>) -> PResult { + let checkpoint = p.checkpoint(); + p.start(SyntaxKind::ColumnRef); + qualified_name(p)?; + p.finish(); + if !p.at(SyntaxKind::LParen) { + return Ok(()); + } + p.open_at(checkpoint, SyntaxKind::FunctionCall); + arg_list(p)?; + if p.at_kw("within") && p.nth_at_kw(1, "group") { + p.start(SyntaxKind::WithinGroupClause); + p.bump(); + p.bump(); + p.expect(SyntaxKind::LParen, "`(`")?; + super::grammar::order_by_clause(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + } + if p.at_kw("filter") && p.nth_at(1, SyntaxKind::LParen) { + p.start(SyntaxKind::FilterClause); + p.bump(); + p.bump(); + p.expect_kw("where")?; + expr(p, 0)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + } + if p.at_kw("over") { + p.start(SyntaxKind::OverClause); + p.bump(); + if p.at(SyntaxKind::LParen) { + super::grammar::window_spec(p)?; + } else { + ident(p, "a window name")?; + } + p.finish(); + } + p.finish(); + Ok(()) +} + +fn case_expr(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::CaseExpr); + p.expect_kw("case")?; + if !p.at_kw("when") { + expr(p, 0)?; + } + while p.at_kw("when") { + p.start(SyntaxKind::WhenClause); + p.bump(); + expr(p, 0)?; + p.expect_kw("then")?; + expr(p, 0)?; + p.finish(); + } + if p.eat_kw("else") { + expr(p, 0)?; + } + p.expect_kw("end")?; + p.finish(); + Ok(()) +} + +fn array_expr(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::ArrayExpr); + p.expect_kw("array")?; + if p.at(SyntaxKind::LParen) { + subquery(p)?; + } else { + p.expect(SyntaxKind::LBracket, "`[`")?; + if !p.at(SyntaxKind::RBracket) { + loop { + expr(p, 0)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + } + p.expect(SyntaxKind::RBracket, "`]`")?; + } + p.finish(); + Ok(()) +} + +/// A type name: qualified name, multi-word suffixes (`double precision`, +/// `character varying`, `with/without time zone`), optional `(...)` +/// modifiers, and array suffixes. +pub(crate) fn type_name(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::TypeName); + ident(p, "a type name")?; + while p.at(SyntaxKind::Dot) { + p.bump(); + ident(p, "a type name")?; + } + while p.at_kw("precision") || p.at_kw("varying") { + p.bump(); + } + if (p.at_kw("with") || p.at_kw("without")) && p.nth_at_kw(1, "time") && p.nth_at_kw(2, "zone") { + p.bump(); + p.bump(); + p.bump(); + } + if p.at(SyntaxKind::LParen) { + paren_expr_list(p)?; + } + if p.at_kw("array") { + p.bump(); + } + while p.at(SyntaxKind::LBracket) { + p.bump(); + p.eat(SyntaxKind::Number); + p.expect(SyntaxKind::RBracket, "`]`")?; + } + p.finish(); + Ok(()) +} diff --git a/crates/parser/src/parser/grammar.rs b/crates/parser/src/parser/grammar.rs new file mode 100644 index 0000000..c137df8 --- /dev/null +++ b/crates/parser/src/parser/grammar.rs @@ -0,0 +1,768 @@ +//! Statement and clause grammar for queries (SELECT and friends). + +use crate::syntax::SyntaxKind; + +use super::expr::expr; +use super::{PResult, Parser}; + +/// Keywords that end an expression in a select-item position, so a bare +/// identifier after an expression can be taken as an alias. +const SELECT_ITEM_STOP: &[&str] = &[ + "from", + "where", + "group", + "having", + "window", + "order", + "limit", + "offset", + "fetch", + "for", + "union", + "intersect", + "except", + "into", + "returning", +]; + +/// Keywords that stop a bare table alias in FROM position. +const TABLE_ALIAS_STOP: &[&str] = &[ + "on", + "using", + "join", + "inner", + "left", + "right", + "full", + "cross", + "natural", + "lateral", + "where", + "group", + "having", + "window", + "order", + "limit", + "offset", + "fetch", + "for", + "union", + "intersect", + "except", + "with", + "returning", +]; + +/// A full query statement: `[WITH ...] query [trailing clauses] [;]`. +pub(crate) fn select_stmt(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::SelectStmt); + query_body(p)?; + if !p.at_eof() { + p.expect(SyntaxKind::Semicolon, "`;` or end of statement")?; + } + p.finish(); + Ok(()) +} + +/// `[WITH ...] set-op-expr [ORDER BY] [LIMIT/OFFSET/FETCH] [FOR ...]` — +/// the reusable query body, also used inside parentheses. +pub(crate) fn query_body(p: &mut Parser<'_>) -> PResult { + if p.at_kw("with") { + with_clause(p)?; + } + query_expr(p, 0)?; + trailing_clauses(p)?; + Ok(()) +} + +fn with_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::WithClause); + p.expect_kw("with")?; + p.eat_kw("recursive"); + loop { + cte(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +fn cte(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::Cte); + ident(p, "CTE name")?; + if p.at(SyntaxKind::LParen) { + paren_name_list(p)?; + } + p.expect_kw("as")?; + if p.at_kw("not") { + p.bump(); + p.expect_kw("materialized")?; + } else { + p.eat_kw("materialized"); + } + p.expect(SyntaxKind::LParen, "`(`")?; + query_body(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + if p.at_kw("search") { + search_clause(p)?; + } + if p.at_kw("cycle") { + cycle_clause(p)?; + } + p.finish(); + Ok(()) +} + +fn search_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::SearchClause); + p.expect_kw("search")?; + if !p.eat_kw("breadth") && !p.eat_kw("depth") { + return Err(p.error("expected `BREADTH` or `DEPTH`")); + } + p.expect_kws(&["first", "by"])?; + name_list(p)?; + p.expect_kws(&["set"])?; + ident(p, "column name")?; + p.finish(); + Ok(()) +} + +fn cycle_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::CycleClause); + p.expect_kw("cycle")?; + name_list(p)?; + p.expect_kw("set")?; + ident(p, "column name")?; + if p.eat_kw("to") { + expr(p, 0)?; + p.expect_kw("default")?; + expr(p, 0)?; + } + p.expect_kw("using")?; + ident(p, "column name")?; + p.finish(); + Ok(()) +} + +/// Set operations with precedence: INTERSECT binds tighter than +/// UNION/EXCEPT; all left-associative. +fn query_expr(p: &mut Parser<'_>, min_bp: u8) -> PResult { + let checkpoint = p.checkpoint(); + query_primary(p)?; + loop { + let bp = if p.at_kw("intersect") { + 2 + } else if p.at_kw("union") || p.at_kw("except") { + 1 + } else { + break; + }; + if bp <= min_bp { + break; + } + p.open_at(checkpoint, SyntaxKind::SetOperation); + p.bump(); // UNION | INTERSECT | EXCEPT + if !p.eat_kw("all") { + p.eat_kw("distinct"); + } + query_expr(p, bp)?; + p.finish(); + } + Ok(()) +} + +fn query_primary(p: &mut Parser<'_>) -> PResult { + if p.at(SyntaxKind::LParen) { + p.start(SyntaxKind::ParenSelect); + p.bump(); + query_body(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) + } else if p.at_kw("select") { + select_core(p) + } else if p.at_kw("values") { + values_clause(p) + } else if p.at_kw("table") { + p.start(SyntaxKind::TableCore); + p.bump(); + qualified_name(p)?; + p.finish(); + Ok(()) + } else { + Err(p.error("expected `SELECT`, `VALUES`, `TABLE`, or `(`")) + } +} + +fn select_core(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::SelectCore); + p.expect_kw("select")?; + if p.at_kw("distinct") { + p.bump(); + if p.at_kw("on") { + p.bump(); + paren_expr_list(p)?; + } + } else { + p.eat_kw("all"); + } + select_list(p)?; + if p.at_kw("from") { + from_clause(p)?; + } + if p.at_kw("where") { + p.start(SyntaxKind::WhereClause); + p.bump(); + expr(p, 0)?; + p.finish(); + } + if p.at_kw("group") { + group_by_clause(p)?; + } + if p.at_kw("having") { + p.start(SyntaxKind::HavingClause); + p.bump(); + expr(p, 0)?; + p.finish(); + } + if p.at_kw("window") { + window_clause(p)?; + } + p.finish(); + Ok(()) +} + +fn select_list(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::SelectList); + loop { + select_item(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +fn select_item(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::SelectItem); + if p.at_op("*") { + p.bump(); + } else { + expr(p, 0)?; + if p.eat_kw("as") || at_bare_alias(p, SELECT_ITEM_STOP) { + alias_name(p)?; + } + } + p.finish(); + Ok(()) +} + +/// A bare (no `AS`) alias: an identifier that is not a clause keyword. +fn at_bare_alias(p: &Parser<'_>, stop: &[&str]) -> bool { + (p.at(SyntaxKind::Ident) || p.at(SyntaxKind::QuotedIdent)) && !p.at_any_kw(stop) +} + +fn alias_name(p: &mut Parser<'_>) -> PResult { + if p.at(SyntaxKind::Ident) || p.at(SyntaxKind::QuotedIdent) { + p.bump(); + Ok(()) + } else { + Err(p.error("expected an alias name")) + } +} + +fn from_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::FromClause); + p.expect_kw("from")?; + loop { + from_item(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +/// One FROM element: a primary table reference plus any number of joins, +/// built left-associatively as nested `JoinExpr` nodes. +fn from_item(p: &mut Parser<'_>) -> PResult { + let checkpoint = p.checkpoint(); + table_primary(p)?; + while at_join_start(p) { + p.open_at(checkpoint, SyntaxKind::JoinExpr); + join_keywords(p)?; + table_primary(p)?; + if p.at_kw("on") { + p.start(SyntaxKind::JoinCondition); + p.bump(); + expr(p, 0)?; + p.finish(); + } else if p.at_kw("using") { + p.start(SyntaxKind::JoinCondition); + p.bump(); + paren_name_list(p)?; + if p.eat_kw("as") { + alias_name(p)?; + } + p.finish(); + } + p.finish(); + } + Ok(()) +} + +fn at_join_start(p: &Parser<'_>) -> bool { + p.at_any_kw(&["join", "inner", "left", "right", "full", "cross", "natural"]) +} + +fn join_keywords(p: &mut Parser<'_>) -> PResult { + p.eat_kw("natural"); + if p.eat_kw("cross") { + p.expect_kw("join")?; + return Ok(()); + } + if p.eat_kw("inner") { + p.expect_kw("join")?; + return Ok(()); + } + if p.eat_kw("left") || p.eat_kw("right") || p.eat_kw("full") { + p.eat_kw("outer"); + p.expect_kw("join")?; + return Ok(()); + } + p.expect_kw("join") +} + +fn table_primary(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::TableRef); + p.eat_kw("lateral"); + if p.at(SyntaxKind::LParen) { + let paren_join = |p: &mut Parser<'_>| -> PResult { + // Parenthesized join: `(a join b on ...)`. + p.start(SyntaxKind::ParenTableRef); + p.bump(); + from_item(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) + }; + let subquery = |p: &mut Parser<'_>| -> PResult { + p.start(SyntaxKind::SubqueryExpr); + p.bump(); + query_body(p)?; + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) + }; + if !at_subquery_start(p, 1) { + paren_join(p)?; + } else if p.nth_at(1, SyntaxKind::LParen) { + // `((SELECT ...` may be a parenthesized set-op subquery or a + // paren join whose first table is a subquery; try, back off. + let state = p.state(); + if subquery(p).is_err() { + p.backtrack(state); + paren_join(p)?; + } + } else { + subquery(p)?; + } + } else { + p.eat_kw("only"); + qualified_name(p)?; + if p.at(SyntaxKind::LParen) { + // Table function call: name(args). + arg_list(p)?; + if p.at_kw("with") && p.nth_at_kw(1, "ordinality") { + p.bump(); + p.bump(); + } + } else { + p.eat_op_star(); + } + } + if p.eat_kw("as") || at_bare_alias(p, TABLE_ALIAS_STOP) { + alias_name(p)?; + if p.at(SyntaxKind::LParen) { + paren_name_list(p)?; + } + } + p.finish(); + Ok(()) +} + +/// Does a `(` at offset `n` open a subquery (`SELECT`/`WITH`/`VALUES`/ +/// `TABLE`), looking through further parens? +pub(crate) fn at_subquery_start(p: &Parser<'_>, mut n: usize) -> bool { + while p.nth_at(n, SyntaxKind::LParen) { + n += 1; + } + p.nth_at_kw(n, "select") + || p.nth_at_kw(n, "with") + || p.nth_at_kw(n, "values") + || p.nth_at_kw(n, "table") +} + +fn group_by_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::GroupByClause); + p.expect_kws(&["group", "by"])?; + if !p.eat_kw("all") { + p.eat_kw("distinct"); + } + loop { + grouping_element(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +fn grouping_element(p: &mut Parser<'_>) -> PResult { + if p.at_kw("rollup") || p.at_kw("cube") { + p.start(SyntaxKind::GroupingElement); + p.bump(); + paren_expr_list(p)?; + p.finish(); + Ok(()) + } else if p.at_kw("grouping") && p.nth_at_kw(1, "sets") { + p.start(SyntaxKind::GroupingElement); + p.bump(); + p.bump(); + p.expect(SyntaxKind::LParen, "`(`")?; + loop { + grouping_element(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) + } else if p.at(SyntaxKind::LParen) && p.nth_at(1, SyntaxKind::RParen) { + // Empty grouping set `()`. + p.start(SyntaxKind::GroupingElement); + p.bump(); + p.bump(); + p.finish(); + Ok(()) + } else { + expr(p, 0) + } +} + +fn window_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::WindowClause); + p.expect_kw("window")?; + loop { + p.start(SyntaxKind::WindowDef); + ident(p, "window name")?; + p.expect_kw("as")?; + window_spec(p)?; + p.finish(); + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +/// `( [existing_window] [PARTITION BY ...] [ORDER BY ...] [frame] )` +pub(crate) fn window_spec(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::WindowSpec); + p.expect(SyntaxKind::LParen, "`(`")?; + if (p.at(SyntaxKind::Ident) || p.at(SyntaxKind::QuotedIdent)) + && !p.at_any_kw(&["partition", "order", "rows", "range", "groups"]) + { + p.bump(); + } + if p.at_kw("partition") { + p.bump(); + p.expect_kw("by")?; + loop { + expr(p, 0)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + } + if p.at_kw("order") { + order_by_clause(p)?; + } + if p.at_any_kw(&["rows", "range", "groups"]) { + frame_clause(p)?; + } + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) +} + +fn frame_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::FrameClause); + p.bump(); // rows | range | groups + if p.eat_kw("between") { + frame_bound(p)?; + p.expect_kw("and")?; + frame_bound(p)?; + } else { + frame_bound(p)?; + } + if p.eat_kw("exclude") { + if p.eat_kw("current") { + p.expect_kw("row")?; + } else if p.eat_kw("no") { + p.expect_kw("others")?; + } else if !p.eat_kw("group") && !p.eat_kw("ties") { + return Err(p.error("expected frame exclusion")); + } + } + p.finish(); + Ok(()) +} + +fn frame_bound(p: &mut Parser<'_>) -> PResult { + if p.eat_kw("unbounded") { + if !p.eat_kw("preceding") && !p.eat_kw("following") { + return Err(p.error("expected `PRECEDING` or `FOLLOWING`")); + } + Ok(()) + } else if p.eat_kw("current") { + p.expect_kw("row") + } else { + expr(p, 0)?; + if !p.eat_kw("preceding") && !p.eat_kw("following") { + return Err(p.error("expected `PRECEDING` or `FOLLOWING`")); + } + Ok(()) + } +} + +pub(crate) fn order_by_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::OrderByClause); + p.expect_kws(&["order", "by"])?; + loop { + p.start(SyntaxKind::OrderingTerm); + expr(p, 0)?; + if p.eat_kw("using") { + if !p.at(SyntaxKind::Operator) { + return Err(p.error("expected an operator after `USING`")); + } + p.bump(); + } else if !p.eat_kw("asc") { + p.eat_kw("desc"); + } + if p.eat_kw("nulls") && !p.eat_kw("first") && !p.eat_kw("last") { + return Err(p.error("expected `FIRST` or `LAST`")); + } + p.finish(); + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +fn values_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::ValuesClause); + p.expect_kw("values")?; + loop { + paren_expr_list(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.finish(); + Ok(()) +} + +fn trailing_clauses(p: &mut Parser<'_>) -> PResult { + if p.at_kw("order") { + order_by_clause(p)?; + } + // Postgres accepts LIMIT and OFFSET in either order. + loop { + if p.at_kw("limit") { + p.start(SyntaxKind::LimitClause); + p.bump(); + if !p.eat_kw("all") { + expr(p, 0)?; + } + p.finish(); + } else if p.at_kw("offset") { + p.start(SyntaxKind::OffsetClause); + p.bump(); + expr(p, 0)?; + if !p.eat_kw("rows") { + p.eat_kw("row"); + } + p.finish(); + } else if p.at_kw("fetch") { + fetch_clause(p)?; + } else if p.at_kw("for") { + locking_clause(p)?; + } else { + break; + } + } + Ok(()) +} + +fn fetch_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::FetchClause); + p.expect_kw("fetch")?; + if !p.eat_kw("first") && !p.eat_kw("next") { + return Err(p.error("expected `FIRST` or `NEXT`")); + } + if !p.at_kw("row") && !p.at_kw("rows") { + expr(p, 0)?; + } + if !p.eat_kw("rows") && !p.eat_kw("row") { + return Err(p.error("expected `ROW` or `ROWS`")); + } + if p.eat_kw("with") { + p.expect_kw("ties")?; + } else { + p.expect_kw("only")?; + } + p.finish(); + Ok(()) +} + +fn locking_clause(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::LockingClause); + p.expect_kw("for")?; + if p.eat_kw("update") { + } else if p.eat_kw("no") { + p.expect_kws(&["key", "update"])?; + } else if p.eat_kw("key") { + p.expect_kw("share")?; + } else if !p.eat_kw("share") { + return Err(p.error("expected a lock strength")); + } + if p.eat_kw("of") { + name_list(p)?; + } + if p.eat_kw("skip") { + p.expect_kw("locked")?; + } else { + p.eat_kw("nowait"); + } + p.finish(); + Ok(()) +} + +// ---- small shared pieces ---- + +pub(crate) fn ident(p: &mut Parser<'_>, what: &str) -> PResult { + if p.at(SyntaxKind::Ident) || p.at(SyntaxKind::QuotedIdent) { + p.bump(); + Ok(()) + } else { + Err(p.error(&format!("expected {what}"))) + } +} + +/// `name[.name[.name]]`, with a possible trailing `.*`. +pub(crate) fn qualified_name(p: &mut Parser<'_>) -> PResult { + ident(p, "a name")?; + while p.at(SyntaxKind::Dot) { + p.bump(); + if p.at_op("*") { + p.bump(); + break; + } + ident(p, "a name")?; + } + Ok(()) +} + +fn name_list(p: &mut Parser<'_>) -> PResult { + loop { + qualified_name(p)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + Ok(()) +} + +pub(crate) fn paren_name_list(p: &mut Parser<'_>) -> PResult { + p.expect(SyntaxKind::LParen, "`(`")?; + loop { + ident(p, "a name")?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + p.expect(SyntaxKind::RParen, "`)`") +} + +pub(crate) fn paren_expr_list(p: &mut Parser<'_>) -> PResult { + p.expect(SyntaxKind::LParen, "`(`")?; + if !p.at(SyntaxKind::RParen) { + loop { + expr(p, 0)?; + if !p.eat(SyntaxKind::Comma) { + break; + } + } + } + p.expect(SyntaxKind::RParen, "`)`") +} + +/// Function-call style argument list; also used for table functions. +pub(crate) fn arg_list(p: &mut Parser<'_>) -> PResult { + p.start(SyntaxKind::ArgList); + p.expect(SyntaxKind::LParen, "`(`")?; + if !p.at(SyntaxKind::RParen) { + if !p.eat_kw("distinct") { + p.eat_kw("all"); + } + p.eat_kw("variadic"); + loop { + if p.at_op("*") { + p.bump(); + } else { + expr(p, 0)?; + } + // Tolerate the keyword-separated special forms: EXTRACT(x FROM + // y), SUBSTRING(x FROM y FOR z), POSITION(a IN b), + // TRIM(BOTH x FROM y), OVERLAY(a PLACING b FROM c FOR d). + if p.at_any_kw(&[ + "from", "for", "in", "placing", "as", "both", "leading", "trailing", + ]) { + p.bump(); + continue; + } + if p.at_kw("order") { + order_by_clause(p)?; + } + if !p.eat(SyntaxKind::Comma) { + break; + } + p.eat_kw("variadic"); + } + } + p.expect(SyntaxKind::RParen, "`)`")?; + p.finish(); + Ok(()) +} + +impl Parser<'_> { + /// Eat a `*` operator token if present. + pub(crate) fn eat_op_star(&mut self) -> bool { + if self.at_op("*") { + self.bump(); + true + } else { + false + } + } +} diff --git a/crates/parser/syntax.def b/crates/parser/syntax.def index 6df073c..26d8cbd 100644 --- a/crates/parser/syntax.def +++ b/crates/parser/syntax.def @@ -49,7 +49,86 @@ token Operator node Root many statements Statement -# Placeholder statement node for the toy grammar; TREE-95 introduces the -# real statement kinds. +# Generic statement node, used by the tree-builder tests' toy grammar. node Statement token semicolon Semicolon + +# A statement that failed to parse; contains its raw tokens verbatim. +node ErrorStatement +# A bare `;` with nothing before it. +node EmptyStmt + +# ---- SELECT statements ---- + +node SelectStmt + one with_clause WithClause + one order_by OrderByClause + one limit_clause LimitClause + one offset_clause OffsetClause + one fetch_clause FetchClause + token semicolon Semicolon +node WithClause + many ctes Cte +node Cte +node SearchClause +node CycleClause +node SelectCore + one select_list SelectList + one from_clause FromClause + one where_clause WhereClause + one group_by GroupByClause + one having HavingClause + one window_clause WindowClause +node SetOperation +node ParenSelect +node ValuesClause +node TableCore +node SelectList + many items SelectItem +node SelectItem +node FromClause +node TableRef +node ParenTableRef +node JoinExpr +node JoinCondition +node Alias +node WhereClause +node GroupByClause +node GroupingElement +node HavingClause +node WindowClause +node WindowDef +node WindowSpec +node FrameClause +node OrderByClause + many terms OrderingTerm +node OrderingTerm +node LimitClause +node OffsetClause +node FetchClause +node LockingClause + +# ---- expressions ---- + +node Literal +node ColumnRef +node ParenExpr +node RowExpr +node ArrayExpr +node SubqueryExpr +node QuantifiedExpr +node CaseExpr +node WhenClause +node FunctionCall +node ArgList +node FilterClause +node WithinGroupClause +node OverClause +node CastExpr +node TypeName +node PrefixExpr +node BinaryExpr +node IsExpr +node BetweenExpr +node InExpr +node SubscriptExpr diff --git a/crates/parser/tests/corpus.rs b/crates/parser/tests/corpus.rs index e8a7f16..65e6ab6 100644 --- a/crates/parser/tests/corpus.rs +++ b/crates/parser/tests/corpus.rs @@ -39,3 +39,91 @@ fn corpus_lexes_losslessly_with_no_error_tokens() { } } } + +/// TREE-95: parsing is lossless corpus-wide, including ErrorStatements. +#[test] +fn corpus_parses_losslessly() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"); + let mut files = Vec::new(); + collect_sql_files(&root, &mut files); + + for path in files { + let source = std::fs::read_to_string(&path).expect("read corpus file"); + let tokens = lex(&source, Dialect::Postgres); + let parse = parser::parser::parse(&tokens, Dialect::Postgres); + assert_eq!( + parse.cst.text(), + source, + "parse round-trip failed for {}", + path.display() + ); + } +} + +/// TREE-95: every statement in `corpus/coder/queries/` that does not +/// involve DML parses cleanly. (DML lands with TREE-98.) +#[test] +fn queries_select_statements_all_parse() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/coder/queries"); + let mut files = Vec::new(); + collect_sql_files(&root, &mut files); + assert!(!files.is_empty(), "no query files found"); + + for path in files { + let source = std::fs::read_to_string(&path).expect("read corpus file"); + let tokens = lex(&source, Dialect::Postgres); + let parse = parser::parser::parse(&tokens, Dialect::Postgres); + for node in parse.cst.root().children() { + if node.kind() == SyntaxKind::ErrorStatement { + let text = node.to_string(); + assert!( + !starts_selectish(&text) || involves_dml(&text), + "SELECT statement failed to parse in {}: {}", + path.display(), + text.split_whitespace().collect::>().join(" ") + ); + } + } + } +} + +/// Does the statement start like a query (`SELECT`/`WITH`/`VALUES`/ +/// `TABLE`/`(`)? +fn starts_selectish(sql: &str) -> bool { + let tokens = lex(sql, Dialect::Postgres); + tokens + .iter() + .find(|t| !t.kind.is_trivia()) + .is_some_and(|t| { + t.kind == SyntaxKind::LParen + || ["select", "with", "values", "table"] + .iter() + .any(|kw| t.text.eq_ignore_ascii_case(kw)) + }) +} + +/// Does the failed statement involve DML — a top-level +/// INSERT/UPDATE/DELETE/MERGE or a data-modifying CTE? Mirrors the +/// corpus-report classifier; `FOR [NO KEY] UPDATE` does not count. +fn involves_dml(sql: &str) -> bool { + let tokens = lex(sql, Dialect::Postgres); + let mut depth = 0i32; + let mut prev = String::new(); + for token in tokens.iter().filter(|t| !t.kind.is_trivia()) { + match token.kind { + SyntaxKind::LParen => depth += 1, + SyntaxKind::RParen => depth -= 1, + SyntaxKind::Ident => { + let text = token.text.to_ascii_lowercase(); + let is_dml = ["insert", "update", "delete", "merge"].contains(&text.as_str()); + let after_lock = ["for", "key", "no"].contains(&prev.as_str()); + if is_dml && (prev == "(" || (depth == 0 && !after_lock)) { + return true; + } + } + _ => {} + } + prev = token.text.to_ascii_lowercase(); + } + false +} diff --git a/crates/parser/tests/parser.rs b/crates/parser/tests/parser.rs new file mode 100644 index 0000000..bb1e057 --- /dev/null +++ b/crates/parser/tests/parser.rs @@ -0,0 +1,267 @@ +//! TREE-95 acceptance: SELECT grammar coverage, error recovery, and the +//! lossless invariant. + +use parser::Dialect; +use parser::lexer::lex; +use parser::parser::parse; +use parser::syntax::SyntaxKind; +use parser::tree::Cst; + +/// Parse expecting full success (no ErrorStatements), and assert the +/// lossless round-trip. +#[track_caller] +fn parse_ok(sql: &str) -> Cst { + let tokens = lex(sql, Dialect::Postgres); + let parse = parse(&tokens, Dialect::Postgres); + assert_eq!(parse.cst.text(), sql, "round-trip failed"); + assert!( + parse.diagnostics.is_empty(), + "unexpected diagnostics for {sql:?}: {:?}", + parse.diagnostics + ); + parse.cst +} + +fn top_level_kinds(cst: &Cst) -> Vec { + cst.root().children().map(|node| node.kind()).collect() +} + +// ---- SELECT feature coverage ---- + +#[test] +fn with_recursive_and_materialized() { + parse_ok( + "WITH RECURSIVE t AS MATERIALIZED (SELECT 1), u(a, b) AS NOT MATERIALIZED (SELECT 2, 3) \ + SELECT * FROM t, u;", + ); +} + +#[test] +fn with_search_and_cycle() { + parse_ok( + "WITH RECURSIVE t(a) AS (SELECT 1 UNION ALL SELECT a + 1 FROM t) \ + SEARCH BREADTH FIRST BY a SET ordercol \ + CYCLE a SET is_cycle USING path \ + SELECT * FROM t LIMIT 10;", + ); +} + +#[test] +fn join_forms() { + parse_ok( + "SELECT * FROM a JOIN b ON a.x = b.x \ + LEFT OUTER JOIN c USING (id) AS ualias \ + NATURAL RIGHT JOIN d \ + CROSS JOIN e, \ + f AS alias(x, y), \ + LATERAL (SELECT 1) AS l, \ + ONLY g, \ + generate_series(1, 10) WITH ORDINALITY AS gs(n, ord), \ + (h JOIN i ON h.a = i.a);", + ); +} + +#[test] +fn distinct_forms() { + parse_ok("SELECT DISTINCT a FROM t;"); + parse_ok("SELECT DISTINCT ON (a, b) a, b, c FROM t ORDER BY a, b;"); + parse_ok("SELECT ALL a FROM t;"); +} + +#[test] +fn group_by_forms() { + parse_ok( + "SELECT a, b FROM t \ + GROUP BY GROUPING SETS ((a), (a, b), ()), ROLLUP (a, b), CUBE (a), a \ + HAVING count(*) > 1;", + ); + parse_ok("SELECT a FROM t GROUP BY ALL a;"); + parse_ok("SELECT a FROM t GROUP BY DISTINCT ROLLUP (a);"); +} + +#[test] +fn aggregate_clauses() { + parse_ok( + "SELECT count(*) FILTER (WHERE x > 0), \ + percentile_cont(0.5) WITHIN GROUP (ORDER BY y DESC), \ + string_agg(DISTINCT z, ',' ORDER BY z), \ + array_agg(VARIADIC v) \ + FROM t;", + ); +} + +#[test] +fn window_functions() { + parse_ok( + "SELECT sum(x) OVER w, \ + avg(y) OVER (PARTITION BY a, b ORDER BY c \ + ROWS BETWEEN 1 PRECEDING AND CURRENT ROW EXCLUDE TIES), \ + row_number() OVER (w RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING \ + EXCLUDE NO OTHERS), \ + first_value(z) OVER (GROUPS CURRENT ROW EXCLUDE GROUP), \ + nth_value(z, 2) OVER () \ + FROM t \ + WINDOW w AS (ORDER BY x RANGE UNBOUNDED PRECEDING), \ + w2 AS (w PARTITION BY y);", + ); +} + +#[test] +fn set_operations_and_tail_clauses() { + parse_ok( + "SELECT 1 UNION ALL SELECT 2 INTERSECT DISTINCT SELECT 3 EXCEPT SELECT 4 \ + ORDER BY 1 DESC NULLS LAST OFFSET 2 LIMIT 5;", + ); + parse_ok("SELECT a FROM t OFFSET 5 LIMIT 2;"); + parse_ok("SELECT a FROM t ORDER BY a USING < FETCH FIRST 10 ROWS WITH TIES;"); + parse_ok("SELECT a FROM t FETCH NEXT ROW ONLY;"); + parse_ok("SELECT a FROM t FOR UPDATE OF t SKIP LOCKED FOR SHARE NOWAIT;"); + parse_ok("SELECT a FROM t FOR NO KEY UPDATE;"); + parse_ok("SELECT a FROM t FOR KEY SHARE;"); + parse_ok("(SELECT 1 ORDER BY 1) UNION (SELECT 2 LIMIT 1);"); + parse_ok("TABLE t UNION SELECT 1;"); + parse_ok("VALUES (1, 'a'), (2, 'b') ORDER BY 1;"); +} + +#[test] +fn expression_zoo() { + parse_ok( + "SELECT a::int[], b::character varying(10), c::numeric(10, 2), \ + arr[1], arr[1:2], arr[:2], arr[1:], mx[1][2], \ + x BETWEEN SYMMETRIC 1 AND 10, y NOT BETWEEN 1 AND 2, \ + c1 IS NOT DISTINCT FROM d, c2 IS DISTINCT FROM e, \ + f IS NULL, g IS NOT TRUE, h ISNULL, i NOTNULL, \ + j COLLATE \"en_US\", k AT TIME ZONE 'utc', \ + CASE WHEN x THEN 1 WHEN y THEN 2 ELSE 3 END, \ + CASE x WHEN 1 THEN 'a' END, \ + ARRAY[1, 2, 3], ARRAY[ARRAY[1], ARRAY[2]], ARRAY(SELECT 1), \ + ROW(1, 2), (1, 2, 3), \ + EXISTS (SELECT 1 FROM t), \ + q IN (1, 2), r NOT IN (SELECT z FROM t), \ + s > ANY (SELECT w FROM t), s2 <= ALL (ARRAY[1]), s3 = SOME (1, 2), \ + CAST(v AS timestamp with time zone), \ + n NOT LIKE 'a%' ESCAPE '!', n2 ILIKE 'b_', n3 SIMILAR TO 'x+', \ + interval '1 day', timestamp '2020-01-01', \ + NOT (p AND q2 OR r2), \ + -x + +y, ~bits, @absval, \ + amount * 1.5 ^ 2 % 3 - 7, \ + tags @> ARRAY['a'], meta -> 'k' ->> 'j', \ + ((SELECT 1), 2) \ + FROM t;", + ); +} + +#[test] +fn nested_subquery_disambiguation() { + // `((SELECT ...))` both as scalar and in FROM. + parse_ok("SELECT ((SELECT 1));"); + parse_ok("SELECT * FROM ((SELECT 1) UNION (SELECT 2)) AS u;"); + parse_ok("SELECT * FROM ((SELECT 1 AS a) s JOIN t ON t.a = s.a);"); + parse_ok("SELECT ((SELECT max(x) FROM t), 'k');"); +} + +#[test] +fn sqlc_style_params() { + // sqlc uses `@name` args and `sqlc.arg('x')` calls against Postgres. + parse_ok("SELECT * FROM t WHERE id = @id AND org = sqlc.narg('org')::uuid LIMIT @lim;"); +} + +// ---- statement framework ---- + +#[test] +fn empty_statements() { + let cst = parse_ok(";;"); + assert_eq!( + top_level_kinds(&cst), + [SyntaxKind::EmptyStmt, SyntaxKind::EmptyStmt] + ); +} + +#[test] +fn error_recovery_resumes_at_semicolon() { + let sql = "SELECT 1; FROBNICATE the database; SELECT 2;"; + let tokens = lex(sql, Dialect::Postgres); + let parse = parse(&tokens, Dialect::Postgres); + assert_eq!(parse.cst.text(), sql, "ErrorStatement must keep all tokens"); + assert_eq!( + top_level_kinds(&parse.cst), + [ + SyntaxKind::SelectStmt, + SyntaxKind::ErrorStatement, + SyntaxKind::SelectStmt + ] + ); + assert_eq!(parse.diagnostics.len(), 1); + let diagnostic = &parse.diagnostics[0]; + assert_eq!( + &sql[diagnostic.start..diagnostic.end], + "FROBNICATE", + "diagnostic span should point at the failure" + ); +} + +#[test] +fn error_statement_keeps_partial_select() { + // A select that goes wrong mid-way must roll back into a full + // ErrorStatement, not a half-built tree. + let sql = "SELECT a FROM WHERE ORDER;"; + let tokens = lex(sql, Dialect::Postgres); + let parse = parse(&tokens, Dialect::Postgres); + assert_eq!(parse.cst.text(), sql); + assert_eq!(top_level_kinds(&parse.cst), [SyntaxKind::ErrorStatement]); + assert_eq!(parse.diagnostics.len(), 1); +} + +#[test] +fn sqlite_trigger_body_is_one_statement() { + let sql = "CREATE TRIGGER tr AFTER INSERT ON t BEGIN \ + UPDATE x SET y = 1; DELETE FROM z; END; \ + SELECT 1;"; + let tokens = lex(sql, Dialect::Sqlite); + let parse = parse(&tokens, Dialect::Sqlite); + assert_eq!(parse.cst.text(), sql); + assert_eq!( + top_level_kinds(&parse.cst), + [SyntaxKind::ErrorStatement, SyntaxKind::SelectStmt], + "trigger body semicolons must not split the statement" + ); +} + +#[test] +fn sqlite_begin_transaction_is_not_a_block() { + let sql = "BEGIN TRANSACTION; SELECT 1;"; + let tokens = lex(sql, Dialect::Sqlite); + let parse = parse(&tokens, Dialect::Sqlite); + assert_eq!(parse.cst.text(), sql); + assert_eq!( + top_level_kinds(&parse.cst), + [SyntaxKind::ErrorStatement, SyntaxKind::SelectStmt], + "BEGIN TRANSACTION must end at its own semicolon" + ); +} + +#[test] +fn dollar_quoted_semicolons_are_not_boundaries() { + let sql = "SELECT $fn$ a; b; c $fn$; SELECT 2;"; + let cst = parse_ok(sql); + assert_eq!( + top_level_kinds(&cst), + [SyntaxKind::SelectStmt, SyntaxKind::SelectStmt] + ); +} + +#[test] +fn garbage_never_loses_tokens() { + for sql in [ + "SELECT ((((;", + "'unterminated", + ")))) select ;;; (", + "SELECT FROM WHERE; SELECT 1; GROUP;", + ] { + for dialect in [Dialect::Postgres, Dialect::Sqlite] { + let tokens = lex(sql, dialect); + let parse = parse(&tokens, dialect); + assert_eq!(parse.cst.text(), sql, "{dialect:?} lost tokens for {sql:?}"); + } + } +} -- 2.51.2