mod docgen; mod helpers; mod path; use std::{env, error::Error}; type DynError = Box; pub mod tasks { use crate::DynError; use std::collections::HashSet; pub fn docgen() -> Result<(), DynError> { use crate::docgen::*; write(TYPABLE_COMMANDS_MD_OUTPUT, &typable_commands()?); write(STATIC_COMMANDS_MD_OUTPUT, &static_commands()?); write(LANG_SUPPORT_MD_OUTPUT, &lang_features()?); Ok(()) } pub fn querycheck(languages: impl Iterator) -> Result<(), DynError> { use helix_core::syntax::LanguageData; let languages_to_check: HashSet<_> = languages.collect(); let loader = helix_core::config::default_lang_loader(); for (_language, lang_data) in loader.languages() { if !languages_to_check.is_empty() && !languages_to_check.contains(&lang_data.config().language_id) { continue; } let config = lang_data.config(); let Some(syntax_config) = LanguageData::compile_syntax_config(config, &loader)? else { continue; }; let grammar = syntax_config.grammar; LanguageData::compile_indent_query(grammar, config)?; LanguageData::compile_textobject_query(grammar, config)?; LanguageData::compile_tag_query(grammar, config)?; LanguageData::compile_rainbow_query(grammar, config)?; } println!("Query check succeeded"); Ok(()) } pub fn indentcheck(languages: impl Iterator) -> Result<(), DynError> { use helix_core::{ indent::{ is_opaque_interior, is_outdent_token_at, treesitter_indent_for_pos, IndentStyle, }, Syntax, }; use helix_stdx::rope::RopeSliceExt; use ropey::Rope; let filter: HashSet = languages.collect(); let loader = helix_core::config::default_lang_loader(); let corpus = crate::path::tests_indent(); let tab_width = 4; let mut errors = 0usize; let mut over_notes = 0usize; let mut entries: Vec<_> = std::fs::read_dir(&corpus)? .filter_map(Result::ok) .map(|e| e.path()) .filter(|p| p.is_file()) .collect(); entries.sort(); for path in entries { let stem = path.file_stem().and_then(|s| s.to_str()).unwrap_or(""); if !filter.is_empty() && !filter.contains(stem) { continue; } let file = path.file_name().and_then(|s| s.to_str()).unwrap_or(""); // Corpus files are named .; resolve the language by id. let language = match loader .languages() .find(|(_, data)| data.config().language_id == stem) { Some((language, _)) => language, None => { return Err(format!( "{file}: no configured language with id '{stem}' (corpus files are named .)" ) .into()) } }; let config = loader.language(language).config(); let indent_style = IndentStyle::from_str( &config .indent .as_ref() .ok_or_else(|| format!("{file}: language '{stem}' has no indent config"))? .unit, ); // Lines that are commented out are skipped: they self-document edge // cases (e.g. known indent limitations) without being asserted on. let comment_tokens: Vec = config.comment_tokens.clone().unwrap_or_default(); let doc = Rope::from_reader(&mut std::fs::File::open(&path)?)?; let text = doc.slice(..); let syntax = Syntax::new(text, language, &loader) .map_err(|e| format!("{file}: failed to parse: {e:?}"))?; let indent_query = loader .indent_query(language) .ok_or_else(|| format!("{file}: language '{stem}' has no indent query"))?; for i in 0..doc.len_lines() { let line = text.line(i); let Some(pos) = line.first_non_whitespace_char() else { continue; }; let trimmed = line.slice(pos..).to_string(); if comment_tokens .iter() .any(|tok| trimmed.starts_with(tok.as_str())) { continue; } let suggested = treesitter_indent_for_pos( indent_query, &syntax, &loader, tab_width, indent_style.indent_width(tab_width), text, i, text.line_to_char(i) + pos, false, ) .unwrap() .to_string(&indent_style, tab_width); let actual = line .get_slice(..pos) .map(|s| s.to_string()) .unwrap_or_default(); if actual != suggested { errors += 1; println!( "{file}:{}: reindent expected {} columns, computed {}", i + 1, actual.chars().count(), suggested.chars().count(), ); } // Typing direction: simulate pressing Enter at the end of this line and check the indent computed for the next line. // - under-indent (computed < canonical) is always a failure: nothing pulls the line further in, so the user is left // under-indented. // - over-indent (computed > canonical) is only acceptable when the next line's leading token is @outdent (a closing // bracket, case/else/except keyword, ...): entering it dedents the line. An over-indent on a plain statement (e.g. // a line after `return` that should leave the block but doesn't) has nothing to correct it and is a real failure. if i + 1 < doc.len_lines() { if let Some(next_pos) = text.line(i + 1).first_non_whitespace_char() { let next = text.line(i + 1); let next_trim = next.slice(next_pos..).to_string(); // Lines inside an @opaque body (string/comment) carry literal leading whitespace, not code indent — don't // assert a typing indent for them. let next_byte = text.char_to_byte(text.line_to_char(i + 1) + next_pos) as u32; let next_is_opaque = is_opaque_interior(indent_query, &syntax, text, next_byte); let next_is_comment = next_is_opaque || comment_tokens .iter() .any(|tok| next_trim.starts_with(tok.as_str())); if !next_is_comment { let typed = treesitter_indent_for_pos( indent_query, &syntax, &loader, tab_width, indent_style.indent_width(tab_width), text, i, text.line_to_char(i + 1) - 1, true, ) .unwrap() .to_string(&indent_style, tab_width); let next_actual = next .get_slice(..next_pos) .map(|s| s.to_string()) .unwrap_or_default(); let typed_cols = typed.chars().count(); let want_cols = next_actual.chars().count(); let leading_outdent = || { let byte = text.char_to_byte(text.line_to_char(i + 1) + next_pos); is_outdent_token_at(indent_query, &syntax, text, byte as u32) }; if typed_cols < want_cols { errors += 1; println!( "{file}:{}: typing under-indents: computed {} columns, expected {} | {}", i + 2, typed_cols, want_cols, next_trim.trim_end(), ); } else if typed_cols > want_cols && !leading_outdent() { // Over-indents are reported but not failed: every over-indent sits at a legitimate dedent point, and // for indent-delimited languages (python after a block) the editor genuinely cannot know how far to // dedent. Surfaced so a regression shows up in the diff; gate on under-indents only. over_notes += 1; println!( "{file}:{}: note: typing over-indents: computed {} columns, expected {} (no leading outdent; review) | {}", i + 2, typed_cols, want_cols, next_trim.trim_end(), ); } } } } } } if over_notes > 0 { println!("Indent check: {over_notes} typing over-indent note(s) (not failures; review for regressions)"); } match errors { 0 => { println!("Indent check succeeded"); Ok(()) } n => Err(format!("Indent check failed: {n} line(s) with wrong indentation").into()), } } pub fn themecheck(themes: impl Iterator) -> Result<(), DynError> { use helix_view::theme::Loader; let themes_to_check: HashSet<_> = themes.collect(); let theme_names = [ vec!["default".to_string(), "base16_default".to_string()], Loader::read_names(&crate::path::themes()), ] .concat(); let loader = Loader::new(&[crate::path::runtime()]); let mut errors_present = false; for name in theme_names { if !themes_to_check.is_empty() && !themes_to_check.contains(&name) { continue; } let (_, warnings) = loader.load_with_warnings(&name).unwrap(); if !warnings.is_empty() { errors_present = true; println!("Theme '{name}' loaded with errors:"); for warning in warnings { println!("\t* {}", warning); } } } match errors_present { true => Err("Errors found when loading bundled themes".into()), false => { println!("Theme check successful!"); Ok(()) } } } pub fn highlightcheck(args: impl Iterator) -> Result<(), DynError> { use helix_core::syntax::{HighlightEvent, Loader, Syntax}; use helix_core::Language; use ropey::Rope; // The highlighter yields a `Highlight` index into the loader's scope // list. Feed it the capture names actually used across the bundled // queries, so every capture maps to itself and we read back the exact // `@capture` that won — no hand-maintained scope list. (Inheritance is // irrelevant here: scanning every language's queries unions all names.) fn capture_scopes() -> Vec { let mut set = std::collections::BTreeSet::new(); let Ok(langs) = std::fs::read_dir(crate::path::ts_queries()) else { return Vec::new(); }; for lang in langs.filter_map(Result::ok) { for q in ["highlights.scm", "locals.scm"] { let Ok(text) = std::fs::read_to_string(lang.path().join(q)) else { continue; }; let b = text.as_bytes(); let mut i = 0; while i < b.len() { if b[i] != b'@' { i += 1; continue; } let s = i + 1; let mut j = s; while j < b.len() && (b[j].is_ascii_alphanumeric() || matches!(b[j], b'.' | b'_' | b'-')) { j += 1; } let name = &text[s..j]; i = j; if name.is_empty() || name.starts_with('_') { continue; } // Locals: the highlight applied to a resolved reference is // the class after `local.definition.`; the rest are structural. if let Some(class) = name.strip_prefix("local.definition.") { if !class.is_empty() { set.insert(class.to_string()); } } else if !name.starts_with("local.") && name != "local" { set.insert(name.to_string()); } } } } set.into_iter().collect() } // Winning (top-of-stack) capture per byte range, via the real highlighter. fn spans( loader: &Loader, scopes: &[String], language: Language, source: &str, ) -> Option> { let rope = Rope::from_str(source); // None when the grammar isn't built, so callers can skip rather than // panic (a no-language run only needs the corpus' grammars built). let syntax = Syntax::new(rope.slice(..), language, loader).ok()?; let mut hl = syntax.highlighter(rope.slice(..), loader, ..); let mut active: Vec = Vec::new(); let mut start: u32 = 0; let len = source.len() as u32; let mut out = Vec::new(); loop { let off = hl.next_event_offset(); let cur = if off == u32::MAX { len } else { off }; if cur > start { if let Some(idx) = active.last() { out.push((start as usize, cur as usize, scopes[*idx as usize].clone())); } } if off == u32::MAX { break; } let (event, highlights) = hl.advance(); let v: Vec = highlights.map(|h| h.get()).collect(); match event { HighlightEvent::Refresh => active = v, HighlightEvent::Push => active.extend(v), } start = off; } Some(out) } let scopes = capture_scopes(); let loader = helix_core::config::default_lang_loader(); loader.set_scopes(scopes.clone()); let args: Vec = args.collect(); // `--dump `: print the winning capture per span, for // discovering the exact `@capture` names when authoring assertions. if args.first().map(String::as_str) == Some("--dump") { let lang = args .get(1) .ok_or("usage: highlight-check --dump ")?; let file = args .get(2) .ok_or("usage: highlight-check --dump ")?; let language = loader .language_for_name(lang.as_str()) .ok_or_else(|| format!("unknown language '{lang}'"))?; let source = std::fs::read_to_string(file)?; let sp = spans(&loader, &scopes, language, &source) .ok_or_else(|| format!("could not highlight (grammar for '{lang}' not built?)"))?; for (s, e, scope) in sp { let text = &source[s..e]; if !text.trim().is_empty() { println!("{scope}\t{text:?}"); } } return Ok(()); } // Corpus mode: nvim-treesitter-style highlight tests laid out under // tests/query/highlights//.. A comment line // below the code carries caret assertions whose columns line up with the // tokens above: // // foo(bar) // // ^ @function // // ^^^ @variable // // Each `^` checks the *winning* capture at the column above it; the // expected `@capture` must match exactly (a leading `!` negates). This // catches precedence bugs query-check cannot — e.g. a call captured on a // wrapper node that the inner `(identifier) @variable` then wins. let filter: HashSet = args.into_iter().collect(); let root = crate::path::tests_highlight(); let mut files: Vec<(String, std::path::PathBuf)> = Vec::new(); for lang_dir in std::fs::read_dir(&root)?.filter_map(Result::ok) { if !lang_dir.path().is_dir() { continue; } let lang = lang_dir.file_name().to_string_lossy().into_owned(); if !filter.is_empty() && !filter.contains(&lang) { continue; } for f in std::fs::read_dir(lang_dir.path())?.filter_map(Result::ok) { if f.path().is_file() { files.push((lang.clone(), f.path())); } } } files.sort(); let mut errors = 0usize; let mut checks = 0usize; for (lang, path) in files { let language = match loader .languages() .find(|(_, d)| d.config().language_id == lang) { Some((language, _)) => language, None => { return Err( format!("{}: no configured language '{lang}'", path.display()).into(), ) } }; let source = std::fs::read_to_string(&path)?; let Some(sp) = spans(&loader, &scopes, language, &source) else { eprintln!("{}: skipped ('{lang}' grammar not built)", path.display()); continue; }; let at = |byte: usize| -> Option<&str> { sp.iter() .find(|(s, e, _)| *s <= byte && byte < *e) .map(|(_, _, n)| n.as_str()) }; let lines: Vec<&str> = source.lines().collect(); let mut line_start = Vec::with_capacity(lines.len()); let mut off = 0usize; for l in &lines { line_start.push(off); off += l.len() + 1; } let rel = path.strip_prefix(&root).unwrap_or(&path).display(); let mut base: Option = None; for (li, line) in lines.iter().enumerate() { // An assertion line is a comment whose first `^` is preceded only // by the comment leader (no alphanumerics), so code like `a ^ b` // is never mistaken for one. let marker = line .find('^') .filter(|&c| !line[..c].bytes().any(|b| b.is_ascii_alphanumeric())); let Some(col) = marker else { if !line.trim().is_empty() { base = Some(li); } continue; }; let Some(brow) = base else { continue }; let len = line[col..].bytes().take_while(|b| *b == b'^').count(); let mut exp = line[col + len..].trim(); let negate = exp.starts_with('!'); if negate { exp = exp[1..].trim_start(); } // First whitespace-delimited token after the caret, with a // leading `@` stripped — so a trailing comment close (OCaml // `*)`, HTML `-->`) isn't swallowed into the expected name. let exp = exp .trim_start_matches('@') .split_whitespace() .next() .unwrap_or(""); if exp.is_empty() { continue; } let code = lines[brow]; for c in col..col + len { // Skip carets that fall past the code line or over whitespace. match code.as_bytes().get(c) { None => continue, Some(b) if b.is_ascii_whitespace() => continue, _ => {} } checks += 1; let got = at(line_start[brow] + c); if (got == Some(exp)) != negate { continue; } errors += 1; eprintln!( "{rel}:{}:{}: expected {}{} but got {}", brow + 1, c + 1, if negate { "not " } else { "" }, exp, got.unwrap_or("") ); } } } if errors > 0 { return Err( format!("Highlight check failed: {errors} of {checks} assertion(s) wrong").into(), ); } println!("Highlight check succeeded ({checks} assertions)"); Ok(()) } pub fn print_help() { println!( " Usage: Run with `cargo xtask `, eg. `cargo xtask docgen`. Tasks: docgen Generate files to be included in the mdbook output. query-check [languages] Check that tree-sitter queries are valid for the given languages, or all languages if none are specified. indent-check [languages] Check indentation for the corpus files in tests/indent/ (named .) against the configured grammars, for the given languages, or all corpus files if none are specified. highlight-check [languages] Check highlight queries against the real highlighter using the nvim-treesitter-style tests under tests/query/highlights/ /. (caret comment lines like `// ^^^ @capture` assert the winning capture at the column above), for the given languages, or all if none are specified. `highlight-check --dump ` instead prints the winning capture per span for an arbitrary input file. theme-check [themes] Check that the theme files in runtime/themes/ are valid for the given themes, or all themes if none are specified. " ); } } fn main() -> Result<(), DynError> { let mut args = env::args().skip(1); let task = args.next(); match task { None => tasks::print_help(), Some(t) => match t.as_str() { "docgen" => tasks::docgen()?, "query-check" => tasks::querycheck(args)?, "indent-check" => tasks::indentcheck(args)?, "highlight-check" => tasks::highlightcheck(args)?, "theme-check" => tasks::themecheck(args)?, invalid => return Err(format!("Invalid task name: {}", invalid).into()), }, }; Ok(()) }