From e9ce38a588d7e5c4ca61bffa46d0815202189ab6 Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Wed, 3 Jun 2026 11:20:45 -0500 Subject: [PATCH] test: sample project with metadata/manifest checking --- .gitignore | 3 + Cargo.lock | 3 + crates/cli/src/args.rs | 5 + crates/cli/src/commands.rs | 28 ++ crates/core/Cargo.toml | 3 + crates/core/src/diagnostic.rs | 1 + crates/core/src/lib.rs | 1 + crates/core/src/project.rs | 379 ++++++++++++++++++ .../specs/01_project_model_and_modules.md | 33 +- .../tasks/01_project_model_and_modules.md | 16 +- docs/src/testing_docs.md | 1 + fixtures/README.md | 7 + fixtures/projects/scalar_app/gleam.toml | 8 + fixtures/projects/scalar_app/manifest.toml | 9 + fixtures/projects/scalar_app/src/app.gleam | 3 + 15 files changed, 487 insertions(+), 13 deletions(-) create mode 100644 crates/core/src/project.rs create mode 100644 fixtures/projects/scalar_app/gleam.toml create mode 100644 fixtures/projects/scalar_app/manifest.toml create mode 100644 fixtures/projects/scalar_app/src/app.gleam diff --git a/.gitignore b/.gitignore index c3ec7d2..357e4dd 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ rustc-ice-*.txt # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# gleam build artifacts +build/ diff --git a/Cargo.lock b/Cargo.lock index b7cebc3..fcbbd1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,9 @@ name = "compiler_core" version = "0.1.0" dependencies = [ "insta", + "serde", + "tempfile", + "toml", "tree-sitter", "tree-sitter-gleam", "wasmtime", diff --git a/crates/cli/src/args.rs b/crates/cli/src/args.rs index 8bffe55..1ad8216 100644 --- a/crates/cli/src/args.rs +++ b/crates/cli/src/args.rs @@ -17,4 +17,9 @@ pub enum Command { /// Gleam source file to compile. input: PathBuf, }, + /// Load a Gleam project and print discovered modules. + Project { + /// Project directory containing gleam.toml. + input: PathBuf, + }, } diff --git a/crates/cli/src/commands.rs b/crates/cli/src/commands.rs index 6d99cf7..fdb5a2d 100644 --- a/crates/cli/src/commands.rs +++ b/crates/cli/src/commands.rs @@ -7,6 +7,34 @@ use crate::{args::Command, echo}; pub fn run(command: Command) -> ExitCode { match command { Command::Compile { input } => compile(input), + Command::Project { input } => project(input), + } +} + +fn project(input: std::path::PathBuf) -> ExitCode { + match compiler_core::project::load_project(&input) { + Ok(project) => { + echo::status( + "project", + format!( + "{} {} ({} modules)", + project.config.name, + project.config.version, + project.graph.modules.len() + ), + ); + for module in project.graph.modules { + echo::status("module", format!("{} -> {}", module.name, module.path.display())); + } + ExitCode::SUCCESS + } + Err(diagnostics) => { + echo::error(format!("could not load project {}", input.display())); + for diagnostic in diagnostics { + echo::diagnostic(diagnostic.message); + } + ExitCode::FAILURE + } } } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 82dae87..136b24b 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -4,10 +4,13 @@ version = "0.1.0" edition = "2024" [dependencies] +serde = { version = "1.0.228", features = ["derive"] } +toml = "0.9.12" tree-sitter = "0.26.9" tree-sitter-gleam = "1.0.0" wat = "1.251.0" [dev-dependencies] insta = "1.43.2" +tempfile = "3.27.0" wasmtime = "45.0.0" diff --git a/crates/core/src/diagnostic.rs b/crates/core/src/diagnostic.rs index af84cc1..1349389 100644 --- a/crates/core/src/diagnostic.rs +++ b/crates/core/src/diagnostic.rs @@ -4,6 +4,7 @@ use crate::source::Span; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum DiagnosticCode { Unsupported, + ProjectError, ParseError, AstError, ResolveError, diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 09f08b6..86fea32 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -2,6 +2,7 @@ pub mod ast; pub mod diagnostic; pub mod ir; pub mod parse; +pub mod project; pub mod resolve; pub mod source; pub mod types; diff --git a/crates/core/src/project.rs b/crates/core/src/project.rs new file mode 100644 index 0000000..556ad04 --- /dev/null +++ b/crates/core/src/project.rs @@ -0,0 +1,379 @@ +use std::{ + collections::{BTreeMap, HashMap}, + fs, + path::{Path, PathBuf}, +}; + +use serde::Deserialize; + +use crate::{ + diagnostic::{Diagnostic, DiagnosticCode, Diagnostics}, + source::{SourceFile, SourceFileId}, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Project { + pub root: PathBuf, + pub config: GleamToml, + pub graph: PackageGraph, + pub sources: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageGraph { + pub root_package: PackageNode, + pub dependencies: Vec, + pub modules: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackageNode { + pub name: String, + pub version: String, + pub root: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Dependency { + pub name: String, + pub requirement: String, + pub dev: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ModuleInfo { + pub name: String, + pub path: PathBuf, + pub source_id: SourceFileId, + pub source_root: SourceRoot, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceRoot { + Src, + Test, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct GleamToml { + pub name: String, + pub version: String, + #[serde(default)] + pub description: Option, + #[serde(default)] + pub licences: Vec, + #[serde(default)] + pub repository: Option, + #[serde(default)] + pub links: Vec, + #[serde(default)] + pub gleam: Option, + #[serde(default)] + pub target: Option, + #[serde(default)] + pub dependencies: BTreeMap, + #[serde(rename = "dev-dependencies", default)] + pub dev_dependencies: BTreeMap, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +pub struct Link { + pub title: String, + pub href: String, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Target { + Erlang, + Javascript, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum DependencyToml { + Version(String), + Options { + version: Option, + path: Option, + git: Option, + }, +} + +impl DependencyToml { + fn requirement(&self) -> String { + match self { + DependencyToml::Version(version) => version.clone(), + DependencyToml::Options { version, path, git } => version + .clone() + .or_else(|| path.clone().map(|path| format!("path:{path}"))) + .or_else(|| git.clone().map(|git| format!("git:{git}"))) + .unwrap_or_else(|| "*".into()), + } + } +} + +pub fn load_project(path: impl AsRef) -> Result { + let root = project_root(path.as_ref()); + let config = read_config(&root)?; + let (sources, modules) = discover_modules(&root)?; + let dependencies = dependencies(&config); + + Ok(Project { + graph: PackageGraph { + root_package: PackageNode { + name: config.name.clone(), + version: config.version.clone(), + root: root.clone(), + }, + dependencies, + modules, + }, + root, + config, + sources, + }) +} + +pub fn source_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let text = fs::read_to_string(path).map_err(|error| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("could not read {}: {error}", path.display()), + )] + })?; + Ok(SourceFile::with_path(SourceFileId(0), path, text)) +} + +impl Project { + pub fn module(&self, name: &str) -> Result<&ModuleInfo, Diagnostics> { + self.graph + .modules + .iter() + .find(|module| module.name == name) + .ok_or_else(|| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("missing module `{name}`"), + )] + }) + } +} + +fn project_root(path: &Path) -> PathBuf { + if path.is_file() { path.parent().unwrap_or(path).to_path_buf() } else { path.to_path_buf() } +} + +fn read_config(root: &Path) -> Result { + let path = root.join("gleam.toml"); + let text = fs::read_to_string(&path).map_err(|error| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("could not read {}: {error}", path.display()), + )] + })?; + toml::from_str(&text).map_err(|error| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("could not parse {}: {error}", path.display()), + )] + }) +} + +fn dependencies(config: &GleamToml) -> Vec { + let normal = config.dependencies.iter().map(|(name, dependency)| Dependency { + name: name.clone(), + requirement: dependency.requirement(), + dev: false, + }); + let dev = config.dev_dependencies.iter().map(|(name, dependency)| Dependency { + name: name.clone(), + requirement: dependency.requirement(), + dev: true, + }); + normal.chain(dev).collect() +} + +fn discover_modules(root: &Path) -> Result<(Vec, Vec), Diagnostics> { + let mut entries = Vec::new(); + collect_gleam_files(root, SourceRoot::Src, &mut entries)?; + collect_gleam_files(root, SourceRoot::Test, &mut entries)?; + entries.sort_by(|a, b| a.1.cmp(&b.1)); + + let mut seen = HashMap::new(); + let mut sources = Vec::new(); + let mut modules = Vec::new(); + let mut diagnostics = Vec::new(); + + for (source_root, path) in entries { + let source_id = SourceFileId(sources.len() as u32); + let module_name = module_name(root, source_root, &path); + if let Some(previous) = seen.insert(module_name.clone(), path.clone()) { + diagnostics.push(Diagnostic::new( + DiagnosticCode::ProjectError, + format!( + "duplicate module `{}` in {} and {}", + module_name, + previous.display(), + path.display() + ), + )); + continue; + } + + let text = fs::read_to_string(&path).map_err(|error| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("could not read {}: {error}", path.display()), + )] + })?; + sources.push(SourceFile::with_path(source_id, path.clone(), text)); + modules.push(ModuleInfo { name: module_name, path, source_id, source_root }); + } + + if diagnostics.is_empty() { Ok((sources, modules)) } else { Err(diagnostics) } +} + +fn collect_gleam_files( + root: &Path, source_root: SourceRoot, entries: &mut Vec<(SourceRoot, PathBuf)>, +) -> Result<(), Diagnostics> { + let dir = match source_root { + SourceRoot::Src => root.join("src"), + SourceRoot::Test => root.join("test"), + }; + if !dir.exists() { + return Ok(()); + } + collect_gleam_files_in_dir(source_root, &dir, entries) +} + +fn collect_gleam_files_in_dir( + source_root: SourceRoot, dir: &Path, entries: &mut Vec<(SourceRoot, PathBuf)>, +) -> Result<(), Diagnostics> { + let read_dir = fs::read_dir(dir).map_err(|error| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("could not read directory {}: {error}", dir.display()), + )] + })?; + + for entry in read_dir { + let entry = entry.map_err(|error| { + vec![Diagnostic::new( + DiagnosticCode::ProjectError, + format!("could not read directory entry: {error}"), + )] + })?; + let path = entry.path(); + if path.is_dir() { + collect_gleam_files_in_dir(source_root, &path, entries)?; + } else if path.extension().is_some_and(|extension| extension == "gleam") { + entries.push((source_root, path)); + } + } + + Ok(()) +} + +fn module_name(root: &Path, source_root: SourceRoot, path: &Path) -> String { + let dir = match source_root { + SourceRoot::Src => root.join("src"), + SourceRoot::Test => root.join("test"), + }; + let relative = path.strip_prefix(dir).unwrap_or(path); + let without_extension = relative.with_extension(""); + without_extension + .components() + .map(|component| component.as_os_str().to_string_lossy()) + .collect::>() + .join("/") +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::*; + + fn write(path: &Path, text: &str) { + fs::create_dir_all(path.parent().expect("fixture parent")).expect("create fixture dir"); + fs::write(path, text).expect("write fixture"); + } + + #[test] + fn loads_gleam_toml_and_discovers_modules() { + let dir = tempdir().expect("tempdir"); + write( + &dir.path().join("gleam.toml"), + r#"name = "sample" +version = "1.0.0" +description = "sample project" +licences = ["Apache-2.0"] +target = "javascript" + +[dependencies] +gleam_stdlib = ">= 0.44.0 and < 2.0.0" + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +"#, + ); + write(&dir.path().join("src/app.gleam"), "pub fn main() { Nil }"); + write(&dir.path().join("src/app/view.gleam"), "pub fn view() { Nil }"); + + let project = load_project(dir.path()).expect("load project"); + + assert_eq!(project.config.name, "sample"); + assert_eq!(project.graph.modules.len(), 2); + assert_eq!(project.graph.modules[0].source_id, SourceFileId(0)); + assert!(project.graph.modules.iter().any(|module| module.name == "app/view")); + assert_eq!(project.graph.dependencies.len(), 2); + } + + #[test] + fn reports_duplicate_modules_across_source_roots() { + let dir = tempdir().expect("tempdir"); + write( + &dir.path().join("gleam.toml"), + "name = \"sample\"\nversion = \"1.0.0\"\n", + ); + write(&dir.path().join("src/app.gleam"), "pub fn main() { Nil }"); + write(&dir.path().join("test/app.gleam"), "pub fn test() { Nil }"); + + let diagnostics = load_project(dir.path()).expect_err("duplicate should fail"); + + assert_eq!(diagnostics[0].code, DiagnosticCode::ProjectError); + assert!(diagnostics[0].message.contains("duplicate module `app`")); + } + + #[test] + fn reports_missing_modules() { + let dir = tempdir().expect("tempdir"); + write( + &dir.path().join("gleam.toml"), + "name = \"sample\"\nversion = \"1.0.0\"\n", + ); + write(&dir.path().join("src/app.gleam"), "pub fn main() { Nil }"); + let project = load_project(dir.path()).expect("load project"); + + let diagnostics = project.module("missing").expect_err("missing module should fail"); + + assert_eq!(diagnostics[0].code, DiagnosticCode::ProjectError); + assert!(diagnostics[0].message.contains("missing module `missing`")); + } + + #[test] + fn keeps_single_file_loading_available() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("main.gleam"); + write(&path, "pub fn main() { Nil }"); + + let source = source_file(&path).expect("load source file"); + + assert_eq!(source.id, SourceFileId(0)); + assert_eq!(source.path.as_deref(), Some(path.as_path())); + } +} diff --git a/docs/src/internal/specs/01_project_model_and_modules.md b/docs/src/internal/specs/01_project_model_and_modules.md index 4b08af8..4d9fe63 100644 --- a/docs/src/internal/specs/01_project_model_and_modules.md +++ b/docs/src/internal/specs/01_project_model_and_modules.md @@ -5,27 +5,50 @@ The project model is responsible for discovering modules, reading package configuration, and giving later work a complete view of source files and package dependencies. +## Gleam project configuration + +Project configuration is read from `gleam.toml`. The compiler currently models +the package fields documented by Gleam, including package `name`, `version`, +`description`, `licences`, `repository`, `links`, `gleam`, `target`, +`dependencies`, and `dev-dependencies`. + +Dependency entries may be plain version requirements or option tables containing +metadata such as `version`, `path`, or `git`. The project model records these as +dependency hooks; dependency source loading can be added after the root project +model is stable. + ## Responsibilities - Read `gleam.toml` and project source directories. - Assign stable source file IDs to every module. - Map module names to source files. -- Discover dependency packages and their module interfaces. +- Discover dependency package metadata. - Report duplicate modules and missing modules. - Support single-file tests without requiring a full project. ## Data model -The project model should produce a package graph with module metadata: +The project model produces a package graph with module metadata: -- package name and version where available -- source root +- package name and version +- project root - module name - source path - source file ID -- dependency edges between packages +- dependency requirements + +## Fixture direction + +`fixtures/projects/scalar_app` is the small project fixture that matches what the +compiler can do today. It should expand as language support grows. The long-term +sample project should be a Lustre app, because that exercises real Gleam project +structure, dependencies, UI code, records, custom types, and browser-facing WASM. ## Diagnostics Project diagnostics should use file paths when no source span exists. Once a source file is known, diagnostics should prefer normal source spans. + +## Reference + +- Gleam, `gleam.toml` documentation: https://gleam.run/writing-gleam/gleam-toml/ diff --git a/docs/src/internal/tasks/01_project_model_and_modules.md b/docs/src/internal/tasks/01_project_model_and_modules.md index b031fef..ecc7e34 100644 --- a/docs/src/internal/tasks/01_project_model_and_modules.md +++ b/docs/src/internal/tasks/01_project_model_and_modules.md @@ -6,14 +6,14 @@ Compile projects instead of only isolated source strings. ## Tasks -- [ ] Read `gleam.toml` into a project configuration type. -- [ ] Discover source files under project source directories. -- [ ] Map source files to Gleam module names. -- [ ] Assign stable source file IDs across a project. -- [ ] Build a package/module graph for project modules. -- [ ] Add dependency package metadata hooks. -- [ ] Report duplicate modules and missing module files. -- [ ] Keep single-file compilation available for tests. +- [x] Read `gleam.toml` into a project configuration type. +- [x] Discover source files under project source directories. +- [x] Map source files to Gleam module names. +- [x] Assign stable source file IDs across a project. +- [x] Build a package/module graph for project modules. +- [x] Add dependency package metadata hooks. +- [x] Report duplicate modules and missing module files. +- [x] Keep single-file compilation available for tests. ## Done when diff --git a/docs/src/testing_docs.md b/docs/src/testing_docs.md index 9ae83d5..5afc672 100644 --- a/docs/src/testing_docs.md +++ b/docs/src/testing_docs.md @@ -19,6 +19,7 @@ fixtures/ wasm/ e2e/ diagnostics/ + projects/ ``` Each fixture should focus on one language feature unless it is an end-to-end diff --git a/fixtures/README.md b/fixtures/README.md index 7ff2273..d936395 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -13,7 +13,14 @@ fixtures/ wasm/ # WAT and wasm codegen cases e2e/ # full compile-and-run cases diagnostics/ # diagnostic rendering cases + projects/ # whole Gleam project fixtures ``` Prefer small files with descriptive names. Keep expected output in tests or snapshot files next to the test that owns it. + +`fixtures/projects/scalar_app` is the smallest full project fixture. It should +expand over time toward a realistic Gleam application. + +The planned endgame sample project will be a Lustre app, but the fixture should +grow only as the compiler gains support for the required language and runtime features. diff --git a/fixtures/projects/scalar_app/gleam.toml b/fixtures/projects/scalar_app/gleam.toml new file mode 100644 index 0000000..31e0736 --- /dev/null +++ b/fixtures/projects/scalar_app/gleam.toml @@ -0,0 +1,8 @@ +name = "scalar_app" +version = "1.0.0" +description = "A small project fixture for the Gleam to WASM compiler." +licences = ["Apache-2.0"] +target = "javascript" + +[dependencies] +gleam_stdlib = ">= 0.44.0 and < 2.0.0" diff --git a/fixtures/projects/scalar_app/manifest.toml b/fixtures/projects/scalar_app/manifest.toml new file mode 100644 index 0000000..d3b8d5d --- /dev/null +++ b/fixtures/projects/scalar_app/manifest.toml @@ -0,0 +1,9 @@ +# This file was generated by Gleam +# You typically do not need to edit this file + +packages = [ + { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, +] + +[requirements] +gleam_stdlib = { version = ">= 0.44.0 and < 2.0.0" } diff --git a/fixtures/projects/scalar_app/src/app.gleam b/fixtures/projects/scalar_app/src/app.gleam new file mode 100644 index 0000000..bf267c2 --- /dev/null +++ b/fixtures/projects/scalar_app/src/app.gleam @@ -0,0 +1,3 @@ +pub fn id(x: Int) -> Int { + x +} -- 2.51.2