From 4fcac5c11449cc9322e7407026159b48d3b2638d Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Sun, 23 Nov 2025 20:53:19 -0600 Subject: [PATCH] feat: add status endpoint * restructuring for manpage generation --- Cargo.lock | 18 +++++++ DEPLOYMENT.md | 5 +- README.md | 32 ++++++++++-- cli/Cargo.toml | 8 +++ cli/build.rs | 26 ++++++++++ cli/src/app.rs | 112 ++++++++++++++++++++++++++++++++++++++++++ cli/src/main.rs | 121 +++------------------------------------------- cli/src/server.rs | 68 +++++++++++++++++++++++++- 8 files changed, 269 insertions(+), 121 deletions(-) create mode 100644 cli/build.rs create mode 100644 cli/src/app.rs diff --git a/Cargo.lock b/Cargo.lock index 308cbbe..b6a90e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -242,6 +242,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +[[package]] +name = "clap_mangen" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ea63a92086df93893164221ad4f24142086d535b3a0957b9b9bea2dc86301" +dependencies = [ + "clap", + "roff", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -994,12 +1004,14 @@ dependencies = [ "axum", "chrono", "clap", + "clap_mangen", "dirs", "owo-colors", "pai-core", "rusqlite", "serde", "serde_json", + "tempfile", "tokio", ] @@ -1182,6 +1194,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" + [[package]] name = "rusqlite" version = "0.37.0" diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 56be05a..5c24b0c 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -160,9 +160,10 @@ Use the same `Caddyfile` contents as above, but point `reverse_proxy` to `pai:80 ## Health Checks & Monitoring -- `GET /api/feed?limit=1` ensures the server can read from SQLite. +- `GET /status` – lightweight JSON (`status`, total items, counts per `source_kind`). Ideal for load balancer health probes. +- `GET /api/feed?limit=1` ensures the server can read from SQLite and return real data. - `GET /api/item/{id}` is handy for debugging a specific record. -- Consider adding nginx/Caddy health check endpoints (`/healthz`) that proxy to `/api/feed?limit=1` and monitor via your platform. +- Consider wiring `/status` into nginx/Caddy health checks (`/healthz`) or your platform’s monitoring agents. ## Security Tips diff --git a/README.md b/README.md index 862c553..42040d5 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ A CLI that ingests content from Substack, Bluesky, and Leaflet into SQLite, with - **Bluesky** via AT Protocol - **Leaflet** publications via RSS feeds - Local SQLite storage with full-text search -- Flexible filtering and querying -- Self-hostable or serverless (Cloudflare Workers) +- Flexible filtering and querying via `pai list` / `pai export` +- Self-hostable HTTP API (`pai serve` exposes `/api/feed`, `/api/item/{id}`, and `/status`) +- Cloudflare Worker deployment path (D1) for serverless setups ## Quick Start @@ -36,12 +37,37 @@ pai list -n 10 pai db-check ``` +
+For server mode, run the built-in HTTP server against your SQLite database: + +
+ +```bash +pai serve -d /var/lib/pai/pai.db -a 127.0.0.1:8080 +``` + +Endpoints: + +- `GET /api/feed` – list newest items (supports `source_kind`, `source_id`, `limit`, `since`, `q`) +- `GET /api/item/{id}` – fetch a single item +- `GET /status` – health/status summary (total items, counts per source) + +For reverse-proxy examples (nginx, Caddy, Docker), see [DEPLOYMENT.md](./DEPLOYMENT.md). + +
+ ## Configuration Configuration is loaded from `$XDG_CONFIG_HOME/pai/config.toml` or `$HOME/.config/pai/config.toml`. See [config.example.toml](./config.example.toml) for a complete example with all available options. +## Documentation + +- CLI synopsis: `pai -h`, `pai -h`, or `pai man` for the generated `pai(1)` page. +- Database schema and config reference: [config.example.toml](./config.example.toml). +- Deployment topologies: [DEPLOYMENT.md](./DEPLOYMENT.md). + ## Architecture The project is organized as a Cargo workspace @@ -207,4 +233,4 @@ base_url = "https://stormlightlabs.leaflet.pub" ## License -See [LICENSE file](./LICENSE) for details. +See [LICENSE](./LICENSE) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a47d128..3bf43cf 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -18,3 +18,11 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } axum = "0.7" tokio = { version = "1.40", features = ["macros", "rt-multi-thread", "signal"] } + +[dev-dependencies] +tempfile = "3.13" + +[build-dependencies] +clap = { version = "4.5", features = ["derive"] } +clap_mangen = "0.2" +pai-core = { path = "../core" } diff --git a/cli/build.rs b/cli/build.rs new file mode 100644 index 0000000..8744cba --- /dev/null +++ b/cli/build.rs @@ -0,0 +1,26 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + println!("cargo:rerun-if-changed=src/app.rs"); + println!("cargo:rerun-if-changed=src/main.rs"); + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR not set by Cargo build script environment")); + let man_path = out_dir.join("pai.1"); + + let man = build_manpage(); + fs::write(&man_path, man).expect("failed to write manpage"); + println!("cargo:rustc-env=PAI_MAN_PAGE={}", man_path.display()); +} + +#[path = "src/app.rs"] +mod app; + +fn build_manpage() -> Vec { + use clap::CommandFactory; + + let cmd = app::Cli::command(); + let man = clap_mangen::Man::new(cmd); + let mut buffer = Vec::new(); + man.render(&mut buffer).expect("failed to render man page"); + buffer +} diff --git a/cli/src/app.rs b/cli/src/app.rs new file mode 100644 index 0000000..dff2549 --- /dev/null +++ b/cli/src/app.rs @@ -0,0 +1,112 @@ +use clap::{Parser, Subcommand}; +use pai_core::SourceKind; +use std::path::PathBuf; + +/// Personal Activity Index - POSIX-style CLI for content aggregation +#[derive(Parser, Debug)] +#[command(name = "pai")] +#[command(version, about, long_about = None)] +pub struct Cli { + /// Set configuration directory + #[arg(short = 'C', value_name = "DIR", global = true)] + pub config_dir: Option, + + /// Path to SQLite database file + #[arg(short = 'd', value_name = "PATH", global = true)] + pub db_path: Option, + + #[command(subcommand)] + pub command: Commands, +} + +#[derive(Parser, Debug)] +pub struct ExportOpts { + /// Filter by source kind + #[arg(short = 'k', value_name = "KIND")] + pub kind: Option, + + /// Filter by specific source ID + #[arg(short = 'S', value_name = "ID")] + pub source_id: Option, + + /// Maximum number of items + #[arg(short = 'n', value_name = "NUMBER")] + pub limit: Option, + + /// Only items published at or after this time + #[arg(short = 's', value_name = "TIME")] + pub since: Option, + + /// Filter items by substring + #[arg(short = 'q', value_name = "PATTERN")] + pub query: Option, + + /// Output format + #[arg(short = 'f', value_name = "FORMAT", default_value = "json")] + pub format: String, + + /// Output file (default: stdout) + #[arg(short = 'o', value_name = "FILE")] + pub output: Option, +} + +#[derive(Subcommand, Debug)] +pub enum Commands { + /// Fetch and store content from configured sources + Sync { + /// Sync all configured sources (default) + #[arg(short = 'a')] + all: bool, + + /// Sync only a particular source kind + #[arg(short = 'k', value_name = "KIND")] + kind: Option, + + /// Sync only a specific source instance + #[arg(short = 'S', value_name = "ID")] + source_id: Option, + }, + + /// Inspect stored items + List { + /// Filter by source kind + #[arg(short = 'k', value_name = "KIND")] + kind: Option, + + /// Filter by specific source ID + #[arg(short = 'S', value_name = "ID")] + source_id: Option, + + /// Maximum number of items to display + #[arg(short = 'n', value_name = "NUMBER", default_value = "20")] + limit: usize, + + /// Only show items published at or after this time + #[arg(short = 's', value_name = "TIME")] + since: Option, + + /// Filter items by substring in title/summary + #[arg(short = 'q', value_name = "PATTERN")] + query: Option, + }, + + /// Produce feeds or export files + Export(ExportOpts), + + /// Self-host HTTP API + Serve { + /// Address to bind HTTP server to + #[arg(short = 'a', value_name = "ADDRESS", default_value = "127.0.0.1:8080")] + address: String, + }, + + /// Verify database schema and print statistics + DbCheck, + + /// Initialize configuration file + Init { + /// Force overwrite existing config + #[arg(short = 'f')] + force: bool, + }, +} diff --git a/cli/src/main.rs b/cli/src/main.rs index ffc8e09..69bb1ee 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,9 +1,11 @@ +mod app; mod paths; mod server; mod storage; +use app::{Cli, Commands, ExportOpts}; use chrono::{DateTime, Duration, Utc}; -use clap::{Parser, Subcommand}; +use clap::Parser; use owo_colors::OwoColorize; use pai_core::{Config, Item, ListFilter, PaiError, SourceKind}; use std::fs::File; @@ -12,114 +14,10 @@ use std::path::PathBuf; use std::str::FromStr; use storage::SqliteStorage; -/// Personal Activity Index - POSIX-style CLI for content aggregation -#[derive(Parser, Debug)] -#[command(name = "pai")] -#[command(version, about, long_about = None)] -struct Cli { - /// Set configuration directory - #[arg(short = 'C', value_name = "DIR", global = true)] - config_dir: Option, - - /// Path to SQLite database file - #[arg(short = 'd', value_name = "PATH", global = true)] - db_path: Option, - - #[command(subcommand)] - command: Commands, -} - -#[derive(Parser, Debug)] -struct ExportOpts { - /// Filter by source kind - #[arg(short = 'k', value_name = "KIND")] - kind: Option, - - /// Filter by specific source ID - #[arg(short = 'S', value_name = "ID")] - source_id: Option, - - /// Maximum number of items - #[arg(short = 'n', value_name = "NUMBER")] - limit: Option, - - /// Only items published at or after this time - #[arg(short = 's', value_name = "TIME")] - since: Option, - - /// Filter items by substring - #[arg(short = 'q', value_name = "PATTERN")] - query: Option, - - /// Output format - #[arg(short = 'f', value_name = "FORMAT", default_value = "json")] - format: String, - - /// Output file (default: stdout) - #[arg(short = 'o', value_name = "FILE")] - output: Option, -} - -#[derive(Subcommand, Debug)] -enum Commands { - /// Fetch and store content from configured sources - Sync { - /// Sync all configured sources (default) - #[arg(short = 'a')] - all: bool, - - /// Sync only a particular source kind - #[arg(short = 'k', value_name = "KIND")] - kind: Option, - - /// Sync only a specific source instance - #[arg(short = 'S', value_name = "ID")] - source_id: Option, - }, - - /// Inspect stored items - List { - /// Filter by source kind - #[arg(short = 'k', value_name = "KIND")] - kind: Option, - - /// Filter by specific source ID - #[arg(short = 'S', value_name = "ID")] - source_id: Option, - - /// Maximum number of items to display - #[arg(short = 'n', value_name = "NUMBER", default_value = "20")] - limit: usize, - - /// Only show items published at or after this time - #[arg(short = 's', value_name = "TIME")] - since: Option, - - /// Filter items by substring in title/summary - #[arg(short = 'q', value_name = "PATTERN")] - query: Option, - }, - - /// Produce feeds or export files - Export(ExportOpts), - - /// Self-host HTTP API - Serve { - /// Address to bind HTTP server to - #[arg(short = 'a', value_name = "ADDRESS", default_value = "127.0.0.1:8080")] - address: String, - }, - - /// Verify database schema and print statistics - DbCheck, - - /// Initialize configuration file - Init { - /// Force overwrite existing config - #[arg(short = 'f')] - force: bool, - }, -} +const PUBLISHED_WIDTH: usize = 19; +const KIND_WIDTH: usize = 9; +const SOURCE_WIDTH: usize = 24; +const TITLE_WIDTH: usize = 60; fn main() { let cli = Cli::parse(); @@ -527,11 +425,6 @@ fn render_items_table(items: &[Item]) -> Result<(), PaiError> { } fn write_items_table(items: &[Item], writer: &mut W) -> io::Result<()> { - const PUBLISHED_WIDTH: usize = 19; - const KIND_WIDTH: usize = 9; - const SOURCE_WIDTH: usize = 24; - const TITLE_WIDTH: usize = 60; - let header = format!( "| {published: Result<(), PaiError> { } async fn run_server(db_path: PathBuf, addr: SocketAddr) -> Result<(), PaiError> { - // Ensure the database exists and schema is ready before serving requests. let storage = SqliteStorage::new(&db_path)?; storage.verify_schema()?; drop(storage); @@ -40,6 +40,7 @@ async fn run_server(db_path: PathBuf, addr: SocketAddr) -> Result<(), PaiError> let app = Router::new() .route("/api/feed", get(feed_handler)) .route("/api/item/:id", get(item_handler)) + .route("/status", get(status_handler)) .with_state(state); let listener = TcpListener::bind(addr).await.map_err(PaiError::Io)?; @@ -49,7 +50,7 @@ async fn run_server(db_path: PathBuf, addr: SocketAddr) -> Result<(), PaiError> axum::serve(listener, app.into_make_service()) .with_graceful_shutdown(shutdown_signal()) .await - .map_err(|err| PaiError::Io(std::io::Error::new(std::io::ErrorKind::Other, err))) + .map_err(|e| io::Error::other(e).into()) } #[derive(Clone)] @@ -61,6 +62,18 @@ impl AppState { fn open_storage(&self) -> Result { SqliteStorage::new(self.db_path.as_ref()) } + + fn status_snapshot(&self) -> Result { + let storage = self.open_storage()?; + let total_items = storage.count_items()?; + let sources = storage + .get_stats()? + .into_iter() + .map(|(kind, count)| SourceStat { kind, count }) + .collect(); + + Ok(StatusResponse { status: "ok", database_path: self.db_path.display().to_string(), total_items, sources }) + } } #[derive(Debug, Default, Deserialize)] @@ -95,6 +108,20 @@ struct FeedResponse { items: Vec, } +#[derive(Serialize)] +struct StatusResponse { + status: &'static str, + database_path: String, + total_items: usize, + sources: Vec, +} + +#[derive(Serialize)] +struct SourceStat { + kind: String, + count: usize, +} + async fn feed_handler( State(state): State, Query(query): Query, ) -> Result, ApiError> { @@ -114,6 +141,11 @@ async fn item_handler(State(state): State, Path(id): Path) -> Ok(Json(item)) } +async fn status_handler(State(state): State) -> Result, ApiError> { + let snapshot = state.status_snapshot()?; + Ok(Json(snapshot)) +} + struct ApiError { status: StatusCode, message: String, @@ -160,6 +192,9 @@ async fn shutdown_signal() { #[cfg(test)] mod tests { use super::*; + use chrono::Utc; + use pai_core::Storage; + use tempfile::tempdir; #[test] fn feed_query_defaults() { @@ -200,4 +235,33 @@ mod tests { let resp = ApiError::bad_request("oops").into_response(); assert_eq!(resp.status(), StatusCode::BAD_REQUEST); } + + #[test] + fn status_snapshot_reports_counts() { + let dir = tempdir().unwrap(); + let db_path = dir.path().join("status.db"); + let state = AppState { db_path: Arc::new(db_path.clone()) }; + + let storage = state.open_storage().unwrap(); + let now = Utc::now().to_rfc3339(); + let item = Item { + id: "status-test".to_string(), + source_kind: SourceKind::Substack, + source_id: "status.substack.com".to_string(), + author: None, + title: Some("Status".to_string()), + summary: None, + url: "https://example.com/status".to_string(), + content_html: None, + published_at: now.clone(), + created_at: now, + }; + storage.insert_or_replace_item(&item).unwrap(); + + let snapshot = state.status_snapshot().unwrap(); + assert_eq!(snapshot.status, "ok"); + assert_eq!(snapshot.total_items, 1); + assert_eq!(snapshot.sources.len(), 1); + assert_eq!(snapshot.sources[0].kind, "substack"); + } } -- 2.51.2