diff --git a/Cargo.lock b/Cargo.lock
index 308cbbe..b6a90e0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -242,6 +242,16 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
+[[package]]
+name = "clap_mangen"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ea63a92086df93893164221ad4f24142086d535b3a0957b9b9bea2dc86301"
+dependencies = [
+ "clap",
+ "roff",
+]
+
[[package]]
name = "colorchoice"
version = "1.0.4"
@@ -994,12 +1004,14 @@ dependencies = [
"axum",
"chrono",
"clap",
+ "clap_mangen",
"dirs",
"owo-colors",
"pai-core",
"rusqlite",
"serde",
"serde_json",
+ "tempfile",
"tokio",
]
@@ -1182,6 +1194,12 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "roff"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3"
+
[[package]]
name = "rusqlite"
version = "0.37.0"
diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md
index 56be05a..5c24b0c 100644
--- a/DEPLOYMENT.md
+++ b/DEPLOYMENT.md
@@ -160,9 +160,10 @@ Use the same `Caddyfile` contents as above, but point `reverse_proxy` to `pai:80
## Health Checks & Monitoring
-- `GET /api/feed?limit=1` ensures the server can read from SQLite.
+- `GET /status` – lightweight JSON (`status`, total items, counts per `source_kind`). Ideal for load balancer health probes.
+- `GET /api/feed?limit=1` ensures the server can read from SQLite and return real data.
- `GET /api/item/{id}` is handy for debugging a specific record.
-- Consider adding nginx/Caddy health check endpoints (`/healthz`) that proxy to `/api/feed?limit=1` and monitor via your platform.
+- Consider wiring `/status` into nginx/Caddy health checks (`/healthz`) or your platform’s monitoring agents.
## Security Tips
diff --git a/README.md b/README.md
index 862c553..42040d5 100644
--- a/README.md
+++ b/README.md
@@ -11,8 +11,9 @@ A CLI that ingests content from Substack, Bluesky, and Leaflet into SQLite, with
- **Bluesky** via AT Protocol
- **Leaflet** publications via RSS feeds
- Local SQLite storage with full-text search
-- Flexible filtering and querying
-- Self-hostable or serverless (Cloudflare Workers)
+- Flexible filtering and querying via `pai list` / `pai export`
+- Self-hostable HTTP API (`pai serve` exposes `/api/feed`, `/api/item/{id}`, and `/status`)
+- Cloudflare Worker deployment path (D1) for serverless setups
## Quick Start
@@ -36,12 +37,37 @@ pai list -n 10
pai db-check
```
+
+For server mode, run the built-in HTTP server against your SQLite database:
+
+
+
+```bash
+pai serve -d /var/lib/pai/pai.db -a 127.0.0.1:8080
+```
+
+Endpoints:
+
+- `GET /api/feed` – list newest items (supports `source_kind`, `source_id`, `limit`, `since`, `q`)
+- `GET /api/item/{id}` – fetch a single item
+- `GET /status` – health/status summary (total items, counts per source)
+
+For reverse-proxy examples (nginx, Caddy, Docker), see [DEPLOYMENT.md](./DEPLOYMENT.md).
+
+
+
## Configuration
Configuration is loaded from `$XDG_CONFIG_HOME/pai/config.toml` or `$HOME/.config/pai/config.toml`.
See [config.example.toml](./config.example.toml) for a complete example with all available options.
+## Documentation
+
+- CLI synopsis: `pai -h`, `pai -h`, or `pai man` for the generated `pai(1)` page.
+- Database schema and config reference: [config.example.toml](./config.example.toml).
+- Deployment topologies: [DEPLOYMENT.md](./DEPLOYMENT.md).
+
## Architecture
The project is organized as a Cargo workspace
@@ -207,4 +233,4 @@ base_url = "https://stormlightlabs.leaflet.pub"
## License
-See [LICENSE file](./LICENSE) for details.
+See [LICENSE](./LICENSE)
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index a47d128..3bf43cf 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -18,3 +18,11 @@ serde_json = "1.0"
serde = { version = "1.0", features = ["derive"] }
axum = "0.7"
tokio = { version = "1.40", features = ["macros", "rt-multi-thread", "signal"] }
+
+[dev-dependencies]
+tempfile = "3.13"
+
+[build-dependencies]
+clap = { version = "4.5", features = ["derive"] }
+clap_mangen = "0.2"
+pai-core = { path = "../core" }
diff --git a/cli/build.rs b/cli/build.rs
new file mode 100644
index 0000000..8744cba
--- /dev/null
+++ b/cli/build.rs
@@ -0,0 +1,26 @@
+use std::{env, fs, path::PathBuf};
+
+fn main() {
+ println!("cargo:rerun-if-changed=src/app.rs");
+ println!("cargo:rerun-if-changed=src/main.rs");
+
+ let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR not set by Cargo build script environment"));
+ let man_path = out_dir.join("pai.1");
+
+ let man = build_manpage();
+ fs::write(&man_path, man).expect("failed to write manpage");
+ println!("cargo:rustc-env=PAI_MAN_PAGE={}", man_path.display());
+}
+
+#[path = "src/app.rs"]
+mod app;
+
+fn build_manpage() -> Vec {
+ use clap::CommandFactory;
+
+ let cmd = app::Cli::command();
+ let man = clap_mangen::Man::new(cmd);
+ let mut buffer = Vec::new();
+ man.render(&mut buffer).expect("failed to render man page");
+ buffer
+}
diff --git a/cli/src/app.rs b/cli/src/app.rs
new file mode 100644
index 0000000..dff2549
--- /dev/null
+++ b/cli/src/app.rs
@@ -0,0 +1,112 @@
+use clap::{Parser, Subcommand};
+use pai_core::SourceKind;
+use std::path::PathBuf;
+
+/// Personal Activity Index - POSIX-style CLI for content aggregation
+#[derive(Parser, Debug)]
+#[command(name = "pai")]
+#[command(version, about, long_about = None)]
+pub struct Cli {
+ /// Set configuration directory
+ #[arg(short = 'C', value_name = "DIR", global = true)]
+ pub config_dir: Option,
+
+ /// Path to SQLite database file
+ #[arg(short = 'd', value_name = "PATH", global = true)]
+ pub db_path: Option,
+
+ #[command(subcommand)]
+ pub command: Commands,
+}
+
+#[derive(Parser, Debug)]
+pub struct ExportOpts {
+ /// Filter by source kind
+ #[arg(short = 'k', value_name = "KIND")]
+ pub kind: Option,
+
+ /// Filter by specific source ID
+ #[arg(short = 'S', value_name = "ID")]
+ pub source_id: Option,
+
+ /// Maximum number of items
+ #[arg(short = 'n', value_name = "NUMBER")]
+ pub limit: Option,
+
+ /// Only items published at or after this time
+ #[arg(short = 's', value_name = "TIME")]
+ pub since: Option,
+
+ /// Filter items by substring
+ #[arg(short = 'q', value_name = "PATTERN")]
+ pub query: Option,
+
+ /// Output format
+ #[arg(short = 'f', value_name = "FORMAT", default_value = "json")]
+ pub format: String,
+
+ /// Output file (default: stdout)
+ #[arg(short = 'o', value_name = "FILE")]
+ pub output: Option,
+}
+
+#[derive(Subcommand, Debug)]
+pub enum Commands {
+ /// Fetch and store content from configured sources
+ Sync {
+ /// Sync all configured sources (default)
+ #[arg(short = 'a')]
+ all: bool,
+
+ /// Sync only a particular source kind
+ #[arg(short = 'k', value_name = "KIND")]
+ kind: Option,
+
+ /// Sync only a specific source instance
+ #[arg(short = 'S', value_name = "ID")]
+ source_id: Option,
+ },
+
+ /// Inspect stored items
+ List {
+ /// Filter by source kind
+ #[arg(short = 'k', value_name = "KIND")]
+ kind: Option,
+
+ /// Filter by specific source ID
+ #[arg(short = 'S', value_name = "ID")]
+ source_id: Option,
+
+ /// Maximum number of items to display
+ #[arg(short = 'n', value_name = "NUMBER", default_value = "20")]
+ limit: usize,
+
+ /// Only show items published at or after this time
+ #[arg(short = 's', value_name = "TIME")]
+ since: Option,
+
+ /// Filter items by substring in title/summary
+ #[arg(short = 'q', value_name = "PATTERN")]
+ query: Option,
+ },
+
+ /// Produce feeds or export files
+ Export(ExportOpts),
+
+ /// Self-host HTTP API
+ Serve {
+ /// Address to bind HTTP server to
+ #[arg(short = 'a', value_name = "ADDRESS", default_value = "127.0.0.1:8080")]
+ address: String,
+ },
+
+ /// Verify database schema and print statistics
+ DbCheck,
+
+ /// Initialize configuration file
+ Init {
+ /// Force overwrite existing config
+ #[arg(short = 'f')]
+ force: bool,
+ },
+}
diff --git a/cli/src/main.rs b/cli/src/main.rs
index ffc8e09..69bb1ee 100644
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -1,9 +1,11 @@
+mod app;
mod paths;
mod server;
mod storage;
+use app::{Cli, Commands, ExportOpts};
use chrono::{DateTime, Duration, Utc};
-use clap::{Parser, Subcommand};
+use clap::Parser;
use owo_colors::OwoColorize;
use pai_core::{Config, Item, ListFilter, PaiError, SourceKind};
use std::fs::File;
@@ -12,114 +14,10 @@ use std::path::PathBuf;
use std::str::FromStr;
use storage::SqliteStorage;
-/// Personal Activity Index - POSIX-style CLI for content aggregation
-#[derive(Parser, Debug)]
-#[command(name = "pai")]
-#[command(version, about, long_about = None)]
-struct Cli {
- /// Set configuration directory
- #[arg(short = 'C', value_name = "DIR", global = true)]
- config_dir: Option,
-
- /// Path to SQLite database file
- #[arg(short = 'd', value_name = "PATH", global = true)]
- db_path: Option,
-
- #[command(subcommand)]
- command: Commands,
-}
-
-#[derive(Parser, Debug)]
-struct ExportOpts {
- /// Filter by source kind
- #[arg(short = 'k', value_name = "KIND")]
- kind: Option,
-
- /// Filter by specific source ID
- #[arg(short = 'S', value_name = "ID")]
- source_id: Option,
-
- /// Maximum number of items
- #[arg(short = 'n', value_name = "NUMBER")]
- limit: Option,
-
- /// Only items published at or after this time
- #[arg(short = 's', value_name = "TIME")]
- since: Option,
-
- /// Filter items by substring
- #[arg(short = 'q', value_name = "PATTERN")]
- query: Option,
-
- /// Output format
- #[arg(short = 'f', value_name = "FORMAT", default_value = "json")]
- format: String,
-
- /// Output file (default: stdout)
- #[arg(short = 'o', value_name = "FILE")]
- output: Option,
-}
-
-#[derive(Subcommand, Debug)]
-enum Commands {
- /// Fetch and store content from configured sources
- Sync {
- /// Sync all configured sources (default)
- #[arg(short = 'a')]
- all: bool,
-
- /// Sync only a particular source kind
- #[arg(short = 'k', value_name = "KIND")]
- kind: Option,
-
- /// Sync only a specific source instance
- #[arg(short = 'S', value_name = "ID")]
- source_id: Option,
- },
-
- /// Inspect stored items
- List {
- /// Filter by source kind
- #[arg(short = 'k', value_name = "KIND")]
- kind: Option,
-
- /// Filter by specific source ID
- #[arg(short = 'S', value_name = "ID")]
- source_id: Option,
-
- /// Maximum number of items to display
- #[arg(short = 'n', value_name = "NUMBER", default_value = "20")]
- limit: usize,
-
- /// Only show items published at or after this time
- #[arg(short = 's', value_name = "TIME")]
- since: Option,
-
- /// Filter items by substring in title/summary
- #[arg(short = 'q', value_name = "PATTERN")]
- query: Option,
- },
-
- /// Produce feeds or export files
- Export(ExportOpts),
-
- /// Self-host HTTP API
- Serve {
- /// Address to bind HTTP server to
- #[arg(short = 'a', value_name = "ADDRESS", default_value = "127.0.0.1:8080")]
- address: String,
- },
-
- /// Verify database schema and print statistics
- DbCheck,
-
- /// Initialize configuration file
- Init {
- /// Force overwrite existing config
- #[arg(short = 'f')]
- force: bool,
- },
-}
+const PUBLISHED_WIDTH: usize = 19;
+const KIND_WIDTH: usize = 9;
+const SOURCE_WIDTH: usize = 24;
+const TITLE_WIDTH: usize = 60;
fn main() {
let cli = Cli::parse();
@@ -527,11 +425,6 @@ fn render_items_table(items: &[Item]) -> Result<(), PaiError> {
}
fn write_items_table(items: &[Item], writer: &mut W) -> io::Result<()> {
- const PUBLISHED_WIDTH: usize = 19;
- const KIND_WIDTH: usize = 9;
- const SOURCE_WIDTH: usize = 24;
- const TITLE_WIDTH: usize = 60;
-
let header = format!(
"| {published: Result<(), PaiError> {
}
async fn run_server(db_path: PathBuf, addr: SocketAddr) -> Result<(), PaiError> {
- // Ensure the database exists and schema is ready before serving requests.
let storage = SqliteStorage::new(&db_path)?;
storage.verify_schema()?;
drop(storage);
@@ -40,6 +40,7 @@ async fn run_server(db_path: PathBuf, addr: SocketAddr) -> Result<(), PaiError>
let app = Router::new()
.route("/api/feed", get(feed_handler))
.route("/api/item/:id", get(item_handler))
+ .route("/status", get(status_handler))
.with_state(state);
let listener = TcpListener::bind(addr).await.map_err(PaiError::Io)?;
@@ -49,7 +50,7 @@ async fn run_server(db_path: PathBuf, addr: SocketAddr) -> Result<(), PaiError>
axum::serve(listener, app.into_make_service())
.with_graceful_shutdown(shutdown_signal())
.await
- .map_err(|err| PaiError::Io(std::io::Error::new(std::io::ErrorKind::Other, err)))
+ .map_err(|e| io::Error::other(e).into())
}
#[derive(Clone)]
@@ -61,6 +62,18 @@ impl AppState {
fn open_storage(&self) -> Result {
SqliteStorage::new(self.db_path.as_ref())
}
+
+ fn status_snapshot(&self) -> Result {
+ let storage = self.open_storage()?;
+ let total_items = storage.count_items()?;
+ let sources = storage
+ .get_stats()?
+ .into_iter()
+ .map(|(kind, count)| SourceStat { kind, count })
+ .collect();
+
+ Ok(StatusResponse { status: "ok", database_path: self.db_path.display().to_string(), total_items, sources })
+ }
}
#[derive(Debug, Default, Deserialize)]
@@ -95,6 +108,20 @@ struct FeedResponse {
items: Vec- ,
}
+#[derive(Serialize)]
+struct StatusResponse {
+ status: &'static str,
+ database_path: String,
+ total_items: usize,
+ sources: Vec,
+}
+
+#[derive(Serialize)]
+struct SourceStat {
+ kind: String,
+ count: usize,
+}
+
async fn feed_handler(
State(state): State, Query(query): Query,
) -> Result, ApiError> {
@@ -114,6 +141,11 @@ async fn item_handler(State(state): State, Path(id): Path) ->
Ok(Json(item))
}
+async fn status_handler(State(state): State) -> Result, ApiError> {
+ let snapshot = state.status_snapshot()?;
+ Ok(Json(snapshot))
+}
+
struct ApiError {
status: StatusCode,
message: String,
@@ -160,6 +192,9 @@ async fn shutdown_signal() {
#[cfg(test)]
mod tests {
use super::*;
+ use chrono::Utc;
+ use pai_core::Storage;
+ use tempfile::tempdir;
#[test]
fn feed_query_defaults() {
@@ -200,4 +235,33 @@ mod tests {
let resp = ApiError::bad_request("oops").into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
+
+ #[test]
+ fn status_snapshot_reports_counts() {
+ let dir = tempdir().unwrap();
+ let db_path = dir.path().join("status.db");
+ let state = AppState { db_path: Arc::new(db_path.clone()) };
+
+ let storage = state.open_storage().unwrap();
+ let now = Utc::now().to_rfc3339();
+ let item = Item {
+ id: "status-test".to_string(),
+ source_kind: SourceKind::Substack,
+ source_id: "status.substack.com".to_string(),
+ author: None,
+ title: Some("Status".to_string()),
+ summary: None,
+ url: "https://example.com/status".to_string(),
+ content_html: None,
+ published_at: now.clone(),
+ created_at: now,
+ };
+ storage.insert_or_replace_item(&item).unwrap();
+
+ let snapshot = state.status_snapshot().unwrap();
+ assert_eq!(snapshot.status, "ok");
+ assert_eq!(snapshot.total_items, 1);
+ assert_eq!(snapshot.sources.len(), 1);
+ assert_eq!(snapshot.sources[0].kind, "substack");
+ }
}