diff --git a/Cargo.lock b/Cargo.lock index 9af9bfc..3979719 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,7 +197,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43" dependencies = [ - "bindgen", + "bindgen 0.69.5", "cc", "cmake", "dunce", @@ -372,6 +372,24 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.8.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -2129,11 +2147,11 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "librocksdb-sys" -version = "0.17.1+9.9.3" +version = "0.17.3+10.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" +checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9" dependencies = [ - "bindgen", + "bindgen 0.72.1", "bzip2-sys", "cc", "libc", @@ -3459,9 +3477,9 @@ dependencies = [ [[package]] name = "rocksdb" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" +checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f" dependencies = [ "libc", "librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index facafdf..047438c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,3 +2,10 @@ resolver = "2" members = ["crates/*"] + +[workspace.package] +version = "0.1.0" +edition = "2024" + +[workspace.metadata.crane] +name = "parakeet" diff --git a/crates/consumer/src/config.rs b/crates/consumer/src/config.rs index 6628182..dc4998c 100644 --- a/crates/consumer/src/config.rs +++ b/crates/consumer/src/config.rs @@ -35,6 +35,8 @@ pub struct Config { pub struct ConfigInstruments { #[serde(default)] pub log_json: bool, + #[serde(default = "default_metrics_port")] + pub metrics_port: u16, } #[derive(Debug, Deserialize)] @@ -75,6 +77,10 @@ pub struct BackfillConfig { pub download_tmp_dir: String, } +fn default_metrics_port() -> u16 { + 9000 +} + fn default_backfill_workers() -> u8 { 4 } diff --git a/crates/consumer/src/main.rs b/crates/consumer/src/main.rs index 8682a37..e568647 100644 --- a/crates/consumer/src/main.rs +++ b/crates/consumer/src/main.rs @@ -18,11 +18,16 @@ mod utils; #[tokio::main] async fn main() -> eyre::Result<()> { - PrometheusBuilder::new().install()?; - let cli = cmd::parse(); let conf = config::load_config()?; + PrometheusBuilder::new() + .with_http_listener(std::net::SocketAddr::new( + [0, 0, 0, 0].into(), + conf.instruments.metrics_port, + )) + .install()?; + instrumentation::init_instruments(&conf.instruments); let user_agent = build_ua(&conf.ua_contact); diff --git a/crates/parakeet-index/Cargo.toml b/crates/parakeet-index/Cargo.toml index 0187efb..23dd2a9 100644 --- a/crates/parakeet-index/Cargo.toml +++ b/crates/parakeet-index/Cargo.toml @@ -19,7 +19,7 @@ itertools = { version = "0.14.0", optional = true } opentelemetry = { version = "0.31.0", optional = true } opentelemetry-otlp = { version = "0.31.0", features = ["reqwest-rustls"], optional = true } opentelemetry_sdk = { version = "0.31.0", optional = true } -rocksdb = { version = "0.23", default-features = false, features = ["lz4", "bindgen-runtime"], optional = true } +rocksdb = { version = "0.24", default-features = false, features = ["lz4", "bindgen-runtime"], optional = true } serde = { version = "1.0.217", features = ["derive"], optional = true } tokio = { version = "1.42.0", features = ["full"], optional = true } tonic-health = { version = "0.13.0", optional = true } @@ -47,4 +47,4 @@ server = [ "dep:tracing", "dep:tracing-subscriber", "dep:tracing-opentelemetry" -] \ No newline at end of file +] diff --git a/crates/parakeet-index/build.rs b/crates/parakeet-index/build.rs index ec77e8a..52c95e1 100644 --- a/crates/parakeet-index/build.rs +++ b/crates/parakeet-index/build.rs @@ -1,5 +1,8 @@ fn main() -> Result<(), Box> { - tonic_build::configure().compile_protos(&["proto/parakeet.proto"], &[""])?; + tonic_build::configure().compile_protos( + &[std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("proto/parakeet.proto")], + &[std::path::Path::new(env!("CARGO_MANIFEST_DIR"))], + )?; Ok(()) } diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..e189f63 --- /dev/null +++ b/flake.lock @@ -0,0 +1,98 @@ +{ + "nodes": { + "crane": { + "locked": { + "lastModified": 1768319649, + "narHash": "sha256-VFkNyxHxkqGp8gf8kfFMW1j6XeBy609kv6TE9uF/0Js=", + "owner": "ipetkov", + "repo": "crane", + "rev": "4b6527687cfd20da3c2ef8287e01b74c2d6c705b", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1768395095, + "narHash": "sha256-ZhuYJbwbZT32QA95tSkXd9zXHcdZj90EzHpEXBMabaw=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "13868c071cc73a5e9f610c47d7bb08e5da64fdd5", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "crane": "crane", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1769222645, + "narHash": "sha256-gu6oZ86zLudBZMq8LL1qdtYt/S69GV5keQVXdvBrVSU=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "22da29e7f3d8cff75009cbbcf992c7cb66920cfd", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..bcbf670 --- /dev/null +++ b/flake.nix @@ -0,0 +1,145 @@ +{ + description = "Parakeet - a Rust-based Bluesky AppView for Bluesky client"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + crane.url = "github:ipetkov/crane"; + flake-utils.url = "github:numtide/flake-utils"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = + { + self, + nixpkgs, + crane, + flake-utils, + rust-overlay, + ... + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; + + inherit (pkgs) lib; + inherit (lib.fileset) toSource unions union; + + craneLib = (crane.mkLib pkgs).overrideToolchain (p: p.rust-bin.stable.latest.default); + src = toSource { + root = ./.; + fileset = unions [ (craneLib.fileset.commonCargoSources ./.) ]; + }; + + commonArgs = { + inherit src; + strictDeps = true; + + nativeBuildInputs = with pkgs; [ pkg-config ]; + buildInputs = with pkgs; [ + clang + libclang + openssl + protobuf + ]; + + CLANG_PATH = "${pkgs.clang}/bin/clang"; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + PROTOC = "${pkgs.protobuf}/bin/protoc"; + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; + }; + + cargoArtifacts = craneLib.buildDepsOnly commonArgs; + + individualCrateArgs = commonArgs // { + inherit cargoArtifacts; + inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; + doCheck = false; + }; + + fileSetForCrate = + crate: add: + toSource { + root = ./.; + fileset = union (unions [ + ./Cargo.lock + ./Cargo.toml + (craneLib.fileset.commonCargoSources ./crates/lexica) + (craneLib.fileset.commonCargoSources ./crates/parakeet-db) + (craneLib.fileset.commonCargoSources ./crates/parakeet-index) + ./crates/parakeet-index/proto + (craneLib.fileset.commonCargoSources crate) + ]) add; + }; + + parakeet-appview = craneLib.buildPackage ( + individualCrateArgs + // { + pname = "parakeet"; + cargoExtraArgs = "-p parakeet"; + src = fileSetForCrate ./crates/parakeet (unions [ + ./migrations + (craneLib.fileset.commonCargoSources ./crates/dataloader-rs) + (craneLib.fileset.commonCargoSources ./crates/did-resolver) + (craneLib.fileset.commonCargoSources ./crates/parakeet) + ./crates/parakeet/src/sql + ]); + } + ); + + parakeet-consumer = craneLib.buildPackage ( + individualCrateArgs + // { + pname = "consumer"; + cargoExtraArgs = "-p consumer"; + src = fileSetForCrate ./crates/consumer (unions [ + ./crates/did-resolver + ./crates/consumer/src/db/sql + ]); + } + ); + + parakeet-index = craneLib.buildPackage ( + individualCrateArgs + // { + pname = "parakeet-index"; + cargoExtraArgs = "-p parakeet-index --features server"; + src = fileSetForCrate ./crates/parakeet-index (unions [ ]); + } + ); + in + { + packages = { + default = parakeet-appview; + inherit parakeet-appview parakeet-consumer parakeet-index; + }; + + apps = { + parakeet-appview = flake-utils.lib.mkApp { drv = parakeet-appview; }; + parakeet-consumer = flake-utils.lib.mkApp { drv = parakeet-consumer; }; + parakeet-index = flake-utils.lib.mkApp { drv = parakeet-index; }; + }; + + devShells.default = craneLib.devShell { + packages = with pkgs; [ + diesel-cli + protobuf + ]; + }; + } + ) + // flake-utils.lib.eachDefaultSystemPassThrough (system: { + nixosModules = { + default = import ./nix/services.nix { + inherit system; + parakeetSelf = self; + }; + }; + }); +} diff --git a/nix/services.nix b/nix/services.nix new file mode 100644 index 0000000..ba6ba9b --- /dev/null +++ b/nix/services.nix @@ -0,0 +1,187 @@ +{ parakeetSelf, system, ... }: +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (lib) + getExe + mkEnableOption + mkIf + mkOption + types + ; + + thisSystemPackages = parakeetSelf.packages.${system}; + + cfg = config.services.parakeet; + anyEnabled = cfg.appview.enable || cfg.consumer.enable || cfg.index.enable; + + settingsFormat = pkgs.formats.toml { }; + + toml = { + appview = settingsFormat.generate "Config-appview.toml" cfg.appview.settings; + consumer = settingsFormat.generate "Config-consumer.toml" cfg.consumer.settings; + index = settingsFormat.generate "Config-index.toml" cfg.index.settings; + }; + + buildService = + sub: + let + fullName = "parakeet-${sub}"; + thisCfg = cfg.${sub}; + thisToml = toml.${sub}; + in + { + description = "Parakeet ${sub}"; + + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + preStart = "cp -f ${thisToml} $STATE_DIRECTORY/Config.toml"; + + serviceConfig = { + User = "parakeet"; + Group = "parakeet"; + Restart = "always"; + ExecStart = "${getExe thisCfg.package}"; + + RuntimeDirectory = fullName; + StateDirectory = fullName; + WorkingDirectory = "/var/lib/${fullName}"; + + Environment = [ "RUST_LOG=${thisCfg.settings.RUST_LOG}" ]; + EnvironmentFile = mkIf (thisCfg.environmentFile != null) thisCfg.environmentFile; + }; + }; +in +{ + options.services.parakeet = { + appview = { + enable = mkEnableOption "Parakeet Appview"; + + package = mkOption { + type = types.package; + default = thisSystemPackages.parakeet-appview; + description = "The parakeet-appview package to use"; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "The environment file to use for parakeet-appview"; + }; + + settings = mkOption { + type = types.submodule { + freeformType = settingsFormat.type; + + options = { + RUST_LOG = mkOption { + type = types.str; + default = "info"; + description = "Logging level and filters to use. Uses tracing-subscriber EnvFilter"; + }; + }; + }; + }; + }; + + consumer = { + enable = mkEnableOption "Parakeet Consumer"; + + package = mkOption { + type = types.package; + default = thisSystemPackages.parakeet-consumer; + description = "The parakeet-consumer package to use"; + }; + + environmentFile = mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "The environment file to use for parakeet-consumer"; + }; + + settings = mkOption { + type = types.submodule { + freeformType = settingsFormat.type; + + options = { + RUST_LOG = mkOption { + type = types.str; + default = "info"; + description = "Logging level and filters to use. Uses tracing-subscriber EnvFilter"; + }; + + modes = mkOption { + type = types.listOf types.str; + default = [ "indexer" ]; + description = "Modes to run the consumer in. Accepts one or more of backfill, indexer, labels."; + }; + }; + }; + }; + }; + + index = { + enable = mkEnableOption "Parakeet Index"; + + package = mkOption { + type = types.package; + default = thisSystemPackages.parakeet-index; + description = "The parakeet-index package to use"; + }; + + environmentFile = mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = "The environment file to use for parakeet-index"; + }; + + settings = mkOption { + type = types.submodule { + freeformType = settingsFormat.type; + + options = { + RUST_LOG = mkOption { + type = types.str; + default = "info"; + description = "Logging level and filters to use. Uses tracing-subscriber EnvFilter"; + }; + }; + }; + }; + }; + }; + + config = { + environment.systemPackages = mkIf cfg.consumer.enable [ + thisSystemPackages.parakeet-consumer + ]; + + systemd.services.parakeet-appview = mkIf cfg.appview.enable (buildService "appview"); + + systemd.services.parakeet-consumer = mkIf cfg.consumer.enable ( + lib.recursiveUpdate (buildService "consumer") { + serviceConfig.ExecStart = "${getExe thisSystemPackages.parakeet-consumer} ${ + lib.strings.join " " (lib.map (x: "--${x}") cfg.consumer.settings.modes) + }"; + } + ); + + systemd.services.parakeet-index = mkIf cfg.index.enable (buildService "index"); + + users = mkIf anyEnabled { + users.parakeet = { + group = "parakeet"; + isSystemUser = true; + }; + + groups.parakeet = { }; + }; + }; +}