diff --git a/flake.nix b/flake.nix index f7893229..3414a21f 100644 --- a/flake.nix +++ b/flake.nix @@ -80,98 +80,108 @@ # Build *just* the cargo dependencies, so we can reuse # all of that work (e.g. via cachix) when running in CI - cargoArtifacts = craneLib.buildDepsOnly commonArgs; + # cargoArtifacts = craneLib.buildDepsOnly commonArgs; - individualCrateArgs = commonArgs // { - inherit cargoArtifacts; - inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; - # NB: we disable tests since we'll run them all via cargo-nextest - doCheck = false; - }; - fileSetForCrate = - crate: - lib.fileset.toSource { - root = ./.; - fileset = lib.fileset.unions [ - ./Cargo.toml - ./Cargo.lock - ./migrations - ./consumer/src/sources/jetstream/zstd_dictionary - (craneLib.fileset.commonCargoSources ./consumer) - ./consumer/src/db/sql - (craneLib.fileset.commonCargoSources ./dataloader-rs) - (craneLib.fileset.commonCargoSources ./did-resolver) - (craneLib.fileset.commonCargoSources ./lexica) - (craneLib.fileset.commonCargoSources ./parakeet) - ./parakeet/src/sql - (craneLib.fileset.commonCargoSources ./parakeet-db) - ./parakeet-db/src/dicts/post_content_v1.dict - (craneLib.fileset.commonCargoSources ./parakeet-lexgen) - (craneLib.fileset.commonCargoSources crate) - ]; - }; + # individualCrateArgs = commonArgs // { + # inherit cargoArtifacts; + # inherit (craneLib.crateNameFromCargoToml { inherit src; }) version; + # # NB: we disable tests since we'll run them all via cargo-nextest + # doCheck = false; + # }; + # fileSetForCrate = + # crate: + # lib.fileset.toSource { + # root = ./.; + # fileset = lib.fileset.unions [ + # ./Cargo.toml + # ./Cargo.lock + # ./migrations + # ./consumer/src/sources/jetstream/zstd_dictionary + # (craneLib.fileset.commonCargoSources ./consumer) + # ./consumer/src/db/sql + # (craneLib.fileset.commonCargoSources ./dataloader-rs) + # (craneLib.fileset.commonCargoSources ./did-resolver) + # (craneLib.fileset.commonCargoSources ./lexica) + # (craneLib.fileset.commonCargoSources ./parakeet) + # ./parakeet/src/sql + # (craneLib.fileset.commonCargoSources ./parakeet-db) + # ./parakeet-db/src/dicts/post_content_v1.dict + # (craneLib.fileset.commonCargoSources ./parakeet-lexgen) + # (craneLib.fileset.commonCargoSources crate) + # ]; + # }; # Build the actual crate itself, reusing the dependency # artifacts from above. - consumer = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "consumer"; - cargoExtraArgs = "-p consumer"; - src = fileSetForCrate ./consumer; - postInstall = '' - mkdir -p $out/{bin,lib/consumer} - ''; - } - ); - dataloader = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "dataloader"; - cargoExtraArgs = "-p dataloader"; - src = fileSetForCrate ./dataloader-rs; - } - ); - did-resolver = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "did-resolver"; - cargoExtraArgs = "-p did-resolver"; - src = fileSetForCrate ./did-resolver; - } - ); - lexica = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "lexica"; - cargoExtraArgs = "-p lexica"; - src = fileSetForCrate ./lexica; - } - ); - parakeet = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "parakeet"; - cargoExtraArgs = "-p parakeet"; - src = fileSetForCrate ./parakeet; - } - ); - parakeet-db = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "parakeet-db"; - cargoExtraArgs = "-p parakeet-db"; - src = fileSetForCrate ./parakeet-db; - } - ); - parakeet-lexgen = craneLib.buildPackage ( - individualCrateArgs - // { - pname = "parakeet-lexgen"; - cargoExtraArgs = "-p parakeet-lexgen"; - src = fileSetForCrate ./parakeet-lexgen; - } - ); + # COMMENTED OUT FOR DEV + # consumer = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "consumer"; + # cargoExtraArgs = "-p consumer"; + # src = fileSetForCrate ./consumer; + # postInstall = '' + # mkdir -p $out/{bin,lib/consumer} + # ''; + # } + # ); + # dataloader = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "dataloader"; + # cargoExtraArgs = "-p dataloader"; + # src = fileSetForCrate ./dataloader-rs; + # } + # ); + # did-resolver = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "did-resolver"; + # cargoExtraArgs = "-p did-resolver"; + # src = fileSetForCrate ./did-resolver; + # } + # ); + # lexica = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "lexica"; + # cargoExtraArgs = "-p lexica"; + # src = fileSetForCrate ./lexica; + # } + # ); + # parakeet = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "parakeet"; + # cargoExtraArgs = "-p parakeet"; + # src = fileSetForCrate ./parakeet; + # } + # ); + # parakeet-db = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "parakeet-db"; + # cargoExtraArgs = "-p parakeet-db"; + # src = fileSetForCrate ./parakeet-db; + # } + # ); + # parakeet-lexgen = craneLib.buildPackage ( + # individualCrateArgs + # // { + # pname = "parakeet-lexgen"; + # cargoExtraArgs = "-p parakeet-lexgen"; + # src = fileSetForCrate ./parakeet-lexgen; + # } + # ); + + # Dummy derivations to satisfy references + consumer = pkgs.writeTextDir "bin/consumer" "echo 'Consumer not built in dev mode'"; + dataloader = pkgs.writeTextDir "bin/dataloader" "echo 'Dataloader not built in dev mode'"; + did-resolver = pkgs.writeTextDir "bin/did-resolver" "echo 'DID resolver not built in dev mode'"; + lexica = pkgs.writeTextDir "bin/lexica" "echo 'Lexica not built in dev mode'"; + parakeet = pkgs.writeTextDir "bin/parakeet" "echo 'Parakeet not built in dev mode'"; + parakeet-db = pkgs.writeTextDir "bin/parakeet-db" "echo 'Parakeet-db not built in dev mode'"; + parakeet-lexgen = pkgs.writeTextDir "bin/parakeet-lexgen" "echo 'Parakeet-lexgen not built in dev mode'"; in { checks = { @@ -204,6 +214,23 @@ # Inherit inputs from checks. checks = self.checks.${system}; + # Manually provision devshell + nativeBuildInputs = with pkgs; [ + pkg-config + ]; + buildInputs = [ + pkgs.openssl + pkgs.postgresql + pkgs.libpq + pkgs.clang + pkgs.libclang + pkgs.lld + pkgs.protobuf + ]; + CLANG_PATH = "${pkgs.llvmPackages_18.clang}/bin/clang"; + PROTOC_INCLUDE = "${pkgs.protobuf}/include"; + PROTOC = "${pkgs.protobuf}/bin/protoc"; + # Additional dev-shell environment variables can be set directly RUST_BACKTRACE = 0; NIXOS_OZONE_WL = 1; @@ -227,6 +254,14 @@ tokei ast-grep scc + openssl + postgresql + libpq + clang + libclang + lld + protobuf + pkg-config ]; }; } @@ -277,111 +312,117 @@ }; }; config = mkIf cfg.enable { - environment.systemPackages = [ - self.packages.${pkgs.system}.consumer - ]; - systemd.services.consumer = { - description = "consumer"; - after = [ "network-online.target" "parakeet.service" ]; - wants = [ "network-online.target" ]; - requires = [ "parakeet.service" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - ExecStart = "${self.packages.${pkgs.system}.consumer}/bin/consumer --backfill --indexer"; - Type = "exec"; - WorkingDirectory = cfg.workingDirectory; + # environment.systemPackages = [ + # self.packages.${pkgs.system}.consumer + # ]; + # systemd.services.consumer = { + # description = "consumer"; + # after = [ + # "network-online.target" + # "parakeet.service" + # ]; + # wants = [ "network-online.target" ]; + # requires = [ "parakeet.service" ]; + # wantedBy = [ "multi-user.target" ]; + # serviceConfig = { + # ExecStart = "${self.packages.${pkgs.system}.consumer}/bin/consumer --backfill --indexer"; + # Type = "exec"; + # WorkingDirectory = cfg.workingDirectory; - EnvironmentFile = cfg.environmentFiles; - User = "parakeet"; - Group = "parakeet"; - StateDirectory = "parakeet"; - StateDirectoryMode = "0755"; - Restart = "always"; + # EnvironmentFile = cfg.environmentFiles; + # User = "parakeet"; + # Group = "parakeet"; + # StateDirectory = "parakeet"; + # StateDirectoryMode = "0755"; + # Restart = "always"; - # Hardening - RemoveIPC = true; - CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; - NoNewPrivileges = true; - PrivateDevices = true; - ProtectClock = true; - ProtectKernelLogs = true; - ProtectControlGroups = true; - ProtectKernelModules = true; - PrivateMounts = true; - SystemCallArchitectures = [ "native" ]; - MemoryDenyWriteExecute = false; # required by V8 JIT - RestrictNamespaces = true; - RestrictSUIDSGID = true; - ProtectHostname = true; - LockPersonality = true; - ProtectKernelTunables = true; - RestrictAddressFamilies = [ - "AF_UNIX" - "AF_INET" - "AF_INET6" - ]; - RestrictRealtime = true; - DeviceAllow = [ "" ]; - ProtectSystem = "full"; - ProtectProc = "invisible"; - ProcSubset = "pid"; - ProtectHome = true; - PrivateUsers = true; - PrivateTmp = true; - UMask = "0077"; - }; - }; - systemd.services.parakeet = { - description = "parakeet"; - after = [ "network-online.target" "postgresql.service" ]; - wants = [ "network-online.target" ]; - requires = [ "postgresql.service" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - ExecStart = "${cfg.package}/bin/parakeet"; - Type = "exec"; - WorkingDirectory = cfg.workingDirectory; + # # Hardening + # RemoveIPC = true; + # CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + # NoNewPrivileges = true; + # PrivateDevices = true; + # ProtectClock = true; + # ProtectKernelLogs = true; + # ProtectControlGroups = true; + # ProtectKernelModules = true; + # PrivateMounts = true; + # SystemCallArchitectures = [ "native" ]; + # MemoryDenyWriteExecute = false; # required by V8 JIT + # RestrictNamespaces = true; + # RestrictSUIDSGID = true; + # ProtectHostname = true; + # LockPersonality = true; + # ProtectKernelTunables = true; + # RestrictAddressFamilies = [ + # "AF_UNIX" + # "AF_INET" + # "AF_INET6" + # ]; + # RestrictRealtime = true; + # DeviceAllow = [ "" ]; + # ProtectSystem = "full"; + # ProtectProc = "invisible"; + # ProcSubset = "pid"; + # ProtectHome = true; + # PrivateUsers = true; + # PrivateTmp = true; + # UMask = "0077"; + # }; + # }; + # systemd.services.parakeet = { + # description = "parakeet"; + # after = [ + # "network-online.target" + # "postgresql.service" + # ]; + # wants = [ "network-online.target" ]; + # requires = [ "postgresql.service" ]; + # wantedBy = [ "multi-user.target" ]; + # serviceConfig = { + # ExecStart = "${cfg.package}/bin/parakeet"; + # Type = "exec"; + # WorkingDirectory = cfg.workingDirectory; - EnvironmentFile = cfg.environmentFiles; - User = "parakeet"; - Group = "parakeet"; - StateDirectory = "parakeet"; - StateDirectoryMode = "0755"; - Restart = "always"; + # EnvironmentFile = cfg.environmentFiles; + # User = "parakeet"; + # Group = "parakeet"; + # StateDirectory = "parakeet"; + # StateDirectoryMode = "0755"; + # Restart = "always"; - # Hardening - RemoveIPC = true; - CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; - NoNewPrivileges = true; - PrivateDevices = true; - ProtectClock = true; - ProtectKernelLogs = true; - ProtectControlGroups = true; - ProtectKernelModules = true; - PrivateMounts = true; - SystemCallArchitectures = [ "native" ]; - MemoryDenyWriteExecute = false; # required by V8 JIT - RestrictNamespaces = true; - RestrictSUIDSGID = true; - ProtectHostname = true; - LockPersonality = true; - ProtectKernelTunables = true; - RestrictAddressFamilies = [ - "AF_UNIX" - "AF_INET" - "AF_INET6" - ]; - RestrictRealtime = true; - DeviceAllow = [ "" ]; - ProtectSystem = "full"; - ProtectProc = "invisible"; - ProcSubset = "pid"; - ProtectHome = true; - PrivateUsers = true; - PrivateTmp = true; - UMask = "0077"; - }; - }; + # # Hardening + # RemoveIPC = true; + # CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ]; + # NoNewPrivileges = true; + # PrivateDevices = true; + # ProtectClock = true; + # ProtectKernelLogs = true; + # ProtectControlGroups = true; + # ProtectKernelModules = true; + # PrivateMounts = true; + # SystemCallArchitectures = [ "native" ]; + # MemoryDenyWriteExecute = false; # required by V8 JIT + # RestrictNamespaces = true; + # RestrictSUIDSGID = true; + # ProtectHostname = true; + # LockPersonality = true; + # ProtectKernelTunables = true; + # RestrictAddressFamilies = [ + # "AF_UNIX" + # "AF_INET" + # "AF_INET6" + # ]; + # RestrictRealtime = true; + # DeviceAllow = [ "" ]; + # ProtectSystem = "full"; + # ProtectProc = "invisible"; + # ProcSubset = "pid"; + # ProtectHome = true; + # PrivateUsers = true; + # PrivateTmp = true; + # UMask = "0077"; + # }; + # }; users = { users.parakeet = { group = "parakeet"; @@ -408,10 +449,10 @@ extraPlugins = with pkgs.postgresql16Packages; [ # Note: pg_stat_statements is built into PostgreSQL (contrib module) # and doesn't need to be listed here - just enable via CREATE EXTENSION - pgvector # Vector similarity search (future-proofing) - timescaledb # Time-series optimization - pgrouting # Graph analysis for social graph - postgis # Required dependency for pgrouting + pgvector # Vector similarity search (future-proofing) + timescaledb # Time-series optimization + pgrouting # Graph analysis for social graph + postgis # Required dependency for pgrouting ]; settings = { # Preload extensions that require early initialization @@ -419,8 +460,8 @@ shared_preload_libraries = "pg_stat_statements,timescaledb"; # pg_stat_statements configuration - "pg_stat_statements.max" = "10000"; # Track up to 10k unique queries - "pg_stat_statements.track" = "all"; # Track all queries (top-level + nested) + "pg_stat_statements.max" = "10000"; # Track up to 10k unique queries + "pg_stat_statements.track" = "all"; # Track all queries (top-level + nested) }; }; };