From f0d3a1a643f5c14c8f3cd1640e982a8b71b3dfef Mon Sep 17 00:00:00 2001 From: Lewis Date: Tue, 11 Aug 2026 16:40:10 +0300 Subject: [PATCH] docs,knot2,nix,systemd: write knot 2 switchover guide, add unit Lewis: May this revision serve well! --- docs/DOCS.md | 380 +++++++++++++++++++++ flake.nix | 1 - knot2/Containerfile | 5 +- knot2/README.md | 132 ++++++- knot2/crates/knot-migrate/src/emit.rs | 5 +- knot2/crates/knot-migrate/tests/migrate.rs | 43 ++- knot2/crates/knot-server/src/main.rs | 13 + nix/modules/knot-rs.nix | 26 +- systemd/knot.service | 52 +++ 9 files changed, 599 insertions(+), 58 deletions(-) create mode 100644 systemd/knot.service diff --git a/docs/DOCS.md b/docs/DOCS.md index 2977e3038..f27e8493d 100644 --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -387,6 +387,9 @@ So you want to run your own knot server? Great! Here are a few prerequisites: 2. A (sub)domain name. People generally use `knot.example.com`. 3. A valid SSL certificate for your domain. +Already running a knot and want to move it to knot 2? +See [Migrating to knot 2](#migrating-to-knot-2). + ## NixOS Refer to the [knot @@ -722,6 +725,383 @@ Reload `sshd` after making this change. > repositories have not yet been isolation-migrated. Re-run > `migrate-isolation` if you see this error. +## Migrating to Knot 2 + +Knot 2 is a second implementation of the knot concept. +It serves the same repositories under the same hostname and the same `did:web`, +and every clone URL and every `at://` reference will keep on working. +You'll move to it by running one offline tool while the knot is stopped. +Upgrading the old knot won't do a magic upgrade to knot 2 for you. +This is intentional; we don't want to give anyone a nasty surprise +if we'd switch out implementations under you or accidentally +brick someone's janky-but-working setup by attempting to shift +a jenga block underneath them. + +Here's what a switchover involves: + +- **Rehearsals:** try as many as you like, with the old knot still serving. +- **Downtime:** from the `systemctl disable` in step 1 to `listening on` in step 5, + mostly the copying, depending on how large your repositories are and how fast the disk is. +- **Parity:** the same knot on the same hostname, + with the same owners, members, & collaborators. +- **Untouched stuff for reversion:** the old database, + and under the default-copy-mode, the old repositories too. + +What do you get for upgrading? + +- Knot 2 doesn't have a SQLite database. Git is the only database, + so your members, collaborators, & repository owners will be stored in git itself. +- Knot 2 will serve SSH in-process, so you no longer need `sshd`, + the `AuthorizedKeysCommand`, or the unix `git` account. +- Knot 2 will read SSH keys live from each user's PDS. +- Repositories created from now on will stay SHA-1, matching what you already have. + Pass `--object-format sha256` to the migration if you'd rather modernize. + +The repo directories will come across whole, hooks and commit-graphs included. +Knot 2 doesn't do repo hooks, +and it will rewrite commit-graphs on its own maintenance schedule. + +### Before we begin + +Get tools handy! Install both binaries: + +``` +git clone https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is +cd core +nix build .#knot-rs && sudo install -m755 result/bin/knot-server /usr/local/bin/ +nix build .#knot-migrate && sudo install -m755 result/bin/knot-migrate /usr/local/bin/ +``` + +On NixOS, the `knot-rs` module will write the config file, the service unit, +& the state directory for you, +so read [Migrating on NixOS](#migrating-on-nixos) alongside this guide. +Run `./result/bin/knot-migrate` straight out of `nix build` instead of installing it. +The knot never runs the migrator, so the module leaves it off the system entirely. + +Make the dir that knot 2 will use, owned by the same user that the old knot runs as, +then write a master key for unsealing the per-repository signing keys: + +``` +sudo install -d -o git -g git /var/lib/knot +sudo install -d -m 700 /etc/knot +printf 'KNOT_MASTER_KEY=%s\n' "$(openssl rand -base64 32)" | sudo tee /etc/knot/knot.env > /dev/null +sudo chmod 600 /etc/knot/knot.env +``` + +That writes `/etc/knot/knot.env`, readable only by root: + +``` +KNOT_MASTER_KEY=Xm9x2t2m2WNSVJ+9v0Cq0ftFhwEuLW/8x0aeCcaHiWM= +``` + +**Back it up somewhere off the server.** +Seriously. +Seriously seriously. +The sealed key store is useless without it, +and the keys that it seals will sign every record that the migration writes. + +### Dress rehearsal, with old knot still serving requests + +`knot-migrate` will open its source database read-only, +so you can rehearse as often as you like without stopping anything. +Run it as root, since it'll read your `sshd` host key and the master key alongside the database: + +``` +sudo sh -c 'set -a; . /etc/knot/knot.env; set +a; exec knot-migrate \ + --source-db /home/git/knotserver.db \ + --env-file /home/git/.knot.env \ + --host-key /etc/ssh/ssh_host_ed25519_key \ + --plc-url https://plc.directory \ + --target /var/lib/knot \ + --dry-run' +``` + +The `--env-file` gives your hostname and repository scan path, +so you don't have to repeat them. +`--dry-run` won't touch the target, +though SQLite will still add `knotserver.db-shm` and `knotserver.db-wal` +beside a write-ahead-log database even to read it. +Rehearse until the exit code is 0 and the last block of the report reads +`scan path: writable`, `host key algorithm: ssh-ed25519`, +and `master key: KNOT_MASTER_KEY decodes to a usable key`. +Any line in the block with an error in it will stop the switchover. + +Mount each host directory at the same location inside the container if you'd rather run it that way, +such that every path in the command is still a host path. +Mount the target too, even though the rehearsal won't write there, +since it will measure the free space and the write permission on the target. +The `atcr.io` image runs as its own unprivileged `knot` user, +so pass `--user 0:0` to read the host key. +An image that you built yourself from `knot2/Containerfile` is distroless, +with its binaries in `/usr/local/bin` instead: + +``` +sudo sh -c 'set -a; . /etc/knot/knot.env; set +a; exec docker run --rm \ + --user 0:0 -v /home/git:/home/git -v /etc/ssh:/etc/ssh:ro \ + -v /var/lib/knot:/var/lib/knot -e KNOT_MASTER_KEY \ + --entrypoint /usr/bin/knot-migrate atcr.io/tangled.org/knot:2 \ + ' +``` + +The `:ro` on `/home/git` will work only while the old knot is still running. +Mount it writably from step 1 onward, rehearsals included, +even though the migration only ever reads your database. +SQLite will create an index file beside a write-ahead-log database to read it at all. +Under a `:ro` mount, with the knot stopped and its log checkpointed, it can't, +and `knot-migrate` errors out with `unable to open database file`. +`--consume-source` will move the repositories out of it, so mount it writably for that too. + +### Reading the report + +Then read what the report says! + +``` +knot owner: did:plc:akshay +members to grant: 3 +repos to adopt: 5 +collaborator grants: 4 + +casbin cross-check drift: +acl-only collaborator grants unioned in: 1 +did:plc:squid <- did:plc:boltless +table-only collaborator grants missing from acl: 1 +did:plc:octopus <- did:plc:dawn +repos with no acl owner marker where the owner regains push: 1 +did:plc:scallop + +skipped repos: 2 +did:plc:conch unrepresentable name "Test knot" +drops collaborator grant for did:plc:boltless +did:plc:tuna no git repository at the source path + +transfer mode: copy +the filesystem checks used /var/lib/knot, since the scan path doesn't exist yet +scan path: writable +room to copy: 89.1GiB free is enough for the 580.0KiB that adoption will copy +host key algorithm: ssh-ed25519 +master key: KNOT_MASTER_KEY decodes to a usable key + +we left the target alone. Re-run without --dry-run to migrate. +``` + +At the top is what's coming across, +`repos to adopt`, `members to grant`, and `collaborator grants`, +and they should match what you think you have. +Under them is every repository that `knot-migrate` will skip, each with its reason. +The last block is the preflight on the target, +where `transfer mode: copy` and `room to copy: ... is enough` +join the three lines you rehearsed against. + +**Skips.** If `repos to adopt` is lower than you expect, the skip list is where the rest went: +a name with a slash, a backslash, whitespace, a control character, `..`, +a bare `.`, or more than 100 bytes, and a record key with anything outside letters, +digits and `.-_:~`. Rename those on tangled.org and rehearse again, or accept losing them. + +**Owner drift.** `knot-migrate` will refuse the whole switchover +when the `acl` table and `repo_keys` are recorded with different owners for a repository. +The drift section prints both DIDs, one repository per line, as ` acl , repo_keys `. +Settle each repository in the old knot's database, +either by deleting the `acl` rows for the wrong owner or by correcting `repo_keys.owner_did`, +then rehearse again. +An extra `acl` owner marker beside the owner in `repo_keys` won't stop anything, +since `knot-migrate` reports it and discards it. + +**Room on disk.** Every disk measurement happens at `/repos`, +where your repositories end up. +Point `--target` at a directory whose parent already exists, +and mount or create the disk you mean to fill before you rehearse. +A rehearsal against your root filesystem measures your root filesystem. +Under `copy`, the default, your existing repositories stay where they are, +and `room to copy` counts a second copy of every repository it adopts. +`--consume-source` will move them instead, in seconds, with no extra room on disk, +and only where source and target are on one filesystem +and this process can write the old scan path. +Once they've moved, only a filesystem snapshot taken beforehand will bring them back, +even from a switchover that stopped halfway. +`knot-migrate` will refuse before it writes anything, +so your repos never end up split across two trees. + +> 🦪 **Note:** if you ran Secure Mode, each repo tree has a per-owner uid, +> so any one owner can read only their own. Root can read all of them. +> `knot-migrate` will refuse outright when it can't read a source path, +> and every repository is in the skip list with the error behind it. +> It will rehearse everything else first either way, +> and report that you need root alongside whatever else is outstanding. +> It skips a path that isn't there at all, as usual. +> `--skip-unreadable` will migrate the rest and leave those repositories on the old knot, +> which is what you want when root can't read them either, +> as with a dead disk or a stale network mount. +> `--consume-source` will empty the old scan path of every repository that it could read, +> so running it alongside `--skip-unreadable` leaves only the unreadable repositories on the old knot. + +### Doing the thing + +**Step 1.** Stop the knot and disable it, +so SQLite checkpoints its WAL and no restart of the machine can take port 5555 back from knot 2. +Leave it installed though, for "Going back": + +``` +sudo systemctl disable --now knotserver +``` + +**Step 2.** Copy database through a read-only connection, so the log stays untouched, +and keep the copy until the switchover is settled. +`knot-migrate` will never write to the source database. +The copy is there for everything else that can go wrong on the day, +and "Going back" has a restore-command: + +``` +sudo sqlite3 "file:/home/git/knotserver.db?mode=ro" \ + ".backup /var/tmp/knotserver-cutover.db" +``` + +**Step 3.** Run the same command that you rehearsed, without `--dry-run`, +then give the tree to the user that the service runs as, since root wrote all of it: + +``` +sudo chown -R git:git /var/lib/knot +``` + +`knot-migrate` will write your repositories, a `config.toml`, the imported host key, +the sealed key store, and `repo-signing-keys.json`. +That last file is every repository's private key in the clear, +the same keys that your old database stored, +so move it somewhere safe and off the server once the migration is done. +Towards the end of the copy, install +[`knot.service`](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/systemd/knot.service) +to `/etc/systemd/system/`, so step 5 is only a start. + +**Step 4.** Give port 22 to the knot, +if you want SSH clone URLs to keep working without a port number in them. +The order to do that in is under "Ports, Taylor's version" in the [knot 2 README](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/knot2/README.md), +since getting it wrong will lock you out of your own server. +Two moves there are migration-only: +move `/etc/ssh/sshd_config.d/authorized_keys_command.conf` outside `sshd_config.d`, +keeping it for going back, +and uncomment `ssh_listen_addr` in `/var/lib/knot/config.toml` to set it to `"[::]:22"`. + +**Step 5.** Start it and follow the log, watching the index rebuild from the meta-repo, +until `index ready` and then `listening on`: + +``` +sudo systemctl enable --now knot +sudo journalctl -fu knot +``` + +The knot rebuilds the index before it takes a connection. +The README's systemd section covers editing the unit for a different user, +a different binary path, or an LFS store outside `/var/lib/knot`. + +Both knots serve HTTP on port 5555 by default, +so you don't have to touch a reverse proxy already aimed at the old knot. +Configure the proxy in the knot anyway. +The knot will rate-limit by the address that a request arrives from, +and every one of your users arrives from the proxy until you do. +In `/var/lib/knot/config.toml`, +set `xrpc.trusted_proxy_header` to the header that your proxy appends, +and `xrpc.trusted_proxies` to the address that it connects from. +The README's TLS section spells both out at length. +An empty `trusted_proxies` leaves the knot honoring the header from every address, +so fill it in whenever anything but your proxy can connect to port 5555, +and keep 5555 off the public internet either way. + +### Migrating on NixOS + +> 🦪 +> +> NixOS is not my forte. Please read this and absolutely second-guess me. +> I have of course tested this, but still, a boy gets paranoid. + +Steps 1 and 2 read the same, and the `knot-rs` module replaces the rest, +so configure it as the [knot 2 README](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/knot2/README.md) describes. +Set `settings.git.object_format` to whatever you passed `--object-format`, +since the module defaults to `sha256` where the migration defaults to `sha1`. + +Run the migration before your first rebuild. +A knot that starts on an empty state directory won't serve any repositories, +and it will generate a host key of its own, +which `knot-migrate` will then refuse to overwrite. +A rebuild starts the knot the moment the module is on, +so if you want the module in place before you migrate, add this line first: + +``` +systemd.services.knot-rs.wantedBy = lib.mkForce []; +``` + +The knot then stays stopped until you start it by hand. +Migrate, `chown`, run `systemctl start knot-rs`, and delete the line at your next rebuild. + +If the knot already started once, `--force-host-key` writes the imported key over the one it generated. +The fingerprint everybody cached under the old knot is the one that comes back, +so it only surprises whoever connected while knot 2 was serving a key of its own. +The `config.toml` that `knot-migrate` writes is only a record of what it chose, +because the file the knot reads is the module's. + +Then run the migration and hand the tree to the module's user, +`knot`, where the Go module used `git`, and rebuild. +The `--source-repos` and `--hostname` flags replace the `--env-file` above, +since the Go module configures the old knot through systemd and will never write an env file. +Append `--dry-run` to rehearse it first, as often as you like: + +``` +nix build .#knot-migrate +sudo sh -c 'set -a; . /etc/knot/knot.env; set +a; exec ./result/bin/knot-migrate \ + --source-db /home/git/knotserver.db \ + --source-repos /home/git \ + --hostname knot.example.com \ + --host-key /etc/ssh/ssh_host_ed25519_key \ + --plc-url https://plc.directory \ + --target /var/lib/knot' +sudo chown -R knot:knot /var/lib/knot +sudo nixos-rebuild switch +``` + +The source paths above are the Go module's defaults, +where `stateDir` is `/home/git` and `repo.scanPath` is the state directory itself, +so read your own out of `services.tangled.knot` if you moved either. +"Check it worked" and everything after it apply unchanged, +with `systemctl status knot-rs` in place of `knot`. + +### Checking that it worked + +``` +curl -s https://knot.example.com/xrpc/_health +curl -s https://knot.example.com/xrpc/sh.tangled.owner +ssh-keyscan -t ed25519 your.knot.com +``` + +The health endpoint should answer `{"version":"knot 2.0.0"}`, +`sh.tangled.owner` should answer with your own DID, +and the keyscan should match the fingerprint that your users already have. +Then clone a repository anonymously, and push to a repository that you own. + +> 🦪 +> +> knot 2 will take SSH keys from each user's PDS on an hour's lease, +> where the old knot kept a copy that could outlive the record. +> So when someone's push stops working: is their key record still in their PDS? +> Have they re-added the key in their Tangled settings, which writes it back? +> Have they waited out the lease since? +> Lower `keyfill.ttl_secs` if an hour is too long for you. + +### Going back in case of emergency + +If you ran the migration in its default copy mode, +your old data is exactly where it was. +Put `sshd` back on 22, move `authorized_keys_command.conf` back into `sshd_config.d`, +and start the old knot: + +``` +sudo systemctl disable --now knot +sudo systemctl enable --now knotserver +``` + +Restore `/var/tmp/knotserver-cutover.db` over `/home/git/knotserver.db` only if something wrote to the original, +which the migration itself never will. +If you used `--consume-source`, your repositories moved instead of being copied, +and only a filesystem snapshot taken before the migration +or your own little dir-reverse script would bring repos back. + ## Troubleshooting If you run your own knot, you may run into some of these diff --git a/flake.nix b/flake.nix index 7706fadc5..ea89f025c 100644 --- a/flake.nix +++ b/flake.nix @@ -682,7 +682,6 @@ imports = [./nix/modules/knot-rs.nix]; services.tangled.knot-rs.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knot-rs; - services.tangled.knot-rs.migratePackage = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.knot-migrate; }; nixosModules.spindle = { lib, diff --git a/knot2/Containerfile b/knot2/Containerfile index c6d615215..4b5f6f10b 100644 --- a/knot2/Containerfile +++ b/knot2/Containerfile @@ -12,11 +12,12 @@ COPY lexicons ./lexicons COPY knot2/lexicons ./knot2/lexicons COPY knot2/crates ./knot2/crates COPY knot2/third_party ./knot2/third_party -RUN cargo build --release --package knot-server -RUN strip target/release/knot-server +RUN cargo build --release --package knot-server --package knot-migrate +RUN strip target/release/knot-server target/release/knot-migrate FROM gcr.io/distroless/cc-debian13:latest@sha256:1e3c6d9c255be500eb680cdea0ad07554f52ae92dfcbdf07043a2a435b4c1fe3 COPY --from=builder /src/target/release/knot-server /usr/local/bin/knot-server +COPY --from=builder /src/target/release/knot-migrate /usr/local/bin/knot-migrate EXPOSE 5555 2222 ENTRYPOINT ["/usr/local/bin/knot-server"] CMD ["/etc/knot/config.toml"] diff --git a/knot2/README.md b/knot2/README.md index 56127308f..2f05b09d7 100644 --- a/knot2/README.md +++ b/knot2/README.md @@ -20,6 +20,13 @@ Back to knot 2 today: Have fun! +Already running the Go knot and want this instead? +Stop it once and point `knot-migrate` at its database. +Afterwards the knot will serve the same hostname with the same repos, +owners, members and collaborators. +Upgrading the Go knot won't do a magic upgrade for you. +The walkthrough is in [the docs](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/docs/DOCS.md#migrating-to-knot-2). + # Running a knot2 The following is how I actually run `knot.oyster.cafe`. Please treat it as one possible setup. @@ -90,22 +97,33 @@ If the reader is nodding along instead of saying "no Lewis I won't change the re Moving sshd is something that can lock one out of one's own server, so do it in this order: -1. Open a second SSH session to the server and keep it open for this whole procedure. If step 4 goes wrong, having this session open might be the saving grace. -2. Edit `/etc/ssh/sshd_config` and set `Port 2200` or whatever free port one likes. Leave `Port 22` in place as well for now, so sshd listens on both. -3. Open the new port in the firewall if there is one. On ufw that's `ufw allow 2200/tcp` (I think!! Untested). On a cloud provider one will probably also have to deal with it / open it in their proprietary config. -4. Restart sshd, and **from the local computer, in a third terminal**, confirm `ssh -p 2200 root@the.server` works before continuing. -5. Once confirmed, only now remove `Port 22` from `sshd_config`, restart sshd once more, & give the port to the knot. When running the binary directly, that means `ssh_listen_addr = "[::]:22"`. Comparatively, in a container it entails publishing the container's 2222 as the host's 22, which is what the compose file example below does. +1. Open a second SSH session to the server and keep it open for this whole procedure. If step 5 goes wrong, having this session open might be the saving grace. +2. Find out which thing owns the port with `systemctl is-enabled ssh.socket`, since each answer sends you to a different file in step 3: + - `enabled`: edit the socket unit. A `Port` line in `sshd_config` does nothing at all here, and editing it is the usual way to lose an afternoon. Ask me how I know. + - `disabled`, or no systemd: edit `Port` in `sshd_config`; sshd holds the port itself. +3. Edit `/etc/ssh/sshd_config` and set `Port 2200` or whatever free port one likes. Leave `Port 22` in place as well for now, so sshd listens on both. Under the socket unit, `systemctl edit ssh.socket` takes a `[Socket]` section with `ListenStream=2200` instead, such that the new port joins whatever's already listening. +4. Open the new port in the firewall if there is one. On ufw that's `ufw allow 2200/tcp` (I think!! Untested). On a cloud provider one will probably also have to deal with it / open it in their proprietary config. +5. Restart sshd, or `systemctl restart ssh.socket` for the socket unit, and **from the local computer, in a third terminal**, confirm `ssh -p 2200 root@the.server` works before continuing. +6. Once confirmed, only now remove `Port 22` from `sshd_config`, restart sshd once more, & give the port to the knot. Under the socket unit that's an empty `ListenStream=` above the `ListenStream=2200`, since a drop-in only adds to the port it inherits. When running the binary directly, that means `ssh_listen_addr = "[::]:22"`. Comparatively, in a container it entails publishing the container's 2222 as the host's 22, which is what the compose file example below does. If one would rather not move sshd at all, another cool option for having knot2 on port 22 is a second IP address on the remote computer. Bind sshd to one with `ListenAddress`, bind the knot to the other with `ssh_listen_addr = ":22"`, and just put the knot's DNS record on that second address. Leaving the knot on `[::]:22` would wildcard-bind every address on the box and would collide with sshd no matter which single IP that sshd listens on. HTTP can stay on 5555 behind a reverse proxy, or move to 443 if one wants the knot to terminate TLS itself. +## Running under systemd + +`systemd/knot.service` at the repository root is the unit I'd install to `/etc/systemd/system/` for running the binary straight on the machine, then `systemctl enable --now knot`. + +It runs the knot as `git` from `/usr/local/bin/knot-server`, so edit `User=`, `Group=` and `ExecStart=` if one's setup differs. `ProtectSystem=strict` and `ReadWritePaths=/var/lib/knot` mean anything the knot writes outside the tree gets its own `ReadWritePaths=` entry, an LFS store or an ACME cache on another disk being the usual suspects. `ProtectHome=true` has to go if any of the paths is under `/home`. The two `CAP_NET_BIND_SERVICE` lines let it bind a port below 1024, and both can go when SSH stays on 2222 and HTTP on 5555. + ## Running with containers -The `Containerfile` at project root builds a distroless image with just the `knot-server` binary in it: +`knot2/Containerfile` will build a distroless image with `knot-server` and `knot-migrate` in `/usr/local/bin`. +Its `COPY` lines start at the workspace root, +so build it from the repository root and point `-f` at it: ```sh -podman build -t knot-oyster:latest . +podman build -f knot2/Containerfile -t knot-oyster:latest . ``` I personally run it with a composefile. This is the file from `knot.oyster.cafe` with a few opsec adjustments: @@ -171,14 +189,59 @@ mkdir -p repos ssh secrets lfs podman-compose up -d ``` -The `mkdir` is necessary, since the knot won't start unless the repo and LFS directories are present/writable. Podman would create the bind-mount sources for the operator, but then they belong to whichever unix user podman has rather than to the operator. +The `mkdir` is necessary, since the knot won't start unless the repo and LFS directories are present/writable. Podman would create the bind-mount sources for the operator, but then they belong to whichever unix user podman has. The knot creates the SSH host key on the first run at mode 600, aaand the sealed store on that same first run, because the knot's own signing key needs sealing before any repo exists. The knot purposefully won't load a host key that is group or other readable, so don't loosen those please. -Note that this (my) config turns LFS on, since `lfs.store_path` is set. One can drop that whole `[lfs]` block if one doesn't want it. The floor of 30GiB is what I have judged for my disk (of 500GiB, doing other things at the same time), so pick something that suits one's own rather than copying mine. +Note that this (my) config turns LFS on, since `lfs.store_path` is set. One can drop that whole `[lfs]` block if one doesn't want it. The floor of 30GiB is what I have judged for my disk (of 500GiB, doing other things at the same time), so pick something that suits one's own instead of copying mine. Speaking of LFS, I made the directory different in the first place so that we could specify a whole separate storage medium if wanted. For example, let's say I want my actual git repos to be wicked fast, so everything *else* is on an SSD, and *only* LFS is on a massive-but-relatively-cheap HDD cluster. Wouldn't want terabytes and terabytes of massive files taking up precious SSD space in this economy! +## NixOS + +`nixosModules.knot-rs` renders the config file, defines the same hardened unit as above, and creates the state directory. A `config.toml` the operator writes goes unread under the module. Add the flake as an input, import the module, and write a `knot.nix`: + +```nix +{ + inputs.tangled.url = "git+https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is"; + outputs = {nixpkgs, tangled, ...}: { + nixosConfigurations.knot = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [tangled.nixosModules.knot-rs ./knot.nix]; + }; + }; +} +``` + +```nix +{ + services.tangled.knot-rs = { + enable = true; + environmentFile = "/etc/knot/knot.env"; + settings = { + server = { + hostname = "knot.oyster.cafe"; + admins = ["did:plc:nel"]; + ssh_listen_addr = "[::]:22"; + }; + atproto.plc_directory = "https://plc.directory"; + xrpc = { + trusted_proxy_header = "x-forwarded-for"; + trusted_proxies = ["127.0.0.1" "::1"]; + }; + }; + }; + + services.openssh.ports = [2200]; +} +``` + +- `environmentFile` is where `KNOT_MASTER_KEY=` goes, and the module refuses to build without it. Write the path, never the value, so the key stays out of the nix store. Every `KNOT_*` variable the file sets overrides the matching key in `settings`. +- Moving `services.openssh` to another port frees 22 for the knot, per the port dance above, and the module asserts the collision instead of starting two services on one port. It adds `CAP_NET_BIND_SERVICE` itself once a listen address is below 1024. Confirm a session on the new port before rebuilding, since a rebuild that moves sshd and takes 22 in one go will lock one out if the port is wrong. +- `openFirewall` defaults to true and opens the port of each listen address that isn't loopback, so the knot's 22 opens while the default `listen_addr = "127.0.0.1:5555"` stays shut for a proxy on the same host. `services.openssh` opens its own port. +- `stateDir` defaults to `/var/lib/knot`, and `systemd.tmpfiles` creates it and its `repos` at mode 0750 for the `knot` user. `settings.secrets.sealed_key_file` and `settings.server.ssh_host_key_file` default to `sealed-keys` and `ssh_host_key` inside it, and the module works out `ReadWritePaths=` from wherever the operator puts them. +- The module installs the knot and nothing else. [Migrating from the Go knot](https://tangled.org/did:plc:j5hmlfdrwkvtxm7cjmu7j2is/blob/master/docs/DOCS.md#migrating-to-knot-2) runs `knot-migrate` out of `nix build`, before the module is on the machine at all. + ## TLS My setup has Caddy in front, which gets me certificates for free and lets one machine serve several sites (which it does). The config is: @@ -277,7 +340,7 @@ git clone https://knot.oyster.cafe/did:plc:barnacle Push works over both, of course . For HTTP pushing, it is up to the user to find a good Tangled-CLI or something that can put the right things in the git credential helper such that a service auth token is minted and used on push. -A trailing `.git` on the repo name is optional, so `did:plc:nel/squid.git` goes to the same repo as `did:plc:nel/squid`. That only applies to the repo name variant though - `did:plc:barnacle.git` is read as a DID rather than as a repo-DID with a suffix, and it won't resolve. This would be made better from better DID parsing, since a `did:plc` can't have dots, only a `did:web` can. +A trailing `.git` on the repo name is optional, so `did:plc:nel/squid.git` goes to the same repo as `did:plc:nel/squid`. That only applies to the repo name variant though - `did:plc:barnacle.git` is read as a DID with a `.git` on the end of it, and it won't resolve. This would be made better from better DID parsing, since a `did:plc` can't have dots, only a `did:web` can. ## Things worth knowing before one commits (get it?) to a config @@ -315,11 +378,56 @@ Back these things up or don't come cryin' to me! ## Updating -// TODO: publish to ATCR, maybe nix something something. +Pre-built images are at `atcr.io/tangled.org/knot:2`, +with `knot-server` and `knot-migrate` in them. +The image comes from [@tangled.org/knot-docker](https://tangled.org/did:plc:f5s5la5wlofsxidb3zemdune) instead of the `Containerfile` here. +It's a Debian build, with its binaries in `/usr/bin`, +and the `:latest` tag over there is still the Go knot, +so one has to ask for `:2` on purpose. +My composefile above points at a locally built image under `pull_policy: never`. +Switching to the published image means editing the `image:` line, +or tagging the pulled image with the name the compose file already has: + +```sh +podman pull atcr.io/tangled.org/knot:2 +podman tag atcr.io/tangled.org/knot:2 localhost/knot-oyster:latest +podman-compose up -d +``` + +The published image bakes `KNOT_SCAN_PATH`, +`KNOT_SEALED_KEY_FILE`, +`KNOT_SSH_HOST_KEY_FILE` and `KNOT_MASTER_KEY_ENV` into itself, +and an environment variable overrides the config file. +The knot will then look for the sealed store at `/data/sealed-keys` and the host key at `/data/ssh_host_key`, +whatever the mounted `config.toml` sets, +find neither, +and generate a new identity and a new host key on a path that isn't mounted by anything. +Put my two paths back in the compose environment before switching to the published image: + +```yaml + environment: + KNOT_SEALED_KEY_FILE: /data/secrets/sealed.bin + KNOT_SSH_HOST_KEY_FILE: /data/ssh/host_key +``` + +The published image also runs as its own `knot` user at uid 1000, +where the distroless image here runs as root, +so chown the four bind mounts before the first start: + +```sh +sudo chown -R 1000:1000 repos ssh secrets lfs +``` + +Skipping the chown will stop the knot at startup with `repo.scan_path /data/repos isn't writable`, +since root wrote every one of the directories. +`user: "0:0"` in the compose file is the other way out, +at the cost of running the knot as root. + +Building it oneself is the same three lines as always: ```sh git pull -podman build -t knot-oyster:latest . +podman build -f knot2/Containerfile -t knot-oyster:latest . podman-compose up -d ``` diff --git a/knot2/crates/knot-migrate/src/emit.rs b/knot2/crates/knot-migrate/src/emit.rs index 2bc78a8e4..6cb3948df 100644 --- a/knot2/crates/knot-migrate/src/emit.rs +++ b/knot2/crates/knot-migrate/src/emit.rs @@ -476,10 +476,7 @@ pub fn plan_host_key( writable_target(destination, placement).map(|()| placement) } -fn writable_target( - destination: &Path, - placement: HostKeyPlacement, -) -> Result<(), HostKeyConflict> { +fn writable_target(destination: &Path, placement: HostKeyPlacement) -> Result<(), HostKeyConflict> { match placement { HostKeyPlacement::Fresh => { let blocked = match destination.parent() { diff --git a/knot2/crates/knot-migrate/tests/migrate.rs b/knot2/crates/knot-migrate/tests/migrate.rs index 70421d6a5..110cdf9af 100644 --- a/knot2/crates/knot-migrate/tests/migrate.rs +++ b/knot2/crates/knot-migrate/tests/migrate.rs @@ -1506,7 +1506,11 @@ fn a_host_key_target_this_process_cannot_write_is_refused_before_adoption() { chmod(&sealed, 0o500); assert!( matches!( - emit::plan_host_key(&sealed.join("ssh_host_key"), &key, emit::HostKeyPolicy::Keep), + emit::plan_host_key( + &sealed.join("ssh_host_key"), + &key, + emit::HostKeyPolicy::Keep + ), Err(emit::HostKeyConflict::Uncreatable { .. }) ), "an absent key under a directory nobody can write is a write that fails after adoption" @@ -1560,18 +1564,19 @@ fn a_rehearsal_report_names_what_the_key_at_the_target_costs() { skipped: Vec::new(), drift: mapping::Drift::default(), }; - let render = |host_key_target: Option>| { - let rehearsal = Rehearsal { - host_key_target, - ..ready_rehearsal() + let render = + |host_key_target: Option>| { + let rehearsal = Rehearsal { + host_key_target, + ..ready_rehearsal() + }; + report::Report { + mapping: &mapping, + orphan_alias_count: 0, + phase: report::Phase::Rehearsed(&rehearsal), + } + .to_string() }; - report::Report { - mapping: &mapping, - orphan_alias_count: 0, - phase: report::Phase::Rehearsed(&rehearsal), - } - .to_string() - }; [None, Some(Ok(emit::HostKeyPlacement::Fresh))] .into_iter() @@ -1591,8 +1596,7 @@ fn a_rehearsal_report_names_what_the_key_at_the_target_costs() { let replacing = render(Some(Ok(emit::HostKeyPlacement::Replacing))); assert!( - replacing - .contains("host key: the migration will replace the different key at the target"), + replacing.contains("host key: the migration will replace the different key at the target"), "{replacing}" ); @@ -1601,8 +1605,10 @@ fn a_rehearsal_report_names_what_the_key_at_the_target_costs() { source: rustix::io::Errno::ACCESS, }))); assert!( - unwritable.contains("host key: the migration will write the host key over \ - /srv/knot/ssh_host_key, which this process can't write"), + unwritable.contains( + "host key: the migration will write the host key over \ + /srv/knot/ssh_host_key, which this process can't write" + ), "{unwritable}" ); @@ -1734,8 +1740,9 @@ fn a_real_run_refuses_the_key_at_the_target_before_it_touches_a_repo() { assert!(!refused.status.success(), "{refusal}"); assert!(refusal.contains("your users already trust"), "{refusal}"); assert!( - refusal.contains(&format!("whose fingerprint {found} your users already trust")) - && refusal.contains(&importing.to_string()), + refusal.contains(&format!( + "whose fingerprint {found} your users already trust" + )) && refusal.contains(&importing.to_string()), "an operator deciding whether to force needs both fingerprints, not the word: {refusal}" ); assert!( diff --git a/knot2/crates/knot-server/src/main.rs b/knot2/crates/knot-server/src/main.rs index d062c3bca..1183bb92f 100644 --- a/knot2/crates/knot-server/src/main.rs +++ b/knot2/crates/knot-server/src/main.rs @@ -264,6 +264,12 @@ async fn main() -> anyhow::Result<()> { .context("decode master key as base64")?, ) .context("master key from environment")?; + if !config.secrets.sealed_key_file.exists() { + tracing::info!( + path = %config.secrets.sealed_key_file.display(), + "the knot will seal a new signing identity, since it can't find a key store at this path" + ); + } let secrets = Arc::new( SealedStore::open( &config.secrets.sealed_key_file, @@ -374,6 +380,13 @@ async fn main() -> anyhow::Result<()> { max_objects: ObjectCount::from(config.pack.selection_max_objects), time_budget: Duration::from_secs(config.pack.selection_time_budget_secs), }); + if !config.server.ssh_host_key_file.exists() { + tracing::info!( + path = %config.server.ssh_host_key_file.display(), + "the knot will generate an ssh host key, since it can't find a key at this path. \ + Anyone who already has this knot's fingerprint cached will see it change" + ); + } let host_key = knot_ssh::load_or_create_host_key(&config.server.ssh_host_key_file) .context("load or create SSH host key")?; diff --git a/nix/modules/knot-rs.nix b/nix/modules/knot-rs.nix index 2b3092c39..ad2d5df54 100644 --- a/nix/modules/knot-rs.nix +++ b/nix/modules/knot-rs.nix @@ -82,20 +82,6 @@ in { description = "Package providing the knot-server binary"; }; - migratePackage = mkOption { - type = types.package; - description = "Package providing the knot-migrate binary"; - }; - - installMigrateTool = mkOption { - type = types.bool; - default = false; - description = '' - Whether to instlal {option}`migratePackage` system-wide. - Only needed if doing a one-time migration from the Go knot. - ''; - }; - user = mkOption { type = types.str; default = "knot"; @@ -186,8 +172,8 @@ in { ssh_host_key_file = mkOption { type = absPathType; - default = "${cfg.stateDir}/ssh_host_ed25519_key"; - defaultText = literalExpression ''"''${stateDir}/ssh_host_ed25519_key"''; + default = "${cfg.stateDir}/ssh_host_key"; + defaultText = literalExpression ''"''${stateDir}/ssh_host_key"''; description = '' Private ssh host key the knot presents. The knot creates one on first start when the file is absent, @@ -219,8 +205,8 @@ in { secrets = { sealed_key_file = mkOption { type = absPathType; - default = "${cfg.stateDir}/knot.sealed"; - defaultText = literalExpression ''"''${stateDir}/knot.sealed"''; + default = "${cfg.stateDir}/sealed-keys"; + defaultText = literalExpression ''"''${stateDir}/sealed-keys"''; description = '' Sealed store for the knot signing key. The knot creates one on first start when the file is absent, @@ -397,9 +383,7 @@ in { lib.optional (tls.acme_enabled && listenPort != 443) "services.tangled.knot-rs validates over TLS-ALPN-01, which a certificate authority reaches on TCP 443, and settings.server.listen_addr uses port ${toString listenPort}. Map 443 to that port."; - environment.systemPackages = - [cfg.package] - ++ lib.optional cfg.installMigrateTool cfg.migratePackage; + environment.systemPackages = [cfg.package]; environment.etc."knot/config.toml".source = configFile; diff --git a/systemd/knot.service b/systemd/knot.service new file mode 100644 index 000000000..c7bd3fc78 --- /dev/null +++ b/systemd/knot.service @@ -0,0 +1,52 @@ +[Unit] +Description=tangled knot 2 server +After=network-online.target +Wants=network-online.target +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +ExecStart=/usr/local/bin/knot-server /var/lib/knot/config.toml +User=git +Group=git +StateDirectory=knot +WorkingDirectory=/var/lib/knot +Environment=HOME=/var/lib/knot +EnvironmentFile=/etc/knot/knot.env +UMask=0077 +Restart=on-failure +RestartSec=5 +TimeoutStopSec=120 +LimitNOFILE=65536 +StandardOutput=journal +StandardError=journal + +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=true +ProtectProc=invisible +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/knot +PrivateTmp=true +PrivateDevices=true +ProtectHostname=true +ProtectClock=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_NETLINK AF_UNIX +RestrictNamespaces=true +LockPersonality=true +MemoryDenyWriteExecute=true +RestrictRealtime=true +RestrictSUIDSGID=true +RemoveIPC=true +PrivateMounts=true +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources +SystemCallArchitectures=native + +[Install] +WantedBy=multi-user.target -- 2.51.2