diff --git a/milky-way/environments/stage00/orion-system/main.jsonnet b/milky-way/environments/stage00/orion-system/main.jsonnet index e1dfa9e..1f36cd9 100644 --- a/milky-way/environments/stage00/orion-system/main.jsonnet +++ b/milky-way/environments/stage00/orion-system/main.jsonnet @@ -13,6 +13,7 @@ local testTailscaleIngress = import 'milky-way/lib/test-tailscale-operator-ingre local testTailscaleL3 = import 'milky-way/lib/test-tailscale-operator-network-L3.libsonnet'; local openclaw = import 'milky-way/lib/openclaw.libsonnet'; local qbittorrent = import 'milky-way/lib/qbittorrent.libsonnet'; +local sftp = import 'milky-way/lib/sftp.libsonnet'; local gluetunLeakTest = import 'milky-way/lib/gluetun-leak-test.libsonnet'; local testExampleWhaleImageDigest = import 'milky-way/lib/test-example-whale-image-digest.libsonnet'; local secrets = import 'milky-way/secrets/k8s-secret-values.jsonnet'; @@ -129,6 +130,21 @@ local secrets = import 'milky-way/secrets/k8s-secret-values.jsonnet'; mediaClaimName = this.mdataPvc.metadata.name, ), + // Public-key-only SFTP front door onto the shared mdata volume (read-write), reached over the + // tailnet (mdata-sftp.tail4c9a.ts.net:22) and over the LAN via methanol's mDNS alias + // (mdata-methanol.local:30022 -- alias + firewall port live in venus methanol.nix). Authorized + // identities: sodium's key and the 1Password key (public keys, mirrored from methanol.nix). + mdataSftp: sftp.new( + claimName = this.mdataPvc.metadata.name, + name = "mdata-sftp", + sftpUser = "mdata", + nodePort = 30022, + authorizedKeys = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDLrT2/gQXhOz4E4xSphB8EXouild5qNOnZ6ZVXuTnf167z8xxSB10mxNey2gKDaIVig6I/tRFeYy6/N/QutbBlKI/+GNPjGCcVJI0hf7fTZGL4caTW8ggcXRz4LAsFp3JBf6Li0FVrGz5ojD0Etbl54BDn033q/tlVRhme5bXJ6s73yRg04kqdQsWVBRJwyzbUUmCQPrZd9i5Nh4QFVuhZljEyUWIStajE+c9v8OOiY1svv+XjKBjyWphP16HqgzvnEDf5+MQ5AUxE05IvJx43UY43CKTe3evzt4F/IqSdYwYGIQ55DaseRmf5zmHLU8MTTkksmOPQEzJL0nBzAmxyGV3PsMYPoIN+1/gJmxCO6ZaaCxYr9SFK/yoRW5e0PFX433xPhNsITBq7jUrVg6BQ/lr0ntRfvd7pRhFq8v02R3jWokL/99skxp1kjVF42bXEJXYPpHF3XAUhYscjOwmWj8dJgsIsSIKIjh7gRVYxQGrZQXOcJQjMytFgXy7fWHM= yuto@Yutos-MacBook-Pro.local", // Yuto's Sodium + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPtVvX9uhSWD1DPBIRqgkNzFXqjdqvWB/WtDy4seaiJl", // 1Password "ssh key - main" + ], + ), + // Continuously asserts qbittorrent's egress is the VPN exit (not the home IP) and exercises a real // ipleak.net torrent magnet; crashloops on a detected leak. gluetunLeakTest: gluetunLeakTest.new(), diff --git a/milky-way/lib/images.libsonnet b/milky-way/lib/images.libsonnet index 8164469..0cad583 100644 --- a/milky-way/lib/images.libsonnet +++ b/milky-way/lib/images.libsonnet @@ -47,6 +47,12 @@ local images = { fullyQualifiedRepository: "lscr.io/linuxserver/qbittorrent", defaultDigest: { hash: "sha256:1784d5a65d08d01de308c7d87ff2c1dba328379e180eeca41cc6b96bdf6a0ffc", tagHint: "5.2.1" }, }, + // Minimal OpenSSH SFTP-only server. The :alpine tag is a single-arch (linux/amd64) manifest -- + // matches methanol -- so the digest below is that manifest, not a multi-arch index. + "atmoz-sftp": { + fullyQualifiedRepository: "docker.io/atmoz/sftp", + defaultDigest: { hash: "sha256:a6cb3eb29202ca7f57e73bb7e527286e66e0e822fff65609207c7e0ef2d135a3", tagHint: "alpine" }, + }, // traefik/whoami is shared by the two tailscale-operator smoke tests at the same digest; // one digestFor per consumer, following the per-consumer convention. whoami: { diff --git a/milky-way/lib/sftp.libsonnet b/milky-way/lib/sftp.libsonnet new file mode 100644 index 0000000..d28ffd5 --- /dev/null +++ b/milky-way/lib/sftp.libsonnet @@ -0,0 +1,203 @@ +local utils = import 'milky-way/lib/utils.libsonnet'; +local images = import 'milky-way/lib/images.libsonnet'; + +// SFTP front door onto an arbitrary PVC. Mounts the given claim into a chrooted SFTP user's home and +// exposes it over two L4/TCP paths -- SFTP is SSH, not HTTP, so the cluster's Traefik / +// `ingressClassName: tailscale` (both L7-HTTP) cannot carry it: +// * Tailnet: an L4 Service-expose (`tailscale.com/expose`) -> a tagged tailnet node +// `..ts.net` (same mechanism as mopidy's MPD port). This +// is a plain tagged node, NOT a Tailscale Service/VIP, so it needs none of the L3 +// ProxyGroup autoApprovers/grants dance. +// * LAN: a NodePort on a fixed port, reached via a node's `.local` mDNS alias. NodePort +// (not hostPort) so the port is open on every node and Cilium forwards to wherever +// the pod runs -- multi-node-safe without enabling LoadBalancer IPAM. +// +// Auth is public-key only (atmoz/sftp locks the account for password auth when the password field +// is empty). Authorized keys are PUBLIC, so they're passed in literally rather than via sops. +// +// Two non-obvious storage facts drive the host-key handling below: +// 1. sshd refuses host private keys unless they're owned by root and mode 0600. An NFS-backed PVC +// typically root-squashes (see qbittorrent.libsonnet), so a root-written file there lands owned +// by `nobody` and sshd would reject it. Host keys therefore live on a dedicated iSCSI (block, +// no squash) RWO PVC instead, which forces `strategy: Recreate` (the old pod must release the +// volume). This is independent of whatever PVC is being exposed. +// 2. Without persisted host keys, atmoz regenerates them every boot and clients hit "REMOTE HOST +// IDENTIFICATION HAS CHANGED". An init container seeds them once (only-if-missing); the main +// container subPath-mounts each key file into /etc/ssh so atmoz's `ssh-keygen -A` finds them +// and regenerates nothing. +{ + new( + claimName, // positional, required -> the PVC to expose over SFTP + authorizedKeys, // positional, required -> list of public-key strings + nodePort, // positional, required, must be unique per instance -> sftp -P @.local + name='sftp', + namespace='default', // must match claimName's namespace (PVCs are namespaced) + image=images['atmoz-sftp'].fullyQualifiedImageReferencePinned, + sftpUser='sftp', // login user; set uid/gid to match the PVC's file ownership + sftpUid=1000, + sftpGid=1000, + dataDirName='data', // subdir under the chroot home where the PVC is mounted + dataReadOnly=false, // expose the PVC read-only when true + tailscaleHostname=name, // -> sftp @..ts.net + hostKeysStorageClassName='my-custom-zfs-generic-iscsi', // iSCSI/RWO: real root, no NFS squash + hostKeysStorageSize='1Gi', + ):: { + local this = self, + + // Inside the sshd chroot (ChrootDirectory = the user's home) the user sees this as /. + local dataMountPath = '/home/%s/%s' % [sftpUser, dataDirName], + local keysMountPath = '/home/%s/.ssh/keys' % sftpUser, // atmoz appends every file here to authorized_keys + + // Persist all three host-key types so no client (whatever HostKeyAlgorithms it negotiates) ever + // sees a changed key. One source of truth for the init shell loop and the subPath mounts. + local hostKeyTypes = ['rsa', 'ecdsa', 'ed25519'], + local hostKeyFiles = std.flattenArrays([ + ['ssh_host_%s_key' % t, 'ssh_host_%s_key.pub' % t] + for t in hostKeyTypes + ]), + local hostKeyMounts = [ + { name: 'hostkeys', mountPath: '/etc/ssh/' + f, subPath: f, readOnly: true } + for f in hostKeyFiles + ], + + // Trailing newline per file is REQUIRED: atmoz concatenates every file in .ssh/keys/ into + // authorized_keys with plain `cat`, so without it two keys merge onto one line and only the + // first parses (the second is swallowed into the first line's comment field). + local authorizedKeysData = { + ['authorized-key-%d.pub' % i]: authorizedKeys[i] + '\n' + for i in std.range(0, std.length(authorizedKeys) - 1) + }, + + configMapAuthorizedKeys: { + apiVersion: 'v1', + kind: 'ConfigMap', + metadata: { name: name + '-authorized-keys', namespace: namespace }, + data: authorizedKeysData, + }, + + hostKeysPvc: { + apiVersion: 'v1', + kind: 'PersistentVolumeClaim', + metadata: { name: name + '-hostkeys', namespace: namespace }, + spec: { + accessModes: ['ReadWriteOncePod'], + storageClassName: hostKeysStorageClassName, + resources: { requests: { storage: hostKeysStorageSize } }, + }, + }, + + deployment: { + apiVersion: 'apps/v1', + kind: 'Deployment', + metadata: { name: name, namespace: namespace }, + spec: { + replicas: 1, + strategy: { type: 'Recreate' }, // RWO host-keys PVC: old pod must release before new mounts + selector: { matchLabels: { app: name } }, + template: { + metadata: { + labels: {} + this.deployment.spec.selector.matchLabels, + // atmoz reads authorized_keys once at startup; hash the keys into the template so a key + // change rolls the pod (a ConfigMap edit alone doesn't), mirroring qbittorrent's checksum. + annotations: { 'checksum/authorized-keys': std.md5(std.manifestJsonEx(authorizedKeysData, '')) }, + }, + spec: { + tolerations: [ + { key: 'ephemeral', operator: 'Exists', effect: 'NoSchedule' }, + ], + initContainers: [ + { + // Seed each host key only-if-missing onto the iSCSI volume, root-owned 0600, so sshd + // accepts them and they survive restarts. Reuses the atmoz image (it ships ssh-keygen). + name: 'init-hostkeys', + image: image, + command: ['sh', '-c', (||| + set -eu + for t in %(types)s; do + f="/hostkeys/ssh_host_${t}_key" + [ -f "$f" ] || ssh-keygen -q -t "$t" -f "$f" -N "" + done + |||) % { types: std.join(' ', hostKeyTypes) }], + volumeMounts: [ + { name: 'hostkeys', mountPath: '/hostkeys' }, + ], + resources: { + requests: { memory: '16Mi', cpu: '25m' }, + limits: { memory: '32Mi', cpu: '50m' }, + }, + }, + ], + containers: [ + { + name: name, + image: image, + // atmoz reads user definitions from its args: user:pass[:e][:uid[:gid[:dir...]]]. + // Empty password -> key-only auth. + args: ['%s::%d:%d' % [sftpUser, sftpUid, sftpGid]], + ports: [{ name: 'sftp', containerPort: 22 }], + volumeMounts: [ + { name: 'data', mountPath: dataMountPath } + (if dataReadOnly then { readOnly: true } else {}), + { name: 'authorized-keys', mountPath: keysMountPath, readOnly: true }, + ] + hostKeyMounts, + readinessProbe: { + tcpSocket: { port: 'sftp' }, + initialDelaySeconds: 5, + periodSeconds: 15, + }, + resources: { + requests: { memory: '32Mi', cpu: '25m' }, + limits: { memory: '128Mi', cpu: '200m' }, + }, + }, + ], + volumes: [ + { name: 'data', persistentVolumeClaim: { claimName: claimName } }, + { name: 'hostkeys', persistentVolumeClaim: { claimName: this.hostKeysPvc.metadata.name } }, + { name: 'authorized-keys', configMap: { name: this.configMapAuthorizedKeys.metadata.name } }, + ], + }, + }, + }, + }, + + // Tailnet path: L4 Service-expose -> a tagged tailnet node serving SFTP on port 22. + service: { + apiVersion: 'v1', + kind: 'Service', + metadata: { + name: name, + namespace: namespace, + annotations: { + 'tailscale.com/expose': 'true', + 'tailscale.com/hostname': tailscaleHostname, + }, + }, + spec: { + selector: {} + this.deployment.spec.template.metadata.labels, + ports: [{ + name: 'sftp', + port: 22, + targetPort: utils.assertEqualAndReturn(this.deployment.spec.template.spec.containers[0].ports[0].name, 'sftp'), + }], + type: 'ClusterIP', + }, + }, + + // LAN path: NodePort on every node (Cilium forwards to the pod), reached via a `.local` alias. + nodePortService: { + apiVersion: 'v1', + kind: 'Service', + metadata: { name: name + '-nodeport', namespace: namespace }, + spec: { + selector: {} + this.deployment.spec.template.metadata.labels, + ports: [{ + name: 'sftp', + port: 22, + nodePort: nodePort, + targetPort: utils.assertEqualAndReturn(this.deployment.spec.template.spec.containers[0].ports[0].name, 'sftp'), + }], + type: 'NodePort', + }, + }, + }, +} diff --git a/venus/modules/nixos-darwin/methanol.nix b/venus/modules/nixos-darwin/methanol.nix index 2c2e553..8dc5de4 100644 --- a/venus/modules/nixos-darwin/methanol.nix +++ b/venus/modules/nixos-darwin/methanol.nix @@ -100,7 +100,7 @@ services.avahi-aliases = { enable = true; device = "enp42s0"; - aliases = [ "cwa-methanol.local" ]; + aliases = [ "cwa-methanol.local" "mdata-methanol.local" ]; }; services.tailscale = { @@ -320,6 +320,7 @@ config.services.nfs.server.statdPort config.services.nfs.server.mountdPort config.services.nfs.server.lockdPort + 30022 # mdata-sftp NodePort (SFTP over .local; see milky-way lib/sftp.libsonnet) ]; networking.firewall.allowedUDPPorts = config.networking.firewall.allowedTCPPorts; # Or disable the firewall altogether.