import { type ChildProcess, spawn } from 'node:child_process' import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync, } from 'node:fs' import { type HarnessConfig, readProcCmdline, readRunningPid, syncServerCmdlineFile, syncServerDataDir, syncServerLogFile, syncServerPidFile, syncServerStorageIdFile, } from './env.ts' const STARTUP_TIMEOUT_MS = Number(process.env.SYNC_SERVER_STARTUP_TIMEOUT_MS ?? 10_000) /** * Outcome of `startSyncServer`. Surfaces both "already running" and "just * started" so callers can act idempotently. */ export type StartResult = | { state: 'already-running'; pid: number; storageId: string } | { state: 'started'; pid: number; storageId: string } /** * Start the sync server. * * Idempotent: if a server is already running (PID file present and the * live process matches its recorded cmdline), returns `already-running` * without spawning. * * Detached + piped stdio: the spawned process lives past this script's * exit, but we keep its stdio piped through us long enough to scrape the * "Storage ID: " line. Once captured, we persist the PID, cmdline * fingerprint, and storage ID to disk, then `unref()` the child. * * Bugs we go out of our way to NOT have: * - PID file written *before* the await on storage-id, so a Ctrl+C during * startup leaves a recoverable trail (next `up` can find and clean it). * - cmdline fingerprint captured so PID recycling can't fool us. * - On startup-timeout we SIGTERM the spawned child AND clean up our state * files so the next `up` starts from a known-empty state. * - Logs rotated, not truncated, so a failed-then-restarted session can * still read the previous failure log. */ export async function startSyncServer(config: HarnessConfig): Promise { mkdirSync(config.stateRoot, { recursive: true }) mkdirSync(syncServerDataDir(config), { recursive: true }) const existing = readRunningPid(config) if (existing !== null) { const storageId = readFileSync(syncServerStorageIdFile(config), 'utf8').trim() return { state: 'already-running', pid: existing, storageId } } // Stale state from a previous crashed run, if any. cleanupStateFiles(config) if (!existsSync(config.syncServerBin)) { throw new Error( `Sync server binary not found${ config.syncServerBin ? ` at ${config.syncServerBin}` : ' on $PATH' }.\n` + `Enter the Nix dev shell (direnv allow / nix develop), which provides ` + `automerge-repo-sync-server, or set SYNC_SERVER_BIN=.`, ) } rotateLog(config) const logPath = syncServerLogFile(config) const child = spawn(config.syncServerBin, [], { env: { ...process.env, PORT: String(config.syncServerPort), DATA_DIR: syncServerDataDir(config), }, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }) // Persist PID IMMEDIATELY so a Ctrl+C while waiting for storage-id // doesn't leak the child (next `up` finds and cleans it up). writeFileSync(syncServerPidFile(config), String(child.pid)) let storageId: string | null = null const onChunk = (chunk: Buffer) => { appendFileSync(logPath, chunk) if (storageId !== null) return const match = String(chunk).match(/Storage ID:\s*([0-9a-f-]+)/i) if (match) storageId = match[1] } child.stdout.on('data', onChunk) child.stderr.on('data', onChunk) try { const captured = await waitForStorageId(child, logPath, () => storageId) // Record the cmdline fingerprint AFTER startup completes. The // sync-server binary is a bash wrapper that `exec`s into node, so // the cmdline at spawn time is bash but the live cmdline (post-exec) // is node. Capturing here ensures we record the post-exec cmdline, // which is what subsequent `readRunningPid` calls will see. // // A null result here is a hard failure: without the fingerprint, the // PID-recycling guard in readRunningPid silently degrades to "trust // the PID." Better to fail loudly now (cleanup-on-error below) than // to leave the harness running with a defeated safety check. if (child.pid === undefined) { throw new Error('Sync server spawn did not yield a PID') } const cmdline = readProcCmdline(child.pid) if (cmdline === null) { throw new Error( `Could not read /proc/${child.pid}/cmdline after startup. ` + `The PID-recycling guard requires this; running without it would ` + `silently risk killing the wrong process on stop. ` + `If /proc is unavailable (macOS, restricted container), this ` + `harness is unsupported on your platform.`, ) } writeFileSync(syncServerCmdlineFile(config), cmdline) writeFileSync(syncServerStorageIdFile(config), captured) // Stop listening AND destroy the streams so the event loop can exit. // `unref()` alone isn't sufficient when there are open pipes from // the parent to the child. child.stdout.removeAllListeners('data') child.stderr.removeAllListeners('data') child.stdout.destroy() child.stderr.destroy() child.unref() return { state: 'started', pid: child.pid!, storageId: captured } } catch (err) { // Cleanup: kill the spawned child, remove our state files. try { child.kill('SIGTERM') } catch { /* already dead */ } cleanupStateFiles(config) throw err } } /** * Block until the server prints its storage ID, or fail within the * configured timeout. Logs are flushed to disk by `onChunk` so the timeout * branch can include log tail in its error. */ function waitForStorageId( child: ChildProcess, logPath: string, getCaptured: () => string | null, ): Promise { return new Promise((resolve, reject) => { const start = Date.now() const timer = setInterval(() => { const captured = getCaptured() if (captured !== null) { clearInterval(timer) resolve(captured) return } if (Date.now() - start > STARTUP_TIMEOUT_MS) { clearInterval(timer) reject(new Error( `Sync server did not print a Storage ID within ${STARTUP_TIMEOUT_MS}ms.\n` + `Override with SYNC_SERVER_STARTUP_TIMEOUT_MS=.\n` + `Last log output:\n${readLogTail(logPath)}`, )) } }, 100) child.once('exit', (code, signal) => { if (getCaptured() === null) { clearInterval(timer) reject(new Error( `Sync server exited before printing Storage ID ` + `(code=${code}, signal=${signal}).\n` + `Last log output:\n${readLogTail(logPath)}`, )) } }) }) } function readLogTail(path: string): string { if (!existsSync(path)) return '(no log)' return readFileSync(path, 'utf8').slice(-2000) } /** * Rotate the previous log file to `server.log.prev` instead of truncating. * Preserves diagnostic info from the most recent crash across restarts. */ function rotateLog(config: HarnessConfig): void { const current = syncServerLogFile(config) if (existsSync(current)) { renameSync(current, `${current}.prev`) } writeFileSync(current, '') } function cleanupStateFiles(config: HarnessConfig): void { for (const path of [ syncServerPidFile(config), syncServerCmdlineFile(config), syncServerStorageIdFile(config), ]) { if (existsSync(path)) rmSync(path) } } /** * Stop the running server with graceful SIGTERM, escalating to SIGKILL after * a 2s grace period. No-op if no server is running. * * Returns the stopped PID, or null if nothing was running. * * Throws if SIGKILL fails to actually kill the process (e.g. EPERM, or a * truly stuck process). We never silently lie about whether the server * is gone — that would corrupt state on the next `up` cycle (stale data * dir picked up by an unkillable second server fighting for port 3030). */ export async function stopSyncServer(config: HarnessConfig): Promise { const pid = readRunningPid(config) if (pid === null) { cleanupStateFiles(config) return null } try { process.kill(pid, 'SIGTERM') } catch { // Process died between readRunningPid and now. Race, but safe. cleanupStateFiles(config) return pid } // Wait up to 2s for graceful shutdown. if (await waitForDeath(pid, 2000)) { cleanupStateFiles(config) return pid } // Still alive — force-kill. try { process.kill(pid, 'SIGKILL') } catch (err) { // EPERM, ESRCH, etc. Don't clean up state; the user needs to see // that something is wrong. throw new Error( `Failed to SIGKILL sync server pid ${pid}: ${(err as Error).message}. ` + `State files left in place; investigate manually.`, ) } // Verify it actually died after SIGKILL — uninterruptible-sleep processes // (D state in Linux) can survive SIGKILL until they wake. if (!(await waitForDeath(pid, 1000))) { throw new Error( `Sync server pid ${pid} survived SIGKILL. ` + `It may be in uninterruptible sleep. State files left in place; ` + `investigate manually (try: 'ps -o pid,stat,cmd ${pid}').`, ) } cleanupStateFiles(config) return pid } /** * Poll `process.kill(pid, 0)` until the process is gone or `timeoutMs` * elapses. Returns true if the process is gone, false on timeout. */ async function waitForDeath(pid: number, timeoutMs: number): Promise { const start = Date.now() while (Date.now() - start < timeoutMs) { try { process.kill(pid, 0) } catch { return true } await sleep(100) } return false } // --- Status reporting ----------------------------------------------------- export interface ServerStatus { running: boolean pid: number | null url: string storageId: string | null uptime: string | null dataDir: string logFile: string } export function getServerStatus(config: HarnessConfig): ServerStatus { const pid = readRunningPid(config) if (pid === null) { return { running: false, pid: null, url: config.syncServerUrl, storageId: null, uptime: null, dataDir: syncServerDataDir(config), logFile: syncServerLogFile(config), } } const storageIdPath = syncServerStorageIdFile(config) const storageId = existsSync(storageIdPath) ? readFileSync(storageIdPath, 'utf8').trim() : null let uptime: string | null = null try { // /proc//cmdline's mtime is roughly when the process started on Linux. const procStart = statSync(`/proc/${pid}/cmdline`).mtime const secs = Math.floor((Date.now() - procStart.getTime()) / 1000) uptime = secs < 60 ? `${secs}s` : `${Math.floor(secs / 60)}m${secs % 60}s` } catch { /* /proc not available */ } return { running: true, pid, url: config.syncServerUrl, storageId, uptime, dataDir: syncServerDataDir(config), logFile: syncServerLogFile(config), } } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) }