#!/usr/bin/env bash # dev/run-dev.sh - a three-node local Halcyon raft cluster, no Docker # required (dev/PLAN.md P8-T1). # # The raft and admin listener ports are fixed (internal/cluster.RaftPort # / AdminPort, decision D3) and always bind 0.0.0.0, so three real # "halcyon serve" processes cannot share one network namespace - the # second and third would fail with "address already in use" no matter # what --advertise-addr they are given. This script gives each node its # own network namespace (unprivileged, via "unshare --user --net"; no # root and no Docker needed) and joins the three with veth pairs onto a # small bridge, so each node's fixed ports are genuinely isolated the # same way three separate hosts - or three containers - would be. # # Usage: dev/run-dev.sh [routes-file] # routes-file defaults to dev/routes-dev.yaml. # # Once up, node 1's data plane answers Host: demo.local on port 18080 # and its admin listener answers /healthz on 6072 (likewise node 2 and # 3), all proxied through the same veth/bridge topology this script # builds - see its own "proxy check" step for a live example. Each # node's namespace exists only as long as this script's process tree # does, so testing from another shell means nsenter-ing a namespace # that only exists here; there is no persistent /run/netns entry to # reach it from outside. Ctrl-C tears the whole cluster down. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" ROUTES_FILE="${1:-$SCRIPT_DIR/routes-dev.yaml}" BRIDGE_ADDR="10.99.0.1" NODE_ADDRS=(10.99.0.11 10.99.0.12 10.99.0.13) DATA_PORT=18080 BACKEND_PORT=8090 # Re-exec inside a fresh unprivileged user+network namespace so this # script owns a private network stack to build the bridge and node # namespaces in, without touching the operator's real network. if [[ "${HALCYON_DEV_NETNS:-0}" != "1" ]]; then for cmd in unshare nsenter ip curl python3; do if ! command -v "$cmd" >/dev/null 2>&1; then echo "dev/run-dev.sh: missing required command: $cmd (needs iproute2 and util-linux)" >&2 exit 1 fi done export HALCYON_DEV_NETNS=1 exec unshare --user --net --map-root-user -- "$0" "$ROUTES_FILE" fi if [[ ! -f "$ROUTES_FILE" ]]; then echo "dev/run-dev.sh: routes file not found: $ROUTES_FILE" >&2 exit 1 fi WORK_DIR="$(mktemp -d)" BIN="$WORK_DIR/halcyon" declare -a NODE_NS_PIDS=() declare -a NODE_LOG_PIDS=() BACKEND_PID="" cleanup() { trap - EXIT INT TERM echo "dev/run-dev.sh: shutting down" >&2 for pid in "${NODE_LOG_PIDS[@]:-}"; do [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true done for pid in "${NODE_NS_PIDS[@]:-}"; do [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true done if [[ -n "$BACKEND_PID" ]]; then kill "$BACKEND_PID" 2>/dev/null || true fi wait 2>/dev/null || true rm -rf "$WORK_DIR" } trap cleanup EXIT INT TERM echo "dev/run-dev.sh: building halcyon" >&2 GO="${GO:-go}" export GOCACHE="${GOCACHE:-$WORK_DIR/gocache}" export GOPATH="${GOPATH:-$WORK_DIR/gopath}" ( cd "$REPO_ROOT" && "$GO" build -o "$BIN" ./cmd/halcyon ) echo "dev/run-dev.sh: setting up bridge $BRIDGE_ADDR/24" >&2 ip link set lo up ip link add br-halcyon type bridge ip addr add "$BRIDGE_ADDR/24" dev br-halcyon ip link set br-halcyon up # demoBackend answers every request on $BACKEND_PORT so the routes # file (dev/routes-dev.yaml) has something real to proxy to. python3 -c " import http.server, sys class H(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header('Content-Type', 'text/plain') self.end_headers() self.wfile.write(b'hello from the halcyon dev backend\n') def log_message(self, *a): pass http.server.HTTPServer(('$BRIDGE_ADDR', $BACKEND_PORT), H).serve_forever() " & BACKEND_PID=$! RAFT_PEERS="${NODE_ADDRS[0]}|${NODE_ADDRS[1]}|${NODE_ADDRS[2]}" start_node() { local index="$1" addr="$2" local data_dir="$WORK_DIR/node$index-raft" local log_file="$WORK_DIR/node$index.log" mkdir -p "$data_dir" # Each node gets its own net namespace, held open by a background # "sleep infinity" whose PID nsenter targets - the namespace lives # exactly as long as that process does. unshare --net sleep infinity & local ns_pid=$! NODE_NS_PIDS+=("$ns_pid") sleep 0.2 nsenter --net="/proc/$ns_pid/ns/net" ip link set lo up ip link add "veth-n$index" type veth peer name "veth-p$index" ip link set "veth-p$index" netns "$ns_pid" ip link set "veth-n$index" master br-halcyon ip link set "veth-n$index" up nsenter --net="/proc/$ns_pid/ns/net" ip addr add "$addr/24" dev "veth-p$index" nsenter --net="/proc/$ns_pid/ns/net" ip link set "veth-p$index" up echo "dev/run-dev.sh: starting node $index at $addr (log: $log_file)" >&2 nsenter --net="/proc/$ns_pid/ns/net" -- "$BIN" serve \ --dev \ --entrypoint "web=$DATA_PORT" \ --advertise-addr "$addr" \ --raft-peer "$RAFT_PEERS" \ --raft-data-dir "$data_dir" \ --routes-file "$ROUTES_FILE" \ --log-level info \ >"$log_file" 2>&1 & NODE_LOG_PIDS+=("$!") } for i in 1 2 3; do start_node "$i" "${NODE_ADDRS[$((i - 1))]}" done echo "dev/run-dev.sh: waiting for nodes to become healthy" >&2 for i in 1 2 3; do ns_pid="${NODE_NS_PIDS[$((i - 1))]}" ok=0 for _ in $(seq 1 30); do if nsenter --net="/proc/$ns_pid/ns/net" -- curl -fsS -o /dev/null "http://127.0.0.1:6072/healthz" 2>/dev/null; then ok=1 break fi sleep 1 done if [[ "$ok" -eq 1 ]]; then echo "dev/run-dev.sh: node $i healthy" >&2 else echo "dev/run-dev.sh: node $i did NOT become healthy - see ${WORK_DIR}/node$i.log" >&2 fi done echo "dev/run-dev.sh: proxy check through node 1" >&2 proxy_ok=0 for _ in $(seq 1 20); do if nsenter --net="/proc/${NODE_NS_PIDS[0]}/ns/net" -- curl -fsS -H "Host: demo.local" "http://127.0.0.1:$DATA_PORT/" 2>/dev/null; then proxy_ok=1 break fi sleep 1 done if [[ "$proxy_ok" -ne 1 ]]; then echo "dev/run-dev.sh: proxy check failed - see ${WORK_DIR}/node1.log" >&2 fi echo "dev/run-dev.sh: cluster is up. Node logs are in $WORK_DIR/nodeN.log. Press Ctrl-C to stop." >&2 wait