Something went wrong. Try again.
A traefik replacement that is less braindead
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182#!/usr/bin/env bash# dev/run-dev.sh - a three-node local Halcyon raft cluster, no Docker# required (dev/PLAN.md P8-T1).## The raft and admin listener ports are fixed (internal/cluster.RaftPort# / AdminPort, decision D3) and always bind 0.0.0.0, so three real# "halcyon serve" processes cannot share one network namespace - the# second and third would fail with "address already in use" no matter# what --advertise-addr they are given. This script gives each node its# own network namespace (unprivileged, via "unshare --user --net"; no# root and no Docker needed) and joins the three with veth pairs onto a# small bridge, so each node's fixed ports are genuinely isolated the# same way three separate hosts - or three containers - would be.## Usage: dev/run-dev.sh [routes-file]# routes-file defaults to dev/routes-dev.yaml.## Once up, node 1's data plane answers Host: demo.local on port 18080# and its admin listener answers /healthz on 6072 (likewise node 2 and# 3), all proxied through the same veth/bridge topology this script# builds - see its own "proxy check" step for a live example. Each# node's namespace exists only as long as this script's process tree# does, so testing from another shell means nsenter-ing a namespace# that only exists here; there is no persistent /run/netns entry to# reach it from outside. Ctrl-C tears the whole cluster down.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"ROUTES_FILE="${1:-$SCRIPT_DIR/routes-dev.yaml}"
BRIDGE_ADDR="10.99.0.1"NODE_ADDRS=(10.99.0.11 10.99.0.12 10.99.0.13)DATA_PORT=18080BACKEND_PORT=8090
# Re-exec inside a fresh unprivileged user+network namespace so this# script owns a private network stack to build the bridge and node# namespaces in, without touching the operator's real network.if [[ "${HALCYON_DEV_NETNS:-0}" != "1" ]]; then for cmd in unshare nsenter ip curl python3; do if ! command -v "$cmd" >/dev/null 2>&1; then echo "dev/run-dev.sh: missing required command: $cmd (needs iproute2 and util-linux)" >&2 exit 1 fi done export HALCYON_DEV_NETNS=1 exec unshare --user --net --map-root-user -- "$0" "$ROUTES_FILE"fi
if [[ ! -f "$ROUTES_FILE" ]]; then echo "dev/run-dev.sh: routes file not found: $ROUTES_FILE" >&2 exit 1fi
WORK_DIR="$(mktemp -d)"BIN="$WORK_DIR/halcyon"declare -a NODE_NS_PIDS=()declare -a NODE_LOG_PIDS=()BACKEND_PID=""
cleanup() { trap - EXIT INT TERM echo "dev/run-dev.sh: shutting down" >&2 for pid in "${NODE_LOG_PIDS[@]:-}"; do [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true done for pid in "${NODE_NS_PIDS[@]:-}"; do [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true done if [[ -n "$BACKEND_PID" ]]; then kill "$BACKEND_PID" 2>/dev/null || true fi wait 2>/dev/null || true rm -rf "$WORK_DIR"}trap cleanup EXIT INT TERM
echo "dev/run-dev.sh: building halcyon" >&2GO="${GO:-go}"export GOCACHE="${GOCACHE:-$WORK_DIR/gocache}"export GOPATH="${GOPATH:-$WORK_DIR/gopath}"( cd "$REPO_ROOT" && "$GO" build -o "$BIN" ./cmd/halcyon )
echo "dev/run-dev.sh: setting up bridge $BRIDGE_ADDR/24" >&2ip link set lo upip link add br-halcyon type bridgeip addr add "$BRIDGE_ADDR/24" dev br-halcyonip link set br-halcyon up
# demoBackend answers every request on $BACKEND_PORT so the routes# file (dev/routes-dev.yaml) has something real to proxy to.python3 -c "import http.server, sysclass H(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header('Content-Type', 'text/plain') self.end_headers() self.wfile.write(b'hello from the halcyon dev backend\n') def log_message(self, *a): passhttp.server.HTTPServer(('$BRIDGE_ADDR', $BACKEND_PORT), H).serve_forever()" &BACKEND_PID=$!
RAFT_PEERS="${NODE_ADDRS[0]}|${NODE_ADDRS[1]}|${NODE_ADDRS[2]}"
start_node() { local index="$1" addr="$2" local data_dir="$WORK_DIR/node$index-raft" local log_file="$WORK_DIR/node$index.log" mkdir -p "$data_dir"
# Each node gets its own net namespace, held open by a background # "sleep infinity" whose PID nsenter targets - the namespace lives # exactly as long as that process does. unshare --net sleep infinity & local ns_pid=$! NODE_NS_PIDS+=("$ns_pid") sleep 0.2
nsenter --net="/proc/$ns_pid/ns/net" ip link set lo up ip link add "veth-n$index" type veth peer name "veth-p$index" ip link set "veth-p$index" netns "$ns_pid" ip link set "veth-n$index" master br-halcyon ip link set "veth-n$index" up nsenter --net="/proc/$ns_pid/ns/net" ip addr add "$addr/24" dev "veth-p$index" nsenter --net="/proc/$ns_pid/ns/net" ip link set "veth-p$index" up
echo "dev/run-dev.sh: starting node $index at $addr (log: $log_file)" >&2 nsenter --net="/proc/$ns_pid/ns/net" -- "$BIN" serve \ --dev \ --entrypoint "web=$DATA_PORT" \ --advertise-addr "$addr" \ --raft-peer "$RAFT_PEERS" \ --raft-data-dir "$data_dir" \ --routes-file "$ROUTES_FILE" \ --log-level info \ >"$log_file" 2>&1 & NODE_LOG_PIDS+=("$!")}
for i in 1 2 3; do start_node "$i" "${NODE_ADDRS[$((i - 1))]}"done
echo "dev/run-dev.sh: waiting for nodes to become healthy" >&2for i in 1 2 3; do ns_pid="${NODE_NS_PIDS[$((i - 1))]}" ok=0 for _ in $(seq 1 30); do if nsenter --net="/proc/$ns_pid/ns/net" -- curl -fsS -o /dev/null "http://127.0.0.1:6072/healthz" 2>/dev/null; then ok=1 break fi sleep 1 done if [[ "$ok" -eq 1 ]]; then echo "dev/run-dev.sh: node $i healthy" >&2 else echo "dev/run-dev.sh: node $i did NOT become healthy - see ${WORK_DIR}/node$i.log" >&2 fidone
echo "dev/run-dev.sh: proxy check through node 1" >&2proxy_ok=0for _ in $(seq 1 20); do if nsenter --net="/proc/${NODE_NS_PIDS[0]}/ns/net" -- curl -fsS -H "Host: demo.local" "http://127.0.0.1:$DATA_PORT/" 2>/dev/null; then proxy_ok=1 break fi sleep 1doneif [[ "$proxy_ok" -ne 1 ]]; then echo "dev/run-dev.sh: proxy check failed - see ${WORK_DIR}/node1.log" >&2fi
echo "dev/run-dev.sh: cluster is up. Node logs are in $WORK_DIR/nodeN.log. Press Ctrl-C to stop." >&2wait