package mill import ( "context" "crypto/ed25519" "crypto/rand" "encoding/pem" "fmt" "net" "os" "path/filepath" "sync" "time" "github.com/gliderlabs/ssh" gossh "golang.org/x/crypto/ssh" "tangled.org/core/spindle/observability" ) const ( jumpIdleTimeout = 5 * time.Minute jumpMaxTimeout = 24 * time.Hour maxJumpConnectionsPerIP = 8 ) type jumpContextKey string const jumpRouteOpened jumpContextKey = "route-opened" func (m *Mill) ServeJump(ctx context.Context, listenAddr, hostKeyPath string, executorPort uint32, maxConnections int) { if listenAddr == "" { return } srv, err := m.newJumpServer(hostKeyPath, executorPort, maxConnections) if err != nil { m.l.Error("setup debug ssh jump server", "err", err) return } go func() { <-ctx.Done() _ = srv.Close() }() m.l.Info("starting debug ssh jump server", "address", listenAddr) srv.Addr = listenAddr if err := srv.ListenAndServe(); err != nil && err != ssh.ErrServerClosed { m.l.Error("debug ssh jump server stopped", "err", err) } } func (m *Mill) newJumpServer(hostKeyPath string, executorPort uint32, maxConnections int) (*ssh.Server, error) { if executorPort == 0 { executorPort = 2223 } if maxConnections <= 0 { return nil, fmt.Errorf("max jump connections must be greater than zero") } if err := ensureJumpHostKey(hostKeyPath); err != nil { return nil, fmt.Errorf("prepare jump host key: %w", err) } limiter := newJumpConnectionLimiter(maxConnections, maxJumpConnectionsPerIP) limiter.metrics = m.metrics m.metrics.RecordJumpLimit(int64(maxConnections)) srv := &ssh.Server{ PublicKeyHandler: func(ctx ssh.Context, _ ssh.PublicKey) bool { allowed := ctx.User() == "debug" if !allowed { m.metrics.RecordJumpRejection("unauthorized_user") } return allowed }, ConnCallback: func(ctx ssh.Context, conn net.Conn) net.Conn { if !limiter.acquire(conn.RemoteAddr()) { _ = conn.Close() return conn } go func() { <-ctx.Done() limiter.release(conn.RemoteAddr()) }() return conn }, LocalPortForwardingCallback: func(ctx ssh.Context, host string, port uint32) bool { if port != executorPort { m.metrics.RecordJumpRejection("invalid_port") return false } if !m.hasLiveExecutor(host) { m.metrics.RecordJumpRejection("executor_unavailable") return false } ctx.Lock() defer ctx.Unlock() if opened, _ := ctx.Value(jumpRouteOpened).(bool); opened { m.metrics.RecordJumpRejection("route_already_open") return false } ctx.SetValue(jumpRouteOpened, true) return true }, ChannelHandlers: map[string]ssh.ChannelHandler{ "direct-tcpip": ssh.DirectTCPIPHandler, }, IdleTimeout: jumpIdleTimeout, MaxTimeout: jumpMaxTimeout, } if err := srv.SetOption(ssh.HostKeyFile(hostKeyPath)); err != nil { return nil, fmt.Errorf("load jump host key: %w", err) } return srv, nil } type jumpConnectionLimiter struct { mu sync.Mutex total int perIP map[string]int maxTotal int maxPerIP int metrics *observability.Metrics } func newJumpConnectionLimiter(maxTotal, maxPerIP int) *jumpConnectionLimiter { return &jumpConnectionLimiter{ perIP: make(map[string]int), maxTotal: maxTotal, maxPerIP: maxPerIP, } } func (l *jumpConnectionLimiter) acquire(addr net.Addr) bool { host := jumpRemoteHost(addr) l.mu.Lock() defer l.mu.Unlock() metrics := l.metrics if l.total >= l.maxTotal { metrics.RecordJumpRejection("max_total_reached") return false } if l.perIP[host] >= l.maxPerIP { metrics.RecordJumpRejection("max_per_ip_reached") return false } l.total++ l.perIP[host]++ metrics.RecordJumpActive(int64(l.total)) return true } func (l *jumpConnectionLimiter) release(addr net.Addr) { host := jumpRemoteHost(addr) l.mu.Lock() defer l.mu.Unlock() l.total-- l.perIP[host]-- if l.perIP[host] == 0 { delete(l.perIP, host) } l.metrics.RecordJumpActive(int64(l.total)) } func jumpRemoteHost(addr net.Addr) string { if addr == nil { return "" } host, _, err := net.SplitHostPort(addr.String()) if err != nil { return addr.String() } return host } func ensureJumpHostKey(path string) error { if _, err := os.Stat(path); err == nil { return nil } else if !os.IsNotExist(err) { return fmt.Errorf("stat host key: %w", err) } _, privateKey, err := ed25519.GenerateKey(rand.Reader) if err != nil { return fmt.Errorf("generate host key: %w", err) } block, err := gossh.MarshalPrivateKey(privateKey, "") if err != nil { return fmt.Errorf("marshal host key: %w", err) } if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return fmt.Errorf("create host key directory: %w", err) } return os.WriteFile(path, pem.EncodeToMemory(block), 0o600) }