Something went wrong. Try again.
Routing embedding stack for Qwen3-Embedding-0.6B on Apple Silicon: orchestrator + MLX + llama.cpp backends
Something went wrong. Try again.
1.7 kB · 46 lines
Swift
at main
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647import Foundationimport Hummingbirdimport Hub
@mainstruct EmbedServerMain { static func main() async throws { let env = ProcessInfo.processInfo.environment
let port = Int(env["EMBED_PORT"] ?? "8080") ?? 8080 let maxBatch = Int(env["EMBED_MAX_BATCH"] ?? "64") ?? 64 let maxLen = Int(env["EMBED_MAX_LEN"] ?? "512") ?? 512
// Preload list: comma-separated sizes e.g. "0.6B,4B". Default is just 0.6B. let preloadRaw = env["EMBED_PRELOAD"] ?? "0.6B" let preload: [ModelSize] = preloadRaw .split(separator: ",") .compactMap { ModelSize(rawValue: $0.trimmingCharacters(in: .whitespaces)) }
fputs( "[embed] starting on port \(port) maxBatch=\(maxBatch) maxLen=\(maxLen) preload=\(preload.map(\.rawValue).joined(separator: ","))\n", stderr)
let registry = ModelRegistry(hub: HubApi()) let service = EmbedService(registry: registry, maxLen: maxLen) let health = HealthState()
// Kick off preload + warmup in the background. /healthz flips to ok when done. Task.detached { do { try await registry.preload(preload, maxBatch: maxBatch, maxLen: maxLen) await health.markReady() fputs("[embed] ready\n", stderr) } catch { fputs("[embed] preload failed: \(error)\n", stderr) } }
let router = buildRouter(service: service, health: health) let app = Application( router: router, configuration: .init(address: .hostname("0.0.0.0", port: port)) ) try await app.runService() }}