Something went wrong. Try again.
Routing embedding stack for Qwen3-Embedding-0.6B on Apple Silicon: orchestrator + MLX + llama.cpp backends
Something went wrong. Try again.
2.0 kB · 73 lines
Rust
at main
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374use serde::{Deserialize, Serialize};use std::fmt;
/// Which Qwen3-Embedding size to use.#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]pub enum ModelSize { #[serde(rename = "0.6B")] S06, #[serde(rename = "2B")] S2, #[serde(rename = "4B")] S4,}
impl ModelSize { /// HuggingFace GGUF repo id. pub fn repo_id(self) -> &'static str { match self { Self::S06 => "Qwen/Qwen3-Embedding-0.6B-GGUF", Self::S2 => "Qwen/Qwen3-Embedding-2B-GGUF", Self::S4 => "Qwen/Qwen3-Embedding-4B-GGUF", } }
/// GGUF filename inside the repo (F16 quant). pub fn gguf_filename(self) -> &'static str { match self { Self::S06 => "Qwen3-Embedding-0.6B-f16.gguf", Self::S2 => "Qwen3-Embedding-2B-f16.gguf", Self::S4 => "Qwen3-Embedding-4B-f16.gguf", } }
/// Human-readable label returned in API responses. pub fn label(self) -> &'static str { match self { Self::S06 => "qwen3-embedding-0.6B", Self::S2 => "qwen3-embedding-2B", Self::S4 => "qwen3-embedding-4B", } }}
impl fmt::Display for ModelSize { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.label()) }}
/// Whether the input is a document (no prefix) or a query (instruction prefix).#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]#[serde(rename_all = "lowercase")]pub enum InputKind { Document, Query,}
/// JSON body for POST /embed.#[derive(Debug, Deserialize)]pub struct EmbedRequest { pub texts: Vec<String>, pub input_type: InputKind, /// Model size to use. Defaults to 0.6B if omitted. pub size: Option<ModelSize>,}
/// JSON response for POST /embed.#[derive(Debug, Serialize)]pub struct EmbedResponse { pub vectors: Vec<Vec<f32>>, pub dim: usize, pub model: String,}