//! Embeddings for gloss canonicalization. A fixed relation ontology would be //! our quilt imposed at the substrate (docs/04), so canonical relations are //! discovered by clustering gloss-text embeddings. The embedder is pluggable: //! an OpenAI-compatible HTTP endpoint when configured, else a deterministic //! character-trigram feature-hashing embedder — crude, but stable, offline, //! and sufficient for incremental clustering to function. use serde::Deserialize; pub const DIM: usize = 256; pub enum Embedder { Http { url: String, api_key: Option, model: String, http: reqwest::Client, }, Hashing, } impl Embedder { pub fn from_config(config: &crate::config::Config) -> Self { match &config.embeddings_url { Some(url) => Embedder::Http { url: url.clone(), api_key: config.embeddings_api_key.clone(), model: config.embeddings_model.clone(), http: reqwest::Client::new(), }, None => Embedder::Hashing, } } #[allow(dead_code)] pub fn id(&self) -> String { match self { Embedder::Http { model, .. } => format!("http:{model}"), Embedder::Hashing => "hash-trigram-v1".into(), } } pub async fn embed(&self, text: &str) -> anyhow::Result> { match self { Embedder::Http { url, api_key, model, http } => { #[derive(Deserialize)] struct Resp { data: Vec, } #[derive(Deserialize)] struct Item { embedding: Vec, } let mut req = http .post(format!("{}/embeddings", url.trim_end_matches('/'))) .json(&serde_json::json!({"model": model, "input": text})); if let Some(key) = api_key { req = req.bearer_auth(key); } let resp: Resp = req.send().await?.error_for_status()?.json().await?; let mut v = resp .data .into_iter() .next() .ok_or_else(|| anyhow::anyhow!("empty embedding response"))? .embedding; normalize(&mut v); Ok(v) } Embedder::Hashing => Ok(hash_embed(text)), } } } /// Character-trigram feature hashing into DIM buckets with a sign hash, /// L2-normalized. Deterministic across runs and platforms. pub fn hash_embed(text: &str) -> Vec { let normalized: String = text .to_lowercase() .chars() .map(|c| if c.is_alphanumeric() || c == ' ' { c } else { ' ' }) .collect(); let padded = format!(" {normalized} "); let chars: Vec = padded.chars().collect(); let mut v = vec![0f32; DIM]; for w in chars.windows(3) { let mut h: u64 = 1469598103934665603; // FNV-1a for c in w { h ^= *c as u64; h = h.wrapping_mul(1099511628211); } let bucket = (h % DIM as u64) as usize; let sign = if (h >> 32) & 1 == 1 { 1.0 } else { -1.0 }; v[bucket] += sign; } normalize(&mut v); v } pub fn normalize(v: &mut [f32]) { let norm: f32 = v.iter().map(|x| x * x).sum::().sqrt(); if norm > 0.0 { for x in v.iter_mut() { *x /= norm; } } } pub fn cosine(a: &[f32], b: &[f32]) -> f32 { let n = a.len().min(b.len()); a[..n].iter().zip(&b[..n]).map(|(x, y)| x * y).sum() } #[cfg(test)] mod tests { use super::*; #[test] fn hash_embed_is_deterministic_and_normalized() { let a = hash_embed("is institutionalized by"); let b = hash_embed("is institutionalized by"); assert_eq!(a, b); let norm: f32 = a.iter().map(|x| x * x).sum::().sqrt(); assert!((norm - 1.0).abs() < 1e-5); } #[test] fn similar_texts_are_closer_than_dissimilar() { let a = hash_embed("is threatened by the state"); let b = hash_embed("is threatened by government"); let c = hash_embed("tastes like breakfast cereal"); assert!(cosine(&a, &b) > cosine(&a, &c)); } #[test] fn blob_roundtrip() { let v = hash_embed("freedom"); assert_eq!(from_blob(&to_blob(&v)), v); } } pub fn to_blob(v: &[f32]) -> Vec { v.iter().flat_map(|x| x.to_le_bytes()).collect() } pub fn from_blob(blob: &[u8]) -> Vec { blob.chunks_exact(4) .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) .collect() }