Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
4.7 kB · 150 lines
Rust
at dev
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151//! Embeddings for gloss canonicalization. A fixed relation ontology would be//! our quilt imposed at the substrate (docs/04), so canonical relations are//! discovered by clustering gloss-text embeddings. The embedder is pluggable://! an OpenAI-compatible HTTP endpoint when configured, else a deterministic//! character-trigram feature-hashing embedder — crude, but stable, offline,//! and sufficient for incremental clustering to function.
use serde::Deserialize;
pub const DIM: usize = 256;
pub enum Embedder { Http { url: String, api_key: Option<String>, model: String, http: reqwest::Client, }, Hashing,}
impl Embedder { pub fn from_config(config: &crate::config::Config) -> Self { match &config.embeddings_url { Some(url) => Embedder::Http { url: url.clone(), api_key: config.embeddings_api_key.clone(), model: config.embeddings_model.clone(), http: reqwest::Client::new(), }, None => Embedder::Hashing, } }
#[allow(dead_code)] pub fn id(&self) -> String { match self { Embedder::Http { model, .. } => format!("http:{model}"), Embedder::Hashing => "hash-trigram-v1".into(), } }
pub async fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>> { match self { Embedder::Http { url, api_key, model, http } => { #[derive(Deserialize)] struct Resp { data: Vec<Item>, } #[derive(Deserialize)] struct Item { embedding: Vec<f32>, } let mut req = http .post(format!("{}/embeddings", url.trim_end_matches('/'))) .json(&serde_json::json!({"model": model, "input": text})); if let Some(key) = api_key { req = req.bearer_auth(key); } let resp: Resp = req.send().await?.error_for_status()?.json().await?; let mut v = resp .data .into_iter() .next() .ok_or_else(|| anyhow::anyhow!("empty embedding response"))? .embedding; normalize(&mut v); Ok(v) } Embedder::Hashing => Ok(hash_embed(text)), } }}
/// Character-trigram feature hashing into DIM buckets with a sign hash,/// L2-normalized. Deterministic across runs and platforms.pub fn hash_embed(text: &str) -> Vec<f32> { let normalized: String = text .to_lowercase() .chars() .map(|c| if c.is_alphanumeric() || c == ' ' { c } else { ' ' }) .collect(); let padded = format!(" {normalized} "); let chars: Vec<char> = padded.chars().collect(); let mut v = vec![0f32; DIM]; for w in chars.windows(3) { let mut h: u64 = 1469598103934665603; // FNV-1a for c in w { h ^= *c as u64; h = h.wrapping_mul(1099511628211); } let bucket = (h % DIM as u64) as usize; let sign = if (h >> 32) & 1 == 1 { 1.0 } else { -1.0 }; v[bucket] += sign; } normalize(&mut v); v}
pub fn normalize(v: &mut [f32]) { let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt(); if norm > 0.0 { for x in v.iter_mut() { *x /= norm; } }}
pub fn cosine(a: &[f32], b: &[f32]) -> f32 { let n = a.len().min(b.len()); a[..n].iter().zip(&b[..n]).map(|(x, y)| x * y).sum()}
#[cfg(test)]mod tests { use super::*;
#[test] fn hash_embed_is_deterministic_and_normalized() { let a = hash_embed("is institutionalized by"); let b = hash_embed("is institutionalized by"); assert_eq!(a, b); let norm: f32 = a.iter().map(|x| x * x).sum::<f32>().sqrt(); assert!((norm - 1.0).abs() < 1e-5); }
#[test] fn similar_texts_are_closer_than_dissimilar() { let a = hash_embed("is threatened by the state"); let b = hash_embed("is threatened by government"); let c = hash_embed("tastes like breakfast cereal"); assert!(cosine(&a, &b) > cosine(&a, &c)); }
#[test] fn blob_roundtrip() { let v = hash_embed("freedom"); assert_eq!(from_blob(&to_blob(&v)), v); }}
pub fn to_blob(v: &[f32]) -> Vec<u8> { v.iter().flat_map(|x| x.to_le_bytes()).collect()}
pub fn from_blob(blob: &[u8]) -> Vec<f32> { blob.chunks_exact(4) .map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]])) .collect()}