Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
15 kB · 446 lines
Rust
at dev
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447//! The personal term-graph (docs/05): nodes are terms, edges are canonical//! relations instantiated by the user's completions, weights from gloss//! weights, signs from gloss valence. Rebuilt per-user by the medium loop —//! a user contributes dozens of completions, so full rebuild is cheap.
use std::collections::{BTreeMap, BTreeSet, HashMap};
use sqlx::SqlitePool;
#[allow(dead_code)] // retained: the in-memory form for future incremental updatespub struct UserGraph { pub terms: Vec<String>, pub index: HashMap<String, usize>, /// Adjacency as |weight| sums (undirected) for centrality/communities. pub adj: Vec<Vec<(usize, f64)>>, /// Signed edges: (src, dst, relation, weight, sign, variance, completion_uri). pub edges: Vec<(usize, usize, i64, f64, f64, f64, String)>, pub centrality: Vec<f64>, pub community: Vec<usize>, pub entropy: Vec<f64>, pub gloss_variance: Vec<f64>,}
fn norm_term(t: &str) -> String { t.trim().to_lowercase()}
/// Aggregate the gloss ensemble for one completion: readings grouped by/// canonical relation; disagreement across members is kept as variance.struct AggregatedReading { relation: i64, weight: f64, valence: f64, variance: f64,}
fn aggregate(readings_by_member: &[Vec<(i64, f64, f64)>]) -> Vec<AggregatedReading> { let mut per_relation: BTreeMap<i64, Vec<(f64, f64)>> = BTreeMap::new(); for member in readings_by_member { for (rel, w, v) in member { per_relation.entry(*rel).or_default().push((*w, *v)); } } let n_members = readings_by_member.len().max(1) as f64; per_relation .into_iter() .map(|(relation, wv)| { // Members that didn't propose this relation count as weight 0: // absence is disagreement. let mut weights: Vec<f64> = wv.iter().map(|(w, _)| *w).collect(); while (weights.len() as f64) < n_members { weights.push(0.0); } let mean = weights.iter().sum::<f64>() / weights.len() as f64; let var = weights.iter().map(|w| (w - mean).powi(2)).sum::<f64>() / weights.len() as f64; let valence = wv.iter().map(|(_, v)| *v).sum::<f64>() / wv.len() as f64; AggregatedReading { relation, weight: mean, valence, variance: var } }) .filter(|r| r.weight > 0.01) .collect()}
pub async fn rebuild_user_graph(pool: &SqlitePool, did: &str) -> anyhow::Result<()> { // Completions joined with their squares. let completions: Vec<(String, Option<String>, String, String, Option<String>, Option<String>, Option<String>, Option<String>, String)> = sqlx::query_as( "SELECT c.uri, c.text, c.mode, c.slot, s.a, s.b, s.c, s.d, s.open_slot FROM completions c JOIN squares s ON s.uri = c.square_uri WHERE c.did = ?", ) .bind(did) .fetch_all(pool) .await?;
// Glosses grouped by completion, then by ensemble member. let gloss_rows: Vec<(String, String, String)> = sqlx::query_as( "SELECT completion_uri, ensemble_member, readings FROM glosses WHERE did = ? AND orphaned = 0", ) .bind(did) .fetch_all(pool) .await?;
let mut glosses: HashMap<String, BTreeMap<String, Vec<(i64, f64, f64)>>> = HashMap::new(); for (curi, member, readings_json) in gloss_rows { let readings: Vec<serde_json::Value> = serde_json::from_str(&readings_json).unwrap_or_default(); let parsed: Vec<(i64, f64, f64)> = readings .iter() .filter_map(|r| { Some(( r.get("canonicalRelation")?.as_i64()?, r.get("weight")?.as_f64().unwrap_or(0.0), r.get("valence").and_then(|v| v.as_f64()).unwrap_or(0.0), )) }) .collect(); glosses.entry(curi).or_default().insert(member, parsed); }
let mut index: HashMap<String, usize> = HashMap::new(); let mut terms: Vec<String> = Vec::new(); let intern = |t: &str, terms: &mut Vec<String>, index: &mut HashMap<String, usize>| { let key = norm_term(t); *index.entry(key.clone()).or_insert_with(|| { terms.push(key); terms.len() - 1 }) };
let mut edges: Vec<(usize, usize, i64, f64, f64, f64, String)> = Vec::new(); // For the empty-signifier entropy: term -> multiset of completion texts // received in squares where the term was fixed. let mut received: HashMap<usize, Vec<String>> = HashMap::new();
for (uri, text, mode, slot, a, b, c, d, _open) in &completions { // Resolve the four positions with the user's text in the open slot. let get = |fixed: &Option<String>, s: &str| -> Option<String> { if slot == s { text.clone() } else { fixed.clone() } }; let ra = get(a, "a"); let rb = get(b, "b"); let rc = get(c, "c"); let rd = get(d, "d");
// Every fixed term hears the user's completion (entropy denominator). if mode == "completed" { if let Some(t) = text { for fixed in [a, b, c, d].into_iter().flatten() { let i = intern(fixed, &mut terms, &mut index); received.entry(i).or_default().push(norm_term(t)); } } }
if mode != "completed" { continue; // refusals enter the likelihood (docs/04), not the graph }
let members: Vec<Vec<(i64, f64, f64)>> = glosses .get(uri) .map(|m| m.values().cloned().collect()) .unwrap_or_default(); if members.is_empty() { continue; // not yet glossed; graph will pick it up next rebuild }
for agg in aggregate(&members) { for (x, y) in [(&ra, &rb), (&rc, &rd)] { if let (Some(x), Some(y)) = (x, y) { let xi = intern(x, &mut terms, &mut index); let yi = intern(y, &mut terms, &mut index); if xi != yi { edges.push(( xi, yi, agg.relation, agg.weight, agg.valence, agg.variance, uri.clone(), )); } } } } }
let n = terms.len(); let mut adj: Vec<Vec<(usize, f64)>> = vec![Vec::new(); n]; for (x, y, _, w, _, _, _) in &edges { adj[*x].push((*y, w.abs())); adj[*y].push((*x, w.abs())); }
let centrality = eigenvector_centrality(&adj); let community = label_propagation(&adj); let entropy: Vec<f64> = (0..n) .map(|i| received.get(&i).map(|texts| shannon(texts)).unwrap_or(0.0)) .collect(); let mut gloss_variance = vec![0.0f64; n]; let mut var_count = vec![0usize; n]; for (x, y, _, _, _, var, _) in &edges { for i in [*x, *y] { gloss_variance[i] += var; var_count[i] += 1; } } for i in 0..n { if var_count[i] > 0 { gloss_variance[i] /= var_count[i] as f64; } }
// Master signifier: highest-centrality term (fast approximation; the slow // loop's screening-off test in infer::slow is the principled version). let master = centrality .iter() .enumerate() .max_by(|a, b| a.1.total_cmp(b.1)) .map(|(i, _)| i);
let mut tx = pool.begin().await?; sqlx::query("DELETE FROM terms WHERE did = ?").bind(did).execute(&mut *tx).await?; sqlx::query("DELETE FROM edges WHERE did = ?").bind(did).execute(&mut *tx).await?; for i in 0..n { sqlx::query( "INSERT INTO terms (did, term, centrality, degree, community, entropy, gloss_variance, is_master) VALUES (?,?,?,?,?,?,?,?)", ) .bind(did) .bind(&terms[i]) .bind(centrality[i]) .bind(adj[i].len() as i64) .bind(community[i] as i64) .bind(entropy[i]) .bind(gloss_variance[i]) .bind(master == Some(i)) .execute(&mut *tx) .await?; } for (x, y, rel, w, sign, var, curi) in &edges { sqlx::query( "INSERT INTO edges (did, src, dst, relation, weight, sign, variance, completion_uri) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT (did, src, dst, relation, completion_uri) DO UPDATE SET weight = excluded.weight, sign = excluded.sign, variance = excluded.variance", ) .bind(did) .bind(&terms[*x]) .bind(&terms[*y]) .bind(rel) .bind(w) .bind(sign) .bind(var) .bind(curi) .execute(&mut *tx) .await?; } tx.commit().await?; Ok(())}
pub fn shannon(items: &[String]) -> f64 { let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); for it in items { *counts.entry(it.as_str()).or_default() += 1; } let total = items.len() as f64; if total == 0.0 { return 0.0; } -counts .values() .map(|&c| { let p = c as f64 / total; p * p.ln() }) .sum::<f64>()}
pub fn eigenvector_centrality(adj: &[Vec<(usize, f64)>]) -> Vec<f64> { let n = adj.len(); if n == 0 { return Vec::new(); } let mut v = vec![1.0 / (n as f64).sqrt(); n]; for _ in 0..100 { let mut next = vec![0.0; n]; for (i, nbrs) in adj.iter().enumerate() { for (j, w) in nbrs { next[*j] += v[i] * w; } } // Damping keeps disconnected nodes finite and iteration stable. for (i, x) in next.iter_mut().enumerate() { *x = 0.85 * *x + 0.15 * v[i]; } let norm: f64 = next.iter().map(|x| x * x).sum::<f64>().sqrt(); if norm == 0.0 { break; } for x in next.iter_mut() { *x /= norm; } let delta: f64 = next.iter().zip(&v).map(|(a, b)| (a - b).abs()).sum(); v = next; if delta < 1e-9 { break; } } // Normalize to [0, 1] for storage. let max = v.iter().cloned().fold(0.0f64, f64::max); if max > 0.0 { for x in v.iter_mut() { *x /= max; } } v}
/// Deterministic label propagation: fixed iteration order (term index),/// ties broken by smallest label — same data, same communities.pub fn label_propagation(adj: &[Vec<(usize, f64)>]) -> Vec<usize> { let n = adj.len(); let mut labels: Vec<usize> = (0..n).collect(); for _ in 0..50 { let mut changed = false; for i in 0..n { if adj[i].is_empty() { continue; } let mut tally: BTreeMap<usize, f64> = BTreeMap::new(); for (j, w) in &adj[i] { *tally.entry(labels[*j]).or_default() += w; } let best = tally .iter() .max_by(|a, b| a.1.total_cmp(b.1).then(b.0.cmp(a.0))) .map(|(l, _)| *l) .unwrap_or(labels[i]); if best != labels[i] { labels[i] = best; changed = true; } } if !changed { break; } } // Renumber to dense 0..k in first-seen order. let mut remap: BTreeMap<usize, usize> = BTreeMap::new(); let mut out = Vec::with_capacity(n); for l in labels { let next = remap.len(); out.push(*remap.entry(l).or_insert(next)); } out}
/// Load the stored graph for rendering/analysis.pub struct StoredGraph { pub terms: Vec<StoredTerm>, pub edges: Vec<StoredEdge>,}
pub struct StoredTerm { pub term: String, pub centrality: f64, #[allow(dead_code)] pub degree: i64, pub community: i64, pub entropy: f64, pub gloss_variance: f64, pub is_master: bool,}
pub struct StoredEdge { pub src: String, pub dst: String, #[allow(dead_code)] pub relation: i64, pub weight: f64, pub sign: f64, pub variance: f64,}
pub async fn load_user_graph(pool: &SqlitePool, did: &str) -> anyhow::Result<StoredGraph> { let term_rows: Vec<(String, f64, i64, i64, f64, f64, bool)> = sqlx::query_as( "SELECT term, centrality, degree, community, entropy, gloss_variance, is_master FROM terms WHERE did = ? ORDER BY centrality DESC, term", ) .bind(did) .fetch_all(pool) .await?; let edge_rows: Vec<(String, String, i64, f64, f64, f64)> = sqlx::query_as( "SELECT src, dst, relation, SUM(weight), AVG(sign), AVG(variance) FROM edges WHERE did = ? GROUP BY src, dst, relation", ) .bind(did) .fetch_all(pool) .await?; Ok(StoredGraph { terms: term_rows .into_iter() .map(|(term, centrality, degree, community, entropy, gloss_variance, is_master)| { StoredTerm { term, centrality, degree, community, entropy, gloss_variance, is_master } }) .collect(), edges: edge_rows .into_iter() .map(|(src, dst, relation, weight, sign, variance)| StoredEdge { src, dst, relation, weight, sign, variance, }) .collect(), })}
/// Distinct communities of the stored graph, for sector allocation.pub fn communities(g: &StoredGraph) -> BTreeSet<i64> { g.terms.iter().map(|t| t.community).collect()}
#[cfg(test)]mod tests { use super::*;
#[test] fn hub_dominates_eigenvector_centrality() { // Star: node 0 connected to 1..5. let mut adj: Vec<Vec<(usize, f64)>> = vec![Vec::new(); 6]; for i in 1..6 { adj[0].push((i, 1.0)); adj[i].push((0, 1.0)); } let c = eigenvector_centrality(&adj); assert!((c[0] - 1.0).abs() < 1e-9); for i in 1..6 { assert!(c[i] < c[0]); } }
#[test] fn label_propagation_finds_two_cliques() { // Two triangles joined by nothing. let mut adj: Vec<Vec<(usize, f64)>> = vec![Vec::new(); 6]; for &(a, b) in &[(0, 1), (1, 2), (0, 2), (3, 4), (4, 5), (3, 5)] { adj[a].push((b, 1.0)); adj[b].push((a, 1.0)); } let labels = label_propagation(&adj); assert_eq!(labels[0], labels[1]); assert_eq!(labels[1], labels[2]); assert_eq!(labels[3], labels[4]); assert_eq!(labels[4], labels[5]); assert_ne!(labels[0], labels[3]); }
#[test] fn shannon_entropy_behaves() { let uniform: Vec<String> = ["a", "b", "c", "d"].iter().map(|s| s.to_string()).collect(); let constant: Vec<String> = ["a", "a", "a", "a"].iter().map(|s| s.to_string()).collect(); assert!(shannon(&uniform) > shannon(&constant)); assert!(shannon(&constant).abs() < 1e-12); }}