diff --git a/experiments/for_you_v2.py b/experiments/for_you_v2.py index 75675e9..1530dd1 100644 --- a/experiments/for_you_v2.py +++ b/experiments/for_you_v2.py @@ -26,11 +26,41 @@ from collections import defaultdict import numpy as np DIVEPOOL_SEARCH = "https://divepool.social/api/v1/search" +DIVEPOOL_MEDOIDS = "https://divepool.social/api/v1/medoids" DIVEPOOL_STREAM = "https://divepool.social/api/v1/embeddings" BSKY_API = "https://public.api.bsky.app/xrpc" STATS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "for_you_v2_stats.json") LOG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "for_you_v2.log") +# ── Scoring pipeline (applied in order) ──────────────────────────────────── +# +# 1. IDF-weighted similarity: raw_sims * ref_weights → best_sim +# Per-reference specificity weight suppresses generic catch-all matches. +# +# 2. Multi-interest bonus: comp_score = best_sim + bonus for cross-cluster hits +# Posts matching multiple user interests get a small boost (max +0.15). +# +# 3. Z-score normalization (per cluster, Welford's online): +# Normalizes comp_score relative to each cluster's running distribution, +# so a "good" match for a rare cluster competes fairly with common ones. +# Raw comp_score used as fallback during warmup (first 100 scored posts). +# +# --- above computed per firehose post; below computed per window at flush --- +# +# 4. Cluster diversity malus: -MALUS per prior win for that cluster. +# Prevents one dominant cluster from winning every window. +# +# 5. Account affinity bonus: +AFFINITY_BONUS * max_sim(account_medoids, user_centroids) +# Accounts whose overall posting profile overlaps the user's interests. +# +# 6. Account credibility: +CREDIBILITY_WEIGHT * credibility_score(-1 to +1) +# Penalizes spam bots, boosts small legit accounts, neutral for large ones. +# +WARMUP = 100 # min scored posts before z-scores kick in +MALUS = 0.5 # z-score penalty per prior cluster win +AFFINITY_BONUS = 1.0 # max z-score bonus for account affinity +CREDIBILITY_WEIGHT = 1.5 # scales credibility (-1 to +1) + # ── API helpers ────────────────────────────────────────────────────────────── @@ -53,11 +83,8 @@ def divepool_search(token, query, limit=100, did=None, cluster=False, return json.loads(resp.read()) -DIVEPOOL_MEDOIDS = "https://divepool.social/api/v1/medoids" - - def fetch_account_medoids(token, dids): - """Batch-fetch up to 3 cluster medoids per account for up to 10 DIDs.""" + """Batch-fetch up to 3 cluster medoids per account (max 25 DIDs).""" payload = {"dids": dids[:25]} data = json.dumps(payload).encode() headers = {"Content-Type": "application/json"} @@ -474,21 +501,16 @@ def main(): seen_rkeys = set() did_freq = defaultdict(int) stats = FilterStats() - shown_scores = [] window_start = time.time() candidates = [] # (z, comp_score, best_label, other_labels, did, col, rkey) - cluster_shown_count = defaultdict(int) # cumulative malus: how many times each cluster won - MALUS = 0.5 # z-score penalty per prior win — after 2 wins, cluster needs z 1.0 higher - AFFINITY_BONUS = 1.0 # max z-score bonus for account affinity (scaled by similarity) - CREDIBILITY_WEIGHT = 1.5 # scales credibility score (now -1 to +1) - account_affinity_cache = {} # did -> affinity score + cluster_shown_count = defaultdict(int) # how many times each cluster won a window + account_affinity_cache = {} # did -> affinity score (0-1) profile_cache = {} # did -> (followers, following, posts) # Per-cluster running stats for z-score normalization (Welford's online algo) cluster_stats = defaultdict(lambda: [0, 0.0, 0.0]) # [count, mean, M2] total_scored = 0 - WARMUP = 100 # min scored posts before z-scores kick in def update_cluster_stats(label, score): s = cluster_stats[label] @@ -517,19 +539,24 @@ def main(): return True return False - def show(handle, text, rkey, adj_z, comp_score, label, affinity=0.0, cred=0.5): - link = bsky_link(handle, rkey) - aff_str = f" aff={affinity:.2f}" if affinity > 0 else "" - cred_str = f" cred={cred:+.2f}" if abs(cred) > 0.1 else "" + def format_age(rkey): age_sec = time.time() - tid_to_timestamp(rkey) - if age_sec < 60: - age_str = f"{age_sec:.0f}s" + if age_sec < 90: + return f"{age_sec:.0f} seconds ago" elif age_sec < 3600: - age_str = f"{age_sec / 60:.0f}m" + return f"{age_sec / 60:.0f} minutes ago" + elif age_sec < 86400: + return f"{age_sec / 3600:.1f} hours ago" else: - age_str = f"{age_sec / 3600:.1f}h" - print(f"[z={adj_z:.1f} {comp_score:.2f}{aff_str}{cred_str} {age_str} {label}]\n" - f"{text}\n{link} — @{handle}\n", + return f"{age_sec / 86400:.1f} days ago" + + def show(handle, text, rkey, adj_z, comp_score, label, affinity=0.0, cred=0.0): + link = bsky_link(handle, rkey) + aff_str = f" aff={affinity:.2f}" if affinity > 0 else "" + cred_str = f" cred={cred:+.2f}" if abs(cred) > 0.1 else "" + print(f"[z={adj_z:.1f} {comp_score:.2f}{aff_str}{cred_str} {label}]\n" + f"{text}\n{link} — @{handle}\n" + f"{format_age(rkey)}\n", flush=True) stats.record_shown(comp_score) @@ -581,7 +608,7 @@ def main(): for z, cs, bl, ol, d, c, rk in candidates: affinity = account_affinity_cache.get(d, 0.0) prof = profile_cache.get(d) - cred = account_credibility(*prof) if prof else 0.5 + cred = account_credibility(*prof) if prof else 0.0 adj = (z - MALUS * cluster_shown_count[bl] + AFFINITY_BONUS * affinity @@ -670,10 +697,10 @@ def main(): z, comp_score, best_label, other_labels, did, col, rkey, )) - # Keep candidate list bounded - if len(candidates) > 20: + # Keep candidate list bounded (generous: flush adjustments can reorder) + if len(candidates) > 50: candidates.sort(key=lambda c: -c[0]) - candidates = candidates[:10] + candidates = candidates[:25] # Check if window is up now = time.time()