Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
31 kB · 836 lines
Rust
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837use crate::{error::ApiError, state::AppState};use axum::{ Json, Router, extract::{Path, Query, State}, routing::{get, patch, post},};use chrono::{DateTime, Utc};use serde::{Deserialize, Serialize};use serde_json::{Value, json};use sqlx::{Postgres, Transaction};use std::collections::HashSet;use uuid::Uuid;
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]pub(crate) struct Material { pub id: Uuid, pub note_id: Option<Uuid>, pub kind: String, pub title: String, pub subject: String, pub body: String, pub sources: Value, pub filename: Option<String>, pub page_count: i32, pub status: String, pub progress: i32, pub error: Option<String>, pub position: Value, pub version: i64, pub asset_id: Option<Uuid>, pub created_at: DateTime<Utc>, pub updated_at: DateTime<Utc>,}
#[derive(Debug, Clone, Deserialize)]pub(crate) struct CreateMaterialRequest { pub title: String, pub subject: Option<String>, pub content: String, pub sources: Option<Value>,}
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]pub(crate) struct Chunk { pub id: i64, pub material_id: Uuid, pub page: Option<i32>, pub heading: String, pub text: String,}
#[derive(Debug, Serialize)]#[serde(rename_all = "camelCase")]pub(crate) struct ReaderContext { pub text: String, pub citations: Vec<Value>, pub estimated_tokens: usize, pub mode: String,}
pub fn routes() -> Router<AppState> { Router::new() .route( "/api/reader/health", get(|| async { Json(json!({"ok":true,"version":"2.0.0"})) }), ) .route( "/api/reader/materials", get(list_materials).post(create_article), ) .route( "/api/reader/materials/{id}", get(get_material_route) .patch(update_material) .delete(delete_material), ) .route( "/api/reader/materials/{id}/sources/{index}", axum::routing::delete(delete_source), ) .route("/api/reader/materials/{id}/search", get(search_material)) .route("/api/reader/materials/{id}/reindex", post(reindex_material)) .route( "/api/reader/materials/{id}/highlights", get(list_highlights).post(create_highlight), ) .route( "/api/reader/highlights/{id}", patch(update_highlight).delete(delete_highlight), ) .route("/api/reader/materials/{id}/revisions", get(list_revisions)) .route( "/api/reader/materials/{id}/revisions/{revision_id}/restore", post(restore_revision), ) .route("/api/reader/search", get(global_search)) .route( "/api/reader/from-note/{id}", post(crate::reader_native::from_note), ) .route( "/api/reader/from-learning/{id}", post(crate::reader_native::from_learning), ) .merge(crate::reader_learning::routes()) .merge(crate::reader_import::routes()) .merge(crate::reader_pdf::routes())}
pub(crate) fn bounded_text( value: &str, name: &str, max: usize, required: bool,) -> Result<(), ApiError> { if (required && value.trim().is_empty()) || value.chars().count() > max { return Err(ApiError::BadRequest(format!( "{name} must {}contain at most {max} characters", if required { "be nonempty and " } else { "" } ))); } Ok(())}
pub(crate) fn validate_sources(value: &Value) -> Result<(), ApiError> { let sources = value .as_array() .ok_or_else(|| ApiError::BadRequest("sources must be an array".into()))?; if sources.len() > 100 { return Err(ApiError::BadRequest( "at most 100 sources are supported".into(), )); } for source in sources { let title = source .get("title") .and_then(Value::as_str) .ok_or_else(|| ApiError::BadRequest("source title is required".into()))?; bounded_text(title, "source title", 400, false)?; let url = source .get("url") .and_then(Value::as_str) .ok_or_else(|| ApiError::BadRequest("source URL is required".into()))?; let parsed = reqwest::Url::parse(url) .map_err(|_| ApiError::BadRequest("invalid source URL".into()))?; if !matches!(parsed.scheme(), "http" | "https") { return Err(ApiError::BadRequest( "source URL must use HTTP or HTTPS".into(), )); } if let Some(excerpt) = source.get("excerpt") { let excerpt = excerpt .as_str() .ok_or_else(|| ApiError::BadRequest("source excerpt must be text".into()))?; bounded_text(excerpt, "source excerpt", 100000, false)?; } } Ok(())}
pub(crate) async fn get_material(state: &AppState, id: Uuid) -> Result<Material, ApiError> { Ok( sqlx::query_as::<_, Material>("select * from reader_materials where id=$1") .bind(id) .fetch_one(&state.platform_db) .await?, )}
async fn get_material_route( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<Json<Material>, ApiError> { Ok(Json(get_material(&state, id).await?))}
async fn list_materials(State(state): State<AppState>) -> Result<Json<Value>, ApiError> { let rows: Vec<Value> = sqlx::query_scalar("select to_jsonb(summary) from (select id,note_id,kind,title,subject,page_count,status,progress,error,position,version,asset_id,created_at,updated_at from reader_materials order by updated_at desc) summary") .fetch_all(&state.platform_db).await?; Ok(Json(Value::Array(rows)))}
pub(crate) fn article_chunks(body: &str) -> Vec<(String, String)> { let mut sections: Vec<(String, String)> = Vec::new(); let mut heading = "Introduction".to_owned(); let mut section = String::new(); for line in body.split_inclusive('\n') { let hashes = line.chars().take_while(|c| *c == '#').count(); if (1..=6).contains(&hashes) && line.as_bytes().get(hashes) == Some(&b' ') { if !section.is_empty() { sections.push((heading.clone(), std::mem::take(&mut section))); } heading = line[hashes + 1..].trim().to_owned(); } section.push_str(line); } if !section.is_empty() { sections.push((heading, section)); } let mut chunks = Vec::new(); for (heading, section) in sections { let chars: Vec<char> = section.chars().collect(); for start in (0..chars.len()).step_by(2000) { chunks.push(( heading.clone(), chars[start.saturating_sub(200)..(start + 2200).min(chars.len())] .iter() .collect(), )); } } chunks}
pub(crate) async fn index_article_tx( tx: &mut Transaction<'_, Postgres>, id: Uuid, body: &str,) -> Result<(), ApiError> { sqlx::query("delete from reader_chunks where material_id=$1") .bind(id) .execute(&mut **tx) .await?; for (heading, text) in article_chunks(body) { sqlx::query("insert into reader_chunks(material_id,heading,text) values($1,$2,$3)") .bind(id) .bind(heading) .bind(text) .execute(&mut **tx) .await?; } Ok(())}
pub(crate) async fn index_article(state: &AppState, id: Uuid, body: &str) -> Result<(), ApiError> { let mut tx = state.platform_db.begin().await?; index_article_tx(&mut tx, id, body).await?; tx.commit().await?; Ok(())}
pub(crate) async fn sync_catalog_tx( tx: &mut Transaction<'_, Postgres>, material: &Material,) -> Result<Material, ApiError> { let material = crate::reader_native::sync_material_note_tx(tx, material).await?; if material.kind == "book" { sqlx::query("insert into shelf_items(kind,title,asset_id,metadata,created_by) values($1,$2,$3,jsonb_build_object('reader_material_id',$4::text),'owner') on conflict ((metadata->>'reader_material_id')) where metadata ? 'reader_material_id' do update set kind=excluded.kind,title=excluded.title,asset_id=excluded.asset_id,updated_at=now()") .bind(&material.kind).bind(&material.title).bind(material.asset_id).bind(material.id.to_string()).execute(&mut **tx).await?; } crate::reader_native::sync_lesson_titles_tx(tx, material.id, &material.title).await?; Ok(material)}
pub(crate) async fn create_article_record( state: &AppState, input: CreateMaterialRequest,) -> Result<Material, ApiError> { bounded_text(&input.title, "title", 200, true)?; bounded_text(&input.content, "content", 4_000_000, false)?; let subject = input.subject.unwrap_or_default(); bounded_text(&subject, "subject", 200, false)?; let sources = input.sources.unwrap_or_else(|| json!([])); validate_sources(&sources)?; let mut tx = state.platform_db.begin().await?; let material=sqlx::query_as::<_,Material>("insert into reader_materials(kind,title,subject,body,sources) values('article',$1,$2,$3,$4) returning *") .bind(input.title.trim()).bind(subject).bind(&input.content).bind(sources).fetch_one(&mut *tx).await?; index_article_tx(&mut tx, material.id, &material.body).await?; let material = sync_catalog_tx(&mut tx, &material).await?; tx.commit().await?; Ok(material)}
async fn create_article( State(state): State<AppState>, Json(input): Json<CreateMaterialRequest>,) -> Result<(axum::http::StatusCode, Json<Material>), ApiError> { Ok(( axum::http::StatusCode::CREATED, Json(create_article_record(&state, input).await?), ))}
pub(crate) async fn snapshot_revision( tx: &mut Transaction<'_, Postgres>, material: &Material, reason: &str,) -> Result<(), ApiError> { sqlx::query("insert into reader_material_revisions(material_id,version,title,subject,body,sources,reason) values($1,$2,$3,$4,$5,$6,$7) on conflict(material_id,version) do nothing") .bind(material.id).bind(material.version).bind(&material.title).bind(&material.subject).bind(&material.body).bind(&material.sources).bind(reason).execute(&mut **tx).await?; Ok(())}
pub(crate) fn validate_position(position: &Value, material: &Material) -> Result<(), ApiError> { let object = position .as_object() .ok_or_else(|| ApiError::BadRequest("position must be an object".into()))?; for (key, value) in object { let number = value .as_f64() .filter(|n| n.is_finite()) .ok_or_else(|| ApiError::BadRequest(format!("invalid position {key}")))?; let valid = match key.as_str() { "page" => { number.fract() == 0.0 && number >= 1.0 && number <= material.page_count.max(1) as f64 } "zoom" => (0.4..=3.0).contains(&number), "scrollRatio" | "pageOffset" => (0.0..=1.0).contains(&number), _ => false, }; if !valid { return Err(ApiError::BadRequest(format!("invalid position {key}"))); } } Ok(())}
async fn update_material( State(state): State<AppState>, Path(id): Path<Uuid>, Json(input): Json<Value>,) -> Result<Json<Material>, ApiError> { let current = get_material(&state, id).await?; if let Some(position) = input.get("position") { validate_position(position, ¤t)?; let material = sqlx::query_as::<_, Material>( "update reader_materials set position=position || $2 where id=$1 returning *", ) .bind(id) .bind(position) .fetch_one(&state.platform_db) .await?; return Ok(Json(material)); } let expected = input .get("expected_version") .and_then(Value::as_i64) .ok_or_else(|| { ApiError::BadRequest( "expected_version is required when editing material content".into(), ) })?; if expected != current.version { return Err(ApiError::Conflict( "material changed; reload before saving".into(), )); } for key in ["title", "subject", "content"] { if input .get(key) .is_some_and(|value| !value.is_null() && !value.is_string()) { return Err(ApiError::BadRequest(format!("{key} must be text"))); } } let title = input .get("title") .and_then(Value::as_str) .unwrap_or(¤t.title); let subject = input .get("subject") .and_then(Value::as_str) .unwrap_or(¤t.subject); let body = input .get("content") .and_then(Value::as_str) .unwrap_or(¤t.body); let sources = input.get("sources").unwrap_or(¤t.sources); bounded_text(title, "title", 200, true)?; bounded_text(subject, "subject", 200, false)?; bounded_text(body, "content", 4_000_000, false)?; validate_sources(sources)?; if current.kind == "book" && (input.get("content").is_some() || input.get("sources").is_some()) { return Err(ApiError::BadRequest( "PDF contents cannot be overwritten as an article".into(), )); } let mut tx = state.platform_db.begin().await?; let material=sqlx::query_as::<_,Material>("update reader_materials set title=$3,subject=$4,body=$5,sources=$6,version=version+1,updated_at=now() where id=$1 and version=$2 returning *") .bind(id).bind(expected).bind(title.trim()).bind(subject).bind(body).bind(sources).fetch_optional(&mut *tx).await? .ok_or_else(||ApiError::Conflict("material changed; reload before saving".into()))?; snapshot_revision(&mut tx, ¤t, "edited").await?; if body != current.body { index_article_tx(&mut tx, id, body).await?; } let material = sync_catalog_tx(&mut tx, &material).await?; tx.commit().await?; Ok(Json(material))}
async fn delete_material( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<axum::http::StatusCode, ApiError> { let mut tx = state.platform_db.begin().await?; let material = sqlx::query_as::<_, Material>("select * from reader_materials where id=$1 for update") .bind(id) .fetch_one(&mut *tx) .await?; crate::reader_native::delete_projection_tx(&mut tx, &material).await?; if let Some(note) = material.note_id { sqlx::query("delete from notes where id=$1") .bind(note) .execute(&mut *tx) .await?; } tx.commit().await?; // Book assets may also be referenced by existing native notes or shelf entries. // Preserve those shared records; only material-owned bundle files are removed. let _ = tokio::fs::remove_dir_all(state.config.storage_dir.join("reader").join(id.to_string())) .await; Ok(axum::http::StatusCode::NO_CONTENT)}
async fn delete_source( State(state): State<AppState>, Path((id, index)): Path<(Uuid, usize)>, input: Option<Json<Value>>,) -> Result<Json<Material>, ApiError> { let material = get_material(&state, id).await?; if material.kind != "article" { return Err(ApiError::BadRequest( "The PDF is part of its book. Delete the book to remove it.".into(), )); } let mut sources = material.sources.as_array().cloned().unwrap_or_default(); if index >= sources.len() { return Err(ApiError::NotFound("source not found".into())); } sources.remove(index); update_material(State(state),Path(id),Json(json!({"expected_version":input.as_ref().and_then(|Json(value)|value.get("expected_version")).and_then(Value::as_i64).unwrap_or(material.version),"sources":sources}))).await}
#[derive(Deserialize)]struct SearchQuery { #[serde(default)] q: String,}
pub(crate) async fn search_chunks( state: &AppState, id: Uuid, query: &str, limit: i64,) -> Result<Vec<Chunk>, ApiError> { let tokens: Vec<String> = query .split(|c: char| !c.is_alphanumeric()) .filter(|s| s.chars().count() >= 2) .take(18) .map(|s| format!("'{}':*", s.replace('\'', "''"))) .collect(); if tokens.is_empty() { return Ok(Vec::new()); } let query = tokens.join(" | "); Ok(sqlx::query_as::<_,Chunk>("select id,material_id,page,heading,text from reader_chunks where material_id=$1 and search_vector @@ to_tsquery('simple',$2) order by ts_rank(search_vector,to_tsquery('simple',$2)) desc,id limit $3") .bind(id).bind(query).bind(limit.clamp(1,100)).fetch_all(&state.platform_db).await?)}
async fn search_material( State(state): State<AppState>, Path(id): Path<Uuid>, Query(query): Query<SearchQuery>,) -> Result<Json<Vec<Chunk>>, ApiError> { get_material(&state, id).await?; Ok(Json(search_chunks(&state, id, &query.q, 30).await?))}
async fn reindex_material( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<Json<Material>, ApiError> { let material = get_material(&state, id).await?; if material.kind == "book" && material.status != "indexing" { sqlx::query( "update reader_materials set status='indexing',progress=0,error=null where id=$1", ) .bind(id) .execute(&state.platform_db) .await?; crate::reader_pdf::enqueue_index(state.clone(), id); } else if material.kind == "article" { index_article(&state, id, &material.body).await?; } Ok(Json(get_material(&state, id).await?))}
#[derive(Debug, Serialize, sqlx::FromRow)]struct Highlight { id: Uuid, material_id: Uuid, quote: String, prefix: String, suffix: String, page: Option<i32>, rects: Value, note: String, content_version: i64, created_at: DateTime<Utc>,}
pub(crate) fn validate_highlight(input: &Value, material: &Material) -> Result<(), ApiError> { bounded_text( input.get("quote").and_then(Value::as_str).unwrap_or(""), "quote", 12000, true, )?; for (key, max) in [("prefix", 200), ("suffix", 200), ("note", 10000)] { if let Some(value) = input.get(key) { let value = value .as_str() .ok_or_else(|| ApiError::BadRequest(format!("{key} must be text")))?; bounded_text(value, key, max, false)?; } } if let Some(page) = input.get("page").filter(|p| !p.is_null()) { let page = page .as_i64() .ok_or_else(|| ApiError::BadRequest("invalid PDF page".into()))?; if material.kind != "book" || page < 1 || page > material.page_count as i64 { return Err(ApiError::BadRequest("invalid PDF page".into())); } } if let Some(rects) = input.get("rects") { let rects = rects .as_array() .ok_or_else(|| ApiError::BadRequest("rects must be an array".into()))?; if rects.len() > 200 { return Err(ApiError::BadRequest("too many selection rectangles".into())); } for rect in rects { for key in ["x", "y", "width", "height"] { if !rect .get(key) .and_then(Value::as_f64) .is_some_and(|v| (0.0..=1.0).contains(&v)) { return Err(ApiError::BadRequest("invalid normalized rectangle".into())); } } } } Ok(())}
async fn list_highlights( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<Json<Vec<Highlight>>, ApiError> { get_material(&state, id).await?; Ok(Json( sqlx::query_as::<_, Highlight>( "select * from reader_highlights where material_id=$1 order by created_at,id", ) .bind(id) .fetch_all(&state.platform_db) .await?, ))}async fn create_highlight( State(state): State<AppState>, Path(id): Path<Uuid>, Json(input): Json<Value>,) -> Result<(axum::http::StatusCode, Json<Highlight>), ApiError> { let material = get_material(&state, id).await?; validate_highlight(&input, &material)?; let row=sqlx::query_as::<_,Highlight>("insert into reader_highlights(material_id,quote,prefix,suffix,page,rects,note,content_version) values($1,$2,$3,$4,$5,$6,$7,$8) returning *") .bind(id).bind(input["quote"].as_str().unwrap_or_default()).bind(input["prefix"].as_str().unwrap_or_default()).bind(input["suffix"].as_str().unwrap_or_default()).bind(input["page"].as_i64().map(|n|n as i32)).bind(input.get("rects").cloned().unwrap_or_else(||json!([]))).bind(input["note"].as_str().unwrap_or_default()).bind(material.version).fetch_one(&state.platform_db).await?; Ok((axum::http::StatusCode::CREATED, Json(row)))}async fn update_highlight( State(state): State<AppState>, Path(id): Path<Uuid>, Json(input): Json<Value>,) -> Result<axum::http::StatusCode, ApiError> { let note = input["note"] .as_str() .ok_or_else(|| ApiError::BadRequest("note is required".into()))?; bounded_text(note, "note", 10000, false)?; if sqlx::query("update reader_highlights set note=$2 where id=$1") .bind(id) .bind(note) .execute(&state.platform_db) .await? .rows_affected() == 0 { return Err(ApiError::NotFound("highlight not found".into())); } Ok(axum::http::StatusCode::NO_CONTENT)}async fn delete_highlight( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<axum::http::StatusCode, ApiError> { sqlx::query("delete from reader_highlights where id=$1") .bind(id) .execute(&state.platform_db) .await?; Ok(axum::http::StatusCode::NO_CONTENT)}
async fn list_revisions( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<Json<Value>, ApiError> { get_material(&state, id).await?; let rows:Value=sqlx::query_scalar("select coalesce(jsonb_agg(to_jsonb(r) order by version desc),'[]'::jsonb) from reader_material_revisions r where material_id=$1").bind(id).fetch_one(&state.platform_db).await?; Ok(Json(rows))}async fn restore_revision( State(state): State<AppState>, Path((id, revision_id)): Path<(Uuid, Uuid)>, Json(input): Json<Value>,) -> Result<Json<Material>, ApiError> { let revision: Value = sqlx::query_scalar( "select to_jsonb(r) from reader_material_revisions r where id=$1 and material_id=$2", ) .bind(revision_id) .bind(id) .fetch_one(&state.platform_db) .await?; let expected = input .get("expected_version") .and_then(Value::as_i64) .ok_or_else(|| ApiError::BadRequest("expected_version is required".into()))?; let material = get_material(&state, id).await?; let mut restore = json!({"expected_version":expected,"title":revision["title"],"subject":revision["subject"]}); if material.kind == "article" { restore["content"] = revision["body"].clone(); restore["sources"] = revision["sources"].clone(); } update_material(State(state), Path(id), Json(restore)).await}
fn take_chars(value: &str, limit: usize) -> String { value.chars().take(limit).collect()}pub(crate) async fn context_for( state: &AppState, material: &Material, query: &str, selection: &str, page: Option<i32>, context_tokens: usize,) -> Result<ReaderContext, ApiError> { let budget = context_tokens.clamp(1000, 64000) * 3; let mut pieces = Vec::new(); let mut citations = Vec::new(); let mut size = 0usize; let mut seen = HashSet::new(); if material.kind == "article" && material.body.chars().count() < budget * 8 / 10 { pieces.push(material.body.clone()); size = material.body.chars().count(); } else { let mut chunks = Vec::new(); if let Some(page) = page { chunks.extend(sqlx::query_as::<_,Chunk>("select id,material_id,page,heading,text from reader_chunks where material_id=$1 and page between $2 and $3 order by abs(page-$4),id").bind(material.id).bind((page-1).max(1)).bind(page+1).bind(page).fetch_all(&state.platform_db).await?); } chunks .extend(search_chunks(state, material.id, &format!("{query} {selection}"), 14).await?); if chunks.is_empty() { chunks=sqlx::query_as::<_,Chunk>("select id,material_id,page,heading,text from reader_chunks where material_id=$1 order by id limit 5").bind(material.id).fetch_all(&state.platform_db).await?; } for chunk in chunks { if !seen.insert(chunk.id) || chunk.text.is_empty() || budget.saturating_sub(size) < 200 { continue; } let text = take_chars(&chunk.text, budget - size); size += text.chars().count(); let label = chunk .page .map(|p| format!("[p. {p}]")) .unwrap_or_else(|| format!("[Section: {}]", chunk.heading)); pieces.push(format!("{label}\n{text}")); if let Some(page) = chunk.page { if !citations .iter() .any(|c: &Value| c["page"].as_i64() == Some(page as i64)) { citations.push(json!({"label":format!("Page {page}"),"page":page,"excerpt":take_chars(&text,300)})); } } } } for (i, source) in material .sources .as_array() .into_iter() .flatten() .enumerate() { let metadata = format!( "[s{}] {}: {}", i + 1, source["title"].as_str().unwrap_or_default(), source["url"].as_str().unwrap_or_default() ); let length = metadata.chars().count(); if size + length > budget { break; } let excerpt = take_chars( source["excerpt"].as_str().unwrap_or_default(), budget.saturating_sub(size + length + 1), ); size += length + excerpt.chars().count() + usize::from(!excerpt.is_empty()); pieces.push(if excerpt.is_empty() { metadata } else { format!("{metadata}\n{excerpt}") }); citations.push(json!({"label":source["title"],"url":source["url"],"source":i+1})); } Ok(ReaderContext { text: pieces.join("\n\n"), citations, estimated_tokens: size.div_ceil(3), mode: if material.kind == "book" { "Page context + local full-text retrieval" } else { "Lesson + source excerpts" } .into(), })}
async fn global_search( State(state): State<AppState>, Query(query): Query<SearchQuery>,) -> Result<Json<Value>, ApiError> { bounded_text(&query.q, "query", 200, false)?; if query.q.trim().chars().count() < 2 { return Ok(Json(json!([]))); } let pattern = format!( "%{}%", query .q .trim() .replace('\\', "\\\\") .replace('%', "\\%") .replace('_', "\\_") ); let selections = [ ( "material", "select m.id::text item_id,m.id material_id,m.title,m.kind,null::int page,''::text heading,null::uuid chat_id,null::uuid message_id,m.title || ' ' || m.subject text from reader_materials m where m.title ilike $1 or m.subject ilike $1", ), ( "passage", "select c.id::text item_id,m.id material_id,m.title,m.kind,c.page,c.heading,null::uuid chat_id,null::uuid message_id,c.text from reader_chunks c join reader_materials m on m.id=c.material_id where c.text ilike $1", ), ( "highlight", "select h.id::text item_id,m.id material_id,m.title,m.kind,h.page,''::text heading,null::uuid chat_id,null::uuid message_id,h.quote || chr(10) || h.note text,h.quote from reader_highlights h join reader_materials m on m.id=h.material_id where h.quote ilike $1 or h.note ilike $1", ), ( "note", "select n.id::text item_id,m.id material_id,m.title,m.kind,n.page,n.title heading,n.chat_id,n.message_id,n.body text,n.quote from reader_notes n join reader_materials m on m.id=n.material_id where n.title ilike $1 or n.body ilike $1", ), ( "chat", "select a.id::text item_id,m.id material_id,m.title,m.kind,null::int page,c.title heading,c.id chat_id,a.id message_id,a.content text from agent_messages a join agent_conversations c on c.id=a.conversation_id join reader_materials m on m.id=c.context_id where c.context_type='reader' and a.content ilike $1", ), ]; let mut results = Vec::new(); for (kind, sql) in selections { let sql = format!("select to_jsonb(hit) from ({sql} limit 20) hit"); let rows: Vec<Value> = sqlx::query_scalar(&sql) .bind(&pattern) .fetch_all(&state.platform_db) .await?; for mut row in rows { if let Some(object) = row.as_object_mut() { let text = object .remove("text") .and_then(|v| v.as_str().map(ToOwned::to_owned)) .unwrap_or_default(); let chars: Vec<char> = text.chars().collect(); let lower = text.to_lowercase(); let offset = lower .find(&query.q.to_lowercase()) .map(|byte| lower[..byte].chars().count()) .unwrap_or(0) .saturating_sub(65); let excerpt: String = chars.iter().skip(offset).take(220).collect(); object.insert("type".into(), json!(kind)); object.insert( "excerpt".into(), json!(format!("{}{excerpt}", if offset > 0 { "…" } else { "" })), ); object.insert("query".into(), json!(query.q)); } results.push(row); } } Ok(Json(Value::Array(results)))}
#[cfg(test)]mod tests { use super::*; #[test] fn chunks_preserve_unicode_and_section_boundaries() { let chunks = article_chunks(&format!("Начало\n## Память\n{}", "🧠".repeat(4500))); assert_eq!(chunks[0].0, "Introduction"); assert!( chunks .iter() .skip(1) .all(|(heading, _)| heading == "Память") ); assert!(chunks.iter().all(|(_, text)| !text.contains('\u{fffd}'))); assert_eq!(chunks.len(), 4); } #[test] fn source_validation_rejects_executable_urls() { assert!(validate_sources(&json!([{"title":"x","url":"javascript:alert(1)"}])).is_err()); assert!( validate_sources( &json!([{"title":"x","url":"https://example.org","excerpt":"summary"}]) ) .is_ok() ); }}