Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
38 kB · 874 lines
Rust
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875//! Candidate generation never mutates the canonical lesson; applying is version checked.use crate::{ completion::{CompletionRequest, completion_stream}, domains::provider, error::ApiError, reader::{self, Material}, reader_chats::{event, validate_provider, validate_text}, state::AppState,};use axum::{ Json, Router, extract::{Path, Query, State}, response::sse::{Event, KeepAlive, KeepAliveStream, Sse}, routing::{get, post},};use chrono::{DateTime, Utc};use serde::{Deserialize, Serialize};use serde_json::{Value, json};use std::{ collections::HashMap, convert::Infallible, sync::{Mutex, OnceLock}, time::{Duration, Instant},};use tokio::sync::{mpsc, watch};use tokio_stream::wrappers::ReceiverStream;use uuid::Uuid;
struct ActiveProposal { material_id: Option<Uuid>, cancel: watch::Sender<bool>,}static ACTIVE: OnceLock<Mutex<HashMap<Uuid, ActiveProposal>>> = OnceLock::new();fn active() -> &'static Mutex<HashMap<Uuid, ActiveProposal>> { ACTIVE.get_or_init(|| Mutex::new(HashMap::new()))}pub(crate) fn cancel_material(material_id: Uuid) { if let Ok(active) = active().lock() { for turn in active .values() .filter(|t| t.material_id == Some(material_id)) { let _ = turn.cancel.send(true); } }}
pub(crate) const AUTHORING_RULES: &str = include_str!("authoring.md");type EventStream = Sse<KeepAliveStream<ReceiverStream<Result<Event, Infallible>>>>;#[derive(Clone, Debug, Serialize, sqlx::FromRow)]struct Proposal { id: Uuid, material_id: Option<Uuid>, operation: String, instruction: String, title: String, selection: Value, replacement: String, base_version: Option<i64>, status: String, error: Option<String>, provider: String, model: String, metadata: Value, created_at: DateTime<Utc>, updated_at: DateTime<Utc>,}
pub(crate) fn routes() -> Router<AppState> { Router::new() .route("/api/reader/generate", post(generate)) .route("/api/reader/materials/{id}/rewrite", post(rewrite)) .route("/api/reader/proposals", get(list_proposals)) .route("/api/reader/proposals/{id}", get(get_proposal)) .route("/api/reader/proposals/{id}/apply", post(apply)) .route( "/api/reader/authoring-rules", get(|| async { Json(json!({"version":1,"instructions":AUTHORING_RULES})) }), ) .route("/api/reader/lessons/{id}/revisions", get(native_revisions)) .route( "/api/reader/lessons/{id}/revisions/{revision}/restore", post(restore_native_revision), )}#[derive(Deserialize)]#[serde(rename_all = "camelCase")]struct GenerateInput { title: String, instruction: String, space_id: Option<Uuid>, lesson_id: Option<Uuid>, provider_id: String, model: String, reasoning_effort: Option<String>, sources: Option<Value>,}#[derive(Deserialize)]#[serde(rename_all = "camelCase")]struct RewriteInput { instruction: String, selection: Option<Selection>, provider_id: String, model: String, reasoning_effort: Option<String>, #[serde(alias = "expected_version")] expected_version: Option<i64>,}#[derive(Clone, Debug, Serialize, Deserialize)]struct Selection { quote: String, #[serde(default)] prefix: String, #[serde(default)] suffix: String,}
#[derive(Deserialize)]#[serde(rename_all = "camelCase")]struct ProposalQuery { material_id: Option<Uuid>, lesson_id: Option<Uuid>,}async fn list_proposals( State(state): State<AppState>, Query(query): Query<ProposalQuery>,) -> Result<Json<Vec<Value>>, ApiError> { let rows:Vec<Value>=sqlx::query_scalar("select to_jsonb(p)-'replacement'-'selection' from reader_rewrite_proposals p where status<>'applied' and ($1::uuid is null or material_id=$1) and ($2::text is null or metadata->>'lesson_id'=$2) order by updated_at desc limit 50").bind(query.material_id).bind(query.lesson_id.map(|id|id.to_string())).fetch_all(&state.platform_db).await?; Ok(Json(rows))}
async fn get_proposal( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<Json<Proposal>, ApiError> { Ok(Json( sqlx::query_as::<_, Proposal>("select * from reader_rewrite_proposals where id=$1") .bind(id) .fetch_one(&state.platform_db) .await?, ))}async fn generate( State(state): State<AppState>, Json(input): Json<GenerateInput>,) -> Result<EventStream, ApiError> { validate_text(&input.title, 200, "Lesson title")?; validate_text(&input.instruction, 20000, "Instruction")?; let setting = provider::provider_setting(&state, &input.provider_id).await?; validate_provider(&setting, &input.model)?; let sources = input.sources.unwrap_or_else(|| json!([])); reader::validate_sources(&sources)?; let budget = setting.options["contextTokens"] .as_u64() .unwrap_or(6000) .clamp(1000, 64000) as usize; let mut metadata = json!({"space_id":input.space_id,"lesson_id":input.lesson_id,"sources":sources}); let mut reference = String::new(); let mut target = None; if let Some(space) = input.space_id { let exists: bool = sqlx::query_scalar("select exists(select 1 from learning_spaces where id=$1)") .bind(space) .fetch_one(&state.platform_db) .await?; if !exists { return Err(ApiError::NotFound("Learning space not found.".into())); } } if let Some(lesson) = input.lesson_id { let data: Value = sqlx::query_scalar("select to_jsonb(l) from learning_lessons l where id=$1") .bind(lesson) .fetch_one(&state.platform_db) .await?; if input .space_id .is_some_and(|id| data["space_id"].as_str() != Some(id.to_string().as_str())) { return Err(ApiError::BadRequest( "Lesson belongs to a different learning space.".into(), )); } metadata["lesson_version"] = data["version"].clone(); metadata["space_id"] = data["space_id"].clone(); if let Some(id) = data["document"]["reader_material_id"] .as_str() .and_then(|s| Uuid::parse_str(s).ok()) { let material = reader::get_material(&state, id).await?; reference = rewrite_reference(&state, &material, &input.instruction, None, budget).await?; metadata["sources"] = material.sources.clone(); target = Some(material); } else { reference = format!( "Existing native lesson document (reference data):\n{}", serde_json::to_string(&data["document"]).map_err(anyhow::Error::from)? ); if reference.chars().count() > budget * 3 { return Err(ApiError::BadRequest("The native lesson is too large for the configured context. Increase the context budget before regenerating it.".into())); } let resources: Vec<Uuid> = data["resource_ids"] .as_array() .into_iter() .flatten() .filter_map(|v| v.as_str().and_then(|s| Uuid::parse_str(s).ok())) .take(20) .collect(); if !resources.is_empty() { let refs: Vec<Value> = sqlx::query_scalar( "select to_jsonb(i) from shelf_items i where id=any($1) order by created_at", ) .bind(resources) .fetch_all(&state.platform_db) .await?; let mut remaining = (budget * 3).saturating_sub(reference.chars().count()); for resource in refs { if remaining < 300 { break; } let head = format!( "\nAttached resource: {}\nURL: {}\n", resource["title"].as_str().unwrap_or_default(), resource["url"].as_str().unwrap_or_default() ); let head: String = head.chars().take(remaining).collect(); remaining = remaining.saturating_sub(head.chars().count()); reference.push_str(&head); if let Some(mid) = resource["metadata"]["reader_material_id"] .as_str() .and_then(|s| Uuid::parse_str(s).ok()) { if let Ok(material) = reader::get_material(&state, mid).await { let context = reader::context_for( &state, &material, &input.instruction, "", None, (remaining / 3).clamp(1000, 4000), ) .await?; let mut text = context.text; let manifest = metadata["sources"] .as_array_mut() .expect("validated source array"); let offset = manifest.len(); for (i, source) in material .sources .as_array() .into_iter() .flatten() .enumerate() { if manifest.len() >= 100 { return Err(ApiError::BadRequest( "Attached resources exceed the 100 source limit.".into(), )); } manifest.push(source.clone()); text = text.replace( &format!("[s{}]", i + 1), &format!("[attached_source_{}]", offset + i + 1), ); } for i in offset + 1..=manifest.len() { text = text .replace(&format!("[attached_source_{i}]"), &format!("[s{i}]")); } // A source PDF's page numbers refer to that named resource, not the new article. text = text.replace("[p. ", &format!("[{} p. ", material.title)); let text: String = text.chars().take(remaining).collect(); remaining = remaining.saturating_sub(text.chars().count()); reference.push_str(&text); } } } } // Snapshot is taken at apply time under a lesson row lock. } } if reference.is_empty() && sources.as_array().is_some_and(|s| !s.is_empty()) { reference = source_context(&sources, budget * 3); } let request = CompletionRequest { provider: input.provider_id.clone(), model: input.model.clone(), reasoning_effort: input.reasoning_effort, system: AUTHORING_RULES.into(), messages: vec![ json!({"role":"user","content":format!("Lesson title: {}\nOperation: {}\nUser requirements:\n{}\n\n{}",input.title,if target.is_some(){"rewrite the whole lesson"}else{"generate the complete lesson"},input.instruction,reference)}), ], max_output_tokens: setting.options["maxTokens"] .as_u64() .unwrap_or(8192) .clamp(256, 32000) as u32, }; let proposal=sqlx::query_as::<_,Proposal>("insert into reader_rewrite_proposals(material_id,operation,instruction,title,base_version,provider,model,metadata) values($1,$2,$3,$4,$5,$6,$7,$8) returning *") .bind(target.as_ref().map(|m|m.id)).bind(if target.is_some(){"rewrite"}else{"generate"}).bind(input.instruction).bind(input.title.trim()).bind(target.as_ref().map(|m|m.version)).bind(input.provider_id).bind(input.model).bind(metadata).fetch_one(&state.platform_db).await?; Ok(stream_proposal(state, proposal, request))}
async fn rewrite( State(state): State<AppState>, Path(id): Path<Uuid>, Json(input): Json<RewriteInput>,) -> Result<EventStream, ApiError> { validate_text(&input.instruction, 20000, "Instruction")?; let material = reader::get_material(&state, id).await?; if material.kind != "article" { return Err(ApiError::BadRequest( "PDF source text cannot be rewritten. Create a lesson from it instead.".into(), )); } if input .expected_version .is_some_and(|v| v != material.version) { return Err(ApiError::Conflict( "Material changed after opening the editor. Reload before requesting a rewrite.".into(), )); } let selection = if let Some(selection) = &input.selection { validate_text(&selection.quote, 12000, "Selected passage")?; if selection.prefix.chars().count() > 200 || selection.suffix.chars().count() > 200 { return Err(ApiError::BadRequest( "Selection context is too long.".into(), )); } let (start, end) = resolve_selection(&material.body, selection)?; json!({"quote":selection.quote,"prefix":selection.prefix,"suffix":selection.suffix,"start":start,"end":end,"original":&material.body[start..end]}) } else { json!({}) }; let setting = provider::provider_setting(&state, &input.provider_id).await?; validate_provider(&setting, &input.model)?; let budget = setting.options["contextTokens"] .as_u64() .unwrap_or(6000) .clamp(1000, 64000) as usize; let reference = rewrite_reference( &state, &material, &input.instruction, input.selection.as_ref(), budget, ) .await?; let scope = if input.selection.is_some() { format!( "Replace only this Markdown passage, returning just its replacement:\n{}", selection["original"].as_str().unwrap_or_default() ) } else { "Rewrite the complete lesson, returning the whole Markdown document.".into() }; let request = CompletionRequest { provider: input.provider_id.clone(), model: input.model.clone(), reasoning_effort: input.reasoning_effort, system: AUTHORING_RULES.into(), messages: vec![ json!({"role":"user","content":format!("Lesson: {}\nScope: {}\nUser request:\n{}\n\nREFERENCE DATA:\n{}",material.title,scope,input.instruction,reference)}), ], max_output_tokens: setting.options["maxTokens"] .as_u64() .unwrap_or(8192) .clamp(256, 32000) as u32, }; let proposal=sqlx::query_as::<_,Proposal>("insert into reader_rewrite_proposals(material_id,operation,instruction,title,selection,base_version,provider,model,metadata) values($1,'rewrite',$2,$3,$4,$5,$6,$7,$8) returning *").bind(id).bind(input.instruction).bind(&material.title).bind(selection).bind(material.version).bind(input.provider_id).bind(input.model).bind(json!({"sources":material.sources})).fetch_one(&state.platform_db).await?; Ok(stream_proposal(state, proposal, request))}fn source_context(sources: &Value, budget: usize) -> String { let sources: Vec<&Value> = sources.as_array().into_iter().flatten().collect(); let mut text = String::new(); let mut used = 0; for (index, source) in sources.iter().enumerate() { let head = format!( "[s{}] {}: {}\n", index + 1, source["title"].as_str().unwrap_or_default(), source["url"].as_str().unwrap_or_default() ); if used + head.chars().count() + 2 >= budget { break; } used += head.chars().count(); text.push_str(&head); let remaining_sources = sources.len() - index; let excerpt: String = source["excerpt"] .as_str() .unwrap_or_default() .chars() .take((budget - used).saturating_sub(2) / remaining_sources) .collect(); used += excerpt.chars().count(); text.push_str(&excerpt); text.push_str("\n\n"); used += 2; } text}
async fn rewrite_reference( state: &AppState, material: &Material, instruction: &str, selection: Option<&Selection>, budget: usize,) -> Result<String, ApiError> { if selection.is_none() && material.body.chars().count() > budget * 3 * 3 / 4 { return Err(ApiError::BadRequest("This lesson exceeds the configured context budget for a complete rewrite. Select a passage or increase Context tokens in Settings.".into())); } let context = reader::context_for( state, material, instruction, selection.map(|s| s.quote.as_str()).unwrap_or_default(), None, budget, ) .await?; let mut text = context.text; if selection.is_none() && !text.contains(&material.body) { text = format!("CURRENT LESSON:\n{}\n\nSOURCES:\n{}", material.body, text); } if let Some(s) = selection { let (start, end) = resolve_selection(&material.body, s)?; let before_start = material.body[..start] .char_indices() .rev() .nth(2000) .map(|(i, _)| i) .unwrap_or(0); let after_end = material.body[end..] .char_indices() .nth(2000) .map(|(i, _)| end + i) .unwrap_or(material.body.len()); text = format!( "Immediate surrounding Markdown:\n{}\n\nRetrieved lesson and source context:\n{}", &material.body[before_start..after_end], text ); } if text.chars().count() > budget * 3 + 6000 { return Err(ApiError::BadRequest("Selected passage and source context exceed the budget. Increase Context tokens or shorten the selection.".into())); } Ok(text)}
fn stream_proposal(state: AppState, proposal: Proposal, request: CompletionRequest) -> EventStream { let (tx, rx) = mpsc::channel(32); let (cancel, mut cancelled) = watch::channel(false); if let Ok(mut active) = active().lock() { active.insert( proposal.id, ActiveProposal { material_id: proposal.material_id, cancel, }, ); } tokio::spawn(async move { let mut replacement = String::new(); let mut status = "complete"; let mut error = None; let _=tx.send(event(json!({"type":"context","proposalId":proposal.id,"base_version":proposal.base_version,"mode":"Learning content proposal"}))).await; match completion_stream(state.clone(), request).await { Err(e) => { status = "error"; error = Some(e.to_string()); } Ok(mut upstream) => { let mut checkpoint = Instant::now(); loop { let delta = tokio::select! {_=tx.closed()=>{status="cancelled";break;},_=cancelled.changed()=>{status="cancelled";break;},d=upstream.recv()=>d}; match delta { None => break, Some(Err(e)) => { status = "error"; error = Some(e); break; } Some(Ok(text)) => { replacement.push_str(&text); let sent = tokio::select! { _=cancelled.changed()=>false, result=tx.send(event(json!({"type":"delta","text":text})))=>result.is_ok(), }; if !sent { status = "cancelled"; break; } if checkpoint.elapsed() > Duration::from_millis(500) { match sqlx::query("update reader_rewrite_proposals set replacement=$2,updated_at=now() where id=$1").bind(proposal.id).bind(&replacement).execute(&state.platform_db).await { Ok(r) if r.rows_affected()>0=>{}, _=>{status="error";error=Some("Could not save the proposal.".into());break;}, } checkpoint = Instant::now(); } } } } drop(upstream); } } if let Ok(mut active) = active().lock() { active.remove(&proposal.id); } if status == "complete" && replacement.trim().is_empty() { status = "error"; error = Some("The model returned no content.".into()); } let result=sqlx::query_as::<_,Proposal>("update reader_rewrite_proposals set replacement=$2,status=$3,error=$4,updated_at=now() where id=$1 returning *").bind(proposal.id).bind(replacement).bind(status).bind(&error).fetch_one(&state.platform_db).await; match result { Ok(saved) => { if let Some(error) = error { let _ = tx.send(event(json!({"type":"error","error":error}))).await; } let _ = tx .send(event( json!({"type":"done","id":proposal.id,"status":status,"proposal":saved}), )) .await; } Err(e) => { tracing::error!(?e, "could not persist generation proposal"); let _=tx.send(event(json!({"type":"error","error":"Could not save the proposal. Copy the visible text before reloading."}))).await; let _ = tx .send(event( json!({"type":"done","id":proposal.id,"status":"error"}), )) .await; } } }); Sse::new(ReceiverStream::new(rx)).keep_alive(KeepAlive::new().interval(Duration::from_secs(10)))}
#[derive(Deserialize)]struct ApplyInput { expected_version: Option<i64>,}async fn apply( State(state): State<AppState>, Path(id): Path<Uuid>, Json(input): Json<ApplyInput>,) -> Result<Json<Material>, ApiError> { let mut tx = state.platform_db.begin().await?; let p = sqlx::query_as::<_, Proposal>( "select * from reader_rewrite_proposals where id=$1 for update", ) .bind(id) .fetch_one(&mut *tx) .await?; if p.status == "applied" { let material = sqlx::query_as::<_, Material>("select * from reader_materials where id=$1") .bind(p.material_id) .fetch_one(&mut *tx) .await?; return Ok(Json(material)); } if p.status != "complete" { return Err(ApiError::Conflict( "Only a completed proposal can be applied.".into(), )); } validate_text(&p.replacement, 1_000_000, "Generated content")?; let lesson_id = p.metadata["lesson_id"] .as_str() .and_then(|s| Uuid::parse_str(s).ok()); let mut native = None; if let Some(lesson) = lesson_id { let data: Value = sqlx::query_scalar("select to_jsonb(l) from learning_lessons l where id=$1 for update") .bind(lesson) .fetch_one(&mut *tx) .await?; if data["version"].as_i64() != p.metadata["lesson_version"].as_i64() { return Err(ApiError::Conflict( "The lesson changed during generation. Regenerate against its current version." .into(), )); } native = Some(data); } let material = if let Some(material_id) = p.material_id { let current = sqlx::query_as::<_, Material>("select * from reader_materials where id=$1 for update") .bind(material_id) .fetch_one(&mut *tx) .await?; if p.base_version != Some(current.version) || input.expected_version.is_some_and(|v| v != current.version) { return Err(ApiError::Conflict("Material changed during generation. The original and proposal are preserved; regenerate after reloading.".into())); } let body = apply_patch(¤t.body, &p.selection, &p.replacement)?; reader::snapshot_revision(&mut tx, ¤t, "agent rewrite").await?; sqlx::query_as::<_,Material>("update reader_materials set body=$2,version=version+1,updated_at=now() where id=$1 returning *").bind(material_id).bind(body).fetch_one(&mut *tx).await? } else { let sources = p .metadata .get("sources") .cloned() .unwrap_or_else(|| json!([])); reader::validate_sources(&sources)?; sqlx::query_as::<_,Material>("insert into reader_materials(kind,title,body,sources) values('article',$1,$2,$3) returning *").bind(&p.title).bind(&p.replacement).bind(sources).fetch_one(&mut *tx).await? }; reader::index_article_tx(&mut tx, material.id, &material.body).await?; if let Some(data) = native { let lesson = lesson_id.expect("native lesson is loaded only with an ID"); sqlx::query("insert into agent_lesson_revisions(lesson_id,version,title,document) values($1,$2,$3,$4) on conflict(lesson_id,version) do nothing").bind(lesson).bind(data["version"].as_i64().unwrap_or(1)).bind(data["title"].as_str().unwrap_or_default()).bind(&data["document"]).execute(&mut *tx).await?; let mut document = data["document"].clone(); if !document.is_object() { return Err(ApiError::Conflict( "Native lesson document is malformed; original content was preserved.".into(), )); } document["reader_material_id"] = json!(material.id); sqlx::query("update learning_lessons set document=$2,title=$3,version=version+1,updated_at=now() where id=$1").bind(lesson).bind(document).bind(&material.title).execute(&mut *tx).await?; } else if let Some(space) = p.metadata["space_id"] .as_str() .and_then(|s| Uuid::parse_str(s).ok()) { sqlx::query("insert into learning_lessons(space_id,position,title,document,created_by) values($1,(select coalesce(max(position),-1)+1 from learning_lessons where space_id=$1),$2,$3,'agent')").bind(space).bind(&material.title).bind(json!({"schema_version":1,"content":{"type":"doc","content":[]},"reader_material_id":material.id})).execute(&mut *tx).await?; } let material = reader::sync_catalog_tx(&mut tx, &material).await?; sqlx::query("update reader_rewrite_proposals set material_id=$2,status='applied',updated_at=now() where id=$1").bind(p.id).bind(material.id).execute(&mut *tx).await?; tx.commit().await?; Ok(Json(material))}fn apply_patch(body: &str, selection: &Value, replacement: &str) -> Result<String, ApiError> { if selection.get("start").is_none() { return Ok(replacement.to_owned()); } let start = selection["start"].as_u64().unwrap_or(u64::MAX) as usize; let end = selection["end"].as_u64().unwrap_or(u64::MAX) as usize; if start > end || !body.is_char_boundary(start) || !body.is_char_boundary(end) || body.get(start..end) != selection["original"].as_str() { return Err(ApiError::Conflict( "Selected passage no longer matches the saved article.".into(), )); } Ok(format!("{}{}{}", &body[..start], replacement, &body[end..]))}
// Normalize whitespace while retaining original byte spans; rendered Markdown is resolved below.fn normalized_map(text: &str, base: usize) -> (String, Vec<(usize, usize)>) { let mut normalized = String::new(); let mut map = Vec::new(); let mut whitespace = false; for (i, c) in text.char_indices() { if c.is_whitespace() { if !whitespace { normalized.push(' '); map.push((base + i, base + i + c.len_utf8())); } else if let Some(last) = map.last_mut() { last.1 = base + i + c.len_utf8(); } whitespace = true; } else { normalized.push(c); for _ in 0..c.len_utf8() { map.push((base + i, base + i + c.len_utf8())); } whitespace = false; } } (normalized, map)}fn choose_match( text: &str, map: &[(usize, usize)], selection: &Selection,) -> Result<(usize, usize), ApiError> { let quote = normalized_map(&selection.quote, 0).0.trim().to_owned(); let prefix = normalized_map(&selection.prefix, 0).0; let suffix = normalized_map(&selection.suffix, 0).0; let all: Vec<(usize, usize)> = text .match_indices("e) .map(|(start, _)| (start, start + quote.len())) .collect(); let matches: Vec<(usize, usize)> = all .iter() .filter(|(start, end)| { all.len() == 1 || ((prefix.trim().is_empty() || text[..*start].trim_end().ends_with(prefix.trim_end())) && (suffix.trim().is_empty() || text[*end..].trim_start().starts_with(suffix.trim_start()))) }) .map(|(start, end)| (map[*start].0, map[*end - 1].1)) .collect(); match matches.as_slice(){[one]=>Ok(*one),[]=>Err(ApiError::BadRequest("The selected passage could not be located in the current Markdown. Select a complete paragraph or edit the source.".into())),_=>Err(ApiError::BadRequest("The selected passage occurs more than once. Include surrounding text to identify it uniquely.".into()))}}fn resolve_selection(body: &str, selection: &Selection) -> Result<(usize, usize), ApiError> { use pulldown_cmark::{Event as MdEvent, Parser, Tag}; let mut rendered = String::new(); let mut mapping = Vec::new(); let mut inline = Vec::new(); let mut stack = Vec::new(); for (event, range) in Parser::new(body).into_offset_iter() { match event { MdEvent::Start(tag) => { let is_inline = matches!( tag, Tag::Emphasis | Tag::Strong | Tag::Strikethrough | Tag::Link { .. } ); stack.push((is_inline, range.clone(), rendered.len())); } MdEvent::End(_) => { if let Some((is_inline, source, start)) = stack.pop() { if is_inline { inline.push((start, rendered.len(), source)); } else if !rendered.ends_with(' ') { rendered.push(' '); mapping.push((range.end, range.end)); } } } MdEvent::Text(text) | MdEvent::Code(text) => { // Pulldown may decode entities/escapes. Resolve only text with a precise source span. let raw = &body[range.clone()]; let Some(offset) = raw.find(text.as_ref()) else { continue; }; let (text, map) = normalized_map(&text, range.start + offset); rendered.push_str(&text); mapping.extend(map); } MdEvent::SoftBreak | MdEvent::HardBreak => { rendered.push(' '); mapping.push((range.start, range.end)); } _ => {} } } let range = choose_match(&rendered, &mapping, selection).or_else(|error| { let normalized_quote = normalized_map(&selection.quote, 0).0.trim().to_owned(); if rendered.contains(&normalized_quote) { return Err(error); } let (raw, map) = normalized_map(body, 0); choose_match(&raw, &map, selection) })?; let (mut start, mut end) = range; for (render_start, render_end, range) in inline { if render_start >= render_end { continue; } let visible_start = mapping[render_start].0; let visible_end = mapping[render_end - 1].1; if start <= visible_start && end >= visible_end { start = start.min(range.start); end = end.max(range.end); } else if (start > visible_start && start < visible_end && end > range.end) || (end < visible_end && end > visible_start && start < range.start) { return Err(ApiError::BadRequest("The selection crosses part of Markdown formatting. Select the complete formatted phrase or paragraph.".into())); } } Ok((start, end))}async fn native_revisions( State(state): State<AppState>, Path(id): Path<Uuid>,) -> Result<Json<Value>, ApiError> { let revisions:Value=sqlx::query_scalar("select coalesce(jsonb_agg(to_jsonb(r) order by version desc),'[]'::jsonb) from agent_lesson_revisions r where lesson_id=$1").bind(id).fetch_one(&state.platform_db).await?; Ok(Json(revisions))}async fn restore_native_revision( State(state): State<AppState>, Path((id, revision)): Path<(Uuid, Uuid)>, Json(input): Json<ApplyInput>,) -> Result<Json<Value>, ApiError> { let expected = input .expected_version .ok_or_else(|| ApiError::BadRequest("expected_version is required.".into()))?; let mut tx = state.platform_db.begin().await?; let current: Value = sqlx::query_scalar("select to_jsonb(l) from learning_lessons l where id=$1 for update") .bind(id) .fetch_one(&mut *tx) .await?; if current["version"].as_i64() != Some(expected) { return Err(ApiError::Conflict( "Lesson changed. Reload before restoring.".into(), )); } let saved: Value = sqlx::query_scalar( "select to_jsonb(r) from agent_lesson_revisions r where id=$1 and lesson_id=$2", ) .bind(revision) .bind(id) .fetch_one(&mut *tx) .await?; sqlx::query("insert into agent_lesson_revisions(lesson_id,version,title,document) values($1,$2,$3,$4) on conflict(lesson_id,version) do nothing").bind(id).bind(expected).bind(current["title"].as_str().unwrap_or_default()).bind(¤t["document"]).execute(&mut *tx).await?; let lesson:Value=sqlx::query_scalar("update learning_lessons l set title=$2,document=$3,version=version+1,updated_at=now() where id=$1 returning to_jsonb(l)").bind(id).bind(saved["title"].as_str().unwrap_or_default()).bind(&saved["document"]).fetch_one(&mut *tx).await?; tx.commit().await?; Ok(Json(lesson))}
#[cfg(test)]mod tests { use super::*; fn selection(quote: &str, prefix: &str) -> Selection { Selection { quote: quote.into(), prefix: prefix.into(), suffix: String::new(), } } #[test] fn repeated_quote_requires_a_unique_anchor() { assert!(resolve_selection("first: word\nsecond: word", &selection("word", "")).is_err()); let range = resolve_selection("first: word\nsecond: word", &selection("word", "second: ")).unwrap(); assert_eq!(range, (20, 24)); } #[test] fn unicode_and_formatted_selection_preserve_other_content() { let body = "## Урок\nTLB **хранит адреса** процессора.\nКонец."; let (start, end) = resolve_selection(body, &selection("TLB хранит адреса процессора.", "")).unwrap(); let changed = apply_patch( body, &json!({"start":start,"end":end,"original":&body[start..end]}), "Новый текст.", ) .unwrap(); assert_eq!(changed, "## Урок\nНовый текст.\nКонец."); } #[test] fn patch_rejects_stale_text_and_invalid_utf8_boundary() { assert!(apply_patch("язык", &json!({"start":1,"end":2,"original":"я"}), "a").is_err()); assert!( apply_patch( "current", &json!({"start":0,"end":3,"original":"old"}), "new" ) .is_err() ); } #[test] fn formatted_duplicate_is_still_ambiguous() { assert!( resolve_selection( "First foo bar. Second **foo** bar.", &selection("foo bar", "") ) .is_err() ); let body = "See [documentation](https://example.org) now."; let (start, end) = resolve_selection(body, &selection("See documentation now.", "")).unwrap(); assert_eq!(&body[start..end], body); } #[test] fn source_excerpts_remain_bounded_and_labeled() { let sources = json!([{"title":"A","url":"https://a.test","excerpt":"x".repeat(10000)},{"title":"B","url":"https://b.test","excerpt":"y".repeat(10000)}]); let reference = source_context(&sources, 1000); assert!(reference.chars().count() <= 1000); assert!(reference.contains("[s1]")); assert!(reference.contains("[s2]")); }}