diff --git a/src/config.rs b/src/config.rs --- a/src/config.rs +++ b/src/config.rs @@ -104,6 +104,13 @@ /// timeout for fetching a full repository CAR during backfill. /// set via `HYDRANT_REPO_FETCH_TIMEOUT` (humantime duration, e.g. `5min`). pub repo_fetch_timeout: Duration, + /// maximum size in bytes of a full-repository `getRepo` CAR response accepted during + /// backfill. the body is streamed and rejected before it is fully buffered once it would + /// exceed this ceiling, bounding per-task memory regardless of what a PDS returns (the same + /// ceiling also bounds sparse `getBlocks` fetches). legitimate repositories are far smaller; + /// lower this on memory-constrained hosts running high backfill concurrency. + /// set via `HYDRANT_MAX_CAR_BODY_BYTES`. + pub max_car_body_bytes: usize, /// maximum number of concurrent backfill tasks. /// set via `HYDRANT_BACKFILL_CONCURRENCY_LIMIT`. pub backfill_concurrency_limit: usize, @@ -315,6 +322,9 @@ firehose_max_failures: 15, cursor_save_interval: Duration::from_secs(3), repo_fetch_timeout: Duration::from_secs(300), + // 2 GiB safety ceiling: never rejects a legitimate repository, only a runaway or + // malicious response. memory-constrained/high-concurrency hosts should lower it. + max_car_body_bytes: 2 * 1024 * 1024 * 1024, backfill_concurrency_limit: 16, backfill_strategy: BackfillStrategy::Auto, backfill_proxies: vec![], @@ -436,6 +446,11 @@ f, "repo fetch timeout", format_args!("{}sec", self.repo_fetch_timeout.as_secs()) + )?; + config_line!( + f, + "max car body", + format_args!("{} mb", self.max_car_body_bytes / (1024 * 1024)) )?; config_line!(f, "ephemeral", self.ephemeral)?; config_line!(f, "database path", self.database_path.to_string_lossy())?; diff --git a/src/state.rs b/src/state.rs --- a/src/state.rs +++ b/src/state.rs @@ -44,6 +44,7 @@ pub ephemeral_ttl: Duration, pub only_index_links: bool, pub verify_cids: bool, + pub max_car_body_bytes: usize, pub get_repo_semaphore: Semaphore, } @@ -124,6 +125,7 @@ ephemeral_ttl: config.ephemeral_ttl, only_index_links: config.only_index_links, verify_cids: config.verify_cids, + max_car_body_bytes: config.max_car_body_bytes, get_repo_semaphore: Semaphore::new(config.get_repo_concurrency_limit), throttler: Throttler::new( config.backfill_proxies.len() + 1, diff --git a/src/backfill/client.rs b/src/backfill/client.rs --- a/src/backfill/client.rs +++ b/src/backfill/client.rs @@ -1,4 +1,5 @@ use crate::util::throttle::Throttler; +use bytes::{Bytes, BytesMut}; use jacquard_common::http_client::HttpClient; use reqwest::StatusCode; use std::sync::Arc; @@ -89,4 +90,32 @@ res } +} + +/// streams a response body into a single contiguous [`Bytes`], enforcing a hard byte ceiling. +/// +/// returns `Ok(None)` when the body exceeds `max_bytes`: the response is dropped without +/// buffering the remainder, so per-task memory stays bounded regardless of what the PDS sends +/// (a chunked response with no `Content-Length`, or a decompression bomb — `chunk` yields +/// already-decoded bytes, so the ceiling applies to the decompressed size). the buffer is grown +/// in place and never double-buffered: at most `buf + one chunk` is resident at any point. +pub(crate) async fn collect_body_bounded( + mut resp: reqwest::Response, + max_bytes: usize, +) -> Result, reqwest::Error> { + // reject early when a declared (uncompressed) length already exceeds the ceiling. reqwest + // strips `Content-Length` from decoded responses, so a present value is the real body size. + let content_length = resp.content_length(); + if content_length.is_some_and(|len| len > max_bytes as u64) { + return Ok(None); + } + let reserve = content_length.unwrap_or(0).min(max_bytes as u64) as usize; + let mut buf = BytesMut::with_capacity(reserve); + while let Some(chunk) = resp.chunk().await? { + if buf.len() + chunk.len() > max_bytes { + return Ok(None); + } + buf.extend_from_slice(&chunk); + } + Ok(Some(buf.freeze())) } diff --git a/src/backfill/sparse.rs b/src/backfill/sparse.rs --- a/src/backfill/sparse.rs +++ b/src/backfill/sparse.rs @@ -1,4 +1,4 @@ -use crate::backfill::client::ThrottledHttpClient; +use crate::backfill::client::{ThrottledHttpClient, collect_body_bounded}; use crate::backfill::error::BackfillError; use crate::config::{BackfillStrategy, RateTier}; use crate::db::types::{DbAction, DbRkey}; @@ -199,6 +199,7 @@ &throttle, &tier, app_state.verify_cids, + app_state.max_car_body_bytes, ) .await?; if missing.iter().all(|cid| !blocks.contains_key(cid)) { @@ -243,6 +244,7 @@ &throttle, &tier, app_state.verify_cids, + app_state.max_car_body_bytes, ) .await?, ); @@ -287,12 +289,24 @@ throttle: &ThrottleHandle, tier: &RateTier, verify_cids: bool, + max_body_bytes: usize, ) -> Result, BackfillError> { let mut out = BTreeMap::new(); let fetches = cids .chunks(SPARSE_GET_BLOCKS_CHUNK) - .map(|chunk| fetch_block_chunk(http, pds, did, chunk, throttle, tier, verify_cids)) + .map(|chunk| { + fetch_block_chunk( + http, + pds, + did, + chunk, + throttle, + tier, + verify_cids, + max_body_bytes, + ) + }) .collect::>(); let fetches = stream::iter(fetches).buffer_unordered(SPARSE_GET_BLOCKS_PARALLELISM); futures::pin_mut!(fetches); @@ -312,6 +326,7 @@ throttle: &ThrottleHandle, tier: &RateTier, verify_cids: bool, + max_body_bytes: usize, ) -> Result, BackfillError> { let mut url = pds .join("xrpc/com.atproto.sync.getBlocks") @@ -355,10 +370,14 @@ let status = resp.status(); if status.is_success() { - let body = resp - .bytes() + let Some(body) = collect_body_bounded(resp, max_body_bytes) .await - .map_err(|e| BackfillError::Transport(e.to_string().into()))?; + .map_err(|e| BackfillError::Transport(e.to_string().into()))? + else { + return Err(BackfillError::Generic(miette::miette!( + "getBlocks response for {did} exceeded max body size of {max_body_bytes} bytes" + ))); + }; let blocks = crate::car::parse_car_blocks(body).map_err(BackfillError::from)?; if verify_cids { crate::car::validate_block_cids(&blocks)?; @@ -660,6 +679,7 @@ &throttle, &RateTier::trusted(), false, + 64 * 1024 * 1024, ), ) .await @@ -708,6 +728,7 @@ pds: &url::Url, wanted: IpldCid, verify_cids: bool, + max_body_bytes: usize, ) -> Result, BackfillError> { let throttler = crate::util::throttle::Throttler::new(1, 10); let http = ThrottledHttpClient::new(vec![reqwest::Client::new()], throttler.clone()); @@ -721,6 +742,7 @@ &throttle, &RateTier::trusted(), verify_cids, + max_body_bytes, ) .await } @@ -729,7 +751,9 @@ async fn verify_cids_rejects_mismatched_get_blocks_car() { let pds = spawn_car_server(car_with_block(cid(1), b"forged").await).await; - let err = fetch_one(&pds, cid(1), true).await.unwrap_err(); + let err = fetch_one(&pds, cid(1), true, 64 * 1024 * 1024) + .await + .unwrap_err(); assert!(err.to_string().contains("CAR block CID mismatch")); } @@ -738,7 +762,9 @@ let real_cid = jacquard_repo::mst::util::compute_cid(b"trusted").unwrap(); let pds = spawn_car_server(car_with_block(real_cid, b"trusted").await).await; - let blocks = fetch_one(&pds, real_cid, true).await.unwrap(); + let blocks = fetch_one(&pds, real_cid, true, 64 * 1024 * 1024) + .await + .unwrap(); assert_eq!(blocks.get(&real_cid).unwrap().as_ref(), b"trusted"); } @@ -746,7 +772,17 @@ async fn verify_cids_disabled_accepts_mismatched_get_blocks_car() { let pds = spawn_car_server(car_with_block(cid(1), b"forged").await).await; - let blocks = fetch_one(&pds, cid(1), false).await.unwrap(); + let blocks = fetch_one(&pds, cid(1), false, 64 * 1024 * 1024) + .await + .unwrap(); assert_eq!(blocks.get(&cid(1)).unwrap().as_ref(), b"forged"); + } + + #[tokio::test] + async fn get_blocks_rejects_oversized_body() { + let pds = spawn_car_server(car_with_block(cid(1), &[0u8; 4096]).await).await; + + let err = fetch_one(&pds, cid(1), false, 64).await.unwrap_err(); + assert!(err.to_string().contains("exceeded max body size")); } } diff --git a/src/config/env.rs b/src/config/env.rs --- a/src/config/env.rs +++ b/src/config/env.rs @@ -119,6 +119,7 @@ let cursor_save_interval = cfg!("CURSOR_SAVE_INTERVAL", defaults.cursor_save_interval, sec); let repo_fetch_timeout = cfg!("REPO_FETCH_TIMEOUT", defaults.repo_fetch_timeout, sec); + let max_car_body_bytes = cfg!("MAX_CAR_BODY_BYTES", defaults.max_car_body_bytes); let ephemeral: bool = cfg!("EPHEMERAL", defaults.ephemeral); let ephemeral_ttl = cfg!("EPHEMERAL_TTL", defaults.ephemeral_ttl, sec); @@ -360,6 +361,7 @@ cursor_save_interval, enable_backfill, repo_fetch_timeout, + max_car_body_bytes, backfill_concurrency_limit, backfill_strategy, backfill_proxies, diff --git a/src/backfill/worker/process.rs b/src/backfill/worker/process.rs --- a/src/backfill/worker/process.rs +++ b/src/backfill/worker/process.rs @@ -2,23 +2,24 @@ use std::sync::Arc; use std::time::Instant; +use bytes::Bytes; use fjall::Slice; use miette::{IntoDiagnostic, Result}; +use reqwest::StatusCode; use smol_str::{SmolStr, ToSmolStr}; use tracing::{debug, error, trace, warn}; -use jacquard_api::com_atproto::sync::get_repo::{GetRepo, GetRepoError}; +use jacquard_api::com_atproto::sync::get_repo::GetRepoError; use jacquard_common::IntoStatic; use jacquard_common::types::cid::Cid as AtCid; use jacquard_common::types::did::Did; -use jacquard_common::xrpc::{XrpcError, XrpcExt}; use jacquard_repo::mst::Mst; use jacquard_repo::{BlockStore, MemoryBlockStore}; -use crate::backfill::client::ThrottledHttpClient; +use crate::backfill::client::{ThrottledHttpClient, collect_body_bounded}; use crate::backfill::error::BackfillError; use crate::backfill::sparse::{SparseBackfillResult, process_did_sparse}; -use crate::config::BackfillStrategy; +use crate::config::{BackfillStrategy, RateTier}; use crate::db::types::{DbAction, DbRkey}; use crate::db::{self, Txn as DbTxn, keys}; use crate::filter::FilterMode; @@ -26,7 +27,7 @@ use crate::sparse_mst::sparse_probe_collection; use crate::state::AppState; use crate::types::{Commit, GaugeState, RepoState, RepoStatus, ResyncState}; -use crate::util::url_to_fluent_uri; +use crate::util::{parse_retry_after, throttle::ThrottleHandle}; #[cfg(feature = "indexer_stream")] use crate::types::{AccountEvt, BroadcastEvent}; @@ -194,106 +195,78 @@ // 2. fetch repo (car) let start = Instant::now(); - let req = GetRepo::new().did(did.clone()).build(); let throttle = app_state.throttler.get_handle(&pds).await; - if throttle.is_throttled() { - return Err(BackfillError::PreemptivelyThrottled); - } let tier = app_state.resolve_pds_tier(pds.host_str().unwrap_or("")); - let resp = { - let _permit = throttle.acquire().await; - if throttle.is_throttled() { - return Err(BackfillError::PreemptivelyThrottled); - } - throttle.wait_for_allow(1, &tier).await; - if throttle.is_throttled() { - return Err(BackfillError::PreemptivelyThrottled); - } - match http.xrpc(url_to_fluent_uri(&pds)).send(&req).await { - Ok(resp) => { - if !throttle.is_throttled() { - throttle.record_success(); + let car_bytes = match fetch_full_repo_car( + http, + &pds, + did, + &throttle, + &tier, + app_state.max_car_body_bytes, + ) + .await? + { + FullRepoOutcome::Car(body) => body, + FullRepoOutcome::NotFound => { + warn!("repo not found, deleting"); + let mut txn = DbTxn::new(db); + let applied = + txn.transition_pending_key(did, pending_key.as_ref(), GaugeState::Synced)?; + txn.batch.remove(&db.indexer.pending, pending_key.clone()); + if applied { + if let Err(e) = crate::ops::delete_repo(&mut txn.batch, db, did, &state) { + error!(err = %e, "failed to wipe repo during backfill"); } - resp } - Err(e) => return Err(BackfillError::from_sparse_client(e, &throttle)), + txn.commit()?; + // return None so did_task skips sending BackfillFinished (nothing to drain for a deleted repo) + return Ok(None); } - }; + FullRepoOutcome::Inactive(status) => { + warn!(?status, "repo is inactive, stopping backfill"); - let car_bytes = match resp.into_output() { - Ok(o) => o, - Err(XrpcError::Xrpc(e)) => { - if matches!(e, GetRepoError::RepoNotFound(_)) { - warn!("repo not found, deleting"); - let mut txn = DbTxn::new(db); - let applied = - txn.transition_pending_key(did, pending_key.as_ref(), GaugeState::Synced)?; - txn.batch.remove(&db.indexer.pending, pending_key.clone()); - if applied { - if let Err(e) = crate::ops::delete_repo(&mut txn.batch, db, did, &state) { - error!(err = %e, "failed to wipe repo during backfill"); - } - } - txn.commit()?; - // return None so did_task skips sending BackfillFinished (nothing to drain for a deleted repo) - return Ok(None); - } + #[cfg(feature = "indexer_stream")] + emit_identity(&status, false); - let inactive_status = match e { - GetRepoError::RepoDeactivated(_) => Some(RepoStatus::Deactivated), - GetRepoError::RepoTakendown(_) => Some(RepoStatus::Takendown), - GetRepoError::RepoSuspended(_) => Some(RepoStatus::Suspended), - _ => None, + let resync_state = ResyncState::Gone { + status: status.clone(), }; + let resync_bytes = rmp_serde::to_vec(&resync_state).into_diagnostic()?; - if let Some(status) = inactive_status { - warn!(?status, "repo is inactive, stopping backfill"); - - #[cfg(feature = "indexer_stream")] - emit_identity(&status, false); - - let resync_state = ResyncState::Gone { - status: status.clone(), - }; - let resync_bytes = rmp_serde::to_vec(&resync_state).into_diagnostic()?; - - let app_state_clone = app_state.clone(); - let did = did.clone(); - let pending_key = pending_key.clone(); - app_state_clone - .db - .run(move |db| { - let mut txn = DbTxn::new(db); - let applied = txn.transition_pending_key( + let app_state_clone = app_state.clone(); + let did = did.clone(); + let pending_key = pending_key.clone(); + app_state_clone + .db + .run(move |db| { + let mut txn = DbTxn::new(db); + let applied = txn.transition_pending_key( + &did, + pending_key.as_ref(), + GaugeState::Resync(None), + )?; + txn.batch.remove(&db.indexer.pending, pending_key.clone()); + if applied { + crate::db::Db::update_repo_state( + &mut txn.batch, + &db.repos, &did, - pending_key.as_ref(), - GaugeState::Resync(None), + move |state, (key, batch)| { + state.active = false; + state.status = status; + batch.insert(&db.indexer.resync, key, resync_bytes); + Ok((true, ())) + }, )?; - txn.batch.remove(&db.indexer.pending, pending_key.clone()); - if applied { - crate::db::Db::update_repo_state( - &mut txn.batch, - &db.repos, - &did, - move |state, (key, batch)| { - state.active = false; - state.status = status; - batch.insert(&db.indexer.resync, key, resync_bytes); - Ok((true, ())) - }, - )?; - } - txn.commit()?; - Ok::<_, miette::Report>(()) - }) - .await?; + } + txn.commit()?; + Ok::<_, miette::Report>(()) + }) + .await?; - return Ok(None); - } - - Err(e).into_diagnostic()? + return Ok(None); } - Err(e) => Err(e).into_diagnostic()?, }; // emit identity event so any consumers know, but only if something changed @@ -306,19 +279,14 @@ } trace!( - bytes = car_bytes.body.len(), + bytes = car_bytes.len(), elapsed = ?start.elapsed(), "fetched car bytes" ); - // TODO: enforce a max_car_body_bytes limit here before parsing to prevent a malicious - // or compromised PDS from returning an unbounded response and causing OOM. - // also apply to the sparse path in sparse.rs. - // (finding d87fbd22402c81919fceb9cdfee53cd4) - // 3. import repo let start = Instant::now(); - let parsed = crate::car::parse_car(car_bytes.body.into())?; + let parsed = crate::car::parse_car(car_bytes)?; trace!(elapsed = %start.elapsed().as_secs_f32(), "parsed car"); let start = Instant::now(); @@ -554,4 +522,207 @@ trace!("complete"); Ok(Some(previous_state)) +} + +/// upper bound on a buffered xrpc error body. error responses are tiny json objects; this only +/// guards against a peer streaming an unbounded body on the error path. +const ERROR_BODY_MAX_BYTES: usize = 64 * 1024; + +/// outcome of a full-repository `getRepo` fetch that the caller must act on. transport and +/// rate-limit failures are surfaced as [`BackfillError`] instead. +#[derive(Debug)] +enum FullRepoOutcome { + /// the streamed CAR body, within the configured size ceiling. + Car(Bytes), + /// the PDS reported that the repository does not exist (`RepoNotFound`). + NotFound, + /// the PDS reported the repository is inactive; carries the status to record. + Inactive(RepoStatus), +} + +/// fetches a full repository CAR by streaming `com.atproto.sync.getRepo` directly through the +/// backfill client, enforcing `max_body_bytes` as chunks arrive so a runaway or malicious +/// response cannot exhaust memory. this bypasses the typed xrpc layer (which materializes the +/// entire decompressed body as one `Vec` before the caller sees it) while preserving the +/// same handling: `RepoNotFound`/`RepoDeactivated`/`RepoTakendown`/`RepoSuspended` are decoded +/// from the xrpc error body, and 429 responses feed the throttler from the rate-limit headers. +async fn fetch_full_repo_car( + http: &ThrottledHttpClient, + pds: &url::Url, + did: &Did<'_>, + throttle: &ThrottleHandle, + tier: &RateTier, + max_body_bytes: usize, +) -> Result { + let mut url = pds + .join("xrpc/com.atproto.sync.getRepo") + .map_err(|e| BackfillError::Generic(miette::miette!("invalid PDS URL: {e}")))?; + url.query_pairs_mut().append_pair("did", did.as_str()); + + // hold the per-pds permit for the entire download, not just until response headers: full + // repo bodies are large, so releasing early would let unbounded concurrent downloads run + // against a single pds and defeat per-pds concurrency (the original xrpc getRepo held it + // for the whole download, since its transport read the full body before returning). + let _permit = throttle.acquire().await; + if throttle.is_throttled() { + return Err(BackfillError::PreemptivelyThrottled); + } + throttle.wait_for_allow(1, tier).await; + if throttle.is_throttled() { + return Err(BackfillError::PreemptivelyThrottled); + } + let resp = match http + .get(url) + .header(reqwest::header::ACCEPT, "application/vnd.ipld.car") + .send() + .await + { + Ok(resp) => { + if !throttle.is_throttled() { + throttle.record_success(); + } + resp + } + Err(e) => { + let reason = e.to_string(); + if let Some(secs) = throttle.record_failure_detail("transport", reason.clone()) { + warn!(%pds, reason, "PDS offline, blacklisting for {secs}s"); + } + return Err(BackfillError::Transport(reason.into())); + } + }; + + let status = resp.status(); + if status.is_success() { + return match collect_body_bounded(resp, max_body_bytes) + .await + .map_err(|e| BackfillError::Transport(e.to_string().into()))? + { + Some(body) => Ok(FullRepoOutcome::Car(body)), + None => Err(BackfillError::Generic(miette::miette!( + "getRepo response for {did} exceeded max body size of {max_body_bytes} bytes" + ))), + }; + } + + // non-success: decode the (tiny) xrpc error body to preserve typed error handling. + let retry_after = (status == StatusCode::TOO_MANY_REQUESTS) + .then(|| parse_retry_after(&resp)) + .flatten(); + let body = collect_body_bounded(resp, ERROR_BODY_MAX_BYTES) + .await + .map_err(|e| BackfillError::Transport(e.to_string().into()))? + .unwrap_or_default(); + + if let Ok(err) = serde_json::from_slice::(&body) { + match err { + GetRepoError::RepoNotFound(_) => return Ok(FullRepoOutcome::NotFound), + GetRepoError::RepoDeactivated(_) => { + return Ok(FullRepoOutcome::Inactive(RepoStatus::Deactivated)); + } + GetRepoError::RepoTakendown(_) => { + return Ok(FullRepoOutcome::Inactive(RepoStatus::Takendown)); + } + GetRepoError::RepoSuspended(_) => { + return Ok(FullRepoOutcome::Inactive(RepoStatus::Suspended)); + } + _ => {} + } + } + + if status == StatusCode::TOO_MANY_REQUESTS { + throttle.record_ratelimit(retry_after); + return Err(BackfillError::Ratelimited); + } + + Err(BackfillError::Generic(miette::miette!( + "getRepo failed with HTTP {status}: {}", + String::from_utf8_lossy(&body) + ))) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::Router; + use axum::routing::get; + + /// spawns a server answering `com.atproto.sync.getRepo` with a fixed status and body. + async fn spawn_get_repo(status: StatusCode, body: Vec) -> url::Url { + let app = Router::new().route( + "/xrpc/com.atproto.sync.getRepo", + get(move || { + let body = body.clone(); + async move { (status, body) } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = url::Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + url + } + + async fn fetch( + pds: &url::Url, + max_body_bytes: usize, + ) -> Result { + let throttler = crate::util::throttle::Throttler::new(1, 10); + let http = ThrottledHttpClient::new(vec![reqwest::Client::new()], throttler.clone()); + let throttle = throttler.get_handle(pds).await; + let did = Did::new_static("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa").unwrap(); + fetch_full_repo_car( + &http, + pds, + &did, + &throttle, + &RateTier::trusted(), + max_body_bytes, + ) + .await + } + + #[tokio::test] + async fn full_get_repo_streams_body_within_ceiling() { + let pds = spawn_get_repo(StatusCode::OK, b"car-bytes".to_vec()).await; + match fetch(&pds, 1024).await.unwrap() { + FullRepoOutcome::Car(body) => assert_eq!(body.as_ref(), b"car-bytes"), + other => panic!("expected Car, got {other:?}"), + } + } + + #[tokio::test] + async fn full_get_repo_rejects_oversized_body() { + let pds = spawn_get_repo(StatusCode::OK, vec![0u8; 4096]).await; + let err = fetch(&pds, 64).await.unwrap_err(); + assert!( + err.to_string().contains("exceeded max body size"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn full_get_repo_maps_repo_not_found() { + let pds = spawn_get_repo( + StatusCode::BAD_REQUEST, + br#"{"error":"RepoNotFound","message":"nope"}"#.to_vec(), + ) + .await; + assert!(matches!( + fetch(&pds, 1024).await.unwrap(), + FullRepoOutcome::NotFound + )); + } + + #[tokio::test] + async fn full_get_repo_maps_inactive_status() { + let pds = spawn_get_repo( + StatusCode::BAD_REQUEST, + br#"{"error":"RepoDeactivated"}"#.to_vec(), + ) + .await; + match fetch(&pds, 1024).await.unwrap() { + FullRepoOutcome::Inactive(status) => assert_eq!(status, RepoStatus::Deactivated), + other => panic!("expected Inactive, got {other:?}"), + } + } }