diff --git a/packages/core/test/private-sync.test.mjs b/packages/core/test/private-sync.test.mjs --- a/packages/core/test/private-sync.test.mjs +++ b/packages/core/test/private-sync.test.mjs @@ -347,6 +347,67 @@ assert.deepEqual(await left.bus.catchUp(SPACE, { summaries: [], wants: [] }), []) }) + it('drops an established link that stops answering catch-up frames', async () => { + const failures = [] + let closed = 0 + const link = { + id: 'hung-peer', + request: () => new Promise(() => {}), + send: async () => {}, + close: () => { closed += 1 }, + } + const bus = new WirePrivateBus({ + spaceUri: SPACE, + envelopes: storeOf([]), + authorizePeer: allowPeer, + frameTimeoutMs: 20, + peers: { + links: async () => [link], + failed: (id, error) => { + failures.push([id, error]) + link.close() + }, + }, + }) + + await bus.join(SPACE) + const startedAt = Date.now() + assert.deepEqual(await bus.catchUp(SPACE, { summaries: [], wants: [] }), []) + assert.ok(Date.now() - startedAt < 1_000, 'catch-up waited for the transport’s failure timeout') + assert.equal(closed, 1) + assert.equal(failures.length, 1) + assert.equal(failures[0][0], 'hung-peer') + assert.match(failures[0][1].message, /did not answer a catch-up frame within 20ms/) + assert.match(bus.status()[0].lastError, /did not answer a catch-up frame within 20ms/) + }) + + it('stops waiting for a peer that accepts gossip but never acknowledges it', async () => { + const [envelope] = await corpus(1) + let failed = 0 + const link = { + id: 'hung-peer', + request: async () => { throw new Error('unused') }, + send: () => new Promise(() => {}), + } + const bus = new WirePrivateBus({ + spaceUri: SPACE, + envelopes: storeOf([]), + authorizePeer: allowPeer, + frameTimeoutMs: 20, + peers: { + links: async () => [link], + failed: () => { failed += 1 }, + }, + }) + + const startedAt = Date.now() + await bus.publish(SPACE, envelope) + assert.ok(Date.now() - startedAt < 1_000, 'publish waited for the transport’s failure timeout') + assert.equal(failed, 1) + assert.equal(bus.status()[0].delivered, 0) + assert.match(bus.status()[0].lastError, /did not answer a gossip frame within 20ms/) + }) + it('recovers what a partition dropped at the next catch-up', async () => { const [envelope] = await corpus(1) const { left, right, offline } = await pair([], []) diff --git a/packages/core/src/private/sync.ts b/packages/core/src/private/sync.ts --- a/packages/core/src/private/sync.ts +++ b/packages/core/src/private/sync.ts @@ -290,11 +290,24 @@ * the same bounded prefix. */ maxRounds?: number + /** + * Maximum time one peer may hold a frame exchange open. + * + * Dialling has its own bound in `PeerConnections`; this is the matching bound after a connection + * has been established. Without it, a peer that disappears while retaining a QUIC connection can + * leave `request()` waiting for the transport's minutes-long failure detection, which in turn + * stalls the replica sync cycle containing the exchange. + */ + frameTimeoutMs?: number } const DEFAULT_MAX_ROUNDS = 32 +/** A dead established link gets no longer than two ordinary daemon poll intervals to answer. */ +export const DEFAULT_FRAME_TIMEOUT_MS = 10_000 /** A bootstrap cursor is tiny, but endpoint ids are still a peer-controlled namespace. */ const MAX_BOOTSTRAP_CURSORS = 64 + +class FrameTimeout extends Error {} /** * A `PrivateBus` over frames. @@ -313,6 +326,7 @@ readonly #peers: PeerSource readonly #authorizePeer: (endpointId: string) => PeerAuthorization readonly #maxRounds: number + readonly #frameTimeoutMs: number readonly #endpointId: string readonly #blobs: Pick | undefined readonly #blobLimits: BlobLimits @@ -333,6 +347,7 @@ this.#peers = options.peers this.#authorizePeer = options.authorizePeer this.#maxRounds = options.maxRounds ?? DEFAULT_MAX_ROUNDS + this.#frameTimeoutMs = Math.max(1, options.frameTimeoutMs ?? DEFAULT_FRAME_TIMEOUT_MS) this.#endpointId = options.endpointId ?? '' this.#blobs = options.blobs this.#blobLimits = options.blobLimits ?? DEFAULT_BLOB_LIMITS @@ -398,22 +413,44 @@ this.#peers.failed?.(status.id, error) } + /** + * Bound an exchange on a link that was reachable when the round began. + * + * The losing operation remains observed by `Promise.race`, and `#failLink` closes the managed + * connection as soon as this timeout is caught by the caller. That makes the transport operation + * reject too, instead of leaving an unhandled promise or a live stream behind the completed sync. + */ + async #withinFrame(link: PeerLink, kind: PrivateFrame['kind'], operation: Promise): Promise { + let timer: ReturnType | undefined + const expiry = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new FrameTimeout(`peer ${link.id} did not answer a ${kind} frame within ${this.#frameTimeoutMs}ms`)), + this.#frameTimeoutMs, + ) + }) + try { + return await Promise.race([operation, expiry]) + } finally { + if (timer) clearTimeout(timer) + } + } + async publish(spaceUri: string, envelope: PrivateEnvelope): Promise { this.#assertSpace(spaceUri) const topic = await this.#topicFor(spaceUri) const frame: PrivateFrame = { kind: 'gossip', version: WIRE_VERSION, topic, envelope } - for (const link of await this.#links(spaceUri)) { + await Promise.all((await this.#links(spaceUri)).map(async (link) => { // A retained bootstrap route is not authorization. Until its inventory-free scan ends it gets // only the catch-up requests that advance that scan, never newly written private content. - if (this.#bootstrapCursors.has(link.id)) continue + if (this.#bootstrapCursors.has(link.id)) return const status = this.#peerStatus(link.id) try { - await link.send(frame) + await this.#withinFrame(link, frame.kind, link.send(frame)) status.delivered += 1 } catch (error) { this.#failLink(status, error) } - } + })) } subscribe(spaceUri: string, onEnvelope: (envelope: PrivateEnvelope) => void): () => void { @@ -449,14 +486,15 @@ for (let round = 0; round < this.#maxRounds; round += 1) { let answer: PrivateFrame try { - answer = await link.request({ + const frame: PrivateFrame = { kind: 'catch-up', version: WIRE_VERSION, topic, summaries, wants, ...(cursor !== undefined ? { cursor } : {}), - }) + } + answer = await this.#withinFrame(link, frame.kind, link.request(frame)) } catch (error) { this.#failLink(status, error) break @@ -543,13 +581,14 @@ for (let round = 0; round < rounds; round += 1) { let answer: PrivateFrame try { - answer = await link.request({ + const frame: PrivateFrame = { kind: 'blob-request', version: WIRE_VERSION, topic, cid: need.cid, ...(offset > 0 ? { offset } : {}), - }) + } + answer = await this.#withinFrame(link, frame.kind, link.request(frame)) } catch (error) { this.#failLink(status, error) return undefined