Open History to see the last commit
1/**
2 * Durable-run wiring for the sandbox-web example — the journal-only tier of
3 * durable runs (see docs/sandbox/durable-runs), in one module the routes share:
4 *
5 * - one `memoryPersistence()` so the run record, the transcript, and the
6 * delivery log all live in THIS process (zero infra — which also means a dev
7 * server restart forgets everything; that is the tier's documented trade);
8 * - `buildRunStream`, the ONE chat() assembly both the fresh-run POST and the
9 * takeover GET use, so a replayed run re-derives the exact same stream;
10 * - `driveRun`, the takeover drive (`durability.attach: true`) shared by the
11 * attach route and the reaper;
12 * - `ensureReaper`, the scheduled sweep durable runs require (a detached run
13 * nobody rejoins must not keep its sandbox alive forever).
14 *
15 * The whole module is small because the stack is FIXED (Claude Code on Docker):
16 * takeover requires rebuilding a run's chat() from its `runId` alone, and with
17 * one adapter and one provider the rebuild is just `threadId` → sandbox +
18 * transcript. Make the stack a per-request browser choice and every route that
19 * arrives with only a `runId` needs that choice stored server-side — which is
20 * most of the wiring this example used to carry.
21 *
22 * The `InMemoryLockStore` is fine here ONLY because this example is one
23 * process. On real replicas it cannot stop two hosts driving one run — use a
24 * distributed LockStore (see docs/sandbox/takeover, "Requirements").
25 */
26import { chat, memoryStream } from '@tanstack/ai'
27import { resolveDebugOption } from '@tanstack/ai/adapter-internals'
28import { InMemoryLockStore, withLocks } from '@tanstack/ai/locks'
29import { memoryPersistence, withPersistence } from '@tanstack/ai-persistence'
30import {
31 InMemorySandboxInstanceStore,
32 JournalAttachUnavailableError,
33 RunClaimNotAcquiredError,
34 probeRunExit,
35 reapDetachedRuns,
36 sandboxRunDriver,
37 withSandbox,
38} from '@tanstack/ai-sandbox'
39import {
40 PREVIEW_GUIDANCE,
41 buildAdapter,
42 buildSandbox,
43 makeExposePreviewTool,
44 tanstackStartRecipe,
45} from './sandbox-agent'
46import { isTerminalRunStatus } from '@tanstack/ai'
47import type {
48 ModelMessage,
49 RunDriverOptions,
50 RunRecord,
51 RunStore,
52 StreamChunk,
53 StreamDurability,
54} from '@tanstack/ai'
55import type { LockStore } from '@tanstack/ai/locks'
56import type { RunExitProbe } from '@tanstack/ai-sandbox'
57
58export const persistence = memoryPersistence()
59export const { runs, messages: messageStore } = persistence.stores
60export const locks = new InMemoryLockStore()
61
62/**
63 * ONE instance store, shared by every route, drive, and probe. This is what
64 * gives the sandbox a cross-request identity: `buildSandbox()` mints a fresh
65 * definition per call, and a definition's fallback bookkeeping is its own —
66 * without a shared store, the takeover GET and every reaper probe `ensure` a
67 * brand-new container instead of resuming the one the POST created (and the
68 * takeover then waits on the empty journal of a sandbox nobody started).
69 */
70export const instances = new InMemorySandboxInstanceStore()
71
72/**
73 * Errors-only console logger for the run driver and the reaper. Both are
74 * total-by-construction (a drive failure is logged, never thrown to the
75 * response), so WITHOUT a logger a failed takeover is completely invisible —
76 * the log stays open, the record stays `running`, and nothing prints.
77 */
78export const runLogger = resolveDebugOption(undefined)
79
80/**
81 * Runs THIS process is currently driving, for the cancel endpoint's fast path.
82 * Only ever a fast path: the durable band (`requestRunCancel`) is what reaches
83 * a run nobody is driving right now.
84 */
85export const driving = new Map<string, AbortController>()
86
87/**
88 * Mirror a claim/request signal onto a fresh AbortController for chat(), so a
89 * lost claim (or a reaper budget) actually stops the drive.
90 */
91export function controllerFor(signal: AbortSignal): AbortController {
92 const controller = new AbortController()
93 const abort = (): void => controller.abort(signal.reason)
94 if (signal.aborted) abort()
95 else signal.addEventListener('abort', abort, { once: true })
96 return controller
97}
98
99/**
100 * The one chat() assembly. `attach: false` starts the agent; `attach: true`
101 * tails the run's existing journal instead of starting a second one. Same
102 * sandbox, adapter, prompts, and tools either way — determinism is what lets a
103 * takeover's replay align against the already-delivered log.
104 */
105export function buildRunStream(input: {
106 threadId: string
107 messages: Array<ModelMessage>
108 runId: string
109 abortController: AbortController
110 attach: boolean
111 durability: StreamDurability
112 /** Claude Code session to resume for a follow-up turn (fresh runs only). */
113 sessionId?: string
114}): AsyncIterable<StreamChunk> {
115 const { threadId, runId, abortController, attach, durability, sessionId } =
116 input
117 const sandbox = buildSandbox(threadId)
118
119 return chat({
120 threadId,
121 // Forwarded, never generated here: the journal path, the deterministic
122 // message ids, and the delivery-log stream name all derive from it.
123 runId,
124 adapter: buildAdapter(),
125 messages: input.messages,
126 systemPrompts: [PREVIEW_GUIDANCE],
127 tools: [
128 tanstackStartRecipe,
129 makeExposePreviewTool(sandbox, threadId, { store: instances, locks }),
130 ],
131 ...(sessionId !== undefined ? { modelOptions: { sessionId } } : {}),
132 abortController,
133 middleware: [
134 withPersistence(persistence),
135 withLocks(locks),
136 withSandbox(sandbox, {
137 runs,
138 instances,
139 durability: {
140 adapter: durability,
141 ...(attach ? { attach: true } : {}),
142 },
143 }),
144 ],
145 }) as AsyncIterable<StreamChunk>
146}
147
148/**
149 * The takeover drive, shaped for `sandboxRunDriver` and `reapDetachedRuns`:
150 * rebuild the run's chat() from the persisted transcript, with `attach: true`.
151 */
152export async function* driveRun(input: {
153 runId: string
154 threadId: string
155 signal: AbortSignal
156}): AsyncIterable<StreamChunk> {
157 const controller = controllerFor(input.signal)
158 driving.set(input.runId, controller)
159 console.log(`[driveRun] claimed ${input.runId}; starting attach drive`)
160 try {
161 // The client sent no history — it is reconnecting, not asking a question.
162 const messages = await messageStore.loadThread(input.threadId)
163 let restart = false
164 try {
165 const attachStream = buildRunStream({
166 threadId: input.threadId,
167 messages,
168 runId: input.runId,
169 abortController: controller,
170 attach: true,
171 durability: memoryStream({ runId: input.runId }),
172 })
173 for await (const chunk of attachStream) {
174 // chat() reports a failed attach as an in-stream RUN_ERROR chunk (the
175 // detached run has no caller to throw to). Intercept it BEFORE
176 // yielding — a yielded chunk would be appended to the log and
177 // delivered, turning a recoverable condition into a terminal error.
178 if (isAttachUnavailableChunk(chunk)) {
179 restart = true
180 break
181 }
182 yield chunk
183 }
184 if (!restart) return
185 } catch (error) {
186 // The thrown shape of the same condition. Matched by `name`/`reason`
187 // rather than `instanceof`: under vite dev the example's import of
188 // `@tanstack/ai-sandbox` and the adapter's own can be two module
189 // instances, and a cross-instance `instanceof` is always false.
190 if (!isAttachUnavailableError(error)) throw error
191 restart = true
192 }
193 // The claimed run has NO journal (or an empty one): its agent never wrote
194 // a byte. The routine cause is a refresh during sandbox boot — the
195 // original drive unwinds mid-setup, before the agent is even spawned, so
196 // there is nothing to attach to and never will be. Nothing beyond the
197 // run-accepted marker was delivered or stored, so restarting the run
198 // FRESH (attach: false — start the agent) cannot duplicate anything the
199 // client saw. Without this the run is a zombie: record `running`, log
200 // open at the marker, every joiner parked until the reaper's TTL.
201 console.warn(
202 `[run-durable] attach found no journal for run ${input.runId} — its agent never started; restarting the run fresh`,
203 )
204 yield* buildRunStream({
205 threadId: input.threadId,
206 messages,
207 runId: input.runId,
208 abortController: controller,
209 attach: false,
210 durability: memoryStream({ runId: input.runId }),
211 })
212 } finally {
213 driving.delete(input.runId)
214 }
215}
216
217/** The two waited-then-gave-up preflight outcomes a fresh restart can fix. */
218function isAttachUnavailableError(error: unknown): boolean {
219 return (
220 error instanceof Error &&
221 error.name === JournalAttachUnavailableError.name &&
222 'reason' in error &&
223 (error.reason === 'journal-timeout' || error.reason === 'journal-stalled')
224 )
225}
226
227/**
228 * The in-stream shape of `JournalAttachUnavailableError`'s recoverable cases.
229 * The chunk carries no machine code (the error class exposes `reason`, which
230 * `toRunErrorPayload` does not map), so this matches the class's stable
231 * message prefix + the two waited-then-gave-up reasons' wording.
232 */
233function isAttachUnavailableChunk(chunk: StreamChunk): boolean {
234 if (chunk.type !== 'RUN_ERROR') return false
235 const message = 'message' in chunk ? chunk.message : undefined
236 return (
237 typeof message === 'string' &&
238 message.startsWith('cannot attach to run') &&
239 /journal/i.test(message)
240 )
241}
242
243/**
244 * Whether a detached run's agent already finished, by probing its journal tail
245 * for the exit sentinel. `ensure` resumes the thread's sandbox (`reuse:
246 * 'thread'`); if it had to CREATE one the probe finds no journal and answers
247 * `producing` — the fail-safe direction, and the TTL still bounds the run.
248 */
249async function hasFinished(record: RunRecord): Promise<RunExitProbe> {
250 try {
251 const handle = await buildSandbox(record.threadId).ensure({
252 threadId: record.threadId,
253 runId: 'run',
254 // The SHARED bookkeeping, so this resumes the run's real sandbox instead
255 // of creating a fresh (journal-less) one per probe.
256 store: instances,
257 locks,
258 })
259 return await probeRunExit({ handle, runId: record.runId })
260 } catch (error) {
261 return { state: 'unknown', error }
262 }
263}
264
265/** Tear the reclaimed run's sandbox down through the same definition that built it. */
266function reclaim(record: RunRecord): Promise<void> {
267 return buildSandbox(record.threadId).destroy({
268 threadId: record.threadId,
269 runId: 'run',
270 store: instances,
271 locks,
272 })
273}
274
275const CLAIM_RETRY_MS = 3_000
276/** ~2 minutes of retries — spans a slow sandbox boot with room to spare. */
277const CLAIM_RETRY_LIMIT = 40
278
279/**
280 * The takeover driver for the attach GET: `sandboxRunDriver` with a RETRYING
281 * claim.
282 *
283 * Core's driver tries the claim exactly once and — correctly — declines to
284 * drive when another host holds it. But on a refresh during sandbox boot the
285 * holder is the run's ORIGINAL drive, which only observes its own disconnect
286 * once the boot completes; the client has already attached to the log by then,
287 * so nothing ever issues another attach and the run is stranded. Retrying the
288 * claim (bounded, and abandoned the moment the run settles or a cancel is
289 * recorded) bridges that window: when the original drive finally detaches and
290 * releases, this driver claims, finds no journal, and `driveRun` restarts the
291 * agent.
292 */
293export function takeoverDriver(request: Request): RunDriverOptions {
294 const base = sandboxRunDriver({
295 request,
296 runs,
297 locks,
298 durability: () => memoryStream(request),
299 drive: driveRun,
300 // The driver is total: a failed drive is logged, never thrown to the
301 // (already streaming) response. Without a logger the failure is invisible.
302 logger: runLogger,
303 })
304 return {
305 ...base,
306 claim: async <T>(
307 input: { runs: RunStore; locks: LockStore; runId: string },
308 fn: (claim: {
309 runId: string
310 epoch: number
311 signal: AbortSignal
312 }) => Promise<T>,
313 ): Promise<T> => {
314 for (let attempt = 0; ; attempt += 1) {
315 try {
316 return await base.claim(input, fn)
317 } catch (error) {
318 if (
319 !(error instanceof RunClaimNotAcquiredError) ||
320 attempt >= CLAIM_RETRY_LIMIT
321 ) {
322 throw error
323 }
324 console.log(
325 `[takeover] claim for ${input.runId} refused (attempt ${attempt + 1}); retrying — the original drive likely still holds it`,
326 )
327 await new Promise((resolve) => setTimeout(resolve, CLAIM_RETRY_MS))
328 // Re-check between attempts: retrying past a settle would be noise,
329 // and retrying past a CANCEL would resurrect a run the user stopped.
330 const record = await runs.get(input.runId)
331 if (
332 record === null ||
333 isTerminalRunStatus(record.status) ||
334 record.cancelRequested === true
335 ) {
336 throw error
337 }
338 }
339 }
340 },
341 }
342}
343
344const SWEEP_INTERVAL_MS = 60_000
345
346let reaperStarted = false
347let sweepInFlight = false
348
349/**
350 * Schedule the detached-run sweep — the second, easy-to-forget half of the
351 * durable-runs opt-in (see docs/sandbox/reaping). Guarded so overlapping ticks
352 * never race for the same claims. Call it from the run routes; the first call
353 * arms it for the life of the dev server.
354 */
355export function ensureReaper(): void {
356 if (reaperStarted) return
357 reaperStarted = true
358 setInterval(() => {
359 if (sweepInFlight) return
360 sweepInFlight = true
361 void reapDetachedRuns({
362 runs,
363 locks,
364 durability: (runId) => memoryStream({ runId }),
365 hasFinished,
366 drive: driveRun,
367 logger: runLogger,
368 now: Date.now(),
369 // Short for a demo: a run nobody rejoins is finalized (or expired and its
370 // sandbox destroyed) within ~5 minutes of its last viewer leaving.
371 detachedRunTtlMs: 5 * 60_000,
372 maxRuns: 5,
373 reclaim,
374 })
375 .then((result) => {
376 if (result.considered > 0) console.log('[reaper]', result)
377 })
378 .catch((error: unknown) => console.error('[reaper] sweep failed:', error))
379 .finally(() => {
380 sweepInFlight = false
381 })
382 }, SWEEP_INTERVAL_MS).unref?.()
383}