Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

durability-attach.test.ts

301 lines11.2 KB

1/**
2 * The OpenCode adapter never journals: it drives the harness over the opencode
3 * HTTP server, so there is no `spawnNdjson`, no journal to tail, no
4 * `awaitAttachableJournal` to refuse a hopeless attach up front, and no
5 * `alignedIfAttaching` to suppress an already-delivered prefix. An app that
6 * wires `withSandbox({ runs, durability })` and routes runs here therefore gets
7 * something that LOOKS durable while being unrecoverable.
8 *
9 * The response splits on whether a first attempt has already run:
10 *
11 * - A FRESH durable run only fails to be recoverable LATER, which an app may
12 * knowingly accept. That is a WARN — audible, not fatal — once per run.
13 * - An ATTACH has already spent its first attempt (`sandboxRunDriver`'s `drive()`
14 * sets `attach: true` only when a previous host was streaming this run), so
15 * proceeding re-runs the agent against the workspace that attempt mutated and
16 * double-appends its output to the run log. That is a THROW.
17 *
18 * The gate sits immediately after the sandbox is resolved and BEFORE
19 * `startOpencodeServerInSandbox`, which is what makes the refusal testable
20 * without a live opencode server: on the attach path nothing is ever spawned, and
21 * `mockSandbox` below records whether a spawn was attempted. That recorded flag
22 * is the real assertion — the error message alone would not prove the agent
23 * didn't run.
24 */
25import { describe, expect, it, vi } from 'vitest'
26import {
27 SandboxCapability,
28 SandboxDurabilityCapability,
29} from '@tanstack/ai-sandbox'
30import { InMemoryRunStore } from '@tanstack/ai'
31import { opencodeText } from '../src/index'
32import type { InternalLogger } from '@tanstack/ai/adapter-internals'
33import type {
34 CapabilityContext,
35 StreamChunk,
36 StreamDurability,
37} from '@tanstack/ai'
38import type {
39 SandboxHandle,
40 SandboxRunDurability,
41 SpawnHandle,
42} from '@tanstack/ai-sandbox'
43​
44const MODEL = 'anthropic/claude-sonnet-4-5'
45​
46function noopLogger(): InternalLogger & { warn: ReturnType<typeof vi.fn> } {
47 return {
48 request: () => {},
49 provider: () => {},
50 errors: () => {},
51 agentLoop: () => {},
52 warnings: () => {},
53 debug: () => {},
54 warn: vi.fn(),
55 } as unknown as InternalLogger & { warn: ReturnType<typeof vi.fn> }
57​
58async function* stdoutOf(values: Array<string>): AsyncIterable<string> {
59 for (const v of values) {
60 await Promise.resolve()
61 yield v
62 }
64​
65/**
66 * A sandbox that RECORDS whether the adapter tried to spawn anything. The
67 * server-readiness line is scripted so the fresh-durable case gets past the
68 * launch helper and proves the run continued rather than being refused.
69 */
70function mockSandbox(): SandboxHandle & { spawns: Array<string> } {
71 const spawns: Array<string> = []
72 const spawnHandle: SpawnHandle = {
73 pid: 1,
74 stdout: stdoutOf(['opencode server listening on http://0.0.0.0:4096\n']),
75 stderr: stdoutOf([]),
76 stdin: { write: () => Promise.resolve(), end: () => Promise.resolve() },
77 wait: () => Promise.resolve(0),
78 kill: () => Promise.resolve(),
79 }
80 const handle = {
81 id: 'sbx',
82 provider: 'mock',
83 capabilities: {} as SandboxHandle['capabilities'],
84 fs: { write: () => Promise.resolve() } as unknown as SandboxHandle['fs'],
85 git: {} as SandboxHandle['git'],
86 process: {
87 exec: () => Promise.reject(new Error('unused')),
88 spawn: (command: string) => {
89 spawns.push(command)
90 return Promise.resolve(spawnHandle)
91 },
92 },
93 ports: {
94 connect: (port: number) =>
95 Promise.resolve({ url: `http://127.0.0.1:${port}` }),
96 },
97 env: { set: () => Promise.resolve() },
98 destroy: () => Promise.resolve(),
99 spawns,
100 }
101 return handle as unknown as SandboxHandle & { spawns: Array<string> }
103​
104/** Never expected to be exercised: this adapter does not journal, which is the condition under test. */
105function fakeAdapterLog(): StreamDurability {
106 return {
107 resumeFrom: () => null,
108 append: (chunks) => Promise.resolve(chunks.map((_, i) => `o:${i}`)),
109 read: () => (async function* empty() {})(),
110 close: () => Promise.resolve(),
111 snapshot: () => Promise.resolve([]),
112 }
114​
115function durability(attach = false): SandboxRunDurability {
116 return {
117 runs: new InMemoryRunStore(),
118 adapter: fakeAdapterLog(),
119 journalDir: '/tmp/tanstack-opencode-attach-unused',
120 attach,
121 detachOnDisconnect: true,
122 }
124​
125function contextWith(
126 handle: SandboxHandle,
127 durabilityCapability?: SandboxRunDurability,
128): CapabilityContext {
129 const [, provideSandbox] = SandboxCapability
130 const [, provideSandboxDurability] = SandboxDurabilityCapability
131 const ctx = {
132 capabilities: { markProvided: () => {}, has: () => true },
133 } as unknown as CapabilityContext
134 provideSandbox(ctx, handle)
135 if (durabilityCapability) provideSandboxDurability(ctx, durabilityCapability)
136 return ctx
138​
139/**
140 * Drain the stream, but give up after `ms` and return what arrived.
141 *
142 * A plain `collect` would be right for the refusal (it terminates at once) and
143 * wrong for diagnosing its ABSENCE: if the refusal ever regressed, the run would
144 * proceed and then hang against the mock's advertised-but-unlistened port, so the
145 * failure would surface as "test timed out" instead of the assertion that
146 * actually caught it. Bounding the drain keeps the red pointing at the spawn that
147 * should not have happened.
148 */
149async function collectWithin(
150 stream: AsyncIterable<StreamChunk>,
151 ms: number,
152): Promise<Array<StreamChunk>> {
153 const out: Array<StreamChunk> = []
154 const iterator = stream[Symbol.asyncIterator]()
155 const deadline = Date.now() + ms
156 for (;;) {
157 const remaining = deadline - Date.now()
158 if (remaining <= 0) break
159 const step = await Promise.race([
160 iterator.next(),
161 new Promise<'timeout'>((resolve) =>
162 setTimeout(() => resolve('timeout'), remaining),
163 ),
164 ])
165 if (step === 'timeout') break
166 if (step.done === true) return out
167 out.push(step.value)
168 }
169 // Abandoned, not awaited: an async generator serializes its queue, so
170 // `return()` would wait behind the `next()` that just failed to settle.
171 void Promise.resolve(iterator.return?.()).catch(() => {})
172 return out
174​
175function errorMessageOf(chunks: Array<StreamChunk>): string | undefined {
176 const err = chunks.find((c) => c.type === 'RUN_ERROR')
177 return (err as { message?: string } | undefined)?.message
179​
180/**
181 * Start the stream and stop as soon as the harness spawn is observed.
182 *
183 * The NON-refused cases cannot be collected to completion: past the gate the
184 * adapter opens a real HTTP session against the mock's advertised port, where
185 * nothing is listening, so the stream neither yields nor settles. That hang is
186 * irrelevant to what these tests assert — the gate's decision is fully made
187 * before the spawn, so observing the spawn is observing the decision. Kicking the
188 * generator with one `next()` we never await, then polling for the spawn, tests
189 * exactly that and terminates.
190 *
191 * Neither the `next()` nor the `return()` may be awaited. An async generator
192 * serializes its own queue, so `return()` waits behind the still-pending
193 * `next()` and would hang just as long as the thing we are avoiding. Both are
194 * therefore fired and abandoned, with rejections swallowed so an abandoned
195 * promise cannot surface as an unhandled rejection. Nothing real leaks: the
196 * spawn is a mock and the only OS resource is a socket to a port with no
197 * listener, which the runtime tears down on its own.
198 */
199async function startUntilSpawn(
200 stream: AsyncIterable<StreamChunk>,
201 sandbox: { spawns: Array<string> },
202): Promise<void> {
203 const iterator = stream[Symbol.asyncIterator]()
204 void Promise.resolve(iterator.next()).catch(() => {})
205 const deadline = Date.now() + 4000
206 while (sandbox.spawns.length === 0 && Date.now() < deadline) {
207 await new Promise((resolve) => setTimeout(resolve, 10))
208 }
209 void Promise.resolve(iterator.return?.()).catch(() => {})
211​
212describe('opencode durability on a never-journaling adapter', () => {
213 it('REFUSES an attach outright, without spawning the harness', async () => {
214 const sandbox = mockSandbox()
215 const logger = noopLogger()
216​
217 const chunks = await collectWithin(
218 opencodeText(MODEL).chatStream({
219 model: MODEL,
220 runId: 'r-attach-refused',
221 messages: [{ role: 'user', content: 'hi' }],
222 logger,
223 capabilities: contextWith(sandbox, durability(true)),
224 }),
225 3000,
226 )
227​
228 // THE POINT OF THE THROW, asserted FIRST because it is the load-bearing
229 // safety property and the one whose failure must be legible: nothing ran. No
230 // opencode server was spawned, so the workspace the previous attempt mutated
231 // was not re-driven and nothing was appended to the run log.
232 expect(sandbox.spawns).toEqual([])
233 expect(chunks.some((c) => c.type === 'RUN_STARTED')).toBe(false)
234​
235 // The adapter's `catch` turns any throw into a RUN_ERROR chunk, so assert on
236 // that rather than on a rejection.
237 const message = errorMessageOf(chunks)
238 expect(message).toContain('cannot ATTACH')
239​
240 // The refusal must state the CONSEQUENCE, not merely the condition — that is
241 // what makes it actionable instead of something an operator retries.
242 expect(message).toMatch(/re-run the agent from scratch/i)
243 expect(message).toMatch(/double-append/i)
244 // And rule the retry out explicitly, because the neighbouring
245 // `JournalAttachUnavailableError` IS retryable.
246 expect(message).toMatch(/not a transient condition/i)
247​
248 // A refusal, not the warn escalated: warning here would mean it proceeded.
249 expect(logger.warn).not.toHaveBeenCalled()
250 })
251​
252 it('warns exactly once and PROCEEDS past the gate on a fresh durable run', async () => {
253 const sandbox = mockSandbox()
254 const logger = noopLogger()
255​
256 await startUntilSpawn(
257 opencodeText(MODEL).chatStream({
258 model: MODEL,
259 runId: 'r-fresh-durable',
260 messages: [{ role: 'user', content: 'hi' }],
261 logger,
262 capabilities: contextWith(sandbox, durability(false)),
263 }),
264 sandbox,
265 )
266​
267 // Once per run, not per chunk — a per-chunk warning would be worse than
268 // none, so the COUNT is the assertion, not merely that it fired.
269 expect(logger.warn).toHaveBeenCalledTimes(1)
270 const [message] = logger.warn.mock.calls[0] as [string]
271 expect(message).toMatch(/not be recoverable/i)
272 // It must name a fix, or it is only noise.
273 expect(message).toMatch(/journaling harness adapter/i)
274​
275 // PROCEEDED: the fresh run is not refused, and the proof is that it reached
276 // the spawn the attach case never got to.
277 expect(sandbox.spawns.length).toBeGreaterThan(0)
278 })
279​
280 it('is byte-identical to today when no durability is wired', async () => {
281 const sandbox = mockSandbox()
282 const logger = noopLogger()
283​
284 await startUntilSpawn(
285 opencodeText(MODEL).chatStream({
286 model: MODEL,
287 runId: 'r-no-durability',
288 messages: [{ role: 'user', content: 'hi' }],
289 logger,
290 capabilities: contextWith(sandbox),
291 }),
292 sandbox,
293 )
294​
295 // The common default stays silent and unchanged: neither check may fire when
296 // the app never asked for durability at all, and the run proceeds exactly as
297 // before to the same spawn.
298 expect(logger.warn).not.toHaveBeenCalled()
299 expect(sandbox.spawns.length).toBeGreaterThan(0)
300 })
301})