Skip to content
Source
Sign inSign up
Open History to see the last commit

OrchestratorReplayRestartBackgroundNote.integration.test.ts

298 lines11.6 KB

1import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { assert, describe, it } from "@effect/vitest";
4import { EventId, ProviderDriverKind, type ProviderReplayEntry } from "@t3tools/contracts";
5import * as DateTime from "effect/DateTime";
6import * as Effect from "effect/Effect";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";
10import * as SqlClient from "effect/sql/SqlClient";
11​
12import {
13 ClaudeOrchestratorReplayHarness,
14 makeClaudeRestartReplayHarness,
15} from "../Adapters/ClaudeAdapterV2.testkit.ts";
16import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
17import * as EffectWorker from "../EffectWorker.ts";
18import * as EventSink from "../EventSink.ts";
19import * as Orchestrator from "../Orchestrator.ts";
20import * as SqlitePersistence from "../../persistence/Sqlite.ts";
21import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
22import { CLAUDE_BACKGROUND_SUBAGENT_AFTER_ROOT_PROMPT } from "./fixtures/claude_background_subagent_after_root/input.ts";
23import {
24 CLAUDE_MODEL_SELECTION,
25 materializeFixtureInput,
26 projectionFor,
27} from "./fixtures/shared.ts";
28import { runOrchestratorV2ProviderReplayScenario } from "./ProviderReplayHarness.ts";
29import * as ProviderReplayHarness from "./ProviderReplayHarness.ts";
30import { runOrchestratorV2Scenario } from "./OrchestratorScenario.ts";
31import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
32import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
33​
34const SCENARIO = "claude_background_subagent_after_root";
35const SESSION_ID = "cca274e4-25ae-4171-b972-bbb31118517e";
36const FIRST_AFTER_RESTART = "Is the background subagent done yet?";
37const SECOND_AFTER_RESTART = "Thanks. Anything else?";
38const NOTE = [
39 "Note: the T3 server restarted, and this background work was cancelled before it finished. It will not report back on its own, so resume or restart what you still need and wait for its result:",
40 "- subagent: Background subagent test (id a2995bfced8019363)",
41].join("\n");
42​
43const readRestartTranscript = Effect.fn("readRestartTranscript")(function* (
44 resumedPrompts: ReadonlyArray<string>,
45) {
46 const recorded = yield* readProviderReplayTranscript(
47 new URL(`./fixtures/${SCENARIO}/claude_transcript.ndjson`, import.meta.url),
48 );
49 const rootResult = recorded.entries.findIndex(
50 (entry) => entry.type === "emit_inbound" && entry.label === "result",
51 );
52​
53 const queryOpen = recorded.entries[0];
54 const openFrame = queryOpen?.type === "expect_outbound" ? queryOpen.frame : undefined;
55 const queryOptions: unknown =
56 typeof openFrame === "object" && openFrame !== null
57 ? Reflect.get(openFrame, "options")
58 : undefined;
59 if (typeof queryOptions !== "object" || queryOptions === null) {
60 throw new Error(`${SCENARIO} must open with a recorded query.open frame.`);
61 }
62 const { sessionId: _sessionId, ...resumeOptions } = queryOptions as Record<string, unknown>;
63 const resumedTurn = (label: string, text: string): ReadonlyArray<ProviderReplayEntry> => [
64 {
65 type: "expect_outbound",
66 label: `prompt.offer:${label}`,
67 frame: {
68 type: "prompt.offer",
69 message: {
70 type: "user",
71 message: { role: "user", content: text },
72 parent_tool_use_id: null,
73 },
74 },
75 },
76 {
77 type: "emit_inbound",
78 label: `assistant:${label}`,
79 frame: {
80 type: "assistant",
81 message: {
82 model: "claude-sonnet-4-6",
83 id: `msg_restart_${label}`,
84 type: "message",
85 role: "assistant",
86 content: [{ type: "text", text: `REPLY_${label}` }],
87 stop_reason: null,
88 stop_sequence: null,
89 usage: { input_tokens: 1, output_tokens: 1 },
90 },
91 parent_tool_use_id: null,
92 session_id: SESSION_ID,
93 uuid: `restart-assistant-${label}`,
94 },
95 },
96 {
97 type: "emit_inbound",
98 label: `result:${label}`,
99 frame: {
100 type: "result",
101 subtype: "success",
102 is_error: false,
103 num_turns: 1,
104 result: `REPLY_${label}`,
105 stop_reason: "end_turn",
106 duration_ms: 1,
107 duration_api_ms: 1,
108 total_cost_usd: 0,
109 usage: { input_tokens: 1, output_tokens: 1 },
110 modelUsage: {},
111 permission_denials: [],
112 session_id: SESSION_ID,
113 uuid: `restart-result-${label}`,
114 },
115 },
116 ];
117 return yield* ClaudeOrchestratorReplayHarness.decodeTranscript({
118 ...recorded,
119 entries: [
120 ...recorded.entries.slice(0, rootResult + 1),
121 // Keep the query alive until the crash snapshot is captured.
122 {
123 type: "expect_outbound",
124 label: "query.open:resume",
125 frame: { type: "query.open", options: { ...resumeOptions, resume: SESSION_ID } },
126 },
127 ...resumedPrompts.flatMap((prompt, index) => resumedTurn(String(index + 1), prompt)),
128 ],
129 });
130});
131​
132const runRestart = Effect.fn("runRestart")(function* (input: {
133 readonly resumedPrompts: ReadonlyArray<string>;
134 readonly userMessagesAfterRestart: ReadonlyArray<string>;
135 readonly continueThreadsAfterServerUpdate: boolean;
136}) {
137 const transcript = yield* readRestartTranscript(input.resumedPrompts);
138 const workspace = yield* checkpointWorkspace(SCENARIO);
139 const fs = yield* FileSystem.FileSystem;
140 const path = yield* Path.Path;
141 const tempDir = yield* Effect.acquireRelease(
142 fs.makeTempDirectory({ prefix: "t3-orchestration-v2-restart-note-" }),
143 (directory) => fs.remove(directory, { recursive: true, force: true }).pipe(Effect.orDie),
144 );
145 const materialized = yield* materializeFixtureInput({
146 scenario: SCENARIO,
147 fixtureInput: {
148 steps: [
149 { type: "message", text: CLAUDE_BACKGROUND_SUBAGENT_AFTER_ROOT_PROMPT },
150 ...input.userMessagesAfterRestart.map((text) => ({ type: "message" as const, text })),
151 ],
152 },
153 driver: ProviderDriverKind.make("claudeAgent"),
154 modelSelection: CLAUDE_MODEL_SELECTION,
155 });
156​
157 const firstIdle = materialized.steps.findIndex((step) => step.type === "await_thread_idle");
158 const phase1Steps = materialized.steps.slice(0, firstIdle + 1);
159 const phase2Steps = materialized.steps.slice(firstIdle + 1);
160 const { harness, assertComplete } = makeClaudeRestartReplayHarness(transcript);
161 const layerDatabase = SqlitePersistence.layerFromPath(path.join(tempDir, "state.sqlite")).pipe(
162 Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
163 );
164 const snapshotPath = path.join(tempDir, "restart-state.sqlite");
165 const scenario = (name: string, steps: typeof materialized.steps) => ({
166 name: `${SCENARIO}:${name}`,
167 transcript,
168 commands: steps.flatMap((step) => (step.type === "dispatch" ? [step.command] : [])),
169 steps,
170 projectionThreadIds: materialized.projectionThreadIds,
171 runtimePolicyOverride: { cwd: workspace },
172 });
173​
174 const beforeScenario = scenario("before-restart", phase1Steps);
175 const before = yield* Effect.scoped(
176 Effect.gen(function* () {
177 const result = yield* runOrchestratorV2Scenario(beforeScenario);
178 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
179 yield* worker.drain();
180 const orchestrator = yield* Orchestrator.OrchestratorV2;
181 const eventSink = yield* EventSink.EventSinkV2;
182 const threadId = materialized.projectionThreadIds[0];
183 if (threadId === undefined) {
184 return yield* Effect.die("restart scenario has no projection thread");
185 }
186 const projection = yield* orchestrator.getThreadProjection(threadId);
187 const session = projection.providerSessions[0];
188 if (session === undefined) {
189 return yield* Effect.die("restart scenario has no provider session");
190 }
191 // Snapshot the legacy running task before exit cleanup settles it, then
192 // seed the stopped session that older runtimes could leave after a crash.
193 const now = yield* DateTime.now;
194 yield* eventSink.write({
195 events: [
196 {
197 id: EventId.make("event:restart-background-note:stopped-provider-session"),
198 type: "provider-session.updated",
199 threadId,
200 providerInstanceId: session.providerInstanceId,
201 occurredAt: now,
202 payload: { ...session, status: "stopped", updatedAt: now, lastError: null },
203 },
204 ],
205 });
206 const sql = yield* SqlClient.SqlClient;
207 yield* sql`VACUUM INTO ${snapshotPath}`;
208 return result;
209 }).pipe(
210 Effect.provide(
211 Layer.merge(
212 ProviderReplayHarness.layerProviderReplay(beforeScenario, harness, {
213 databaseLayer: layerDatabase,
214 }),
215 layerDatabase,
216 ),
217 ),
218 ),
219 );
220 const snapshotDatabase = SqlitePersistence.layerFromPath(snapshotPath).pipe(
221 Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
222 );
223 const settled = projectionFor(before, SCENARIO);
224 assert.equal(settled.runs[0]?.status, "completed");
225 assert.equal(settled.subagents[0]?.status, "running");
226​
227 const restartScenario = scenario("restarted", []);
228 const restarted = yield* Effect.scoped(
229 Effect.gen(function* () {
230 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
231 yield* worker.drain();
232 return yield* runOrchestratorV2Scenario(restartScenario);
233 }).pipe(
234 Effect.provide(
235 ProviderReplayHarness.layerProviderReplay(restartScenario, harness, {
236 databaseLayer: snapshotDatabase,
237 recoverOnStartup: true,
238 continueThreadsAfterServerUpdate: input.continueThreadsAfterServerUpdate,
239 }),
240 ),
241 ),
242 );
243 const recovered = projectionFor(restarted, SCENARIO);
244 assert.deepEqual(
245 recovered.runs.map((run) => run.status),
246 ["completed"],
247 );
248 assert.equal(recovered.subagents[0]?.status, "cancelled");
249 assert.equal(recovered.runs[0]?.restartCancelledBackgroundWork?.[0]?.kind, "subagent");
250​
251 const after = yield* Effect.scoped(
252 runOrchestratorV2ProviderReplayScenario(scenario("after-restart", phase2Steps), harness, {
253 databaseLayer: snapshotDatabase,
254 continueThreadsAfterServerUpdate: input.continueThreadsAfterServerUpdate,
255 }),
256 );
257​
258 yield* assertComplete;
259 const projection = projectionFor(after, SCENARIO);
260 assert.equal(projection.subagents[0]?.status, "cancelled");
261 return projection;
262});
263​
264const userTexts = (projection: ReturnType<typeof projectionFor>) =>
265 projection.turnItems.flatMap((item) => (item.type === "user_message" ? [item.text] : []));
266​
267describe("restart-cancelled background work", () => {
268 it.effect.each([false, true])(
269 "keeps the thread settled and tells the next user turn once when continuation is %s",
270 (continueThreadsAfterServerUpdate) =>
271 Effect.gen(function* () {
272 const projection = yield* runRestart({
273 resumedPrompts: [
274 `${NOTE}\n\nUser message:\n${FIRST_AFTER_RESTART}`,
275 SECOND_AFTER_RESTART,
276 ],
277 userMessagesAfterRestart: [FIRST_AFTER_RESTART, SECOND_AFTER_RESTART],
278 continueThreadsAfterServerUpdate,
279 });
280 assert.deepEqual(
281 projection.runs.map((run) => run.status),
282 ["completed", "completed", "completed"],
283 );
284 assert.isFalse(projection.runs.some((run) => run.restartContinuationOfRunId !== undefined));
285​
286 assert.deepEqual(userTexts(projection), [
287 CLAUDE_BACKGROUND_SUBAGENT_AFTER_ROOT_PROMPT,
288 FIRST_AFTER_RESTART,
289 SECOND_AFTER_RESTART,
290 ]);
291 }).pipe(
292 provideDeterministicTestRuntime,
293 Effect.provide(
294 Layer.merge(IdAllocator.layer, Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
295 ),
296 ),
297 );
298});