Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

ClaudeReplayFixtures.integration.test.ts

332 lines14.0 KB

1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, describe, it } from "@effect/vitest";
3import {
4 CommandId,
5 isProviderNativeSubagentThread,
6 MessageId,
7 ProviderDriverKind,
8 type ProviderReplayEntry,
9} from "@t3tools/contracts";
10import * as Effect from "effect/Effect";
11​
12import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts";
13import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
14import * as Orchestrator from "../Orchestrator.ts";
15import { userFacingDispatchErrorMessage } from "../UserFacingErrors.ts";
16import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
17import { subagentInput } from "./fixtures/subagent/input.ts";
18import { runOrchestratorV2Scenario } from "./OrchestratorScenario.ts";
19import * as ProviderReplayHarness from "./ProviderReplayHarness.ts";
20import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
21import {
22 CLAUDE_MODEL_SELECTION,
23 materializeFixtureInput,
24 projectionFor,
25} from "./fixtures/shared.ts";
26import {
27 findReplayFixture,
28 itRunsReplayFixtures,
29 runReplayFixture,
30 type ReplayFixtureCase,
31} from "./ProviderReplayFixtures.ts";
32import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
33​
34const readTranscript = Effect.fn("readClaudeReplayFixture")(function* (file: URL) {
35 return yield* readProviderReplayTranscript(file);
36}, Effect.provide(NodeServices.layer));
37​
38function readClaudeTranscriptFixture(path: string) {
39 return readTranscript(new URL(`./fixtures/${path}/claude_transcript.ndjson`, import.meta.url));
41​
42function runClaudeReplayFixture(input: ReplayFixtureCase) {
43 return runReplayFixture({ ...input, harness: ClaudeOrchestratorReplayHarness });
45​
46describe("Claude Agent SDK replay fixtures", () => {
47 itRunsReplayFixtures("claudeAgent", runClaudeReplayFixture);
48​
49 it.effect("refuses messages to a native subagent thread without touching it", () =>
50 Effect.gen(function* () {
51 const raw = yield* readClaudeTranscriptFixture("subagent");
52 const transcript = yield* ClaudeOrchestratorReplayHarness.decodeTranscript(
53 materializeReplayTranscriptRuntimeInstructions(raw, {
54 driver: ProviderDriverKind.make("claudeAgent"),
55 model: CLAUDE_MODEL_SELECTION.model,
56 }),
57 );
58 const materialized = yield* materializeFixtureInput({
59 scenario: "subagent",
60 fixtureInput: subagentInput(),
61 driver: ProviderDriverKind.make("claudeAgent"),
62 modelSelection: CLAUDE_MODEL_SELECTION,
63 }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
64 const scenario = {
65 name: "subagent/claudeAgent:read-only-child",
66 transcript,
67 commands: materialized.commands,
68 steps: materialized.steps,
69 projectionThreadIds: materialized.projectionThreadIds,
70 };
71 yield* Effect.gen(function* () {
72 const result = yield* runOrchestratorV2Scenario(scenario);
73 const orchestrator = yield* Orchestrator.OrchestratorV2;
74 const child = [...result.projections.values()].find((projection) =>
75 isProviderNativeSubagentThread(projection.thread),
76 );
77 assert.isDefined(child);
78 const before = yield* orchestrator.getThreadEventSequence(child.thread.id);
79​
80 const refused = yield* orchestrator
81 .dispatch({
82 type: "message.dispatch",
83 createdBy: "user",
84 creationSource: "web",
85 commandId: CommandId.make("command:subagent:message-native-child"),
86 threadId: child.thread.id,
87 messageId: MessageId.make("message:subagent:message-native-child"),
88 text: "Also check the tests.",
89 attachments: [],
90 dispatchMode: { type: "start_immediately" },
91 })
92 .pipe(Effect.flip);
93 assert.equal(refused._tag, "OrchestratorSubagentThreadReadOnlyError");
94 assert.equal(
95 userFacingDispatchErrorMessage(refused),
96 "This subagent is run by its provider and cannot take messages. Message the parent thread instead.",
97 );
98 const refusedContinuation = yield* orchestrator
99 .dispatch({
100 type: "thread.metadata.update",
101 commandId: CommandId.make("command:subagent:continue-native-child"),
102 threadId: child.thread.id,
103 title: "Renamed",
104 continuation: {
105 messageId: MessageId.make("message:subagent:continue-native-child"),
106 text: "Also check the tests.",
107 },
108 })
109 .pipe(Effect.flip);
110 assert.equal(refusedContinuation._tag, "OrchestratorSubagentThreadReadOnlyError");
111 assert.equal(yield* orchestrator.getThreadEventSequence(child.thread.id), before);
112 const after = yield* orchestrator.getThreadProjection(child.thread.id);
113 assert.lengthOf(after.runs, 0);
114 assert.deepEqual(after.messages, child.messages);
115 }).pipe(
116 Effect.provide(
117 ProviderReplayHarness.layerProviderReplay(scenario, ClaudeOrchestratorReplayHarness),
118 ),
119 provideDeterministicTestRuntime,
120 Effect.scoped,
121 );
122 }),
123 );
124​
125 const afterRootFixture = findReplayFixture("claude_background_subagent_after_root");
126 const afterRootProvider = afterRootFixture?.providers[0];
127 if (afterRootFixture !== undefined && afterRootProvider !== undefined) {
128 const OPEN_TOOL_ID = "toolu_01QDa5jV5g1H9h6QyDfeJohD";
129 const SUBAGENT_TASK_ID = "a2995bfced8019363";
130​
131 const moveOpenToolBeforeRootResult = (
132 move: boolean,
133 entries: ReadonlyArray<ProviderReplayEntry>,
134 ): ReadonlyArray<ProviderReplayEntry> => {
135 if (!move) return entries;
136 const toolIndex = entries.findIndex(
137 (entry) =>
138 entry.type === "emit_inbound" &&
139 (entry.frame as Record<string, any>).message?.content?.[0]?.id === OPEN_TOOL_ID,
140 );
141 const resultIndex = entries.findIndex(
142 (entry) => entry.type === "emit_inbound" && entry.label === "result",
143 );
144 if (toolIndex < 0 || resultIndex < 0) throw new Error("transcript shape changed");
145 const toolEntry = entries[toolIndex]!;
146 const rest = entries.filter((_, index) => index !== toolIndex);
147 return [...rest.slice(0, resultIndex), toolEntry, ...rest.slice(resultIndex)];
148 };
149 // A subagent's foreground Bash run during the root's turn is tailed like a
150 // root call's, so the child's command row streams live.
151 const SUBAGENT_BASH_ID = "toolu_01JMN3SSQCHzUTNua9AWdkiH";
152 it.effect("streams a subagent's foreground Bash output live during the root turn", () =>
153 runClaudeReplayFixture({
154 fixtureName: afterRootFixture.name,
155 buildInput: afterRootFixture.buildInput,
156 driver: {
157 ...afterRootProvider,
158 assertOutput: (result) => {
159 const childThreadId = projectionFor(result, afterRootFixture.name).subagents[0]
160 ?.childThreadId;
161 assert.exists(childThreadId);
162 const command = result.projections
163 .get(childThreadId)
164 ?.turnItems.find(
165 (item) =>
166 item.type === "command_execution" &&
167 JSON.stringify(item.input).includes("SUB_DONE_1"),
168 );
169 assert.exists(command);
170 const streamed = result.commandOutput?.get(command.id);
171 assert.exists(streamed, "the subagent's command must stream its output");
172 assert.include(streamed.output.text, "SUB_DONE_1");
173 },
174 },
175 transformTranscript: (transcript) => {
176 const isBashFrame = (entry: ProviderReplayEntry) => {
177 if (entry.type !== "emit_inbound") return false;
178 const frame = entry.frame as Record<string, any>;
179 return (
180 frame.message?.content?.[0]?.id === SUBAGENT_BASH_ID ||
181 frame.message?.content?.[0]?.tool_use_id === SUBAGENT_BASH_ID ||
182 (frame.type === "system" && frame.tool_use_id === SUBAGENT_BASH_ID)
183 );
184 };
185 // Recorded without the structured result the replay writes output files from.
186 const bashFrames = transcript.entries.filter(isBashFrame).map((entry) => {
187 const frame = (entry as { readonly frame: Record<string, any> }).frame;
188 return frame.type === "user"
189 ? {
190 ...entry,
191 frame: { ...frame, tool_use_result: { stdout: "SUB_DONE_1\n", stderr: "" } },
192 }
193 : entry;
194 });
195 const rest = transcript.entries.filter((entry) => !isBashFrame(entry));
196 const resultIndex = rest.findIndex(
197 (entry) => entry.type === "emit_inbound" && entry.label === "result",
198 );
199 assert.lengthOf(bashFrames, 4, "transcript shape changed");
200 assert.isAtLeast(resultIndex, 0, "transcript shape changed");
201 return {
202 ...transcript,
203 entries: [...rest.slice(0, resultIndex), ...bashFrames, ...rest.slice(resultIndex)],
204 };
205 },
206 }),
207 );
208​
209 // regression: #16486 a background subagent's tool left open while its parent is idle never settled
210 it.effect.each([
211 ["WebFetch", "while root is idle", "stopped"],
212 ["Write", "before root settles", "failed"],
213 ] as const)(
214 "ends a background subagent's open %s (opened %s) when its notification is %s",
215 ([tool, opens, notificationStatus]) =>
216 runClaudeReplayFixture({
217 fixtureName: afterRootFixture.name,
218 buildInput: afterRootFixture.buildInput,
219 driver: {
220 ...afterRootProvider,
221 assertOutput: (result) => {
222 const parent = projectionFor(result, afterRootFixture.name);
223 assert.deepEqual(
224 parent.runs.map((run) => run.status),
225 ["completed", "completed"],
226 );
227 const subagent = parent.subagents[0];
228 assert.equal(
229 subagent?.status,
230 notificationStatus === "stopped" ? "cancelled" : "failed",
231 );
232 const childThreadId = subagent?.childThreadId;
233 assert.exists(childThreadId);
234 const child = result.projections.get(childThreadId);
235 assert.exists(child);
236 const fetches = child.turnItems.filter(
237 (item) => item.type === (tool === "WebFetch" ? "web_search" : "file_change"),
238 );
239 assert.lengthOf(fetches, 1, `the ${tool} reached the child thread`);
240 assert.deepEqual(
241 {
242 openTool: fetches.map((item) => item.status),
243 openChildNodes: child.nodes
244 .filter(
245 (node) =>
246 node.status === "running" ||
247 node.status === "pending" ||
248 node.status === "waiting",
249 )
250 .map((node) => `${node.kind}:${node.status}`),
251 },
252 {
253 openTool: [notificationStatus === "stopped" ? "interrupted" : "failed"],
254 openChildNodes: [],
255 },
256 );
257 },
258 },
259 transformTranscript: (transcript) => ({
260 ...transcript,
261 entries: moveOpenToolBeforeRootResult(
262 opens === "before root settles",
263 transcript.entries.flatMap((entry): ReadonlyArray<ProviderReplayEntry> => {
264 if (entry.type !== "emit_inbound") return [entry];
265 const frame = entry.frame as Record<string, any>;
266 if (
267 frame.type === "assistant" &&
268 frame.message?.content?.[0]?.id === OPEN_TOOL_ID
269 ) {
270 return [
271 {
272 ...entry,
273 frame: {
274 ...frame,
275 message: {
276 ...frame.message,
277 content: [
278 {
279 ...frame.message.content[0],
280 name: tool,
281 input:
282 tool === "WebFetch"
283 ? { url: "https://example.com", prompt: "Summarize" }
284 : { file_path: "notes.txt", content: "SUB_DONE_2" },
285 },
286 ],
287 },
288 },
289 },
290 ];
291 }
292 if (
293 (frame.type === "system" &&
294 (frame.subtype === "task_started" || frame.subtype === "task_notification") &&
295 frame.tool_use_id === OPEN_TOOL_ID) ||
296 (frame.type === "user" &&
297 frame.message?.content?.[0]?.tool_use_id === OPEN_TOOL_ID) ||
298 (frame.type === "assistant" &&
299 frame.parent_tool_use_id !== null &&
300 frame.message?.content?.[0]?.text === "SUB_FINAL_REPORT")
301 ) {
302 return [];
303 }
304 if (frame.type === "system" && frame.subtype === "task_updated") {
305 return [
306 {
307 ...entry,
308 frame: {
309 ...frame,
310 patch: {
311 ...frame.patch,
312 status: notificationStatus === "stopped" ? "killed" : "failed",
313 },
314 },
315 },
316 ];
317 }
318 if (
319 frame.type === "system" &&
320 frame.subtype === "task_notification" &&
321 frame.task_id === SUBAGENT_TASK_ID
322 ) {
323 return [{ ...entry, frame: { ...frame, status: notificationStatus } }];
324 }
325 return [entry];
326 }),
327 ),
328 }),
329 }),
330 );
331 }
332});