Skip to content
Source
Sign inSign up
Open History to see the last commit

record-fx-acp-replay-fixture.ts

540 lines20.4 KB

1/**
2 * Records an fx ACP replay transcript through the real orchestrator and
3 * FxAdapterV2, teeing only the protocol logger. Run from apps/server with
4 * fx 0.0.13 on PATH (or T3_FX_BIN); HOME stays the user's real login:
5 * node scripts/record-fx-acp-replay-fixture.ts --scenario simple
6 */
7import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
8import * as NodeServices from "@effect/platform-node/NodeServices";
9import { ProviderInstanceId, type ProviderReplayEntry } from "@t3tools/contracts";
10import { FxSettings } from "@t3tools/provider-fx/settings";
11import { FX_DRIVER_KIND, makeFxAcpRuntime, makeFxAdapterV2 } from "@t3tools/provider-fx/testing";
12import * as HostProcess from "@t3tools/shared/HostProcess";
13import { resolveSelfInvocation } from "@t3tools/shared/nodeRuntime";
14import * as Clock from "effect/Clock";
15import * as Console from "effect/Console";
16import * as Config from "effect/Config";
17import * as Option from "effect/Option";
18import * as Effect from "effect/Effect";
19import * as FileSystem from "effect/FileSystem";
20import * as Layer from "effect/Layer";
21import * as Path from "effect/Path";
22import * as Schema from "effect/Schema";
23import type * as EffectAcpProtocol from "effect-acp/protocol";
24​
25import { ACP_PROTOCOL } from "@t3tools/provider-acp/server/adapter";
26import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
27import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
28import { buildRuntimeInstructions } from "@t3tools/provider-core/server/runtimeInstructions";
29import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost";
30import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
31import * as ProviderAdapterRegistry from "../src/orchestration-v2/ProviderAdapterRegistry.ts";
32import { provideDeterministicTestRuntime } from "../src/orchestration-v2/testkit/DeterministicRuntime.ts";
33import { ORCHESTRATOR_REPLAY_FIXTURES } from "../src/orchestration-v2/testkit/fixtures/index.ts";
34import { materializeFixtureInput } from "../src/orchestration-v2/testkit/fixtures/shared.ts";
35import { runOrchestratorV2Scenario } from "../src/orchestration-v2/testkit/OrchestratorScenario.ts";
36import * as ProviderReplayHarness from "../src/orchestration-v2/testkit/ProviderReplayHarness.ts";
37import * as Result from "effect/Result";
38​
39const wallClock = Clock.Clock.defaultValue();
40const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
41const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
42const HOME_PLACEHOLDER = "/home/fx-replay";
43const FX_INSTANCE_ID = ProviderInstanceId.make("fx");
44const T3_INSTRUCTIONS_BODY = /<t3_code_instructions>\n[\s\S]*?\n<\/t3_code_instructions>/u;
45const EMAIL = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/giu;
46const TOKEN = /\b(?:sk-|gho_|ghp_|xai-|vercel_)[A-Za-z0-9_-]{8,}\b/gu;
47​
48interface JsonRpcMessage {
49 readonly id?: string | number | null;
50 readonly method?: string;
51 readonly params?: unknown;
52 readonly result?: unknown;
53 readonly error?: unknown;
55​
56interface WireMessage {
57 readonly direction: "incoming" | "outgoing";
58 readonly message: JsonRpcMessage;
60​
61function readArgValues(name: string): ReadonlyArray<string> {
62 const args = process.argv.slice(2);
63 return args.flatMap((arg, index) => (arg === name && args[index + 1] ? [args[index + 1]!] : []));
65​
66function isRecord(value: unknown): value is Record<string, unknown> {
67 return typeof value === "object" && value !== null && !Array.isArray(value);
69​
70function mapStrings(value: unknown, map: (text: string) => string): unknown {
71 if (typeof value === "string") return map(value);
72 if (Array.isArray(value)) return value.map((entry) => mapStrings(entry, map));
73 if (!isRecord(value)) return value;
74 return Object.fromEntries(
75 Object.entries(value).map(([key, entry]) => [key, mapStrings(entry, map)]),
76 );
78​
79const readUserSkillNames = Effect.fn("readFxUserSkillNames")(function* (home: string) {
80 const fs = yield* FileSystem.FileSystem;
81 const path = yield* Path.Path;
82 const entries = yield* fs
83 .readDirectory(path.join(home, ".fx", "skills"))
84 .pipe(Effect.orElseSucceed(() => [] as ReadonlyArray<string>));
85 return entries
86 .filter((name) => name.length > 0)
87 .toSorted((left, right) => right.length - left.length);
88});
89​
90/** Tees raw ACP lines in wire order. Replay drives one process, so a restart is an error. */
91function makeWireTee() {
92 const wire: Array<WireMessage> = [];
93 let runtimeCount = 0;
94 const attachRuntime = () => {
95 runtimeCount += 1;
96 if (runtimeCount > 1) {
97 throw new Error("The fx recording spawned a second ACP process; replay drives one.");
98 }
99 let incomingBuffer = "";
100 const push = (direction: WireMessage["direction"], line: string) => {
101 if (line.trim().length === 0) return;
102 wire.push({ direction, message: decodeJson(line) as JsonRpcMessage });
103 };
104 return {
105 logIncoming: true,
106 logOutgoing: true,
107 logger: (event: EffectAcpProtocol.AcpProtocolLogEvent) =>
108 Effect.sync(() => {
109 if (event.stage !== "raw" || typeof event.payload !== "string") return;
110 if (event.direction === "outgoing") {
111 for (const line of event.payload.split("\n")) push("outgoing", line);
112 return;
113 }
114 incomingBuffer += event.payload;
115 const lines = incomingBuffer.split("\n");
116 incomingBuffer = lines.pop() ?? "";
117 for (const line of lines) push("incoming", line);
118 }),
119 };
120 };
121 return { wire, attachRuntime };
123​
124function frameLabel(kind: string, method: string, params: unknown): string {
125 const update = isRecord(params) && isRecord(params.update) ? params.update : undefined;
126 const updateType = typeof update?.sessionUpdate === "string" ? `:${update.sessionUpdate}` : "";
127 return `${kind}:${method}${updateType}`;
129​
130function wireToEntries(wire: ReadonlyArray<WireMessage>): {
131 readonly entries: Array<ProviderReplayEntry>;
132 readonly droppedFrames: number;
133} {
134 const entries: Array<ProviderReplayEntry> = [];
135 const t3Requests = new Map<string, string>();
136 const agentRequests = new Map<string, string>();
137 let droppedFrames = 0;
138 for (const { direction, message } of wire) {
139 const type = direction === "outgoing" ? "expect_outbound" : "emit_inbound";
140 if (typeof message.method === "string") {
141 const isRequest = message.id !== undefined && message.id !== null;
142 if (isRequest) {
143 (direction === "outgoing" ? t3Requests : agentRequests).set(
144 String(message.id),
145 message.method,
146 );
147 }
148 const kind = isRequest ? "request" : "notification";
149 entries.push({
150 type,
151 label: frameLabel(kind, message.method, message.params),
152 frame: {
153 kind,
154 method: message.method,
155 ...(message.params === undefined ? {} : { params: message.params }),
156 },
157 });
158 continue;
159 }
160 const pending = direction === "outgoing" ? agentRequests : t3Requests;
161 const method = pending.get(String(message.id));
162 if (method === undefined) {
163 droppedFrames += 1;
164 continue;
165 }
166 pending.delete(String(message.id));
167 entries.push({
168 type,
169 label: `response:${method}`,
170 frame: {
171 kind: "response",
172 method,
173 ...(message.result === undefined ? {} : { result: message.result }),
174 ...(message.error === undefined ? {} : { error: message.error }),
175 },
176 });
177 }
178 return { entries, droppedFrames };
180​
181/** Replaces T3-owned request content. User prompt text, clientCapabilities, and _meta stay. */
182function normalizeOutboundFrame(frame: Record<string, unknown>, runtimeInstructions: string) {
183 const params = isRecord(frame.params) ? frame.params : undefined;
184 if (params === undefined) return frame;
185 switch (frame.method) {
186 case "initialize":
187 return {
188 ...frame,
189 params: Object.fromEntries(
190 Object.keys(params).map((key) => [
191 key,
192 key === "protocolVersion" || key === "clientCapabilities" || key === "_meta"
193 ? params[key]
194 : "<any>",
195 ]),
196 ),
197 };
198 case "session/new":
199 case "session/load":
200 case "session/resume":
201 return { ...frame, params: { ...params, mcpServers: "<any>" } };
202 case "session/prompt": {
203 if (!Array.isArray(params.prompt)) return frame;
204 const prompt = params.prompt.filter(
205 (part) => !(isRecord(part) && part.type === "text" && part.text === runtimeInstructions),
206 );
207 return {
208 ...frame,
209 params: {
210 ...params,
211 prompt: prompt.map((part) =>
212 isRecord(part) && typeof part.text === "string"
213 ? {
214 ...part,
215 text: part.text.replace(
216 T3_INSTRUCTIONS_BODY,
217 "<t3_code_instructions>\n<any>\n</t3_code_instructions>",
218 ),
219 }
220 : part,
221 ),
222 },
223 };
224 }
225 default:
226 return frame;
227 }
229​
230function normalizeInboundFrame(frame: Record<string, unknown>): Record<string, unknown> {
231 const params = isRecord(frame.params) ? frame.params : undefined;
232 const update = isRecord(params?.update) ? params.update : undefined;
233 if (
234 update?.sessionUpdate === "available_commands_update" &&
235 Array.isArray(update.availableCommands)
236 ) {
237 return {
238 ...frame,
239 params: {
240 ...params,
241 update: {
242 ...update,
243 availableCommands: update.availableCommands.filter(
244 (command) =>
245 !(isRecord(command) && isRecord(command._meta) && command._meta.scope === "user"),
246 ),
247 },
248 },
249 };
250 }
251 return frame;
253​
254function collectSessionIds(entries: ReadonlyArray<ProviderReplayEntry>): ReadonlyArray<string> {
255 const ids: Array<string> = [];
256 const add = (value: unknown) => {
257 if (typeof value === "string" && value.length > 0 && !ids.includes(value)) ids.push(value);
258 };
259 for (const entry of entries) {
260 if (entry.type === "runtime_exit" || !isRecord(entry.frame)) continue;
261 const frame = entry.frame;
262 if (frame.kind === "response" && frame.method === "session/new" && isRecord(frame.result)) {
263 add(frame.result.sessionId);
264 }
265 if (entry.type === "emit_inbound" && isRecord(frame.params)) {
266 add(frame.params.sessionId);
267 }
268 }
269 return ids;
271​
272function normalizeEntries(input: {
273 readonly entries: ReadonlyArray<ProviderReplayEntry>;
274 readonly workspace: string;
275 readonly home: string;
276 readonly user: string;
277 readonly runtimeInstructions: string;
278 readonly skillNames: ReadonlyArray<string>;
279 readonly gatewayApiKey?: string;
280}): Array<ProviderReplayEntry> {
281 const sessionIds = collectSessionIds(input.entries);
282 const replacements: Array<readonly [string, string]> = [
283 ...sessionIds.map(
284 (id, index) => [id, `fx-session-${String(index + 1).padStart(2, "0")}`] as const,
285 ),
286 [input.workspace, "<workspace>"],
287 [encodeURIComponent(input.workspace), "%3Cworkspace%3E"],
288 [input.home, HOME_PLACEHOLDER],
289 ...input.skillNames.flatMap((name, index) => {
290 const token = `skill-${index + 1}`;
291 return [
292 [`$${name}`, `$${token}`],
293 [`/skills/${name}/`, `/skills/${token}/`],
294 [`/skills/${name}`, `/skills/${token}`],
295 ] as const;
296 }),
297 ];
298 const user = /^[a-z_][a-z0-9_-]*$/iu.test(input.user) ? input.user : "";
299 const userPattern = user.length === 0 ? undefined : new RegExp(`\\b${user}\\b`, "gu");
300 const gatewayApiKey = input.gatewayApiKey ?? "";
301 const replaceAll = (text: string) => {
302 const withoutGatewayKey =
303 gatewayApiKey.length === 0 ? text : text.replaceAll(gatewayApiKey, "<token>");
304 const replaced = replacements.reduce(
305 (current, [from, to]) => (from.length === 0 ? current : current.replaceAll(from, to)),
306 withoutGatewayKey,
307 );
308 const withoutUser =
309 userPattern === undefined ? replaced : replaced.replace(userPattern, "fx-replay");
310 return withoutUser.replace(EMAIL, "user@fx-replay.example").replace(TOKEN, "<token>");
311 };
312 return input.entries.map((entry) => {
313 if (entry.type === "runtime_exit" || !isRecord(entry.frame)) return entry;
314 const frame =
315 entry.type === "expect_outbound"
316 ? normalizeOutboundFrame(entry.frame, input.runtimeInstructions)
317 : normalizeInboundFrame(entry.frame);
318 return {
319 ...entry,
320 ...(entry.label === undefined ? {} : { label: replaceAll(entry.label) }),
321 frame: mapStrings(frame, replaceAll),
322 };
323 });
325​
326const DEFAULT_FX_SETTINGS = Schema.decodeSync(FxSettings)({});
327​
328const RecorderEnv = Config.all({
329 T3_FX_BIN: Config.String("T3_FX_BIN").pipe(Config.option, Config.map(Option.getOrUndefined)),
330 HOME: Config.String("HOME").pipe(Config.option, Config.map(Option.getOrUndefined)),
331 USER: Config.String("USER").pipe(Config.option, Config.map(Option.getOrUndefined)),
332 AI_GATEWAY_API_KEY: Config.String("AI_GATEWAY_API_KEY").pipe(
333 Config.option,
334 Config.map(Option.getOrUndefined),
335 ),
336 TMPDIR: Config.String("TMPDIR").pipe(Config.option, Config.map(Option.getOrUndefined)),
337});
338​
339const recordScenario = Effect.fn("recordFxScenario")(function* (fixtureName: string) {
340 const recorderEnv = yield* RecorderEnv;
341 const fixture = ORCHESTRATOR_REPLAY_FIXTURES.find((candidate) => candidate.name === fixtureName);
342 const variant = fixture?.providers.find((provider) => provider.driver === FX_DRIVER_KIND);
343 if (fixture === undefined || variant === undefined) {
344 return yield* Effect.die(new Error(`No fx replay fixture named '${fixtureName}'.`));
345 }
346 const fs = yield* FileSystem.FileSystem;
347 const path = yield* Path.Path;
348 const fixtureInput = fixture.buildInput();
349 const workspace = yield* checkpointWorkspace(fixtureName, fixtureInput.workspaceFiles);
350 const realWorkspace = yield* fs.realPath(workspace);
351 const materialized = yield* materializeFixtureInput({
352 scenario: fixtureName,
353 fixtureInput,
354 driver: FX_DRIVER_KIND,
355 modelSelection: variant.modelSelection,
356 }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
357 const scenario = {
358 name: `${fixtureName}/fx-record`,
359 commands: materialized.commands,
360 steps: materialized.steps.some(
361 (step) =>
362 step.type === "release_replay_gate" || step.type === "release_replay_gate_after_waiting",
363 )
364 ? materialized.steps.filter(
365 (step) =>
366 step.type !== "release_replay_gate" &&
367 step.type !== "release_replay_gate_after_waiting",
368 )
369 : materialized.steps.map((step) =>
370 step.type === "finish_held_run" ? { ...step, type: "await_run_status" as const } : step,
371 ),
372 projectionThreadIds: materialized.projectionThreadIds,
373 runtimePolicyOverride: { ...variant.runtimePolicyOverride, cwd: realWorkspace },
374 };
375​
376 const tee = makeWireTee();
377 const settings = { ...DEFAULT_FX_SETTINGS, binaryPath: recorderEnv.T3_FX_BIN ?? "fx" };
378 const layerRegistry = ProviderAdapterRegistry.layerFromAdaptersEffect(
379 Effect.gen(function* () {
380 const environment = yield* HostProcess.Environment;
381 const adapter = yield* makeFxAdapterV2({
382 instanceId: FX_INSTANCE_ID,
383 settings,
384 environment,
385 selfInvocation: yield* resolveSelfInvocation(),
386 makeRuntime: (input) =>
387 makeFxAcpRuntime({
388 ...input,
389 protocolLogging: tee.attachRuntime(),
390 fxSettings: settings,
391 environment,
392 }),
393 });
394 const onWallClock = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
395 Effect.provideService(effect, Clock.Clock, wallClock);
396 return [
397 {
398 ...adapter,
399 openSession: (input) =>
400 adapter.openSession(input).pipe(
401 Effect.map((session): ProviderAdapter.ProviderAdapterV2SessionRuntime => ({
402 ...session,
403 startTurn: (turnInput) => onWallClock(session.startTurn(turnInput)),
404 steerTurn: (turnInput) => onWallClock(session.steerTurn(turnInput)),
405 interruptTurn: (turnInput) => onWallClock(session.interruptTurn(turnInput)),
406 respondToRuntimeRequest: (turnInput) =>
407 onWallClock(session.respondToRuntimeRequest(turnInput)),
408 })),
409 onWallClock,
410 ),
411 },
412 ];
413 }),
414 ).pipe(
415 Layer.provide(
416 Layer.mergeAll(
417 TestProviderHost.layer().pipe(
418 Layer.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
419 ),
420 NodeServices.layer,
421 NodeCrypto.layer,
422 IdAllocator.layer,
423 ),
424 ),
425 );
426​
427 let quietPolls = 0;
428 let seenFrames = -1;
429 const waitForFxIdle = Effect.sleep("1 second").pipe(
430 Effect.andThen(
431 Effect.sync(() => {
432 quietPolls = seenFrames === tee.wire.length ? quietPolls + 1 : 0;
433 seenFrames = tee.wire.length;
434 return quietPolls >= 5;
435 }),
436 ),
437 Effect.repeat({ until: (idle) => idle }),
438 Effect.timeout("5 minutes"),
439 Effect.orDie,
440 Effect.asVoid,
441 Effect.provideService(Clock.Clock, wallClock),
442 );
443 const result = yield* runOrchestratorV2Scenario(scenario, { afterSteps: waitForFxIdle }).pipe(
444 Effect.provide(
445 ProviderReplayHarness.layerWithRegistry(
446 scenario,
447 layerRegistry,
448 variant.runContinuationWorker === true ? { runContinuationWorker: true } : {},
449 ),
450 ),
451 provideDeterministicTestRuntime,
452 Effect.scoped,
453 );
454​
455 const { entries, droppedFrames } = wireToEntries(tee.wire);
456 const closedCleanly = entries.some(
457 (entry) =>
458 entry.type === "expect_outbound" &&
459 isRecord(entry.frame) &&
460 entry.frame.method === "session/close",
461 );
462 const initialize = entries.find(
463 (entry) =>
464 entry.type === "emit_inbound" && isRecord(entry.frame) && entry.frame.method === "initialize",
465 );
466 const initializeAgent =
467 initialize?.type === "emit_inbound" &&
468 isRecord(initialize.frame) &&
469 isRecord(initialize.frame.result) &&
470 isRecord(initialize.frame.result.agentInfo)
471 ? initialize.frame.result.agentInfo
472 : {};
473 const home = recorderEnv.HOME ?? "";
474 const transcript = {
475 provider: FX_DRIVER_KIND,
476 protocol: ACP_PROTOCOL,
477 version: "1",
478 scenario: fixtureName,
479 metadata: {
480 generatedBy: "live-fx-recorder",
481 fxVersion: initializeAgent.version ?? "unknown",
482 normalization:
483 "Session ids are fx-session-NN, the workspace is <workspace>, HOME is /home/fx-replay and the recording user is fx-replay. T3-owned prompt text, MCP servers and initialize params other than clientCapabilities and _meta are <any>. User skill names, emails and token-shaped strings are removed. Timestamps are kept as recorded.",
484 droppedFrames,
485 },
486 entries: [
487 ...normalizeEntries({
488 entries,
489 workspace: realWorkspace,
490 home,
491 user: recorderEnv.USER ?? "",
492 skillNames: yield* readUserSkillNames(home),
493 ...(recorderEnv.AI_GATEWAY_API_KEY === undefined
494 ? {}
495 : { gatewayApiKey: recorderEnv.AI_GATEWAY_API_KEY }),
496 runtimeInstructions: buildRuntimeInstructions({
497 harness: "fx",
498 model: variant.modelSelection.model,
499 }),
500 }),
501 { type: "runtime_exit", status: closedCleanly ? "success" : "cancelled" } as const,
502 ],
503 };
504​
505 const outputPath = readArgValues("--out")[0] ?? (yield* path.fromFileUrl(variant.transcriptFile));
506 const liveFailure = Result.try(() => variant.assertOutput(result, transcript));
507 if (Result.isFailure(liveFailure)) {
508 const dumpPath = path.join(
509 recorderEnv.TMPDIR ?? "/tmp",
510 `fx-replay-${fixtureName}-failed.json`,
511 );
512 yield* fs.writeFileString(dumpPath, encodeJson(transcript));
513 yield* Console.log(
514 `Live orchestration failed ${fixtureName} assertions; left ${outputPath} unchanged. Dump: ${dumpPath}`,
515 liveFailure.failure,
516 );
517 return yield* Effect.die(liveFailure.failure);
518 }
519​
520 const { entries: transcriptEntries, ...header } = transcript;
521 yield* fs.writeFileString(
522 outputPath,
523 [
524 encodeJson({ type: "transcript_start", ...header }),
525 ...transcriptEntries.map((entry) => encodeJson(entry)),
526 "",
527 ].join("\n"),
528 );
529 yield* Console.log(`Wrote ${transcriptEntries.length} fx ACP replay entries to ${outputPath}`);
530});
531​
532const scenarios = readArgValues("--scenario").flatMap((value) => value.split(","));
533if (scenarios.length === 0) {
534 throw new Error("Pass --scenario <fixture name>[,<fixture name>...]");
536await Effect.runPromise(
537 Effect.forEach(scenarios, (name) => Effect.scoped(recordScenario(name)), {
538 discard: true,
539 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
540);