Skip to content
Source
Sign inSign up
Open History to see the last commit

record-pi-rpc-replay-fixture.ts

311 lines12.1 KB

1import { pipe } from "effect/Function";
2import * as NodeServices from "@effect/platform-node/NodeServices";
3import type { ProviderReplayEntry, ProviderReplayTranscript } from "@t3tools/contracts";
4import * as Console from "effect/Console";
5import * as DateTime from "effect/DateTime";
6import * as Duration from "effect/Duration";
7import * as Effect from "effect/Effect";
8import * as FileSystem from "effect/FileSystem";
9import * as Layer from "effect/Layer";
10import * as Path from "effect/Path";
11import * as Schema from "effect/Schema";
12import { TestClock } from "effect/testing";
13import { ChildProcess, ChildProcessSpawner } from "effect/process";
14​
15import {
16 makePiRecordingSpawner,
17 PI_REPLAY_ANY,
18 PI_RPC_REPLAY_PROTOCOL,
19 PiOrchestratorReplayHarness,
20} from "../src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts";
21import * as PiAdapterV2Testkit from "../src/orchestration-v2/Adapters/PiAdapterV2.testkit.ts";
22import { PI_PROVIDER } from "@t3tools/provider-pi/server";
23import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
24import { provideDeterministicTestRuntime } from "../src/orchestration-v2/testkit/DeterministicRuntime.ts";
25import { ORCHESTRATOR_REPLAY_FIXTURES } from "../src/orchestration-v2/testkit/fixtures/index.ts";
26import { materializeFixtureInput } from "../src/orchestration-v2/testkit/fixtures/shared.ts";
27import { runOrchestratorV2ProviderReplayScenario } from "../src/orchestration-v2/testkit/ProviderReplayHarness.ts";
28import {
29 checkpointWorkspace,
30 makeCheckpointWorkspace,
31} from "@t3tools/provider-testing/replayWorkspace";
32​
33// Exclude user context; --approve lets the throwaway workspace's `.pi/settings.json` load.
34const HERMETIC_LAUNCH_ARGS =
35 "--no-extensions --no-skills --no-prompt-templates --no-context-files --approve";
36// Pi excludes ~20k recent tokens from compaction; lower the budget for this short fixture.
37const WORKSPACE_PI_SETTINGS = { compaction: { keepRecentTokens: 50 } };
38const CLOCK_TICK = Duration.millis(20);
39​
40function readArgValue(name: string): string | undefined {
41 const index = process.argv.indexOf(name);
42 return index === -1 ? undefined : process.argv[index + 1];
44​
45const scenario = readArgValue("--scenario");
46const fixture = ORCHESTRATOR_REPLAY_FIXTURES.find((entry) => entry.name === scenario);
47const variant = fixture?.providers.find((provider) => provider.driver === PI_PROVIDER);
48if (fixture === undefined || variant === undefined) {
49 const names = ORCHESTRATOR_REPLAY_FIXTURES.filter((entry) =>
50 entry.providers.some((provider) => provider.driver === PI_PROVIDER),
51 ).map((entry) => entry.name);
52 throw new Error(`Pass --scenario with a fixture that registers Pi: ${names.join(", ")}`);
54​
55const piBinary = process.env.T3_PI_BIN ?? "pi";
56const [modelProvider, ...modelId] = variant.modelSelection.model.split("/");
57const launchArgs = `--provider ${modelProvider} --model ${modelId.join("/")} ${HERMETIC_LAUNCH_ARGS}`;
58const home = process.env.HOME ?? "";
59​
60const UUID_PATTERN = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/giu;
61const ISO_TIMESTAMP_PATTERN = /\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\b/gu;
62const NORMALIZED_EPOCH_MS = DateTime.toEpochMillis(DateTime.makeUnsafe("2026-01-01T00:00:00.000Z"));
63​
64function escapeRegExp(value: string): string {
65 return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
67​
68function isRecord(value: unknown): value is Record<string, unknown> {
69 return typeof value === "object" && value !== null && !Array.isArray(value);
71​
72// Normalize both directions consistently so echoed paths, UUIDs and times still match on replay.
73function normalizeEntries(input: {
74 readonly entries: ReadonlyArray<ProviderReplayEntry>;
75 readonly workspace: string;
76 readonly sessionDir: string;
77 readonly modelSlug: string;
78}): ReadonlyArray<ProviderReplayEntry> {
79 const sessionFiles = new Map<string, string>();
80 const uuids = new Map<string, string>();
81 let timeAnchorMs: number | undefined;
82 const shiftTime = (ms: number): number => {
83 timeAnchorMs ??= ms;
84 return NORMALIZED_EPOCH_MS + (ms - timeAnchorMs);
85 };
86 const sessionFilePattern = new RegExp(
87 `${escapeRegExp(input.sessionDir)}/[^\\s"']+?\\.jsonl`,
88 "gu",
89 );
90 const normalizeString = (value: string): string =>
91 value
92 .replace(sessionFilePattern, (file) => {
93 const existing = sessionFiles.get(file);
94 if (existing !== undefined) return existing;
95 const next = `/pi-sessions/session-${sessionFiles.size + 1}.jsonl`;
96 sessionFiles.set(file, next);
97 return next;
98 })
99 .replaceAll(input.sessionDir, "/pi-sessions")
100 .replaceAll(input.workspace, "<workspace>")
101 .replaceAll(home.length > 1 ? home : "\0", "~")
102 .replace(ISO_TIMESTAMP_PATTERN, (iso) =>
103 pipe(
104 DateTime.makeUnsafe(iso),
105 DateTime.toEpochMillis,
106 shiftTime,
107 DateTime.makeUnsafe,
108 DateTime.formatIso,
109 ),
110 )
111 .replace(UUID_PATTERN, (uuid) => {
112 const existing = uuids.get(uuid.toLowerCase());
113 if (existing !== undefined) return existing;
114 const next = `00000000-0000-4000-8000-${String(uuids.size + 1).padStart(12, "0")}`;
115 uuids.set(uuid.toLowerCase(), next);
116 return next;
117 });
118 const normalizeValue = (value: unknown): unknown => {
119 if (typeof value === "string") return normalizeString(value);
120 if (Array.isArray(value)) return value.map(normalizeValue);
121 if (!isRecord(value)) return value;
122 // Pi's unread system messages contain machine-specific context; retain only their shape.
123 if (value.role === "system" && isRecord(value.sections)) {
124 return {
125 ...value,
126 sections: Object.fromEntries(
127 Object.keys(value.sections).map((key) => [key, "<system-prompt-section>"]),
128 ),
129 };
130 }
131 return Object.fromEntries(
132 Object.entries(value).map(([key, entry]) => [
133 key,
134 key === "thinkingSignature" || key === "textSignature"
135 ? "<signature>"
136 : key === "timestamp" && typeof entry === "number"
137 ? shiftTime(entry)
138 : normalizeValue(entry),
139 ]),
140 );
141 };
142 const normalizeFrame = (frame: unknown): unknown => {
143 if (!isRecord(frame)) return normalizeValue(frame);
144 const args = frame.args;
145 if (frame.type === "process_start" && Array.isArray(args)) {
146 return normalizeValue({
147 ...frame,
148 args: args.map((arg, index) => (args[index - 1] === "--extension" ? PI_REPLAY_ANY : arg)),
149 });
150 }
151 const data = frame.data;
152 if (frame.command === "get_available_models" && isRecord(data) && Array.isArray(data.models)) {
153 return normalizeValue({
154 ...frame,
155 data: {
156 ...data,
157 models: data.models.filter(
158 (model) => isRecord(model) && `${model.provider}/${model.id}` === input.modelSlug,
159 ),
160 },
161 });
162 }
163 return normalizeValue(frame);
164 };
165 return input.entries.map((entry) =>
166 entry.type === "runtime_exit" ? entry : { ...entry, frame: normalizeFrame(entry.frame) },
167 );
169​
170const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
171​
172function encodeTranscriptNdjson(transcript: ProviderReplayTranscript): string {
173 const { entries, ...metadata } = transcript;
174 return [
175 encodeJson({ type: "transcript_start", ...metadata }),
176 ...entries.map((entry) => encodeJson(entry)),
177 "",
178 ].join("\n");
180​
181// Follow wall time for live Pi timers while retaining replay's TestClock steps.
182const followWallClock = Effect.gen(function* () {
183 while (true) {
184 yield* TestClock.withLive(Effect.sleep(CLOCK_TICK));
185 yield* TestClock.adjust(CLOCK_TICK);
186 }
187});
188​
189const readPiVersion = Effect.gen(function* () {
190 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
191 return (yield* spawner.string(ChildProcess.make(piBinary, ["--version"]))).trim();
192});
193​
194const record = Effect.gen(function* () {
195 const fs = yield* FileSystem.FileSystem;
196 const path = yield* Path.Path;
197 const piVersion = yield* readPiVersion;
198 const outputPath = readArgValue("--out") ?? (yield* path.fromFileUrl(variant.transcriptFile));
199 const fixtureInput = fixture.buildInput();
200 const workspace = yield* Effect.promise(() =>
201 makeCheckpointWorkspace(`pi-rpc-record-${fixture.name}`, fixtureInput.workspaceFiles),
202 );
203 yield* fs.makeDirectory(path.join(workspace, ".pi"));
204 yield* fs.writeFileString(
205 path.join(workspace, ".pi", "settings.json"),
206 encodeJson(WORKSPACE_PI_SETTINGS),
207 );
208 const sessionDir = yield* fs.makeTempDirectory({ prefix: `t3-pi-record-sessions-` });
209 yield* Effect.addFinalizer(() =>
210 Effect.all([
211 fs.remove(workspace, { recursive: true, force: true }),
212 fs.remove(sessionDir, { recursive: true, force: true }),
213 ]).pipe(Effect.ignore),
214 );
215​
216 const entries: Array<ProviderReplayEntry> = [];
217 const placeholder: ProviderReplayTranscript = {
218 provider: PI_PROVIDER,
219 protocol: PI_RPC_REPLAY_PROTOCOL,
220 version: piVersion,
221 scenario: fixture.name,
222 entries: [],
223 };
224 const layerRecordingSpawner = Layer.effect(
225 ChildProcessSpawner.ChildProcessSpawner,
226 Effect.map(Effect.service(ChildProcessSpawner.ChildProcessSpawner), (live) =>
227 makePiRecordingSpawner(live, entries),
228 ),
229 ).pipe(Layer.provide(NodeServices.layer));
230​
231 const materializeInput = materializeFixtureInput({
232 scenario: fixture.name,
233 fixtureInput,
234 driver: variant.driver,
235 modelSelection: variant.modelSelection,
236 }).pipe(Effect.provide(IdAllocator.layer));
237 const continuationOptions =
238 variant.runContinuationWorker === true ? { runContinuationWorker: true } : {};
239​
240 yield* Effect.gen(function* () {
241 yield* followWallClock.pipe(Effect.forkScoped);
242 const materialized = yield* materializeInput;
243 yield* runOrchestratorV2ProviderReplayScenario(
244 {
245 name: `${fixture.name}/pi:record`,
246 transcript: placeholder,
247 commands: materialized.commands,
248 steps: materialized.steps,
249 projectionThreadIds: materialized.projectionThreadIds,
250 runtimePolicyOverride: { ...variant.runtimePolicyOverride, cwd: workspace },
251 },
252 {
253 driver: PI_PROVIDER,
254 decodeTranscript: Effect.succeed,
255 makeProviderAdapterRegistryLayer: () =>
256 PiAdapterV2Testkit.layer({
257 scenario: fixture.name,
258 binaryPath: piBinary,
259 launchArgs,
260 environment: [
261 { name: "PI_CODING_AGENT_SESSION_DIR", value: sessionDir, sensitive: false },
262 { name: "PI_SKIP_VERSION_CHECK", value: "1", sensitive: false },
263 ],
264 spawner: layerRecordingSpawner,
265 }),
266 },
267 continuationOptions,
268 );
269 }).pipe(Effect.scoped, provideDeterministicTestRuntime);
270​
271 const transcript = {
272 ...placeholder,
273 metadata: {
274 generatedBy: "live-pi-recorder",
275 piVersion,
276 model: variant.modelSelection.model,
277 launchArgs,
278 },
279 entries: normalizeEntries({
280 entries,
281 workspace,
282 sessionDir,
283 modelSlug: variant.modelSelection.model,
284 }),
285 } satisfies ProviderReplayTranscript;
286​
287 // Assert on normalized replay; the live projection has raw paths. Failed assertions preserve the fixture.
288 yield* Effect.gen(function* () {
289 const replayWorkspace = yield* checkpointWorkspace(fixture.name, fixtureInput.workspaceFiles);
290 const materialized = yield* materializeInput;
291 const replayResult = yield* runOrchestratorV2ProviderReplayScenario(
292 {
293 name: `${fixture.name}/pi:verify`,
294 transcript: yield* PiOrchestratorReplayHarness.decodeTranscript(transcript),
295 commands: materialized.commands,
296 steps: materialized.steps,
297 projectionThreadIds: materialized.projectionThreadIds,
298 runtimePolicyOverride: { ...variant.runtimePolicyOverride, cwd: replayWorkspace },
299 },
300 PiOrchestratorReplayHarness,
301 continuationOptions,
302 );
303 variant.assertOutput(replayResult, transcript);
304 }).pipe(Effect.scoped, provideDeterministicTestRuntime);
305​
306 yield* fs.makeDirectory(path.dirname(outputPath), { recursive: true });
307 yield* fs.writeFileString(outputPath, encodeTranscriptNdjson(transcript));
308 yield* Console.log(`Wrote ${transcript.entries.length} Pi RPC replay entries to ${outputPath}`);
309});
310​
311await Effect.runPromise(record.pipe(Effect.scoped, Effect.provide(NodeServices.layer)));