Skip to content
Source
Sign inSign up
Open History to see the last commit

output.ts

86 lines3.4 KB

1import { assert } from "@effect/vitest";
2import type { ProviderReplayTranscript } from "@t3tools/contracts";
3​
4import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts";
5import {
6 assertBaseProjection,
7 assertSemanticProjectionIntegrity,
8 projectionFor,
9} from "../shared.ts";
10import { PI_COMPACTION_MARKER } from "./input.ts";
11​
12function field(value: unknown, key: string): unknown {
13 return typeof value === "object" && value !== null ? Reflect.get(value, key) : undefined;
15​
16export function assertPiCompactionOutput(
17 result: OrchestratorV2ScenarioResult,
18 transcript: ProviderReplayTranscript,
19) {
20 const outbound = transcript.entries.flatMap((entry) =>
21 entry.type === "expect_outbound" ? [entry.frame] : [],
22 );
23 assert.isFalse(
24 outbound.some(
25 (frame) =>
26 field(frame, "type") === "prompt" && String(field(frame, "message")).startsWith("/compact"),
27 ),
28 "/compact must not be sent as a prompt",
29 );
30 assert.deepEqual(
31 outbound.filter((frame) => field(frame, "type") === "compact"),
32 [
33 { type: "compact", customInstructions: `keep the opaque marker ${PI_COMPACTION_MARKER}` },
34 { type: "compact" },
35 ],
36 );
37 const compactionEnds = transcript.entries
38 .flatMap((entry) => (entry.type === "emit_inbound" ? [entry.frame] : []))
39 .filter((frame) => field(frame, "type") === "compaction_end");
40 assert.lengthOf(compactionEnds, 2);
41 const [refused, compacted] = compactionEnds;
42 // Pi omits `result` on failure rather than sending null.
43 assert.isFalse(typeof refused === "object" && refused !== null && "result" in refused);
44 const compactionResult = field(compacted, "result");
45 assert.isDefined(compactionResult, "the second compaction must succeed");
46​
47 assertBaseProjection({
48 result,
49 transcript,
50 runCount: 6,
51 runStatuses: ["completed", "completed", "completed", "completed", "completed", "completed"],
52 });
53 const projection = projectionFor(result, transcript.scenario);
54 assertSemanticProjectionIntegrity(projection);
55 const compactionRuns = [projection.runs[1], projection.runs[4]];
56 const compactions = projection.turnItems.filter((item) => item.type === "compaction");
57 assert.deepEqual(
58 compactions.map((item) => [item.runId, item.status, item.title]),
59 [
60 [compactionRuns[0]?.id ?? null, "failed", "Context compaction failed"],
61 [compactionRuns[1]?.id ?? null, "completed", "Context compacted"],
62 ],
63 );
64 assert.equal(compactions[0]?.summary, field(refused, "errorMessage"));
65 assert.equal(compactions[1]?.summary, field(compactionResult, "summary"));
66 assert.equal(compactions[1]?.beforeTokenCount, field(compactionResult, "tokensBefore"));
67 assert.equal(compactions[1]?.afterTokenCount, field(compactionResult, "estimatedTokensAfter"));
68 assert.isFalse(
69 projection.turnItems.some(
70 (item) =>
71 compactionRuns.some((run) => run?.id === item.runId) && item.type === "assistant_message",
72 ),
73 "compaction must not run a model turn",
74 );
75 // Pi reports tokens null after compaction; retain its post-compaction estimate.
76 const compactionTurn = projection.providerTurns.find(
77 (turn) => turn.runAttemptId === compactionRuns[1]?.activeAttemptId,
78 );
79 assert.equal(
80 compactionTurn?.tokenUsage?.usedTokens,
81 field(compactionResult, "estimatedTokensAfter"),
82 );
83​
84 const recall = projection.turnItems.findLast((item) => item.type === "assistant_message");
85 assert.include(recall?.text, PI_COMPACTION_MARKER);