Skip to content
Source
Sign inSign up
Open History to see the last commit

OrchestratorScenario.ts

785 lines29.1 KB

1import type {
2 OrchestrationV2Command,
3 OrchestrationV2DomainEvent,
4 OrchestrationV2RuntimeRequest,
5 OrchestrationV2Run,
6 OrchestrationV2ThreadShellSnapshot,
7 OrchestrationV2StoredEvent,
8 OrchestrationV2ThreadProjection,
9 OrchestrationV2TurnItem,
10 ProviderApprovalDecision,
11 ProviderUserInputAnswers,
12 CommandId,
13 ThreadId,
14 TurnItemId,
15} from "@t3tools/contracts";
16import * as Clock from "effect/Clock";
17import * as Deferred from "effect/Deferred";
18import * as Duration from "effect/Duration";
19import * as Effect from "effect/Effect";
20import * as Fiber from "effect/Fiber";
21import * as Option from "effect/Option";
22import * as Ref from "effect/Ref";
23import * as Schema from "effect/Schema";
24import * as Stream from "effect/Stream";
25import { TestClock } from "effect/testing";
26​
27import * as Orchestrator from "../Orchestrator.ts";
28import type { RecordedCommandOutput } from "./CommandOutputRecorder.ts";
29import type { ProviderReplayGate } from "@t3tools/provider-testing/replayGate";
30​
31export type OrchestratorV2ScenarioStep =
32 | {
33 readonly type: "dispatch";
34 readonly command: OrchestrationV2Command;
35 readonly await?: boolean;
36 readonly key?: string;
37 }
38 | {
39 readonly type: "advance_clock";
40 readonly duration: Duration.Input;
41 /** Wait until the jump idle-releases this thread's provider session. */
42 readonly awaitProviderSessionRelease?: ThreadId;
43 }
44 | {
45 readonly type: "await";
46 readonly key: string;
47 }
48 | {
49 readonly type: "await_all";
50 }
51 | {
52 readonly type: "await_thread_idle";
53 readonly threadId: ThreadId;
54 }
55 | {
56 readonly type: "await_no_background_work";
57 readonly threadId: ThreadId;
58 }
59 | {
60 readonly type: "await_run_steerable";
61 readonly threadId: ThreadId;
62 readonly runId: OrchestrationV2Run["id"];
63 }
64 | {
65 readonly type: "await_run_status";
66 readonly threadId: ThreadId;
67 readonly runId: OrchestrationV2Run["id"];
68 readonly status: OrchestrationV2Run["status"];
69 }
70 | {
71 readonly type: "finish_held_run";
72 readonly threadId: ThreadId;
73 readonly runId: OrchestrationV2Run["id"];
74 readonly status: OrchestrationV2Run["status"];
75 }
76 | {
77 readonly type: "await_run_turn_item";
78 readonly threadId: ThreadId;
79 readonly runId: OrchestrationV2Run["id"];
80 readonly itemType: OrchestrationV2TurnItem["type"];
81 /** Wait until the item also has this status. */
82 readonly itemStatus?: OrchestrationV2TurnItem["status"];
83 /** Look in the threads of the run's provider subagents instead of the run itself. */
84 readonly inSubagentThreads?: boolean;
85 }
86 | {
87 readonly type: "release_replay_gate_after_waiting";
88 readonly label: string;
89 readonly threadId: ThreadId;
90 readonly runId: OrchestrationV2Run["id"];
91 }
92 | {
93 readonly type: "release_replay_gate";
94 readonly label: string;
95 }
96 | {
97 readonly type: "capture_shell_snapshot";
98 readonly key: string;
99 }
100 | {
101 readonly type: "respond_to_next_runtime_request";
102 readonly threadId: ThreadId;
103 readonly commandId: CommandId;
104 readonly decision?: ProviderApprovalDecision;
105 readonly answers?: ProviderUserInputAnswers;
106 readonly shellSnapshotKeyWhilePending?: string;
107 };
108​
109export interface OrchestratorV2Scenario {
110 readonly name: string;
111 readonly commands: ReadonlyArray<OrchestrationV2Command>;
112 readonly steps?: ReadonlyArray<OrchestratorV2ScenarioStep>;
113 readonly projectionThreadIds?: ReadonlyArray<ThreadId>;
115​
116export interface OrchestratorV2ScenarioResult {
117 readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
118 readonly domainEvents: ReadonlyArray<OrchestrationV2DomainEvent>;
119 readonly projections: ReadonlyMap<ThreadId, OrchestrationV2ThreadProjection>;
120 readonly shellSnapshot: OrchestrationV2ThreadShellSnapshot;
121 readonly capturedShellSnapshots: ReadonlyMap<string, OrchestrationV2ThreadShellSnapshot>;
122 /** Live output each command streamed (never persisted). Provider replays fill it in. */
123 readonly commandOutput?: ReadonlyMap<TurnItemId, RecordedCommandOutput>;
125​
126export class OrchestratorV2ScenarioStepError extends Schema.TaggedError<OrchestratorV2ScenarioStepError>()(
127 "OrchestratorV2ScenarioStepError",
128 {
129 scenario: Schema.String,
130 step: Schema.String,
131 },
132) {
133 override get message(): string {
134 return `Invalid orchestrator scenario step ${this.step} in ${this.scenario}.`;
135 }
137​
138function commandThreadIds(command: OrchestrationV2Command): ReadonlyArray<ThreadId> {
139 switch (command.type) {
140 case "thread.create":
141 case "thread.stop":
142 case "thread.archive":
143 case "thread.unarchive":
144 case "thread.delete":
145 case "thread.guest-share.close":
146 case "thread.guest-message.dismiss":
147 case "thread.settle":
148 case "thread.auto-settle":
149 case "thread.unsettle":
150 case "thread.snooze":
151 case "thread.unsnooze":
152 case "thread.auto-settle.set":
153 case "thread.pin":
154 case "thread.unpin":
155 case "thread.pin.reorder":
156 case "thread.active.reorder":
157 case "thread.tag.set":
158 case "thread.visit":
159 case "thread.mark-unread":
160 case "thread.metadata.update":
161 case "thread.pull-request.link":
162 case "thread.pull-request.unlink":
163 case "thread.pull-request-link.sync":
164 case "thread.pull-request.watch":
165 case "thread.pull-request.sync":
166 case "thread.title.regeneration.complete":
167 case "thread.runtime-mode.set":
168 case "thread.interaction-mode.set":
169 case "thread.model-selection.set":
170 case "provider-session.detach":
171 case "message.dispatch":
172 case "notification.delivery.accept":
173 case "prepared-run.release":
174 case "prepared-run.progress":
175 case "prepared-run.fail":
176 case "prepared-run.retry":
177 case "subagent.stop":
178 case "run.interrupt":
179 case "queued-message.promote-to-steer":
180 case "queued-run.start":
181 case "queue.resume":
182 case "queued-run.reorder":
183 case "queued-run.cancel":
184 case "queued-run.edit":
185 case "runtime-request.respond":
186 case "thread.user-input.dismiss":
187 case "checkpoint.rollback":
188 case "provider.switch":
189 return [command.threadId];
190 case "delegated_task.request":
191 case "delegated_task.wake-policy":
192 case "delegated_task.completion-delivery.acknowledge":
193 case "delegated_task.completion-delivery.dispose":
194 case "thread.created.record":
195 return [command.parentThreadId];
196 case "thread.fork":
197 case "thread.merge_back":
198 return [command.sourceThreadId, command.targetThreadId];
199 }
201​
202function scenarioSteps(
203 scenario: OrchestratorV2Scenario,
204): ReadonlyArray<OrchestratorV2ScenarioStep> {
205 return (
206 scenario.steps ??
207 scenario.commands.map((command) => ({
208 type: "dispatch" as const,
209 command,
210 await: true,
211 }))
212 );
214​
215function scenarioCommands(scenario: OrchestratorV2Scenario): ReadonlyArray<OrchestrationV2Command> {
216 return scenarioSteps(scenario).flatMap((step) =>
217 step.type === "dispatch" ? [step.command] : [],
218 );
220​
221const findPendingRuntimeRequest = (projection: OrchestrationV2ThreadProjection) =>
222 projection.runtimeRequests.find((request) => request.status === "pending");
223​
224const hasActiveRun = (projection: OrchestrationV2ThreadProjection) =>
225 projection.runs.some((run) =>
226 ["preparing", "queued", "starting", "running", "waiting"].includes(run.status),
227 );
228​
229const SCENARIO_WAIT_ATTEMPTS = 10_000;
230// Iteration budgets burn through idle event-loop turns during async I/O; exhaust both iterations and wall-clock deadline.
231const SCENARIO_WAIT_DEADLINE_MS = 60_000;
232const scenarioWaitDeadline = () => performance.now() + SCENARIO_WAIT_DEADLINE_MS;
233const scenarioWaitExhausted = (attemptsRemaining: number, deadlineAt: number) =>
234 attemptsRemaining <= 0 && performance.now() >= deadlineAt;
235​
236const yieldToRuntime = Effect.yieldNow.pipe(
237 Effect.andThen(
238 Effect.promise(
239 () =>
240 new Promise<void>((resolve) => {
241 setImmediate(resolve);
242 }),
243 ),
244 ),
245);
246​
247function collectProjectionThreadIds(scenario: OrchestratorV2Scenario): ReadonlyArray<ThreadId> {
248 if (scenario.projectionThreadIds) {
249 return scenario.projectionThreadIds;
250 }
251​
252 const ids = new Set<ThreadId>();
253 for (const command of scenarioCommands(scenario)) {
254 for (const threadId of commandThreadIds(command)) {
255 ids.add(threadId);
256 }
257 }
258 return Array.from(ids);
260​
261export function runOrchestratorV2Scenario(
262 scenario: OrchestratorV2Scenario,
263 options: {
264 readonly replayGate?: ProviderReplayGate;
265 readonly afterSteps?: Effect.Effect<void>;
266 } = {},
267): Effect.Effect<
268 OrchestratorV2ScenarioResult,
269 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
270 Orchestrator.OrchestratorV2
271> {
272 return Effect.scoped(
273 Effect.gen(function* () {
274 const orchestrator = yield* Orchestrator.OrchestratorV2;
275 const storedEventGroups: Array<ReadonlyArray<OrchestrationV2StoredEvent>> = [];
276 const observedStoredEvents = yield* Ref.make<Array<OrchestrationV2StoredEvent>>([]);
277 let releaseWatch:
278 | { readonly threadId: ThreadId; readonly released: Deferred.Deferred<void> }
279 | undefined;
280 yield* orchestrator.streamStoredEvents.pipe(
281 Stream.runForEach((stored) =>
282 Ref.update(observedStoredEvents, (existing) => [...existing, stored]).pipe(
283 Effect.andThen(
284 releaseWatch !== undefined &&
285 stored.event.type === "provider-session.updated" &&
286 stored.event.threadId === releaseWatch.threadId &&
287 stored.event.payload.status === "stopped"
288 ? Deferred.succeed(releaseWatch.released, undefined)
289 : Effect.void,
290 ),
291 ),
292 ),
293 Effect.forkScoped,
294 );
295 const backgroundDispatches = new Map<
296 string,
297 Fiber.Fiber<ReadonlyArray<OrchestrationV2StoredEvent>, Orchestrator.OrchestratorV2Error>
298 >();
299 const capturedShellSnapshots = new Map<string, OrchestrationV2ThreadShellSnapshot>();
300 let anonymousBackgroundDispatchIndex = 0;
301​
302 const awaitDispatch = Effect.fn("awaitDispatch")(function* (key: string) {
303 const fiber = backgroundDispatches.get(key);
304 if (!fiber) {
305 return yield* new OrchestratorV2ScenarioStepError({
306 scenario: scenario.name,
307 step: `await:${key}`,
308 });
309 }
310 const events = yield* Fiber.join(fiber);
311 backgroundDispatches.delete(key);
312 storedEventGroups.push(events);
313 });
314​
315 const waitForPendingRuntimeRequest = (
316 threadId: ThreadId,
317 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
318 deadlineAt = scenarioWaitDeadline(),
319 ): Effect.Effect<
320 OrchestrationV2RuntimeRequest,
321 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
322 never
323 > =>
324 Effect.gen(function* () {
325 const projection = yield* orchestrator.getThreadProjection(threadId);
326 const request = findPendingRuntimeRequest(projection);
327 if (request !== undefined) {
328 return request;
329 }
330 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
331 const runState = projection.runs.map((run) => `${run.id}:${run.status}`).join(",");
332 return yield* new OrchestratorV2ScenarioStepError({
333 scenario: scenario.name,
334 step: `respond_to_next_runtime_request:${threadId}:runs=${runState}:providerTurns=${projection.providerTurns.length}`,
335 });
336 }
337 yield* yieldToRuntime;
338 return yield* waitForPendingRuntimeRequest(threadId, attemptsRemaining - 1, deadlineAt);
339 });
340​
341 const waitForThreadIdle = (
342 threadId: ThreadId,
343 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
344 deadlineAt = scenarioWaitDeadline(),
345 ): Effect.Effect<
346 void,
347 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
348 never
349 > =>
350 Effect.gen(function* () {
351 const projection = yield* orchestrator.getThreadProjection(threadId);
352 if (!hasActiveRun(projection)) {
353 return;
354 }
355 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
356 const activeRuns = projection.runs
357 .filter((run) =>
358 ["preparing", "queued", "starting", "running", "waiting"].includes(run.status),
359 )
360 .map((run) => `${run.id}:${run.status}`)
361 .join(",");
362 const pendingRequests = projection.runtimeRequests
363 .filter((request) => request.status === "pending")
364 .map((request) => `${request.id}:${request.kind}`)
365 .join(",");
366 return yield* new OrchestratorV2ScenarioStepError({
367 scenario: scenario.name,
368 step: `await_thread_idle:${threadId}:runs=${activeRuns}:requests=${pendingRequests}`,
369 });
370 }
371 yield* yieldToRuntime;
372 return yield* waitForThreadIdle(threadId, attemptsRemaining - 1, deadlineAt);
373 });
374​
375 const waitForNoBackgroundWork = (
376 threadId: ThreadId,
377 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
378 deadlineAt = scenarioWaitDeadline(),
379 ): Effect.Effect<
380 void,
381 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
382 never
383 > =>
384 Effect.gen(function* () {
385 const pending = (yield* orchestrator.getThreadShell(threadId))?.pendingBackgroundTasks;
386 if ((pending?.length ?? 0) === 0) {
387 return;
388 }
389 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
390 return yield* new OrchestratorV2ScenarioStepError({
391 scenario: scenario.name,
392 step: `await_no_background_work:${threadId}:pending=${pending?.map((task) => task.taskId).join(",")}`,
393 });
394 }
395 yield* yieldToRuntime;
396 return yield* waitForNoBackgroundWork(threadId, attemptsRemaining - 1, deadlineAt);
397 });
398​
399 const waitForRunSteerable = (
400 threadId: ThreadId,
401 runId: OrchestrationV2Run["id"],
402 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
403 deadlineAt = scenarioWaitDeadline(),
404 ): Effect.Effect<
405 void,
406 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
407 never
408 > =>
409 Effect.gen(function* () {
410 const projection = yield* orchestrator.getThreadProjection(threadId);
411 const run = projection.runs.find((candidate) => candidate.id === runId);
412 const providerTurn = projection.providerTurns.find(
413 (candidate) =>
414 run?.activeAttemptId !== null &&
415 candidate.runAttemptId === run?.activeAttemptId &&
416 candidate.status === "running",
417 );
418 if (run?.status === "running" && providerTurn !== undefined) {
419 return;
420 }
421 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
422 return yield* new OrchestratorV2ScenarioStepError({
423 scenario: scenario.name,
424 step: `await_run_steerable:${runId}`,
425 });
426 }
427 yield* yieldToRuntime;
428 return yield* waitForRunSteerable(threadId, runId, attemptsRemaining - 1, deadlineAt);
429 });
430​
431 const waitForRunStatus = (
432 threadId: ThreadId,
433 runId: OrchestrationV2Run["id"],
434 status: OrchestrationV2Run["status"],
435 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
436 deadlineAt = scenarioWaitDeadline(),
437 ): Effect.Effect<
438 void,
439 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
440 never
441 > =>
442 Effect.gen(function* () {
443 const projection = yield* orchestrator.getThreadProjection(threadId);
444 const run = projection.runs.find((candidate) => candidate.id === runId);
445 if (run?.status === status) {
446 return;
447 }
448 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
449 return yield* new OrchestratorV2ScenarioStepError({
450 scenario: scenario.name,
451 step: `await_run_status:${runId}:${status}:actual=${run?.status ?? "missing"}`,
452 });
453 }
454 yield* yieldToRuntime;
455 return yield* waitForRunStatus(
456 threadId,
457 runId,
458 status,
459 attemptsRemaining - 1,
460 deadlineAt,
461 );
462 });
463​
464 const waitForRunTurnItem = (
465 threadId: ThreadId,
466 runId: OrchestrationV2Run["id"],
467 itemType: OrchestrationV2TurnItem["type"],
468 itemStatus: OrchestrationV2TurnItem["status"] | undefined,
469 inSubagentThreads: boolean,
470 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
471 deadlineAt = scenarioWaitDeadline(),
472 ): Effect.Effect<
473 void,
474 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
475 never
476 > =>
477 Effect.gen(function* () {
478 const projection = yield* orchestrator.getThreadProjection(threadId);
479 const matches = (item: OrchestrationV2TurnItem) =>
480 item.type === itemType && (itemStatus === undefined || item.status === itemStatus);
481 if (!inSubagentThreads) {
482 if (projection.turnItems.some((item) => item.runId === runId && matches(item))) return;
483 } else {
484 for (const subagent of projection.subagents) {
485 if (subagent.runId !== runId || subagent.childThreadId === null) continue;
486 const child = yield* orchestrator.getThreadProjection(subagent.childThreadId);
487 if (child.turnItems.some(matches)) return;
488 }
489 }
490 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
491 return yield* new OrchestratorV2ScenarioStepError({
492 scenario: scenario.name,
493 step: `await_run_turn_item:${runId}:${itemType}${itemStatus === undefined ? "" : `:${itemStatus}`}`,
494 });
495 }
496 yield* yieldToRuntime;
497 return yield* waitForRunTurnItem(
498 threadId,
499 runId,
500 itemType,
501 itemStatus,
502 inSubagentThreads,
503 attemptsRemaining - 1,
504 deadlineAt,
505 );
506 });
507​
508 const releaseReplayGateAfterWaiting = (
509 label: string,
510 threadId: ThreadId,
511 runId: OrchestrationV2Run["id"],
512 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
513 ): Effect.Effect<
514 void,
515 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
516 never
517 > =>
518 Effect.gen(function* () {
519 const projection = yield* orchestrator.getThreadProjection(threadId);
520 const run = projection.runs.find((candidate) => candidate.id === runId);
521 const providerThread = projection.providerThreads.find(
522 (candidate) => candidate.id === run?.providerThreadId,
523 );
524 const pendingTaskCount = providerThread?.pendingBackgroundTasks?.length ?? 0;
525 const gateReached = options.replayGate?.hasReached(label) ?? false;
526 if (
527 gateReached &&
528 run?.status === "completed" &&
529 providerThread !== undefined &&
530 pendingTaskCount > 0
531 ) {
532 options.replayGate?.release(label);
533 yield* waitForProviderBackgroundTasksCleared(threadId, providerThread.id);
534 return;
535 }
536 if (attemptsRemaining <= 0) {
537 options.replayGate?.release(label);
538 return yield* new OrchestratorV2ScenarioStepError({
539 scenario: scenario.name,
540 step: `release_replay_gate_after_waiting:${label}:reached=${gateReached}:run=${run?.status ?? "missing"}:providerThread=${run?.providerThreadId ?? "missing"}:pending=${pendingTaskCount}`,
541 });
542 }
543 yield* yieldToRuntime;
544 return yield* releaseReplayGateAfterWaiting(
545 label,
546 threadId,
547 runId,
548 attemptsRemaining - 1,
549 );
550 });
551​
552 const waitForProviderBackgroundTasksCleared = (
553 threadId: ThreadId,
554 providerThreadId: NonNullable<OrchestrationV2Run["providerThreadId"]>,
555 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
556 ): Effect.Effect<
557 void,
558 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
559 never
560 > =>
561 Effect.gen(function* () {
562 const projection = yield* orchestrator.getThreadProjection(threadId);
563 const providerThread = projection.providerThreads.find(
564 (candidate) => candidate.id === providerThreadId,
565 );
566 const hasPendingTasks = (providerThread?.pendingBackgroundTasks?.length ?? 0) > 0;
567 if (!hasPendingTasks && providerThread?.status === "idle") {
568 return;
569 }
570 if (attemptsRemaining <= 0) {
571 const providerState = projection.providerThreads
572 .map(
573 (candidate) =>
574 `${candidate.id}:${candidate.status}:pending=${candidate.pendingBackgroundTasks?.length ?? 0}`,
575 )
576 .join(",");
577 return yield* new OrchestratorV2ScenarioStepError({
578 scenario: scenario.name,
579 step: `await_provider_background_tasks_cleared:${threadId}:target=${providerThreadId}:providers=${providerState}`,
580 });
581 }
582 yield* yieldToRuntime;
583 return yield* waitForProviderBackgroundTasksCleared(
584 threadId,
585 providerThreadId,
586 attemptsRemaining - 1,
587 );
588 });
589​
590 let consumedFinishReceipts = 0;
591 const finishHeldRun = Effect.fn("scenario.finishHeldRun")(function* (
592 step: Extract<OrchestratorV2ScenarioStep, { readonly type: "finish_held_run" }>,
593 ) {
594 const gate = options.replayGate;
595 if (gate === undefined) {
596 return yield* new OrchestratorV2ScenarioStepError({
597 scenario: scenario.name,
598 step: `finish_held_run:${step.runId}:no_replay_gate`,
599 });
600 }
601 while (true) {
602 // Only the latest debounce receipt settles the run; late frames rearm it.
603 const armed = yield* Effect.promise(() =>
604 gate.waitForFinishArmed(consumedFinishReceipts),
605 ).pipe(
606 Effect.timeoutOption(Duration.millis(SCENARIO_WAIT_DEADLINE_MS)),
607 Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()),
608 );
609 if (Option.isNone(armed)) {
610 const projection = yield* orchestrator.getThreadProjection(step.threadId);
611 const run = projection.runs.find((candidate) => candidate.id === step.runId);
612 return yield* new OrchestratorV2ScenarioStepError({
613 scenario: scenario.name,
614 step: `finish_held_run:${step.runId}:${step.status}:actual=${run?.status ?? "missing"}:no_finish_armed`,
615 });
616 }
617 consumedFinishReceipts = gate.finishArmedCount();
618 yield* TestClock.adjust(armed.value);
619 const settled = yield* Effect.raceFirst(
620 waitForRunStatus(step.threadId, step.runId, step.status).pipe(Effect.as(true)),
621 Effect.promise(() => gate.waitForFinishArmed(consumedFinishReceipts)).pipe(
622 Effect.as(false),
623 ),
624 );
625 if (settled) {
626 consumedFinishReceipts = gate.finishArmedCount();
627 return;
628 }
629 }
630 });
631​
632 const releaseReplayGate = Effect.fn("scenario.releaseReplayGate")(function* (label: string) {
633 const gate = options.replayGate;
634 const reached =
635 gate === undefined ? false : yield* Effect.promise(() => gate.waitForReached(label));
636 if (!reached) {
637 return yield* new OrchestratorV2ScenarioStepError({
638 scenario: scenario.name,
639 step: `release_replay_gate:${label}:not_configured`,
640 });
641 }
642 gate?.release(label);
643 });
644​
645 for (const step of scenarioSteps(scenario)) {
646 switch (step.type) {
647 case "dispatch": {
648 if (step.await ?? true) {
649 const result = yield* orchestrator.dispatch(step.command);
650 storedEventGroups.push(result.storedEvents);
651 break;
652 }
653​
654 anonymousBackgroundDispatchIndex += 1;
655 const key = step.key ?? `dispatch:${anonymousBackgroundDispatchIndex}`;
656 backgroundDispatches.set(
657 key,
658 yield* orchestrator.dispatch(step.command).pipe(
659 Effect.map((result) => result.storedEvents),
660 Effect.forkScoped,
661 ),
662 );
663 break;
664 }
665 case "advance_clock": {
666 if (step.awaitProviderSessionRelease === undefined) {
667 yield* TestClock.adjust(step.duration);
668 break;
669 }
670 // The release runs on a woken fiber; a dispatch before it commits cancels it.
671 const released = yield* Deferred.make<void>();
672 releaseWatch = { threadId: step.awaitProviderSessionRelease, released };
673 yield* TestClock.adjust(step.duration);
674 const settled = yield* Deferred.await(released).pipe(
675 Effect.timeoutOption(Duration.millis(SCENARIO_WAIT_DEADLINE_MS)),
676 Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()),
677 );
678 releaseWatch = undefined;
679 if (Option.isNone(settled)) {
680 return yield* new OrchestratorV2ScenarioStepError({
681 scenario: scenario.name,
682 step: `advance_clock:${step.awaitProviderSessionRelease}:no_provider_session_release`,
683 });
684 }
685 break;
686 }
687 case "await":
688 yield* awaitDispatch(step.key);
689 break;
690 case "await_all":
691 for (const key of Array.from(backgroundDispatches.keys())) {
692 yield* awaitDispatch(key);
693 }
694 break;
695 case "await_thread_idle":
696 yield* waitForThreadIdle(step.threadId);
697 break;
698 case "await_no_background_work":
699 yield* waitForNoBackgroundWork(step.threadId);
700 break;
701 case "await_run_steerable":
702 yield* waitForRunSteerable(step.threadId, step.runId);
703 break;
704 case "await_run_status":
705 yield* waitForRunStatus(step.threadId, step.runId, step.status);
706 break;
707 case "finish_held_run":
708 yield* finishHeldRun(step);
709 break;
710 case "await_run_turn_item":
711 yield* waitForRunTurnItem(
712 step.threadId,
713 step.runId,
714 step.itemType,
715 step.itemStatus,
716 step.inSubagentThreads === true,
717 );
718 break;
719 case "release_replay_gate_after_waiting":
720 yield* releaseReplayGateAfterWaiting(step.label, step.threadId, step.runId);
721 break;
722 case "release_replay_gate":
723 yield* releaseReplayGate(step.label);
724 break;
725 case "capture_shell_snapshot":
726 capturedShellSnapshots.set(step.key, yield* orchestrator.getShellSnapshot());
727 break;
728 case "respond_to_next_runtime_request": {
729 const request = yield* waitForPendingRuntimeRequest(step.threadId);
730 if (step.shellSnapshotKeyWhilePending !== undefined) {
731 capturedShellSnapshots.set(
732 step.shellSnapshotKeyWhilePending,
733 yield* orchestrator.getShellSnapshot(),
734 );
735 }
736 const result = yield* orchestrator.dispatch({
737 type: "runtime-request.respond",
738 commandId: step.commandId,
739 threadId: step.threadId,
740 requestId: request.id,
741 ...(step.decision === undefined ? {} : { decision: step.decision }),
742 ...(step.answers === undefined ? {} : { answers: step.answers }),
743 });
744 storedEventGroups.push(result.storedEvents);
745 break;
746 }
747 }
748 }
749​
750 for (const key of Array.from(backgroundDispatches.keys())) {
751 yield* awaitDispatch(key);
752 }
753 if (options.afterSteps !== undefined) {
754 yield* options.afterSteps;
755 }
756​
757 const shellSnapshot = yield* orchestrator.getShellSnapshot();
758 const projectionThreadIds = new Set(collectProjectionThreadIds(scenario));
759 for (const thread of shellSnapshot.threads) {
760 projectionThreadIds.add(thread.id);
761 }
762 const projections = new Map<ThreadId, OrchestrationV2ThreadProjection>();
763 for (const threadId of projectionThreadIds) {
764 projections.set(threadId, yield* orchestrator.getThreadProjection(threadId));
765 }
766​
767 yield* Effect.yieldNow;
768​
769 const observedEvents = yield* Ref.get(observedStoredEvents);
770 const storedEvents = (
771 observedEvents.length > 0 ? observedEvents : storedEventGroups.flat()
772 ).toSorted((left, right) => left.sequence - right.sequence);
773 return {
774 storedEvents,
775 domainEvents: storedEvents.map((stored) => stored.event),
776 projections,
777 shellSnapshot,
778 capturedShellSnapshots,
779 };
780 }),
781 ).pipe(
782 // Release held provider frames before shutdown so teardown cannot block on gates.
783 Effect.ensuring(Effect.sync(() => options.replayGate?.releaseAll())),
784 );