1import type {
2 NodeId,
3 OrchestrationV2ProviderFailure,
4 OrchestrationV2ProviderFailureClass,
5 OrchestrationV2ProviderRetry,
6 OrchestrationV2TurnItem,
7 ProviderDriverKind,
8 ProviderThreadId,
9 ProviderTurnId,
10 RunId,
11 ThreadId,
12} from "@t3tools/contracts";
13import * as DateTime from "effect/DateTime";
14import * as Cause from "effect/Cause";
15
16import * as IdAllocator from "./IdAllocator.ts";
17import type { ProviderAdapterFailureReason } from "./ProviderAdapter.ts";
18import * as Schema from "effect/Schema";
19
20export class ContextHandoffBudgetError extends Schema.TaggedError<ContextHandoffBudgetError>()(
21 "ContextHandoffBudgetError",
22 {},
23) {
24 override get message() {
25 return "The request leaves too little context for a provider handoff. Shorten the request or use a larger-context model; the current request has not been truncated.";
26 }
27}
28
29export class ContextHandoffBackgroundWorkError extends Schema.TaggedError<ContextHandoffBackgroundWorkError>()(
30 "ContextHandoffBackgroundWorkError",
31 {},
32) {
33 override get message() {
34 return "The target provider still has background work running. Wait for it to finish or stop it before retrying the handoff.";
35 }
36}
37
38export const MAX_PROVIDER_FAILURE_MESSAGE_LENGTH = 4_096;
39const MAX_PROVIDER_FAILURE_CODE_LENGTH = 128;
40
41const DEFAULT_PROVIDER_FAILURE_MESSAGE = "Provider turn failed.";
42
43const PROVIDER_FAILURE_CATEGORIES: Partial<
44 Record<string, { readonly summary: string; readonly guidance: string }>
45> = {
46 ProviderAdapterTurnStartError: {
47 summary: "The provider could not start this turn",
48 guidance: "Retry the turn; if it keeps failing, check the provider setup and server logs.",
49 },
50 ProviderAdapterEventStreamError: {
51 summary: "The provider event stream closed unexpectedly",
52 guidance: "Retry the turn; if it keeps failing, check the provider and server logs.",
53 },
54 ProviderAdapterOpenSessionError: {
55 summary: "The provider session could not be opened",
56 guidance: "Check that the provider is installed and signed in, then retry the turn.",
57 },
58 ProviderAdapterResumeThreadError: {
59 summary: "The provider conversation could not be resumed",
60 guidance: "Retry the turn; if it keeps failing, check the provider and server logs.",
61 },
62};
63
64// codex_protocol::protocol::validate_thread_goal_objective
65const CODEX_GOAL_VALIDATION_MESSAGE =
66 /^goal objective must (?:not be empty|be at most \d{1,6} characters)$/u;
67
68// codex_config::ConstraintError::InvalidValue for a managed requirements.toml or cloud policy.
69const CODEX_ACCESS_REQUIREMENT_REJECTION =
70 /invalid value for `(approval_policy|sandbox_mode)`: `([A-Za-z]{1,32})` is not in the allowed set /u;
71
72const PROVIDER_FAILURE_REASONS: Record<ProviderAdapterFailureReason, string> = {
73 previous_turn_active: "the previous turn is still running",
74 attachment_unreadable: "an attachment could not be read",
75};
76
77/**
78 * Walks a cause chain for curated text without reading arbitrary provider
79 * defect text. `known` is the message of a domain error T3 raised itself;
80 * `category` is the generic message of the innermost provider wrapper seen.
81 */
82function curatedCauseMessages(cause: unknown): {
83 readonly known: string | undefined;
84 readonly category: string | undefined;
85} {
86 const seen = new Set<unknown>();
87 let category: (typeof PROVIDER_FAILURE_CATEGORIES)[string];
88 let reason: string | undefined;
89 const categoryMessage = () => {
90 if (category === undefined) return undefined;
91 return reason
92 ? `${category.summary}: ${reason}. ${category.guidance}`
93 : `${category.summary}. ${category.guidance}`;
94 };
95 for (let depth = 0; depth < 16 && cause != null && !seen.has(cause); depth++) {
96 seen.add(cause);
97 try {
98 if (Cause.isCause(cause)) {
99 cause = Cause.squash(cause);
100 continue;
101 }
102 if (typeof cause !== "object") break;
103 const tag = (cause as Record<string, unknown>)._tag;
104 switch (tag) {
105 case "ContextHandoffBudgetError":
106 return { known: new ContextHandoffBudgetError().message, category: categoryMessage() };
107 case "ContextHandoffBackgroundWorkError":
108 return {
109 known: new ContextHandoffBackgroundWorkError().message,
110 category: categoryMessage(),
111 };
112 case "AcpModelSelectionError":
113 case "ClaudeBackgroundWorkBlocksQueryReplacementError":
114 case "ClaudeAppendSystemPromptFileError":
115 case "ClaudeAppendSystemPromptMissingValueError":
116 case "ClaudeSettingsLaunchArgError":
117 case "ClaudeMcpRegistrationTimeoutError":
118 case "ClaudeCodeRootRefusalError":
119 case "CursorModelUnavailableError":
120 case "CodexGoalObjectiveTooLongError":
121 case "ProviderSetupError":
122 case "ProviderAdapterThreadInUseError":
123 case "ProviderCompactionUnsupportedError":
124 return { known: stringField(cause, "message"), category: categoryMessage() };
125 case "CodexAppServerRequestError": {
126 // Only Codex's fixed-shape goal validation and managed-requirements
127 // text is actionable and safe to show; other request error text stays hidden.
128 const rejection = stringField(cause, "message");
129 if (
130 (cause as Record<string, unknown>).method === "thread/goal/set" &&
131 rejection !== undefined &&
132 CODEX_GOAL_VALIDATION_MESSAGE.test(rejection)
133 ) {
134 return { known: `Codex rejected the goal: ${rejection}.`, category: categoryMessage() };
135 }
136 const access =
137 rejection === undefined ? null : CODEX_ACCESS_REQUIREMENT_REJECTION.exec(rejection);
138 if (access !== null) {
139 return {
140 known: `Codex's managed requirements do not allow \`${access[1]}\` \`${access[2]}\`, which this thread's access mode uses. Switch the thread to an access mode the policy allows.`,
141 category: categoryMessage(),
142 };
143 }
144 break;
145 }
146 case "AcpProcessExitedError": {
147 const exit = `provider process exited${processExitStatus(cause)}`;
148 const stage = category === undefined ? `The ${exit}` : `${category.summary}: the ${exit}`;
149 return {
150 known: `${stage}. Check the server logs for the provider's output, then retry the turn.`,
151 category: categoryMessage(),
152 };
153 }
154 case "ContextHandoffDeliveryUncertainError":
155 return {
156 known:
157 "T3 could not confirm whether conversation history reached the provider. Retry the turn to recover the session.",
158 category: categoryMessage(),
159 };
160 case "ProviderAdapterProtocolError": {
161 const typedReason = (cause as Record<string, unknown>).reason;
162 if (
163 category !== undefined &&
164 typeof typedReason === "string" &&
165 Object.hasOwn(PROVIDER_FAILURE_REASONS, typedReason)
166 ) {
167 reason ??= PROVIDER_FAILURE_REASONS[typedReason as ProviderAdapterFailureReason];
168 }
169 break;
170 }
171 }
172 if (typeof tag === "string" && Object.hasOwn(PROVIDER_FAILURE_CATEGORIES, tag)) {
173 category = PROVIDER_FAILURE_CATEGORIES[tag];
174 reason = undefined;
175 }
176 cause = (cause as Record<string, unknown>).cause;
177 } catch {
178 break;
179 }
180 }
181 return { known: undefined, category: categoryMessage() };
182}
183
184// Only the structured status; stderr and the spawner cause can carry provider output.
185function processExitStatus(exited: object): string {
186 const { signal, code } = exited as Record<string, unknown>;
187 if (typeof signal === "string" && /^SIG[A-Z0-9]{1,16}$/u.test(signal)) {
188 return ` after signal ${signal}`;
189 }
190 return typeof code === "number" && Number.isSafeInteger(code) ? ` with code ${code}` : "";
191}
192
193/**
194 * The curated message of a domain error in a failure's cause chain, if any.
195 * Callers that pass their own `message` to `makeProviderFailure` check it
196 * first, so a known reason is not hidden behind a wrapper's text.
197 */
198export function knownProviderFailureMessage(cause: unknown): string | undefined {
199 return curatedCauseMessages(cause).known;
200}
201
202function stringField(value: unknown, key: "message" | "code"): string | undefined {
203 if (typeof value !== "object" || value === null) return undefined;
204 try {
205 const candidate = (value as Record<string, unknown>)[key];
206 return typeof candidate === "string" ? candidate : undefined;
207 } catch {
208 return undefined;
209 }
210}
211
212function redactUrl(match: string): string {
213 const trailing = /[),.;!?]+$/u.exec(match)?.[0] ?? "";
214 const candidate = trailing.length === 0 ? match : match.slice(0, -trailing.length);
215 try {
216 const url = new URL(candidate);
217 url.username = "";
218 url.password = "";
219 url.search = "";
220 url.hash = "";
221 return `${url.toString()}${trailing}`;
222 } catch {
223 return "[REDACTED_URL]";
224 }
225}
226
227function replaceUnsafeControlCharacters(value: string): string {
228 const sanitized: Array<string> = [];
229 for (const character of value) {
230 const codePoint = character.codePointAt(0) ?? 0;
231 sanitized.push(
232 codePoint <= 0x08 ||
233 (codePoint >= 0x0b && codePoint <= 0x0c) ||
234 (codePoint >= 0x0e && codePoint <= 0x1f) ||
235 codePoint === 0x7f
236 ? " "
237 : character,
238 );
239 }
240 return sanitized.join("");
241}
242
243function redactProviderFailureText(value: string): string {
244 return replaceUnsafeControlCharacters(value)
245 .replace(/\bhttps?:\/\/[^\s<>"']+/giu, redactUrl)
246 .replace(/\b(Bearer|Basic)\s+[^\s,;]+/giu, "$1 [REDACTED]")
247 .replace(
248 /(["'][\w-]*(?:access[_-]?token|api[_-]?key|authorization|credential|password|secret|token)["']\s*:\s*["'])[^"']*(["'])/giu,
249 "$1[REDACTED]$2",
250 )
251 .replace(
252 /((?<![\w-])[\w-]*(?:access[_-]?token|api[_-]?key|authorization|credential|password|secret|token)\b\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;]+)/giu,
253 "$1[REDACTED]",
254 )
255 .replace(/\bsk-[A-Za-z0-9_-]{16,}\b/gu, "[REDACTED]")
256 .trim();
257}
258
259function boundedText(value: string, maxLength: number): string {
260 const redacted = redactProviderFailureText(value);
261 if (redacted.length <= maxLength) return redacted;
262 let end = Math.max(0, maxLength - 1);
263 const finalCodeUnit = redacted.charCodeAt(end - 1);
264 if (finalCodeUnit >= 0xd800 && finalCodeUnit <= 0xdbff) {
265 end -= 1;
266 }
267 return `${redacted.slice(0, end)}…`;
268}
269
270export function makeProviderFailure(input: {
271 readonly cause?: unknown;
272 readonly message?: string | undefined;
273 readonly code?: string | null | undefined;
274 readonly class?: OrchestrationV2ProviderFailureClass;
275 readonly retryable?: boolean | null;
276 readonly resetAt?: string | null;
277}): OrchestrationV2ProviderFailure {
278 const curated = curatedCauseMessages(input.cause);
279 const rawMessage =
280 input.message ?? curated.known ?? curated.category ?? DEFAULT_PROVIDER_FAILURE_MESSAGE;
281 const message = boundedText(rawMessage, MAX_PROVIDER_FAILURE_MESSAGE_LENGTH);
282 const rawCode = input.code ?? stringField(input.cause, "code") ?? null;
283 const code =
284 rawCode === null ? null : boundedText(rawCode, MAX_PROVIDER_FAILURE_CODE_LENGTH) || null;
285
286 return {
287 class: input.class ?? "unknown",
288 message: message || DEFAULT_PROVIDER_FAILURE_MESSAGE,
289 code,
290 retryable: input.retryable ?? null,
291 ...(input.class === "usage_limit" &&
292 input.resetAt != null &&
293 Number.isFinite(Date.parse(input.resetAt))
294 ? { resetAt: DateTime.formatIso(DateTime.makeUnsafe(input.resetAt)) }
295 : {}),
296 };
297}
298
299export function makeProviderFailureTurnItem(input: {
300 readonly driver: ProviderDriverKind;
301 readonly threadId: ThreadId;
302 readonly runId: RunId | null;
303 readonly nodeId: NodeId | null;
304 readonly providerThreadId: ProviderThreadId;
305 readonly providerTurnId: ProviderTurnId;
306 readonly itemOrdinal: number;
307 readonly failure: OrchestrationV2ProviderFailure;
308 readonly retry?: OrchestrationV2ProviderRetry;
309 readonly retryStartedAt?: DateTime.Utc;
310 readonly occurredAt: DateTime.Utc;
311}): Extract<OrchestrationV2TurnItem, { readonly type: "error" }> {
312 return {
313 id: IdAllocator.derive.turnItemFromProviderItem({
314 driver: input.driver,
315 nativeItemId: `terminal-failure:${input.providerTurnId}`,
316 }),
317 threadId: input.threadId,
318 runId: input.runId,
319 nodeId: input.nodeId,
320 providerThreadId: input.providerThreadId,
321 providerTurnId: input.providerTurnId,
322 nativeItemRef: null,
323 parentItemId: null,
324 ordinal: input.itemOrdinal,
325 status: "failed",
326 title: input.failure.class === "usage_limit" ? "Usage limit reached" : "Provider error",
327 startedAt: input.retryStartedAt ?? input.occurredAt,
328 completedAt: input.occurredAt,
329 updatedAt: input.occurredAt,
330 type: "error",
331 failure: input.failure,
332 ...(input.retry === undefined ? {} : { retry: input.retry }),
333 };
334}
335
336export function makeProviderRetryTurnItem(input: {
337 readonly driver: ProviderDriverKind;
338 readonly threadId: ThreadId;
339 readonly runId: RunId | null;
340 readonly nodeId: NodeId | null;
341 readonly providerThreadId: ProviderThreadId;
342 readonly providerTurnId: ProviderTurnId;
343 readonly itemOrdinal: number;
344 readonly failure: OrchestrationV2ProviderFailure;
345 readonly retry: OrchestrationV2ProviderRetry;
346 readonly status: Extract<
347 OrchestrationV2TurnItem["status"],
348 "running" | "completed" | "failed" | "interrupted" | "cancelled"
349 >;
350 readonly startedAt: DateTime.Utc;
351 readonly updatedAt: DateTime.Utc;
352}): Extract<OrchestrationV2TurnItem, { readonly type: "error" }> {
353 const completed = input.status !== "running";
354 let title = "Provider retry";
355 if (input.status === "completed") {
356 title = "Provider recovered";
357 } else if (input.status === "failed") {
358 title = input.failure.class === "usage_limit" ? "Usage limit reached" : "Provider error";
359 } else if (input.status === "interrupted" || input.status === "cancelled") {
360 title = "Provider retry stopped";
361 }
362 return {
363 id: IdAllocator.derive.turnItemFromProviderItem({
364 driver: input.driver,
365 nativeItemId: `terminal-failure:${input.providerTurnId}`,
366 }),
367 threadId: input.threadId,
368 runId: input.runId,
369 nodeId: input.nodeId,
370 providerThreadId: input.providerThreadId,
371 providerTurnId: input.providerTurnId,
372 nativeItemRef: null,
373 parentItemId: null,
374 ordinal: input.itemOrdinal,
375 status: input.status,
376 title,
377 startedAt: input.startedAt,
378 completedAt: completed ? input.updatedAt : null,
379 updatedAt: input.updatedAt,
380 type: "error",
381 failure: input.failure,
382 retry: input.retry,
383 };
384}