1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { it } from "@effect/vitest";
3import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts";
4import * as HostProcess from "@t3tools/shared/HostProcess";
5import { createModelSelection } from "@t3tools/shared/model";
6import * as Effect from "effect/Effect";
7import * as FileSystem from "effect/FileSystem";
8import * as Layer from "effect/Layer";
9import * as Path from "effect/Path";
10import * as Schema from "effect/Schema";
11import { expect } from "vite-plus/test";
12
13import * as ServerConfig from "../config.ts";
14import { processProbeSource } from "../testUtils/processProbe.ts";
15import {
16 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
17 SYNTHETIC_CLAUDE_COLLIDING_ALIAS,
18 SYNTHETIC_CLAUDE_MODEL_CATALOG,
19 SYNTHETIC_CLAUDE_STANDARD_MODEL,
20} from "../provider/ClaudeModelCatalog.testFixtures.ts";
21import * as TextGeneration from "./TextGeneration.ts";
22import { sanitizeThreadTitle } from "@t3tools/provider-core/server/textGenerationUtils";
23import { makeClaudeTextGeneration } from "./ClaudeTextGeneration.ts";
24import { writeFakeCli } from "@t3tools/provider-testing/fakeCli";
25const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings);
26
27const layerClaudeTextGenerationTest = ServerConfig.ServerConfig.layerTest(process.cwd(), {
28 prefix: "t3code-claude-text-generation-test-",
29}).pipe(Layer.provideMerge(NodeServices.layer));
30
31// The stub behaviour lives in Node so the same implementation runs on Windows,
32// where a shebang file is not executable and would fall through to the real
33// Claude CLI on PATH; `writeFakeCli` picks the launcher shape per host.
34function makeFakeClaudeBinary(dir: string, lifecyclePort?: number) {
35 return Effect.gen(function* () {
36 const path = yield* Path.Path;
37 const fs = yield* FileSystem.FileSystem;
38 const platform = yield* HostProcess.Platform;
39 const binDir = path.join(dir, "bin");
40 const fixturePath = yield* path.fromFileUrl(
41 new URL("./testing/ClaudeTextGeneration.fixture.mjs", import.meta.url),
42 );
43 const source = yield* fs.readFileString(fixturePath);
44 writeFakeCli({
45 directory: binDir,
46 name: "claude",
47 platform,
48 source:
49 lifecyclePort === undefined
50 ? source
51 : `process.on("SIGTERM", () => {}); ${processProbeSource(lifecyclePort)}`,
52 });
53 return binDir;
54 });
55}
56
57function withFakeClaudeEnv<A, E, R>(
58 input: {
59 output: string;
60 rejectsNoSessionPersistence?: boolean;
61 lifecyclePort?: number;
62 exitCode?: number;
63 stderr?: string;
64 argsMustContain?: string;
65 argsMustNotContain?: string;
66 stdinMustContain?: string;
67 configDirMustBe?: string;
68 cwdMustNotBe?: string;
69 claudeConfig?: Partial<ClaudeSettings>;
70 },
71 effectFn: (textGeneration: TextGeneration.TextGeneration["Service"]) => Effect.Effect<A, E, R>,
72) {
73 return Effect.gen(function* () {
74 const fs = yield* FileSystem.FileSystem;
75 const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-claude-text-" });
76 const binDir = yield* makeFakeClaudeBinary(tempDir, input.lifecyclePort);
77 const pathDelimiter = (yield* HostProcess.isWindows) ? ";" : ":";
78 const hostEnv = yield* HostProcess.Environment;
79 const fakeClaudeEnv: Record<string, string | undefined> = {
80 T3_FAKE_CLAUDE_OUTPUT: input.output,
81 T3_FAKE_CLAUDE_REJECTS_NO_SESSION_PERSISTENCE: input.rejectsNoSessionPersistence
82 ? "1"
83 : undefined,
84 T3_FAKE_CLAUDE_EXIT_CODE: input.exitCode === undefined ? undefined : String(input.exitCode),
85 T3_FAKE_CLAUDE_STDERR: input.stderr,
86 T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN: input.argsMustContain,
87 T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN: input.argsMustNotContain,
88 T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN: input.stdinMustContain,
89 T3_FAKE_CLAUDE_CWD_MUST_NOT_BE: input.cwdMustNotBe,
90 T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE: input.configDirMustBe,
91 };
92 const environment: NodeJS.ProcessEnv = {
93 ...Object.fromEntries(
94 Object.entries(hostEnv).filter(([name]) => !name.startsWith("T3_FAKE_CLAUDE_")),
95 ),
96 ...Object.fromEntries(
97 Object.entries(fakeClaudeEnv).filter(([, value]) => value !== undefined),
98 ),
99 PATH: `${binDir}${pathDelimiter}${hostEnv.PATH ?? ""}`,
100 };
101
102 const config = decodeClaudeSettings(input.claudeConfig ?? {});
103 const textGeneration = yield* makeClaudeTextGeneration(
104 config,
105 environment,
106 Effect.succeed(SYNTHETIC_CLAUDE_MODEL_CATALOG),
107 );
108 return yield* effectFn(textGeneration);
109 }).pipe(Effect.scoped);
110}
111
112it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => {
113 it.effect(
114 "keeps canonical built-in capabilities when a custom model collides with its alias",
115 () =>
116 withFakeClaudeEnv(
117 {
118 output: JSON.stringify({
119 structured_output: {
120 title: "Improve orchestration flow",
121 body: "Body",
122 },
123 }),
124 argsMustContain: `--model ${SYNTHETIC_CLAUDE_CAPABLE_MODEL}[expanded] --effort max --settings {"disableAllHooks":true,"fastMode":true}`,
125 claudeConfig: { customModels: [SYNTHETIC_CLAUDE_COLLIDING_ALIAS] },
126 },
127 (textGeneration) =>
128 Effect.gen(function* () {
129 const generated = yield* textGeneration.generatePrContent({
130 cwd: process.cwd(),
131 baseBranch: "main",
132 headBranch: "feature/claude-effect",
133 commitSummary: "Improve orchestration",
134 diffSummary: "1 file changed",
135 diffPatch: "diff --git a/README.md b/README.md",
136 modelSelection: {
137 ...createModelSelection(
138 ProviderInstanceId.make("claudeAgent"),
139 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
140 [
141 { id: "effort", value: "max" },
142 { id: "fastMode", value: true },
143 ],
144 ),
145 },
146 });
147
148 expect(generated.title).toBe("Improve orchestration flow");
149 }),
150 ),
151 );
152
153 it.effect(
154 "generates thread titles outside the project with tools, skills, and hooks disabled",
155 () =>
156 withFakeClaudeEnv(
157 {
158 output: JSON.stringify({
159 structured_output: {
160 title:
161 ' "Reconnect failures after restart because the session state does not recover" ',
162 },
163 }),
164 cwdMustNotBe: process.cwd(),
165 stdinMustContain: "/call-script",
166 },
167 (textGeneration) =>
168 Effect.gen(function* () {
169 const generated = yield* textGeneration.generateThreadTitle({
170 cwd: process.cwd(),
171 message: "/call-script",
172 modelSelection: {
173 instanceId: ProviderInstanceId.make("claudeAgent"),
174 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
175 },
176 });
177
178 expect(generated.title).toBe(
179 sanitizeThreadTitle(
180 '"Reconnect failures after restart because the session state does not recover"',
181 ),
182 );
183 }),
184 ),
185 );
186
187 it.effect("sends thread title instructions to the provider prompt", () =>
188 withFakeClaudeEnv(
189 {
190 output: JSON.stringify({ structured_output: { title: "Reconnect failures" } }),
191 stdinMustContain: "Additional instructions:\nUse two to four words.",
192 },
193 (textGeneration) =>
194 textGeneration.generateThreadTitle({
195 cwd: process.cwd(),
196 message: "Investigate reconnect failures",
197 instructions: "Use two to four words.",
198 modelSelection: {
199 instanceId: ProviderInstanceId.make("claudeAgent"),
200 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
201 },
202 }),
203 ),
204 );
205
206 it.effect("runs Claude text generation with the configured CLAUDE_CONFIG_DIR", () =>
207 Effect.gen(function* () {
208 const path = yield* Path.Path;
209 const claudeConfigDir = path.join(process.cwd(), ".claude-work-test");
210 return yield* withFakeClaudeEnv(
211 {
212 output: JSON.stringify({
213 structured_output: {
214 title: "Use Claude home",
215 },
216 }),
217 configDirMustBe: claudeConfigDir,
218 claudeConfig: { homePath: claudeConfigDir },
219 },
220 (textGeneration) =>
221 Effect.gen(function* () {
222 const generated = yield* textGeneration.generateThreadTitle({
223 cwd: process.cwd(),
224 message: "thread title",
225 modelSelection: {
226 instanceId: ProviderInstanceId.make("claudeAgent"),
227 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
228 },
229 });
230
231 expect(generated.title).toBe(sanitizeThreadTitle("Use Claude home"));
232 }),
233 );
234 }),
235 );
236
237 it.effect("retries without --no-session-persistence when the Claude CLI rejects it", () =>
238 withFakeClaudeEnv(
239 {
240 output: JSON.stringify({ structured_output: { title: "Reconnect failures" } }),
241 rejectsNoSessionPersistence: true,
242 },
243 (textGeneration) =>
244 Effect.gen(function* () {
245 const generated = yield* textGeneration.generateThreadTitle({
246 cwd: process.cwd(),
247 message: "Investigate reconnect failures",
248 modelSelection: {
249 instanceId: ProviderInstanceId.make("claudeAgent"),
250 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
251 },
252 });
253
254 expect(generated.title).toBe("Reconnect failures");
255 }),
256 ),
257 );
258
259 it.effect.each([
260 { verbose: false, label: "normal" },
261 { verbose: true, label: "verbose" },
262 ])("unwraps a JSON title in $label Claude output", ({ verbose }) => {
263 const result = {
264 type: "result",
265 structured_output: { title: '{"title": "Refresh ev-stg APP ASG instances"}' },
266 };
267 return withFakeClaudeEnv(
268 { output: JSON.stringify(verbose ? [result] : result) },
269 (textGeneration) =>
270 Effect.gen(function* () {
271 const generated = yield* textGeneration.generateThreadTitle({
272 cwd: process.cwd(),
273 message: "Refresh ev-stg APP ASG instances",
274 modelSelection: {
275 instanceId: ProviderInstanceId.make("claudeAgent"),
276 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
277 },
278 });
279
280 expect(generated.title).toBe("Refresh ev-stg APP ASG instances");
281 }),
282 );
283 });
284
285 it.effect.each([
286 { previousTitle: undefined, label: "generating" },
287 { previousTitle: "Old thread title", label: "regenerating" },
288 ])("reads the result from verbose Claude output when $label a title", ({ previousTitle }) =>
289 withFakeClaudeEnv(
290 {
291 output: JSON.stringify([
292 { type: "system", subtype: "init" },
293 { type: "assistant", message: { content: [] } },
294 { type: "user", message: { content: [] } },
295 { type: "rate_limit_event" },
296 {
297 type: "result",
298 subtype: "success",
299 result: '{"title":"Refresh ev-stg APP ASG Instances"}',
300 structured_output: { title: "Refresh ev-stg APP ASG Instances" },
301 },
302 ]),
303 },
304 (textGeneration) =>
305 Effect.gen(function* () {
306 const generated = yield* textGeneration.generateThreadTitle({
307 cwd: process.cwd(),
308 message: "Refresh ev-stg APP ASG instances",
309 previousTitle,
310 modelSelection: {
311 instanceId: ProviderInstanceId.make("claudeAgent"),
312 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
313 },
314 });
315
316 expect(generated.title).toBe("Refresh ev-stg APP ASG Instances");
317 }),
318 ),
319 );
320
321 it.effect.each([
322 ["empty message array", []],
323 ["missing result", [{ type: "assistant", structured_output: { title: "Not a result" } }]],
324 ["invalid title", [{ type: "result", structured_output: { title: 42 } }]],
325 [
326 "final result without structured output",
327 [
328 { type: "result", structured_output: { title: "Earlier result" } },
329 { type: "result", subtype: "error_max_structured_output_retries" },
330 ],
331 ],
332 ] as const)("rejects verbose Claude output with %s", ([, output]) =>
333 withFakeClaudeEnv({ output: JSON.stringify(output) }, (textGeneration) =>
334 Effect.gen(function* () {
335 const error = yield* Effect.flip(
336 textGeneration.generateThreadTitle({
337 cwd: process.cwd(),
338 message: "Name this thread",
339 modelSelection: {
340 instanceId: ProviderInstanceId.make("claudeAgent"),
341 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
342 },
343 }),
344 );
345
346 expect(error._tag).toBe("TextGenerationError");
347 }),
348 ),
349 );
350
351 const apiError = "API Error: 400 Model blocked by guardrail: 4 endpoints excluded";
352 const wrapperNotice = "Using the OpenRouter credential from the global credential file.";
353 const structuredOutputError = "Failed to provide valid structured output after 5 attempts";
354 const structuredOutputFailure = {
355 type: "result",
356 subtype: "error_max_structured_output_retries",
357 is_error: true,
358 errors: [structuredOutputError],
359 };
360 it.effect.each([
361 [
362 "the errors list of a result with no result text",
363 JSON.stringify(structuredOutputFailure),
364 structuredOutputError,
365 ],
366 [
367 "the errors list from verbose output",
368 JSON.stringify([{ type: "system", subtype: "init" }, structuredOutputFailure]),
369 structuredOutputError,
370 ],
371 [
372 "the errors list without CLI telemetry",
373 JSON.stringify({
374 ...structuredOutputFailure,
375 subtype: "error_during_execution",
376 errors: ["[ede_diagnostic] result_type=user last_content_type=n/a", apiError],
377 }),
378 apiError,
379 ],
380 [
381 "the stdout error over a stderr notice",
382 JSON.stringify({ is_error: true, api_error_status: 400, result: apiError }),
383 apiError,
384 ],
385 [
386 "the verbose stdout error over a stderr notice",
387 JSON.stringify([
388 { type: "system", subtype: "init" },
389 { type: "result", is_error: true, result: apiError },
390 { type: "result", is_error: true, result: " " },
391 ]),
392 apiError,
393 ],
394 ["stderr when stdout has no error", "not json", wrapperNotice],
395 [
396 "a stdout error naming --no-session-persistence without retrying",
397 JSON.stringify({
398 is_error: true,
399 result: "error: unknown option '--no-session-persistence'",
400 }),
401 "error: unknown option '--no-session-persistence'",
402 ],
403 ] as const)("reports %s when the Claude CLI fails", ([, output, expected]) =>
404 withFakeClaudeEnv({ output, stderr: wrapperNotice, exitCode: 1 }, (textGeneration) =>
405 Effect.gen(function* () {
406 const error = yield* Effect.flip(
407 textGeneration.generateThreadTitle({
408 cwd: process.cwd(),
409 message: "Name this thread",
410 modelSelection: {
411 instanceId: ProviderInstanceId.make("claudeAgent"),
412 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
413 },
414 }),
415 );
416
417 expect(error.detail).toBe(`Claude CLI command failed: ${expected}`);
418 }),
419 ),
420 );
421
422 it.effect("does not report the raw result JSON when a failed run writes nothing to stderr", () =>
423 withFakeClaudeEnv(
424 { output: JSON.stringify(structuredOutputFailure), exitCode: 1 },
425 (textGeneration) =>
426 Effect.gen(function* () {
427 const error = yield* Effect.flip(
428 textGeneration.generateThreadTitle({
429 cwd: process.cwd(),
430 message: "Name this thread",
431 modelSelection: {
432 instanceId: ProviderInstanceId.make("claudeAgent"),
433 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
434 },
435 }),
436 );
437
438 expect(error.detail).toBe(`Claude CLI command failed: ${structuredOutputError}`);
439 }),
440 ),
441 );
442});