Open History to see the last commit
1#!/usr/bin/env node
2
3/**
4 * CLI for testing the TanStack AI Code Mode Skills system
5 *
6 * Commands:
7 * - run: Run tests across multiple adapters (like smoke-tests)
8 * - list: List available adapters and tests
9 * - live: Run test with a real LLM (single adapter, legacy)
10 * - simulated: Run deterministic test with mock adapter
11 * - structured: Run structured output test (legacy)
12 */
13
14import { dirname, join } from 'node:path'
15import { fileURLToPath } from 'node:url'
16import { config } from 'dotenv'
17import { Command } from 'commander'
18import { runSimulatedTest } from './simulated-test'
19import { runRegistryTest } from './registry-test'
20import { runLiveTest } from './live-test'
21import { runStructuredOutputTest } from './structured-output-test'
22import { ADAPTERS, getAdapter } from './adapters'
23import { TESTS, getTest, getDefaultTests } from './tests'
24import type { AdapterDefinition, AdapterSet } from './adapters'
25import type { TestDefinition, TestOutcome } from './tests'
26import { colors, logError, logInfo } from './test-utils'
27
28// Load environment variables from test-cli directory
29const __dirname = dirname(fileURLToPath(import.meta.url))
30const envPath = join(__dirname, '.env.local')
31const result = config({ path: envPath })
32if (result.parsed) {
33 // Manually set process.env from parsed values (dotenv may not set them in ESM)
34 for (const [key, value] of Object.entries(result.parsed)) {
35 process.env[key] = value
36 }
37}
38config({ path: join(__dirname, '.env') })
39
40interface AdapterResult {
41 adapter: string
42 tests: Record<string, TestOutcome>
43}
44
45interface TestTask {
46 adapterDef: AdapterDefinition | null // null for simulated test
47 adapterSet: AdapterSet | null
48 test: TestDefinition
49 verbose: boolean
50}
51
52/**
53 * Get the display width of a string, accounting for emojis
54 */
55function displayWidth(str: string): number {
56 const emojiRegex =
57 /[\u{1F300}-\u{1F9FF}]|[\u{2600}-\u{26FF}]|[\u{2700}-\u{27BF}]|✅|❌|⚠️/gu
58 const emojiCount = (str.match(emojiRegex) || []).length
59 return str.length + emojiCount
60}
61
62/**
63 * Pad a string to a display width, accounting for emojis
64 */
65function padEnd(str: string, width: number): string {
66 const currentWidth = displayWidth(str)
67 const padding = Math.max(0, width - currentWidth)
68 return str + ' '.repeat(padding)
69}
70
71/**
72 * List available adapters and/or tests
73 */
74function listCommand(options: { adapters?: boolean; tests?: boolean }) {
75 const showAll = !options.adapters && !options.tests
76
77 if (showAll || options.adapters) {
78 console.log('\n📦 Available Adapters:\n')
79 console.log(' ID Name Env Key Status')
80 console.log(' ---------- ---------- ------------------- ------')
81 for (const adapter of ADAPTERS) {
82 const envValue = process.env[adapter.envKey]
83 const status = envValue ? '✅ Ready' : '⚠️ Missing env'
84
85 console.log(
86 ` ${adapter.id.padEnd(10)} ${adapter.name.padEnd(10)} ${adapter.envKey.padEnd(19)} ${status}`,
87 )
88 }
89 }
90
91 if (showAll || options.tests) {
92 console.log('\n🧪 Available Tests:\n')
93 console.log(' ID Name Requires Adapter Description')
94 console.log(' --- -------------------- ---------------- -----------')
95 for (const test of TESTS) {
96 const requires = test.requiresAdapter ? 'Yes' : 'No (mock)'
97 console.log(
98 ` ${test.id} ${test.name.padEnd(20)} ${requires.padEnd(16)} ${test.description}`,
99 )
100 }
101 }
102
103 console.log('')
104}
105
106/**
107 * Format the results grid with proper emoji alignment
108 */
109function formatGrid(results: AdapterResult[], testsRun: TestDefinition[]) {
110 const headers = ['Adapter', ...testsRun.map((t) => t.id)]
111
112 // Build rows with result indicators
113 const rows = results.map((result) => [
114 result.adapter,
115 ...testsRun.map((test) => {
116 const outcome = result.tests[test.id]
117 if (!outcome) return '—'
118 if (outcome.ignored) return '—'
119 return outcome.passed ? '✅' : '❌'
120 }),
121 ])
122
123 // Calculate column widths based on display width
124 const colWidths = headers.map((header, index) => {
125 const headerWidth = displayWidth(header)
126 const maxCellWidth = Math.max(
127 ...rows.map((row) => displayWidth(row[index] || '')),
128 )
129 return Math.max(headerWidth, maxCellWidth)
130 })
131
132 const separator = colWidths.map((w) => '-'.repeat(w)).join('-+-')
133 const formatRow = (row: string[]) =>
134 row.map((cell, idx) => padEnd(cell, colWidths[idx]!)).join(' | ')
135
136 console.log(formatRow(headers))
137 console.log(separator)
138 rows.forEach((row) => console.log(formatRow(row)))
139}
140
141/**
142 * Clear the current line and move cursor to beginning
143 */
144function clearLine() {
145 process.stdout.write('\r\x1b[K')
146}
147
148/**
149 * Update progress display
150 */
151function updateProgress(
152 completed: number,
153 total: number,
154 running: string[],
155 failed: number,
156) {
157 clearLine()
158 const runningStr =
159 running.length > 0 ? ` | Running: ${running.join(', ')}` : ''
160 const failedStr = failed > 0 ? ` | ❌ ${failed} failed` : ''
161 process.stdout.write(
162 `⏳ Progress: ${completed}/${total} completed${failedStr}${runningStr}`,
163 )
164}
165
166/**
167 * Run tests in parallel with progress display
168 */
169async function runParallel(
170 adaptersToRun: AdapterDefinition[],
171 testsToRun: TestDefinition[],
172 concurrency: number,
173 verbose: boolean,
174): Promise<AdapterResult[]> {
175 // Build task queue
176 const tasks: TestTask[] = []
177 const resultsMap = new Map<string, AdapterResult>()
178 const skippedAdapters: string[] = []
179
180 // Handle tests that don't need an adapter (mock/simulated tests)
181 const mockTests = testsToRun.filter((t) => !t.requiresAdapter)
182 if (mockTests.length > 0) {
183 // Add Mock adapter result for simulated tests
184 const mockResult: AdapterResult = {
185 adapter: 'Mock',
186 tests: {},
187 }
188 resultsMap.set('mock', mockResult)
189 for (const mockTest of mockTests) {
190 tasks.push({
191 adapterDef: null,
192 adapterSet: null,
193 test: mockTest,
194 verbose,
195 })
196 }
197 }
198
199 // Handle tests that require adapters
200 const adapterTests = testsToRun.filter((t) => t.requiresAdapter)
201
202 for (const adapterDef of adaptersToRun) {
203 const adapterSet = await adapterDef.create()
204
205 if (!adapterSet) {
206 skippedAdapters.push(`${adapterDef.name} (${adapterDef.envKey} not set)`)
207 continue
208 }
209
210 // Initialize result for this adapter
211 const adapterResult: AdapterResult = {
212 adapter: adapterDef.name,
213 tests: {},
214 }
215 resultsMap.set(adapterDef.id, adapterResult)
216
217 for (const test of adapterTests) {
218 tasks.push({ adapterDef, adapterSet, test, verbose })
219 }
220 }
221
222 // Show skipped adapters
223 if (skippedAdapters.length > 0) {
224 console.log(`⚠️ Skipping: ${skippedAdapters.join(', ')}`)
225 }
226
227 const total = tasks.length
228 let completed = 0
229 let failed = 0
230 const running = new Set<string>()
231 const failedTests: Array<{ name: string; error: string }> = []
232
233 // Show initial progress
234 console.log(
235 `\n🔄 Running ${total} tests with ${concurrency} parallel workers\n`,
236 )
237 updateProgress(completed, total, Array.from(running), failed)
238
239 // Suppress console.log during parallel execution
240 const originalLog = console.log
241 console.log = () => {}
242
243 // Process tasks with limited concurrency
244 const taskQueue = [...tasks]
245
246 async function runTask(task: TestTask): Promise<void> {
247 const taskName = task.adapterDef
248 ? `${task.adapterDef.name}/${task.test.id}`
249 : `Mock/${task.test.id}`
250 running.add(taskName)
251 updateProgress(completed, total, Array.from(running), failed)
252
253 try {
254 const adapter = task.adapterSet?.adapter || null
255 const outcome = await task.test.run(adapter, task.verbose)
256
257 const resultKey = task.adapterDef ? task.adapterDef.id : 'mock'
258 const adapterResult = resultsMap.get(resultKey)!
259 adapterResult.tests[task.test.id] = outcome
260
261 if (!outcome.passed && !outcome.ignored) {
262 failed++
263 failedTests.push({
264 name: taskName,
265 error: outcome.error || 'Unknown error',
266 })
267 }
268 } catch (error: any) {
269 const resultKey = task.adapterDef ? task.adapterDef.id : 'mock'
270 const adapterResult = resultsMap.get(resultKey)!
271 const errorMsg = error?.message || String(error)
272 adapterResult.tests[task.test.id] = { passed: false, error: errorMsg }
273 failed++
274 failedTests.push({ name: taskName, error: errorMsg })
275 }
276
277 running.delete(taskName)
278 completed++
279 updateProgress(completed, total, Array.from(running), failed)
280 }
281
282 // Run with concurrency limit
283 const workers: Promise<void>[] = []
284
285 async function worker() {
286 while (taskQueue.length > 0) {
287 const task = taskQueue.shift()
288 if (task) {
289 await runTask(task)
290 }
291 }
292 }
293
294 // Start workers
295 for (let i = 0; i < Math.min(concurrency, tasks.length); i++) {
296 workers.push(worker())
297 }
298
299 // Wait for all workers to complete
300 await Promise.all(workers)
301
302 // Restore console.log
303 console.log = originalLog
304
305 // Clear progress line and show completion
306 clearLine()
307 console.log(`✅ Completed ${total} tests (${failed} failed)\n`)
308
309 // Show failed tests summary
310 if (failedTests.length > 0) {
311 console.log('Failed tests:')
312 for (const ft of failedTests) {
313 console.log(` ❌ ${ft.name}: ${ft.error}`)
314 }
315 console.log('')
316 }
317
318 // Return results in order: Mock first (if exists), then adapters
319 const orderedResults: AdapterResult[] = []
320 const mockResult = resultsMap.get('mock')
321 if (mockResult) {
322 orderedResults.push(mockResult)
323 }
324 for (const adapterDef of adaptersToRun) {
325 const result = resultsMap.get(adapterDef.id)
326 if (result) {
327 orderedResults.push(result)
328 }
329 }
330 return orderedResults
331}
332
333/**
334 * Run tests with optional filtering
335 */
336async function runCommand(options: {
337 adapters?: string
338 tests?: string
339 parallel?: string
340 verbose?: boolean
341}) {
342 console.log(`${colors.bright}${colors.cyan}`)
343 console.log('╔═══════════════════════════════════════════════════════════╗')
344 console.log('║ TanStack AI Code Mode Skills - Multi-Adapter Tests ║')
345 console.log('╚═══════════════════════════════════════════════════════════╝')
346 console.log(colors.reset)
347
348 // Parse adapter filter
349 const adapterFilter = options.adapters
350 ? options.adapters.split(',').map((a) => a.trim().toLowerCase())
351 : null
352
353 // Parse test filter
354 const testFilter = options.tests
355 ? options.tests.split(',').map((t) => t.trim().toUpperCase())
356 : null
357
358 // Parse parallel option (default to 3)
359 const parallel = options.parallel ? parseInt(options.parallel, 10) : 3
360 const verbose = options.verbose || false
361
362 // Determine which adapters to run
363 const adaptersToRun = adapterFilter
364 ? ADAPTERS.filter((a) => adapterFilter.includes(a.id.toLowerCase()))
365 : ADAPTERS
366
367 // Validate adapter filter
368 if (adapterFilter) {
369 for (const id of adapterFilter) {
370 if (!getAdapter(id)) {
371 console.error(`❌ Unknown adapter: "${id}"`)
372 console.error(
373 ` Valid adapters: ${ADAPTERS.map((a) => a.id).join(', ')}`,
374 )
375 process.exit(1)
376 }
377 }
378 }
379
380 // Determine which tests to run
381 let testsToRun: TestDefinition[]
382 if (testFilter) {
383 testsToRun = []
384 for (const id of testFilter) {
385 const test = getTest(id)
386 if (!test) {
387 console.error(`❌ Unknown test: "${id}"`)
388 console.error(` Valid tests: ${TESTS.map((t) => t.id).join(', ')}`)
389 process.exit(1)
390 }
391 testsToRun.push(test)
392 }
393 } else {
394 testsToRun = getDefaultTests()
395 }
396
397 console.log('🚀 Starting skills tests')
398 console.log(` Adapters: ${adaptersToRun.map((a) => a.name).join(', ')}`)
399 console.log(` Tests: ${testsToRun.map((t) => t.id).join(', ')}`)
400 console.log(` Parallel: ${parallel}`)
401
402 // Run tests
403 const results = await runParallel(
404 adaptersToRun,
405 testsToRun,
406 parallel,
407 verbose,
408 )
409
410 console.log('\n')
411
412 if (results.length === 0) {
413 console.log('⚠️ No tests were run.')
414 if (adapterFilter) {
415 console.log(
416 ' The specified adapters may not be configured or available.',
417 )
418 }
419 process.exit(1)
420 }
421
422 // Print results grid
423 formatGrid(results, testsToRun)
424
425 // Check for failures
426 const allPassed = results.every((result) =>
427 testsToRun.every((test) => {
428 const outcome = result.tests[test.id]
429 return !outcome || outcome.ignored || outcome.passed
430 }),
431 )
432
433 console.log('\n' + '='.repeat(60))
434 if (allPassed) {
435 console.log('✅ All tests passed!')
436 process.exit(0)
437 } else {
438 console.log('❌ Some tests failed')
439 process.exit(1)
440 }
441}
442
443const program = new Command()
444 .name('skills-test')
445 .description('Test the TanStack AI Code Mode Skills system')
446 .version('0.0.1')
447
448// Run command (primary)
449program
450 .command('run')
451 .description('Run tests across adapters (like smoke-tests)')
452 .option(
453 '--adapters <names>',
454 'Comma-separated list of adapters (e.g., openai,anthropic,gemini)',
455 )
456 .option(
457 '--tests <ids>',
458 'Comma-separated list of test IDs (e.g., SIM,SKL,STR)',
459 )
460 .option(
461 '--parallel <n>',
462 'Number of tests to run in parallel (default: 3)',
463 '3',
464 )
465 .option('-v, --verbose', 'Enable verbose output')
466 .action(runCommand)
467
468// List command
469program
470 .command('list')
471 .description('List available adapters and tests')
472 .option('--adapters', 'List adapters only')
473 .option('--tests', 'List tests only')
474 .action(listCommand)
475
476// Simulated test command (legacy)
477program
478 .command('simulated')
479 .description('Run deterministic test with mock adapter (no API key needed)')
480 .action(async () => {
481 console.log(`${colors.bright}${colors.cyan}`)
482 console.log('╔═══════════════════════════════════════════════════════════╗')
483 console.log('║ TanStack AI Code Mode Skills - Simulated Test ║')
484 console.log('╚═══════════════════════════════════════════════════════════╝')
485 console.log(colors.reset)
486
487 try {
488 const result = await runSimulatedTest()
489
490 if (result.passed) {
491 process.exit(0)
492 } else {
493 process.exit(1)
494 }
495 } catch (error) {
496 logError(
497 `Unexpected error: ${error instanceof Error ? error.message : String(error)}`,
498 )
499 process.exit(1)
500 }
501 })
502
503// Registry test command
504program
505 .command('registry')
506 .description('Run ToolRegistry dynamic registration test (no API key needed)')
507 .action(async () => {
508 console.log(`${colors.bright}${colors.cyan}`)
509 console.log('╔═══════════════════════════════════════════════════════════╗')
510 console.log('║ TanStack AI Code Mode Skills - Registry Test ║')
511 console.log('╚═══════════════════════════════════════════════════════════╝')
512 console.log(colors.reset)
513
514 try {
515 const result = await runRegistryTest()
516
517 if (result.passed) {
518 process.exit(0)
519 } else {
520 process.exit(1)
521 }
522 } catch (error) {
523 logError(
524 `Unexpected error: ${error instanceof Error ? error.message : String(error)}`,
525 )
526 process.exit(1)
527 }
528 })
529
530// Live test command (legacy)
531program
532 .command('live')
533 .description('Run test with a real LLM (requires API key)')
534 .option(
535 '--provider <provider>',
536 'LLM provider to use (openai, anthropic, or gemini)',
537 'openai',
538 )
539 .option('--model <model>', 'Model to use (default depends on provider)')
540 .option('-v, --verbose', 'Enable verbose output')
541 .action(async (options) => {
542 console.log(`${colors.bright}${colors.cyan}`)
543 console.log('╔═══════════════════════════════════════════════════════════╗')
544 console.log('║ TanStack AI Code Mode Skills - Live Test ║')
545 console.log('╚═══════════════════════════════════════════════════════════╝')
546 console.log(colors.reset)
547
548 const { provider, model, verbose } = options
549
550 // Get the appropriate adapter
551 let adapter: any
552
553 if (provider === 'openai') {
554 const apiKey = process.env.OPENAI_API_KEY
555 if (!apiKey) {
556 logError('OPENAI_API_KEY environment variable is not set')
557 logInfo(
558 'Set it in test-cli/.env.local or export it: export OPENAI_API_KEY=sk-...',
559 )
560 process.exit(1)
561 }
562
563 try {
564 const { openaiText } = await import('@tanstack/ai-openai')
565 const modelName = model || 'gpt-4o'
566 adapter = openaiText(modelName, { apiKey })
567 logInfo(`Using OpenAI adapter with model: ${modelName}`)
568 } catch (error) {
569 logError(
570 'Failed to load OpenAI adapter. Make sure @tanstack/ai-openai is installed.',
571 )
572 logInfo('Run: pnpm add @tanstack/ai-openai')
573 process.exit(1)
574 }
575 } else if (provider === 'anthropic') {
576 const apiKey = process.env.ANTHROPIC_API_KEY
577 if (!apiKey) {
578 logError('ANTHROPIC_API_KEY environment variable is not set')
579 logInfo(
580 'Set it in test-cli/.env.local or export it: export ANTHROPIC_API_KEY=sk-ant-...',
581 )
582 process.exit(1)
583 }
584
585 try {
586 const { anthropicText } = await import('@tanstack/ai-anthropic')
587 const modelName = model || 'claude-sonnet-4-20250514'
588 adapter = anthropicText(modelName, { apiKey })
589 logInfo(`Using Anthropic adapter with model: ${modelName}`)
590 } catch (error) {
591 logError(
592 'Failed to load Anthropic adapter. Make sure @tanstack/ai-anthropic is installed.',
593 )
594 logInfo('Run: pnpm add @tanstack/ai-anthropic')
595 process.exit(1)
596 }
597 } else if (provider === 'gemini') {
598 const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY
599 if (!apiKey) {
600 logError('GEMINI_API_KEY environment variable is not set')
601 logInfo(
602 'Set it in test-cli/.env.local or export it: export GEMINI_API_KEY=...',
603 )
604 process.exit(1)
605 }
606
607 try {
608 const { geminiText } = await import('@tanstack/ai-gemini')
609 const modelName = model || 'gemini-2.5-flash'
610 adapter = geminiText(modelName, { apiKey })
611 logInfo(`Using Gemini adapter with model: ${modelName}`)
612 } catch (error) {
613 logError(
614 'Failed to load Gemini adapter. Make sure @tanstack/ai-gemini is installed.',
615 )
616 logInfo('Run: pnpm add @tanstack/ai-gemini')
617 process.exit(1)
618 }
619 } else {
620 logError(`Unknown provider: ${provider}`)
621 logInfo('Supported providers: openai, anthropic, gemini')
622 process.exit(1)
623 }
624
625 try {
626 const result = await runLiveTest({
627 adapter,
628 verbose,
629 })
630
631 if (result.passed) {
632 process.exit(0)
633 } else {
634 process.exit(1)
635 }
636 } catch (error) {
637 logError(
638 `Unexpected error: ${error instanceof Error ? error.message : String(error)}`,
639 )
640 if (error instanceof Error && error.stack) {
641 console.error(error.stack)
642 }
643 process.exit(1)
644 }
645 })
646
647// Structured output test command (legacy)
648program
649 .command('structured')
650 .description('Test structured output with code mode (requires API key)')
651 .option(
652 '--provider <provider>',
653 'LLM provider to use (openai, anthropic, or gemini)',
654 'openai',
655 )
656 .option('--model <model>', 'Model to use (default depends on provider)')
657 .option('-v, --verbose', 'Enable verbose output')
658 .action(async (options) => {
659 console.log(`${colors.bright}${colors.cyan}`)
660 console.log('╔═══════════════════════════════════════════════════════════╗')
661 console.log('║ TanStack AI Code Mode - Structured Output Test ║')
662 console.log('╚═══════════════════════════════════════════════════════════╝')
663 console.log(colors.reset)
664
665 const { provider, model, verbose } = options
666
667 // Get the appropriate adapter
668 let adapter: any
669
670 if (provider === 'openai') {
671 const apiKey = process.env.OPENAI_API_KEY
672 if (!apiKey) {
673 logError('OPENAI_API_KEY environment variable is not set')
674 logInfo(
675 'Set it in test-cli/.env.local or export it: export OPENAI_API_KEY=sk-...',
676 )
677 process.exit(1)
678 }
679
680 try {
681 const { openaiText } = await import('@tanstack/ai-openai')
682 const modelName = model || 'gpt-4o'
683 adapter = openaiText(modelName, { apiKey })
684 logInfo(`Using OpenAI adapter with model: ${modelName}`)
685 } catch (error) {
686 logError(
687 'Failed to load OpenAI adapter. Make sure @tanstack/ai-openai is installed.',
688 )
689 process.exit(1)
690 }
691 } else if (provider === 'anthropic') {
692 const apiKey = process.env.ANTHROPIC_API_KEY
693 if (!apiKey) {
694 logError('ANTHROPIC_API_KEY environment variable is not set')
695 logInfo(
696 'Set it in test-cli/.env.local or export it: export ANTHROPIC_API_KEY=sk-ant-...',
697 )
698 process.exit(1)
699 }
700
701 try {
702 const { anthropicText } = await import('@tanstack/ai-anthropic')
703 const modelName = model || 'claude-sonnet-4-20250514'
704 adapter = anthropicText(modelName, { apiKey })
705 logInfo(`Using Anthropic adapter with model: ${modelName}`)
706 } catch (error) {
707 logError(
708 'Failed to load Anthropic adapter. Make sure @tanstack/ai-anthropic is installed.',
709 )
710 process.exit(1)
711 }
712 } else if (provider === 'gemini') {
713 const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY
714 if (!apiKey) {
715 logError('GEMINI_API_KEY environment variable is not set')
716 logInfo(
717 'Set it in test-cli/.env.local or export it: export GEMINI_API_KEY=...',
718 )
719 process.exit(1)
720 }
721
722 try {
723 const { geminiText } = await import('@tanstack/ai-gemini')
724 const modelName = model || 'gemini-2.5-flash'
725 adapter = geminiText(modelName, { apiKey })
726 logInfo(`Using Gemini adapter with model: ${modelName}`)
727 } catch (error) {
728 logError(
729 'Failed to load Gemini adapter. Make sure @tanstack/ai-gemini is installed.',
730 )
731 process.exit(1)
732 }
733 } else {
734 logError(`Unknown provider: ${provider}`)
735 logInfo('Supported providers: openai, anthropic, gemini')
736 process.exit(1)
737 }
738
739 try {
740 const result = await runStructuredOutputTest({
741 adapter,
742 verbose,
743 })
744
745 if (result.passed) {
746 process.exit(0)
747 } else {
748 process.exit(1)
749 }
750 } catch (error) {
751 logError(
752 `Unexpected error: ${error instanceof Error ? error.message : String(error)}`,
753 )
754 if (error instanceof Error && error.stack) {
755 console.error(error.stack)
756 }
757 process.exit(1)
758 }
759 })
760
761// Default action - run all tests
762program.action(() => {
763 runCommand({})
764})
765
766// Parse command line
767program.parse()