Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

api.chat.ts

279 lines8.9 KB

1import * as path from 'node:path'
2import * as fs from 'node:fs'
3import { createFileRoute } from '@tanstack/react-router'
4import {
5 chat,
6 createChatOptions,
7 maxIterations,
8 toServerSentEventsResponse,
9} from '@tanstack/ai'
10import { anthropicText } from '@tanstack/ai-anthropic'
11import { geminiText } from '@tanstack/ai-gemini'
12import { grokText } from '@tanstack/ai-grok'
13import { openaiText } from '@tanstack/ai-openai'
14import { ollamaText } from '@tanstack/ai-ollama'
15import { openRouterText } from '@tanstack/ai-openrouter'
16import type { AIAdapter, StreamChunk } from '@tanstack/ai'
17import type { ChunkRecording } from '@/lib/recording'
18import {
19 addToCartToolDef,
20 addToWishListToolDef,
21 getGuitars,
22 getPersonalGuitarPreferenceToolDef,
23 recommendGuitarToolDef,
24} from '@/lib/guitar-tools'
25​
26const SYSTEM_PROMPT = `You are a helpful assistant for a guitar store.
27​
28CRITICAL INSTRUCTIONS - YOU MUST FOLLOW THIS EXACT WORKFLOW:
29​
30When a user asks for a guitar recommendation:
311. FIRST: Use the getGuitars tool (no parameters needed)
322. SECOND: Use the recommendGuitar tool with the ID of the guitar you want to recommend
333. NEVER write a recommendation directly - ALWAYS use the recommendGuitar tool
34​
35IMPORTANT:
36- The recommendGuitar tool will display the guitar in a special, appealing format
37- You MUST use recommendGuitar for ANY guitar recommendation
38- ONLY recommend guitars from our inventory (use getGuitars first)
39- The recommendGuitar tool has a buy button - this is how customers purchase
40- Do NOT describe the guitar yourself - let the recommendGuitar tool do it
41​
42Example workflow:
43User: "I want an acoustic guitar"
44Step 1: Call getGuitars()
45Step 2: Call recommendGuitar(id: "6")
46Step 3: Done - do NOT add any text after calling recommendGuitar
48const addToCartToolServer = addToCartToolDef.server((args) => ({
49 success: true,
50 cartId: 'CART_' + Date.now(),
51 guitarId: args.guitarId,
52 quantity: args.quantity,
53 totalItems: args.quantity,
54}))
55​
56type Provider =
57 | 'openai'
58 | 'anthropic'
59 | 'gemini'
60 | 'ollama'
61 | 'grok'
62 | 'openrouter'
63​
64/**
65 * Wraps an adapter to intercept chatStream and record raw chunks from the adapter
66 * before they're processed by the stream processor.
67 */
68function wrapAdapterForRecording<TAdapter extends AIAdapter>(
69 adapter: TAdapter,
70 recordingFilePath: string,
71 model: string,
72 provider: string,
73): TAdapter {
74 // Type guard to check if adapter has chatStream
75 if (!('chatStream' in adapter) || typeof adapter.chatStream !== 'function') {
76 return adapter
77 }
78​
79 const originalChatStream = adapter.chatStream.bind(adapter)
80​
81 // Track chunks for recording
82 const chunks: Array<{
83 chunk: StreamChunk
84 timestamp: number
85 index: number
86 }> = []
87 let chunkIndex = 0
88​
89 // Create a wrapper that intercepts chatStream
90 const wrappedAdapter = {
91 ...adapter,
92 chatStream: async function* (
93 options: Parameters<typeof originalChatStream>[0],
94 ): AsyncIterable<StreamChunk> {
95 const startTime = Date.now()
96​
97 try {
98 // Iterate over chunks from the original adapter
99 for await (const chunk of originalChatStream(options)) {
100 const timestamp = Date.now()
101 const index = chunkIndex++
102​
103 // Record the chunk
104 chunks.push({
105 chunk,
106 timestamp,
107 index,
108 })
109​
110 // Yield the chunk to continue normal processing
111 yield chunk
112 }
113 } finally {
114 // Save recording when stream completes
115 try {
116 const recording: ChunkRecording = {
117 version: '1.0',
118 timestamp: startTime,
119 model,
120 provider,
121 chunks,
122 }
123​
124 // Ensure directory exists
125 const dir = path.dirname(recordingFilePath)
126 if (!fs.existsSync(dir)) {
127 fs.mkdirSync(dir, { recursive: true })
128 }
129​
130 // Write recording
131 fs.writeFileSync(
132 recordingFilePath,
133 JSON.stringify(recording, null, 2),
134 'utf-8',
135 )
136​
137 console.log(`Adapter chunks recorded to: ${recordingFilePath}`)
138 } catch (error) {
139 console.error('Failed to save adapter chunk recording:', error)
140 }
141 }
142 },
143 } as TAdapter
144​
145 return wrappedAdapter
147​
148export const Route = createFileRoute('/api/chat')({
149 server: {
150 handlers: {
151 POST: async ({ request }) => {
152 // Capture request signal before reading body (it may be aborted after body is consumed)
153 const requestSignal = request.signal
154​
155 // If request is already aborted, return early
156 if (requestSignal.aborted) {
157 return new Response(null, { status: 499 }) // 499 = Client Closed Request
158 }
159​
160 const abortController = new AbortController()
161​
162 const body = await request.json()
163 const messages = body.messages
164 const data = body.data || {}
165​
166 // Extract provider, model, and traceId from data
167 const provider: Provider = data.provider || 'openai'
168 const model: string = data.model || 'gpt-4o'
169 const traceId: string | undefined = data.traceId
170​
171 try {
172 // Pre-define typed adapter configurations with full type inference
173 // Model is passed to the adapter factory function for type-safe autocomplete
174 const adapterConfig = {
175 anthropic: () =>
176 createChatOptions({
177 adapter: anthropicText((model || 'claude-sonnet-4-5') as any),
178 }),
179 gemini: () =>
180 createChatOptions({
181 adapter: geminiText((model || 'gemini-2.0-flash') as any),
182 }),
183 grok: () =>
184 createChatOptions({
185 adapter: grokText((model || 'grok-3') as any),
186 }),
187 ollama: () =>
188 createChatOptions({
189 adapter: ollamaText((model || 'mistral:7b') as any),
190 }),
191 openai: () =>
192 createChatOptions({
193 adapter: openaiText((model || 'gpt-4o') as any),
194 }),
195 openrouter: () =>
196 createChatOptions({
197 adapter: openRouterText((model || 'openai/gpt-4o') as any),
198 }),
199 }
200​
201 // Get typed adapter options using createChatOptions pattern
202 const options = adapterConfig[provider]()
203 let { adapter } = options
204​
205 console.log(`>> model: ${model} on provider: ${provider}`)
206​
207 // If we have a traceId, wrap the adapter to record raw chunks from chatStream
208 if (traceId) {
209 const traceDir = path.join(process.cwd(), 'test-traces')
210 if (!fs.existsSync(traceDir)) {
211 fs.mkdirSync(traceDir, { recursive: true })
212 }
213 const traceFile = path.join(traceDir, `${traceId}.json`)
214 adapter = wrapAdapterForRecording(
215 adapter,
216 traceFile,
217 model,
218 provider,
219 )
220 }
221​
222 // Use the stream abort signal for proper cancellation handling
223 const stream = chat({
224 ...options,
225 adapter, // Use potentially wrapped adapter
226 tools: [
227 getGuitars, // Server tool
228 recommendGuitarToolDef, // No server execute - client will handle
229 addToCartToolServer,
230 addToWishListToolDef,
231 getPersonalGuitarPreferenceToolDef,
232 ],
233 systemPrompts: [SYSTEM_PROMPT],
234 agentLoopStrategy: maxIterations(20),
235 messages,
236 modelOptions: {
237 // Enable reasoning for OpenAI (gpt-5, o3 models):
238 // reasoning: {
239 // effort: "medium", // or "low", "high", "minimal", "none" (for gpt-5.1)
240 // },
241 // Enable thinking for Anthropic:
242 /* thinking: {
243 type: "enabled",
244 budget_tokens: 2048,
245 }, */
246 },
247 abortController,
248 })
249​
250 return toServerSentEventsResponse(stream, { abortController })
251 } catch (error: any) {
252 console.error('[API Route] Error in chat request:', {
253 message: error?.message,
254 name: error?.name,
255 status: error?.status,
256 statusText: error?.statusText,
257 code: error?.code,
258 type: error?.type,
259 stack: error?.stack,
260 error: error,
261 })
262 // If request was aborted, return early (don't send error response)
263 if (error.name === 'AbortError' || abortController.signal.aborted) {
264 return new Response(null, { status: 499 }) // 499 = Client Closed Request
265 }
266 return new Response(
267 JSON.stringify({
268 error: error.message || 'An error occurred',
269 }),
270 {
271 status: 500,
272 headers: { 'Content-Type': 'application/json' },
273 },
274 )
275 }
276 },
277 },
278 },
279})