Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

api.codemode.ts

199 lines6.5 KB

1import { createFileRoute } from '@tanstack/react-router'
2import { chat, maxIterations, toServerSentEventsStream } from '@tanstack/ai'
3import { createCodeMode } from '@tanstack/ai-code-mode'
4import { anthropicText } from '@tanstack/ai-anthropic'
5import { openaiText } from '@tanstack/ai-openai'
6import { geminiText } from '@tanstack/ai-gemini'
7import type { AnyTextAdapter, StreamChunk } from '@tanstack/ai'
8import { allTools } from '@/lib/tools'
9import { CODE_MODE_SYSTEM_PROMPT } from '@/lib/prompts'
10import { exportConversationToPdfTool } from '@/lib/tools/export-pdf-tool'
11import { maxTokensModelOptions } from '@/lib/max-tokens-model-options'
12​
13type Provider = 'anthropic' | 'openai' | 'gemini'
14​
15function getAdapter(provider: Provider, model?: string): AnyTextAdapter {
16 switch (provider) {
17 case 'openai':
18 return openaiText((model || 'gpt-4o') as 'gpt-4o')
19 case 'gemini':
20 return geminiText((model || 'gemini-2.5-flash') as 'gemini-2.5-flash')
21 case 'anthropic':
22 default:
23 return anthropicText((model || 'claude-haiku-4-5') as 'claude-haiku-4-5')
24 }
26​
27import {
28 createIsolateDriver,
29 type IsolateVM,
30} from '@/lib/create-isolate-driver'
31​
32type CodeModeEntry = {
33 tool: ReturnType<typeof createCodeMode>['tool']
34 systemPrompt: string
36const codeModeCacheByVM = new Map<IsolateVM, CodeModeEntry>()
37​
38async function getCodeModeTools(vm: IsolateVM = 'node') {
39 let entry = codeModeCacheByVM.get(vm)
40 if (!entry) {
41 const driver = await createIsolateDriver(vm)
42 const { tool, systemPrompt } = createCodeMode({
43 driver,
44 tools: allTools,
45 timeout: 60000,
46 memoryLimit: 128,
47 })
48 entry = { tool, systemPrompt }
49 codeModeCacheByVM.set(vm, entry)
50 }
51 return entry
53​
54export const Route = createFileRoute('/_npm-github-chat/api/codemode')({
55 server: {
56 handlers: {
57 POST: async ({ request }) => {
58 const requestSignal = request.signal
59 if (requestSignal.aborted) {
60 return new Response(null, { status: 499 })
61 }
62​
63 const abortController = new AbortController()
64 const body = await request.json()
65 const { messages, data } = body
66​
67 const provider: Provider = data?.provider || 'anthropic'
68 const model: string | undefined = data?.model
69 const vm: IsolateVM = data?.vm || 'node'
70 const serverRuntime =
71 typeof (globalThis as { Bun?: unknown }).Bun !== 'undefined'
72 ? 'bun'
73 : 'node'
74 console.info(
75 `[api/codemode] request provider=${provider} model=${model ?? 'default'} vm=${vm} serverRuntime=${serverRuntime}`,
76 )
77​
78 const adapter = getAdapter(provider, model)
79 const baseChatStream = adapter.chatStream.bind(adapter)
80 let llmCallCount = 0
81 let totalContextBytes = 0
82 const textEncoder = new TextEncoder()
83 const instrumentedAdapter: AnyTextAdapter = {
84 ...adapter,
85 chatStream: (options) => {
86 llmCallCount += 1
87 let contextBytes = 0
88 try {
89 contextBytes = textEncoder.encode(
90 JSON.stringify(options.messages ?? []),
91 ).length
92 } catch {
93 contextBytes = 0
94 }
95 totalContextBytes += contextBytes
96 const averageContextBytes =
97 llmCallCount > 0
98 ? Math.round(totalContextBytes / llmCallCount)
99 : 0
100 const stream = baseChatStream(options)
101 async function* instrumentedStream(): AsyncGenerator<StreamChunk> {
102 yield {
103 type: 'CUSTOM',
104 model: adapter.model,
105 timestamp: Date.now(),
106 name: 'code_mode:llm_call',
107 data: {
108 count: llmCallCount,
109 contextBytes,
110 totalContextBytes,
111 averageContextBytes,
112 },
113 } as StreamChunk
114 for await (const chunk of stream) {
115 yield chunk
116 }
117 }
118 return instrumentedStream()
119 },
120 }
121 const { tool, systemPrompt } = await getCodeModeTools(vm)
122​
123 try {
124 const stream = chat({
125 adapter: instrumentedAdapter,
126 messages,
127 tools: [tool, exportConversationToPdfTool],
128 systemPrompts: [CODE_MODE_SYSTEM_PROMPT, systemPrompt],
129 agentLoopStrategy: maxIterations(15),
130 abortController,
131 // Increase max tokens to allow for complex code generation
132 modelOptions: maxTokensModelOptions(adapter, 8192),
133 })
134​
135 const requestStartTimeMs = Date.now()
136 const instrumentedStream =
137 (async function* (): AsyncGenerator<StreamChunk> {
138 yield {
139 type: 'CUSTOM',
140 model: adapter.model,
141 timestamp: requestStartTimeMs,
142 name: 'code_mode:chat_start',
143 data: { startTimeMs: requestStartTimeMs },
144 } as StreamChunk
145 for await (const chunk of stream) {
146 if (chunk.type === 'RUN_FINISHED') {
147 const endTimeMs = Date.now()
148 yield {
149 type: 'CUSTOM',
150 model: adapter.model,
151 timestamp: endTimeMs,
152 name: 'code_mode:chat_end',
153 data: {
154 endTimeMs,
155 durationMs: endTimeMs - requestStartTimeMs,
156 },
157 } as StreamChunk
158 }
159 yield chunk
160 }
161 })()
162​
163 const sseStream = toServerSentEventsStream(
164 instrumentedStream,
165 abortController,
166 )
167​
168 return new Response(sseStream, {
169 headers: {
170 'Content-Type': 'text/event-stream',
171 'Cache-Control': 'no-cache',
172 Connection: 'keep-alive',
173 },
174 })
175 } catch (error: unknown) {
176 console.error('[API Route] Error:', error)
177​
178 if (
179 (error instanceof Error && error.name === 'AbortError') ||
180 abortController.signal.aborted
181 ) {
182 return new Response(null, { status: 499 })
183 }
184​
185 return new Response(
186 JSON.stringify({
187 error:
188 error instanceof Error ? error.message : 'An error occurred',
189 }),
190 {
191 status: 500,
192 headers: { 'Content-Type': 'application/json' },
193 },
194 )
195 }
196 },
197 },
198 },
199})