Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

efficiency.ts

236 lines7.3 KB

1/**
2 * Efficiency metrics calculation for Code Mode.
3 *
4 * Code Mode provides significant efficiency gains over traditional tool-calling patterns
5 * by keeping data in the sandbox and only returning final results.
6 */
7​
8// Model ID mapping from API format to pricing key
9const MODEL_MAPPING: Record<string, string> = {
10 'claude-haiku-4-5': 'claude-haiku-4.5',
11 'claude-haiku-4-20250514': 'claude-haiku-4',
12 'claude-haiku': 'claude-haiku',
13 'gpt-4o': 'gpt-4o',
14 'gemini-2.5-flash': 'gemini-2.5-flash',
16​
17export const EFFICIENCY_CONSTANTS = {
18 // Conversion
19 BYTES_PER_TOKEN: 4, // ~4 bytes per token (English text average)
20​
21 // Tool call patterns
22 AVG_TOOL_RESULT_BYTES: 2400, // Average size of a tool result (~600 tokens)
23 AVG_ASSISTANT_OVERHEAD: 600, // LLM response between tool calls (~150 tokens)
24 BASE_CONTEXT_BYTES: 8000, // System prompt + user message (~2000 tokens)
25​
26 // Timing
27 AVG_ROUND_TRIP_MS: 3000, // Average LLM round-trip time
28 CODE_EXECUTION_OVERHEAD_MS: 500, // Sandbox startup + execution overhead
29​
30 // Pricing (per million tokens, as of 2025)
31 PRICING: {
32 'claude-haiku-4.5': { input: 3, output: 15 },
33 'claude-haiku-4': { input: 3, output: 15 },
34 'claude-haiku': { input: 0.25, output: 1.25 },
35 'gpt-4o': { input: 2.5, output: 10 },
36 'gemini-2.5-flash': { input: 0.15, output: 0.6 },
37 } as Record<string, { input: number; output: number }>,
39​
40export interface EfficiencyInput {
41 actualBytes: number // Actual context used with Code Mode
42 theoreticalBytes: number // Estimated context without Code Mode
43 model?: string // Model ID for cost calculation
45​
46export interface EfficiencyMetrics {
47 // Context (direct from input)
48 contextActual: number
49 contextTheoretical: number
50 contextSavedBytes: number
51 contextSavedPercent: number
52​
53 // Round-trips (estimated)
54 roundTripsActual: number
55 roundTripsTheoretical: number
56 roundTripsSaved: number
57 roundTripsSavedPercent: number
58​
59 // Time (estimated)
60 timeActualMs: number
61 timeTheoreticalMs: number
62 timeSavedMs: number
63 timeSavedPercent: number
64​
65 // Cost (estimated)
66 costActual: number
67 costTheoretical: number
68 costSaved: number
69 costSavedPercent: number
71​
72/**
73 * Calculate compounded token usage for traditional tool-calling.
74 * Each round resends all previous context — this is the key inefficiency.
75 */
76function calculateCompoundedTokens(
77 toolCalls: number,
78 C: typeof EFFICIENCY_CONSTANTS,
79): number {
80 const baseTokens = C.BASE_CONTEXT_BYTES / C.BYTES_PER_TOKEN
81 const toolResultTokens = C.AVG_TOOL_RESULT_BYTES / C.BYTES_PER_TOKEN
82 const overheadTokens = C.AVG_ASSISTANT_OVERHEAD / C.BYTES_PER_TOKEN
83​
84 let totalInputTokens = 0
85 let accumulatedContext = baseTokens
86​
87 for (let i = 0; i < toolCalls; i++) {
88 totalInputTokens += accumulatedContext
89 accumulatedContext += toolResultTokens + overheadTokens
90 }
91​
92 // Add final round for summary
93 totalInputTokens += accumulatedContext
94​
95 // Output tokens: LLM responds each round
96 const totalOutputTokens = toolCalls * overheadTokens
97​
98 return totalInputTokens + totalOutputTokens
100​
101/**
102 * Calculate cost from tokens using model pricing.
103 */
104function calculateCost(
105 tokens: number,
106 pricing: { input: number; output: number },
107): number {
108 // Assume 80% input, 20% output
109 const inputTokens = tokens * 0.8
110 const outputTokens = tokens * 0.2
111​
112 const inputCost = (inputTokens / 1_000_000) * pricing.input
113 const outputCost = (outputTokens / 1_000_000) * pricing.output
114​
115 return inputCost + outputCost
117​
118/**
119 * Calculate comprehensive efficiency metrics from byte measurements.
120 */
121export function calculateEfficiency(input: EfficiencyInput): EfficiencyMetrics {
122 const { actualBytes, theoreticalBytes, model } = input
123 const C = EFFICIENCY_CONSTANTS
124​
125 // Map model ID to pricing key
126 const pricingKey = model ? MODEL_MAPPING[model] || model : 'claude-haiku-4.5'
127​
128 // === Context (direct) ===
129 const contextSavedBytes = theoreticalBytes - actualBytes
130 const contextSavedPercent =
131 theoreticalBytes > 0
132 ? Math.round((contextSavedBytes / theoreticalBytes) * 100)
133 : 0
134​
135 // === Round-trips (estimated from theoretical bytes) ===
136 // theoreticalBytes ≈ base + n * (toolResult + overhead)
137 // Solve for n (number of tool calls)
138 const toolCallBytes = C.AVG_TOOL_RESULT_BYTES + C.AVG_ASSISTANT_OVERHEAD
139 const estimatedToolCalls = Math.max(
140 1,
141 Math.round((theoreticalBytes - C.BASE_CONTEXT_BYTES) / toolCallBytes),
142 )
143​
144 // Without Code Mode: each tool call = 1 round-trip
145 // Plus ~30% overhead for LLM "thinking" between calls
146 const roundTripsTheoretical = Math.ceil(estimatedToolCalls * 1.3)
147​
148 // With Code Mode: typically 2 round-trips
149 // (1) User message → LLM writes code
150 // (2) Code execution → LLM summarizes
151 const roundTripsActual = 2
152​
153 const roundTripsSaved = roundTripsTheoretical - roundTripsActual
154 const roundTripsSavedPercent =
155 roundTripsTheoretical > 0
156 ? Math.round((roundTripsSaved / roundTripsTheoretical) * 100)
157 : 0
158​
159 // === Time (estimated from round-trips) ===
160 const timeTheoreticalMs = roundTripsTheoretical * C.AVG_ROUND_TRIP_MS
161 const timeActualMs =
162 roundTripsActual * C.AVG_ROUND_TRIP_MS + C.CODE_EXECUTION_OVERHEAD_MS
163 const timeSavedMs = timeTheoreticalMs - timeActualMs
164 const timeSavedPercent =
165 timeTheoreticalMs > 0
166 ? Math.round((timeSavedMs / timeTheoreticalMs) * 100)
167 : 0
168​
169 // === Cost (estimated from tokens) ===
170 const pricing = C.PRICING[pricingKey] ?? C.PRICING['claude-haiku-4.5']
171​
172 // Convert bytes to tokens
173 const actualTokens = actualBytes / C.BYTES_PER_TOKEN
174 const theoreticalTokens = calculateCompoundedTokens(estimatedToolCalls, C)
175​
176 const costActual = calculateCost(actualTokens, pricing)
177 const costTheoretical = calculateCost(theoreticalTokens, pricing)
178 const costSaved = costTheoretical - costActual
179 const costSavedPercent =
180 costTheoretical > 0 ? Math.round((costSaved / costTheoretical) * 100) : 0
181​
182 return {
183 contextActual: actualBytes,
184 contextTheoretical: theoreticalBytes,
185 contextSavedBytes,
186 contextSavedPercent,
187​
188 roundTripsActual,
189 roundTripsTheoretical,
190 roundTripsSaved,
191 roundTripsSavedPercent,
192​
193 timeActualMs,
194 timeTheoreticalMs,
195 timeSavedMs,
196 timeSavedPercent,
197​
198 costActual,
199 costTheoretical,
200 costSaved,
201 costSavedPercent,
202 }
204​
205export function estimateRoundTripTimeMs(calls: number): number {
206 const safeCalls = Math.max(0, calls)
207 return safeCalls * EFFICIENCY_CONSTANTS.AVG_ROUND_TRIP_MS
209​
210export function estimateCostFromBytes(bytes: number, model?: string): number {
211 const pricingKey = model ? MODEL_MAPPING[model] || model : 'claude-haiku-4.5'
212 const pricing =
213 EFFICIENCY_CONSTANTS.PRICING[pricingKey] ??
214 EFFICIENCY_CONSTANTS.PRICING['claude-haiku-4.5']
215 const tokens = bytes / EFFICIENCY_CONSTANTS.BYTES_PER_TOKEN
216 return calculateCost(tokens, pricing)
218​
219// === Formatters ===
220​
221export function formatBytes(bytes: number): string {
222 if (bytes < 1024) return `${bytes} B`
223 if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
224 return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
226​
227export function formatDuration(ms: number): string {
228 if (ms < 1000) return `${ms}ms`
229 if (ms < 60000) return `${Math.round(ms / 1000)}s`
230 return `${Math.round(ms / 60000)}m ${Math.round((ms % 60000) / 1000)}s`
232​
233export function formatCurrency(amount: number): string {
234 if (amount < 0.01) return '<$0.01'
235 return `$${amount.toFixed(2)}`