Open History to see the last commit
1/**
2 * Efficiency metrics calculation for Code Mode.
3 *
4 * Code Mode provides significant efficiency gains over traditional tool-calling patterns
5 * by keeping data in the sandbox and only returning final results.
6 */
7
8// Model ID mapping from API format to pricing key
9const MODEL_MAPPING: Record<string, string> = {
10 'claude-haiku-4-5': 'claude-haiku-4.5',
11 'claude-haiku-4-20250514': 'claude-haiku-4',
12 'claude-haiku': 'claude-haiku',
13 'gpt-4o': 'gpt-4o',
14 'gemini-2.5-flash': 'gemini-2.5-flash',
15}
16
17export const EFFICIENCY_CONSTANTS = {
18 // Conversion
19 BYTES_PER_TOKEN: 4, // ~4 bytes per token (English text average)
20
21 // Tool call patterns
22 AVG_TOOL_RESULT_BYTES: 2400, // Average size of a tool result (~600 tokens)
23 AVG_ASSISTANT_OVERHEAD: 600, // LLM response between tool calls (~150 tokens)
24 BASE_CONTEXT_BYTES: 8000, // System prompt + user message (~2000 tokens)
25
26 // Timing
27 AVG_ROUND_TRIP_MS: 3000, // Average LLM round-trip time
28 CODE_EXECUTION_OVERHEAD_MS: 500, // Sandbox startup + execution overhead
29
30 // Pricing (per million tokens, as of 2025)
31 PRICING: {
32 'claude-haiku-4.5': { input: 3, output: 15 },
33 'claude-haiku-4': { input: 3, output: 15 },
34 'claude-haiku': { input: 0.25, output: 1.25 },
35 'gpt-4o': { input: 2.5, output: 10 },
36 'gemini-2.5-flash': { input: 0.15, output: 0.6 },
37 } as Record<string, { input: number; output: number }>,
38}
39
40export interface EfficiencyInput {
41 actualBytes: number // Actual context used with Code Mode
42 theoreticalBytes: number // Estimated context without Code Mode
43 model?: string // Model ID for cost calculation
44}
45
46export interface EfficiencyMetrics {
47 // Context (direct from input)
48 contextActual: number
49 contextTheoretical: number
50 contextSavedBytes: number
51 contextSavedPercent: number
52
53 // Round-trips (estimated)
54 roundTripsActual: number
55 roundTripsTheoretical: number
56 roundTripsSaved: number
57 roundTripsSavedPercent: number
58
59 // Time (estimated)
60 timeActualMs: number
61 timeTheoreticalMs: number
62 timeSavedMs: number
63 timeSavedPercent: number
64
65 // Cost (estimated)
66 costActual: number
67 costTheoretical: number
68 costSaved: number
69 costSavedPercent: number
70}
71
72/**
73 * Calculate compounded token usage for traditional tool-calling.
74 * Each round resends all previous context — this is the key inefficiency.
75 */
76function calculateCompoundedTokens(
77 toolCalls: number,
78 C: typeof EFFICIENCY_CONSTANTS,
79): number {
80 const baseTokens = C.BASE_CONTEXT_BYTES / C.BYTES_PER_TOKEN
81 const toolResultTokens = C.AVG_TOOL_RESULT_BYTES / C.BYTES_PER_TOKEN
82 const overheadTokens = C.AVG_ASSISTANT_OVERHEAD / C.BYTES_PER_TOKEN
83
84 let totalInputTokens = 0
85 let accumulatedContext = baseTokens
86
87 for (let i = 0; i < toolCalls; i++) {
88 totalInputTokens += accumulatedContext
89 accumulatedContext += toolResultTokens + overheadTokens
90 }
91
92 // Add final round for summary
93 totalInputTokens += accumulatedContext
94
95 // Output tokens: LLM responds each round
96 const totalOutputTokens = toolCalls * overheadTokens
97
98 return totalInputTokens + totalOutputTokens
99}
100
101/**
102 * Calculate cost from tokens using model pricing.
103 */
104function calculateCost(
105 tokens: number,
106 pricing: { input: number; output: number },
107): number {
108 // Assume 80% input, 20% output
109 const inputTokens = tokens * 0.8
110 const outputTokens = tokens * 0.2
111
112 const inputCost = (inputTokens / 1_000_000) * pricing.input
113 const outputCost = (outputTokens / 1_000_000) * pricing.output
114
115 return inputCost + outputCost
116}
117
118/**
119 * Calculate comprehensive efficiency metrics from byte measurements.
120 */
121export function calculateEfficiency(input: EfficiencyInput): EfficiencyMetrics {
122 const { actualBytes, theoreticalBytes, model } = input
123 const C = EFFICIENCY_CONSTANTS
124
125 // Map model ID to pricing key
126 const pricingKey = model ? MODEL_MAPPING[model] || model : 'claude-haiku-4.5'
127
128 // === Context (direct) ===
129 const contextSavedBytes = theoreticalBytes - actualBytes
130 const contextSavedPercent =
131 theoreticalBytes > 0
132 ? Math.round((contextSavedBytes / theoreticalBytes) * 100)
133 : 0
134
135 // === Round-trips (estimated from theoretical bytes) ===
136 // theoreticalBytes ≈ base + n * (toolResult + overhead)
137 // Solve for n (number of tool calls)
138 const toolCallBytes = C.AVG_TOOL_RESULT_BYTES + C.AVG_ASSISTANT_OVERHEAD
139 const estimatedToolCalls = Math.max(
140 1,
141 Math.round((theoreticalBytes - C.BASE_CONTEXT_BYTES) / toolCallBytes),
142 )
143
144 // Without Code Mode: each tool call = 1 round-trip
145 // Plus ~30% overhead for LLM "thinking" between calls
146 const roundTripsTheoretical = Math.ceil(estimatedToolCalls * 1.3)
147
148 // With Code Mode: typically 2 round-trips
149 // (1) User message → LLM writes code
150 // (2) Code execution → LLM summarizes
151 const roundTripsActual = 2
152
153 const roundTripsSaved = roundTripsTheoretical - roundTripsActual
154 const roundTripsSavedPercent =
155 roundTripsTheoretical > 0
156 ? Math.round((roundTripsSaved / roundTripsTheoretical) * 100)
157 : 0
158
159 // === Time (estimated from round-trips) ===
160 const timeTheoreticalMs = roundTripsTheoretical * C.AVG_ROUND_TRIP_MS
161 const timeActualMs =
162 roundTripsActual * C.AVG_ROUND_TRIP_MS + C.CODE_EXECUTION_OVERHEAD_MS
163 const timeSavedMs = timeTheoreticalMs - timeActualMs
164 const timeSavedPercent =
165 timeTheoreticalMs > 0
166 ? Math.round((timeSavedMs / timeTheoreticalMs) * 100)
167 : 0
168
169 // === Cost (estimated from tokens) ===
170 const pricing = C.PRICING[pricingKey] ?? C.PRICING['claude-haiku-4.5']
171
172 // Convert bytes to tokens
173 const actualTokens = actualBytes / C.BYTES_PER_TOKEN
174 const theoreticalTokens = calculateCompoundedTokens(estimatedToolCalls, C)
175
176 const costActual = calculateCost(actualTokens, pricing)
177 const costTheoretical = calculateCost(theoreticalTokens, pricing)
178 const costSaved = costTheoretical - costActual
179 const costSavedPercent =
180 costTheoretical > 0 ? Math.round((costSaved / costTheoretical) * 100) : 0
181
182 return {
183 contextActual: actualBytes,
184 contextTheoretical: theoreticalBytes,
185 contextSavedBytes,
186 contextSavedPercent,
187
188 roundTripsActual,
189 roundTripsTheoretical,
190 roundTripsSaved,
191 roundTripsSavedPercent,
192
193 timeActualMs,
194 timeTheoreticalMs,
195 timeSavedMs,
196 timeSavedPercent,
197
198 costActual,
199 costTheoretical,
200 costSaved,
201 costSavedPercent,
202 }
203}
204
205export function estimateRoundTripTimeMs(calls: number): number {
206 const safeCalls = Math.max(0, calls)
207 return safeCalls * EFFICIENCY_CONSTANTS.AVG_ROUND_TRIP_MS
208}
209
210export function estimateCostFromBytes(bytes: number, model?: string): number {
211 const pricingKey = model ? MODEL_MAPPING[model] || model : 'claude-haiku-4.5'
212 const pricing =
213 EFFICIENCY_CONSTANTS.PRICING[pricingKey] ??
214 EFFICIENCY_CONSTANTS.PRICING['claude-haiku-4.5']
215 const tokens = bytes / EFFICIENCY_CONSTANTS.BYTES_PER_TOKEN
216 return calculateCost(tokens, pricing)
217}
218
219// === Formatters ===
220
221export function formatBytes(bytes: number): string {
222 if (bytes < 1024) return `${bytes} B`
223 if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
224 return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
225}
226
227export function formatDuration(ms: number): string {
228 if (ms < 1000) return `${ms}ms`
229 if (ms < 60000) return `${Math.round(ms / 1000)}s`
230 return `${Math.round(ms / 60000)}m ${Math.round((ms % 60000) / 1000)}s`
231}
232
233export function formatCurrency(amount: number): string {
234 if (amount < 0.01) return '<$0.01'
235 return `$${amount.toFixed(2)}`
236}