Open History to see the last commit
1// The eval arms (E2). An arm is a mode (tool-calls vs code-mode) plus an
2// optimize() config (or none). System prompt and turn limits are held constant
3// by the runner; only the mode + middleware change between arms.
4
5import {
6 truncateHeadTail,
7 dropWithRef,
8 aboveTokens,
9} from '@tanstack/ai-optimize'
10
11const HEAD_TAIL = aboveTokens(1500, truncateHeadTail({ maxTokens: 750 }))
12
13// --- tool-call arms ---------------------------------------------------------
14// Author-adorned rules for the runner's own file tools:
15// - write_file.content is redundant once the file is on disk -> drop the
16// completed call's args from history (D3). This is the big, safe saving, and
17// it compounds across a multi-write session.
18// - run_shell.stdout is a log -> safe to clip.
19// - read_file.content is load-bearing while editing -> keep it WHOLE. Pin an
20// effectively-infinite threshold so the leaves-mode default can never clip it.
21const KEEP_WHOLE = aboveTokens(
22 1_000_000_000,
23 truncateHeadTail({ maxTokens: 750 }),
24)
25const ADORNED_TOOL_RULES = {
26 write_file: { args: { fields: { content: dropWithRef() } } },
27 run_shell: { result: { fields: { stdout: HEAD_TAIL } } },
28 read_file: { result: { fields: { content: KEEP_WHOLE } } },
29}
30
31// --- code-mode arms ---------------------------------------------------------
32// In code mode the only visible tool is execute_typescript. The program source
33// (`typescriptCode`) is write-once-run-once: after it executes the model needs
34// the *result*, not the source it already submitted — exactly the write_file
35// story. `logs` is console spam, safe to clip. This is the whole code-mode
36// thesis: every bundled program evaporates from history once it has run.
37const CODE_MODE_ADORNED_RULES = {
38 execute_typescript: {
39 args: { fields: { typescriptCode: dropWithRef() } },
40 result: { fields: { logs: HEAD_TAIL } },
41 },
42}
43
44// Each arm: { codeMode, config }. `config` is the optimize() config (null = none).
45const ARM_SPECS = {
46 baseline: { codeMode: false, config: null },
47 // Strands replica: flatten every result, 1500/750 head_tail, errors exempt.
48 global: {
49 codeMode: false,
50 config: { defaults: { projection: HEAD_TAIL, mode: 'flatten' } },
51 },
52 adorned: {
53 codeMode: false,
54 config: {
55 defaults: { projection: HEAD_TAIL, mode: 'leaves' },
56 tools: ADORNED_TOOL_RULES,
57 },
58 },
59 universal: {
60 codeMode: false,
61 config: { defaults: { projection: HEAD_TAIL, mode: 'leaves' } },
62 },
63 // Code-mode arms: same workload, run via execute_typescript.
64 'cm-baseline': { codeMode: true, config: null },
65 'cm-adorned': { codeMode: true, config: { tools: CODE_MODE_ADORNED_RULES } },
66 'cm-global': {
67 codeMode: true,
68 config: { defaults: { projection: HEAD_TAIL, mode: 'flatten' } },
69 },
70}
71
72/** Returns the spec { codeMode, config } for an arm. */
73export function armSpec(arm) {
74 const spec = ARM_SPECS[arm]
75 if (!spec)
76 throw new Error(
77 `Unknown arm "${arm}". Use one of: ${Object.keys(ARM_SPECS).join(', ')}.`,
78 )
79 return spec
80}
81
82/** Back-compat: just the optimize() config (or null) for an arm. */
83export function armConfig(arm) {
84 return armSpec(arm).config
85}
86
87export const ARMS = Object.keys(ARM_SPECS)
88export const TOOL_ARMS = ARMS.filter((a) => !ARM_SPECS[a].codeMode)
89export const CODE_MODE_ARMS = ARMS.filter((a) => ARM_SPECS[a].codeMode)