Open History to see the last commit
1import { aiEventClient } from '@tanstack/ai-event-client'
2import type {
3 ChatMiddleware,
4 ChatMiddlewareConfig,
5 ChatMiddlewareContext,
6 ModelMessage,
7 StreamChunk,
8} from '@tanstack/ai'
9import type {
10 MemoryAdapter,
11 MemoryFact,
12 MemoryScope,
13 MemoryTurn,
14 RecallResult,
15 SaveReceipt,
16} from './types'
17
18/**
19 * CUSTOM stream-event name carrying server-side memory state to the browser.
20 * The middleware injects one of these per turn (via `onChunk`); the client
21 * devtools bridge (`@tanstack/ai-client`) recognizes it and re-emits `memory:*`
22 * on the browser event bus. This is how server-side memory reaches the browser
23 * DevTools panel — server-emitted `aiEventClient` events never cross runtimes;
24 * everything the panel shows is re-derived client-side from the chat stream
25 * (mirrors how generation results ride `CUSTOM` events — see `GENERATION_EVENTS`).
26 */
27export const MEMORY_STATE_EVENT = 'memory:state'
28
29/** Payload of the {@link MEMORY_STATE_EVENT} CUSTOM chunk. Captures memory state
30 * as of the turn's START — the snapshot reflects every prior turn's save; this
31 * turn's own save (deferred) surfaces in the next turn's snapshot. */
32export interface MemoryStateEventValue {
33 scope: MemoryScope
34 adapter: string
35 /** The recall query (last user text). */
36 query: string
37 /** Recall metrics for the operations timeline. */
38 recall: {
39 fragmentCount: number
40 hasTools: boolean
41 systemPromptChars: number
42 durationMs: number
43 }
44 /** Live store snapshot, when the adapter supports `inspect`/`listFacts`. */
45 snapshot?: {
46 takenAt: string
47 data: unknown
48 facts: Array<MemoryFact>
49 }
50}
51
52/**
53 * How the middleware participates in the run:
54 * - `'recall+save'` (default): recall on init (inject prompt + tools), save on finish.
55 * - `'save-only'`: skip recall entirely — persist the turn but never read/inject.
56 */
57export type MemoryMiddlewareRole = 'recall+save' | 'save-only'
58
59export interface MemoryRecallInfo {
60 scope: MemoryScope
61 query: string
62 result: RecallResult
63}
64
65export interface MemorySaveInfo {
66 scope: MemoryScope
67 turn: MemoryTurn
68 receipts: Array<SaveReceipt>
69}
70
71export interface MemoryMiddlewareOptions {
72 /** The memory backend to recall from / save to. */
73 adapter: MemoryAdapter
74 /**
75 * Scope for every adapter call. The function form is the safer default for
76 * multi-tenant apps: derive scope per request from trusted, server-validated
77 * chat context — never from client input.
78 */
79 scope:
80 | MemoryScope
81 | ((ctx: ChatMiddlewareContext) => MemoryScope | Promise<MemoryScope>)
82 /** Participation role. Defaults to `'recall+save'`. */
83 role?: MemoryMiddlewareRole
84 /** Fired after `recall` completes (post-injection), for app telemetry. */
85 onRecall?: (info: MemoryRecallInfo) => void | Promise<void>
86 /** Fired after the deferred `save` completes, for app telemetry. */
87 onSave?: (info: MemorySaveInfo) => void | Promise<void>
88}
89
90/** Per-request scratch state, keyed by context in a module-level WeakMap so the
91 * same middleware instance is safe across concurrent `chat()` calls. */
92interface MemoryRequestState {
93 resolvedScope?: MemoryScope
94 lastUserText: string
95 /** Pending devtools transport chunk, injected once by the first `onChunk`. */
96 stateChunk?: { emitted: boolean; value: MemoryStateEventValue }
97}
98
99const stateByCtx = new WeakMap<ChatMiddlewareContext, MemoryRequestState>()
100
101/**
102 * Server-side memory middleware. Recalls relevant memory into the prompt before
103 * the model runs, then defers `save` of the completed turn after it finishes.
104 * All extraction/ranking/rendering lives in the adapter — this middleware only
105 * wires `recall`/`save` into the chat lifecycle and emits devtools events.
106 */
107export function memoryMiddleware(
108 options: MemoryMiddlewareOptions,
109): ChatMiddleware {
110 const role = options.role ?? 'recall+save'
111
112 async function resolveScope(
113 ctx: ChatMiddlewareContext,
114 state: MemoryRequestState,
115 ): Promise<MemoryScope> {
116 if (state.resolvedScope) return state.resolvedScope
117 state.resolvedScope =
118 typeof options.scope === 'function'
119 ? await options.scope(ctx)
120 : options.scope
121 return state.resolvedScope
122 }
123
124 return {
125 name: `memory:${options.adapter.id}`,
126
127 async onConfig(ctx, config) {
128 if (ctx.phase !== 'init') return
129
130 const state: MemoryRequestState = { lastUserText: '' }
131 stateByCtx.set(ctx, state)
132
133 state.lastUserText = getMessageText(findLastUserMessage(config.messages))
134 if (!state.lastUserText || role === 'save-only') return
135
136 const startedAt = Date.now()
137 let scope: MemoryScope
138 let result: RecallResult
139 try {
140 scope = await resolveScope(ctx, state)
141 safeEmit('memory:retrieve:started', {
142 scope,
143 adapter: options.adapter.id,
144 query: state.lastUserText,
145 timestamp: startedAt,
146 })
147 result = await options.adapter.recall(scope, state.lastUserText)
148 } catch (error) {
149 safeEmit('memory:error', {
150 // Only attach scope when resolve already succeeded; otherwise omit
151 // (no empty-string / partial fake identity).
152 ...(state.resolvedScope ? { scope: state.resolvedScope } : {}),
153 adapter: options.adapter.id,
154 phase: 'recall',
155 error: errorInfo(error),
156 timestamp: Date.now(),
157 })
158 return
159 }
160
161 const tools = result.tools ?? []
162 const recallMetrics = {
163 fragmentCount: result.fragments?.length ?? 0,
164 hasTools: tools.length > 0,
165 systemPromptChars: result.systemPrompt.length,
166 durationMs: Date.now() - startedAt,
167 }
168 safeEmit('memory:retrieve:completed', {
169 scope,
170 adapter: options.adapter.id,
171 ...recallMetrics,
172 timestamp: Date.now(),
173 })
174 await options.onRecall?.({ scope, query: state.lastUserText, result })
175
176 // Stage the devtools transport chunk (recall metrics + current store
177 // snapshot). Injected into the stream by `onChunk` so it reaches the
178 // browser panel; see MEMORY_STATE_EVENT.
179 const snapshot = await gatherSnapshot(options.adapter, scope)
180 state.stateChunk = {
181 emitted: false,
182 value: {
183 scope,
184 adapter: options.adapter.id,
185 query: state.lastUserText,
186 recall: recallMetrics,
187 ...(snapshot ? { snapshot } : {}),
188 },
189 }
190
191 const memoryPrompts = [result.toolGuidance ?? '', result.systemPrompt]
192 const additions = memoryPrompts.filter((p) => p.length > 0)
193 if (additions.length === 0 && tools.length === 0) return
194
195 return {
196 systemPrompts: [...config.systemPrompts, ...additions],
197 tools: [...config.tools, ...tools],
198 } satisfies Partial<ChatMiddlewareConfig>
199 },
200
201 onChunk(ctx, chunk) {
202 // Inject the staged memory-state chunk exactly once, riding alongside the
203 // first stream chunk (typically RUN_STARTED) so the browser devtools sees
204 // it. Returning an array expands the stream; see ChatMiddleware.onChunk.
205 const state = stateByCtx.get(ctx)
206 if (!state?.stateChunk || state.stateChunk.emitted) return
207 state.stateChunk.emitted = true
208 const custom: StreamChunk = {
209 type: 'CUSTOM',
210 name: MEMORY_STATE_EVENT,
211 value: state.stateChunk.value,
212 timestamp: Date.now(),
213 }
214 return [chunk, custom]
215 },
216
217 onFinish(ctx, info) {
218 const state = stateByCtx.get(ctx)
219 stateByCtx.delete(ctx)
220 const userText =
221 state?.lastUserText || getMessageText(findLastUserMessage(ctx.messages))
222 const assistant = info.content
223 if (!userText || !assistant) return
224 const scope = state?.resolvedScope
225
226 ctx.defer(
227 (async () => {
228 // Resolve scope defensively — a throwing resolver must not escape the
229 // terminal hook. Memory failures are always non-fatal + observable.
230 let resolved: MemoryScope
231 try {
232 resolved =
233 scope ?? (await resolveScope(ctx, { lastUserText: userText }))
234 } catch (error) {
235 safeEmit('memory:error', {
236 adapter: options.adapter.id,
237 phase: 'save',
238 error: errorInfo(error),
239 timestamp: Date.now(),
240 })
241 return
242 }
243
244 const turn: MemoryTurn = { user: userText, assistant }
245 const startedAt = Date.now()
246 safeEmit('memory:persist:started', {
247 scope: resolved,
248 adapter: options.adapter.id,
249 timestamp: startedAt,
250 })
251 let receipts: Array<SaveReceipt>
252 try {
253 receipts = await options.adapter.save(resolved, turn)
254 } catch (error) {
255 receipts = [{ ok: false, error: String(error) }]
256 safeEmit('memory:error', {
257 scope: resolved,
258 adapter: options.adapter.id,
259 phase: 'save',
260 error: errorInfo(error),
261 timestamp: Date.now(),
262 })
263 }
264 safeEmit('memory:persist:completed', {
265 scope: resolved,
266 adapter: options.adapter.id,
267 receiptCount: receipts.length,
268 okCount: receipts.filter((r) => r.ok).length,
269 durationMs: Date.now() - startedAt,
270 timestamp: Date.now(),
271 })
272 await emitSnapshot(options.adapter, resolved)
273 await options.onSave?.({ scope: resolved, turn, receipts })
274 })(),
275 )
276 },
277 }
278}
279
280// ===========================
281// Internals
282// ===========================
283
284/**
285 * Read the adapter's current stored state via the optional `inspect`/`listFacts`
286 * introspection methods. Returns `undefined` for adapters that don't implement
287 * `inspect` (they degrade to the metrics-only timeline). Fully guarded:
288 * introspection must never affect chat.
289 */
290async function gatherSnapshot(
291 adapter: MemoryAdapter,
292 scope: MemoryScope,
293): Promise<
294 { takenAt: string; data: unknown; facts: Array<MemoryFact> } | undefined
295> {
296 if (!adapter.inspect) return undefined
297 try {
298 const snapshot = await adapter.inspect(scope)
299 const facts = (await adapter.listFacts?.(scope)) ?? []
300 return { takenAt: snapshot.takenAt, data: snapshot.data, facts }
301 } catch {
302 // ignored — introspection is best-effort telemetry.
303 return undefined
304 }
305}
306
307/**
308 * DevTools-only: after a save, emit the adapter's current stored state on the
309 * (in-process) event bus, so a devtools consumer running in the SAME runtime as
310 * the chat (client-side execution / server-side listener) sees "what's in
311 * memory". For the standard server-side topology, the browser panel instead
312 * gets state via the {@link MEMORY_STATE_EVENT} stream chunk (see `onChunk`).
313 */
314async function emitSnapshot(
315 adapter: MemoryAdapter,
316 scope: MemoryScope,
317): Promise<void> {
318 const snapshot = await gatherSnapshot(adapter, scope)
319 if (!snapshot) return
320 safeEmit('memory:snapshot', {
321 scope,
322 adapter: adapter.id,
323 ...snapshot,
324 timestamp: Date.now(),
325 })
326}
327
328function findLastUserMessage(
329 messages: ReadonlyArray<ModelMessage>,
330): ModelMessage | undefined {
331 for (let i = messages.length - 1; i >= 0; i--) {
332 const message = messages[i]
333 if (message && message.role === 'user') return message
334 }
335 return undefined
336}
337
338/**
339 * Extract plain text from a `ModelMessage`. Text lives on `part.content` for
340 * `TextPart`; bare strings in the content array are tolerated. All other
341 * content kinds (tool-call, image, …) yield '' so they don't pollute the
342 * recall query.
343 */
344function getMessageText(message?: ModelMessage): string {
345 if (!message) return ''
346 if (typeof message.content === 'string') return message.content
347 if (Array.isArray(message.content)) {
348 return message.content
349 .map((part) => {
350 if (typeof part === 'string') return part
351 if (part.type === 'text' && typeof part.content === 'string') {
352 return part.content
353 }
354 return ''
355 })
356 .filter(Boolean)
357 .join('\n')
358 }
359 return ''
360}
361
362function errorInfo(error: unknown): { name: string; message: string } {
363 if (error instanceof Error)
364 return { name: error.name, message: error.message }
365 if (
366 error &&
367 typeof error === 'object' &&
368 'name' in error &&
369 typeof error.name === 'string'
370 ) {
371 return {
372 name: error.name,
373 message: String((error as { message?: unknown }).message ?? error),
374 }
375 }
376 return { name: 'Error', message: String(error) }
377}
378
379/** Fire-and-forget devtools emit — telemetry failures must never affect chat. */
380function safeEmit(...args: Parameters<typeof aiEventClient.emit>): void {
381 try {
382 aiEventClient.emit(...args)
383 } catch {
384 // ignored — telemetry must not affect chat behaviour
385 }
386}