Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

chat.py

552 lines17.6 KB

1"""
2Chat functionality with automatic tool execution loop.
3​
4This module provides the main chat() function and ChatEngine class that
5orchestrate the agentic flow with tool calling support.
6"""
7​
8import json
9import time
10from enum import Enum
11from typing import Any, AsyncIterator, Dict, List, Optional
12​
13from .agent_strategies import max_iterations
14from .base_adapter import BaseAdapter
15from .tool_manager import (
16 ApprovalRequest,
17 ClientToolRequest,
18 ExecuteToolCallsResult,
19 ToolCallManager,
20 ToolResult,
21 execute_tool_calls,
23from .types import (
24 AgentLoopStrategy,
25 ApprovalRequestedStreamChunk,
26 ChatOptions,
27 DoneStreamChunk,
28 ModelMessage,
29 StreamChunk,
30 Tool,
31 ToolCall,
32 ToolInputAvailableStreamChunk,
33 ToolResultStreamChunk,
35​
36​
37class CyclePhase(str, Enum):
38 """Phase of the agent cycle."""
39​
40 PROCESS_CHAT = "processChat"
41 EXECUTE_TOOL_CALLS = "executeToolCalls"
42​
43​
44class ToolPhaseResult(str, Enum):
45 """Result from tool execution phase."""
46​
47 CONTINUE = "continue"
48 STOP = "stop"
49 WAIT = "wait"
50​
51​
52def _prepend_system_prompts(
53 messages: List[ModelMessage],
54 system_prompts: Optional[List[str]] = None,
55 default_prompts: Optional[List[str]] = None,
56) -> List[ModelMessage]:
57 """
58 Prepend system prompts to messages.
59​
60 Args:
61 messages: Existing messages
62 system_prompts: Optional system prompts to prepend
63 default_prompts: Default prompts to use if system_prompts is empty
64​
65 Returns:
66 Messages with system prompts prepended
67 """
68 prompts = system_prompts if system_prompts else (default_prompts or [])
69​
70 if not prompts:
71 return messages
72​
73 system_messages: List[ModelMessage] = [
74 {"role": "system", "content": content} for content in prompts
75 ]
76​
77 return system_messages + messages
78​
79​
80class ChatEngine:
81 """
82 Orchestrates the chat loop with automatic tool execution.
83​
84 This class manages the conversation state, handles the agentic loop,
85 executes tool calls, and emits appropriate stream chunks.
86 """
87​
88 def __init__(
89 self,
90 adapter: BaseAdapter,
91 options: ChatOptions,
92 system_prompts: Optional[List[str]] = None,
93 ):
94 """
95 Initialize the chat engine.
96​
97 Args:
98 adapter: AI adapter to use
99 options: Chat options
100 system_prompts: Optional default system prompts
101 """
102 self.adapter = adapter
103 self.options = options
104 self.system_prompts = system_prompts or []
105 self.tools: List[Tool] = options.tools or []
106 self.loop_strategy: AgentLoopStrategy = (
107 options.agent_loop_strategy or max_iterations(5)
108 )
109 self.tool_call_manager = ToolCallManager(self.tools)
110 self.initial_message_count = len(options.messages)
111​
112 # State
113 self.messages = _prepend_system_prompts(
114 options.messages,
115 options.system_prompts,
116 self.system_prompts,
117 )
118 self.iteration_count = 0
119 self.last_finish_reason: Optional[str] = None
120 self.current_message_id: Optional[str] = None
121 self.accumulated_content = ""
122 self.done_chunk: Optional[DoneStreamChunk] = None
123 self.should_emit_stream_end = True
124 self.early_termination = False
125 self.tool_phase: ToolPhaseResult = ToolPhaseResult.CONTINUE
126 self.cycle_phase: CyclePhase = CyclePhase.PROCESS_CHAT
127​
128 # Generate IDs
129 self.request_id = self._create_id("chat")
130 self.stream_id = self._create_id("stream")
131​
132 async def chat(self) -> AsyncIterator[StreamChunk]:
133 """
134 Main chat loop with automatic tool execution.
135​
136 Yields:
137 StreamChunk objects
138 """
139 try:
140 # Check for pending tool calls first
141 async for chunk in self._check_for_pending_tool_calls():
142 yield chunk
143​
144 if self.tool_phase == ToolPhaseResult.WAIT:
145 return
146​
147 # Main agentic loop
148 while self._should_continue():
149 if self.early_termination:
150 return
151​
152 self._begin_cycle()
153​
154 if self.cycle_phase == CyclePhase.PROCESS_CHAT:
155 async for chunk in self._stream_model_response():
156 yield chunk
157 else:
158 async for chunk in self._process_tool_calls():
159 yield chunk
160​
161 self._end_cycle()
162​
163 finally:
164 pass
165​
166 def _begin_cycle(self) -> None:
167 """Begin a cycle (either chat or tool execution)."""
168 if self.cycle_phase == CyclePhase.PROCESS_CHAT:
169 self._begin_iteration()
170​
171 def _end_cycle(self) -> None:
172 """End a cycle and switch phase."""
173 if self.cycle_phase == CyclePhase.PROCESS_CHAT:
174 self.cycle_phase = CyclePhase.EXECUTE_TOOL_CALLS
175 return
176​
177 self.cycle_phase = CyclePhase.PROCESS_CHAT
178 self.iteration_count += 1
179​
180 def _begin_iteration(self) -> None:
181 """Begin a new iteration."""
182 self.current_message_id = self._create_id("msg")
183 self.accumulated_content = ""
184 self.done_chunk = None
185​
186 async def _stream_model_response(self) -> AsyncIterator[StreamChunk]:
187 """
188 Stream response from the model.
189​
190 Yields:
191 StreamChunk objects
192 """
193 # Create chat options for the adapter
194 adapter_options = ChatOptions(
195 model=self.options.model,
196 messages=self.messages,
197 tools=self.tools,
198 options=self.options.options,
199 provider_options=self.options.provider_options,
200 )
201​
202 async for chunk in self.adapter.chat_stream(adapter_options):
203 yield chunk
204 self._handle_stream_chunk(chunk)
205​
206 if self.early_termination:
207 break
208​
209 def _handle_stream_chunk(self, chunk: StreamChunk) -> None:
210 """
211 Handle a stream chunk and update state.
212​
213 Args:
214 chunk: Stream chunk to handle
215 """
216 chunk_type = chunk.get("type")
217​
218 if chunk_type == "content":
219 self.accumulated_content = chunk["content"]
220 elif chunk_type == "tool_call":
221 self.tool_call_manager.add_tool_call_chunk(chunk)
222 elif chunk_type == "done":
223 self._handle_done_chunk(chunk)
224 elif chunk_type == "error":
225 self.early_termination = True
226 self.should_emit_stream_end = False
227​
228 def _handle_done_chunk(self, chunk: DoneStreamChunk) -> None:
229 """Handle a done chunk."""
230 # Don't overwrite a tool_calls finishReason with a stop finishReason
231 if (
232 self.done_chunk
233 and self.done_chunk.get("finishReason") == "tool_calls"
234 and chunk.get("finishReason") == "stop"
235 ):
236 self.last_finish_reason = chunk.get("finishReason")
237 return
238​
239 self.done_chunk = chunk
240 self.last_finish_reason = chunk.get("finishReason")
241​
242 async def _check_for_pending_tool_calls(self) -> AsyncIterator[StreamChunk]:
243 """
244 Check for pending tool calls in messages and execute them.
245​
246 Yields:
247 StreamChunk objects
248 """
249 pending_tool_calls = self._get_pending_tool_calls_from_messages()
250 if not pending_tool_calls:
251 return
252​
253 done_chunk = self._create_synthetic_done_chunk()
254​
255 # Collect client state
256 approvals, client_tool_results = self._collect_client_state()
257​
258 # Execute tool calls
259 execution_result = await execute_tool_calls(
260 pending_tool_calls,
261 self.tools,
262 approvals,
263 client_tool_results,
264 )
265​
266 # Handle approval requests
267 if execution_result.needs_approval or execution_result.needs_client_execution:
268 async for chunk in self._emit_approval_requests(
269 execution_result.needs_approval, done_chunk
270 ):
271 yield chunk
272​
273 async for chunk in self._emit_client_tool_inputs(
274 execution_result.needs_client_execution, done_chunk
275 ):
276 yield chunk
277​
278 self.should_emit_stream_end = False
279 self.tool_phase = ToolPhaseResult.WAIT
280 return
281​
282 # Emit tool results
283 async for chunk in self._emit_tool_results(execution_result.results, done_chunk):
284 yield chunk
285​
286 async def _process_tool_calls(self) -> AsyncIterator[StreamChunk]:
287 """
288 Process tool calls from the current iteration.
289​
290 Yields:
291 StreamChunk objects
292 """
293 if not self._should_execute_tool_phase():
294 self._set_tool_phase(ToolPhaseResult.STOP)
295 return
296​
297 tool_calls = self.tool_call_manager.get_tool_calls()
298 done_chunk = self.done_chunk
299​
300 if not done_chunk or not tool_calls:
301 self._set_tool_phase(ToolPhaseResult.STOP)
302 return
303​
304 # Add assistant message with tool calls
305 self._add_assistant_tool_call_message(tool_calls)
306​
307 # Collect client state
308 approvals, client_tool_results = self._collect_client_state()
309​
310 # Execute tool calls
311 execution_result = await execute_tool_calls(
312 tool_calls,
313 self.tools,
314 approvals,
315 client_tool_results,
316 )
317​
318 # Handle approval requests
319 if execution_result.needs_approval or execution_result.needs_client_execution:
320 async for chunk in self._emit_approval_requests(
321 execution_result.needs_approval, done_chunk
322 ):
323 yield chunk
324​
325 async for chunk in self._emit_client_tool_inputs(
326 execution_result.needs_client_execution, done_chunk
327 ):
328 yield chunk
329​
330 self._set_tool_phase(ToolPhaseResult.WAIT)
331 return
332​
333 # Emit tool results
334 async for chunk in self._emit_tool_results(execution_result.results, done_chunk):
335 yield chunk
336​
337 self.tool_call_manager.clear()
338 self._set_tool_phase(ToolPhaseResult.CONTINUE)
339​
340 def _should_execute_tool_phase(self) -> bool:
341 """Check if we should execute the tool phase."""
342 return (
343 self.done_chunk is not None
344 and self.done_chunk.get("finishReason") == "tool_calls"
345 and len(self.tools) > 0
346 and self.tool_call_manager.has_tool_calls()
347 )
348​
349 def _add_assistant_tool_call_message(self, tool_calls: List[ToolCall]) -> None:
350 """Add an assistant message with tool calls to the conversation."""
351 message: ModelMessage = {
352 "role": "assistant",
353 "content": self.accumulated_content or None,
354 "toolCalls": tool_calls,
355 }
356 self.messages.append(message)
357​
358 def _collect_client_state(self) -> tuple[Dict[str, bool], Dict[str, Any]]:
359 """
360 Collect approval decisions and client tool results from messages.
361​
362 Returns:
363 Tuple of (approvals map, client tool results map)
364 """
365 approvals: Dict[str, bool] = {}
366 client_tool_results: Dict[str, Any] = {}
367​
368 # TODO: Parse message parts for approval state and client results
369 # This would require extending ModelMessage to support parts
370​
371 return approvals, client_tool_results
372​
373 async def _emit_approval_requests(
374 self,
375 approval_requests: List[ApprovalRequest],
376 done_chunk: DoneStreamChunk,
377 ) -> AsyncIterator[StreamChunk]:
378 """Emit approval request chunks."""
379 for approval in approval_requests:
380 chunk: ApprovalRequestedStreamChunk = {
381 "type": "approval-requested",
382 "id": done_chunk["id"],
383 "model": done_chunk["model"],
384 "timestamp": int(time.time() * 1000),
385 "toolCallId": approval.tool_call_id,
386 "toolName": approval.tool_name,
387 "input": approval.input,
388 "approval": {
389 "id": approval.approval_id,
390 "needsApproval": True,
391 },
392 }
393 yield chunk
394​
395 async def _emit_client_tool_inputs(
396 self,
397 client_requests: List[ClientToolRequest],
398 done_chunk: DoneStreamChunk,
399 ) -> AsyncIterator[StreamChunk]:
400 """Emit tool-input-available chunks for client execution."""
401 for client_tool in client_requests:
402 chunk: ToolInputAvailableStreamChunk = {
403 "type": "tool-input-available",
404 "id": done_chunk["id"],
405 "model": done_chunk["model"],
406 "timestamp": int(time.time() * 1000),
407 "toolCallId": client_tool.tool_call_id,
408 "toolName": client_tool.tool_name,
409 "input": client_tool.input,
410 }
411 yield chunk
412​
413 async def _emit_tool_results(
414 self,
415 results: List[ToolResult],
416 done_chunk: DoneStreamChunk,
417 ) -> AsyncIterator[StreamChunk]:
418 """Emit tool result chunks and add to messages."""
419 for result in results:
420 content = json.dumps(result.result)
421​
422 chunk: ToolResultStreamChunk = {
423 "type": "tool_result",
424 "id": done_chunk["id"],
425 "model": done_chunk["model"],
426 "timestamp": int(time.time() * 1000),
427 "toolCallId": result.tool_call_id,
428 "content": content,
429 }
430 yield chunk
431​
432 # Add tool result message
433 tool_message: ModelMessage = {
434 "role": "tool",
435 "content": content,
436 "toolCallId": result.tool_call_id,
437 }
438 self.messages.append(tool_message)
439​
440 def _get_pending_tool_calls_from_messages(self) -> List[ToolCall]:
441 """Get tool calls that don't have results yet."""
442 completed_tool_ids = set(
443 msg.get("toolCallId")
444 for msg in self.messages
445 if msg.get("role") == "tool" and msg.get("toolCallId")
446 )
447​
448 pending: List[ToolCall] = []
449 for message in self.messages:
450 if message.get("role") == "assistant" and message.get("toolCalls"):
451 for tool_call in message["toolCalls"]:
452 if tool_call["id"] not in completed_tool_ids:
453 pending.append(tool_call)
454​
455 return pending
456​
457 def _create_synthetic_done_chunk(self) -> DoneStreamChunk:
458 """Create a synthetic done chunk for pending tool calls."""
459 return {
460 "type": "done",
461 "id": self._create_id("pending"),
462 "model": self.options.model,
463 "timestamp": int(time.time() * 1000),
464 "finishReason": "tool_calls",
465 }
466​
467 def _should_continue(self) -> bool:
468 """Check if the loop should continue."""
469 if self.cycle_phase == CyclePhase.EXECUTE_TOOL_CALLS:
470 return True
471​
472 return self.loop_strategy(
473 {
474 "iterationCount": self.iteration_count,
475 "messages": self.messages,
476 "finishReason": self.last_finish_reason,
477 }
478 ) and self.tool_phase == ToolPhaseResult.CONTINUE
479​
480 def _set_tool_phase(self, phase: ToolPhaseResult) -> None:
481 """Set the tool phase."""
482 self.tool_phase = phase
483 if phase == ToolPhaseResult.WAIT:
484 self.should_emit_stream_end = False
485​
486 def _create_id(self, prefix: str) -> str:
487 """Create a unique ID with a prefix."""
488 import random
489 import string
490​
491 timestamp = int(time.time() * 1000)
492 random_suffix = "".join(
493 random.choices(string.ascii_lowercase + string.digits, k=7)
494 )
495 return f"{prefix}-{timestamp}-{random_suffix}"
496​
497​
498# ============================================================================
499# Main chat function
500# ============================================================================
501​
502​
503async def chat(
504 adapter: BaseAdapter,
505 model: str,
506 messages: List[ModelMessage],
507 tools: Optional[List[Tool]] = None,
508 system_prompts: Optional[List[str]] = None,
509 agent_loop_strategy: Optional[AgentLoopStrategy] = None,
510 options: Optional[Dict[str, Any]] = None,
511 provider_options: Optional[Dict[str, Any]] = None,
512) -> AsyncIterator[StreamChunk]:
513 """
514 Standalone chat streaming function with automatic tool execution loop.
515​
516 Args:
517 adapter: AI adapter instance to use
518 model: Model name (must be supported by the adapter)
519 messages: Conversation messages
520 tools: Optional tools for function calling (auto-executed)
521 system_prompts: Optional system prompts to prepend
522 agent_loop_strategy: Optional strategy for controlling tool execution loop
523 options: Common options (temperature, max_tokens, etc.)
524 provider_options: Provider-specific options
525​
526 Yields:
527 StreamChunk objects representing different events during the conversation
528​
529 Example:
530 >>> async for chunk in chat(
531 ... adapter=anthropic_adapter,
532 ... model="claude-3-5-sonnet-20241022",
533 ... messages=[{"role": "user", "content": "Hello!"}],
534 ... tools=[weather_tool],
535 ... ):
536 ... if chunk["type"] == "content":
537 ... print(chunk["delta"], end="", flush=True)
538 """
539 chat_options = ChatOptions(
540 model=model,
541 messages=messages,
542 tools=tools,
543 system_prompts=system_prompts,
544 agent_loop_strategy=agent_loop_strategy,
545 options=options,
546 provider_options=provider_options,
547 )
548​
549 engine = ChatEngine(adapter, chat_options)
550​
551 async for chunk in engine.chat():
552 yield chunk