Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

chat.py

560 lines18.0 KB

1"""
2Chat functionality with automatic tool execution loop.
3​
4This module provides the main chat() function and ChatEngine class that
5orchestrate the agentic flow with tool calling support.
6"""
7​
8import json
9import time
10from enum import Enum
11from typing import Any, AsyncIterator, Dict, List, Optional
12​
13from .agent_strategies import max_iterations
14from .base_adapter import BaseAdapter
15from .tool_manager import (
16 ApprovalRequest,
17 ClientToolRequest,
18 ExecuteToolCallsResult,
19 ToolCallManager,
20 ToolResult,
21 execute_tool_calls,
23from .types import (
24 AgentLoopStrategy,
25 ChatOptions,
26 CustomEvent,
27 ModelMessage,
28 RunFinishedEvent,
29 StreamChunk,
30 Tool,
31 ToolCall,
32 ToolCallEndEvent,
34​
35​
36class CyclePhase(str, Enum):
37 """Phase of the agent cycle."""
38​
39 PROCESS_CHAT = "processChat"
40 EXECUTE_TOOL_CALLS = "executeToolCalls"
41​
42​
43class ToolPhaseResult(str, Enum):
44 """Result from tool execution phase."""
45​
46 CONTINUE = "continue"
47 STOP = "stop"
48 WAIT = "wait"
49​
50​
51def _prepend_system_prompts(
52 messages: List[ModelMessage],
53 system_prompts: Optional[List[str]] = None,
54 default_prompts: Optional[List[str]] = None,
55) -> List[ModelMessage]:
56 """
57 Prepend system prompts to messages.
58​
59 Args:
60 messages: Existing messages
61 system_prompts: Optional system prompts to prepend
62 default_prompts: Default prompts to use if system_prompts is empty
63​
64 Returns:
65 Messages with system prompts prepended
66 """
67 prompts = system_prompts if system_prompts else (default_prompts or [])
68​
69 if not prompts:
70 return messages
71​
72 system_messages: List[ModelMessage] = [
73 {"role": "system", "content": content} for content in prompts
74 ]
75​
76 return system_messages + messages
77​
78​
79class ChatEngine:
80 """
81 Orchestrates the chat loop with automatic tool execution.
82​
83 This class manages the conversation state, handles the agentic loop,
84 executes tool calls, and emits appropriate stream chunks.
85 """
86​
87 def __init__(
88 self,
89 adapter: BaseAdapter,
90 options: ChatOptions,
91 system_prompts: Optional[List[str]] = None,
92 ):
93 """
94 Initialize the chat engine.
95​
96 Args:
97 adapter: AI adapter to use
98 options: Chat options
99 system_prompts: Optional default system prompts
100 """
101 self.adapter = adapter
102 self.options = options
103 self.system_prompts = system_prompts or []
104 self.tools: List[Tool] = options.tools or []
105 self.loop_strategy: AgentLoopStrategy = (
106 options.agent_loop_strategy or max_iterations(5)
107 )
108 self.tool_call_manager = ToolCallManager(self.tools)
109 self.initial_message_count = len(options.messages)
110​
111 # State
112 self.messages = _prepend_system_prompts(
113 options.messages,
114 options.system_prompts,
115 self.system_prompts,
116 )
117 self.iteration_count = 0
118 self.last_finish_reason: Optional[str] = None
119 self.current_message_id: Optional[str] = None
120 self.accumulated_content = ""
121 self.finished_event: Optional[RunFinishedEvent] = None
122 self.should_emit_stream_end = True
123 self.early_termination = False
124 self.tool_phase: ToolPhaseResult = ToolPhaseResult.CONTINUE
125 self.cycle_phase: CyclePhase = CyclePhase.PROCESS_CHAT
126​
127 # Generate IDs
128 self.request_id = self._create_id("chat")
129 self.stream_id = self._create_id("stream")
130​
131 async def chat(self) -> AsyncIterator[StreamChunk]:
132 """
133 Main chat loop with automatic tool execution.
134​
135 Yields:
136 StreamChunk objects
137 """
138 try:
139 # Check for pending tool calls first
140 async for chunk in self._check_for_pending_tool_calls():
141 yield chunk
142​
143 if self.tool_phase == ToolPhaseResult.WAIT:
144 return
145​
146 # Main agentic loop
147 while self._should_continue():
148 if self.early_termination:
149 return
150​
151 self._begin_cycle()
152​
153 if self.cycle_phase == CyclePhase.PROCESS_CHAT:
154 async for chunk in self._stream_model_response():
155 yield chunk
156 else:
157 async for chunk in self._process_tool_calls():
158 yield chunk
159​
160 self._end_cycle()
161​
162 finally:
163 pass
164​
165 def _begin_cycle(self) -> None:
166 """Begin a cycle (either chat or tool execution)."""
167 if self.cycle_phase == CyclePhase.PROCESS_CHAT:
168 self._begin_iteration()
169​
170 def _end_cycle(self) -> None:
171 """End a cycle and switch phase."""
172 if self.cycle_phase == CyclePhase.PROCESS_CHAT:
173 self.cycle_phase = CyclePhase.EXECUTE_TOOL_CALLS
174 return
175​
176 self.cycle_phase = CyclePhase.PROCESS_CHAT
177 self.iteration_count += 1
178​
179 def _begin_iteration(self) -> None:
180 """Begin a new iteration."""
181 self.current_message_id = self._create_id("msg")
182 self.accumulated_content = ""
183 self.finished_event = None
184​
185 async def _stream_model_response(self) -> AsyncIterator[StreamChunk]:
186 """
187 Stream response from the model.
188​
189 Yields:
190 StreamChunk objects
191 """
192 # Create chat options for the adapter
193 adapter_options = ChatOptions(
194 model=self.options.model,
195 messages=self.messages,
196 tools=self.tools,
197 options=self.options.options,
198 provider_options=self.options.provider_options,
199 )
200​
201 async for chunk in self.adapter.chat_stream(adapter_options):
202 yield chunk
203 self._handle_stream_chunk(chunk)
204​
205 if self.early_termination:
206 break
207​
208 def _handle_stream_chunk(self, chunk: StreamChunk) -> None:
209 """
210 Handle a stream chunk and update state.
211​
212 Args:
213 chunk: Stream chunk to handle
214 """
215 chunk_type = chunk.get("type")
216​
217 if chunk_type == "TEXT_MESSAGE_CONTENT":
218 if chunk.get("content"):
219 self.accumulated_content = chunk["content"]
220 else:
221 self.accumulated_content += chunk.get("delta", "")
222 elif chunk_type == "TOOL_CALL_START":
223 self.tool_call_manager.add_tool_call_start_event(chunk)
224 elif chunk_type == "TOOL_CALL_ARGS":
225 self.tool_call_manager.add_tool_call_args_event(chunk)
226 elif chunk_type == "RUN_FINISHED":
227 self._handle_run_finished_event(chunk)
228 elif chunk_type == "RUN_ERROR":
229 self.early_termination = True
230 self.should_emit_stream_end = False
231​
232 def _handle_run_finished_event(self, chunk: RunFinishedEvent) -> None:
233 """Handle a RUN_FINISHED event."""
234 # Don't overwrite a tool_calls finishReason with a stop finishReason
235 if (
236 self.finished_event
237 and self.finished_event.get("finishReason") == "tool_calls"
238 and chunk.get("finishReason") == "stop"
239 ):
240 self.last_finish_reason = chunk.get("finishReason")
241 return
242​
243 self.finished_event = chunk
244 self.last_finish_reason = chunk.get("finishReason")
245​
246 async def _check_for_pending_tool_calls(self) -> AsyncIterator[StreamChunk]:
247 """
248 Check for pending tool calls in messages and execute them.
249​
250 Yields:
251 StreamChunk objects
252 """
253 pending_tool_calls = self._get_pending_tool_calls_from_messages()
254 if not pending_tool_calls:
255 return
256​
257 finish_event = self._create_synthetic_finished_event()
258​
259 # Collect client state
260 approvals, client_tool_results = self._collect_client_state()
261​
262 # Execute tool calls
263 execution_result = await execute_tool_calls(
264 pending_tool_calls,
265 self.tools,
266 approvals,
267 client_tool_results,
268 )
269​
270 # Handle approval requests
271 if execution_result.needs_approval or execution_result.needs_client_execution:
272 async for chunk in self._emit_approval_requests(
273 execution_result.needs_approval, finish_event
274 ):
275 yield chunk
276​
277 async for chunk in self._emit_client_tool_inputs(
278 execution_result.needs_client_execution, finish_event
279 ):
280 yield chunk
281​
282 self.should_emit_stream_end = False
283 self.tool_phase = ToolPhaseResult.WAIT
284 return
285​
286 # Emit tool results
287 async for chunk in self._emit_tool_results(execution_result.results, finish_event):
288 yield chunk
289​
290 async def _process_tool_calls(self) -> AsyncIterator[StreamChunk]:
291 """
292 Process tool calls from the current iteration.
293​
294 Yields:
295 StreamChunk objects
296 """
297 if not self._should_execute_tool_phase():
298 self._set_tool_phase(ToolPhaseResult.STOP)
299 return
300​
301 tool_calls = self.tool_call_manager.get_tool_calls()
302 finish_event = self.finished_event
303​
304 if not finish_event or not tool_calls:
305 self._set_tool_phase(ToolPhaseResult.STOP)
306 return
307​
308 # Add assistant message with tool calls
309 self._add_assistant_tool_call_message(tool_calls)
310​
311 # Collect client state
312 approvals, client_tool_results = self._collect_client_state()
313​
314 # Execute tool calls
315 execution_result = await execute_tool_calls(
316 tool_calls,
317 self.tools,
318 approvals,
319 client_tool_results,
320 )
321​
322 # Handle approval requests
323 if execution_result.needs_approval or execution_result.needs_client_execution:
324 async for chunk in self._emit_approval_requests(
325 execution_result.needs_approval, finish_event
326 ):
327 yield chunk
328​
329 async for chunk in self._emit_client_tool_inputs(
330 execution_result.needs_client_execution, finish_event
331 ):
332 yield chunk
333​
334 self._set_tool_phase(ToolPhaseResult.WAIT)
335 return
336​
337 # Emit tool results
338 async for chunk in self._emit_tool_results(execution_result.results, finish_event):
339 yield chunk
340​
341 self.tool_call_manager.clear()
342 self._set_tool_phase(ToolPhaseResult.CONTINUE)
343​
344 def _should_execute_tool_phase(self) -> bool:
345 """Check if we should execute the tool phase."""
346 return (
347 self.finished_event is not None
348 and self.finished_event.get("finishReason") == "tool_calls"
349 and len(self.tools) > 0
350 and self.tool_call_manager.has_tool_calls()
351 )
352​
353 def _add_assistant_tool_call_message(self, tool_calls: List[ToolCall]) -> None:
354 """Add an assistant message with tool calls to the conversation."""
355 message: ModelMessage = {
356 "role": "assistant",
357 "content": self.accumulated_content or None,
358 "toolCalls": tool_calls,
359 }
360 self.messages.append(message)
361​
362 def _collect_client_state(self) -> tuple[Dict[str, bool], Dict[str, Any]]:
363 """
364 Collect approval decisions and client tool results from messages.
365​
366 Returns:
367 Tuple of (approvals map, client tool results map)
368 """
369 approvals: Dict[str, bool] = {}
370 client_tool_results: Dict[str, Any] = {}
371​
372 # TODO: Parse message parts for approval state and client results
373 # This would require extending ModelMessage to support parts
374​
375 return approvals, client_tool_results
376​
377 async def _emit_approval_requests(
378 self,
379 approval_requests: List[ApprovalRequest],
380 finish_event: RunFinishedEvent,
381 ) -> AsyncIterator[StreamChunk]:
382 """Emit approval request events using CUSTOM event type."""
383 for approval in approval_requests:
384 chunk: CustomEvent = {
385 "type": "CUSTOM",
386 "timestamp": int(time.time() * 1000),
387 "model": finish_event.get("model"),
388 "name": "approval-requested",
389 "value": {
390 "toolCallId": approval.tool_call_id,
391 "toolName": approval.tool_name,
392 "input": approval.input,
393 "approval": {
394 "id": approval.approval_id,
395 "needsApproval": True,
396 },
397 },
398 }
399 yield chunk
400​
401 async def _emit_client_tool_inputs(
402 self,
403 client_requests: List[ClientToolRequest],
404 finish_event: RunFinishedEvent,
405 ) -> AsyncIterator[StreamChunk]:
406 """Emit tool-input-available events using CUSTOM event type."""
407 for client_tool in client_requests:
408 chunk: CustomEvent = {
409 "type": "CUSTOM",
410 "timestamp": int(time.time() * 1000),
411 "model": finish_event.get("model"),
412 "name": "tool-input-available",
413 "value": {
414 "toolCallId": client_tool.tool_call_id,
415 "toolName": client_tool.tool_name,
416 "input": client_tool.input,
417 },
418 }
419 yield chunk
420​
421 async def _emit_tool_results(
422 self,
423 results: List[ToolResult],
424 finish_event: RunFinishedEvent,
425 ) -> AsyncIterator[StreamChunk]:
426 """Emit TOOL_CALL_END events and add to messages."""
427 for result in results:
428 content = json.dumps(result.result)
429​
430 chunk: ToolCallEndEvent = {
431 "type": "TOOL_CALL_END",
432 "timestamp": int(time.time() * 1000),
433 "model": finish_event.get("model"),
434 "toolCallId": result.tool_call_id,
435 "toolName": result.tool_name,
436 "result": content,
437 }
438 yield chunk
439​
440 # Add tool result message
441 tool_message: ModelMessage = {
442 "role": "tool",
443 "content": content,
444 "toolCallId": result.tool_call_id,
445 }
446 self.messages.append(tool_message)
447​
448 def _get_pending_tool_calls_from_messages(self) -> List[ToolCall]:
449 """Get tool calls that don't have results yet."""
450 completed_tool_ids = set(
451 msg.get("toolCallId")
452 for msg in self.messages
453 if msg.get("role") == "tool" and msg.get("toolCallId")
454 )
455​
456 pending: List[ToolCall] = []
457 for message in self.messages:
458 if message.get("role") == "assistant" and message.get("toolCalls"):
459 for tool_call in message["toolCalls"]:
460 if tool_call["id"] not in completed_tool_ids:
461 pending.append(tool_call)
462​
463 return pending
464​
465 def _create_synthetic_finished_event(self) -> RunFinishedEvent:
466 """Create a synthetic RUN_FINISHED event for pending tool calls."""
467 return {
468 "type": "RUN_FINISHED",
469 "runId": self._create_id("pending"),
470 "model": self.options.model,
471 "timestamp": int(time.time() * 1000),
472 "finishReason": "tool_calls",
473 }
474​
475 def _should_continue(self) -> bool:
476 """Check if the loop should continue."""
477 if self.cycle_phase == CyclePhase.EXECUTE_TOOL_CALLS:
478 return True
479​
480 return self.loop_strategy(
481 {
482 "iterationCount": self.iteration_count,
483 "messages": self.messages,
484 "finishReason": self.last_finish_reason,
485 }
486 ) and self.tool_phase == ToolPhaseResult.CONTINUE
487​
488 def _set_tool_phase(self, phase: ToolPhaseResult) -> None:
489 """Set the tool phase."""
490 self.tool_phase = phase
491 if phase == ToolPhaseResult.WAIT:
492 self.should_emit_stream_end = False
493​
494 def _create_id(self, prefix: str) -> str:
495 """Create a unique ID with a prefix."""
496 import random
497 import string
498​
499 timestamp = int(time.time() * 1000)
500 random_suffix = "".join(
501 random.choices(string.ascii_lowercase + string.digits, k=7)
502 )
503 return f"{prefix}-{timestamp}-{random_suffix}"
504​
505​
506# ============================================================================
507# Main chat function
508# ============================================================================
509​
510​
511async def chat(
512 adapter: BaseAdapter,
513 model: str,
514 messages: List[ModelMessage],
515 tools: Optional[List[Tool]] = None,
516 system_prompts: Optional[List[str]] = None,
517 agent_loop_strategy: Optional[AgentLoopStrategy] = None,
518 options: Optional[Dict[str, Any]] = None,
519 provider_options: Optional[Dict[str, Any]] = None,
520) -> AsyncIterator[StreamChunk]:
521 """
522 Standalone chat streaming function with automatic tool execution loop.
523​
524 Args:
525 adapter: AI adapter instance to use
526 model: Model name (must be supported by the adapter)
527 messages: Conversation messages
528 tools: Optional tools for function calling (auto-executed)
529 system_prompts: Optional system prompts to prepend
530 agent_loop_strategy: Optional strategy for controlling tool execution loop
531 options: Common options (temperature, max_tokens, etc.)
532 provider_options: Provider-specific options
533​
534 Yields:
535 StreamChunk objects representing different events during the conversation
536​
537 Example:
538 >>> async for chunk in chat(
539 ... adapter=anthropic_adapter,
540 ... model="claude-3-5-sonnet-20241022",
541 ... messages=[{"role": "user", "content": "Hello!"}],
542 ... tools=[weather_tool],
543 ... ):
544 ... if chunk["type"] == "content":
545 ... print(chunk["delta"], end="", flush=True)
546 """
547 chat_options = ChatOptions(
548 model=model,
549 messages=messages,
550 tools=tools,
551 system_prompts=system_prompts,
552 agent_loop_strategy=agent_loop_strategy,
553 options=options,
554 provider_options=provider_options,
555 )
556​
557 engine = ChatEngine(adapter, chat_options)
558​
559 async for chunk in engine.chat():
560 yield chunk