diff --git a/src/langbot/pkg/api/http/controller/groups/monitoring.py b/src/langbot/pkg/api/http/controller/groups/monitoring.py index 65640b6e6..29dedcafb 100644 --- a/src/langbot/pkg/api/http/controller/groups/monitoring.py +++ b/src/langbot/pkg/api/http/controller/groups/monitoring.py @@ -138,6 +138,39 @@ class MonitoringRouterGroup(group.RouterGroup): } ) + @self.route('/tool-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) + async def get_tool_calls() -> str: + """Get tool call records""" + bot_ids = quart.request.args.getlist('botId') + pipeline_ids = quart.request.args.getlist('pipelineId') + session_ids = quart.request.args.getlist('sessionId') + start_time_str = quart.request.args.get('startTime') + end_time_str = quart.request.args.get('endTime') + limit = int(quart.request.args.get('limit', 100)) + offset = int(quart.request.args.get('offset', 0)) + + start_time = parse_iso_datetime(start_time_str) + end_time = parse_iso_datetime(end_time_str) + + tool_calls, total = await self.ap.monitoring_service.get_tool_calls( + bot_ids=bot_ids if bot_ids else None, + pipeline_ids=pipeline_ids if pipeline_ids else None, + session_ids=session_ids if session_ids else None, + start_time=start_time, + end_time=end_time, + limit=limit, + offset=offset, + ) + + return self.success( + data={ + 'tool_calls': tool_calls, + 'total': total, + 'limit': limit, + 'offset': offset, + } + ) + @self.route('/embedding-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) async def get_embedding_calls() -> str: """Get embedding call records""" @@ -284,6 +317,16 @@ class MonitoringRouterGroup(group.RouterGroup): offset=0, ) + # Get tool calls + tool_calls, tool_calls_total = await self.ap.monitoring_service.get_tool_calls( + bot_ids=bot_ids if bot_ids else None, + pipeline_ids=pipeline_ids if pipeline_ids else None, + start_time=start_time, + end_time=end_time, + limit=limit, + offset=0, + ) + # Get sessions sessions, sessions_total = await self.ap.monitoring_service.get_sessions( bot_ids=bot_ids if bot_ids else None, @@ -318,12 +361,14 @@ class MonitoringRouterGroup(group.RouterGroup): 'overview': overview, 'messages': messages, 'llmCalls': llm_calls, + 'toolCalls': tool_calls, 'embeddingCalls': embedding_calls, 'sessions': sessions, 'errors': errors, 'totalCount': { 'messages': messages_total, 'llmCalls': llm_calls_total, + 'toolCalls': tool_calls_total, 'embeddingCalls': embedding_calls_total, 'sessions': sessions_total, 'errors': errors_total, diff --git a/src/langbot/pkg/api/http/service/maintenance.py b/src/langbot/pkg/api/http/service/maintenance.py index e755800ee..fa7359cba 100644 --- a/src/langbot/pkg/api/http/service/maintenance.py +++ b/src/langbot/pkg/api/http/service/maintenance.py @@ -243,6 +243,7 @@ class MaintenanceService: tables = { 'messages': persistence_monitoring.MonitoringMessage.id, 'llm_calls': persistence_monitoring.MonitoringLLMCall.id, + 'tool_calls': persistence_monitoring.MonitoringToolCall.id, 'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id, 'errors': persistence_monitoring.MonitoringError.id, 'sessions': persistence_monitoring.MonitoringSession.session_id, diff --git a/src/langbot/pkg/api/http/service/monitoring.py b/src/langbot/pkg/api/http/service/monitoring.py index 3e8e0cde0..46a352ad1 100644 --- a/src/langbot/pkg/api/http/service/monitoring.py +++ b/src/langbot/pkg/api/http/service/monitoring.py @@ -2,6 +2,7 @@ from __future__ import annotations import uuid import datetime +import json import sqlalchemy from ....core import app @@ -50,6 +51,12 @@ class MonitoringService: persistence_monitoring.MonitoringLLMCall.timestamp, persistence_monitoring.MonitoringLLMCall.id, ), + ( + 'monitoring_tool_calls', + persistence_monitoring.MonitoringToolCall, + persistence_monitoring.MonitoringToolCall.timestamp, + persistence_monitoring.MonitoringToolCall.id, + ), ( 'monitoring_embedding_calls', persistence_monitoring.MonitoringEmbeddingCall, @@ -131,6 +138,68 @@ class MonitoringService: await autocommit_conn.execute(sqlalchemy.text('PRAGMA wal_checkpoint(TRUNCATE)')) await autocommit_conn.execute(sqlalchemy.text('VACUUM')) + def _serialize_tool_payload(self, payload: object, max_length: int = 20000) -> str | None: + """Serialize tool arguments/results for monitoring storage.""" + if payload is None: + return None + + if isinstance(payload, str): + text = payload + else: + try: + text = json.dumps(payload, ensure_ascii=False, default=str) + except Exception: + text = str(payload) + + if len(text) <= max_length: + return text + + return f'{text[:max_length]}... [truncated {len(text) - max_length} chars]' + + async def _get_message_for_tool_context( + self, + message_id: str | None = None, + session_id: str | None = None, + ): + if message_id: + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_monitoring.MonitoringMessage).where( + persistence_monitoring.MonitoringMessage.id == message_id + ) + ) + row = result.first() + if row: + return row[0] + + if not session_id: + return None + + user_query = ( + sqlalchemy.select(persistence_monitoring.MonitoringMessage) + .where( + sqlalchemy.and_( + persistence_monitoring.MonitoringMessage.session_id == session_id, + persistence_monitoring.MonitoringMessage.role == 'user', + ) + ) + .order_by(persistence_monitoring.MonitoringMessage.timestamp.desc()) + .limit(1) + ) + result = await self.ap.persistence_mgr.execute_async(user_query) + row = result.first() + if row: + return row[0] + + any_query = ( + sqlalchemy.select(persistence_monitoring.MonitoringMessage) + .where(persistence_monitoring.MonitoringMessage.session_id == session_id) + .order_by(persistence_monitoring.MonitoringMessage.timestamp.desc()) + .limit(1) + ) + result = await self.ap.persistence_mgr.execute_async(any_query) + row = result.first() + return row[0] if row else None + # ========== Recording Methods ========== async def record_message( @@ -220,6 +289,57 @@ class MonitoringService: return call_id + async def record_tool_call( + self, + tool_name: str, + tool_source: str, + duration: int, + status: str = 'success', + bot_id: str | None = None, + bot_name: str | None = None, + pipeline_id: str | None = None, + pipeline_name: str | None = None, + session_id: str | None = None, + message_id: str | None = None, + arguments: object | None = None, + result: object | None = None, + error_message: str | None = None, + ) -> str: + """Record a tool call.""" + context_message = await self._get_message_for_tool_context(message_id=message_id, session_id=session_id) + if context_message: + bot_id = bot_id or context_message.bot_id + bot_name = bot_name or context_message.bot_name + pipeline_id = pipeline_id or context_message.pipeline_id + pipeline_name = pipeline_name or context_message.pipeline_name + session_id = session_id or context_message.session_id + message_id = message_id or context_message.id + + call_id = str(uuid.uuid4()) + call_data = { + 'id': call_id, + 'timestamp': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), + 'tool_name': tool_name, + 'tool_source': tool_source, + 'duration': max(0, duration), + 'status': status, + 'bot_id': bot_id or 'unknown', + 'bot_name': bot_name or 'Unknown', + 'pipeline_id': pipeline_id or 'unknown', + 'pipeline_name': pipeline_name or 'Unknown', + 'session_id': session_id, + 'message_id': message_id, + 'arguments': self._serialize_tool_payload(arguments), + 'result': self._serialize_tool_payload(result), + 'error_message': self._serialize_tool_payload(error_message), + } + + await self.ap.persistence_mgr.execute_async( + sqlalchemy.insert(persistence_monitoring.MonitoringToolCall).values(call_data) + ) + + return call_id + async def record_embedding_call( self, model_name: str, @@ -749,6 +869,58 @@ class MonitoringService: total, ) + async def get_tool_calls( + self, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + session_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + limit: int = 100, + offset: int = 0, + ) -> tuple[list[dict], int]: + """Get tool calls with filters""" + conditions = [] + + if bot_ids: + conditions.append(persistence_monitoring.MonitoringToolCall.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(persistence_monitoring.MonitoringToolCall.pipeline_id.in_(pipeline_ids)) + if session_ids: + conditions.append(persistence_monitoring.MonitoringToolCall.session_id.in_(session_ids)) + if start_time: + conditions.append(persistence_monitoring.MonitoringToolCall.timestamp >= start_time) + if end_time: + conditions.append(persistence_monitoring.MonitoringToolCall.timestamp <= end_time) + + count_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringToolCall.id)) + if conditions: + count_query = count_query.where(sqlalchemy.and_(*conditions)) + + count_result = await self.ap.persistence_mgr.execute_async(count_query) + total = count_result.scalar() or 0 + + query = sqlalchemy.select(persistence_monitoring.MonitoringToolCall).order_by( + persistence_monitoring.MonitoringToolCall.timestamp.desc() + ) + if conditions: + query = query.where(sqlalchemy.and_(*conditions)) + + query = query.limit(limit).offset(offset) + + result = await self.ap.persistence_mgr.execute_async(query) + tool_calls_rows = result.all() + + return ( + [ + self.ap.persistence_mgr.serialize_model( + persistence_monitoring.MonitoringToolCall, row[0] if isinstance(row, tuple) else row + ) + for row in tool_calls_rows + ], + total, + ) + async def get_embedding_calls( self, start_time: datetime.datetime | None = None, @@ -971,6 +1143,34 @@ class MonitoringService: else: error_llm_calls += 1 + # Get tool calls for this session + tool_query = ( + sqlalchemy.select(persistence_monitoring.MonitoringToolCall) + .where(persistence_monitoring.MonitoringToolCall.session_id == session_id) + .order_by(persistence_monitoring.MonitoringToolCall.timestamp.asc()) + ) + tool_result = await self.ap.persistence_mgr.execute_async(tool_query) + tool_rows = tool_result.all() + + tool_calls = [ + self.ap.persistence_mgr.serialize_model( + persistence_monitoring.MonitoringToolCall, row[0] if isinstance(row, tuple) else row + ) + for row in tool_rows + ] + + total_tool_calls = len(tool_rows) + success_tool_calls = 0 + error_tool_calls = 0 + total_tool_duration = 0 + for row in tool_rows: + tool_call = row[0] if isinstance(row, tuple) else row + total_tool_duration += tool_call.duration + if tool_call.status == 'success': + success_tool_calls += 1 + else: + error_tool_calls += 1 + # Get errors for this session error_query = ( sqlalchemy.select(persistence_monitoring.MonitoringError) @@ -1014,6 +1214,14 @@ class MonitoringService: 'total_tokens': total_tokens, 'average_duration_ms': int(total_duration / total_llm_calls) if total_llm_calls > 0 else 0, }, + 'tool_calls': tool_calls, + 'tool_stats': { + 'total_calls': total_tool_calls, + 'success_calls': success_tool_calls, + 'error_calls': error_tool_calls, + 'total_duration_ms': total_tool_duration, + 'average_duration_ms': int(total_tool_duration / total_tool_calls) if total_tool_calls > 0 else 0, + }, 'errors': errors, 'session_duration_seconds': session_duration_seconds, } diff --git a/src/langbot/pkg/entity/persistence/monitoring.py b/src/langbot/pkg/entity/persistence/monitoring.py index 01e4fdd39..f594b8187 100644 --- a/src/langbot/pkg/entity/persistence/monitoring.py +++ b/src/langbot/pkg/entity/persistence/monitoring.py @@ -49,6 +49,28 @@ class MonitoringLLMCall(Base): message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) # Associated message ID +class MonitoringToolCall(Base): + """Tool call records""" + + __tablename__ = 'monitoring_tool_calls' + + id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True) + timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True) + tool_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + tool_source = sqlalchemy.Column(sqlalchemy.String(50), nullable=False) # native, plugin, mcp, skill + duration = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) # milliseconds + status = sqlalchemy.Column(sqlalchemy.String(50), nullable=False) # success, error + bot_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True) + bot_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + pipeline_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True) + pipeline_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + session_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) + message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) + arguments = sqlalchemy.Column(sqlalchemy.Text, nullable=True) + result = sqlalchemy.Column(sqlalchemy.Text, nullable=True) + error_message = sqlalchemy.Column(sqlalchemy.Text, nullable=True) + + class MonitoringSession(Base): """Session tracking records""" diff --git a/src/langbot/pkg/persistence/migrations/dbm026_monitoring_tool_calls.py b/src/langbot/pkg/persistence/migrations/dbm026_monitoring_tool_calls.py new file mode 100644 index 000000000..8bd71b10c --- /dev/null +++ b/src/langbot/pkg/persistence/migrations/dbm026_monitoring_tool_calls.py @@ -0,0 +1,17 @@ +from langbot.pkg.entity.persistence import monitoring as persistence_monitoring +from .. import migration + + +@migration.migration_class(26) +class DBMigrateMonitoringToolCalls(migration.DBMigration): + """Add monitoring_tool_calls table""" + + async def upgrade(self): + """Upgrade""" + async with self.ap.persistence_mgr.get_db_engine().begin() as conn: + await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.create, checkfirst=True) + + async def downgrade(self): + """Downgrade""" + async with self.ap.persistence_mgr.get_db_engine().begin() as conn: + await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.drop, checkfirst=True) diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index 642024824..60a16ce7e 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -1,6 +1,7 @@ from __future__ import annotations import typing +import time from typing import TYPE_CHECKING import langbot_plugin.api.entities.builtin.resource.tool as resource_tool @@ -142,21 +143,130 @@ class ToolManager: return tools + def _get_query_session_id(self, query: pipeline_query.Query) -> str | None: + launcher_type = getattr(query, 'launcher_type', None) + launcher_id = getattr(query, 'launcher_id', None) + if launcher_type is None or launcher_id is None: + return None + + launcher_type_value = launcher_type.value if hasattr(launcher_type, 'value') else launcher_type + return f'{launcher_type_value}_{launcher_id}' + + async def _record_tool_call( + self, + *, + name: str, + source: str, + parameters: dict, + query: pipeline_query.Query, + duration_ms: int, + status: str, + result: typing.Any = None, + error_message: str | None = None, + ) -> None: + monitoring_service = getattr(self.ap, 'monitoring_service', None) + if not monitoring_service: + return + + variables = getattr(query, 'variables', {}) or {} + message_id = variables.get('_monitoring_message_id') if isinstance(variables, dict) else None + bot_name = variables.get('_monitoring_bot_name') if isinstance(variables, dict) else None + pipeline_name = variables.get('_monitoring_pipeline_name') if isinstance(variables, dict) else None + + try: + await monitoring_service.record_tool_call( + tool_name=name, + tool_source=source, + duration=duration_ms, + status=status, + bot_id=getattr(query, 'bot_uuid', None), + bot_name=bot_name, + pipeline_name=pipeline_name, + session_id=self._get_query_session_id(query), + message_id=message_id, + arguments=parameters, + result=result, + error_message=error_message, + ) + except Exception as e: + self.ap.logger.warning(f'Failed to record tool call: {e}') + + async def _invoke_tool_with_monitoring( + self, + *, + source: str, + name: str, + parameters: dict, + query: pipeline_query.Query, + invoke: typing.Callable[[], typing.Awaitable[typing.Any]], + ) -> typing.Any: + start_time = time.perf_counter() + try: + result = await invoke() + except Exception as e: + duration_ms = int((time.perf_counter() - start_time) * 1000) + await self._record_tool_call( + name=name, + source=source, + parameters=parameters, + query=query, + duration_ms=duration_ms, + status='error', + error_message=str(e), + ) + raise + + duration_ms = int((time.perf_counter() - start_time) * 1000) + await self._record_tool_call( + name=name, + source=source, + parameters=parameters, + query=query, + duration_ms=duration_ms, + status='success', + result=result, + ) + return result + async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any: from langbot.pkg.telemetry import features as telemetry_features if await self.native_tool_loader.has_tool(name): telemetry_features.increment(query, 'tool_calls', 'native') - return await self.native_tool_loader.invoke_tool(name, parameters, query) + return await self._invoke_tool_with_monitoring( + source='native', + name=name, + parameters=parameters, + query=query, + invoke=lambda: self.native_tool_loader.invoke_tool(name, parameters, query), + ) if await self.plugin_tool_loader.has_tool(name): telemetry_features.increment(query, 'tool_calls', 'plugin') - return await self.plugin_tool_loader.invoke_tool(name, parameters, query) + return await self._invoke_tool_with_monitoring( + source='plugin', + name=name, + parameters=parameters, + query=query, + invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query), + ) if await self.mcp_tool_loader.has_tool(name): telemetry_features.increment(query, 'tool_calls', 'mcp') - return await self.mcp_tool_loader.invoke_tool(name, parameters, query) + return await self._invoke_tool_with_monitoring( + source='mcp', + name=name, + parameters=parameters, + query=query, + invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query), + ) if await self.skill_tool_loader.has_tool(name): telemetry_features.increment(query, 'tool_calls', 'skill') - return await self.skill_tool_loader.invoke_tool(name, parameters, query) + return await self._invoke_tool_with_monitoring( + source='skill', + name=name, + parameters=parameters, + query=query, + invoke=lambda: self.skill_tool_loader.invoke_tool(name, parameters, query), + ) raise ToolNotFoundError(name) async def shutdown(self): diff --git a/web/src/app/home/bots/BotDetailContent.tsx b/web/src/app/home/bots/BotDetailContent.tsx index 8440e47ac..6c0a9ccc1 100644 --- a/web/src/app/home/bots/BotDetailContent.tsx +++ b/web/src/app/home/bots/BotDetailContent.tsx @@ -191,45 +191,47 @@ export default function BotDetailContent({ id }: { id: string }) { onValueChange={setActiveTab} className="flex flex-1 flex-col min-h-0" > - - - - {t('bots.configuration')} - - - - {t('bots.logs')} - - - - {t('bots.sessionMonitor.title')} - {activeTab === 'sessions' && ( - - )} - - +
+ + + + {t('bots.configuration')} + + + + {t('bots.logs')} + + + + {t('bots.sessionMonitor.title')} + + + {activeTab === 'sessions' && ( + + )} +
{/* Tab: Configuration */} Promise; } @@ -100,6 +133,10 @@ const BotSessionMonitor = forwardRef< const [feedbackMap, setFeedbackMap] = useState< Record >({}); + const [toolCalls, setToolCalls] = useState([]); + const [expandedToolCallIds, setExpandedToolCallIds] = useState< + Record + >({}); const messagesContainerRef = useRef(null); const { admins, reload: reloadAdmins } = useBotAdmins(botId); const [adminsDialogOpen, setAdminsDialogOpen] = useState(false); @@ -189,6 +226,7 @@ const BotSessionMonitor = forwardRef< const loadMessages = useCallback( async (sessionId: string) => { setLoadingMessages(true); + setExpandedToolCallIds({}); try { const messagesRes = await httpClient.getSessionMessages(sessionId); const sorted = (messagesRes.messages ?? []).sort( @@ -197,6 +235,18 @@ const BotSessionMonitor = forwardRef< ); setMessages(sorted); + try { + const analysisRes = await httpClient.get<{ + tool_calls?: SessionToolCall[]; + }>( + `/api/v1/monitoring/sessions/${encodeURIComponent(sessionId)}/analysis`, + ); + setToolCalls(analysisRes?.tool_calls ?? []); + } catch (analysisError) { + console.error('Failed to load session tool calls:', analysisError); + setToolCalls([]); + } + // Collect user message IDs for feedback matching const userMsgIds = new Set( sorted.filter((m) => !m.role || m.role === 'user').map((m) => m.id), @@ -240,11 +290,14 @@ const BotSessionMonitor = forwardRef< loadMessages(selectedSessionId); } else { setMessages([]); + setToolCalls([]); + setExpandedToolCallIds({}); + setFeedbackMap({}); } }, [selectedSessionId, loadMessages]); useEffect(() => { - if (messages.length === 0) return; + if (messages.length === 0 && toolCalls.length === 0) return; // Wait for DOM to render the new messages before scrolling requestAnimationFrame(() => { const container = messagesContainerRef.current; @@ -256,7 +309,7 @@ const BotSessionMonitor = forwardRef< scrollTarget.scrollTop = scrollTarget.scrollHeight; } }); - }, [messages]); + }, [messages, toolCalls]); const parseMessageChain = (content: string): MessageChainComponent[] => { try { @@ -431,6 +484,69 @@ const BotSessionMonitor = forwardRef< return `${diffDays}d`; }; + const formatDuration = (durationMs: number): string => { + if (!durationMs) return '0ms'; + if (durationMs < 1000) return `${durationMs}ms`; + return `${(durationMs / 1000).toFixed(2)}s`; + }; + + const truncateToolDetail = (value?: string | null): string => { + if (!value) return ''; + return value.length > 600 ? `${value.slice(0, 600)}...` : value; + }; + + const toggleToolCallDetails = (toolCallId: string) => { + setExpandedToolCallIds((previous) => ({ + ...previous, + [toolCallId]: !previous[toolCallId], + })); + }; + + const feedbackByMessageId = useMemo(() => { + const map: Record = {}; + + for (let index = 0; index < messages.length; index++) { + const msg = messages[index]; + if (isUserMessage(msg)) continue; + + for (let previousIndex = index - 1; previousIndex >= 0; previousIndex--) { + const previousMessage = messages[previousIndex]; + if (isUserMessage(previousMessage)) { + const feedback = feedbackMap[previousMessage.id]; + if (feedback) { + map[msg.id] = feedback; + } + break; + } + } + } + + return map; + }, [feedbackMap, messages]); + + const timelineItems = useMemo(() => { + const messageItems: SessionTimelineItem[] = messages.map( + (message, index) => ({ + id: `message-${message.id}`, + type: 'message', + timestamp: parseTimestamp(message.timestamp).getTime(), + order: index * 2, + message, + }), + ); + const toolItems: SessionTimelineItem[] = toolCalls.map((toolCall, index) => ({ + id: `tool-${toolCall.id}`, + type: 'tool', + timestamp: parseTimestamp(toolCall.timestamp).getTime(), + order: index * 2 + 1, + toolCall, + })); + + return [...messageItems, ...toolItems].sort( + (a, b) => a.timestamp - b.timestamp || a.order - b.order, + ); + }, [messages, toolCalls]); + const selectedSession = sessions.find( (s) => s.session_id === selectedSessionId, ); @@ -612,29 +728,164 @@ const BotSessionMonitor = forwardRef<
{t('bots.sessionMonitor.loading')}
- ) : messages.length === 0 ? ( + ) : timelineItems.length === 0 ? (
{t('bots.sessionMonitor.noMessages')}
) : ( - messages.map((msg, msgIndex) => { + timelineItems.map((item) => { + if (item.type === 'tool') { + const call = item.toolCall; + const hasToolDetails = Boolean( + call.arguments || call.result || call.error_message, + ); + const expandedToolCall = Boolean( + expandedToolCallIds[call.id], + ); + const detailsId = `tool-call-details-${call.id}`; + return ( +
+
+ + + {hasToolDetails && expandedToolCall && ( +
+ {(call.arguments || call.result) && ( +
+ {call.arguments && ( +
+
+ {t( + 'monitoring.toolCalls.arguments', + { + defaultValue: '参数', + }, + )} +
+
+                                            {truncateToolDetail(
+                                              call.arguments,
+                                            )}
+                                          
+
+ )} + {call.result && ( +
+
+ {t('monitoring.toolCalls.result', { + defaultValue: '结果', + })} +
+
+                                            {truncateToolDetail(call.result)}
+                                          
+
+ )} +
+ )} + + {call.error_message && ( +
+ {call.error_message} +
+ )} +
+ )} + +
+ + {t('monitoring.toolCalls.title', { + defaultValue: '工具调用', + })} + + + {formatTime(call.timestamp)} + + {hasToolDetails && ( + <> + · + + {expandedToolCall + ? t( + 'monitoring.toolCalls.hideDetails', + { + defaultValue: '隐藏详情', + }, + ) + : t( + 'monitoring.toolCalls.showDetails', + { + defaultValue: '查看详情', + }, + )} + + + )} +
+
+
+ ); + } + + const msg = item.message; const isUser = isUserMessage(msg); const isDiscarded = msg.status === 'discarded' || msg.pipeline_id === PIPELINE_DISCARD; - // For bot replies, find feedback linked to the preceding user message - let msgFeedback: SessionFeedback | undefined; - if (!isUser) { - for (let i = msgIndex - 1; i >= 0; i--) { - if (isUserMessage(messages[i])) { - msgFeedback = feedbackMap[messages[i].id]; - break; - } - } - } + const msgFeedback = feedbackByMessageId[msg.id]; return (
1200 ? `${value.slice(0, 1200)}...` : value; +} + function roleLabel(message: MonitoringMessage | undefined) { const role = message?.role?.toLowerCase(); if (role === 'assistant') return 'assistant'; @@ -147,6 +153,16 @@ export function ConversationTurnList({ onToggleTurn, }: ConversationTurnListProps) { const { t } = useTranslation(); + const [expandedToolCallIds, setExpandedToolCallIds] = React.useState< + Record + >({}); + + const toggleToolCallDetails = (toolCallKey: string) => { + setExpandedToolCallIds((previous) => ({ + ...previous, + [toolCallKey]: !previous[toolCallKey], + })); + }; return (
@@ -268,6 +284,12 @@ export function ConversationTurnList({ icon={} label={`${turn.llmCalls.length} LLM`} /> + {turn.toolCalls.length > 0 && ( + } + label={`${turn.toolCalls.length} tools`} + /> + )} } label={`${turn.totalTokens.toLocaleString()} tokens`} @@ -434,6 +456,157 @@ export function ConversationTurnList({
+
+

+ + {t('monitoring.toolCalls.title', { + defaultValue: '工具调用', + })}{' '} + ({turn.toolCalls.length}) +

+ +
+ + + call.status === 'error', + ).length, + )} + /> +
+ +
+ {turn.toolCalls.length > 0 ? ( + turn.toolCalls.map((call, index) => { + const toolCallKey = `${turn.id}:${call.id}`; + const hasToolDetails = Boolean( + call.arguments || call.result || call.errorMessage, + ); + const expandedToolCall = Boolean( + expandedToolCallIds[toolCallKey], + ); + const detailsId = `monitoring-tool-call-details-${call.id}`; + + return ( +
+ + + {hasToolDetails && expandedToolCall && ( +
+ {call.arguments && ( +
+
+ {t('monitoring.toolCalls.arguments', { + defaultValue: '参数', + })} +
+
+                                        {truncateDetail(call.arguments)}
+                                      
+
+ )} + {call.result && ( +
+
+ {t('monitoring.toolCalls.result', { + defaultValue: '结果', + })} +
+
+                                        {truncateDetail(call.result)}
+                                      
+
+ )} + {call.errorMessage && ( +
+ {call.errorMessage} +
+ )} +
+ )} +
+ ); + }) + ) : ( +
+ {t('monitoring.toolCalls.noToolCalls', { + defaultValue: '未记录工具调用', + })} +
+ )} +
+
+ {turn.errors.length > 0 && (

diff --git a/web/src/app/home/monitoring/hooks/useMonitoringData.ts b/web/src/app/home/monitoring/hooks/useMonitoringData.ts index d88d25ab4..8b587cc43 100644 --- a/web/src/app/home/monitoring/hooks/useMonitoringData.ts +++ b/web/src/app/home/monitoring/hooks/useMonitoringData.ts @@ -106,6 +106,9 @@ export function useMonitoringData(filterState: FilterState) { const llmCalls = Array.isArray(response?.llmCalls) ? response.llmCalls : []; + const toolCalls = Array.isArray(response?.toolCalls) + ? response.toolCalls + : []; const embeddingCalls = Array.isArray(response?.embeddingCalls) ? response.embeddingCalls : []; @@ -116,6 +119,7 @@ export function useMonitoringData(filterState: FilterState) { const totalCount = response?.totalCount ?? { messages: messages.length, llmCalls: llmCalls.length, + toolCalls: toolCalls.length, embeddingCalls: embeddingCalls.length, sessions: sessions.length, errors: errors.length, @@ -207,6 +211,41 @@ export function useMonitoringData(filterState: FilterState) { messageId: call.message_id, }), ), + toolCalls: toolCalls.map( + (call: { + id: string; + timestamp: string; + tool_name: string; + tool_source: string; + duration: number; + status: string; + bot_id: string; + bot_name: string; + pipeline_id: string; + pipeline_name: string; + session_id?: string; + message_id?: string; + arguments?: string; + result?: string; + error_message?: string; + }) => ({ + id: call.id, + timestamp: parseUTCTimestamp(call.timestamp), + toolName: call.tool_name, + toolSource: call.tool_source, + duration: call.duration, + status: call.status as 'success' | 'error', + botId: call.bot_id, + botName: call.bot_name, + pipelineId: call.pipeline_id, + pipelineName: call.pipeline_name, + sessionId: call.session_id, + messageId: call.message_id, + arguments: call.arguments, + result: call.result, + errorMessage: call.error_message, + }), + ), embeddingCalls: embeddingCalls.map( (call: { id: string; @@ -300,6 +339,7 @@ export function useMonitoringData(filterState: FilterState) { totalCount: { messages: totalCount.messages, llmCalls: totalCount.llmCalls, + toolCalls: totalCount.toolCalls ?? toolCalls.length, embeddingCalls: totalCount.embeddingCalls || 0, sessions: totalCount.sessions, errors: totalCount.errors, diff --git a/web/src/app/home/monitoring/page.tsx b/web/src/app/home/monitoring/page.tsx index 5c7eff221..1ae5dd945 100644 --- a/web/src/app/home/monitoring/page.tsx +++ b/web/src/app/home/monitoring/page.tsx @@ -104,8 +104,9 @@ function MonitoringPageContent() { data?.messages || [], data?.llmCalls || [], data?.errors || [], + data?.toolCalls || [], ), - [data?.messages, data?.llmCalls, data?.errors], + [data?.messages, data?.llmCalls, data?.errors, data?.toolCalls], ); // State for expanded errors diff --git a/web/src/app/home/monitoring/types/monitoring.ts b/web/src/app/home/monitoring/types/monitoring.ts index 2fc52137b..06e6cc677 100644 --- a/web/src/app/home/monitoring/types/monitoring.ts +++ b/web/src/app/home/monitoring/types/monitoring.ts @@ -38,6 +38,24 @@ export interface LLMCall { messageId?: string; } +export interface ToolCall { + id: string; + timestamp: Date; + toolName: string; + toolSource: 'native' | 'plugin' | 'mcp' | 'skill' | string; + duration: number; + status: 'success' | 'error'; + botId: string; + botName: string; + pipelineId: string; + pipelineName: string; + sessionId?: string; + messageId?: string; + arguments?: string; + result?: string; + errorMessage?: string; +} + export interface EmbeddingCall { id: string; timestamp: Date; @@ -202,6 +220,7 @@ export interface MonitoringData { overview: OverviewMetrics; messages: MonitoringMessage[]; llmCalls: LLMCall[]; + toolCalls: ToolCall[]; embeddingCalls: EmbeddingCall[]; modelCalls: ModelCall[]; sessions: SessionInfo[]; @@ -211,6 +230,7 @@ export interface MonitoringData { totalCount: { messages: number; llmCalls: number; + toolCalls?: number; embeddingCalls: number; sessions: number; errors: number; diff --git a/web/src/app/home/monitoring/utils/conversationTurns.ts b/web/src/app/home/monitoring/utils/conversationTurns.ts index 49b49fb6b..658540900 100644 --- a/web/src/app/home/monitoring/utils/conversationTurns.ts +++ b/web/src/app/home/monitoring/utils/conversationTurns.ts @@ -1,4 +1,9 @@ -import { ErrorLog, LLMCall, MonitoringMessage } from '../types/monitoring'; +import { + ErrorLog, + LLMCall, + MonitoringMessage, + ToolCall, +} from '../types/monitoring'; type MessageRole = 'user' | 'assistant' | 'unknown'; @@ -19,6 +24,7 @@ export interface ConversationTurn { assistantMessages: MonitoringMessage[]; messages: MonitoringMessage[]; llmCalls: LLMCall[]; + toolCalls: ToolCall[]; errors: ErrorLog[]; status: 'success' | 'error' | 'pending'; level: 'info' | 'warning' | 'error' | 'debug'; @@ -26,6 +32,7 @@ export interface ConversationTurn { outputTokens: number; totalTokens: number; totalDuration: number; + totalToolDuration: number; } function normalizeRole( @@ -91,6 +98,7 @@ function createTurn(message: MonitoringMessage): ConversationTurn { assistantMessages: [], messages: [], llmCalls: [], + toolCalls: [], errors: [], status: message.status, level: message.level, @@ -98,6 +106,7 @@ function createTurn(message: MonitoringMessage): ConversationTurn { outputTokens: 0, totalTokens: 0, totalDuration: 0, + totalToolDuration: 0, }; } @@ -174,12 +183,16 @@ export function buildConversationTurns( messages: MonitoringMessage[], llmCalls: LLMCall[], errors: ErrorLog[], + toolCalls: ToolCall[] = [], ): ConversationTurn[] { - const llmMessageIds = new Set( - llmCalls + const activityMessageIds = new Set([ + ...llmCalls .map((call) => call.messageId) .filter((messageId): messageId is string => Boolean(messageId)), - ); + ...toolCalls + .map((call) => call.messageId) + .filter((messageId): messageId is string => Boolean(messageId)), + ]); const visibleMessages = messages .filter((message) => hasRenderableMessageContent(message.messageContent)) .sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); @@ -189,7 +202,7 @@ export function buildConversationTurns( const messageIdToTurn = new Map(); for (const message of visibleMessages) { - const role = normalizeRole(message, llmMessageIds); + const role = normalizeRole(message, activityMessageIds); const previousTurn = lastTurnBySession.get(message.sessionId); const shouldStartTurn = role === 'user' || !previousTurn; const turn = shouldStartTurn ? createTurn(message) : previousTurn; @@ -229,6 +242,25 @@ export function buildConversationTurns( } } + for (const call of toolCalls) { + const turn = + (call.messageId ? messageIdToTurn.get(call.messageId) : undefined) ?? + findTurnBySessionTime(sessionTurns, call.sessionId, call.timestamp); + + if (!turn) { + continue; + } + + turn.toolCalls.push(call); + turn.totalToolDuration += call.duration; + updateTurnActivity(turn, call.timestamp); + + if (call.status === 'error') { + turn.status = 'error'; + turn.level = 'error'; + } + } + for (const error of errors) { const turn = (error.messageId ? messageIdToTurn.get(error.messageId) : undefined) ?? @@ -250,6 +282,9 @@ export function buildConversationTurns( (a, b) => a.timestamp.getTime() - b.timestamp.getTime(), ); turn.llmCalls.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); + turn.toolCalls.sort( + (a, b) => a.timestamp.getTime() - b.timestamp.getTime(), + ); turn.errors.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()); } diff --git a/web/src/app/infra/http/BackendClient.ts b/web/src/app/infra/http/BackendClient.ts index 61995cffb..c4f8b6e3c 100644 --- a/web/src/app/infra/http/BackendClient.ts +++ b/web/src/app/infra/http/BackendClient.ts @@ -1199,8 +1199,10 @@ export class BackendClient extends BaseHttpClient { level: string; platform?: string; user_id?: string; + user_name?: string; runner_name?: string; variables?: string; + role?: string; }>; llmCalls: Array<{ id: string; @@ -1216,9 +1218,27 @@ export class BackendClient extends BaseHttpClient { bot_name: string; pipeline_id: string; pipeline_name: string; + session_id?: string; error_message?: string; message_id?: string; }>; + toolCalls: Array<{ + id: string; + timestamp: string; + tool_name: string; + tool_source: string; + duration: number; + status: string; + bot_id: string; + bot_name: string; + pipeline_id: string; + pipeline_name: string; + session_id?: string; + message_id?: string; + arguments?: string; + result?: string; + error_message?: string; + }>; embeddingCalls: Array<{ id: string; timestamp: string; @@ -1263,6 +1283,7 @@ export class BackendClient extends BaseHttpClient { totalCount: { messages: number; llmCalls: number; + toolCalls?: number; embeddingCalls: number; sessions: number; errors: number; diff --git a/web/src/i18n/locales/en-US.ts b/web/src/i18n/locales/en-US.ts index 61e20ed90..f30ba8536 100644 --- a/web/src/i18n/locales/en-US.ts +++ b/web/src/i18n/locales/en-US.ts @@ -1357,6 +1357,15 @@ const enUS = { avgDuration: 'Avg Duration', calls: 'Calls', }, + toolCalls: { + title: 'Tool Calls', + totalCalls: 'Calls', + duration: 'Tool Duration', + errorCalls: 'Failed Calls', + arguments: 'Arguments', + result: 'Result', + noToolCalls: 'No tool calls recorded', + }, tokens: { totalTokens: 'Total Tokens', inputTokens: 'Input Tokens', diff --git a/web/src/i18n/locales/zh-Hans.ts b/web/src/i18n/locales/zh-Hans.ts index 200a57a5b..61cc70f57 100644 --- a/web/src/i18n/locales/zh-Hans.ts +++ b/web/src/i18n/locales/zh-Hans.ts @@ -1294,6 +1294,15 @@ const zhHans = { avgDuration: '平均耗时', calls: '调用次数', }, + toolCalls: { + title: '工具调用', + totalCalls: '调用次数', + duration: '工具耗时', + errorCalls: '失败次数', + arguments: '参数', + result: '结果', + noToolCalls: '未记录工具调用', + }, tokens: { totalTokens: '总 Token 数', inputTokens: '输入 Token', diff --git a/web/tests/e2e/bot-session-tool-timeline.spec.ts b/web/tests/e2e/bot-session-tool-timeline.spec.ts new file mode 100644 index 000000000..7214f1780 --- /dev/null +++ b/web/tests/e2e/bot-session-tool-timeline.spec.ts @@ -0,0 +1,179 @@ +import { expect, test } from '@playwright/test'; + +import { installLangBotApiMocks } from './fixtures/langbot-api'; + +const botId = 'bot-tool-timeline'; +const sessionId = 'person-tool-timeline-user'; +const botName = 'Tool Timeline Bot'; +const pipelineId = 'pipeline-tool-timeline'; +const pipelineName = 'Tool Timeline Pipeline'; + +function at(minute: number, second = 0) { + return `2026-07-02T10:${String(minute).padStart(2, '0')}:${String( + second, + ).padStart(2, '0')}Z`; +} + +function sessionMessage( + id: string, + role: 'user' | 'assistant', + minute: number, + content: string, +) { + return { + id, + timestamp: at(minute), + bot_id: botId, + bot_name: botName, + pipeline_id: pipelineId, + pipeline_name: pipelineName, + message_content: content, + session_id: sessionId, + status: 'success', + level: 'info', + platform: role === 'user' ? 'person' : 'bot', + user_id: 'timeline-user', + user_name: 'Timeline User', + runner_name: role === 'assistant' ? 'local-agent' : null, + variables: '{}', + role, + }; +} + +function toolCall( + id: string, + minute: number, + toolName: string, + duration: number, + status: 'success' | 'error' = 'success', +) { + return { + id, + timestamp: at(minute, 30), + tool_name: toolName, + tool_source: 'native', + duration, + status, + bot_id: botId, + bot_name: botName, + pipeline_id: pipelineId, + pipeline_name: pipelineName, + session_id: sessionId, + message_id: 'user-message', + arguments: JSON.stringify({ target: toolName }), + result: status === 'success' ? JSON.stringify({ ok: true }) : null, + error_message: status === 'error' ? 'Tool execution failed' : null, + }; +} + +test.describe('bot session monitor tool timeline', () => { + test('renders tool calls as left-side agent events interleaved with messages', async ({ + page, + }) => { + await installLangBotApiMocks(page, { + authenticated: true, + monitoringSessions: [ + { + session_id: sessionId, + bot_id: botId, + bot_name: botName, + pipeline_id: pipelineId, + pipeline_name: pipelineName, + message_count: 3, + start_time: at(0), + last_activity: at(4), + is_active: true, + platform: 'person', + user_id: 'timeline-user', + user_name: 'Timeline User', + }, + ], + sessionMessages: { + [sessionId]: [ + sessionMessage('user-message', 'user', 0, 'Need a timeline check'), + sessionMessage( + 'assistant-step-1', + 'assistant', + 2, + 'Agent step 1: inspected repository files', + ), + sessionMessage( + 'assistant-step-2', + 'assistant', + 4, + 'Agent step 2: test suite finished', + ), + ], + }, + sessionAnalyses: { + [sessionId]: { + session_id: sessionId, + found: true, + tool_calls: [ + toolCall('tool-repo-read', 1, 'repo_file_read', 80), + toolCall('tool-test-run', 3, 'run_test_suite', 140), + ], + }, + }, + }); + + await page.goto(`/home/bots?id=${botId}`); + await page.getByRole('tab', { name: /Sessions/ }).click(); + await page.getByRole('button', { name: /Timeline User/ }).click(); + + await expect(page.getByText('Need a timeline check')).toBeVisible(); + await expect( + page.getByText('repo_file_read', { exact: true }), + ).toBeVisible(); + await expect( + page.getByText('Agent step 1: inspected repository files'), + ).toBeVisible(); + await expect( + page.getByText('run_test_suite', { exact: true }), + ).toBeVisible(); + await expect( + page.getByText('Agent step 2: test suite finished'), + ).toBeVisible(); + await expect(page.getByText('{"target":"repo_file_read"}')).toHaveCount(0); + await expect(page.getByText('{"ok":true}')).toHaveCount(0); + + await expect( + page.locator('div.flex.justify-start').filter({ + hasText: 'repo_file_read', + }), + ).toHaveCount(1); + await expect( + page.locator('div.flex.justify-start').filter({ + hasText: 'run_test_suite', + }), + ).toHaveCount(1); + await expect( + page.locator('div.flex.justify-end').filter({ + hasText: 'repo_file_read', + }), + ).toHaveCount(0); + await expect( + page.locator('div.flex.justify-end').filter({ + hasText: 'run_test_suite', + }), + ).toHaveCount(0); + + const text = await page.locator('body').innerText(); + expect(text.indexOf('Need a timeline check')).toBeLessThan( + text.indexOf('repo_file_read'), + ); + expect(text.indexOf('repo_file_read')).toBeLessThan( + text.indexOf('Agent step 1: inspected repository files'), + ); + expect( + text.indexOf('Agent step 1: inspected repository files'), + ).toBeLessThan(text.indexOf('run_test_suite')); + expect(text.indexOf('run_test_suite')).toBeLessThan( + text.indexOf('Agent step 2: test suite finished'), + ); + + await page.getByText('repo_file_read', { exact: true }).click(); + await expect(page.getByText('{"target":"repo_file_read"}')).toBeVisible(); + await expect(page.getByText('{"ok":true}').first()).toBeVisible(); + }); +}); diff --git a/web/tests/e2e/fixtures/langbot-api.ts b/web/tests/e2e/fixtures/langbot-api.ts index a3c70887f..20f21086d 100644 --- a/web/tests/e2e/fixtures/langbot-api.ts +++ b/web/tests/e2e/fixtures/langbot-api.ts @@ -73,7 +73,10 @@ interface LangBotApiMockState { knowledgeBases: KnowledgeBaseMock[]; mcpServers: MCPServerMock[]; monitoringData: unknown; + monitoringSessions: unknown[]; pipelines: PipelineMock[]; + sessionAnalyses: Record; + sessionMessages: Record; skills: SkillMock[]; } @@ -123,12 +126,14 @@ function emptyMonitoringData() { }, messages: [], llmCalls: [], + toolCalls: [], embeddingCalls: [], sessions: [], errors: [], totalCount: { messages: 0, llmCalls: 0, + toolCalls: 0, embeddingCalls: 0, sessions: 0, errors: 0, @@ -693,6 +698,37 @@ async function handleBackendApi(route: Route, state: LangBotApiMockState) { return fulfillJson(route, state.monitoringData); } + if (path === '/api/v1/monitoring/sessions') { + return fulfillJson(route, { + sessions: state.monitoringSessions, + total: state.monitoringSessions.length, + }); + } + + if (path === '/api/v1/monitoring/messages') { + const sessionId = url.searchParams.get('sessionId') || ''; + const messages = state.sessionMessages[sessionId] || []; + return fulfillJson(route, { + messages, + total: messages.length, + }); + } + + const sessionAnalysisMatch = path.match( + /^\/api\/v1\/monitoring\/sessions\/([^/]+)\/analysis$/, + ); + if (sessionAnalysisMatch) { + const sessionId = decodeURIComponent(sessionAnalysisMatch[1]); + return fulfillJson( + route, + state.sessionAnalyses[sessionId] || { + session_id: sessionId, + found: true, + tool_calls: [], + }, + ); + } + if (path === '/api/v1/monitoring/overview') { const data = state.monitoringData as { overview?: unknown }; return fulfillJson(route, data.overview || emptyMonitoringData().overview); @@ -803,17 +839,30 @@ export async function installLangBotApiMocks( options: { authenticated?: boolean; monitoringData?: unknown; + monitoringSessions?: unknown[]; + sessionAnalyses?: Record; + sessionMessages?: Record; storage?: JsonRecord; } = {}, ) { - const { authenticated = false, monitoringData, storage = {} } = options; + const { + authenticated = false, + monitoringData, + monitoringSessions, + sessionAnalyses, + sessionMessages, + storage = {}, + } = options; const state: LangBotApiMockState = { bots: [], counters: {}, knowledgeBases: [], mcpServers: [], monitoringData: monitoringData || emptyMonitoringData(), + monitoringSessions: monitoringSessions || [], pipelines: [], + sessionAnalyses: sessionAnalyses || {}, + sessionMessages: sessionMessages || {}, skills: [], }; diff --git a/web/tests/e2e/monitoring-turns.spec.ts b/web/tests/e2e/monitoring-turns.spec.ts index 53dbb2870..89c98554f 100644 --- a/web/tests/e2e/monitoring-turns.spec.ts +++ b/web/tests/e2e/monitoring-turns.spec.ts @@ -6,6 +6,7 @@ import { ErrorLog, LLMCall, MonitoringMessage, + ToolCall, } from '../../src/app/home/monitoring/types/monitoring'; const bot = { @@ -93,6 +94,34 @@ function errorLog(id: string, minute: number, messageId: string): ErrorLog { }; } +function toolCall( + id: string, + minute: number, + messageId: string | undefined, + toolName: string, + duration: number, + sessionId = 'session-agent', + status: 'success' | 'error' = 'success', +): ToolCall { + return { + id, + timestamp: time(minute), + toolName, + toolSource: 'native', + duration, + status, + botId: bot.id, + botName: bot.name, + pipelineId: pipeline.id, + pipelineName: pipeline.name, + sessionId, + messageId, + arguments: JSON.stringify({ query: toolName }), + result: status === 'success' ? JSON.stringify({ ok: true }) : undefined, + errorMessage: status === 'error' ? 'Tool failed' : undefined, + }; +} + function rawMessage(message: MonitoringMessage) { return { id: message.id, @@ -151,6 +180,26 @@ function rawError(error: ErrorLog) { }; } +function rawToolCall(call: ToolCall) { + return { + id: call.id, + timestamp: call.timestamp.toISOString(), + tool_name: call.toolName, + tool_source: call.toolSource, + duration: call.duration, + status: call.status, + bot_id: call.botId, + bot_name: call.botName, + pipeline_id: call.pipelineId, + pipeline_name: call.pipelineName, + session_id: call.sessionId, + message_id: call.messageId, + arguments: call.arguments, + result: call.result, + error_message: call.errorMessage, + }; +} + function monitoringScenario() { const messages = [ message( @@ -179,10 +228,16 @@ function monitoringScenario() { llmCall('agent-call-4', 20, 'agent-user-2', 50, 25, 80), ]; const errors = [errorLog('agent-error-1', 12, 'agent-user-1')]; + const toolCalls = [ + toolCall('agent-tool-1', 11, 'agent-user-1', 'repo_search', 90), + toolCall('agent-tool-2', 12, 'agent-user-1', 'run_tests', 150), + toolCall('agent-tool-3', 20, 'agent-user-2', 'rollback_lookup', 70), + ]; return { messages, llmCalls, + toolCalls, errors, }; } @@ -201,12 +256,14 @@ function rawMonitoringData() { }, messages: scenario.messages.map(rawMessage), llmCalls: scenario.llmCalls.map(rawLlmCall), + toolCalls: scenario.toolCalls.map(rawToolCall), embeddingCalls: [], sessions: [], errors: scenario.errors.map(rawError), totalCount: { messages: scenario.messages.length, llmCalls: scenario.llmCalls.length, + toolCalls: scenario.toolCalls.length, embeddingCalls: 0, sessions: 0, errors: scenario.errors.length, @@ -240,6 +297,7 @@ test.describe('monitoring conversation turn grouping', () => { scenario.messages, scenario.llmCalls, scenario.errors, + scenario.toolCalls, ); const agentTurn = turns.find((turn) => turn.id === 'agent-user-1'); @@ -254,9 +312,11 @@ test.describe('monitoring conversation turn grouping', () => { 'Final answer: deployment plan ready', ]); expect(agentTurn?.llmCalls).toHaveLength(3); + expect(agentTurn?.toolCalls).toHaveLength(2); expect(agentTurn?.errors).toHaveLength(1); expect(agentTurn?.totalTokens).toBe(790); expect(agentTurn?.totalDuration).toBe(600); + expect(agentTurn?.totalToolDuration).toBe(240); }); test('starts a new turn for each later user message in the same session', () => { @@ -314,6 +374,30 @@ test.describe('monitoring conversation turn grouping', () => { expect(turns[0].totalTokens).toBe(30); }); + test('attaches tool calls without message ids by session time', () => { + const user = message('tool-fallback-user', 'user', 1, 'Use tool fallback'); + const assistant = message( + 'tool-fallback-assistant', + 'assistant', + 2, + 'Tool fallback reply', + ); + const call = toolCall( + 'tool-fallback-call', + 2, + undefined, + 'memory_lookup', + 45, + ); + + const turns = buildConversationTurns([user, assistant], [], [], [call]); + + expect(turns).toHaveLength(1); + expect(turns[0].toolCalls).toHaveLength(1); + expect(turns[0].toolCalls[0].id).toBe(call.id); + expect(turns[0].totalToolDuration).toBe(45); + }); + test('renders user-only, multi-agent, and multi-turn cases in the monitoring page', async ({ page, }) => { @@ -333,6 +417,7 @@ test.describe('monitoring conversation turn grouping', () => { await expect(page.getByText('Agent step 1: inspect repo')).toBeVisible(); await expect(page.getByText('Assistant +2')).toBeVisible(); await expect(page.getByText('3 LLM')).toBeVisible(); + await expect(page.getByText('2 tools')).toBeVisible(); await expect(page.getByText('790 tokens')).toBeVisible(); await expect(page.getByText('1 errors')).toBeVisible(); await expect(page.getByText('Continue with rollback plan')).toBeVisible(); @@ -354,6 +439,15 @@ test.describe('monitoring conversation turn grouping', () => { await expect(page.getByText('In: 300')).toBeVisible(); await expect(page.getByText('Out: 90')).toBeVisible(); await expect(page.getByText('Total: 390')).toBeVisible(); + await expect(page.getByText('Tool Calls (2)')).toBeVisible(); + await expect(page.getByText('#1 repo_search')).toBeVisible(); + await expect(page.getByText('#2 run_tests')).toBeVisible(); + await expect(page.getByText('Arguments')).toHaveCount(0); + await expect(page.getByText('Result')).toHaveCount(0); + + await page.getByText('#1 repo_search').click(); + await expect(page.getByText('Arguments').first()).toBeVisible(); + await expect(page.getByText('Result').first()).toBeVisible(); await expect(page.getByText('Tool retry failed')).toBeVisible(); }); });