mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-10 13:10:57 +00:00
Add tool call observability
This commit is contained in:
@@ -138,6 +138,39 @@ class MonitoringRouterGroup(group.RouterGroup):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@self.route('/tool-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||||
|
async def get_tool_calls() -> str:
|
||||||
|
"""Get tool call records"""
|
||||||
|
bot_ids = quart.request.args.getlist('botId')
|
||||||
|
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||||
|
session_ids = quart.request.args.getlist('sessionId')
|
||||||
|
start_time_str = quart.request.args.get('startTime')
|
||||||
|
end_time_str = quart.request.args.get('endTime')
|
||||||
|
limit = int(quart.request.args.get('limit', 100))
|
||||||
|
offset = int(quart.request.args.get('offset', 0))
|
||||||
|
|
||||||
|
start_time = parse_iso_datetime(start_time_str)
|
||||||
|
end_time = parse_iso_datetime(end_time_str)
|
||||||
|
|
||||||
|
tool_calls, total = await self.ap.monitoring_service.get_tool_calls(
|
||||||
|
bot_ids=bot_ids if bot_ids else None,
|
||||||
|
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||||
|
session_ids=session_ids if session_ids else None,
|
||||||
|
start_time=start_time,
|
||||||
|
end_time=end_time,
|
||||||
|
limit=limit,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
|
||||||
|
return self.success(
|
||||||
|
data={
|
||||||
|
'tool_calls': tool_calls,
|
||||||
|
'total': total,
|
||||||
|
'limit': limit,
|
||||||
|
'offset': offset,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
@self.route('/embedding-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
@self.route('/embedding-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||||
async def get_embedding_calls() -> str:
|
async def get_embedding_calls() -> str:
|
||||||
"""Get embedding call records"""
|
"""Get embedding call records"""
|
||||||
@@ -284,6 +317,16 @@ class MonitoringRouterGroup(group.RouterGroup):
|
|||||||
offset=0,
|
offset=0,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Get tool calls
|
||||||
|
tool_calls, tool_calls_total = await self.ap.monitoring_service.get_tool_calls(
|
||||||
|
bot_ids=bot_ids if bot_ids else None,
|
||||||
|
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||||
|
start_time=start_time,
|
||||||
|
end_time=end_time,
|
||||||
|
limit=limit,
|
||||||
|
offset=0,
|
||||||
|
)
|
||||||
|
|
||||||
# Get sessions
|
# Get sessions
|
||||||
sessions, sessions_total = await self.ap.monitoring_service.get_sessions(
|
sessions, sessions_total = await self.ap.monitoring_service.get_sessions(
|
||||||
bot_ids=bot_ids if bot_ids else None,
|
bot_ids=bot_ids if bot_ids else None,
|
||||||
@@ -318,12 +361,14 @@ class MonitoringRouterGroup(group.RouterGroup):
|
|||||||
'overview': overview,
|
'overview': overview,
|
||||||
'messages': messages,
|
'messages': messages,
|
||||||
'llmCalls': llm_calls,
|
'llmCalls': llm_calls,
|
||||||
|
'toolCalls': tool_calls,
|
||||||
'embeddingCalls': embedding_calls,
|
'embeddingCalls': embedding_calls,
|
||||||
'sessions': sessions,
|
'sessions': sessions,
|
||||||
'errors': errors,
|
'errors': errors,
|
||||||
'totalCount': {
|
'totalCount': {
|
||||||
'messages': messages_total,
|
'messages': messages_total,
|
||||||
'llmCalls': llm_calls_total,
|
'llmCalls': llm_calls_total,
|
||||||
|
'toolCalls': tool_calls_total,
|
||||||
'embeddingCalls': embedding_calls_total,
|
'embeddingCalls': embedding_calls_total,
|
||||||
'sessions': sessions_total,
|
'sessions': sessions_total,
|
||||||
'errors': errors_total,
|
'errors': errors_total,
|
||||||
|
|||||||
@@ -243,6 +243,7 @@ class MaintenanceService:
|
|||||||
tables = {
|
tables = {
|
||||||
'messages': persistence_monitoring.MonitoringMessage.id,
|
'messages': persistence_monitoring.MonitoringMessage.id,
|
||||||
'llm_calls': persistence_monitoring.MonitoringLLMCall.id,
|
'llm_calls': persistence_monitoring.MonitoringLLMCall.id,
|
||||||
|
'tool_calls': persistence_monitoring.MonitoringToolCall.id,
|
||||||
'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id,
|
'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||||
'errors': persistence_monitoring.MonitoringError.id,
|
'errors': persistence_monitoring.MonitoringError.id,
|
||||||
'sessions': persistence_monitoring.MonitoringSession.session_id,
|
'sessions': persistence_monitoring.MonitoringSession.session_id,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import uuid
|
import uuid
|
||||||
import datetime
|
import datetime
|
||||||
|
import json
|
||||||
import sqlalchemy
|
import sqlalchemy
|
||||||
|
|
||||||
from ....core import app
|
from ....core import app
|
||||||
@@ -50,6 +51,12 @@ class MonitoringService:
|
|||||||
persistence_monitoring.MonitoringLLMCall.timestamp,
|
persistence_monitoring.MonitoringLLMCall.timestamp,
|
||||||
persistence_monitoring.MonitoringLLMCall.id,
|
persistence_monitoring.MonitoringLLMCall.id,
|
||||||
),
|
),
|
||||||
|
(
|
||||||
|
'monitoring_tool_calls',
|
||||||
|
persistence_monitoring.MonitoringToolCall,
|
||||||
|
persistence_monitoring.MonitoringToolCall.timestamp,
|
||||||
|
persistence_monitoring.MonitoringToolCall.id,
|
||||||
|
),
|
||||||
(
|
(
|
||||||
'monitoring_embedding_calls',
|
'monitoring_embedding_calls',
|
||||||
persistence_monitoring.MonitoringEmbeddingCall,
|
persistence_monitoring.MonitoringEmbeddingCall,
|
||||||
@@ -131,6 +138,68 @@ class MonitoringService:
|
|||||||
await autocommit_conn.execute(sqlalchemy.text('PRAGMA wal_checkpoint(TRUNCATE)'))
|
await autocommit_conn.execute(sqlalchemy.text('PRAGMA wal_checkpoint(TRUNCATE)'))
|
||||||
await autocommit_conn.execute(sqlalchemy.text('VACUUM'))
|
await autocommit_conn.execute(sqlalchemy.text('VACUUM'))
|
||||||
|
|
||||||
|
def _serialize_tool_payload(self, payload: object, max_length: int = 20000) -> str | None:
|
||||||
|
"""Serialize tool arguments/results for monitoring storage."""
|
||||||
|
if payload is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if isinstance(payload, str):
|
||||||
|
text = payload
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
text = json.dumps(payload, ensure_ascii=False, default=str)
|
||||||
|
except Exception:
|
||||||
|
text = str(payload)
|
||||||
|
|
||||||
|
if len(text) <= max_length:
|
||||||
|
return text
|
||||||
|
|
||||||
|
return f'{text[:max_length]}... [truncated {len(text) - max_length} chars]'
|
||||||
|
|
||||||
|
async def _get_message_for_tool_context(
|
||||||
|
self,
|
||||||
|
message_id: str | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
):
|
||||||
|
if message_id:
|
||||||
|
result = await self.ap.persistence_mgr.execute_async(
|
||||||
|
sqlalchemy.select(persistence_monitoring.MonitoringMessage).where(
|
||||||
|
persistence_monitoring.MonitoringMessage.id == message_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
row = result.first()
|
||||||
|
if row:
|
||||||
|
return row[0]
|
||||||
|
|
||||||
|
if not session_id:
|
||||||
|
return None
|
||||||
|
|
||||||
|
user_query = (
|
||||||
|
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
|
||||||
|
.where(
|
||||||
|
sqlalchemy.and_(
|
||||||
|
persistence_monitoring.MonitoringMessage.session_id == session_id,
|
||||||
|
persistence_monitoring.MonitoringMessage.role == 'user',
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by(persistence_monitoring.MonitoringMessage.timestamp.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
result = await self.ap.persistence_mgr.execute_async(user_query)
|
||||||
|
row = result.first()
|
||||||
|
if row:
|
||||||
|
return row[0]
|
||||||
|
|
||||||
|
any_query = (
|
||||||
|
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
|
||||||
|
.where(persistence_monitoring.MonitoringMessage.session_id == session_id)
|
||||||
|
.order_by(persistence_monitoring.MonitoringMessage.timestamp.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
result = await self.ap.persistence_mgr.execute_async(any_query)
|
||||||
|
row = result.first()
|
||||||
|
return row[0] if row else None
|
||||||
|
|
||||||
# ========== Recording Methods ==========
|
# ========== Recording Methods ==========
|
||||||
|
|
||||||
async def record_message(
|
async def record_message(
|
||||||
@@ -220,6 +289,57 @@ class MonitoringService:
|
|||||||
|
|
||||||
return call_id
|
return call_id
|
||||||
|
|
||||||
|
async def record_tool_call(
|
||||||
|
self,
|
||||||
|
tool_name: str,
|
||||||
|
tool_source: str,
|
||||||
|
duration: int,
|
||||||
|
status: str = 'success',
|
||||||
|
bot_id: str | None = None,
|
||||||
|
bot_name: str | None = None,
|
||||||
|
pipeline_id: str | None = None,
|
||||||
|
pipeline_name: str | None = None,
|
||||||
|
session_id: str | None = None,
|
||||||
|
message_id: str | None = None,
|
||||||
|
arguments: object | None = None,
|
||||||
|
result: object | None = None,
|
||||||
|
error_message: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
"""Record a tool call."""
|
||||||
|
context_message = await self._get_message_for_tool_context(message_id=message_id, session_id=session_id)
|
||||||
|
if context_message:
|
||||||
|
bot_id = bot_id or context_message.bot_id
|
||||||
|
bot_name = bot_name or context_message.bot_name
|
||||||
|
pipeline_id = pipeline_id or context_message.pipeline_id
|
||||||
|
pipeline_name = pipeline_name or context_message.pipeline_name
|
||||||
|
session_id = session_id or context_message.session_id
|
||||||
|
message_id = message_id or context_message.id
|
||||||
|
|
||||||
|
call_id = str(uuid.uuid4())
|
||||||
|
call_data = {
|
||||||
|
'id': call_id,
|
||||||
|
'timestamp': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
|
||||||
|
'tool_name': tool_name,
|
||||||
|
'tool_source': tool_source,
|
||||||
|
'duration': max(0, duration),
|
||||||
|
'status': status,
|
||||||
|
'bot_id': bot_id or 'unknown',
|
||||||
|
'bot_name': bot_name or 'Unknown',
|
||||||
|
'pipeline_id': pipeline_id or 'unknown',
|
||||||
|
'pipeline_name': pipeline_name or 'Unknown',
|
||||||
|
'session_id': session_id,
|
||||||
|
'message_id': message_id,
|
||||||
|
'arguments': self._serialize_tool_payload(arguments),
|
||||||
|
'result': self._serialize_tool_payload(result),
|
||||||
|
'error_message': self._serialize_tool_payload(error_message),
|
||||||
|
}
|
||||||
|
|
||||||
|
await self.ap.persistence_mgr.execute_async(
|
||||||
|
sqlalchemy.insert(persistence_monitoring.MonitoringToolCall).values(call_data)
|
||||||
|
)
|
||||||
|
|
||||||
|
return call_id
|
||||||
|
|
||||||
async def record_embedding_call(
|
async def record_embedding_call(
|
||||||
self,
|
self,
|
||||||
model_name: str,
|
model_name: str,
|
||||||
@@ -749,6 +869,58 @@ class MonitoringService:
|
|||||||
total,
|
total,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def get_tool_calls(
|
||||||
|
self,
|
||||||
|
bot_ids: list[str] | None = None,
|
||||||
|
pipeline_ids: list[str] | None = None,
|
||||||
|
session_ids: list[str] | None = None,
|
||||||
|
start_time: datetime.datetime | None = None,
|
||||||
|
end_time: datetime.datetime | None = None,
|
||||||
|
limit: int = 100,
|
||||||
|
offset: int = 0,
|
||||||
|
) -> tuple[list[dict], int]:
|
||||||
|
"""Get tool calls with filters"""
|
||||||
|
conditions = []
|
||||||
|
|
||||||
|
if bot_ids:
|
||||||
|
conditions.append(persistence_monitoring.MonitoringToolCall.bot_id.in_(bot_ids))
|
||||||
|
if pipeline_ids:
|
||||||
|
conditions.append(persistence_monitoring.MonitoringToolCall.pipeline_id.in_(pipeline_ids))
|
||||||
|
if session_ids:
|
||||||
|
conditions.append(persistence_monitoring.MonitoringToolCall.session_id.in_(session_ids))
|
||||||
|
if start_time:
|
||||||
|
conditions.append(persistence_monitoring.MonitoringToolCall.timestamp >= start_time)
|
||||||
|
if end_time:
|
||||||
|
conditions.append(persistence_monitoring.MonitoringToolCall.timestamp <= end_time)
|
||||||
|
|
||||||
|
count_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringToolCall.id))
|
||||||
|
if conditions:
|
||||||
|
count_query = count_query.where(sqlalchemy.and_(*conditions))
|
||||||
|
|
||||||
|
count_result = await self.ap.persistence_mgr.execute_async(count_query)
|
||||||
|
total = count_result.scalar() or 0
|
||||||
|
|
||||||
|
query = sqlalchemy.select(persistence_monitoring.MonitoringToolCall).order_by(
|
||||||
|
persistence_monitoring.MonitoringToolCall.timestamp.desc()
|
||||||
|
)
|
||||||
|
if conditions:
|
||||||
|
query = query.where(sqlalchemy.and_(*conditions))
|
||||||
|
|
||||||
|
query = query.limit(limit).offset(offset)
|
||||||
|
|
||||||
|
result = await self.ap.persistence_mgr.execute_async(query)
|
||||||
|
tool_calls_rows = result.all()
|
||||||
|
|
||||||
|
return (
|
||||||
|
[
|
||||||
|
self.ap.persistence_mgr.serialize_model(
|
||||||
|
persistence_monitoring.MonitoringToolCall, row[0] if isinstance(row, tuple) else row
|
||||||
|
)
|
||||||
|
for row in tool_calls_rows
|
||||||
|
],
|
||||||
|
total,
|
||||||
|
)
|
||||||
|
|
||||||
async def get_embedding_calls(
|
async def get_embedding_calls(
|
||||||
self,
|
self,
|
||||||
start_time: datetime.datetime | None = None,
|
start_time: datetime.datetime | None = None,
|
||||||
@@ -971,6 +1143,34 @@ class MonitoringService:
|
|||||||
else:
|
else:
|
||||||
error_llm_calls += 1
|
error_llm_calls += 1
|
||||||
|
|
||||||
|
# Get tool calls for this session
|
||||||
|
tool_query = (
|
||||||
|
sqlalchemy.select(persistence_monitoring.MonitoringToolCall)
|
||||||
|
.where(persistence_monitoring.MonitoringToolCall.session_id == session_id)
|
||||||
|
.order_by(persistence_monitoring.MonitoringToolCall.timestamp.asc())
|
||||||
|
)
|
||||||
|
tool_result = await self.ap.persistence_mgr.execute_async(tool_query)
|
||||||
|
tool_rows = tool_result.all()
|
||||||
|
|
||||||
|
tool_calls = [
|
||||||
|
self.ap.persistence_mgr.serialize_model(
|
||||||
|
persistence_monitoring.MonitoringToolCall, row[0] if isinstance(row, tuple) else row
|
||||||
|
)
|
||||||
|
for row in tool_rows
|
||||||
|
]
|
||||||
|
|
||||||
|
total_tool_calls = len(tool_rows)
|
||||||
|
success_tool_calls = 0
|
||||||
|
error_tool_calls = 0
|
||||||
|
total_tool_duration = 0
|
||||||
|
for row in tool_rows:
|
||||||
|
tool_call = row[0] if isinstance(row, tuple) else row
|
||||||
|
total_tool_duration += tool_call.duration
|
||||||
|
if tool_call.status == 'success':
|
||||||
|
success_tool_calls += 1
|
||||||
|
else:
|
||||||
|
error_tool_calls += 1
|
||||||
|
|
||||||
# Get errors for this session
|
# Get errors for this session
|
||||||
error_query = (
|
error_query = (
|
||||||
sqlalchemy.select(persistence_monitoring.MonitoringError)
|
sqlalchemy.select(persistence_monitoring.MonitoringError)
|
||||||
@@ -1014,6 +1214,14 @@ class MonitoringService:
|
|||||||
'total_tokens': total_tokens,
|
'total_tokens': total_tokens,
|
||||||
'average_duration_ms': int(total_duration / total_llm_calls) if total_llm_calls > 0 else 0,
|
'average_duration_ms': int(total_duration / total_llm_calls) if total_llm_calls > 0 else 0,
|
||||||
},
|
},
|
||||||
|
'tool_calls': tool_calls,
|
||||||
|
'tool_stats': {
|
||||||
|
'total_calls': total_tool_calls,
|
||||||
|
'success_calls': success_tool_calls,
|
||||||
|
'error_calls': error_tool_calls,
|
||||||
|
'total_duration_ms': total_tool_duration,
|
||||||
|
'average_duration_ms': int(total_tool_duration / total_tool_calls) if total_tool_calls > 0 else 0,
|
||||||
|
},
|
||||||
'errors': errors,
|
'errors': errors,
|
||||||
'session_duration_seconds': session_duration_seconds,
|
'session_duration_seconds': session_duration_seconds,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,28 @@ class MonitoringLLMCall(Base):
|
|||||||
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) # Associated message ID
|
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) # Associated message ID
|
||||||
|
|
||||||
|
|
||||||
|
class MonitoringToolCall(Base):
|
||||||
|
"""Tool call records"""
|
||||||
|
|
||||||
|
__tablename__ = 'monitoring_tool_calls'
|
||||||
|
|
||||||
|
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||||
|
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||||
|
tool_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||||
|
tool_source = sqlalchemy.Column(sqlalchemy.String(50), nullable=False) # native, plugin, mcp, skill
|
||||||
|
duration = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) # milliseconds
|
||||||
|
status = sqlalchemy.Column(sqlalchemy.String(50), nullable=False) # success, error
|
||||||
|
bot_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
|
||||||
|
bot_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||||
|
pipeline_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
|
||||||
|
pipeline_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||||
|
session_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||||
|
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||||
|
arguments = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||||
|
result = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||||
|
error_message = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||||
|
|
||||||
|
|
||||||
class MonitoringSession(Base):
|
class MonitoringSession(Base):
|
||||||
"""Session tracking records"""
|
"""Session tracking records"""
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from langbot.pkg.entity.persistence import monitoring as persistence_monitoring
|
||||||
|
from .. import migration
|
||||||
|
|
||||||
|
|
||||||
|
@migration.migration_class(26)
|
||||||
|
class DBMigrateMonitoringToolCalls(migration.DBMigration):
|
||||||
|
"""Add monitoring_tool_calls table"""
|
||||||
|
|
||||||
|
async def upgrade(self):
|
||||||
|
"""Upgrade"""
|
||||||
|
async with self.ap.persistence_mgr.get_db_engine().begin() as conn:
|
||||||
|
await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.create, checkfirst=True)
|
||||||
|
|
||||||
|
async def downgrade(self):
|
||||||
|
"""Downgrade"""
|
||||||
|
async with self.ap.persistence_mgr.get_db_engine().begin() as conn:
|
||||||
|
await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.drop, checkfirst=True)
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import typing
|
import typing
|
||||||
|
import time
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
|
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
|
||||||
@@ -142,21 +143,130 @@ class ToolManager:
|
|||||||
|
|
||||||
return tools
|
return tools
|
||||||
|
|
||||||
|
def _get_query_session_id(self, query: pipeline_query.Query) -> str | None:
|
||||||
|
launcher_type = getattr(query, 'launcher_type', None)
|
||||||
|
launcher_id = getattr(query, 'launcher_id', None)
|
||||||
|
if launcher_type is None or launcher_id is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
launcher_type_value = launcher_type.value if hasattr(launcher_type, 'value') else launcher_type
|
||||||
|
return f'{launcher_type_value}_{launcher_id}'
|
||||||
|
|
||||||
|
async def _record_tool_call(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
source: str,
|
||||||
|
parameters: dict,
|
||||||
|
query: pipeline_query.Query,
|
||||||
|
duration_ms: int,
|
||||||
|
status: str,
|
||||||
|
result: typing.Any = None,
|
||||||
|
error_message: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
monitoring_service = getattr(self.ap, 'monitoring_service', None)
|
||||||
|
if not monitoring_service:
|
||||||
|
return
|
||||||
|
|
||||||
|
variables = getattr(query, 'variables', {}) or {}
|
||||||
|
message_id = variables.get('_monitoring_message_id') if isinstance(variables, dict) else None
|
||||||
|
bot_name = variables.get('_monitoring_bot_name') if isinstance(variables, dict) else None
|
||||||
|
pipeline_name = variables.get('_monitoring_pipeline_name') if isinstance(variables, dict) else None
|
||||||
|
|
||||||
|
try:
|
||||||
|
await monitoring_service.record_tool_call(
|
||||||
|
tool_name=name,
|
||||||
|
tool_source=source,
|
||||||
|
duration=duration_ms,
|
||||||
|
status=status,
|
||||||
|
bot_id=getattr(query, 'bot_uuid', None),
|
||||||
|
bot_name=bot_name,
|
||||||
|
pipeline_name=pipeline_name,
|
||||||
|
session_id=self._get_query_session_id(query),
|
||||||
|
message_id=message_id,
|
||||||
|
arguments=parameters,
|
||||||
|
result=result,
|
||||||
|
error_message=error_message,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
self.ap.logger.warning(f'Failed to record tool call: {e}')
|
||||||
|
|
||||||
|
async def _invoke_tool_with_monitoring(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
source: str,
|
||||||
|
name: str,
|
||||||
|
parameters: dict,
|
||||||
|
query: pipeline_query.Query,
|
||||||
|
invoke: typing.Callable[[], typing.Awaitable[typing.Any]],
|
||||||
|
) -> typing.Any:
|
||||||
|
start_time = time.perf_counter()
|
||||||
|
try:
|
||||||
|
result = await invoke()
|
||||||
|
except Exception as e:
|
||||||
|
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||||
|
await self._record_tool_call(
|
||||||
|
name=name,
|
||||||
|
source=source,
|
||||||
|
parameters=parameters,
|
||||||
|
query=query,
|
||||||
|
duration_ms=duration_ms,
|
||||||
|
status='error',
|
||||||
|
error_message=str(e),
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||||
|
await self._record_tool_call(
|
||||||
|
name=name,
|
||||||
|
source=source,
|
||||||
|
parameters=parameters,
|
||||||
|
query=query,
|
||||||
|
duration_ms=duration_ms,
|
||||||
|
status='success',
|
||||||
|
result=result,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
|
async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
|
||||||
from langbot.pkg.telemetry import features as telemetry_features
|
from langbot.pkg.telemetry import features as telemetry_features
|
||||||
|
|
||||||
if await self.native_tool_loader.has_tool(name):
|
if await self.native_tool_loader.has_tool(name):
|
||||||
telemetry_features.increment(query, 'tool_calls', 'native')
|
telemetry_features.increment(query, 'tool_calls', 'native')
|
||||||
return await self.native_tool_loader.invoke_tool(name, parameters, query)
|
return await self._invoke_tool_with_monitoring(
|
||||||
|
source='native',
|
||||||
|
name=name,
|
||||||
|
parameters=parameters,
|
||||||
|
query=query,
|
||||||
|
invoke=lambda: self.native_tool_loader.invoke_tool(name, parameters, query),
|
||||||
|
)
|
||||||
if await self.plugin_tool_loader.has_tool(name):
|
if await self.plugin_tool_loader.has_tool(name):
|
||||||
telemetry_features.increment(query, 'tool_calls', 'plugin')
|
telemetry_features.increment(query, 'tool_calls', 'plugin')
|
||||||
return await self.plugin_tool_loader.invoke_tool(name, parameters, query)
|
return await self._invoke_tool_with_monitoring(
|
||||||
|
source='plugin',
|
||||||
|
name=name,
|
||||||
|
parameters=parameters,
|
||||||
|
query=query,
|
||||||
|
invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query),
|
||||||
|
)
|
||||||
if await self.mcp_tool_loader.has_tool(name):
|
if await self.mcp_tool_loader.has_tool(name):
|
||||||
telemetry_features.increment(query, 'tool_calls', 'mcp')
|
telemetry_features.increment(query, 'tool_calls', 'mcp')
|
||||||
return await self.mcp_tool_loader.invoke_tool(name, parameters, query)
|
return await self._invoke_tool_with_monitoring(
|
||||||
|
source='mcp',
|
||||||
|
name=name,
|
||||||
|
parameters=parameters,
|
||||||
|
query=query,
|
||||||
|
invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query),
|
||||||
|
)
|
||||||
if await self.skill_tool_loader.has_tool(name):
|
if await self.skill_tool_loader.has_tool(name):
|
||||||
telemetry_features.increment(query, 'tool_calls', 'skill')
|
telemetry_features.increment(query, 'tool_calls', 'skill')
|
||||||
return await self.skill_tool_loader.invoke_tool(name, parameters, query)
|
return await self._invoke_tool_with_monitoring(
|
||||||
|
source='skill',
|
||||||
|
name=name,
|
||||||
|
parameters=parameters,
|
||||||
|
query=query,
|
||||||
|
invoke=lambda: self.skill_tool_loader.invoke_tool(name, parameters, query),
|
||||||
|
)
|
||||||
raise ToolNotFoundError(name)
|
raise ToolNotFoundError(name)
|
||||||
|
|
||||||
async def shutdown(self):
|
async def shutdown(self):
|
||||||
|
|||||||
@@ -191,45 +191,47 @@ export default function BotDetailContent({ id }: { id: string }) {
|
|||||||
onValueChange={setActiveTab}
|
onValueChange={setActiveTab}
|
||||||
className="flex flex-1 flex-col min-h-0"
|
className="flex flex-1 flex-col min-h-0"
|
||||||
>
|
>
|
||||||
<TabsList className="shrink-0">
|
<div className="flex shrink-0 items-center gap-1">
|
||||||
<TabsTrigger value="config" className="gap-1.5">
|
<TabsList>
|
||||||
<Settings className="size-3.5" />
|
<TabsTrigger value="config" className="gap-1.5">
|
||||||
{t('bots.configuration')}
|
<Settings className="size-3.5" />
|
||||||
</TabsTrigger>
|
{t('bots.configuration')}
|
||||||
<TabsTrigger value="logs" className="gap-1.5">
|
</TabsTrigger>
|
||||||
<FileText className="size-3.5" />
|
<TabsTrigger value="logs" className="gap-1.5">
|
||||||
{t('bots.logs')}
|
<FileText className="size-3.5" />
|
||||||
</TabsTrigger>
|
{t('bots.logs')}
|
||||||
<TabsTrigger value="sessions" className="gap-1.5">
|
</TabsTrigger>
|
||||||
<Users className="size-3.5" />
|
<TabsTrigger value="sessions" className="gap-1.5">
|
||||||
{t('bots.sessionMonitor.title')}
|
<Users className="size-3.5" />
|
||||||
{activeTab === 'sessions' && (
|
{t('bots.sessionMonitor.title')}
|
||||||
<button
|
</TabsTrigger>
|
||||||
type="button"
|
</TabsList>
|
||||||
className="inline-flex items-center justify-center ml-0.5"
|
{activeTab === 'sessions' && (
|
||||||
onPointerDown={(e) => e.stopPropagation()}
|
<button
|
||||||
onClick={(e) => {
|
type="button"
|
||||||
e.stopPropagation();
|
aria-label={t('bots.sessionMonitor.refresh')}
|
||||||
e.preventDefault();
|
title={t('bots.sessionMonitor.refresh')}
|
||||||
if (isRefreshingSessions) return;
|
className="inline-flex h-8 w-8 items-center justify-center rounded-md text-muted-foreground transition-colors hover:bg-accent hover:text-foreground disabled:pointer-events-none disabled:opacity-50"
|
||||||
setIsRefreshingSessions(true);
|
disabled={isRefreshingSessions}
|
||||||
const minDelay = new Promise((r) => setTimeout(r, 500));
|
onClick={() => {
|
||||||
Promise.all([
|
if (isRefreshingSessions) return;
|
||||||
sessionMonitorRef.current?.refreshSessions(),
|
setIsRefreshingSessions(true);
|
||||||
minDelay,
|
const minDelay = new Promise((r) => setTimeout(r, 500));
|
||||||
]).finally(() => setIsRefreshingSessions(false));
|
Promise.all([
|
||||||
}}
|
sessionMonitorRef.current?.refreshSessions(),
|
||||||
>
|
minDelay,
|
||||||
<RefreshCw
|
]).finally(() => setIsRefreshingSessions(false));
|
||||||
className={cn(
|
}}
|
||||||
'size-3 text-muted-foreground hover:text-foreground transition-colors',
|
>
|
||||||
isRefreshingSessions && 'animate-spin',
|
<RefreshCw
|
||||||
)}
|
className={cn(
|
||||||
/>
|
'size-3.5',
|
||||||
</button>
|
isRefreshingSessions && 'animate-spin',
|
||||||
)}
|
)}
|
||||||
</TabsTrigger>
|
/>
|
||||||
</TabsList>
|
</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
{/* Tab: Configuration */}
|
{/* Tab: Configuration */}
|
||||||
<TabsContent
|
<TabsContent
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import React, {
|
|||||||
useEffect,
|
useEffect,
|
||||||
useRef,
|
useRef,
|
||||||
useCallback,
|
useCallback,
|
||||||
|
useMemo,
|
||||||
forwardRef,
|
forwardRef,
|
||||||
useImperativeHandle,
|
useImperativeHandle,
|
||||||
} from 'react';
|
} from 'react';
|
||||||
@@ -15,11 +16,14 @@ import {
|
|||||||
Bot,
|
Bot,
|
||||||
Copy,
|
Copy,
|
||||||
Check,
|
Check,
|
||||||
|
ChevronDown,
|
||||||
|
ChevronRight,
|
||||||
Workflow,
|
Workflow,
|
||||||
ThumbsUp,
|
ThumbsUp,
|
||||||
ThumbsDown,
|
ThumbsDown,
|
||||||
ShieldCheck,
|
ShieldCheck,
|
||||||
ShieldOff,
|
ShieldOff,
|
||||||
|
Wrench,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { toast } from 'sonner';
|
import { toast } from 'sonner';
|
||||||
import BotAdminsDialog, {
|
import BotAdminsDialog, {
|
||||||
@@ -76,6 +80,35 @@ interface SessionFeedback {
|
|||||||
stream_id?: string | null;
|
stream_id?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface SessionToolCall {
|
||||||
|
id: string;
|
||||||
|
timestamp: string;
|
||||||
|
tool_name: string;
|
||||||
|
tool_source: string;
|
||||||
|
duration: number;
|
||||||
|
status: string;
|
||||||
|
message_id?: string | null;
|
||||||
|
arguments?: string | null;
|
||||||
|
result?: string | null;
|
||||||
|
error_message?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SessionTimelineItem =
|
||||||
|
| {
|
||||||
|
id: string;
|
||||||
|
type: 'message';
|
||||||
|
timestamp: number;
|
||||||
|
order: number;
|
||||||
|
message: SessionMessage;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
id: string;
|
||||||
|
type: 'tool';
|
||||||
|
timestamp: number;
|
||||||
|
order: number;
|
||||||
|
toolCall: SessionToolCall;
|
||||||
|
};
|
||||||
|
|
||||||
export interface BotSessionMonitorHandle {
|
export interface BotSessionMonitorHandle {
|
||||||
refreshSessions: () => Promise<void>;
|
refreshSessions: () => Promise<void>;
|
||||||
}
|
}
|
||||||
@@ -100,6 +133,10 @@ const BotSessionMonitor = forwardRef<
|
|||||||
const [feedbackMap, setFeedbackMap] = useState<
|
const [feedbackMap, setFeedbackMap] = useState<
|
||||||
Record<string, SessionFeedback>
|
Record<string, SessionFeedback>
|
||||||
>({});
|
>({});
|
||||||
|
const [toolCalls, setToolCalls] = useState<SessionToolCall[]>([]);
|
||||||
|
const [expandedToolCallIds, setExpandedToolCallIds] = useState<
|
||||||
|
Record<string, boolean>
|
||||||
|
>({});
|
||||||
const messagesContainerRef = useRef<HTMLDivElement>(null);
|
const messagesContainerRef = useRef<HTMLDivElement>(null);
|
||||||
const { admins, reload: reloadAdmins } = useBotAdmins(botId);
|
const { admins, reload: reloadAdmins } = useBotAdmins(botId);
|
||||||
const [adminsDialogOpen, setAdminsDialogOpen] = useState(false);
|
const [adminsDialogOpen, setAdminsDialogOpen] = useState(false);
|
||||||
@@ -189,6 +226,7 @@ const BotSessionMonitor = forwardRef<
|
|||||||
const loadMessages = useCallback(
|
const loadMessages = useCallback(
|
||||||
async (sessionId: string) => {
|
async (sessionId: string) => {
|
||||||
setLoadingMessages(true);
|
setLoadingMessages(true);
|
||||||
|
setExpandedToolCallIds({});
|
||||||
try {
|
try {
|
||||||
const messagesRes = await httpClient.getSessionMessages(sessionId);
|
const messagesRes = await httpClient.getSessionMessages(sessionId);
|
||||||
const sorted = (messagesRes.messages ?? []).sort(
|
const sorted = (messagesRes.messages ?? []).sort(
|
||||||
@@ -197,6 +235,18 @@ const BotSessionMonitor = forwardRef<
|
|||||||
);
|
);
|
||||||
setMessages(sorted);
|
setMessages(sorted);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const analysisRes = await httpClient.get<{
|
||||||
|
tool_calls?: SessionToolCall[];
|
||||||
|
}>(
|
||||||
|
`/api/v1/monitoring/sessions/${encodeURIComponent(sessionId)}/analysis`,
|
||||||
|
);
|
||||||
|
setToolCalls(analysisRes?.tool_calls ?? []);
|
||||||
|
} catch (analysisError) {
|
||||||
|
console.error('Failed to load session tool calls:', analysisError);
|
||||||
|
setToolCalls([]);
|
||||||
|
}
|
||||||
|
|
||||||
// Collect user message IDs for feedback matching
|
// Collect user message IDs for feedback matching
|
||||||
const userMsgIds = new Set(
|
const userMsgIds = new Set(
|
||||||
sorted.filter((m) => !m.role || m.role === 'user').map((m) => m.id),
|
sorted.filter((m) => !m.role || m.role === 'user').map((m) => m.id),
|
||||||
@@ -240,11 +290,14 @@ const BotSessionMonitor = forwardRef<
|
|||||||
loadMessages(selectedSessionId);
|
loadMessages(selectedSessionId);
|
||||||
} else {
|
} else {
|
||||||
setMessages([]);
|
setMessages([]);
|
||||||
|
setToolCalls([]);
|
||||||
|
setExpandedToolCallIds({});
|
||||||
|
setFeedbackMap({});
|
||||||
}
|
}
|
||||||
}, [selectedSessionId, loadMessages]);
|
}, [selectedSessionId, loadMessages]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (messages.length === 0) return;
|
if (messages.length === 0 && toolCalls.length === 0) return;
|
||||||
// Wait for DOM to render the new messages before scrolling
|
// Wait for DOM to render the new messages before scrolling
|
||||||
requestAnimationFrame(() => {
|
requestAnimationFrame(() => {
|
||||||
const container = messagesContainerRef.current;
|
const container = messagesContainerRef.current;
|
||||||
@@ -256,7 +309,7 @@ const BotSessionMonitor = forwardRef<
|
|||||||
scrollTarget.scrollTop = scrollTarget.scrollHeight;
|
scrollTarget.scrollTop = scrollTarget.scrollHeight;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}, [messages]);
|
}, [messages, toolCalls]);
|
||||||
|
|
||||||
const parseMessageChain = (content: string): MessageChainComponent[] => {
|
const parseMessageChain = (content: string): MessageChainComponent[] => {
|
||||||
try {
|
try {
|
||||||
@@ -431,6 +484,69 @@ const BotSessionMonitor = forwardRef<
|
|||||||
return `${diffDays}d`;
|
return `${diffDays}d`;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const formatDuration = (durationMs: number): string => {
|
||||||
|
if (!durationMs) return '0ms';
|
||||||
|
if (durationMs < 1000) return `${durationMs}ms`;
|
||||||
|
return `${(durationMs / 1000).toFixed(2)}s`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const truncateToolDetail = (value?: string | null): string => {
|
||||||
|
if (!value) return '';
|
||||||
|
return value.length > 600 ? `${value.slice(0, 600)}...` : value;
|
||||||
|
};
|
||||||
|
|
||||||
|
const toggleToolCallDetails = (toolCallId: string) => {
|
||||||
|
setExpandedToolCallIds((previous) => ({
|
||||||
|
...previous,
|
||||||
|
[toolCallId]: !previous[toolCallId],
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
|
||||||
|
const feedbackByMessageId = useMemo(() => {
|
||||||
|
const map: Record<string, SessionFeedback> = {};
|
||||||
|
|
||||||
|
for (let index = 0; index < messages.length; index++) {
|
||||||
|
const msg = messages[index];
|
||||||
|
if (isUserMessage(msg)) continue;
|
||||||
|
|
||||||
|
for (let previousIndex = index - 1; previousIndex >= 0; previousIndex--) {
|
||||||
|
const previousMessage = messages[previousIndex];
|
||||||
|
if (isUserMessage(previousMessage)) {
|
||||||
|
const feedback = feedbackMap[previousMessage.id];
|
||||||
|
if (feedback) {
|
||||||
|
map[msg.id] = feedback;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return map;
|
||||||
|
}, [feedbackMap, messages]);
|
||||||
|
|
||||||
|
const timelineItems = useMemo<SessionTimelineItem[]>(() => {
|
||||||
|
const messageItems: SessionTimelineItem[] = messages.map(
|
||||||
|
(message, index) => ({
|
||||||
|
id: `message-${message.id}`,
|
||||||
|
type: 'message',
|
||||||
|
timestamp: parseTimestamp(message.timestamp).getTime(),
|
||||||
|
order: index * 2,
|
||||||
|
message,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const toolItems: SessionTimelineItem[] = toolCalls.map((toolCall, index) => ({
|
||||||
|
id: `tool-${toolCall.id}`,
|
||||||
|
type: 'tool',
|
||||||
|
timestamp: parseTimestamp(toolCall.timestamp).getTime(),
|
||||||
|
order: index * 2 + 1,
|
||||||
|
toolCall,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return [...messageItems, ...toolItems].sort(
|
||||||
|
(a, b) => a.timestamp - b.timestamp || a.order - b.order,
|
||||||
|
);
|
||||||
|
}, [messages, toolCalls]);
|
||||||
|
|
||||||
const selectedSession = sessions.find(
|
const selectedSession = sessions.find(
|
||||||
(s) => s.session_id === selectedSessionId,
|
(s) => s.session_id === selectedSessionId,
|
||||||
);
|
);
|
||||||
@@ -612,29 +728,164 @@ const BotSessionMonitor = forwardRef<
|
|||||||
<div className="text-center text-muted-foreground py-12 text-sm">
|
<div className="text-center text-muted-foreground py-12 text-sm">
|
||||||
{t('bots.sessionMonitor.loading')}
|
{t('bots.sessionMonitor.loading')}
|
||||||
</div>
|
</div>
|
||||||
) : messages.length === 0 ? (
|
) : timelineItems.length === 0 ? (
|
||||||
<div className="text-center text-muted-foreground py-12 text-sm">
|
<div className="text-center text-muted-foreground py-12 text-sm">
|
||||||
{t('bots.sessionMonitor.noMessages')}
|
{t('bots.sessionMonitor.noMessages')}
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
messages.map((msg, msgIndex) => {
|
timelineItems.map((item) => {
|
||||||
|
if (item.type === 'tool') {
|
||||||
|
const call = item.toolCall;
|
||||||
|
const hasToolDetails = Boolean(
|
||||||
|
call.arguments || call.result || call.error_message,
|
||||||
|
);
|
||||||
|
const expandedToolCall = Boolean(
|
||||||
|
expandedToolCallIds[call.id],
|
||||||
|
);
|
||||||
|
const detailsId = `tool-call-details-${call.id}`;
|
||||||
|
return (
|
||||||
|
<div key={item.id} className="flex justify-start">
|
||||||
|
<div className="max-w-3xl rounded-2xl rounded-bl-sm border bg-muted px-3 py-2 text-sm">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={cn(
|
||||||
|
'flex w-full items-start justify-between gap-3 rounded-lg text-left outline-none transition-colors',
|
||||||
|
hasToolDetails &&
|
||||||
|
'cursor-pointer hover:bg-background/50 focus-visible:ring-2 focus-visible:ring-ring',
|
||||||
|
)}
|
||||||
|
aria-expanded={
|
||||||
|
hasToolDetails ? expandedToolCall : undefined
|
||||||
|
}
|
||||||
|
aria-controls={
|
||||||
|
hasToolDetails ? detailsId : undefined
|
||||||
|
}
|
||||||
|
aria-disabled={!hasToolDetails}
|
||||||
|
onClick={() =>
|
||||||
|
hasToolDetails &&
|
||||||
|
toggleToolCallDetails(call.id)
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<div className="flex min-w-0 flex-wrap items-center gap-1.5">
|
||||||
|
{hasToolDetails &&
|
||||||
|
(expandedToolCall ? (
|
||||||
|
<ChevronDown className="mt-0.5 h-3.5 w-3.5 text-muted-foreground" />
|
||||||
|
) : (
|
||||||
|
<ChevronRight className="mt-0.5 h-3.5 w-3.5 text-muted-foreground" />
|
||||||
|
))}
|
||||||
|
<Wrench className="mt-0.5 h-3.5 w-3.5 text-muted-foreground" />
|
||||||
|
<span className="min-w-0 max-w-[18rem] truncate font-medium text-foreground">
|
||||||
|
{call.tool_name}
|
||||||
|
</span>
|
||||||
|
<span className="rounded bg-background px-1.5 py-0.5 text-[11px] text-muted-foreground">
|
||||||
|
{call.tool_source}
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
className={cn(
|
||||||
|
'rounded px-1.5 py-0.5 text-[11px] font-medium',
|
||||||
|
call.status === 'success'
|
||||||
|
? 'bg-green-100 text-green-700 dark:bg-green-950 dark:text-green-300'
|
||||||
|
: 'bg-red-100 text-red-700 dark:bg-red-950 dark:text-red-300',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{call.status}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<span className="shrink-0 text-[11px] tabular-nums text-muted-foreground">
|
||||||
|
{formatDuration(call.duration)}
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{hasToolDetails && expandedToolCall && (
|
||||||
|
<div
|
||||||
|
id={detailsId}
|
||||||
|
className="mt-2 space-y-1.5"
|
||||||
|
>
|
||||||
|
{(call.arguments || call.result) && (
|
||||||
|
<div className="space-y-1.5">
|
||||||
|
{call.arguments && (
|
||||||
|
<div>
|
||||||
|
<div className="mb-1 text-[11px] font-medium text-muted-foreground">
|
||||||
|
{t(
|
||||||
|
'monitoring.toolCalls.arguments',
|
||||||
|
{
|
||||||
|
defaultValue: '参数',
|
||||||
|
},
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<pre className="whitespace-pre-wrap break-words rounded bg-background/80 p-2 font-mono text-[11px] leading-4 text-muted-foreground">
|
||||||
|
{truncateToolDetail(
|
||||||
|
call.arguments,
|
||||||
|
)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{call.result && (
|
||||||
|
<div>
|
||||||
|
<div className="mb-1 text-[11px] font-medium text-muted-foreground">
|
||||||
|
{t('monitoring.toolCalls.result', {
|
||||||
|
defaultValue: '结果',
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
<pre className="whitespace-pre-wrap break-words rounded bg-background/80 p-2 font-mono text-[11px] leading-4 text-muted-foreground">
|
||||||
|
{truncateToolDetail(call.result)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{call.error_message && (
|
||||||
|
<div className="whitespace-pre-wrap break-words rounded bg-red-50 p-2 text-[11px] text-red-600 dark:bg-red-950/40 dark:text-red-400">
|
||||||
|
{call.error_message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="mt-1.5 flex items-center gap-1.5 text-[11px] text-muted-foreground">
|
||||||
|
<span>
|
||||||
|
{t('monitoring.toolCalls.title', {
|
||||||
|
defaultValue: '工具调用',
|
||||||
|
})}
|
||||||
|
</span>
|
||||||
|
<span className="tabular-nums">
|
||||||
|
{formatTime(call.timestamp)}
|
||||||
|
</span>
|
||||||
|
{hasToolDetails && (
|
||||||
|
<>
|
||||||
|
<span>·</span>
|
||||||
|
<span>
|
||||||
|
{expandedToolCall
|
||||||
|
? t(
|
||||||
|
'monitoring.toolCalls.hideDetails',
|
||||||
|
{
|
||||||
|
defaultValue: '隐藏详情',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
: t(
|
||||||
|
'monitoring.toolCalls.showDetails',
|
||||||
|
{
|
||||||
|
defaultValue: '查看详情',
|
||||||
|
},
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const msg = item.message;
|
||||||
const isUser = isUserMessage(msg);
|
const isUser = isUserMessage(msg);
|
||||||
const isDiscarded =
|
const isDiscarded =
|
||||||
msg.status === 'discarded' ||
|
msg.status === 'discarded' ||
|
||||||
msg.pipeline_id === PIPELINE_DISCARD;
|
msg.pipeline_id === PIPELINE_DISCARD;
|
||||||
// For bot replies, find feedback linked to the preceding user message
|
const msgFeedback = feedbackByMessageId[msg.id];
|
||||||
let msgFeedback: SessionFeedback | undefined;
|
|
||||||
if (!isUser) {
|
|
||||||
for (let i = msgIndex - 1; i >= 0; i--) {
|
|
||||||
if (isUserMessage(messages[i])) {
|
|
||||||
msgFeedback = feedbackMap[messages[i].id];
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
key={msg.id}
|
key={item.id}
|
||||||
className={cn(
|
className={cn(
|
||||||
'flex',
|
'flex',
|
||||||
isUser ? 'justify-end' : 'justify-start',
|
isUser ? 'justify-end' : 'justify-start',
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
Cpu,
|
Cpu,
|
||||||
Hash,
|
Hash,
|
||||||
User,
|
User,
|
||||||
|
Wrench,
|
||||||
} from 'lucide-react';
|
} from 'lucide-react';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
import { MessageContentRenderer } from './MessageContentRenderer';
|
import { MessageContentRenderer } from './MessageContentRenderer';
|
||||||
@@ -36,6 +37,11 @@ function formatDuration(ms: number) {
|
|||||||
return `${(ms / 1000).toFixed(2)}s`;
|
return `${(ms / 1000).toFixed(2)}s`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function truncateDetail(value?: string) {
|
||||||
|
if (!value) return '';
|
||||||
|
return value.length > 1200 ? `${value.slice(0, 1200)}...` : value;
|
||||||
|
}
|
||||||
|
|
||||||
function roleLabel(message: MonitoringMessage | undefined) {
|
function roleLabel(message: MonitoringMessage | undefined) {
|
||||||
const role = message?.role?.toLowerCase();
|
const role = message?.role?.toLowerCase();
|
||||||
if (role === 'assistant') return 'assistant';
|
if (role === 'assistant') return 'assistant';
|
||||||
@@ -147,6 +153,16 @@ export function ConversationTurnList({
|
|||||||
onToggleTurn,
|
onToggleTurn,
|
||||||
}: ConversationTurnListProps) {
|
}: ConversationTurnListProps) {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const [expandedToolCallIds, setExpandedToolCallIds] = React.useState<
|
||||||
|
Record<string, boolean>
|
||||||
|
>({});
|
||||||
|
|
||||||
|
const toggleToolCallDetails = (toolCallKey: string) => {
|
||||||
|
setExpandedToolCallIds((previous) => ({
|
||||||
|
...previous,
|
||||||
|
[toolCallKey]: !previous[toolCallKey],
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
@@ -268,6 +284,12 @@ export function ConversationTurnList({
|
|||||||
icon={<Cpu className="h-3.5 w-3.5" />}
|
icon={<Cpu className="h-3.5 w-3.5" />}
|
||||||
label={`${turn.llmCalls.length} LLM`}
|
label={`${turn.llmCalls.length} LLM`}
|
||||||
/>
|
/>
|
||||||
|
{turn.toolCalls.length > 0 && (
|
||||||
|
<Metric
|
||||||
|
icon={<Wrench className="h-3.5 w-3.5" />}
|
||||||
|
label={`${turn.toolCalls.length} tools`}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
<Metric
|
<Metric
|
||||||
icon={<Hash className="h-3.5 w-3.5" />}
|
icon={<Hash className="h-3.5 w-3.5" />}
|
||||||
label={`${turn.totalTokens.toLocaleString()} tokens`}
|
label={`${turn.totalTokens.toLocaleString()} tokens`}
|
||||||
@@ -434,6 +456,157 @@ export function ConversationTurnList({
|
|||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section>
|
||||||
|
<h4 className="mb-3 flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||||
|
<Wrench className="h-4 w-4" />
|
||||||
|
{t('monitoring.toolCalls.title', {
|
||||||
|
defaultValue: '工具调用',
|
||||||
|
})}{' '}
|
||||||
|
({turn.toolCalls.length})
|
||||||
|
</h4>
|
||||||
|
|
||||||
|
<div className="grid grid-cols-2 gap-2 lg:grid-cols-3">
|
||||||
|
<MetaItem
|
||||||
|
label={t('monitoring.toolCalls.totalCalls', {
|
||||||
|
defaultValue: '调用次数',
|
||||||
|
})}
|
||||||
|
value={String(turn.toolCalls.length)}
|
||||||
|
/>
|
||||||
|
<MetaItem
|
||||||
|
label={t('monitoring.toolCalls.duration', {
|
||||||
|
defaultValue: '工具耗时',
|
||||||
|
})}
|
||||||
|
value={formatDuration(turn.totalToolDuration)}
|
||||||
|
/>
|
||||||
|
<MetaItem
|
||||||
|
label={t('monitoring.toolCalls.errorCalls', {
|
||||||
|
defaultValue: '失败次数',
|
||||||
|
})}
|
||||||
|
value={String(
|
||||||
|
turn.toolCalls.filter(
|
||||||
|
(call) => call.status === 'error',
|
||||||
|
).length,
|
||||||
|
)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="mt-3 rounded-lg bg-background px-3 py-3">
|
||||||
|
{turn.toolCalls.length > 0 ? (
|
||||||
|
turn.toolCalls.map((call, index) => {
|
||||||
|
const toolCallKey = `${turn.id}:${call.id}`;
|
||||||
|
const hasToolDetails = Boolean(
|
||||||
|
call.arguments || call.result || call.errorMessage,
|
||||||
|
);
|
||||||
|
const expandedToolCall = Boolean(
|
||||||
|
expandedToolCallIds[toolCallKey],
|
||||||
|
);
|
||||||
|
const detailsId = `monitoring-tool-call-details-${call.id}`;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={call.id}
|
||||||
|
className="border-t border-border/70 py-2 first:border-t-0 first:pt-0 last:pb-0"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className={cn(
|
||||||
|
'flex w-full items-start justify-between gap-3 rounded-md px-2 py-2 text-left outline-none transition-colors',
|
||||||
|
hasToolDetails &&
|
||||||
|
'cursor-pointer hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-ring',
|
||||||
|
)}
|
||||||
|
aria-expanded={
|
||||||
|
hasToolDetails ? expandedToolCall : undefined
|
||||||
|
}
|
||||||
|
aria-controls={
|
||||||
|
hasToolDetails ? detailsId : undefined
|
||||||
|
}
|
||||||
|
aria-disabled={!hasToolDetails}
|
||||||
|
onClick={() =>
|
||||||
|
hasToolDetails &&
|
||||||
|
toggleToolCallDetails(toolCallKey)
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<div className="flex min-w-0 flex-wrap items-center gap-2">
|
||||||
|
{hasToolDetails &&
|
||||||
|
(expandedToolCall ? (
|
||||||
|
<ChevronDown className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
|
||||||
|
) : (
|
||||||
|
<ChevronRight className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
|
||||||
|
))}
|
||||||
|
<span className="min-w-0 truncate text-sm font-medium text-foreground">
|
||||||
|
#{index + 1} {call.toolName}
|
||||||
|
</span>
|
||||||
|
<span className="rounded-md bg-muted px-2 py-1 text-xs font-medium text-muted-foreground">
|
||||||
|
{call.toolSource}
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
className={cn(
|
||||||
|
'rounded-md px-2 py-1 text-xs font-medium',
|
||||||
|
call.status === 'success'
|
||||||
|
? 'bg-green-100 text-green-700 dark:bg-green-950 dark:text-green-300'
|
||||||
|
: 'bg-red-100 text-red-700 dark:bg-red-950 dark:text-red-300',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{call.status}
|
||||||
|
</span>
|
||||||
|
<span className="font-mono text-xs text-muted-foreground">
|
||||||
|
ID: {shortId(call.id)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<span className="shrink-0 text-xs text-muted-foreground">
|
||||||
|
{formatDuration(call.duration)}
|
||||||
|
</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{hasToolDetails && expandedToolCall && (
|
||||||
|
<div
|
||||||
|
id={detailsId}
|
||||||
|
className="mt-1 grid gap-2 px-2 pb-2 text-xs lg:grid-cols-2"
|
||||||
|
>
|
||||||
|
{call.arguments && (
|
||||||
|
<div className="min-w-0 rounded-md bg-muted/50 p-2">
|
||||||
|
<div className="mb-1 font-medium text-foreground">
|
||||||
|
{t('monitoring.toolCalls.arguments', {
|
||||||
|
defaultValue: '参数',
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
<pre className="whitespace-pre-wrap break-words font-mono text-muted-foreground">
|
||||||
|
{truncateDetail(call.arguments)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{call.result && (
|
||||||
|
<div className="min-w-0 rounded-md bg-muted/50 p-2">
|
||||||
|
<div className="mb-1 font-medium text-foreground">
|
||||||
|
{t('monitoring.toolCalls.result', {
|
||||||
|
defaultValue: '结果',
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
<pre className="whitespace-pre-wrap break-words font-mono text-muted-foreground">
|
||||||
|
{truncateDetail(call.result)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{call.errorMessage && (
|
||||||
|
<div className="min-w-0 whitespace-pre-wrap break-words rounded-md bg-red-50 p-2 text-red-600 dark:bg-red-950/40 dark:text-red-400 lg:col-span-2">
|
||||||
|
{call.errorMessage}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})
|
||||||
|
) : (
|
||||||
|
<div className="py-4 text-center text-sm text-muted-foreground">
|
||||||
|
{t('monitoring.toolCalls.noToolCalls', {
|
||||||
|
defaultValue: '未记录工具调用',
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
{turn.errors.length > 0 && (
|
{turn.errors.length > 0 && (
|
||||||
<section>
|
<section>
|
||||||
<h4 className="mb-3 flex items-center gap-2 text-sm font-semibold text-red-700 dark:text-red-300">
|
<h4 className="mb-3 flex items-center gap-2 text-sm font-semibold text-red-700 dark:text-red-300">
|
||||||
|
|||||||
@@ -106,6 +106,9 @@ export function useMonitoringData(filterState: FilterState) {
|
|||||||
const llmCalls = Array.isArray(response?.llmCalls)
|
const llmCalls = Array.isArray(response?.llmCalls)
|
||||||
? response.llmCalls
|
? response.llmCalls
|
||||||
: [];
|
: [];
|
||||||
|
const toolCalls = Array.isArray(response?.toolCalls)
|
||||||
|
? response.toolCalls
|
||||||
|
: [];
|
||||||
const embeddingCalls = Array.isArray(response?.embeddingCalls)
|
const embeddingCalls = Array.isArray(response?.embeddingCalls)
|
||||||
? response.embeddingCalls
|
? response.embeddingCalls
|
||||||
: [];
|
: [];
|
||||||
@@ -116,6 +119,7 @@ export function useMonitoringData(filterState: FilterState) {
|
|||||||
const totalCount = response?.totalCount ?? {
|
const totalCount = response?.totalCount ?? {
|
||||||
messages: messages.length,
|
messages: messages.length,
|
||||||
llmCalls: llmCalls.length,
|
llmCalls: llmCalls.length,
|
||||||
|
toolCalls: toolCalls.length,
|
||||||
embeddingCalls: embeddingCalls.length,
|
embeddingCalls: embeddingCalls.length,
|
||||||
sessions: sessions.length,
|
sessions: sessions.length,
|
||||||
errors: errors.length,
|
errors: errors.length,
|
||||||
@@ -207,6 +211,41 @@ export function useMonitoringData(filterState: FilterState) {
|
|||||||
messageId: call.message_id,
|
messageId: call.message_id,
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
|
toolCalls: toolCalls.map(
|
||||||
|
(call: {
|
||||||
|
id: string;
|
||||||
|
timestamp: string;
|
||||||
|
tool_name: string;
|
||||||
|
tool_source: string;
|
||||||
|
duration: number;
|
||||||
|
status: string;
|
||||||
|
bot_id: string;
|
||||||
|
bot_name: string;
|
||||||
|
pipeline_id: string;
|
||||||
|
pipeline_name: string;
|
||||||
|
session_id?: string;
|
||||||
|
message_id?: string;
|
||||||
|
arguments?: string;
|
||||||
|
result?: string;
|
||||||
|
error_message?: string;
|
||||||
|
}) => ({
|
||||||
|
id: call.id,
|
||||||
|
timestamp: parseUTCTimestamp(call.timestamp),
|
||||||
|
toolName: call.tool_name,
|
||||||
|
toolSource: call.tool_source,
|
||||||
|
duration: call.duration,
|
||||||
|
status: call.status as 'success' | 'error',
|
||||||
|
botId: call.bot_id,
|
||||||
|
botName: call.bot_name,
|
||||||
|
pipelineId: call.pipeline_id,
|
||||||
|
pipelineName: call.pipeline_name,
|
||||||
|
sessionId: call.session_id,
|
||||||
|
messageId: call.message_id,
|
||||||
|
arguments: call.arguments,
|
||||||
|
result: call.result,
|
||||||
|
errorMessage: call.error_message,
|
||||||
|
}),
|
||||||
|
),
|
||||||
embeddingCalls: embeddingCalls.map(
|
embeddingCalls: embeddingCalls.map(
|
||||||
(call: {
|
(call: {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -300,6 +339,7 @@ export function useMonitoringData(filterState: FilterState) {
|
|||||||
totalCount: {
|
totalCount: {
|
||||||
messages: totalCount.messages,
|
messages: totalCount.messages,
|
||||||
llmCalls: totalCount.llmCalls,
|
llmCalls: totalCount.llmCalls,
|
||||||
|
toolCalls: totalCount.toolCalls ?? toolCalls.length,
|
||||||
embeddingCalls: totalCount.embeddingCalls || 0,
|
embeddingCalls: totalCount.embeddingCalls || 0,
|
||||||
sessions: totalCount.sessions,
|
sessions: totalCount.sessions,
|
||||||
errors: totalCount.errors,
|
errors: totalCount.errors,
|
||||||
|
|||||||
@@ -104,8 +104,9 @@ function MonitoringPageContent() {
|
|||||||
data?.messages || [],
|
data?.messages || [],
|
||||||
data?.llmCalls || [],
|
data?.llmCalls || [],
|
||||||
data?.errors || [],
|
data?.errors || [],
|
||||||
|
data?.toolCalls || [],
|
||||||
),
|
),
|
||||||
[data?.messages, data?.llmCalls, data?.errors],
|
[data?.messages, data?.llmCalls, data?.errors, data?.toolCalls],
|
||||||
);
|
);
|
||||||
|
|
||||||
// State for expanded errors
|
// State for expanded errors
|
||||||
|
|||||||
@@ -38,6 +38,24 @@ export interface LLMCall {
|
|||||||
messageId?: string;
|
messageId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ToolCall {
|
||||||
|
id: string;
|
||||||
|
timestamp: Date;
|
||||||
|
toolName: string;
|
||||||
|
toolSource: 'native' | 'plugin' | 'mcp' | 'skill' | string;
|
||||||
|
duration: number;
|
||||||
|
status: 'success' | 'error';
|
||||||
|
botId: string;
|
||||||
|
botName: string;
|
||||||
|
pipelineId: string;
|
||||||
|
pipelineName: string;
|
||||||
|
sessionId?: string;
|
||||||
|
messageId?: string;
|
||||||
|
arguments?: string;
|
||||||
|
result?: string;
|
||||||
|
errorMessage?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface EmbeddingCall {
|
export interface EmbeddingCall {
|
||||||
id: string;
|
id: string;
|
||||||
timestamp: Date;
|
timestamp: Date;
|
||||||
@@ -202,6 +220,7 @@ export interface MonitoringData {
|
|||||||
overview: OverviewMetrics;
|
overview: OverviewMetrics;
|
||||||
messages: MonitoringMessage[];
|
messages: MonitoringMessage[];
|
||||||
llmCalls: LLMCall[];
|
llmCalls: LLMCall[];
|
||||||
|
toolCalls: ToolCall[];
|
||||||
embeddingCalls: EmbeddingCall[];
|
embeddingCalls: EmbeddingCall[];
|
||||||
modelCalls: ModelCall[];
|
modelCalls: ModelCall[];
|
||||||
sessions: SessionInfo[];
|
sessions: SessionInfo[];
|
||||||
@@ -211,6 +230,7 @@ export interface MonitoringData {
|
|||||||
totalCount: {
|
totalCount: {
|
||||||
messages: number;
|
messages: number;
|
||||||
llmCalls: number;
|
llmCalls: number;
|
||||||
|
toolCalls?: number;
|
||||||
embeddingCalls: number;
|
embeddingCalls: number;
|
||||||
sessions: number;
|
sessions: number;
|
||||||
errors: number;
|
errors: number;
|
||||||
|
|||||||
@@ -1,4 +1,9 @@
|
|||||||
import { ErrorLog, LLMCall, MonitoringMessage } from '../types/monitoring';
|
import {
|
||||||
|
ErrorLog,
|
||||||
|
LLMCall,
|
||||||
|
MonitoringMessage,
|
||||||
|
ToolCall,
|
||||||
|
} from '../types/monitoring';
|
||||||
|
|
||||||
type MessageRole = 'user' | 'assistant' | 'unknown';
|
type MessageRole = 'user' | 'assistant' | 'unknown';
|
||||||
|
|
||||||
@@ -19,6 +24,7 @@ export interface ConversationTurn {
|
|||||||
assistantMessages: MonitoringMessage[];
|
assistantMessages: MonitoringMessage[];
|
||||||
messages: MonitoringMessage[];
|
messages: MonitoringMessage[];
|
||||||
llmCalls: LLMCall[];
|
llmCalls: LLMCall[];
|
||||||
|
toolCalls: ToolCall[];
|
||||||
errors: ErrorLog[];
|
errors: ErrorLog[];
|
||||||
status: 'success' | 'error' | 'pending';
|
status: 'success' | 'error' | 'pending';
|
||||||
level: 'info' | 'warning' | 'error' | 'debug';
|
level: 'info' | 'warning' | 'error' | 'debug';
|
||||||
@@ -26,6 +32,7 @@ export interface ConversationTurn {
|
|||||||
outputTokens: number;
|
outputTokens: number;
|
||||||
totalTokens: number;
|
totalTokens: number;
|
||||||
totalDuration: number;
|
totalDuration: number;
|
||||||
|
totalToolDuration: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeRole(
|
function normalizeRole(
|
||||||
@@ -91,6 +98,7 @@ function createTurn(message: MonitoringMessage): ConversationTurn {
|
|||||||
assistantMessages: [],
|
assistantMessages: [],
|
||||||
messages: [],
|
messages: [],
|
||||||
llmCalls: [],
|
llmCalls: [],
|
||||||
|
toolCalls: [],
|
||||||
errors: [],
|
errors: [],
|
||||||
status: message.status,
|
status: message.status,
|
||||||
level: message.level,
|
level: message.level,
|
||||||
@@ -98,6 +106,7 @@ function createTurn(message: MonitoringMessage): ConversationTurn {
|
|||||||
outputTokens: 0,
|
outputTokens: 0,
|
||||||
totalTokens: 0,
|
totalTokens: 0,
|
||||||
totalDuration: 0,
|
totalDuration: 0,
|
||||||
|
totalToolDuration: 0,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,12 +183,16 @@ export function buildConversationTurns(
|
|||||||
messages: MonitoringMessage[],
|
messages: MonitoringMessage[],
|
||||||
llmCalls: LLMCall[],
|
llmCalls: LLMCall[],
|
||||||
errors: ErrorLog[],
|
errors: ErrorLog[],
|
||||||
|
toolCalls: ToolCall[] = [],
|
||||||
): ConversationTurn[] {
|
): ConversationTurn[] {
|
||||||
const llmMessageIds = new Set(
|
const activityMessageIds = new Set([
|
||||||
llmCalls
|
...llmCalls
|
||||||
.map((call) => call.messageId)
|
.map((call) => call.messageId)
|
||||||
.filter((messageId): messageId is string => Boolean(messageId)),
|
.filter((messageId): messageId is string => Boolean(messageId)),
|
||||||
);
|
...toolCalls
|
||||||
|
.map((call) => call.messageId)
|
||||||
|
.filter((messageId): messageId is string => Boolean(messageId)),
|
||||||
|
]);
|
||||||
const visibleMessages = messages
|
const visibleMessages = messages
|
||||||
.filter((message) => hasRenderableMessageContent(message.messageContent))
|
.filter((message) => hasRenderableMessageContent(message.messageContent))
|
||||||
.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
||||||
@@ -189,7 +202,7 @@ export function buildConversationTurns(
|
|||||||
const messageIdToTurn = new Map<string, ConversationTurn>();
|
const messageIdToTurn = new Map<string, ConversationTurn>();
|
||||||
|
|
||||||
for (const message of visibleMessages) {
|
for (const message of visibleMessages) {
|
||||||
const role = normalizeRole(message, llmMessageIds);
|
const role = normalizeRole(message, activityMessageIds);
|
||||||
const previousTurn = lastTurnBySession.get(message.sessionId);
|
const previousTurn = lastTurnBySession.get(message.sessionId);
|
||||||
const shouldStartTurn = role === 'user' || !previousTurn;
|
const shouldStartTurn = role === 'user' || !previousTurn;
|
||||||
const turn = shouldStartTurn ? createTurn(message) : previousTurn;
|
const turn = shouldStartTurn ? createTurn(message) : previousTurn;
|
||||||
@@ -229,6 +242,25 @@ export function buildConversationTurns(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const call of toolCalls) {
|
||||||
|
const turn =
|
||||||
|
(call.messageId ? messageIdToTurn.get(call.messageId) : undefined) ??
|
||||||
|
findTurnBySessionTime(sessionTurns, call.sessionId, call.timestamp);
|
||||||
|
|
||||||
|
if (!turn) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
turn.toolCalls.push(call);
|
||||||
|
turn.totalToolDuration += call.duration;
|
||||||
|
updateTurnActivity(turn, call.timestamp);
|
||||||
|
|
||||||
|
if (call.status === 'error') {
|
||||||
|
turn.status = 'error';
|
||||||
|
turn.level = 'error';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (const error of errors) {
|
for (const error of errors) {
|
||||||
const turn =
|
const turn =
|
||||||
(error.messageId ? messageIdToTurn.get(error.messageId) : undefined) ??
|
(error.messageId ? messageIdToTurn.get(error.messageId) : undefined) ??
|
||||||
@@ -250,6 +282,9 @@ export function buildConversationTurns(
|
|||||||
(a, b) => a.timestamp.getTime() - b.timestamp.getTime(),
|
(a, b) => a.timestamp.getTime() - b.timestamp.getTime(),
|
||||||
);
|
);
|
||||||
turn.llmCalls.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
turn.llmCalls.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
||||||
|
turn.toolCalls.sort(
|
||||||
|
(a, b) => a.timestamp.getTime() - b.timestamp.getTime(),
|
||||||
|
);
|
||||||
turn.errors.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
turn.errors.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1199,8 +1199,10 @@ export class BackendClient extends BaseHttpClient {
|
|||||||
level: string;
|
level: string;
|
||||||
platform?: string;
|
platform?: string;
|
||||||
user_id?: string;
|
user_id?: string;
|
||||||
|
user_name?: string;
|
||||||
runner_name?: string;
|
runner_name?: string;
|
||||||
variables?: string;
|
variables?: string;
|
||||||
|
role?: string;
|
||||||
}>;
|
}>;
|
||||||
llmCalls: Array<{
|
llmCalls: Array<{
|
||||||
id: string;
|
id: string;
|
||||||
@@ -1216,9 +1218,27 @@ export class BackendClient extends BaseHttpClient {
|
|||||||
bot_name: string;
|
bot_name: string;
|
||||||
pipeline_id: string;
|
pipeline_id: string;
|
||||||
pipeline_name: string;
|
pipeline_name: string;
|
||||||
|
session_id?: string;
|
||||||
error_message?: string;
|
error_message?: string;
|
||||||
message_id?: string;
|
message_id?: string;
|
||||||
}>;
|
}>;
|
||||||
|
toolCalls: Array<{
|
||||||
|
id: string;
|
||||||
|
timestamp: string;
|
||||||
|
tool_name: string;
|
||||||
|
tool_source: string;
|
||||||
|
duration: number;
|
||||||
|
status: string;
|
||||||
|
bot_id: string;
|
||||||
|
bot_name: string;
|
||||||
|
pipeline_id: string;
|
||||||
|
pipeline_name: string;
|
||||||
|
session_id?: string;
|
||||||
|
message_id?: string;
|
||||||
|
arguments?: string;
|
||||||
|
result?: string;
|
||||||
|
error_message?: string;
|
||||||
|
}>;
|
||||||
embeddingCalls: Array<{
|
embeddingCalls: Array<{
|
||||||
id: string;
|
id: string;
|
||||||
timestamp: string;
|
timestamp: string;
|
||||||
@@ -1263,6 +1283,7 @@ export class BackendClient extends BaseHttpClient {
|
|||||||
totalCount: {
|
totalCount: {
|
||||||
messages: number;
|
messages: number;
|
||||||
llmCalls: number;
|
llmCalls: number;
|
||||||
|
toolCalls?: number;
|
||||||
embeddingCalls: number;
|
embeddingCalls: number;
|
||||||
sessions: number;
|
sessions: number;
|
||||||
errors: number;
|
errors: number;
|
||||||
|
|||||||
@@ -1357,6 +1357,15 @@ const enUS = {
|
|||||||
avgDuration: 'Avg Duration',
|
avgDuration: 'Avg Duration',
|
||||||
calls: 'Calls',
|
calls: 'Calls',
|
||||||
},
|
},
|
||||||
|
toolCalls: {
|
||||||
|
title: 'Tool Calls',
|
||||||
|
totalCalls: 'Calls',
|
||||||
|
duration: 'Tool Duration',
|
||||||
|
errorCalls: 'Failed Calls',
|
||||||
|
arguments: 'Arguments',
|
||||||
|
result: 'Result',
|
||||||
|
noToolCalls: 'No tool calls recorded',
|
||||||
|
},
|
||||||
tokens: {
|
tokens: {
|
||||||
totalTokens: 'Total Tokens',
|
totalTokens: 'Total Tokens',
|
||||||
inputTokens: 'Input Tokens',
|
inputTokens: 'Input Tokens',
|
||||||
|
|||||||
@@ -1294,6 +1294,15 @@ const zhHans = {
|
|||||||
avgDuration: '平均耗时',
|
avgDuration: '平均耗时',
|
||||||
calls: '调用次数',
|
calls: '调用次数',
|
||||||
},
|
},
|
||||||
|
toolCalls: {
|
||||||
|
title: '工具调用',
|
||||||
|
totalCalls: '调用次数',
|
||||||
|
duration: '工具耗时',
|
||||||
|
errorCalls: '失败次数',
|
||||||
|
arguments: '参数',
|
||||||
|
result: '结果',
|
||||||
|
noToolCalls: '未记录工具调用',
|
||||||
|
},
|
||||||
tokens: {
|
tokens: {
|
||||||
totalTokens: '总 Token 数',
|
totalTokens: '总 Token 数',
|
||||||
inputTokens: '输入 Token',
|
inputTokens: '输入 Token',
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
import { expect, test } from '@playwright/test';
|
||||||
|
|
||||||
|
import { installLangBotApiMocks } from './fixtures/langbot-api';
|
||||||
|
|
||||||
|
const botId = 'bot-tool-timeline';
|
||||||
|
const sessionId = 'person-tool-timeline-user';
|
||||||
|
const botName = 'Tool Timeline Bot';
|
||||||
|
const pipelineId = 'pipeline-tool-timeline';
|
||||||
|
const pipelineName = 'Tool Timeline Pipeline';
|
||||||
|
|
||||||
|
function at(minute: number, second = 0) {
|
||||||
|
return `2026-07-02T10:${String(minute).padStart(2, '0')}:${String(
|
||||||
|
second,
|
||||||
|
).padStart(2, '0')}Z`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sessionMessage(
|
||||||
|
id: string,
|
||||||
|
role: 'user' | 'assistant',
|
||||||
|
minute: number,
|
||||||
|
content: string,
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
timestamp: at(minute),
|
||||||
|
bot_id: botId,
|
||||||
|
bot_name: botName,
|
||||||
|
pipeline_id: pipelineId,
|
||||||
|
pipeline_name: pipelineName,
|
||||||
|
message_content: content,
|
||||||
|
session_id: sessionId,
|
||||||
|
status: 'success',
|
||||||
|
level: 'info',
|
||||||
|
platform: role === 'user' ? 'person' : 'bot',
|
||||||
|
user_id: 'timeline-user',
|
||||||
|
user_name: 'Timeline User',
|
||||||
|
runner_name: role === 'assistant' ? 'local-agent' : null,
|
||||||
|
variables: '{}',
|
||||||
|
role,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function toolCall(
|
||||||
|
id: string,
|
||||||
|
minute: number,
|
||||||
|
toolName: string,
|
||||||
|
duration: number,
|
||||||
|
status: 'success' | 'error' = 'success',
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
timestamp: at(minute, 30),
|
||||||
|
tool_name: toolName,
|
||||||
|
tool_source: 'native',
|
||||||
|
duration,
|
||||||
|
status,
|
||||||
|
bot_id: botId,
|
||||||
|
bot_name: botName,
|
||||||
|
pipeline_id: pipelineId,
|
||||||
|
pipeline_name: pipelineName,
|
||||||
|
session_id: sessionId,
|
||||||
|
message_id: 'user-message',
|
||||||
|
arguments: JSON.stringify({ target: toolName }),
|
||||||
|
result: status === 'success' ? JSON.stringify({ ok: true }) : null,
|
||||||
|
error_message: status === 'error' ? 'Tool execution failed' : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test.describe('bot session monitor tool timeline', () => {
|
||||||
|
test('renders tool calls as left-side agent events interleaved with messages', async ({
|
||||||
|
page,
|
||||||
|
}) => {
|
||||||
|
await installLangBotApiMocks(page, {
|
||||||
|
authenticated: true,
|
||||||
|
monitoringSessions: [
|
||||||
|
{
|
||||||
|
session_id: sessionId,
|
||||||
|
bot_id: botId,
|
||||||
|
bot_name: botName,
|
||||||
|
pipeline_id: pipelineId,
|
||||||
|
pipeline_name: pipelineName,
|
||||||
|
message_count: 3,
|
||||||
|
start_time: at(0),
|
||||||
|
last_activity: at(4),
|
||||||
|
is_active: true,
|
||||||
|
platform: 'person',
|
||||||
|
user_id: 'timeline-user',
|
||||||
|
user_name: 'Timeline User',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
sessionMessages: {
|
||||||
|
[sessionId]: [
|
||||||
|
sessionMessage('user-message', 'user', 0, 'Need a timeline check'),
|
||||||
|
sessionMessage(
|
||||||
|
'assistant-step-1',
|
||||||
|
'assistant',
|
||||||
|
2,
|
||||||
|
'Agent step 1: inspected repository files',
|
||||||
|
),
|
||||||
|
sessionMessage(
|
||||||
|
'assistant-step-2',
|
||||||
|
'assistant',
|
||||||
|
4,
|
||||||
|
'Agent step 2: test suite finished',
|
||||||
|
),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
sessionAnalyses: {
|
||||||
|
[sessionId]: {
|
||||||
|
session_id: sessionId,
|
||||||
|
found: true,
|
||||||
|
tool_calls: [
|
||||||
|
toolCall('tool-repo-read', 1, 'repo_file_read', 80),
|
||||||
|
toolCall('tool-test-run', 3, 'run_test_suite', 140),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await page.goto(`/home/bots?id=${botId}`);
|
||||||
|
await page.getByRole('tab', { name: /Sessions/ }).click();
|
||||||
|
await page.getByRole('button', { name: /Timeline User/ }).click();
|
||||||
|
|
||||||
|
await expect(page.getByText('Need a timeline check')).toBeVisible();
|
||||||
|
await expect(
|
||||||
|
page.getByText('repo_file_read', { exact: true }),
|
||||||
|
).toBeVisible();
|
||||||
|
await expect(
|
||||||
|
page.getByText('Agent step 1: inspected repository files'),
|
||||||
|
).toBeVisible();
|
||||||
|
await expect(
|
||||||
|
page.getByText('run_test_suite', { exact: true }),
|
||||||
|
).toBeVisible();
|
||||||
|
await expect(
|
||||||
|
page.getByText('Agent step 2: test suite finished'),
|
||||||
|
).toBeVisible();
|
||||||
|
await expect(page.getByText('{"target":"repo_file_read"}')).toHaveCount(0);
|
||||||
|
await expect(page.getByText('{"ok":true}')).toHaveCount(0);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
page.locator('div.flex.justify-start').filter({
|
||||||
|
hasText: 'repo_file_read',
|
||||||
|
}),
|
||||||
|
).toHaveCount(1);
|
||||||
|
await expect(
|
||||||
|
page.locator('div.flex.justify-start').filter({
|
||||||
|
hasText: 'run_test_suite',
|
||||||
|
}),
|
||||||
|
).toHaveCount(1);
|
||||||
|
await expect(
|
||||||
|
page.locator('div.flex.justify-end').filter({
|
||||||
|
hasText: 'repo_file_read',
|
||||||
|
}),
|
||||||
|
).toHaveCount(0);
|
||||||
|
await expect(
|
||||||
|
page.locator('div.flex.justify-end').filter({
|
||||||
|
hasText: 'run_test_suite',
|
||||||
|
}),
|
||||||
|
).toHaveCount(0);
|
||||||
|
|
||||||
|
const text = await page.locator('body').innerText();
|
||||||
|
expect(text.indexOf('Need a timeline check')).toBeLessThan(
|
||||||
|
text.indexOf('repo_file_read'),
|
||||||
|
);
|
||||||
|
expect(text.indexOf('repo_file_read')).toBeLessThan(
|
||||||
|
text.indexOf('Agent step 1: inspected repository files'),
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
text.indexOf('Agent step 1: inspected repository files'),
|
||||||
|
).toBeLessThan(text.indexOf('run_test_suite'));
|
||||||
|
expect(text.indexOf('run_test_suite')).toBeLessThan(
|
||||||
|
text.indexOf('Agent step 2: test suite finished'),
|
||||||
|
);
|
||||||
|
|
||||||
|
await page.getByText('repo_file_read', { exact: true }).click();
|
||||||
|
await expect(page.getByText('{"target":"repo_file_read"}')).toBeVisible();
|
||||||
|
await expect(page.getByText('{"ok":true}').first()).toBeVisible();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -73,7 +73,10 @@ interface LangBotApiMockState {
|
|||||||
knowledgeBases: KnowledgeBaseMock[];
|
knowledgeBases: KnowledgeBaseMock[];
|
||||||
mcpServers: MCPServerMock[];
|
mcpServers: MCPServerMock[];
|
||||||
monitoringData: unknown;
|
monitoringData: unknown;
|
||||||
|
monitoringSessions: unknown[];
|
||||||
pipelines: PipelineMock[];
|
pipelines: PipelineMock[];
|
||||||
|
sessionAnalyses: Record<string, unknown>;
|
||||||
|
sessionMessages: Record<string, unknown[]>;
|
||||||
skills: SkillMock[];
|
skills: SkillMock[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,12 +126,14 @@ function emptyMonitoringData() {
|
|||||||
},
|
},
|
||||||
messages: [],
|
messages: [],
|
||||||
llmCalls: [],
|
llmCalls: [],
|
||||||
|
toolCalls: [],
|
||||||
embeddingCalls: [],
|
embeddingCalls: [],
|
||||||
sessions: [],
|
sessions: [],
|
||||||
errors: [],
|
errors: [],
|
||||||
totalCount: {
|
totalCount: {
|
||||||
messages: 0,
|
messages: 0,
|
||||||
llmCalls: 0,
|
llmCalls: 0,
|
||||||
|
toolCalls: 0,
|
||||||
embeddingCalls: 0,
|
embeddingCalls: 0,
|
||||||
sessions: 0,
|
sessions: 0,
|
||||||
errors: 0,
|
errors: 0,
|
||||||
@@ -693,6 +698,37 @@ async function handleBackendApi(route: Route, state: LangBotApiMockState) {
|
|||||||
return fulfillJson(route, state.monitoringData);
|
return fulfillJson(route, state.monitoringData);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (path === '/api/v1/monitoring/sessions') {
|
||||||
|
return fulfillJson(route, {
|
||||||
|
sessions: state.monitoringSessions,
|
||||||
|
total: state.monitoringSessions.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (path === '/api/v1/monitoring/messages') {
|
||||||
|
const sessionId = url.searchParams.get('sessionId') || '';
|
||||||
|
const messages = state.sessionMessages[sessionId] || [];
|
||||||
|
return fulfillJson(route, {
|
||||||
|
messages,
|
||||||
|
total: messages.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionAnalysisMatch = path.match(
|
||||||
|
/^\/api\/v1\/monitoring\/sessions\/([^/]+)\/analysis$/,
|
||||||
|
);
|
||||||
|
if (sessionAnalysisMatch) {
|
||||||
|
const sessionId = decodeURIComponent(sessionAnalysisMatch[1]);
|
||||||
|
return fulfillJson(
|
||||||
|
route,
|
||||||
|
state.sessionAnalyses[sessionId] || {
|
||||||
|
session_id: sessionId,
|
||||||
|
found: true,
|
||||||
|
tool_calls: [],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if (path === '/api/v1/monitoring/overview') {
|
if (path === '/api/v1/monitoring/overview') {
|
||||||
const data = state.monitoringData as { overview?: unknown };
|
const data = state.monitoringData as { overview?: unknown };
|
||||||
return fulfillJson(route, data.overview || emptyMonitoringData().overview);
|
return fulfillJson(route, data.overview || emptyMonitoringData().overview);
|
||||||
@@ -803,17 +839,30 @@ export async function installLangBotApiMocks(
|
|||||||
options: {
|
options: {
|
||||||
authenticated?: boolean;
|
authenticated?: boolean;
|
||||||
monitoringData?: unknown;
|
monitoringData?: unknown;
|
||||||
|
monitoringSessions?: unknown[];
|
||||||
|
sessionAnalyses?: Record<string, unknown>;
|
||||||
|
sessionMessages?: Record<string, unknown[]>;
|
||||||
storage?: JsonRecord;
|
storage?: JsonRecord;
|
||||||
} = {},
|
} = {},
|
||||||
) {
|
) {
|
||||||
const { authenticated = false, monitoringData, storage = {} } = options;
|
const {
|
||||||
|
authenticated = false,
|
||||||
|
monitoringData,
|
||||||
|
monitoringSessions,
|
||||||
|
sessionAnalyses,
|
||||||
|
sessionMessages,
|
||||||
|
storage = {},
|
||||||
|
} = options;
|
||||||
const state: LangBotApiMockState = {
|
const state: LangBotApiMockState = {
|
||||||
bots: [],
|
bots: [],
|
||||||
counters: {},
|
counters: {},
|
||||||
knowledgeBases: [],
|
knowledgeBases: [],
|
||||||
mcpServers: [],
|
mcpServers: [],
|
||||||
monitoringData: monitoringData || emptyMonitoringData(),
|
monitoringData: monitoringData || emptyMonitoringData(),
|
||||||
|
monitoringSessions: monitoringSessions || [],
|
||||||
pipelines: [],
|
pipelines: [],
|
||||||
|
sessionAnalyses: sessionAnalyses || {},
|
||||||
|
sessionMessages: sessionMessages || {},
|
||||||
skills: [],
|
skills: [],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
ErrorLog,
|
ErrorLog,
|
||||||
LLMCall,
|
LLMCall,
|
||||||
MonitoringMessage,
|
MonitoringMessage,
|
||||||
|
ToolCall,
|
||||||
} from '../../src/app/home/monitoring/types/monitoring';
|
} from '../../src/app/home/monitoring/types/monitoring';
|
||||||
|
|
||||||
const bot = {
|
const bot = {
|
||||||
@@ -93,6 +94,34 @@ function errorLog(id: string, minute: number, messageId: string): ErrorLog {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function toolCall(
|
||||||
|
id: string,
|
||||||
|
minute: number,
|
||||||
|
messageId: string | undefined,
|
||||||
|
toolName: string,
|
||||||
|
duration: number,
|
||||||
|
sessionId = 'session-agent',
|
||||||
|
status: 'success' | 'error' = 'success',
|
||||||
|
): ToolCall {
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
timestamp: time(minute),
|
||||||
|
toolName,
|
||||||
|
toolSource: 'native',
|
||||||
|
duration,
|
||||||
|
status,
|
||||||
|
botId: bot.id,
|
||||||
|
botName: bot.name,
|
||||||
|
pipelineId: pipeline.id,
|
||||||
|
pipelineName: pipeline.name,
|
||||||
|
sessionId,
|
||||||
|
messageId,
|
||||||
|
arguments: JSON.stringify({ query: toolName }),
|
||||||
|
result: status === 'success' ? JSON.stringify({ ok: true }) : undefined,
|
||||||
|
errorMessage: status === 'error' ? 'Tool failed' : undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function rawMessage(message: MonitoringMessage) {
|
function rawMessage(message: MonitoringMessage) {
|
||||||
return {
|
return {
|
||||||
id: message.id,
|
id: message.id,
|
||||||
@@ -151,6 +180,26 @@ function rawError(error: ErrorLog) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function rawToolCall(call: ToolCall) {
|
||||||
|
return {
|
||||||
|
id: call.id,
|
||||||
|
timestamp: call.timestamp.toISOString(),
|
||||||
|
tool_name: call.toolName,
|
||||||
|
tool_source: call.toolSource,
|
||||||
|
duration: call.duration,
|
||||||
|
status: call.status,
|
||||||
|
bot_id: call.botId,
|
||||||
|
bot_name: call.botName,
|
||||||
|
pipeline_id: call.pipelineId,
|
||||||
|
pipeline_name: call.pipelineName,
|
||||||
|
session_id: call.sessionId,
|
||||||
|
message_id: call.messageId,
|
||||||
|
arguments: call.arguments,
|
||||||
|
result: call.result,
|
||||||
|
error_message: call.errorMessage,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function monitoringScenario() {
|
function monitoringScenario() {
|
||||||
const messages = [
|
const messages = [
|
||||||
message(
|
message(
|
||||||
@@ -179,10 +228,16 @@ function monitoringScenario() {
|
|||||||
llmCall('agent-call-4', 20, 'agent-user-2', 50, 25, 80),
|
llmCall('agent-call-4', 20, 'agent-user-2', 50, 25, 80),
|
||||||
];
|
];
|
||||||
const errors = [errorLog('agent-error-1', 12, 'agent-user-1')];
|
const errors = [errorLog('agent-error-1', 12, 'agent-user-1')];
|
||||||
|
const toolCalls = [
|
||||||
|
toolCall('agent-tool-1', 11, 'agent-user-1', 'repo_search', 90),
|
||||||
|
toolCall('agent-tool-2', 12, 'agent-user-1', 'run_tests', 150),
|
||||||
|
toolCall('agent-tool-3', 20, 'agent-user-2', 'rollback_lookup', 70),
|
||||||
|
];
|
||||||
|
|
||||||
return {
|
return {
|
||||||
messages,
|
messages,
|
||||||
llmCalls,
|
llmCalls,
|
||||||
|
toolCalls,
|
||||||
errors,
|
errors,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -201,12 +256,14 @@ function rawMonitoringData() {
|
|||||||
},
|
},
|
||||||
messages: scenario.messages.map(rawMessage),
|
messages: scenario.messages.map(rawMessage),
|
||||||
llmCalls: scenario.llmCalls.map(rawLlmCall),
|
llmCalls: scenario.llmCalls.map(rawLlmCall),
|
||||||
|
toolCalls: scenario.toolCalls.map(rawToolCall),
|
||||||
embeddingCalls: [],
|
embeddingCalls: [],
|
||||||
sessions: [],
|
sessions: [],
|
||||||
errors: scenario.errors.map(rawError),
|
errors: scenario.errors.map(rawError),
|
||||||
totalCount: {
|
totalCount: {
|
||||||
messages: scenario.messages.length,
|
messages: scenario.messages.length,
|
||||||
llmCalls: scenario.llmCalls.length,
|
llmCalls: scenario.llmCalls.length,
|
||||||
|
toolCalls: scenario.toolCalls.length,
|
||||||
embeddingCalls: 0,
|
embeddingCalls: 0,
|
||||||
sessions: 0,
|
sessions: 0,
|
||||||
errors: scenario.errors.length,
|
errors: scenario.errors.length,
|
||||||
@@ -240,6 +297,7 @@ test.describe('monitoring conversation turn grouping', () => {
|
|||||||
scenario.messages,
|
scenario.messages,
|
||||||
scenario.llmCalls,
|
scenario.llmCalls,
|
||||||
scenario.errors,
|
scenario.errors,
|
||||||
|
scenario.toolCalls,
|
||||||
);
|
);
|
||||||
|
|
||||||
const agentTurn = turns.find((turn) => turn.id === 'agent-user-1');
|
const agentTurn = turns.find((turn) => turn.id === 'agent-user-1');
|
||||||
@@ -254,9 +312,11 @@ test.describe('monitoring conversation turn grouping', () => {
|
|||||||
'Final answer: deployment plan ready',
|
'Final answer: deployment plan ready',
|
||||||
]);
|
]);
|
||||||
expect(agentTurn?.llmCalls).toHaveLength(3);
|
expect(agentTurn?.llmCalls).toHaveLength(3);
|
||||||
|
expect(agentTurn?.toolCalls).toHaveLength(2);
|
||||||
expect(agentTurn?.errors).toHaveLength(1);
|
expect(agentTurn?.errors).toHaveLength(1);
|
||||||
expect(agentTurn?.totalTokens).toBe(790);
|
expect(agentTurn?.totalTokens).toBe(790);
|
||||||
expect(agentTurn?.totalDuration).toBe(600);
|
expect(agentTurn?.totalDuration).toBe(600);
|
||||||
|
expect(agentTurn?.totalToolDuration).toBe(240);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('starts a new turn for each later user message in the same session', () => {
|
test('starts a new turn for each later user message in the same session', () => {
|
||||||
@@ -314,6 +374,30 @@ test.describe('monitoring conversation turn grouping', () => {
|
|||||||
expect(turns[0].totalTokens).toBe(30);
|
expect(turns[0].totalTokens).toBe(30);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('attaches tool calls without message ids by session time', () => {
|
||||||
|
const user = message('tool-fallback-user', 'user', 1, 'Use tool fallback');
|
||||||
|
const assistant = message(
|
||||||
|
'tool-fallback-assistant',
|
||||||
|
'assistant',
|
||||||
|
2,
|
||||||
|
'Tool fallback reply',
|
||||||
|
);
|
||||||
|
const call = toolCall(
|
||||||
|
'tool-fallback-call',
|
||||||
|
2,
|
||||||
|
undefined,
|
||||||
|
'memory_lookup',
|
||||||
|
45,
|
||||||
|
);
|
||||||
|
|
||||||
|
const turns = buildConversationTurns([user, assistant], [], [], [call]);
|
||||||
|
|
||||||
|
expect(turns).toHaveLength(1);
|
||||||
|
expect(turns[0].toolCalls).toHaveLength(1);
|
||||||
|
expect(turns[0].toolCalls[0].id).toBe(call.id);
|
||||||
|
expect(turns[0].totalToolDuration).toBe(45);
|
||||||
|
});
|
||||||
|
|
||||||
test('renders user-only, multi-agent, and multi-turn cases in the monitoring page', async ({
|
test('renders user-only, multi-agent, and multi-turn cases in the monitoring page', async ({
|
||||||
page,
|
page,
|
||||||
}) => {
|
}) => {
|
||||||
@@ -333,6 +417,7 @@ test.describe('monitoring conversation turn grouping', () => {
|
|||||||
await expect(page.getByText('Agent step 1: inspect repo')).toBeVisible();
|
await expect(page.getByText('Agent step 1: inspect repo')).toBeVisible();
|
||||||
await expect(page.getByText('Assistant +2')).toBeVisible();
|
await expect(page.getByText('Assistant +2')).toBeVisible();
|
||||||
await expect(page.getByText('3 LLM')).toBeVisible();
|
await expect(page.getByText('3 LLM')).toBeVisible();
|
||||||
|
await expect(page.getByText('2 tools')).toBeVisible();
|
||||||
await expect(page.getByText('790 tokens')).toBeVisible();
|
await expect(page.getByText('790 tokens')).toBeVisible();
|
||||||
await expect(page.getByText('1 errors')).toBeVisible();
|
await expect(page.getByText('1 errors')).toBeVisible();
|
||||||
await expect(page.getByText('Continue with rollback plan')).toBeVisible();
|
await expect(page.getByText('Continue with rollback plan')).toBeVisible();
|
||||||
@@ -354,6 +439,15 @@ test.describe('monitoring conversation turn grouping', () => {
|
|||||||
await expect(page.getByText('In: 300')).toBeVisible();
|
await expect(page.getByText('In: 300')).toBeVisible();
|
||||||
await expect(page.getByText('Out: 90')).toBeVisible();
|
await expect(page.getByText('Out: 90')).toBeVisible();
|
||||||
await expect(page.getByText('Total: 390')).toBeVisible();
|
await expect(page.getByText('Total: 390')).toBeVisible();
|
||||||
|
await expect(page.getByText('Tool Calls (2)')).toBeVisible();
|
||||||
|
await expect(page.getByText('#1 repo_search')).toBeVisible();
|
||||||
|
await expect(page.getByText('#2 run_tests')).toBeVisible();
|
||||||
|
await expect(page.getByText('Arguments')).toHaveCount(0);
|
||||||
|
await expect(page.getByText('Result')).toHaveCount(0);
|
||||||
|
|
||||||
|
await page.getByText('#1 repo_search').click();
|
||||||
|
await expect(page.getByText('Arguments').first()).toBeVisible();
|
||||||
|
await expect(page.getByText('Result').first()).toBeVisible();
|
||||||
await expect(page.getByText('Tool retry failed')).toBeVisible();
|
await expect(page.getByText('Tool retry failed')).toBeVisible();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user