mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-22 00:15:20 +08:00
feat(monitoring): unify execution views and trace tool calls
This commit is contained in:
@@ -194,6 +194,21 @@ class AgentRunOrchestrator:
|
||||
|
||||
state_context = build_state_context(event, binding, descriptor)
|
||||
run_id = context['run_id']
|
||||
|
||||
# Monitoring attributes every call to the processor that ran it and to the
|
||||
# record that owns it; a Runner has to claim both itself, since no
|
||||
# Pipeline did. Runs before the Runner is invoked and before the run
|
||||
# context is handed to it, so its own variables stay private.
|
||||
from ...pipeline import monitoring_helper
|
||||
|
||||
await monitoring_helper.prepare_monitoring_identity(
|
||||
self.ap,
|
||||
execution_query,
|
||||
processor_type=binding.processor_type,
|
||||
processor_id=binding.processor_id,
|
||||
runner_id=binding.runner_id,
|
||||
execution_record_id=run_id,
|
||||
)
|
||||
context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any(
|
||||
tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform'
|
||||
for tool in resources.get('tools', [])
|
||||
|
||||
@@ -50,6 +50,47 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=metrics)
|
||||
|
||||
@self.route('/executions', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_executions(request_context: RequestContext) -> str:
|
||||
"""Unified execution list across agent runs and pipeline queries."""
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
agent_ids = quart.request.args.getlist('agentId')
|
||||
statuses = quart.request.args.getlist('status')
|
||||
source = quart.request.args.get('source', 'all')
|
||||
mode = quart.request.args.get('mode', 'all')
|
||||
start_time = parse_iso_datetime(quart.request.args.get('startTime'))
|
||||
end_time = parse_iso_datetime(quart.request.args.get('endTime'))
|
||||
|
||||
result = await self.ap.monitoring_service.get_executions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
agent_ids=agent_ids if agent_ids else None,
|
||||
statuses=statuses if statuses else None,
|
||||
source=source,
|
||||
mode=mode,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=quart.request.args.get('limit', 50),
|
||||
offset=quart.request.args.get('offset', 0),
|
||||
)
|
||||
|
||||
return self.success(data=result)
|
||||
|
||||
@self.route(
|
||||
'/executions/<source>/<execution_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_execution_detail(source: str, execution_id: str, request_context: RequestContext) -> str:
|
||||
"""Full trace of a single execution (agent run or pipeline query)."""
|
||||
try:
|
||||
detail = await self.ap.monitoring_service.get_execution_detail(request_context, source, execution_id)
|
||||
except ValueError as exc:
|
||||
return self.fail(404, str(exc))
|
||||
return self.success(data=detail)
|
||||
|
||||
@self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_token_statistics(request_context: RequestContext) -> str:
|
||||
"""Get detailed token usage statistics (summary, per-model, timeseries)."""
|
||||
@@ -639,7 +680,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
'platform',
|
||||
]
|
||||
else:
|
||||
return self.error(message=f'Invalid export type: {export_type}', code=400)
|
||||
return self.fail(400, f'Invalid export type: {export_type}')
|
||||
|
||||
# Generate CSV content with UTF-8 BOM for Excel compatibility
|
||||
import io
|
||||
|
||||
@@ -10,7 +10,10 @@ from sqlalchemy.dialects import postgresql as postgresql_dialect
|
||||
from sqlalchemy.dialects import sqlite as sqlite_dialect
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import agent as persistence_agent
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
from ....entity.persistence import agent_run as persistence_agent_run
|
||||
from ....entity.persistence import agent_interaction as persistence_agent_interaction
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
@@ -29,6 +32,76 @@ _HARD_MAX_MONITORING_OFFSET = 10000000
|
||||
_DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4
|
||||
_HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100
|
||||
|
||||
# Canonical execution status groups shared by the agent ledger and pipeline query
|
||||
# records so both object types can be summarised with one vocabulary.
|
||||
_AGENT_STATUS_GROUP = {
|
||||
'completed': 'completed',
|
||||
'failed': 'failed',
|
||||
'timeout': 'failed',
|
||||
'cancelled': 'cancelled',
|
||||
'created': 'queued',
|
||||
'queued': 'queued',
|
||||
'claimed': 'running',
|
||||
'running': 'running',
|
||||
}
|
||||
_PIPELINE_STATUS_GROUP = {
|
||||
'success': 'completed',
|
||||
'error': 'failed',
|
||||
'pending': 'running',
|
||||
'discarded': 'ignored',
|
||||
'not_matched': 'ignored',
|
||||
}
|
||||
|
||||
|
||||
def _status_group(status: str, source: str) -> str:
|
||||
table = _AGENT_STATUS_GROUP if source == 'agent' else _PIPELINE_STATUS_GROUP
|
||||
return table.get(status, status)
|
||||
|
||||
|
||||
def _epoch_ms(value: datetime.datetime | None) -> int | None:
|
||||
"""Epoch milliseconds for a DB timestamp, treating naive values as UTC."""
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=datetime.timezone.utc)
|
||||
return int(round(value.timestamp() * 1000))
|
||||
|
||||
|
||||
def _percentile_ms(values: list[float], fraction: float) -> int | None:
|
||||
"""Nearest-rank percentile (ms) over a non-empty, prefix-sorted list."""
|
||||
if not values:
|
||||
return None
|
||||
rank = max(1, min(len(values), int(round(fraction * len(values)))))
|
||||
return int(round(values[rank - 1]))
|
||||
|
||||
|
||||
def _collect_text(node, out: list[str], depth: int = 0) -> None:
|
||||
"""Best-effort extraction of human text from a serialised message chain."""
|
||||
if depth > 6:
|
||||
return
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
if key == 'text' and isinstance(value, str):
|
||||
out.append(value)
|
||||
elif isinstance(value, (dict, list)):
|
||||
_collect_text(value, out, depth + 1)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
_collect_text(item, out, depth + 1)
|
||||
|
||||
|
||||
def _message_preview(raw: str | None, limit: int = 160) -> str:
|
||||
if not raw:
|
||||
return ''
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
return raw[:limit]
|
||||
parts: list[str] = []
|
||||
_collect_text(parsed, parts)
|
||||
text = ' '.join(part.strip() for part in parts if part.strip())
|
||||
return (text or raw)[:limit]
|
||||
|
||||
|
||||
def _normalize_user_id(value: str | int | None) -> str | None:
|
||||
"""Convert numeric platform IDs before binding a VARCHAR with asyncpg.
|
||||
@@ -885,6 +958,10 @@ class MonitoringService:
|
||||
active_sessions_result = await self.ap.persistence_mgr.execute_async(active_session_query)
|
||||
active_sessions = active_sessions_result.scalar() or 0
|
||||
|
||||
# Execution-first KPI values (runs + pipeline queries, latency, token
|
||||
# coverage) are served by `get_executions`, which applies the execution
|
||||
# view filters. This endpoint stays the message/traffic rollup so the
|
||||
# dashboard does not compute the same rollup twice per page load.
|
||||
return {
|
||||
'total_messages': total_messages,
|
||||
'llm_calls': llm_calls,
|
||||
@@ -894,6 +971,527 @@ class MonitoringService:
|
||||
'active_sessions': active_sessions,
|
||||
}
|
||||
|
||||
# ========== Execution view ==========
|
||||
|
||||
def _agent_run_conditions(
|
||||
self,
|
||||
workspace_uuid: str,
|
||||
*,
|
||||
bot_ids: list[str] | None = None,
|
||||
agent_ids: list[str] | None = None,
|
||||
statuses: list[str] | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
debug: bool | None = None,
|
||||
) -> list:
|
||||
AgentRun = persistence_agent_run.AgentRun
|
||||
conditions = [AgentRun.workspace_id == workspace_uuid]
|
||||
if bot_ids:
|
||||
conditions.append(AgentRun.bot_id.in_(bot_ids))
|
||||
if agent_ids:
|
||||
conditions.append(AgentRun.agent_id.in_(agent_ids))
|
||||
if statuses is not None:
|
||||
conditions.append(AgentRun.status.in_(statuses) if statuses else sqlalchemy.false())
|
||||
if start_time:
|
||||
conditions.append(AgentRun.created_at >= start_time)
|
||||
if end_time:
|
||||
conditions.append(AgentRun.created_at <= end_time)
|
||||
if debug is not None:
|
||||
debug_condition = sqlalchemy.or_(
|
||||
AgentRun.binding_id.like('debug:%'),
|
||||
AgentRun.conversation_id.like('debug:%'),
|
||||
)
|
||||
conditions.append(debug_condition if debug else sqlalchemy.not_(debug_condition))
|
||||
return conditions
|
||||
|
||||
def _pipeline_execution_conditions(
|
||||
self,
|
||||
workspace_uuid: str,
|
||||
*,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
statuses: list[str] | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
) -> list:
|
||||
MonitoringMessage = persistence_monitoring.MonitoringMessage
|
||||
conditions = [
|
||||
MonitoringMessage.workspace_uuid == workspace_uuid,
|
||||
# Assistant replies are outputs of a query, not an execution of their own.
|
||||
sqlalchemy.or_(MonitoringMessage.role.is_(None), MonitoringMessage.role != 'assistant'),
|
||||
]
|
||||
if bot_ids:
|
||||
conditions.append(MonitoringMessage.bot_id.in_(bot_ids))
|
||||
if pipeline_ids:
|
||||
conditions.append(MonitoringMessage.pipeline_id.in_(pipeline_ids))
|
||||
if statuses is not None:
|
||||
conditions.append(MonitoringMessage.status.in_(statuses) if statuses else sqlalchemy.false())
|
||||
if start_time:
|
||||
conditions.append(MonitoringMessage.timestamp >= start_time)
|
||||
if end_time:
|
||||
conditions.append(MonitoringMessage.timestamp <= end_time)
|
||||
return conditions
|
||||
|
||||
@staticmethod
|
||||
def _raw_agent_statuses(groups: list[str] | None) -> list[str] | None:
|
||||
"""Raw agent statuses for a group filter; empty means "no group applies"."""
|
||||
if not groups:
|
||||
return None
|
||||
raw: list[str] = []
|
||||
for group in groups:
|
||||
raw.extend(status for status, mapped in _AGENT_STATUS_GROUP.items() if mapped == group)
|
||||
return raw
|
||||
|
||||
@staticmethod
|
||||
def _raw_pipeline_statuses(groups: list[str] | None) -> list[str] | None:
|
||||
"""Raw pipeline statuses for a group filter; empty means "no group applies"."""
|
||||
if not groups:
|
||||
return None
|
||||
raw: list[str] = []
|
||||
for group in groups:
|
||||
raw.extend(status for status, mapped in _PIPELINE_STATUS_GROUP.items() if mapped == group)
|
||||
return raw
|
||||
|
||||
@staticmethod
|
||||
def _model_rows(result) -> list:
|
||||
"""Rows of one ``select(Model)`` executed through ``execute_async``.
|
||||
|
||||
The Core keeps the connection result contract, where ``select(Model)``
|
||||
yields a *flat* row of the model's columns (``row.uuid``, ``row.status``)
|
||||
rather than a one-item ORM row. ``.scalars()`` would therefore return the
|
||||
first column - the id - instead of the row, so the rows are taken as-is.
|
||||
"""
|
||||
return list(result.all())
|
||||
|
||||
@staticmethod
|
||||
def _deserialize_json(raw: str | None) -> dict:
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
async def _agent_names(self, workspace_uuid: str, rows: list) -> dict[str, str]:
|
||||
"""The processor name of every Agent run in one page, in one query.
|
||||
|
||||
A run stores the Agent it executed as a uuid; the list has to name the
|
||||
processor, so the names are resolved for the whole page at once.
|
||||
"""
|
||||
agent_ids = {row.agent_id for row in rows if row.agent_id}
|
||||
if not agent_ids:
|
||||
return {}
|
||||
Agent = persistence_agent.Agent
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Agent.uuid, Agent.name).where(
|
||||
sqlalchemy.and_(
|
||||
Agent.workspace_uuid == workspace_uuid,
|
||||
Agent.uuid.in_(agent_ids),
|
||||
)
|
||||
)
|
||||
)
|
||||
return {row.uuid: row.name for row in self._model_rows(result)}
|
||||
|
||||
def _serialize_agent_execution(self, row, agent_names: dict[str, str] | None = None) -> dict:
|
||||
metadata = self._deserialize_json(row.metadata_json)
|
||||
duration_ms = None
|
||||
if row.started_at and row.finished_at:
|
||||
duration_ms = int(round((row.finished_at - row.started_at).total_seconds() * 1000))
|
||||
created_at_ms = _epoch_ms(row.created_at)
|
||||
started_at_ms = _epoch_ms(row.started_at)
|
||||
finished_at_ms = _epoch_ms(row.finished_at)
|
||||
is_debug = str(row.binding_id or '').startswith('debug:') or str(row.conversation_id or '').startswith('debug:')
|
||||
return {
|
||||
'source': 'agent',
|
||||
'id': row.run_id,
|
||||
'event_id': row.event_id,
|
||||
'status': row.status,
|
||||
'status_group': _status_group(row.status, 'agent'),
|
||||
'title': metadata.get('event_type') or row.queue_name or row.runner_id,
|
||||
'target_kind': 'agent' if row.agent_id else 'processor',
|
||||
'target_id': row.agent_id or row.binding_id,
|
||||
'target_name': metadata.get('agent_name')
|
||||
or metadata.get('target_name')
|
||||
or (agent_names or {}).get(row.agent_id or ''),
|
||||
'bot_id': row.bot_id,
|
||||
'bot_name': metadata.get('bot_name'),
|
||||
'pipeline_id': None,
|
||||
'pipeline_name': None,
|
||||
'runner_id': row.runner_id,
|
||||
'conversation_id': row.conversation_id,
|
||||
'session_id': row.conversation_id,
|
||||
'created_at_ms': created_at_ms,
|
||||
'started_at_ms': started_at_ms,
|
||||
'finished_at_ms': finished_at_ms,
|
||||
'duration_ms': duration_ms,
|
||||
'usage': self._deserialize_json(row.usage_json) or None,
|
||||
'cost': self._deserialize_json(row.cost_json) or None,
|
||||
'queue_name': row.queue_name,
|
||||
'debug': is_debug,
|
||||
'has_error': row.status in {'failed', 'timeout'},
|
||||
}
|
||||
|
||||
def _serialize_pipeline_execution(self, row) -> dict:
|
||||
created_at_ms = _epoch_ms(row.timestamp)
|
||||
return {
|
||||
'source': 'pipeline',
|
||||
'id': row.id,
|
||||
'event_id': None,
|
||||
'status': row.status,
|
||||
'status_group': _status_group(row.status, 'pipeline'),
|
||||
'title': _message_preview(row.message_content),
|
||||
'target_kind': 'pipeline',
|
||||
'target_id': row.pipeline_id,
|
||||
'target_name': row.pipeline_name,
|
||||
'bot_id': row.bot_id,
|
||||
'bot_name': row.bot_name,
|
||||
'pipeline_id': row.pipeline_id,
|
||||
'pipeline_name': row.pipeline_name,
|
||||
'runner_id': row.runner_name,
|
||||
'conversation_id': None,
|
||||
'session_id': row.session_id,
|
||||
'created_at_ms': created_at_ms,
|
||||
'started_at_ms': None,
|
||||
'finished_at_ms': None,
|
||||
'duration_ms': None,
|
||||
'usage': None,
|
||||
'cost': None,
|
||||
'queue_name': None,
|
||||
'debug': False,
|
||||
'has_error': row.status == 'error',
|
||||
'platform': row.platform,
|
||||
'user_id': row.user_id,
|
||||
'user_name': row.user_name,
|
||||
}
|
||||
|
||||
async def _get_token_coverage(
|
||||
self,
|
||||
workspace_uuid: str,
|
||||
*,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
) -> dict:
|
||||
LLMCall = persistence_monitoring.MonitoringLLMCall
|
||||
conditions = [LLMCall.workspace_uuid == workspace_uuid]
|
||||
if bot_ids:
|
||||
conditions.append(LLMCall.bot_id.in_(bot_ids))
|
||||
if pipeline_ids:
|
||||
conditions.append(LLMCall.pipeline_id.in_(pipeline_ids))
|
||||
if start_time:
|
||||
conditions.append(LLMCall.timestamp >= start_time)
|
||||
if end_time:
|
||||
conditions.append(LLMCall.timestamp <= end_time)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.count(LLMCall.id),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0),
|
||||
sqlalchemy.func.sum(sqlalchemy.case((LLMCall.total_tokens > 0, 1), else_=0)),
|
||||
).where(sqlalchemy.and_(*conditions))
|
||||
)
|
||||
row = result.first()
|
||||
calls, total, calls_with_usage = row if row else (0, 0, 0)
|
||||
return {
|
||||
'total_tokens': int(total or 0),
|
||||
'calls': calls or 0,
|
||||
'calls_with_usage': calls_with_usage or 0,
|
||||
}
|
||||
|
||||
async def _get_execution_summary(
|
||||
self,
|
||||
workspace_uuid: str,
|
||||
*,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
) -> dict:
|
||||
AgentRun = persistence_agent_run.AgentRun
|
||||
run_conditions = self._agent_run_conditions(
|
||||
workspace_uuid, bot_ids=bot_ids, start_time=start_time, end_time=end_time
|
||||
)
|
||||
run_status_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(AgentRun.status, sqlalchemy.func.count(AgentRun.id))
|
||||
.where(sqlalchemy.and_(*run_conditions))
|
||||
.group_by(AgentRun.status)
|
||||
)
|
||||
agent_counts = {row[0]: row[1] for row in run_status_result.all()}
|
||||
agent_total = sum(agent_counts.values())
|
||||
|
||||
debug_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where(
|
||||
sqlalchemy.and_(
|
||||
*run_conditions,
|
||||
sqlalchemy.or_(
|
||||
AgentRun.binding_id.like('debug:%'),
|
||||
AgentRun.conversation_id.like('debug:%'),
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
debug_count = debug_count_result.scalar() or 0
|
||||
|
||||
# Percentiles are computed in Python, so the sample must stay bounded:
|
||||
# take the most recent completed runs of the window and label the count
|
||||
# so the UI can state how many runs the numbers rest on.
|
||||
timing_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(AgentRun.started_at, AgentRun.finished_at)
|
||||
.where(
|
||||
sqlalchemy.and_(
|
||||
*run_conditions,
|
||||
AgentRun.status == 'completed',
|
||||
AgentRun.started_at.is_not(None),
|
||||
AgentRun.finished_at.is_not(None),
|
||||
)
|
||||
)
|
||||
.order_by(AgentRun.created_at.desc())
|
||||
.limit(self._detail_limit())
|
||||
)
|
||||
durations = sorted((row[1] - row[0]).total_seconds() * 1000 for row in timing_result.all() if row[0] and row[1])
|
||||
|
||||
pipeline_conditions = self._pipeline_execution_conditions(
|
||||
workspace_uuid, bot_ids=bot_ids, pipeline_ids=pipeline_ids, start_time=start_time, end_time=end_time
|
||||
)
|
||||
pipeline_status_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_monitoring.MonitoringMessage.status,
|
||||
sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id),
|
||||
)
|
||||
.where(sqlalchemy.and_(*pipeline_conditions))
|
||||
.group_by(persistence_monitoring.MonitoringMessage.status)
|
||||
)
|
||||
pipeline_counts = {row[0]: row[1] for row in pipeline_status_result.all()}
|
||||
pipeline_total = sum(pipeline_counts.values())
|
||||
|
||||
# Pending interactions are gated by the same bot/window scope as the runs
|
||||
# so "waiting input" cannot drift from the rest of the card.
|
||||
waiting_conditions = [
|
||||
persistence_agent_interaction.AgentInteraction.workspace_id == workspace_uuid,
|
||||
persistence_agent_interaction.AgentInteraction.status == 'pending',
|
||||
]
|
||||
if bot_ids:
|
||||
waiting_conditions.append(persistence_agent_interaction.AgentInteraction.bot_id.in_(bot_ids))
|
||||
if start_time:
|
||||
waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at >= start_time)
|
||||
if end_time:
|
||||
waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at <= end_time)
|
||||
waiting_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_agent_interaction.AgentInteraction.id)).where(
|
||||
sqlalchemy.and_(*waiting_conditions)
|
||||
)
|
||||
)
|
||||
waiting = waiting_result.scalar() or 0
|
||||
|
||||
by_status: dict[str, int] = {}
|
||||
for status, count in agent_counts.items():
|
||||
group = _status_group(status, 'agent')
|
||||
by_status[group] = by_status.get(group, 0) + count
|
||||
for status, count in pipeline_counts.items():
|
||||
group = _status_group(status, 'pipeline')
|
||||
by_status[group] = by_status.get(group, 0) + count
|
||||
|
||||
completed = by_status.get('completed', 0)
|
||||
failed = by_status.get('failed', 0)
|
||||
denominator = completed + failed
|
||||
total = agent_total + pipeline_total
|
||||
return {
|
||||
'total': total,
|
||||
'by_source': {'agent': agent_total, 'pipeline': pipeline_total},
|
||||
'by_status': by_status,
|
||||
'completed': completed,
|
||||
'failed': failed,
|
||||
'running': by_status.get('running', 0),
|
||||
'queued': by_status.get('queued', 0),
|
||||
'cancelled': by_status.get('cancelled', 0),
|
||||
'ignored': by_status.get('ignored', 0),
|
||||
'waiting': waiting,
|
||||
'success_rate': round(completed / denominator * 100, 2) if denominator > 0 else None,
|
||||
'denominator': denominator,
|
||||
'p50_duration_ms': _percentile_ms(durations, 0.5),
|
||||
'p95_duration_ms': _percentile_ms(durations, 0.95),
|
||||
'duration_sample': len(durations),
|
||||
'debug': debug_count,
|
||||
'real': total - debug_count,
|
||||
}
|
||||
|
||||
async def get_executions(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
bot_ids: list[str] | None = None,
|
||||
pipeline_ids: list[str] | None = None,
|
||||
agent_ids: list[str] | None = None,
|
||||
statuses: list[str] | None = None,
|
||||
source: str = 'all',
|
||||
mode: str = 'all',
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> dict:
|
||||
"""List executions from both object types (agent runs and pipeline queries)."""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
fetch_limit = min(limit + offset, self._configured_query_limit('page_rows', 1000, 5000))
|
||||
if source not in {'all', 'agent', 'pipeline'}:
|
||||
source = 'all'
|
||||
if mode not in {'all', 'real', 'debug'}:
|
||||
mode = 'all'
|
||||
debug = None if mode == 'all' else (mode == 'debug')
|
||||
|
||||
include_agent = source in {'all', 'agent'} and (not pipeline_ids or bool(agent_ids))
|
||||
include_pipeline = source in {'all', 'pipeline'} and (not agent_ids or bool(pipeline_ids))
|
||||
|
||||
items: list[dict] = []
|
||||
total = 0
|
||||
|
||||
if include_agent:
|
||||
run_conditions = self._agent_run_conditions(
|
||||
workspace_uuid,
|
||||
bot_ids=bot_ids,
|
||||
agent_ids=agent_ids,
|
||||
statuses=self._raw_agent_statuses(statuses),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
debug=debug,
|
||||
)
|
||||
AgentRun = persistence_agent_run.AgentRun
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(AgentRun)
|
||||
.where(sqlalchemy.and_(*run_conditions))
|
||||
.order_by(AgentRun.created_at.desc(), AgentRun.id.desc())
|
||||
.limit(fetch_limit)
|
||||
)
|
||||
rows = self._model_rows(result)
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where(sqlalchemy.and_(*run_conditions))
|
||||
)
|
||||
total += count_result.scalar() or 0
|
||||
agent_names = await self._agent_names(workspace_uuid, rows)
|
||||
items.extend(self._serialize_agent_execution(row, agent_names=agent_names) for row in rows)
|
||||
|
||||
if include_pipeline and (debug is None or debug is False):
|
||||
message_conditions = self._pipeline_execution_conditions(
|
||||
workspace_uuid,
|
||||
bot_ids=bot_ids,
|
||||
pipeline_ids=pipeline_ids,
|
||||
statuses=self._raw_pipeline_statuses(statuses),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
MonitoringMessage = persistence_monitoring.MonitoringMessage
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(MonitoringMessage)
|
||||
.where(sqlalchemy.and_(*message_conditions))
|
||||
.order_by(MonitoringMessage.timestamp.desc())
|
||||
.limit(fetch_limit)
|
||||
)
|
||||
rows = self._model_rows(result)
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(MonitoringMessage.id)).where(
|
||||
sqlalchemy.and_(*message_conditions)
|
||||
)
|
||||
)
|
||||
total += count_result.scalar() or 0
|
||||
items.extend(self._serialize_pipeline_execution(row) for row in rows)
|
||||
|
||||
items.sort(key=lambda item: item.get('created_at_ms') or 0, reverse=True)
|
||||
page = items[offset : offset + limit]
|
||||
return {
|
||||
'items': page,
|
||||
'total': total,
|
||||
'limit': limit,
|
||||
'offset': offset,
|
||||
'has_more': total > offset + len(page),
|
||||
'summary': {
|
||||
'executions': await self._get_execution_summary(
|
||||
workspace_uuid,
|
||||
bot_ids=bot_ids,
|
||||
pipeline_ids=pipeline_ids,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
),
|
||||
'tokens': await self._get_token_coverage(
|
||||
workspace_uuid,
|
||||
bot_ids=bot_ids,
|
||||
pipeline_ids=pipeline_ids,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
async def get_execution_detail(
|
||||
self,
|
||||
context: TenantContext,
|
||||
source: str,
|
||||
execution_id: str,
|
||||
) -> dict:
|
||||
"""Full trace of a single execution, keyed by object type."""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
|
||||
if source == 'agent':
|
||||
from ....agent.runner.run_ledger_store import RunLedgerStore
|
||||
|
||||
store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine())
|
||||
run = await store.get_run(execution_id)
|
||||
if run is None or run.get('workspace_id') != workspace_uuid:
|
||||
raise ValueError('Execution not found')
|
||||
items, next_cursor, _, has_more = await store.page_run_events(run_id=execution_id, limit=200)
|
||||
return {
|
||||
'source': 'agent',
|
||||
'run': run,
|
||||
'events': items,
|
||||
'has_more': has_more,
|
||||
'next_cursor': next_cursor,
|
||||
}
|
||||
|
||||
if source == 'pipeline':
|
||||
MonitoringMessage = persistence_monitoring.MonitoringMessage
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(MonitoringMessage).where(
|
||||
sqlalchemy.and_(
|
||||
MonitoringMessage.id == execution_id,
|
||||
MonitoringMessage.workspace_uuid == workspace_uuid,
|
||||
)
|
||||
)
|
||||
)
|
||||
rows = self._model_rows(result)
|
||||
if not rows:
|
||||
raise ValueError('Execution not found')
|
||||
serialized = self.ap.persistence_mgr.serialize_model(MonitoringMessage, rows[0])
|
||||
|
||||
async def _related(model, order_column):
|
||||
related = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(model)
|
||||
.where(
|
||||
sqlalchemy.and_(
|
||||
model.workspace_uuid == workspace_uuid,
|
||||
model.message_id == execution_id,
|
||||
)
|
||||
)
|
||||
.order_by(order_column.asc())
|
||||
)
|
||||
return [self.ap.persistence_mgr.serialize_model(model, row) for row in self._model_rows(related)]
|
||||
|
||||
return {
|
||||
'source': 'pipeline',
|
||||
'message': serialized,
|
||||
'llm_calls': await _related(
|
||||
persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.timestamp
|
||||
),
|
||||
'tool_calls': await _related(
|
||||
persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.timestamp
|
||||
),
|
||||
'errors': await _related(
|
||||
persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.timestamp
|
||||
),
|
||||
}
|
||||
|
||||
raise ValueError('Unknown execution source')
|
||||
|
||||
async def get_token_statistics(
|
||||
self,
|
||||
context: TenantContext,
|
||||
|
||||
@@ -18,6 +18,103 @@ if typing.TYPE_CHECKING:
|
||||
from .pool import get_query_execution_context
|
||||
|
||||
|
||||
async def resolve_processor_name(
|
||||
ap: app.Application,
|
||||
workspace_uuid: str,
|
||||
processor_type: str,
|
||||
processor_id: str | None,
|
||||
runner_id: str = '',
|
||||
) -> str:
|
||||
"""Display name of the processor kind that handled a run.
|
||||
|
||||
Monitoring attributed every call to a Pipeline, so an Agent or a plugin event
|
||||
processor recorded "Unknown" for the entity that actually ran. Each kind is
|
||||
resolved from the table that owns it, falling back to the identity the
|
||||
binding carried.
|
||||
"""
|
||||
import sqlalchemy
|
||||
|
||||
from ..entity.persistence import agent as persistence_agent
|
||||
from ..entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
identifier = str(processor_id or '').strip()
|
||||
if identifier:
|
||||
try:
|
||||
model = persistence_pipeline.LegacyPipeline if processor_type == 'pipeline' else persistence_agent.Agent
|
||||
result = await ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(model.name).where(
|
||||
sqlalchemy.and_(
|
||||
model.uuid == identifier,
|
||||
model.workspace_uuid == workspace_uuid,
|
||||
)
|
||||
)
|
||||
)
|
||||
name = str(result.scalar() or '').strip()
|
||||
if name:
|
||||
return name
|
||||
except Exception as e:
|
||||
ap.logger.warning(f'Failed to resolve monitoring processor name: {e}')
|
||||
# A deleted Agent or an unregistered plugin processor still names the
|
||||
# component that ran, never a bare "Unknown".
|
||||
return identifier or runner_id or processor_type
|
||||
|
||||
|
||||
async def prepare_monitoring_identity(
|
||||
ap: app.Application,
|
||||
query: pipeline_query.Query,
|
||||
*,
|
||||
processor_type: str,
|
||||
processor_id: str | None,
|
||||
runner_id: str = '',
|
||||
execution_record_id: str = '',
|
||||
) -> None:
|
||||
"""Pin the monitoring identity of a runner-owned execution.
|
||||
|
||||
A Pipeline pins these variables when it starts a query; a Runner started by
|
||||
an Agent or a plugin event processor has to pin them itself, otherwise every
|
||||
LLM/tool call it makes is attributed to "Unknown". An enclosing Pipeline
|
||||
already named the run, so its label is never overwritten.
|
||||
|
||||
``_monitoring_message_id`` is the execution record the calls belong to: the
|
||||
Pipeline's message, or the Agent run's id. Both are the id the executions
|
||||
list shows, which is what lets a call jump back to its record.
|
||||
"""
|
||||
try:
|
||||
variables = getattr(query, 'variables', None)
|
||||
if not isinstance(variables, dict):
|
||||
return
|
||||
if variables.get('_monitoring_pipeline_name'):
|
||||
return
|
||||
|
||||
bot_name = 'WebChat'
|
||||
bot_uuid = str(getattr(query, 'bot_uuid', '') or '')
|
||||
workspace_uuid = str(getattr(query, 'workspace_uuid', '') or '')
|
||||
if bot_uuid and workspace_uuid:
|
||||
try:
|
||||
bot = await ap.bot_service.get_bot(
|
||||
workspace_uuid,
|
||||
bot_uuid,
|
||||
include_secret=False,
|
||||
)
|
||||
if bot:
|
||||
bot_name = bot.get('name', bot_name)
|
||||
except Exception as e:
|
||||
ap.logger.warning(f'Failed to resolve monitoring bot name: {e}')
|
||||
|
||||
variables['_monitoring_bot_name'] = bot_name
|
||||
variables['_monitoring_pipeline_name'] = await resolve_processor_name(
|
||||
ap,
|
||||
workspace_uuid,
|
||||
processor_type,
|
||||
processor_id,
|
||||
runner_id,
|
||||
)
|
||||
if execution_record_id:
|
||||
variables['_monitoring_message_id'] = execution_record_id
|
||||
except Exception as e:
|
||||
ap.logger.error(f'Failed to prepare monitoring identity: {e}')
|
||||
|
||||
|
||||
class MonitoringHelper:
|
||||
"""Helper class for monitoring operations"""
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import typing
|
||||
import time
|
||||
import asyncio
|
||||
import inspect
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -448,6 +449,19 @@ class ToolManager:
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
result = await invoke()
|
||||
except asyncio.CancelledError:
|
||||
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||
await self._record_tool_call(
|
||||
name=name,
|
||||
source=source,
|
||||
parameters=parameters,
|
||||
query=query,
|
||||
duration_ms=duration_ms,
|
||||
status='error',
|
||||
error_message='cancelled',
|
||||
)
|
||||
self._trace_tool_call(source=source, name=name, query=query, outcome='cancelled', error='cancelled')
|
||||
raise
|
||||
except Exception as e:
|
||||
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||
await self._record_tool_call(
|
||||
@@ -459,6 +473,14 @@ class ToolManager:
|
||||
status='error',
|
||||
error_message=str(e),
|
||||
)
|
||||
outcome = 'timeout' if isinstance(e, TimeoutError) else 'failed'
|
||||
self._trace_tool_call(
|
||||
source=source,
|
||||
name=name,
|
||||
query=query,
|
||||
outcome=outcome,
|
||||
error=str(e) or type(e).__name__,
|
||||
)
|
||||
raise
|
||||
|
||||
duration_ms = int((time.perf_counter() - start_time) * 1000)
|
||||
@@ -471,8 +493,60 @@ class ToolManager:
|
||||
status='success',
|
||||
result=result,
|
||||
)
|
||||
self._trace_tool_call(source=source, name=name, query=query, outcome='success')
|
||||
return result
|
||||
|
||||
def _trace_tool_call(
|
||||
self,
|
||||
*,
|
||||
source: str,
|
||||
name: str,
|
||||
query: pipeline_query.Query,
|
||||
outcome: str,
|
||||
error: str = '',
|
||||
) -> None:
|
||||
"""Report one tool invocation as its own step of the execution chain.
|
||||
|
||||
Every tool call a Run makes - sandbox, skill, plugin or MCP - is the
|
||||
observable action of that Run, so the chain shows it under the step that
|
||||
issued it instead of collapsing the Run into a single opaque node.
|
||||
|
||||
An Agent's tool call is executed for the Runtime's RPC task, which never
|
||||
inherited the chain context. The Run id the caller already carries is the
|
||||
alias that task's record resolves the owning execution through.
|
||||
"""
|
||||
# Imported here: telemetry pulls the application, and the tool manager is
|
||||
# constructed while the application is still assembling.
|
||||
from ...telemetry.execution import record as record_execution
|
||||
from ...telemetry.platform import processing_mode
|
||||
|
||||
try:
|
||||
record_execution(
|
||||
self.ap,
|
||||
get_query_execution_context(query),
|
||||
execution_id=self._trace_execution_id(query),
|
||||
family='tool',
|
||||
operation=name,
|
||||
adapter=source,
|
||||
mode=processing_mode.get(),
|
||||
outcome=outcome,
|
||||
error=error,
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to record tool call trace: {e}')
|
||||
|
||||
@staticmethod
|
||||
def _trace_execution_id(query: pipeline_query.Query) -> str:
|
||||
"""The execution chain this call belongs to, when the caller knows it.
|
||||
|
||||
``''`` lets the emitter fall back to the task's own trace context, which
|
||||
is what a Pipeline-lane tool call has.
|
||||
"""
|
||||
session = getattr(query, '_agent_run_session', None)
|
||||
if not isinstance(session, dict):
|
||||
return ''
|
||||
return str(session.get('run_id') or '').strip()
|
||||
|
||||
async def execute_func_call(
|
||||
self,
|
||||
name: str,
|
||||
|
||||
@@ -33,7 +33,7 @@ from .trace import TraceState
|
||||
|
||||
MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'})
|
||||
OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'})
|
||||
FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'})
|
||||
FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api', 'tool'})
|
||||
ORIGINS = frozenset({'platform', 'webui', 'api'})
|
||||
CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'})
|
||||
CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'})
|
||||
@@ -255,9 +255,9 @@ class ExecutionCounters:
|
||||
state = trace_mod.current()
|
||||
if state is None:
|
||||
# Cross-task work has no inherited context; resolve the owning
|
||||
# execution through the registry. It has no open step here, so
|
||||
# its node becomes a root of that execution instead of being
|
||||
# mirrored from another task's stack.
|
||||
# execution through the registry. It owns no step here, so the
|
||||
# record attaches to the innermost step the chain has open -
|
||||
# the step that caused this work.
|
||||
lookup = str(execution_id or '').strip() or current_execution_id()
|
||||
if lookup:
|
||||
state = self._resolve(lookup)
|
||||
|
||||
@@ -11,12 +11,17 @@ small execution-scoped context the emitter needs:
|
||||
* the routing/run identity pinned around the block that produced a node;
|
||||
* the open-step stack, which lives in a ContextVar holding *one task's own
|
||||
immutable tuple*. Child tasks inherit a snapshot at creation; no shared
|
||||
mutable stack and no cross-task mirror exists, so sibling tasks can never
|
||||
corrupt each other's parent links.
|
||||
mutable stack exists, so sibling tasks can never corrupt each other's parent
|
||||
links.
|
||||
|
||||
Cross-task callers (plugin/RPC work with no inherited context) resolve their
|
||||
execution through the owner's execution-id registry instead, which only
|
||||
carries the execution state - never node payloads.
|
||||
carries the execution state - never node payloads. Such a caller owns no step
|
||||
of its own, yet its work is still caused by a step of the chain: the sandbox
|
||||
tool a Runner invoked, the platform API a plugin called back over RPC. The
|
||||
execution state therefore also keeps the ordered set of steps any task
|
||||
currently has open, and a record emitted with no open step of its own attaches
|
||||
to the innermost one of those instead of becoming a root of the chain.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -77,6 +82,7 @@ class TraceState:
|
||||
'pipeline_plugins',
|
||||
'node_adapter',
|
||||
'node_runner',
|
||||
'open_frames',
|
||||
)
|
||||
|
||||
def __init__(self, execution_id: str | None = None) -> None:
|
||||
@@ -110,6 +116,10 @@ class TraceState:
|
||||
# First adapter/runner seen on a node, used for the chain record.
|
||||
self.node_adapter = ''
|
||||
self.node_runner = ''
|
||||
# Steps currently open anywhere in this chain, in the order they were
|
||||
# opened. A record emitted by a task that owns no step of its own
|
||||
# attaches to the innermost entry here.
|
||||
self.open_frames: dict[str, NodeFrame] = {}
|
||||
|
||||
def allocate_seq(self) -> int:
|
||||
"""Mint the next chain-local node order."""
|
||||
@@ -117,6 +127,17 @@ class TraceState:
|
||||
self.node_seq += 1
|
||||
return seq
|
||||
|
||||
def open_lane(self) -> str:
|
||||
"""The innermost step open in this chain, whichever task opened it."""
|
||||
frames = list(self.open_frames)
|
||||
return frames[-1] if frames else ''
|
||||
|
||||
def open_step(self, frame: NodeFrame) -> None:
|
||||
self.open_frames[frame.node_id] = frame
|
||||
|
||||
def close_step(self, frame: NodeFrame) -> None:
|
||||
self.open_frames.pop(frame.node_id, None)
|
||||
|
||||
def note_node(
|
||||
self,
|
||||
*,
|
||||
@@ -221,8 +242,10 @@ def resolve_node(
|
||||
|
||||
A record stamped with a ``node`` the calling task has open reuses that
|
||||
step's identity, order and start time; anything else (a plain observation,
|
||||
or a node opened by another task) is a fresh node under this task's
|
||||
innermost open step.
|
||||
or a node opened by another task) is a fresh node under the innermost step
|
||||
open in this task. A task that inherited no step at all - plugin/RPC work
|
||||
that only knows the execution - attaches to the innermost step the chain
|
||||
has open, which is the step that caused the work.
|
||||
"""
|
||||
stack = _node.get()
|
||||
if node:
|
||||
@@ -231,7 +254,7 @@ def resolve_node(
|
||||
if frame.node_id == node:
|
||||
parent = stack[index - 1].node_id if index > 0 else ''
|
||||
return frame.node_id, frame.seq, frame.started_at, parent
|
||||
parent = stack[-1].node_id if stack else ''
|
||||
parent = stack[-1].node_id if stack else state.open_lane()
|
||||
return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent
|
||||
|
||||
|
||||
@@ -250,9 +273,11 @@ def stage_scope() -> typing.Iterator[str]:
|
||||
return
|
||||
frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now())
|
||||
token = _node.set(_node.get() + (frame,))
|
||||
state.open_step(frame)
|
||||
try:
|
||||
yield frame.node_id
|
||||
finally:
|
||||
state.close_step(frame)
|
||||
try:
|
||||
_node.reset(token)
|
||||
except (ValueError, RuntimeError):
|
||||
|
||||
Reference in New Issue
Block a user