feat(monitoring): unify execution views and trace tool calls

This commit is contained in:
Hyu
2026-10-07 22:36:10 +08:00
parent de886edc1b
commit 9f76037e64
42 changed files with 4181 additions and 889 deletions
@@ -194,6 +194,21 @@ class AgentRunOrchestrator:
state_context = build_state_context(event, binding, descriptor)
run_id = context['run_id']
# Monitoring attributes every call to the processor that ran it and to the
# record that owns it; a Runner has to claim both itself, since no
# Pipeline did. Runs before the Runner is invoked and before the run
# context is handed to it, so its own variables stay private.
from ...pipeline import monitoring_helper
await monitoring_helper.prepare_monitoring_identity(
self.ap,
execution_query,
processor_type=binding.processor_type,
processor_id=binding.processor_id,
runner_id=binding.runner_id,
execution_record_id=run_id,
)
context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any(
tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform'
for tool in resources.get('tools', [])
@@ -50,6 +50,47 @@ class MonitoringRouterGroup(group.RouterGroup):
return self.success(data=metrics)
@self.route('/executions', methods=['GET'], permission=Permission.RESOURCE_VIEW)
async def get_executions(request_context: RequestContext) -> str:
"""Unified execution list across agent runs and pipeline queries."""
bot_ids = quart.request.args.getlist('botId')
pipeline_ids = quart.request.args.getlist('pipelineId')
agent_ids = quart.request.args.getlist('agentId')
statuses = quart.request.args.getlist('status')
source = quart.request.args.get('source', 'all')
mode = quart.request.args.get('mode', 'all')
start_time = parse_iso_datetime(quart.request.args.get('startTime'))
end_time = parse_iso_datetime(quart.request.args.get('endTime'))
result = await self.ap.monitoring_service.get_executions(
request_context,
bot_ids=bot_ids if bot_ids else None,
pipeline_ids=pipeline_ids if pipeline_ids else None,
agent_ids=agent_ids if agent_ids else None,
statuses=statuses if statuses else None,
source=source,
mode=mode,
start_time=start_time,
end_time=end_time,
limit=quart.request.args.get('limit', 50),
offset=quart.request.args.get('offset', 0),
)
return self.success(data=result)
@self.route(
'/executions/<source>/<execution_id>',
methods=['GET'],
permission=Permission.RESOURCE_VIEW,
)
async def get_execution_detail(source: str, execution_id: str, request_context: RequestContext) -> str:
"""Full trace of a single execution (agent run or pipeline query)."""
try:
detail = await self.ap.monitoring_service.get_execution_detail(request_context, source, execution_id)
except ValueError as exc:
return self.fail(404, str(exc))
return self.success(data=detail)
@self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW)
async def get_token_statistics(request_context: RequestContext) -> str:
"""Get detailed token usage statistics (summary, per-model, timeseries)."""
@@ -639,7 +680,7 @@ class MonitoringRouterGroup(group.RouterGroup):
'platform',
]
else:
return self.error(message=f'Invalid export type: {export_type}', code=400)
return self.fail(400, f'Invalid export type: {export_type}')
# Generate CSV content with UTF-8 BOM for Excel compatibility
import io
@@ -10,7 +10,10 @@ from sqlalchemy.dialects import postgresql as postgresql_dialect
from sqlalchemy.dialects import sqlite as sqlite_dialect
from ....core import app
from ....entity.persistence import agent as persistence_agent
from ....entity.persistence import monitoring as persistence_monitoring
from ....entity.persistence import agent_run as persistence_agent_run
from ....entity.persistence import agent_interaction as persistence_agent_interaction
from ..authz import WorkspaceRequiredError
from ..context import ExecutionContext
from .tenant import TenantContext, require_workspace_uuid
@@ -29,6 +32,76 @@ _HARD_MAX_MONITORING_OFFSET = 10000000
_DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4
_HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100
# Canonical execution status groups shared by the agent ledger and pipeline query
# records so both object types can be summarised with one vocabulary.
_AGENT_STATUS_GROUP = {
'completed': 'completed',
'failed': 'failed',
'timeout': 'failed',
'cancelled': 'cancelled',
'created': 'queued',
'queued': 'queued',
'claimed': 'running',
'running': 'running',
}
_PIPELINE_STATUS_GROUP = {
'success': 'completed',
'error': 'failed',
'pending': 'running',
'discarded': 'ignored',
'not_matched': 'ignored',
}
def _status_group(status: str, source: str) -> str:
table = _AGENT_STATUS_GROUP if source == 'agent' else _PIPELINE_STATUS_GROUP
return table.get(status, status)
def _epoch_ms(value: datetime.datetime | None) -> int | None:
"""Epoch milliseconds for a DB timestamp, treating naive values as UTC."""
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=datetime.timezone.utc)
return int(round(value.timestamp() * 1000))
def _percentile_ms(values: list[float], fraction: float) -> int | None:
"""Nearest-rank percentile (ms) over a non-empty, prefix-sorted list."""
if not values:
return None
rank = max(1, min(len(values), int(round(fraction * len(values)))))
return int(round(values[rank - 1]))
def _collect_text(node, out: list[str], depth: int = 0) -> None:
"""Best-effort extraction of human text from a serialised message chain."""
if depth > 6:
return
if isinstance(node, dict):
for key, value in node.items():
if key == 'text' and isinstance(value, str):
out.append(value)
elif isinstance(value, (dict, list)):
_collect_text(value, out, depth + 1)
elif isinstance(node, list):
for item in node:
_collect_text(item, out, depth + 1)
def _message_preview(raw: str | None, limit: int = 160) -> str:
if not raw:
return ''
try:
parsed = json.loads(raw)
except (TypeError, ValueError):
return raw[:limit]
parts: list[str] = []
_collect_text(parsed, parts)
text = ' '.join(part.strip() for part in parts if part.strip())
return (text or raw)[:limit]
def _normalize_user_id(value: str | int | None) -> str | None:
"""Convert numeric platform IDs before binding a VARCHAR with asyncpg.
@@ -885,6 +958,10 @@ class MonitoringService:
active_sessions_result = await self.ap.persistence_mgr.execute_async(active_session_query)
active_sessions = active_sessions_result.scalar() or 0
# Execution-first KPI values (runs + pipeline queries, latency, token
# coverage) are served by `get_executions`, which applies the execution
# view filters. This endpoint stays the message/traffic rollup so the
# dashboard does not compute the same rollup twice per page load.
return {
'total_messages': total_messages,
'llm_calls': llm_calls,
@@ -894,6 +971,527 @@ class MonitoringService:
'active_sessions': active_sessions,
}
# ========== Execution view ==========
def _agent_run_conditions(
self,
workspace_uuid: str,
*,
bot_ids: list[str] | None = None,
agent_ids: list[str] | None = None,
statuses: list[str] | None = None,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
debug: bool | None = None,
) -> list:
AgentRun = persistence_agent_run.AgentRun
conditions = [AgentRun.workspace_id == workspace_uuid]
if bot_ids:
conditions.append(AgentRun.bot_id.in_(bot_ids))
if agent_ids:
conditions.append(AgentRun.agent_id.in_(agent_ids))
if statuses is not None:
conditions.append(AgentRun.status.in_(statuses) if statuses else sqlalchemy.false())
if start_time:
conditions.append(AgentRun.created_at >= start_time)
if end_time:
conditions.append(AgentRun.created_at <= end_time)
if debug is not None:
debug_condition = sqlalchemy.or_(
AgentRun.binding_id.like('debug:%'),
AgentRun.conversation_id.like('debug:%'),
)
conditions.append(debug_condition if debug else sqlalchemy.not_(debug_condition))
return conditions
def _pipeline_execution_conditions(
self,
workspace_uuid: str,
*,
bot_ids: list[str] | None = None,
pipeline_ids: list[str] | None = None,
statuses: list[str] | None = None,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
) -> list:
MonitoringMessage = persistence_monitoring.MonitoringMessage
conditions = [
MonitoringMessage.workspace_uuid == workspace_uuid,
# Assistant replies are outputs of a query, not an execution of their own.
sqlalchemy.or_(MonitoringMessage.role.is_(None), MonitoringMessage.role != 'assistant'),
]
if bot_ids:
conditions.append(MonitoringMessage.bot_id.in_(bot_ids))
if pipeline_ids:
conditions.append(MonitoringMessage.pipeline_id.in_(pipeline_ids))
if statuses is not None:
conditions.append(MonitoringMessage.status.in_(statuses) if statuses else sqlalchemy.false())
if start_time:
conditions.append(MonitoringMessage.timestamp >= start_time)
if end_time:
conditions.append(MonitoringMessage.timestamp <= end_time)
return conditions
@staticmethod
def _raw_agent_statuses(groups: list[str] | None) -> list[str] | None:
"""Raw agent statuses for a group filter; empty means "no group applies"."""
if not groups:
return None
raw: list[str] = []
for group in groups:
raw.extend(status for status, mapped in _AGENT_STATUS_GROUP.items() if mapped == group)
return raw
@staticmethod
def _raw_pipeline_statuses(groups: list[str] | None) -> list[str] | None:
"""Raw pipeline statuses for a group filter; empty means "no group applies"."""
if not groups:
return None
raw: list[str] = []
for group in groups:
raw.extend(status for status, mapped in _PIPELINE_STATUS_GROUP.items() if mapped == group)
return raw
@staticmethod
def _model_rows(result) -> list:
"""Rows of one ``select(Model)`` executed through ``execute_async``.
The Core keeps the connection result contract, where ``select(Model)``
yields a *flat* row of the model's columns (``row.uuid``, ``row.status``)
rather than a one-item ORM row. ``.scalars()`` would therefore return the
first column - the id - instead of the row, so the rows are taken as-is.
"""
return list(result.all())
@staticmethod
def _deserialize_json(raw: str | None) -> dict:
if not raw:
return {}
try:
value = json.loads(raw)
except (TypeError, ValueError):
return {}
return value if isinstance(value, dict) else {}
async def _agent_names(self, workspace_uuid: str, rows: list) -> dict[str, str]:
"""The processor name of every Agent run in one page, in one query.
A run stores the Agent it executed as a uuid; the list has to name the
processor, so the names are resolved for the whole page at once.
"""
agent_ids = {row.agent_id for row in rows if row.agent_id}
if not agent_ids:
return {}
Agent = persistence_agent.Agent
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Agent.uuid, Agent.name).where(
sqlalchemy.and_(
Agent.workspace_uuid == workspace_uuid,
Agent.uuid.in_(agent_ids),
)
)
)
return {row.uuid: row.name for row in self._model_rows(result)}
def _serialize_agent_execution(self, row, agent_names: dict[str, str] | None = None) -> dict:
metadata = self._deserialize_json(row.metadata_json)
duration_ms = None
if row.started_at and row.finished_at:
duration_ms = int(round((row.finished_at - row.started_at).total_seconds() * 1000))
created_at_ms = _epoch_ms(row.created_at)
started_at_ms = _epoch_ms(row.started_at)
finished_at_ms = _epoch_ms(row.finished_at)
is_debug = str(row.binding_id or '').startswith('debug:') or str(row.conversation_id or '').startswith('debug:')
return {
'source': 'agent',
'id': row.run_id,
'event_id': row.event_id,
'status': row.status,
'status_group': _status_group(row.status, 'agent'),
'title': metadata.get('event_type') or row.queue_name or row.runner_id,
'target_kind': 'agent' if row.agent_id else 'processor',
'target_id': row.agent_id or row.binding_id,
'target_name': metadata.get('agent_name')
or metadata.get('target_name')
or (agent_names or {}).get(row.agent_id or ''),
'bot_id': row.bot_id,
'bot_name': metadata.get('bot_name'),
'pipeline_id': None,
'pipeline_name': None,
'runner_id': row.runner_id,
'conversation_id': row.conversation_id,
'session_id': row.conversation_id,
'created_at_ms': created_at_ms,
'started_at_ms': started_at_ms,
'finished_at_ms': finished_at_ms,
'duration_ms': duration_ms,
'usage': self._deserialize_json(row.usage_json) or None,
'cost': self._deserialize_json(row.cost_json) or None,
'queue_name': row.queue_name,
'debug': is_debug,
'has_error': row.status in {'failed', 'timeout'},
}
def _serialize_pipeline_execution(self, row) -> dict:
created_at_ms = _epoch_ms(row.timestamp)
return {
'source': 'pipeline',
'id': row.id,
'event_id': None,
'status': row.status,
'status_group': _status_group(row.status, 'pipeline'),
'title': _message_preview(row.message_content),
'target_kind': 'pipeline',
'target_id': row.pipeline_id,
'target_name': row.pipeline_name,
'bot_id': row.bot_id,
'bot_name': row.bot_name,
'pipeline_id': row.pipeline_id,
'pipeline_name': row.pipeline_name,
'runner_id': row.runner_name,
'conversation_id': None,
'session_id': row.session_id,
'created_at_ms': created_at_ms,
'started_at_ms': None,
'finished_at_ms': None,
'duration_ms': None,
'usage': None,
'cost': None,
'queue_name': None,
'debug': False,
'has_error': row.status == 'error',
'platform': row.platform,
'user_id': row.user_id,
'user_name': row.user_name,
}
async def _get_token_coverage(
self,
workspace_uuid: str,
*,
bot_ids: list[str] | None = None,
pipeline_ids: list[str] | None = None,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
) -> dict:
LLMCall = persistence_monitoring.MonitoringLLMCall
conditions = [LLMCall.workspace_uuid == workspace_uuid]
if bot_ids:
conditions.append(LLMCall.bot_id.in_(bot_ids))
if pipeline_ids:
conditions.append(LLMCall.pipeline_id.in_(pipeline_ids))
if start_time:
conditions.append(LLMCall.timestamp >= start_time)
if end_time:
conditions.append(LLMCall.timestamp <= end_time)
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
sqlalchemy.func.count(LLMCall.id),
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0),
sqlalchemy.func.sum(sqlalchemy.case((LLMCall.total_tokens > 0, 1), else_=0)),
).where(sqlalchemy.and_(*conditions))
)
row = result.first()
calls, total, calls_with_usage = row if row else (0, 0, 0)
return {
'total_tokens': int(total or 0),
'calls': calls or 0,
'calls_with_usage': calls_with_usage or 0,
}
async def _get_execution_summary(
self,
workspace_uuid: str,
*,
bot_ids: list[str] | None = None,
pipeline_ids: list[str] | None = None,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
) -> dict:
AgentRun = persistence_agent_run.AgentRun
run_conditions = self._agent_run_conditions(
workspace_uuid, bot_ids=bot_ids, start_time=start_time, end_time=end_time
)
run_status_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(AgentRun.status, sqlalchemy.func.count(AgentRun.id))
.where(sqlalchemy.and_(*run_conditions))
.group_by(AgentRun.status)
)
agent_counts = {row[0]: row[1] for row in run_status_result.all()}
agent_total = sum(agent_counts.values())
debug_count_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where(
sqlalchemy.and_(
*run_conditions,
sqlalchemy.or_(
AgentRun.binding_id.like('debug:%'),
AgentRun.conversation_id.like('debug:%'),
),
)
)
)
debug_count = debug_count_result.scalar() or 0
# Percentiles are computed in Python, so the sample must stay bounded:
# take the most recent completed runs of the window and label the count
# so the UI can state how many runs the numbers rest on.
timing_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(AgentRun.started_at, AgentRun.finished_at)
.where(
sqlalchemy.and_(
*run_conditions,
AgentRun.status == 'completed',
AgentRun.started_at.is_not(None),
AgentRun.finished_at.is_not(None),
)
)
.order_by(AgentRun.created_at.desc())
.limit(self._detail_limit())
)
durations = sorted((row[1] - row[0]).total_seconds() * 1000 for row in timing_result.all() if row[0] and row[1])
pipeline_conditions = self._pipeline_execution_conditions(
workspace_uuid, bot_ids=bot_ids, pipeline_ids=pipeline_ids, start_time=start_time, end_time=end_time
)
pipeline_status_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_monitoring.MonitoringMessage.status,
sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id),
)
.where(sqlalchemy.and_(*pipeline_conditions))
.group_by(persistence_monitoring.MonitoringMessage.status)
)
pipeline_counts = {row[0]: row[1] for row in pipeline_status_result.all()}
pipeline_total = sum(pipeline_counts.values())
# Pending interactions are gated by the same bot/window scope as the runs
# so "waiting input" cannot drift from the rest of the card.
waiting_conditions = [
persistence_agent_interaction.AgentInteraction.workspace_id == workspace_uuid,
persistence_agent_interaction.AgentInteraction.status == 'pending',
]
if bot_ids:
waiting_conditions.append(persistence_agent_interaction.AgentInteraction.bot_id.in_(bot_ids))
if start_time:
waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at >= start_time)
if end_time:
waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at <= end_time)
waiting_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(sqlalchemy.func.count(persistence_agent_interaction.AgentInteraction.id)).where(
sqlalchemy.and_(*waiting_conditions)
)
)
waiting = waiting_result.scalar() or 0
by_status: dict[str, int] = {}
for status, count in agent_counts.items():
group = _status_group(status, 'agent')
by_status[group] = by_status.get(group, 0) + count
for status, count in pipeline_counts.items():
group = _status_group(status, 'pipeline')
by_status[group] = by_status.get(group, 0) + count
completed = by_status.get('completed', 0)
failed = by_status.get('failed', 0)
denominator = completed + failed
total = agent_total + pipeline_total
return {
'total': total,
'by_source': {'agent': agent_total, 'pipeline': pipeline_total},
'by_status': by_status,
'completed': completed,
'failed': failed,
'running': by_status.get('running', 0),
'queued': by_status.get('queued', 0),
'cancelled': by_status.get('cancelled', 0),
'ignored': by_status.get('ignored', 0),
'waiting': waiting,
'success_rate': round(completed / denominator * 100, 2) if denominator > 0 else None,
'denominator': denominator,
'p50_duration_ms': _percentile_ms(durations, 0.5),
'p95_duration_ms': _percentile_ms(durations, 0.95),
'duration_sample': len(durations),
'debug': debug_count,
'real': total - debug_count,
}
async def get_executions(
self,
context: TenantContext,
*,
bot_ids: list[str] | None = None,
pipeline_ids: list[str] | None = None,
agent_ids: list[str] | None = None,
statuses: list[str] | None = None,
source: str = 'all',
mode: str = 'all',
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
limit: int = 50,
offset: int = 0,
) -> dict:
"""List executions from both object types (agent runs and pipeline queries)."""
workspace_uuid = require_workspace_uuid(context)
limit, offset = self.normalize_page_window(limit, offset)
fetch_limit = min(limit + offset, self._configured_query_limit('page_rows', 1000, 5000))
if source not in {'all', 'agent', 'pipeline'}:
source = 'all'
if mode not in {'all', 'real', 'debug'}:
mode = 'all'
debug = None if mode == 'all' else (mode == 'debug')
include_agent = source in {'all', 'agent'} and (not pipeline_ids or bool(agent_ids))
include_pipeline = source in {'all', 'pipeline'} and (not agent_ids or bool(pipeline_ids))
items: list[dict] = []
total = 0
if include_agent:
run_conditions = self._agent_run_conditions(
workspace_uuid,
bot_ids=bot_ids,
agent_ids=agent_ids,
statuses=self._raw_agent_statuses(statuses),
start_time=start_time,
end_time=end_time,
debug=debug,
)
AgentRun = persistence_agent_run.AgentRun
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(AgentRun)
.where(sqlalchemy.and_(*run_conditions))
.order_by(AgentRun.created_at.desc(), AgentRun.id.desc())
.limit(fetch_limit)
)
rows = self._model_rows(result)
count_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where(sqlalchemy.and_(*run_conditions))
)
total += count_result.scalar() or 0
agent_names = await self._agent_names(workspace_uuid, rows)
items.extend(self._serialize_agent_execution(row, agent_names=agent_names) for row in rows)
if include_pipeline and (debug is None or debug is False):
message_conditions = self._pipeline_execution_conditions(
workspace_uuid,
bot_ids=bot_ids,
pipeline_ids=pipeline_ids,
statuses=self._raw_pipeline_statuses(statuses),
start_time=start_time,
end_time=end_time,
)
MonitoringMessage = persistence_monitoring.MonitoringMessage
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(MonitoringMessage)
.where(sqlalchemy.and_(*message_conditions))
.order_by(MonitoringMessage.timestamp.desc())
.limit(fetch_limit)
)
rows = self._model_rows(result)
count_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(sqlalchemy.func.count(MonitoringMessage.id)).where(
sqlalchemy.and_(*message_conditions)
)
)
total += count_result.scalar() or 0
items.extend(self._serialize_pipeline_execution(row) for row in rows)
items.sort(key=lambda item: item.get('created_at_ms') or 0, reverse=True)
page = items[offset : offset + limit]
return {
'items': page,
'total': total,
'limit': limit,
'offset': offset,
'has_more': total > offset + len(page),
'summary': {
'executions': await self._get_execution_summary(
workspace_uuid,
bot_ids=bot_ids,
pipeline_ids=pipeline_ids,
start_time=start_time,
end_time=end_time,
),
'tokens': await self._get_token_coverage(
workspace_uuid,
bot_ids=bot_ids,
pipeline_ids=pipeline_ids,
start_time=start_time,
end_time=end_time,
),
},
}
async def get_execution_detail(
self,
context: TenantContext,
source: str,
execution_id: str,
) -> dict:
"""Full trace of a single execution, keyed by object type."""
workspace_uuid = require_workspace_uuid(context)
if source == 'agent':
from ....agent.runner.run_ledger_store import RunLedgerStore
store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine())
run = await store.get_run(execution_id)
if run is None or run.get('workspace_id') != workspace_uuid:
raise ValueError('Execution not found')
items, next_cursor, _, has_more = await store.page_run_events(run_id=execution_id, limit=200)
return {
'source': 'agent',
'run': run,
'events': items,
'has_more': has_more,
'next_cursor': next_cursor,
}
if source == 'pipeline':
MonitoringMessage = persistence_monitoring.MonitoringMessage
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(MonitoringMessage).where(
sqlalchemy.and_(
MonitoringMessage.id == execution_id,
MonitoringMessage.workspace_uuid == workspace_uuid,
)
)
)
rows = self._model_rows(result)
if not rows:
raise ValueError('Execution not found')
serialized = self.ap.persistence_mgr.serialize_model(MonitoringMessage, rows[0])
async def _related(model, order_column):
related = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(model)
.where(
sqlalchemy.and_(
model.workspace_uuid == workspace_uuid,
model.message_id == execution_id,
)
)
.order_by(order_column.asc())
)
return [self.ap.persistence_mgr.serialize_model(model, row) for row in self._model_rows(related)]
return {
'source': 'pipeline',
'message': serialized,
'llm_calls': await _related(
persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.timestamp
),
'tool_calls': await _related(
persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.timestamp
),
'errors': await _related(
persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.timestamp
),
}
raise ValueError('Unknown execution source')
async def get_token_statistics(
self,
context: TenantContext,
@@ -18,6 +18,103 @@ if typing.TYPE_CHECKING:
from .pool import get_query_execution_context
async def resolve_processor_name(
ap: app.Application,
workspace_uuid: str,
processor_type: str,
processor_id: str | None,
runner_id: str = '',
) -> str:
"""Display name of the processor kind that handled a run.
Monitoring attributed every call to a Pipeline, so an Agent or a plugin event
processor recorded "Unknown" for the entity that actually ran. Each kind is
resolved from the table that owns it, falling back to the identity the
binding carried.
"""
import sqlalchemy
from ..entity.persistence import agent as persistence_agent
from ..entity.persistence import pipeline as persistence_pipeline
identifier = str(processor_id or '').strip()
if identifier:
try:
model = persistence_pipeline.LegacyPipeline if processor_type == 'pipeline' else persistence_agent.Agent
result = await ap.persistence_mgr.execute_async(
sqlalchemy.select(model.name).where(
sqlalchemy.and_(
model.uuid == identifier,
model.workspace_uuid == workspace_uuid,
)
)
)
name = str(result.scalar() or '').strip()
if name:
return name
except Exception as e:
ap.logger.warning(f'Failed to resolve monitoring processor name: {e}')
# A deleted Agent or an unregistered plugin processor still names the
# component that ran, never a bare "Unknown".
return identifier or runner_id or processor_type
async def prepare_monitoring_identity(
ap: app.Application,
query: pipeline_query.Query,
*,
processor_type: str,
processor_id: str | None,
runner_id: str = '',
execution_record_id: str = '',
) -> None:
"""Pin the monitoring identity of a runner-owned execution.
A Pipeline pins these variables when it starts a query; a Runner started by
an Agent or a plugin event processor has to pin them itself, otherwise every
LLM/tool call it makes is attributed to "Unknown". An enclosing Pipeline
already named the run, so its label is never overwritten.
``_monitoring_message_id`` is the execution record the calls belong to: the
Pipeline's message, or the Agent run's id. Both are the id the executions
list shows, which is what lets a call jump back to its record.
"""
try:
variables = getattr(query, 'variables', None)
if not isinstance(variables, dict):
return
if variables.get('_monitoring_pipeline_name'):
return
bot_name = 'WebChat'
bot_uuid = str(getattr(query, 'bot_uuid', '') or '')
workspace_uuid = str(getattr(query, 'workspace_uuid', '') or '')
if bot_uuid and workspace_uuid:
try:
bot = await ap.bot_service.get_bot(
workspace_uuid,
bot_uuid,
include_secret=False,
)
if bot:
bot_name = bot.get('name', bot_name)
except Exception as e:
ap.logger.warning(f'Failed to resolve monitoring bot name: {e}')
variables['_monitoring_bot_name'] = bot_name
variables['_monitoring_pipeline_name'] = await resolve_processor_name(
ap,
workspace_uuid,
processor_type,
processor_id,
runner_id,
)
if execution_record_id:
variables['_monitoring_message_id'] = execution_record_id
except Exception as e:
ap.logger.error(f'Failed to prepare monitoring identity: {e}')
class MonitoringHelper:
"""Helper class for monitoring operations"""
+74
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import typing
import time
import asyncio
import inspect
from typing import TYPE_CHECKING
@@ -448,6 +449,19 @@ class ToolManager:
start_time = time.perf_counter()
try:
result = await invoke()
except asyncio.CancelledError:
duration_ms = int((time.perf_counter() - start_time) * 1000)
await self._record_tool_call(
name=name,
source=source,
parameters=parameters,
query=query,
duration_ms=duration_ms,
status='error',
error_message='cancelled',
)
self._trace_tool_call(source=source, name=name, query=query, outcome='cancelled', error='cancelled')
raise
except Exception as e:
duration_ms = int((time.perf_counter() - start_time) * 1000)
await self._record_tool_call(
@@ -459,6 +473,14 @@ class ToolManager:
status='error',
error_message=str(e),
)
outcome = 'timeout' if isinstance(e, TimeoutError) else 'failed'
self._trace_tool_call(
source=source,
name=name,
query=query,
outcome=outcome,
error=str(e) or type(e).__name__,
)
raise
duration_ms = int((time.perf_counter() - start_time) * 1000)
@@ -471,8 +493,60 @@ class ToolManager:
status='success',
result=result,
)
self._trace_tool_call(source=source, name=name, query=query, outcome='success')
return result
def _trace_tool_call(
self,
*,
source: str,
name: str,
query: pipeline_query.Query,
outcome: str,
error: str = '',
) -> None:
"""Report one tool invocation as its own step of the execution chain.
Every tool call a Run makes - sandbox, skill, plugin or MCP - is the
observable action of that Run, so the chain shows it under the step that
issued it instead of collapsing the Run into a single opaque node.
An Agent's tool call is executed for the Runtime's RPC task, which never
inherited the chain context. The Run id the caller already carries is the
alias that task's record resolves the owning execution through.
"""
# Imported here: telemetry pulls the application, and the tool manager is
# constructed while the application is still assembling.
from ...telemetry.execution import record as record_execution
from ...telemetry.platform import processing_mode
try:
record_execution(
self.ap,
get_query_execution_context(query),
execution_id=self._trace_execution_id(query),
family='tool',
operation=name,
adapter=source,
mode=processing_mode.get(),
outcome=outcome,
error=error,
)
except Exception as e:
self.ap.logger.warning(f'Failed to record tool call trace: {e}')
@staticmethod
def _trace_execution_id(query: pipeline_query.Query) -> str:
"""The execution chain this call belongs to, when the caller knows it.
``''`` lets the emitter fall back to the task's own trace context, which
is what a Pipeline-lane tool call has.
"""
session = getattr(query, '_agent_run_session', None)
if not isinstance(session, dict):
return ''
return str(session.get('run_id') or '').strip()
async def execute_func_call(
self,
name: str,
+4 -4
View File
@@ -33,7 +33,7 @@ from .trace import TraceState
MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'})
OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'})
FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'})
FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api', 'tool'})
ORIGINS = frozenset({'platform', 'webui', 'api'})
CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'})
CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'})
@@ -255,9 +255,9 @@ class ExecutionCounters:
state = trace_mod.current()
if state is None:
# Cross-task work has no inherited context; resolve the owning
# execution through the registry. It has no open step here, so
# its node becomes a root of that execution instead of being
# mirrored from another task's stack.
# execution through the registry. It owns no step here, so the
# record attaches to the innermost step the chain has open -
# the step that caused this work.
lookup = str(execution_id or '').strip() or current_execution_id()
if lookup:
state = self._resolve(lookup)
+31 -6
View File
@@ -11,12 +11,17 @@ small execution-scoped context the emitter needs:
* the routing/run identity pinned around the block that produced a node;
* the open-step stack, which lives in a ContextVar holding *one task's own
immutable tuple*. Child tasks inherit a snapshot at creation; no shared
mutable stack and no cross-task mirror exists, so sibling tasks can never
corrupt each other's parent links.
mutable stack exists, so sibling tasks can never corrupt each other's parent
links.
Cross-task callers (plugin/RPC work with no inherited context) resolve their
execution through the owner's execution-id registry instead, which only
carries the execution state - never node payloads.
carries the execution state - never node payloads. Such a caller owns no step
of its own, yet its work is still caused by a step of the chain: the sandbox
tool a Runner invoked, the platform API a plugin called back over RPC. The
execution state therefore also keeps the ordered set of steps any task
currently has open, and a record emitted with no open step of its own attaches
to the innermost one of those instead of becoming a root of the chain.
"""
from __future__ import annotations
@@ -77,6 +82,7 @@ class TraceState:
'pipeline_plugins',
'node_adapter',
'node_runner',
'open_frames',
)
def __init__(self, execution_id: str | None = None) -> None:
@@ -110,6 +116,10 @@ class TraceState:
# First adapter/runner seen on a node, used for the chain record.
self.node_adapter = ''
self.node_runner = ''
# Steps currently open anywhere in this chain, in the order they were
# opened. A record emitted by a task that owns no step of its own
# attaches to the innermost entry here.
self.open_frames: dict[str, NodeFrame] = {}
def allocate_seq(self) -> int:
"""Mint the next chain-local node order."""
@@ -117,6 +127,17 @@ class TraceState:
self.node_seq += 1
return seq
def open_lane(self) -> str:
"""The innermost step open in this chain, whichever task opened it."""
frames = list(self.open_frames)
return frames[-1] if frames else ''
def open_step(self, frame: NodeFrame) -> None:
self.open_frames[frame.node_id] = frame
def close_step(self, frame: NodeFrame) -> None:
self.open_frames.pop(frame.node_id, None)
def note_node(
self,
*,
@@ -221,8 +242,10 @@ def resolve_node(
A record stamped with a ``node`` the calling task has open reuses that
step's identity, order and start time; anything else (a plain observation,
or a node opened by another task) is a fresh node under this task's
innermost open step.
or a node opened by another task) is a fresh node under the innermost step
open in this task. A task that inherited no step at all - plugin/RPC work
that only knows the execution - attaches to the innermost step the chain
has open, which is the step that caused the work.
"""
stack = _node.get()
if node:
@@ -231,7 +254,7 @@ def resolve_node(
if frame.node_id == node:
parent = stack[index - 1].node_id if index > 0 else ''
return frame.node_id, frame.seq, frame.started_at, parent
parent = stack[-1].node_id if stack else ''
parent = stack[-1].node_id if stack else state.open_lane()
return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent
@@ -250,9 +273,11 @@ def stage_scope() -> typing.Iterator[str]:
return
frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now())
token = _node.set(_node.get() + (frame,))
state.open_step(frame)
try:
yield frame.node_id
finally:
state.close_step(frame)
try:
_node.reset(token)
except (ValueError, RuntimeError):