xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-10-22 00:15:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): attach called platform APIs to their step node on execution traces

Browse Source 
Every host step now opens a stage scope, so observations recorded inside it (including cross-task plugin/RPC calls resolved through the execution registry) carry the owning step as parent; the pipeline lane records its own pipeline/run node so acks and replies issued outside the runner still hang off a node.
This commit is contained in:
Hyu
2026-10-01 18:10:42 +08:00
parent df64f83e12
commit 923ebe12af
12 changed files with 1798 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/langbot/pkg/agent/runner/orchestrator.py
+128 -117
View File
@@ -38,6 +38,7 @@ from .session_registry import AgentRunSessionRegistry, get_session_registry
from .state_scope import build_state_context
from ...provider.tools.loaders import skill as skill_loader
from ...telemetry import trace as trace_mod
from ...telemetry.execution import bind_trace as bind_execution_trace
from ...telemetry.execution import close_trace as close_execution_trace
from ...telemetry.execution import record as record_execution
@@ -213,13 +214,16 @@ class AgentRunOrchestrator:
terminal_reason: str | None = None
terminal_usage: dict[str, typing.Any] | None = None
execution_outcome = 'unknown'
execution_error = ''
# A run reached without a platform ingress (WebUI debug, service API)
# owns its own chain; a run inside an ingress reuses that chain.
trace_binding: trace_mod.TraceBinding | None = None
run_token: typing.Any = None
# Workflow step node for this run; '' until the step scope is opened.
node = ''
try:
trace_binding = trace_mod.bind()
trace_binding = bind_execution_trace(self.ap, run_id)
run_token = trace_mod.set_run(run_id)
await self.journal.create_run(
event=event,
@@ -261,139 +265,140 @@ class AgentRunOrchestrator:
event_log_id=event_log_id,
)
async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results:
async for result_dict in results:
result_dict = dict(result_dict)
sequence = result_dict.get('sequence')
if sequence is not None:
try:
sequence_int = int(sequence)
except (TypeError, ValueError):
self.ap.logger.warning(
f'Runner {descriptor.id} returned invalid result sequence: {sequence}'
)
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
else:
if sequence_int in seen_sequences:
with trace_mod.stage_scope() as node:
async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results:
async for result_dict in results:
result_dict = dict(result_dict)
sequence = result_dict.get('sequence')
if sequence is not None:
try:
sequence_int = int(sequence)
except (TypeError, ValueError):
self.ap.logger.warning(
f'Runner {descriptor.id} returned duplicate result sequence '
f'{sequence_int} for run {run_id}; dropping duplicate'
)
continue
if sequence_int <= 0:
self.ap.logger.warning(
f'Runner {descriptor.id} returned non-positive result sequence '
f'{sequence_int} for run {run_id}'
f'Runner {descriptor.id} returned invalid result sequence: {sequence}'
)
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
elif last_sequence and sequence_int != last_sequence + 1:
self.ap.logger.warning(
f'Runner {descriptor.id} result sequence gap or out-of-order '
f'for run {run_id}: previous={last_sequence}, current={sequence_int}'
)
else:
if sequence_int in seen_sequences:
self.ap.logger.warning(
f'Runner {descriptor.id} returned duplicate result sequence '
f'{sequence_int} for run {run_id}; dropping duplicate'
)
continue
if sequence_int <= 0:
self.ap.logger.warning(
f'Runner {descriptor.id} returned non-positive result sequence '
f'{sequence_int} for run {run_id}'
)
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
elif last_sequence and sequence_int != last_sequence + 1:
self.ap.logger.warning(
f'Runner {descriptor.id} result sequence gap or out-of-order '
f'for run {run_id}: previous={last_sequence}, current={sequence_int}'
)
seen_sequences.add(sequence_int)
last_sequence = max(last_sequence, sequence_int)
else:
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
seen_sequences.add(sequence_int)
last_sequence = max(last_sequence, sequence_int)
else:
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
seen_sequences.add(sequence_int)
last_sequence = sequence_int
last_sequence = sequence_int
result_type = result_dict.get('type')
if result_type and not self.result_normalizer.validate_payload(
result_type,
result_dict.get('data', {}),
descriptor,
):
continue
await self.journal.append_run_result(
result_dict=result_dict,
run_id=run_id,
sequence=sequence_int,
)
# Trusted Host observers receive validated events before message-only normalization.
result_observer = (adapter_context or {}).get('_result_observer')
if result_observer is not None:
await result_observer(result_dict)
if result_type == 'state.updated':
await self.journal.handle_state_updated_event(
result_dict,
event,
binding,
result_type = result_dict.get('type')
if result_type and not self.result_normalizer.validate_payload(
result_type,
result_dict.get('data', {}),
descriptor,
run_id=run_id,
)
await self.result_normalizer.normalize(result_dict, descriptor)
continue
if result_type == 'action.requested':
if await self.interaction_manager.handle_result(
result_dict=result_dict,
event=event,
binding=binding,
descriptor=descriptor,
run_id=run_id,
adapter_context=adapter_context,
):
continue
if result_type == 'run.completed':
terminal_status = 'completed'
terminal_reason = (
result_dict.get('data', {}).get('finish_reason')
if isinstance(result_dict.get('data'), dict)
else None
)
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
elif result_type == 'run.failed':
terminal_status = 'failed'
data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {}
terminal_reason = data.get('error') or data.get('code')
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
has_completed_message = result_type == 'message.completed' or (
result_type == 'run.completed'
and isinstance(result_dict.get('data'), dict)
and bool(result_dict['data'].get('message'))
)
if has_completed_message and event.conversation_id and not assistant_transcript_written:
await self.journal.write_assistant_transcript(
await self.journal.append_run_result(
result_dict=result_dict,
event=event,
run_id=run_id,
runner_id=descriptor.id,
sequence=sequence_int,
)
assistant_transcript_written = True
result = await self.result_normalizer.normalize(result_dict, descriptor)
file_ids = result_dict.get('data', {}).get('file_ids', [])
if file_ids:
if result_type != 'message.completed' or result is None:
raise ValueError('Files must be attached to a completed message')
if binding.processor_type != 'pipeline':
raise ValueError('Agent files must be sent explicitly through the reply API')
from ...box.runner import exported_message, binding_for
# Trusted Host observers receive validated events before message-only normalization.
result_observer = (adapter_context or {}).get('_result_observer')
if result_observer is not None:
await result_observer(result_dict)
async with binding_for(execution_query).lock:
result.attachments = exported_message(execution_query, file_ids, consume=True)
if result_type == 'state.updated':
await self.journal.handle_state_updated_event(
result_dict,
event,
binding,
descriptor,
run_id=run_id,
)
await self.result_normalizer.normalize(result_dict, descriptor)
continue
if result is not None:
yield result
if result_type == 'action.requested':
if await self.interaction_manager.handle_result(
result_dict=result_dict,
event=event,
binding=binding,
descriptor=descriptor,
run_id=run_id,
adapter_context=adapter_context,
):
continue
run_snapshot = await self.journal.get_run(run_id)
if run_snapshot and run_snapshot.get('cancel_requested_at') is not None:
terminal_status = 'cancelled'
terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested'
break
if result_type == 'run.completed':
terminal_status = 'completed'
terminal_reason = (
result_dict.get('data', {}).get('finish_reason')
if isinstance(result_dict.get('data'), dict)
else None
)
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
elif result_type == 'run.failed':
terminal_status = 'failed'
data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {}
terminal_reason = data.get('error') or data.get('code')
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
has_completed_message = result_type == 'message.completed' or (
result_type == 'run.completed'
and isinstance(result_dict.get('data'), dict)
and bool(result_dict['data'].get('message'))
)
if has_completed_message and event.conversation_id and not assistant_transcript_written:
await self.journal.write_assistant_transcript(
result_dict=result_dict,
event=event,
run_id=run_id,
runner_id=descriptor.id,
)
assistant_transcript_written = True
result = await self.result_normalizer.normalize(result_dict, descriptor)
file_ids = result_dict.get('data', {}).get('file_ids', [])
if file_ids:
if result_type != 'message.completed' or result is None:
raise ValueError('Files must be attached to a completed message')
if binding.processor_type != 'pipeline':
raise ValueError('Agent files must be sent explicitly through the reply API')
from ...box.runner import exported_message, binding_for
async with binding_for(execution_query).lock:
result.attachments = exported_message(execution_query, file_ids, consume=True)
if result is not None:
yield result
run_snapshot = await self.journal.get_run(run_id)
if run_snapshot and run_snapshot.get('cancel_requested_at') is not None:
terminal_status = 'cancelled'
terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested'
break
await self.journal.finalize_run(
run_id=run_id,
status=terminal_status or 'completed',
@@ -403,11 +408,15 @@ class AgentRunOrchestrator:
execution_outcome = {'completed': 'success', 'failed': 'failed', 'cancelled': 'cancelled'}.get(
terminal_status or '', 'unknown'
)
execution_error = '' if execution_outcome == 'success' else (terminal_reason or execution_outcome)
except asyncio.CancelledError:
execution_outcome = 'cancelled'
execution_error = 'cancelled'
raise
except Exception as exc:
execution_outcome = 'timeout' if self._is_deadline_exhausted(context) else 'failed'
# A chain that broke must always carry a reason, even for a bare exception.
execution_error = str(exc) or type(exc).__name__
failed_usage = terminal_usage
await self.journal.finalize_run(
run_id=run_id,
@@ -426,6 +435,8 @@ class AgentRunOrchestrator:
runner=descriptor.id,
outcome=execution_outcome,
synthetic=event.source == 'webui',
error=execution_error,
node=node,
)
trace_mod.reset_run(run_token)
if trace_binding is not None and trace_mod.unbind_root(trace_binding):
src/langbot/pkg/agent/runner/platform_tools.py
+62 -40
View File
@@ -684,49 +684,71 @@ async def execute_platform_tool(
normalized = _normalize_platform_params(definition, parameters)
if definition.scope == 'event':
normalized = _event_params(definition, context, normalized)
# This flag is frozen by the Host from the synthetic debug envelope, not tool arguments.
if delivery.get('surface') == 'webui' and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True:
result = _execute_mock_platform_tool(definition, context, normalized)
if message_chain is not None:
result['parameters']['message'] = message_chain.model_dump(mode='json')
from ...telemetry.execution import record
# Plugin/RPC actions run outside the ingress context, so bind the owning
# execution id here: nested adapter observations then join its trace.
from ...telemetry.execution import record, reset_execution_id, set_execution_id
record(
ap,
execution_context,
family='platform_api',
operation=definition.api,
mode=authorization.get('processor_type', 'none'),
synthetic=True,
outcome='success',
)
return result
bot_id = authorization.get('bot_id')
if not bot_id:
raise ValueError('This run is not associated with a platform bot')
bot = await ap.platform_mgr.get_bot_by_uuid(execution_context, bot_id)
if bot is None:
raise ValueError(f'Bot {bot_id} is not running')
if definition.api not in set(bot.adapter.get_supported_apis() or []):
raise ValueError(f'Platform API {definition.api} is no longer supported by bot {bot_id}')
api_func = getattr(bot.adapter, definition.api, None)
if not callable(api_func):
raise ValueError(f'Platform API {definition.api} is declared but not implemented')
if definition.api == 'send_message':
normalized = {
'target_type': _require_string(normalized, 'target_type'),
'target_id': _require_string(normalized, 'target_id'),
'message': message_chain
if message_chain is not None
else platform_message.MessageChain([platform_message.Plain(text=_require_string(normalized, 'text'))]),
}
from ...telemetry.platform import processing_mode
token = processing_mode.set(authorization.get('processor_type', 'none'))
execution_id = str(session.get('run_id') or '').strip()
execution_token = set_execution_id(execution_id)
try:
return await api_func(**normalized)
# This flag is frozen by the Host from the synthetic debug envelope, not tool arguments.
mock = (
delivery.get('surface') == 'webui'
and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True
)
if mock:
outcome = 'unknown'
error_detail = ''
try:
result = _execute_mock_platform_tool(definition, context, normalized)
if message_chain is not None:
result['parameters']['message'] = message_chain.model_dump(mode='json')
outcome = 'success'
except Exception as exc:
outcome = 'failed'
error_detail = str(exc)
raise
finally:
record(
ap,
execution_context,
family='platform_api',
operation=definition.api,
mode=authorization.get('processor_type', 'none'),
synthetic=True,
outcome=outcome,
error=error_detail,
execution_id=execution_id or None,
)
return result
bot_id = authorization.get('bot_id')
if not bot_id:
raise ValueError('This run is not associated with a platform bot')
bot = await ap.platform_mgr.get_bot_by_uuid(execution_context, bot_id)
if bot is None:
raise ValueError(f'Bot {bot_id} is not running')
if definition.api not in set(bot.adapter.get_supported_apis() or []):
raise ValueError(f'Platform API {definition.api} is no longer supported by bot {bot_id}')
api_func = getattr(bot.adapter, definition.api, None)
if not callable(api_func):
raise ValueError(f'Platform API {definition.api} is declared but not implemented')
if definition.api == 'send_message':
normalized = {
'target_type': _require_string(normalized, 'target_type'),
'target_id': _require_string(normalized, 'target_id'),
'message': message_chain
if message_chain is not None
else platform_message.MessageChain([platform_message.Plain(text=_require_string(normalized, 'text'))]),
}
from ...telemetry.platform import processing_mode
token = processing_mode.set(authorization.get('processor_type', 'none'))
try:
return await api_func(**normalized)
finally:
processing_mode.reset(token)
finally:
processing_mode.reset(token)
reset_execution_id(execution_token)
def _execute_mock_platform_tool(
src/langbot/pkg/pipeline/pipelinemgr.py
+94 -2
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import dataclasses
import typing
import traceback
import asyncio
import sqlalchemy
@@ -257,6 +258,9 @@ class RuntimePipeline:
self.ap.logger.error(result.error_notice)
# Mark query as having error
query.variables['_monitoring_has_error'] = True
# The lane reports failures as a value instead of raising, so record the
# reason here: without it the uploaded trace would not explain the break.
self._record_lane_failure(query, str(result.error_notice))
# Record error to monitoring system
try:
await self._assert_execution_active(query)
@@ -368,14 +372,59 @@ class RuntimePipeline:
async def process_query(self, query: pipeline_query.Query):
from ..telemetry.execution import ingress
from ..telemetry.execution import record as record_execution
from ..telemetry.platform import processing_mode
from ..telemetry.trace import stage_scope
token = processing_mode.set('pipeline')
# The Workspace-scoped opaque query uuid is the execution identity Space
# shows for this lane; it is a stable UUID for every pooled query.
execution_id = str(getattr(query, 'query_uuid', '') or '').strip() or str(query.query_id)
try:
# Callers without a platform event (Webchat, HTTP API) still get one
# trace for the whole Pipeline lane; nested calls reuse the trace.
with ingress(self.ap, 'pipeline_done'):
return await self._process_query(query)
with ingress(self.ap, 'pipeline_done', getattr(self, 'execution_context', None), execution_id=execution_id):
# The lane is its own workflow step: acknowledgements and replies it
# sends directly, and the runner it drives, hang under this node.
# When the lane runs under a platform route the node nests there.
with stage_scope() as node:
lane_outcome = 'success'
lane_error = ''
try:
return await self._process_query(query)
except asyncio.CancelledError:
lane_outcome = 'cancelled'
lane_error = 'cancelled'
raise
except BaseException as exc:
lane_outcome = 'failed'
lane_error = str(exc) or type(exc).__name__
raise
finally:
try:
variables = getattr(query, 'variables', None) or {}
lane_has_error = bool(variables.get('_monitoring_has_error'))
except Exception:
lane_has_error = False
if lane_outcome == 'success' and lane_has_error:
# The lane reported the failure as a value, not an exception.
lane_outcome = 'failed'
config = getattr(query, 'pipeline_config', None)
try:
runner_id = (RunnerConfigResolver.resolve_runner_id(config) or '') if config else ''
except Exception:
runner_id = ''
record_execution(
self.ap,
getattr(self, 'execution_context', None),
family='pipeline',
operation='run',
mode='pipeline',
runner=runner_id,
outcome=lane_outcome,
error=lane_error,
node=node,
)
finally:
processing_mode.reset(token)
@@ -490,6 +539,9 @@ class RuntimePipeline:
inst_name = query.current_stage_name if query.current_stage_name else 'unknown'
self.ap.logger.error(f'Error processing query {query.query_id} stage={inst_name} : {e}')
self.ap.logger.error(f'Traceback: {traceback.format_exc()}')
# The lane itself broke: land the reason on the execution trace so the
# chain is uploaded even though the error never became a StageProcessResult.
self._record_lane_failure(query, str(e) or type(e).__name__)
# Record query error
try:
@@ -513,6 +565,46 @@ class RuntimePipeline:
self.ap.logger.debug(f'Query {query.query_id} processed')
await self.ap.query_pool.remove_query(query)
def _record_lane_failure(self, query: pipeline_query.Query, reason: str) -> None:
"""Land one failed ``runner/execute`` stage on the owning execution trace."""
try:
from ..telemetry.execution import record as record_execution
from ..telemetry.trace import current as current_trace
from ..telemetry.trace import stage_scope
detail = reason or 'pipeline_lane_failed'
state = current_trace()
# The runner orchestrator already lands its own failed ``runner/execute``
# node when the break happens inside a runner; in that case only carry the
# reason over instead of repeating the node on the trace.
already_recorded = bool(
state is not None
and any(
stage.get('family') == 'runner' and stage.get('outcome') not in ('success', 'skipped')
for stage in state.stages
)
)
if not already_recorded:
# The lane failure is its own workflow step, hanging under whatever
# step routed into it (or a root when nothing did).
with stage_scope() as node:
record_execution(
self.ap,
getattr(self, 'execution_context', None),
family='runner',
operation='execute',
mode='pipeline',
runner=(RunnerConfigResolver.resolve_runner_id(query.pipeline_config) or ''),
outcome='failed',
error=detail,
node=node,
)
state = current_trace()
if state is not None:
state.mark_failure(detail)
except Exception:
pass
class PipelineManager:
"""流水线管理器"""
src/langbot/pkg/pipeline/process/handlers/chat.py
+34 -62
View File
@@ -3,8 +3,6 @@ from __future__ import annotations
import uuid
import typing
import traceback
import time
from datetime import datetime
from .. import handler
@@ -14,9 +12,7 @@ from ... import plugin_diagnostics
import langbot_plugin.api.entities.events as events
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....agent.runner import config_schema
from ....utils import constants, runner as runner_utils
from ....telemetry import features as telemetry_features
from ....telemetry.identity import workspace_identity
from ....utils import runner as runner_utils
import langbot_plugin.api.entities.builtin.provider.session as provider_session
import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
import langbot_plugin.api.entities.builtin.provider.message as provider_message
@@ -115,9 +111,6 @@ class ChatMessageHandler(handler.MessageHandler):
text_length = 0
runner = None
try:
# Mark start time for telemetry
start_ts = time.time()
try_claim_steering = getattr(
self.ap.agent_run_orchestrator,
'try_claim_steering_from_query',
@@ -258,69 +251,48 @@ class ChatMessageHandler(handler.MessageHandler):
debug_notice=traceback.format_exc(),
)
finally:
# Telemetry reporting
# Telemetry: the per-execution record is built when the owning
# trace closes, so only attach the execution-scoped fields this
# lane knows here.
try:
end_ts = time.time()
duration_ms = None
if 'start_ts' in locals():
duration_ms = int((end_ts - start_ts) * 1000)
from ....telemetry.trace import current as current_trace
state = current_trace()
if state is not None:
adapter_name = query.adapter.__class__.__name__ if hasattr(query, 'adapter') else ''
runner_name = self.ap.agent_run_orchestrator.resolve_runner_id_for_telemetry(query)
state.adapter = state.adapter or adapter_name
state.runner = state.runner or (runner_name or '')
state.runner_category = state.runner_category or runner_utils.get_runner_category_from_runner(
runner_name, None, query.pipeline_config
)
model_name = ''
try:
if getattr(query, 'use_llm_model_uuid', None):
m = await self.ap.model_mgr.get_model_by_uuid(
get_query_execution_context(query),
query.use_llm_model_uuid,
)
if m and getattr(m, 'model_entity', None):
model_name = getattr(m.model_entity, 'name', '') or ''
except Exception:
model_name = ''
state.model_name = state.model_name or model_name
if state.pipeline_plugins is None:
state.pipeline_plugins = query.variables.get('_pipeline_bound_plugins', None)
except Exception as ex:
self.ap.logger.warning(f'Failed to attach execution telemetry fields: {ex}')
# Trigger survey events on successful non-WebSocket responses
try:
adapter_name = query.adapter.__class__.__name__ if hasattr(query, 'adapter') else None
# Use orchestrator to resolve runner ID for telemetry
runner_name = self.ap.agent_run_orchestrator.resolve_runner_id_for_telemetry(query)
# Model name if available
model_name = None
try:
if getattr(query, 'use_llm_model_uuid', None):
m = await self.ap.model_mgr.get_model_by_uuid(
get_query_execution_context(query),
query.use_llm_model_uuid,
)
if m and getattr(m, 'model_entity', None):
model_name = getattr(m.model_entity, 'name', None)
except Exception:
model_name = None
pipeline_plugins = query.variables.get('_pipeline_bound_plugins', None)
runner_category = runner_utils.get_runner_category_from_runner(
runner_name, None, query.pipeline_config
)
# Feature usage collected during query processing (tool calls,
# knowledge base usage, sandbox executions, activated skills, ...)
features = telemetry_features.collect_features(query)
payload = {
'event_type': 'query',
'query_id': query.query_id,
'adapter': adapter_name,
'runner': runner_name,
'runner_category': runner_category,
'duration_ms': duration_ms,
'model_name': model_name,
'version': constants.semantic_version,
**workspace_identity(get_query_execution_context(query)),
'runtime_instance_id': constants.instance_id,
'edition': constants.edition,
'pipeline_plugins': pipeline_plugins,
'features': features,
'error': locals().get('error_info', None),
'timestamp': datetime.utcnow().isoformat(),
}
await self.ap.telemetry.start_send_task(payload)
# Trigger survey events on successful non-WebSocket responses
if not locals().get('error_info') and adapter_name and 'WebSocket' not in adapter_name:
if self.ap.survey:
await self.ap.survey.trigger_event('first_bot_response_success')
# Counts toward the bot_response_success_100 milestone event
await self.ap.survey.record_bot_response_success()
except Exception as ex:
self.ap.logger.warning(f'Failed to send telemetry: {ex}')
self.ap.logger.warning(f'Failed to trigger survey event: {ex}')
async def _ensure_conversation_for_history(
self,
src/langbot/pkg/platform/botmgr.py
+111 -46
View File
@@ -401,15 +401,19 @@ class RuntimeBot:
from ..telemetry.execution import record
with trace_mod.scope(route_ref=self._route_ref(binding, target_type, target_uuid)):
record(
getattr(self, 'ap', None),
getattr(self, 'execution_context', None),
family='event_route',
operation=event_type,
adapter=type(getattr(self, 'adapter', None)).__name__,
mode=target_type if target_type in {'pipeline', 'agent', 'event_processor'} else 'none',
outcome={'delivered': 'success', 'failed': 'failed'}.get(status, 'skipped'),
)
# The routing decision is its own workflow step; the Pipeline or
# processor it delivers to hangs under this node.
with trace_mod.stage_scope() as node:
record(
getattr(self, 'ap', None),
getattr(self, 'execution_context', None),
family='event_route',
operation=event_type,
adapter=type(getattr(self, 'adapter', None)).__name__,
mode=target_type if target_type in {'pipeline', 'agent', 'event_processor'} else 'none',
outcome={'delivered': 'success', 'failed': 'failed'}.get(status, 'skipped'),
node=node,
)
return metadata
def get_pipeline_target_for_event_type(self, event_type: str = 'message.received') -> str | None:
@@ -736,12 +740,19 @@ class RuntimeBot:
self,
event: platform_events.EBAEvent,
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
execution_id: str | None = None,
) -> AgentEventEnvelope:
event_type = getattr(event, 'type', None) or event.__class__.__name__
event_time = getattr(event, 'timestamp', None) or time.time()
event_id = (
getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}'
)
if execution_id:
# Reuse the ingress execution identity so the run view, the journaled
# event and the uploaded chain all share one id.
envelope_event_id = str(execution_id)
prefix = f'platform:{self.bot_entity.uuid}:'
event_id = envelope_event_id[len(prefix) :] if envelope_event_id.startswith(prefix) else envelope_event_id
else:
event_id = self._platform_event_raw_id(event)
envelope_event_id = f'platform:{self.bot_entity.uuid}:{event_id}'
target_type, target_id, target_metadata = self._infer_reply_target(event)
supported_apis = self._get_adapter_supported_apis(adapter)
@@ -773,7 +784,7 @@ class RuntimeBot:
delivery_data['interactions'] = interaction_capabilities
return AgentEventEnvelope(
event_id=f'platform:{self.bot_entity.uuid}:{event_id}',
event_id=envelope_event_id,
event_type=event_type,
event_time=int(event_time) if isinstance(event_time, (int, float)) else None,
source='platform',
@@ -863,6 +874,21 @@ class RuntimeBot:
processor_id=agent.get('uuid'),
)
def _platform_event_execution_id(self, event: platform_events.EBAEvent) -> str:
"""Space-visible execution identity for one inbound platform event.
Matches the ``AgentEventEnvelope.event_id`` the same event produces, so
the stored chain and the Space run view share one identity.
"""
return f'platform:{self.bot_entity.uuid}:{self._platform_event_raw_id(event)}'
@staticmethod
def _platform_event_raw_id(event: platform_events.EBAEvent) -> str:
event_type = getattr(event, 'type', None) or event.__class__.__name__
return str(
getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}'
)
async def _handle_platform_event(
self,
event: platform_events.EBAEvent,
@@ -870,51 +896,85 @@ class RuntimeBot:
) -> None:
# One inbound event owns one execution trace; every stage recorded while
# it is handled (routing, runner, platform API calls) joins that trace.
from ..telemetry import trace as trace_mod
from ..telemetry.execution import ingress
event.bot_uuid = self.bot_entity.uuid
with ingress(getattr(self, 'ap', None), 'event_done'):
await self._handle_platform_event_body(event, adapter)
execution_id = self._platform_event_execution_id(event)
with ingress(getattr(self, 'ap', None), 'event_done', getattr(self, 'execution_context', None), execution_id):
# The inbound event is the root workflow step: everything recorded
# while handling it hangs under this node.
with trace_mod.stage_scope() as node:
await self._handle_platform_event_body(event, adapter, execution_id, node)
async def _handle_platform_event_body(
self,
event: platform_events.EBAEvent,
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
) -> None:
@staticmethod
def _summarize_event_delivery(results: list[typing.Any]) -> tuple[str, str]:
outcome = 'success'
error_detail = ''
for result in results:
if isinstance(result, BaseException):
outcome = 'failed'
error_detail = error_detail or f'{type(result).__name__}: {result}'
return outcome, error_detail
def _record_platform_event(self, event, adapter, outcome: str, error_detail: str, node: str = '') -> None:
from ..telemetry.execution import record
record(
getattr(self, 'ap', None),
getattr(self, 'execution_context', None),
family='platform_event',
operation=event.type,
operation=str(getattr(event, 'type', None) or event.__class__.__name__),
adapter=adapter.__class__.__name__,
outcome='success',
outcome=outcome,
error=error_detail,
node=node,
)
await self._record_adapter_event(event, adapter)
primary = (
self._handle_interaction_submission(event, adapter)
if isinstance(event, platform_events.PlatformSpecificEvent) and event.action == 'interaction.submitted'
else self._dispatch_eba_event_to_processor(event, adapter)
)
subscriptions = self._get_event_bindings_from_value(getattr(self.bot_entity, 'plugin_processors', []))
seen = set()
tasks = [primary]
for subscription in subscriptions:
processor_uuid = subscription.get('processor_uuid')
if not processor_uuid or processor_uuid in seen or not subscription.get('enabled', True):
continue
seen.add(processor_uuid)
tasks.append(self._dispatch_plugin_subscription(event, adapter, processor_uuid))
# Start all deliveries together. A slow or failed subscriber cannot block
# another subscriber or the primary route from receiving the event.
results = await asyncio.gather(*tasks, return_exceptions=True)
async def _handle_platform_event_body(
self,
event: platform_events.EBAEvent,
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
execution_id: str | None = None,
node: str = '',
) -> None:
results: list[typing.Any] = []
try:
await self._record_adapter_event(event, adapter)
primary = (
self._handle_interaction_submission(event, adapter, execution_id)
if isinstance(event, platform_events.PlatformSpecificEvent) and event.action == 'interaction.submitted'
else self._dispatch_eba_event_to_processor(event, adapter, execution_id=execution_id)
)
subscriptions = self._get_event_bindings_from_value(getattr(self.bot_entity, 'plugin_processors', []))
seen = set()
tasks = [primary]
for subscription in subscriptions:
processor_uuid = subscription.get('processor_uuid')
if not processor_uuid or processor_uuid in seen or not subscription.get('enabled', True):
continue
seen.add(processor_uuid)
tasks.append(self._dispatch_plugin_subscription(event, adapter, processor_uuid, execution_id))
# Start all deliveries together. A slow or failed subscriber cannot block
# another subscriber or the primary route from receiving the event.
results = await asyncio.gather(*tasks, return_exceptions=True)
except BaseException as exc:
self._record_platform_event(event, adapter, 'failed', f'{type(exc).__name__}: {exc}', node)
raise
# Recorded after handling so the inbound stage carries the real outcome
# instead of a hardcoded success.
outcome, error_detail = self._summarize_event_delivery(results)
for result in results:
if isinstance(result, BaseException):
await self.logger.error(f'Event delivery failed: {result}')
try:
await self.logger.error(f'Event delivery failed: {result}')
except Exception:
pass
self._record_platform_event(event, adapter, outcome, error_detail, node)
async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid):
async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid, execution_id=None):
event_type = event.type
event_binding = {
'id': f'plugin_processor:{processor_uuid}',
@@ -934,7 +994,9 @@ class RuntimeBot:
if not patterns or not self._agent_supports_event_type(patterns, event_type):
return
agent = {**agent, 'supported_event_patterns': patterns}
return await self._dispatch_eba_event_to_processor(event, adapter, event_binding, agent)
return await self._dispatch_eba_event_to_processor(
event, adapter, event_binding, agent, execution_id=execution_id
)
except Exception as exc:
return await self._record_event_route_trace(
event_type=event_type,
@@ -954,6 +1016,7 @@ class RuntimeBot:
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
event_binding: dict | None = None,
agent: dict | None = None,
execution_id: str | None = None,
) -> dict[str, typing.Any]:
event_type = getattr(event, 'type', None) or event.__class__.__name__
@@ -1101,7 +1164,7 @@ class RuntimeBot:
text=f'Event {event_type} target agent has no runner: {target_uuid}',
)
envelope = self._eba_event_to_agent_envelope(event, adapter)
envelope = self._eba_event_to_agent_envelope(event, adapter, execution_id)
if target_type == 'event_processor':
envelope.data = event.model_dump(mode='json', exclude={'source_platform_object', 'legacy_event'})
from ..telemetry import trace as trace_mod
@@ -1318,6 +1381,7 @@ class RuntimeBot:
self,
event: platform_events.PlatformSpecificEvent,
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
execution_id: str | None = None,
) -> None:
"""Consume a platform interaction callback and resume its original processor."""
data = event.data if isinstance(event.data, dict) else {}
@@ -1346,7 +1410,7 @@ class RuntimeBot:
await self.ap.agent_run_orchestrator.interaction_manager.acknowledge_submission(record, adapter)
if record['processor_type'] == 'agent':
await self._resume_agent_interaction(record, event, adapter, actor_id)
await self._resume_agent_interaction(record, event, adapter, actor_id, execution_id)
return
if record['processor_type'] != 'pipeline':
raise ValueError(f'Unsupported interaction processor type: {record["processor_type"]}')
@@ -1428,6 +1492,7 @@ class RuntimeBot:
event: platform_events.PlatformSpecificEvent,
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
actor_id: str | None,
execution_id: str | None = None,
) -> None:
"""Resume the exact independent Agent that produced an interaction."""
agent = await self.ap.agent_service.get_agent(
@@ -1480,7 +1545,7 @@ class RuntimeBot:
delivery_data['interactions'] = interaction_capabilities
envelope = AgentEventEnvelope(
event_id=f'interaction:{record["id"]}:{uuid.uuid4()}',
event_id=execution_id or f'interaction:{record["id"]}:{uuid.uuid4()}',
event_type='interaction.submitted',
event_time=int(event.timestamp) if event.timestamp else int(time.time()),
source='platform',
src/langbot/pkg/telemetry/execution.py
+315 -137
View File
@@ -1,34 +1,33 @@
"""Bounded, content-free execution counters for the existing telemetry sender.
"""Content-free execution records for the existing telemetry sender.
This module reports observations only. Coverage catalogs and acceptance rules
belong to Space. Aggregation keys include both immutable execution identities.
belong to Space.
Two shapes share one payload type:
One complete execution is one record: the execution identity (Pipeline query,
inbound platform event or Agent run), its bounded ordered stages and its
terminal outcome. Nothing is aggregated or counted across executions — every
record carries the id of the execution it belongs to, so Space can fetch the
whole chain by primary identity. Records are buffered whole and flushed as
``{"records": [...]}`` on the configured cadence and on shutdown; an in-flight
trace is never flushed half-written.
* window counters, aggregated by ``(family, operation, mode, adapter, runner,
outcome, synthetic)`` — unchanged coverage reporting;
* per-event traces, one bounded payload per traced event, keyed by
``query_id = trace_id`` so Space can fetch a whole chain by primary identity.
Traces are additive: they never alter the counters, and both are bounded in
memory, batch size and send concurrency.
Memory is bounded in stages per trace, buffered traces, buffered records, the
cross-task execution registry and flush batch size.
"""
from __future__ import annotations
import asyncio
import contextlib
import contextvars
import json
import time
from datetime import datetime, timezone
from uuid import uuid4
from . import trace as trace_mod
from .identity import workspace_identity
from .trace import TraceState
MAX_KEYS = 512
MAX_BATCH = 32
FLUSH_SECONDS = 60
MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'})
OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'})
@@ -39,15 +38,70 @@ TRACE_MODES = frozenset({'off', 'failures', 'sampled', 'all'})
DEFAULT_TRACE_MODE = 'sampled'
DEFAULT_TRACE_SAMPLE = 20
# Record bounds: buffered records, one flush batch, flush cadence, registry.
MAX_BUFFERED_RECORDS = 512
MAX_RECORDS_PER_FLUSH = 128
MAX_FLUSH_BYTES = 200 * 1024
DEFAULT_FLUSH_SECONDS = 180
MAX_REGISTRY_KEYS = 256
# Set by cross-task call sites (plugin/RPC actions) that run without the
# ingress context but know which execution they belong to.
_execution_id: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_execution_id', default='')
def set_execution_id(execution_id: str) -> contextvars.Token:
return _execution_id.set(str(execution_id or '').strip())
def reset_execution_id(token: contextvars.Token | None) -> None:
if token is None:
return
try:
_execution_id.reset(token)
except (ValueError, RuntimeError):
# A token from another context must never break execution.
pass
def current_execution_id() -> str:
return _execution_id.get()
def _bounded_error(value, limit: int = 400) -> str:
"""Bound and de-fang one failure detail before it leaves the process.
Error text is operator-facing: keep it single-line, printable and short so a
noisy failure cannot smuggle control characters or bloat the payload.
"""
if value is None:
return ''
text = ''.join(ch if ch.isprintable() else ' ' for ch in str(value))
text = ' '.join(text.split())
return text[:limit]
def _duration_ms(started_at: str, ended_at: str) -> int:
"""Wall-clock duration of one execution; 0 when the interval is unusable."""
try:
delta = datetime.fromisoformat(ended_at) - datetime.fromisoformat(started_at)
return max(int(delta.total_seconds() * 1000), 0)
except Exception:
return 0
class ExecutionCounters:
def __init__(self, manager):
self.manager = manager
self.pending: dict[tuple, dict] = {}
self.records: list[dict] = []
self.task: asyncio.Task | None = None
self.dropped = 0
self.traces: dict[str, TraceState] = {}
self.trace_deadlines: dict[str, float] = {}
# Execution identity -> in-flight trace, for stages recorded by tasks
# that no longer share the ingress context (plugin/RPC platform calls).
self.registry: dict[str, TraceState] = {}
self.registry_deadlines: dict[str, float] = {}
self.dropped_traces = 0
# ------------------------------------------------------------------ config
@@ -63,6 +117,13 @@ class ExecutionCounters:
sample = DEFAULT_TRACE_SAMPLE
return sample if 1 <= sample <= 100000 else DEFAULT_TRACE_SAMPLE
def flush_seconds(self) -> int:
try:
seconds = int(self.manager.telemetry_config.get('telemetry_flush_seconds', DEFAULT_FLUSH_SECONDS))
except (TypeError, ValueError):
seconds = DEFAULT_FLUSH_SECONDS
return seconds if 1 <= seconds <= 86400 else DEFAULT_FLUSH_SECONDS
# --------------------------------------------------------------- recording
def record(
@@ -76,67 +137,91 @@ class ExecutionCounters:
runner: str = '',
outcome: str = 'unknown',
synthetic: bool = False,
error: str = '',
execution_id: str | None = None,
node: str = '',
):
try:
cfg = self.manager.telemetry_config
if not cfg or cfg.get('disable_telemetry', False) or not cfg.get('url'):
return
if family not in {'platform_event', 'event_route', 'runner', 'platform_api'}:
if family not in {'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}:
return
if mode not in MODES or outcome not in OUTCOMES:
return
# Only code-defined identifiers are accepted; never pass user values.
if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner)):
if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner, node)):
return
identity = workspace_identity(context)
state = trace_mod.current()
if state is not None:
self._record_trace_stage(
state,
identity=identity,
family=family,
operation=operation,
mode=mode,
adapter=adapter,
runner=runner,
outcome=outcome,
synthetic=synthetic,
)
key = (
identity['instance_id'],
identity['workspace_uuid'],
family,
operation,
mode,
adapter,
runner,
outcome,
bool(synthetic),
if state is None:
# Cross-task work has no inherited trace; resolve the owning
# execution through the registry instead.
lookup = str(execution_id or '').strip() or current_execution_id()
if lookup:
state = self.registry.get(lookup)
if state is None:
# Every observation belongs to exactly one execution chain.
return
try:
identity = workspace_identity(context)
except Exception:
# A cross-task caller may only carry the Workspace id; reuse the
# identity already bound to the owning trace.
identity = state.identity or {}
if not identity.get('instance_id') or not identity.get('workspace_uuid'):
return
self._record_trace_stage(
state,
identity=identity,
family=family,
operation=operation,
mode=mode,
adapter=adapter,
runner=runner,
outcome=outcome,
synthetic=synthetic,
error=_bounded_error(error),
node=node,
)
row = self.pending.get(key)
if row is None:
if len(self.pending) >= MAX_KEYS:
self.dropped += 1
return
row = {'count': 0, 'first_seen': datetime.now(timezone.utc).isoformat()}
self.pending[key] = row
row['count'] = min(row['count'] + 1, 2147483647)
row['last_seen'] = datetime.now(timezone.utc).isoformat()
self._ensure_loop()
except Exception:
# Observability must never change execution behavior.
return
def _record_trace_stage(
self, state: TraceState, *, identity, family, operation, mode, adapter, runner, outcome, synthetic
self,
state: TraceState,
*,
identity,
family,
operation,
mode,
adapter,
runner,
outcome,
synthetic,
error: str = '',
node: str = '',
) -> None:
if state.abandoned:
return
if trace_mod.current() is state:
# Recorded by the task that owns the step: its step stack is exact.
# A step's own record sits under its enclosing step, never itself.
parent = trace_mod.current_parent()
if node and parent == node:
parent = trace_mod.enclosing_parent()
else:
# A cross-task observation (plugin/RPC) cannot see the step
# contextvar; attach it to the innermost step the owning task has
# open on the trace.
parent = state.open_nodes[-1] if state.open_nodes else ''
if state.trace_id not in self.traces:
# Never buffer traces this configuration would discard anyway.
if self.trace_mode() == 'off' or len(self.traces) >= MAX_TRACES:
self.dropped += 1
state.abandoned = True
self._unregister_state(state)
return
self.traces[state.trace_id] = state
self.trace_deadlines[state.trace_id] = time.monotonic() + TRACE_TTL_SECONDS
@@ -149,50 +234,79 @@ class ExecutionCounters:
runner=runner,
outcome=outcome,
synthetic=bool(synthetic),
error=error,
node=node,
parent=parent,
)
def _ensure_loop(self) -> None:
if self.task is None or self.task.done():
self.task = asyncio.create_task(self._loop())
# ------------------------------------------------------------------ traces
# --------------------------------------------------------------- registry
def close_trace(self, state: TraceState, reason: str = 'event_done') -> None:
"""Emit one trace payload when it matches the configured sample."""
def register_trace(self, state: TraceState, execution_id) -> None:
"""Alias one more execution identity onto an in-flight trace."""
try:
if self.traces.pop(state.trace_id, None) is None:
exec_id = str(execution_id or '').strip()
if not exec_id or state.abandoned:
return
self.trace_deadlines.pop(state.trace_id, None)
if not state.stages or not self._trace_emitted(state):
return
payload = self._build_trace_payload(state, reason)
if payload is not None:
self._dispatch_payload(payload)
if exec_id not in self.registry and len(self.registry) >= MAX_REGISTRY_KEYS:
oldest = next(iter(self.registry))
self.registry.pop(oldest, None)
self.registry_deadlines.pop(oldest, None)
self.registry[exec_id] = state
self.registry_deadlines[exec_id] = time.monotonic() + TRACE_TTL_SECONDS
except Exception:
return
def _dispatch_payload(self, payload: dict) -> None:
"""Hand one built payload to the telemetry manager from this sync context.
def _unregister_state(self, state: TraceState) -> None:
for exec_id in [key for key, value in self.registry.items() if value is state]:
self.registry.pop(exec_id, None)
self.registry_deadlines.pop(exec_id, None)
TelemetryManager.start_send_task is a coroutine, so calling it without
scheduling dropped every trace closed here. Stand-ins used by tests and
manual tools schedule synchronously, hence the coroutine check.
"""
result = self.manager.start_send_task(payload)
if not asyncio.iscoroutine(result):
return
def _sweep_registry(self, now: float) -> None:
"""Drop aliases whose owning execution never closed (leaked bindings)."""
for exec_id in [key for key, deadline in self.registry_deadlines.items() if deadline <= now]:
self.registry.pop(exec_id, None)
self.registry_deadlines.pop(exec_id, None)
# ------------------------------------------------------------------ traces
def close_trace(self, state: TraceState, reason: str = 'event_done', error: str = '') -> None:
"""Buffer one complete execution record when it matches the sample."""
try:
asyncio.get_running_loop().create_task(result)
except RuntimeError:
result.close()
logger = getattr(getattr(self.manager, 'ap', None), 'logger', None)
if logger is not None:
logger.debug('Execution trace payload dropped: no running event loop')
if state.closed:
return
state.closed = True
registered = self.traces.pop(state.trace_id, None) is not None
self.trace_deadlines.pop(state.trace_id, None)
self._unregister_state(state)
if not registered and not state.failure_reason:
return
if (not state.stages and not state.failure_reason) or not self._trace_emitted(state):
return
record = self._build_record(state, reason, error)
if record is not None:
self._enqueue_record(record)
except Exception:
return
def _enqueue_record(self, record: dict) -> None:
if len(self.records) >= MAX_BUFFERED_RECORDS:
self.records.pop(0)
self.dropped += 1
self.records.append(record)
self._ensure_loop()
def _trace_emitted(self, state: TraceState) -> bool:
mode = self.trace_mode()
if mode == 'off':
return False
# A chain that failed must always be uploaded, even when no stage was
# recorded before it broke: the reason is the whole point of the trace.
if state.failure_reason:
return True
if mode == 'all':
return True
# A failed, cancelled or timed-out stage is always worth keeping.
@@ -208,31 +322,69 @@ class ExecutionCounters:
except ValueError:
return False
def _build_trace_payload(self, state: TraceState, reason: str) -> dict | None:
def _build_record(self, state: TraceState, reason: str, error: str = '') -> dict | None:
from ..utils import constants
identity = state.identity
if not identity.get('instance_id') or not identity.get('workspace_uuid'):
return None
observations = [{**stage, 'trace_id': state.trace_id, 'count': 1} for stage in state.stages]
ended_at = datetime.now(timezone.utc).isoformat()
stages = state.stages
observations = []
for stage in stages:
observation = {
'seq': stage['seq'],
'family': stage['family'],
'operation': stage['operation'],
'mode': stage['mode'],
'adapter': stage['adapter'],
'runner': stage['runner'],
'outcome': stage['outcome'],
'synthetic': stage['synthetic'],
'error': stage['error'],
'route_ref': stage['route_ref'],
'run_id': stage['run_id'],
'first_seen': stage['first_seen'],
'last_seen': stage['last_seen'],
'trace_id': state.trace_id,
}
# Workflow position is optional: records outside any step omit both
# keys entirely, so their payloads stay byte-identical.
if stage.get('node'):
observation['node'] = stage['node']
if stage.get('parent'):
observation['parent'] = stage['parent']
observations.append(observation)
return {
'event_type': 'feature_execution',
# One payload per trace: lookup uses the indexed query_id column.
# The execution identity: one record == one chain == one Space row.
'query_id': state.trace_id,
'instance_id': identity['instance_id'],
'workspace_uuid': identity['workspace_uuid'],
'version': constants.semantic_version,
'edition': constants.edition,
'timestamp': datetime.now(timezone.utc).isoformat(),
'timestamp': ended_at,
'trusted': True,
# First failure detail in this chain, so Space can show why it broke.
'error': _bounded_error(error)
or state.failure_reason
or next((stage['error'] for stage in stages if stage['error']), ''),
# Legacy-compatible columns Space already maps; best effort.
'duration_ms': _duration_ms(state.started_at, ended_at),
'model_name': state.model_name or '',
'adapter': state.adapter or next((stage['adapter'] for stage in stages if stage['adapter']), ''),
'runner': state.runner or next((stage['runner'] for stage in stages if stage['runner']), ''),
'runner_category': state.runner_category or '',
'pipeline_plugins': state.pipeline_plugins,
'features': {
'schema': 1,
'observations': observations,
'trace': {
'closed_by': reason,
'started_at': state.started_at,
'ended_at': datetime.now(timezone.utc).isoformat(),
'route_ref': next((s['route_ref'] for s in state.stages if s['route_ref']), ''),
'run_id': next((s['run_id'] for s in state.stages if s['run_id']), ''),
'ended_at': ended_at,
'route_ref': next((stage['route_ref'] for stage in stages if stage['route_ref']), ''),
'run_id': next((stage['run_id'] for stage in stages if stage['run_id']), ''),
'outcome': state.outcome(),
'dropped_stages': state.dropped_stages,
},
@@ -248,56 +400,36 @@ class ExecutionCounters:
# ----------------------------------------------------------------- flushing
async def _loop(self):
interval = self.flush_seconds()
last_flush = time.monotonic()
while self.pending or self.traces:
# Traces expire on a short TTL; counters keep their minute cadence.
await asyncio.sleep(1 if self.traces else FLUSH_SECONDS)
while self.records or self.traces or self.registry:
# Traces expire on a short TTL; records keep their own cadence.
await asyncio.sleep(1 if (self.traces or self.registry) else interval)
now = time.monotonic()
self._sweep_traces(now)
if self.pending and now - last_flush >= FLUSH_SECONDS:
self._sweep_registry(now)
if self.records and now - last_flush >= interval:
last_flush = now
await self.flush()
self.task = None
async def flush(self):
from ..utils import constants
# Remove only one bounded batch; subsequent windows drain the remainder.
# Drain one tenant per minute, at most one request, round-robin by insertion.
if not self.pending:
if not self.records:
return
tenant = next(iter(self.pending))[:2]
keys = [key for key in self.pending if key[:2] == tenant][:MAX_BATCH]
groups: dict[tuple[str, str], list[dict]] = {}
for key in keys:
row = self.pending.pop(key)
instance, workspace, family, operation, mode, adapter, runner, outcome, synthetic = key
groups.setdefault((instance, workspace), []).append(
{
**row,
'family': family,
'operation': operation,
'mode': mode,
'adapter': adapter,
'runner': runner,
'outcome': outcome,
'synthetic': synthetic,
}
)
for (instance, workspace), observations in groups.items():
payload = {
'event_type': 'feature_execution',
'query_id': str(uuid4()),
'instance_id': instance,
'workspace_uuid': workspace,
'version': constants.semantic_version,
'edition': constants.edition,
'timestamp': datetime.now(timezone.utc).isoformat(),
'features': {'schema': 1, 'observations': observations},
}
if not await self.manager.send(payload):
await asyncio.sleep(1)
await self.manager.send(payload)
batch: list[dict] = []
size = 0
while self.records and len(batch) < MAX_RECORDS_PER_FLUSH:
encoded = len(json.dumps(self.records[0], ensure_ascii=False, default=str))
if batch and size + encoded > MAX_FLUSH_BYTES:
break
batch.append(self.records.pop(0))
size += encoded
if not batch:
return
payload = {'records': batch}
if not await self.manager.send(payload):
await asyncio.sleep(1)
await self.manager.send(payload)
async def shutdown(self):
if self.task is not None:
@@ -305,47 +437,93 @@ class ExecutionCounters:
await asyncio.gather(self.task, return_exceptions=True)
self.task = None
try:
await asyncio.wait_for(self.flush(), timeout=2)
deadline = time.monotonic() + 5
while self.records and time.monotonic() < deadline:
remaining = len(self.records)
await asyncio.wait_for(self.flush(), timeout=2)
if len(self.records) >= remaining:
break
except (Exception, asyncio.CancelledError):
pass
self.pending.clear()
# Traces of in-flight events cannot be completed during shutdown.
self.records.clear()
# Executions still in flight cannot be completed during shutdown.
self.dropped += len(self.traces)
self.dropped_traces += len(self.traces)
self.traces.clear()
self.trace_deadlines.clear()
self.registry.clear()
self.registry_deadlines.clear()
def _execution_counters(ap):
counters = getattr(getattr(ap, 'telemetry', None), 'execution', None)
return counters if isinstance(counters, ExecutionCounters) else None
def record(ap, context, **observation):
"""Best-effort bridge usable with optional telemetry and test doubles."""
try:
counters = getattr(getattr(ap, 'telemetry', None), 'execution', None)
if isinstance(counters, ExecutionCounters):
counters = _execution_counters(ap)
if counters is not None:
counters.record(context, **observation)
except Exception:
pass
def close_trace(ap, state, reason: str = 'event_done') -> None:
def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None:
"""Best-effort trace close usable with optional telemetry and test doubles."""
try:
counters = getattr(getattr(ap, 'telemetry', None), 'execution', None)
if isinstance(counters, ExecutionCounters):
counters.close_trace(state, reason)
counters = _execution_counters(ap)
if counters is not None:
counters.close_trace(state, reason, error)
except Exception:
pass
def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding:
"""Bind an execution trace and alias its identity for cross-task stages."""
binding = trace_mod.bind(execution_id)
if execution_id:
try:
counters = _execution_counters(ap)
if counters is not None:
counters.register_trace(binding.state, execution_id)
except Exception:
pass
return binding
@contextlib.contextmanager
def ingress(ap, reason: str = 'event_done'):
"""Trace one ingress boundary; only the owner of the trace closes it.
def ingress(ap, reason: str = 'event_done', context=None, execution_id: str | None = None):
"""Trace one execution boundary; only the owner of the trace closes it.
Nested boundaries (an event routed into a Pipeline, a Runner invoked from a
dispatch) reuse the in-flight trace instead of starting a second one.
dispatch) reuse the in-flight trace instead of starting a second one, and
only alias their own execution id onto it.
"""
binding = trace_mod.bind()
binding = trace_mod.bind(execution_id)
if binding.created and context is not None:
try:
binding.state.identity = workspace_identity(context)
except Exception:
pass
if execution_id and not binding.state.abandoned:
try:
counters = _execution_counters(ap)
if counters is not None:
counters.register_trace(binding.state, execution_id)
except Exception:
pass
failure = ''
try:
yield binding
except BaseException as exc:
# The boundary itself failed: keep the reason on the trace so the uploaded
# payload explains the failure instead of reporting a bare outcome.
if binding.created:
failure = f'{type(exc).__name__}: {exc}'
binding.state.mark_failure(failure)
raise
finally:
if trace_mod.unbind_root(binding):
close_trace(ap, binding.state, reason)
close_trace(ap, binding.state, reason, failure)
src/langbot/pkg/telemetry/platform.py
+24 -2
View File
@@ -11,6 +11,21 @@ from .execution import record
processing_mode: ContextVar[str] = ContextVar('telemetry_processing_mode', default='none')
def result_failure_detail(result) -> str:
"""Pull a bounded human-readable reason out of a failed platform API result."""
raw = getattr(result, 'raw', result)
if isinstance(raw, dict):
for key in ('error', 'message', 'msg', 'detail', 'reason'):
value = raw.get(key)
if isinstance(value, str) and value.strip():
return value
if isinstance(value, dict):
for inner in ('message', 'msg', 'detail'):
if isinstance(value.get(inner), str) and value[inner].strip():
return value[inner]
return ''
def result_outcome(result):
# Empty returns do not prove a remote operation succeeded.
if result is None:
@@ -69,18 +84,24 @@ def observe_adapter(ap, context, adapter):
if action in declared:
observed_operation = action
outcome = 'unknown'
error_detail = ''
try:
result = await method(*args, **kwargs)
outcome = result_outcome(result)
if outcome != 'success':
error_detail = result_failure_detail(result)
return result
except asyncio.CancelledError:
outcome = 'cancelled'
error_detail = 'cancelled'
raise
except TimeoutError:
except TimeoutError as exc:
outcome = 'timeout'
error_detail = str(exc) or 'timeout'
raise
except Exception:
except Exception as exc:
outcome = 'failed'
error_detail = str(exc)
raise
finally:
record(
@@ -91,6 +112,7 @@ def observe_adapter(ap, context, adapter):
adapter=adapter.__class__.__name__,
mode=processing_mode.get(),
outcome=outcome,
error=error_detail,
)
return wrapped
src/langbot/pkg/telemetry/trace.py
+136 -30
View File
@@ -1,9 +1,11 @@
"""Content-free per-event execution traces.
"""Content-free per-execution traces.
One trace identity is minted per inbound platform event and survives until the
ingress handler returns. Stage records appended under it describe how that one
event was routed and processed, using only code-defined identifiers. Nothing
here is aggregated: a trace is a bounded, ordered sequence for one event.
One trace identity is one execution identity: the id a Pipeline query, inbound
platform event or Agent run is known by, so a stored chain can be joined with
the Space view of that same execution. Stage records appended under it describe
how that one execution was routed and processed, using only code-defined
identifiers. Nothing here is aggregated: a trace is a bounded, ordered sequence
for one execution.
Three bounds keep memory independent of traffic volume:
@@ -12,7 +14,9 @@ Three bounds keep memory independent of traffic volume:
* a wall-clock TTL, enforced by the sender's sweep.
The identity itself lives in a ContextVar so that asynchronous work spawned
while handling one event inherits it, mirroring ``telemetry.platform``.
while handling one execution inherits it, mirroring ``telemetry.platform``.
Cross-task stages (plugin/RPC work with no inherited context) are resolved by
the owner's execution-id registry instead.
"""
from __future__ import annotations
@@ -28,6 +32,10 @@ MAX_STAGES = 32
_current: contextvars.ContextVar['TraceState | None'] = contextvars.ContextVar('telemetry_trace', default=None)
_route: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_route', default='')
_run: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_run', default='')
# Open workflow step nodes, innermost last. Records emitted inside a step carry
# the innermost node as their ``parent``; the step's own record carries the one
# below it (its enclosing step), never itself.
_node: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar('telemetry_trace_node', default=())
def _now() -> str:
@@ -35,29 +43,55 @@ def _now() -> str:
class TraceState:
"""Bounded stage buffer for exactly one event."""
"""Bounded stage buffer for exactly one execution."""
__slots__ = (
'trace_id',
'execution_id',
'started_at',
'stages',
'dropped_stages',
'synthetic',
'sequence',
'node_sequence',
'open_nodes',
'identity',
'abandoned',
'failure_reason',
'closed',
'adapter',
'runner',
'runner_category',
'model_name',
'pipeline_plugins',
)
def __init__(self) -> None:
self.trace_id = str(uuid4())
def __init__(self, execution_id: str | None = None) -> None:
# The execution identity is the trace identity so a stored chain can be
# joined to the Space view of that same Pipeline query / event / run.
self.execution_id = str(execution_id).strip() if execution_id else ''
self.trace_id = self.execution_id or str(uuid4())
self.started_at = _now()
self.stages: list[dict[str, typing.Any]] = []
self.dropped_stages = 0
self.synthetic = False
self.sequence = 0
# Node ids number the workflow steps of this one execution ("n1", "n2"...).
self.node_sequence = 0
# Mirror of the open step stack, readable from any task: cross-task
# observations (plugin/RPC) resolve their parent through this.
self.open_nodes: tuple[str, ...] = ()
self.identity: dict[str, str] = {}
# Set when the sender refused to buffer this trace: stop appending.
self.abandoned = False
self.failure_reason = ''
self.closed = False
# Execution-scoped record fields attached by the owning lane when known.
self.adapter = ''
self.runner = ''
self.runner_category = ''
self.model_name = ''
self.pipeline_plugins: typing.Any = None
def append(
self,
@@ -69,34 +103,62 @@ class TraceState:
runner: str,
outcome: str,
synthetic: bool,
error: str = '',
node: str = '',
parent: str = '',
) -> None:
if len(self.stages) >= MAX_STAGES:
self.dropped_stages += 1
return
seen = _now()
self.stages.append(
{
'family': family,
'operation': operation,
'mode': mode,
'adapter': adapter,
'runner': runner,
'outcome': outcome,
'synthetic': bool(synthetic),
'seq': self.sequence,
'route_ref': _route.get(),
'run_id': _run.get(),
'first_seen': seen,
'last_seen': seen,
}
)
entry = {
'family': family,
'operation': operation,
'mode': mode,
'adapter': adapter,
'runner': runner,
'outcome': outcome,
'synthetic': bool(synthetic),
'seq': self.sequence,
'route_ref': _route.get(),
'run_id': _run.get(),
'first_seen': seen,
'last_seen': seen,
'error': error,
}
# Only workflow steps carry these; legacy records stay byte-identical.
if node:
entry['node'] = node
if parent:
entry['parent'] = parent
self.stages.append(entry)
self.sequence += 1
if synthetic:
self.synthetic = True
def allocate_node(self) -> str:
"""Mint the next step-node id for this execution ("n1", "n2", ...)."""
self.node_sequence += 1
return f'n{self.node_sequence}'
def mark_failure(self, reason: str) -> None:
"""Remember why this chain broke so the payload can explain itself."""
if not self.failure_reason:
self.failure_reason = reason
def outcome(self) -> str:
"""Terminal outcome of the last recorded stage, for space-side display."""
return self.stages[-1]['outcome'] if self.stages else 'unknown'
"""Terminal outcome of this chain, for space-side display.
A chain that carries a failure reason never reports ``success``: the
reason is the point of the trace.
"""
if self.stages:
last = self.stages[-1]['outcome']
else:
last = 'unknown'
if self.failure_reason and last in ('success', 'skipped', 'unknown'):
return 'failed'
return last
class TraceBinding(typing.NamedTuple):
@@ -105,12 +167,16 @@ class TraceBinding(typing.NamedTuple):
token: typing.Any
def bind() -> TraceBinding:
"""Start a trace unless one is already in flight in this context."""
def bind(execution_id: str | None = None) -> TraceBinding:
"""Start a trace unless one is already in flight in this context.
The execution id becomes the trace id only when this call starts the trace;
a nested boundary keeps the enclosing execution's identity.
"""
existing = _current.get()
if existing is not None:
return TraceBinding(existing, False, None)
state = TraceState()
state = TraceState(execution_id)
return TraceBinding(state, True, _current.set(state))
@@ -136,6 +202,46 @@ def current() -> TraceState | None:
return _current.get()
def current_parent() -> str:
"""The step node that records emitted right now attach to ('' when none)."""
stack = _node.get()
return stack[-1] if stack else ''
def enclosing_parent() -> str:
"""The step enclosing the innermost open one; '' when it is a root step."""
stack = _node.get()
return stack[-2] if len(stack) > 1 else ''
@contextlib.contextmanager
def stage_scope() -> typing.Iterator[str]:
"""Open one workflow step node for the duration of the block.
Allocates a node id unique within the execution, publishes it as the current
parent so that nested observations recorded inside the block attach to it,
and yields that id so the step's own records can be stamped with ``node=``.
Outside any trace nothing is allocated and the empty id is yielded.
"""
state = _current.get()
if state is None:
yield ''
return
node = state.allocate_node()
previous_open = state.open_nodes
state.open_nodes = previous_open + (node,)
token = _node.set(_node.get() + (node,))
try:
yield node
finally:
state.open_nodes = previous_open
try:
_node.reset(token)
except (ValueError, RuntimeError):
# A token from another context must never break execution.
pass
def set_run(run_id: str) -> typing.Any:
return _run.set(run_id)