xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-10-27 02:45:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): attach called platform APIs to their step node on execution traces

Browse Source 
Every host step now opens a stage scope, so observations recorded inside it (including cross-task plugin/RPC calls resolved through the execution registry) carry the owning step as parent; the pipeline lane records its own pipeline/run node so acks and replies issued outside the runner still hang off a node.
This commit is contained in:
Hyu
2026-10-01 18:10:42 +08:00
parent df64f83e12
commit 923ebe12af
12 changed files with 1798 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/langbot/pkg/telemetry/execution.py
+315 -137
View File
@@ -1,34 +1,33 @@
"""Bounded, content-free execution counters for the existing telemetry sender.
"""Content-free execution records for the existing telemetry sender.
This module reports observations only. Coverage catalogs and acceptance rules
belong to Space. Aggregation keys include both immutable execution identities.
belong to Space.
Two shapes share one payload type:
One complete execution is one record: the execution identity (Pipeline query,
inbound platform event or Agent run), its bounded ordered stages and its
terminal outcome. Nothing is aggregated or counted across executions — every
record carries the id of the execution it belongs to, so Space can fetch the
whole chain by primary identity. Records are buffered whole and flushed as
``{"records": [...]}`` on the configured cadence and on shutdown; an in-flight
trace is never flushed half-written.
* window counters, aggregated by ``(family, operation, mode, adapter, runner,
outcome, synthetic)`` — unchanged coverage reporting;
* per-event traces, one bounded payload per traced event, keyed by
``query_id = trace_id`` so Space can fetch a whole chain by primary identity.
Traces are additive: they never alter the counters, and both are bounded in
memory, batch size and send concurrency.
Memory is bounded in stages per trace, buffered traces, buffered records, the
cross-task execution registry and flush batch size.
"""
from __future__ import annotations
import asyncio
import contextlib
import contextvars
import json
import time
from datetime import datetime, timezone
from uuid import uuid4
from . import trace as trace_mod
from .identity import workspace_identity
from .trace import TraceState
MAX_KEYS = 512
MAX_BATCH = 32
FLUSH_SECONDS = 60
MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'})
OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'})
@@ -39,15 +38,70 @@ TRACE_MODES = frozenset({'off', 'failures', 'sampled', 'all'})
DEFAULT_TRACE_MODE = 'sampled'
DEFAULT_TRACE_SAMPLE = 20
# Record bounds: buffered records, one flush batch, flush cadence, registry.
MAX_BUFFERED_RECORDS = 512
MAX_RECORDS_PER_FLUSH = 128
MAX_FLUSH_BYTES = 200 * 1024
DEFAULT_FLUSH_SECONDS = 180
MAX_REGISTRY_KEYS = 256
# Set by cross-task call sites (plugin/RPC actions) that run without the
# ingress context but know which execution they belong to.
_execution_id: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_execution_id', default='')
def set_execution_id(execution_id: str) -> contextvars.Token:
return _execution_id.set(str(execution_id or '').strip())
def reset_execution_id(token: contextvars.Token | None) -> None:
if token is None:
return
try:
_execution_id.reset(token)
except (ValueError, RuntimeError):
# A token from another context must never break execution.
pass
def current_execution_id() -> str:
return _execution_id.get()
def _bounded_error(value, limit: int = 400) -> str:
"""Bound and de-fang one failure detail before it leaves the process.
Error text is operator-facing: keep it single-line, printable and short so a
noisy failure cannot smuggle control characters or bloat the payload.
"""
if value is None:
return ''
text = ''.join(ch if ch.isprintable() else ' ' for ch in str(value))
text = ' '.join(text.split())
return text[:limit]
def _duration_ms(started_at: str, ended_at: str) -> int:
"""Wall-clock duration of one execution; 0 when the interval is unusable."""
try:
delta = datetime.fromisoformat(ended_at) - datetime.fromisoformat(started_at)
return max(int(delta.total_seconds() * 1000), 0)
except Exception:
return 0
class ExecutionCounters:
def __init__(self, manager):
self.manager = manager
self.pending: dict[tuple, dict] = {}
self.records: list[dict] = []
self.task: asyncio.Task | None = None
self.dropped = 0
self.traces: dict[str, TraceState] = {}
self.trace_deadlines: dict[str, float] = {}
# Execution identity -> in-flight trace, for stages recorded by tasks
# that no longer share the ingress context (plugin/RPC platform calls).
self.registry: dict[str, TraceState] = {}
self.registry_deadlines: dict[str, float] = {}
self.dropped_traces = 0
# ------------------------------------------------------------------ config
@@ -63,6 +117,13 @@ class ExecutionCounters:
sample = DEFAULT_TRACE_SAMPLE
return sample if 1 <= sample <= 100000 else DEFAULT_TRACE_SAMPLE
def flush_seconds(self) -> int:
try:
seconds = int(self.manager.telemetry_config.get('telemetry_flush_seconds', DEFAULT_FLUSH_SECONDS))
except (TypeError, ValueError):
seconds = DEFAULT_FLUSH_SECONDS
return seconds if 1 <= seconds <= 86400 else DEFAULT_FLUSH_SECONDS
# --------------------------------------------------------------- recording
def record(
@@ -76,67 +137,91 @@ class ExecutionCounters:
runner: str = '',
outcome: str = 'unknown',
synthetic: bool = False,
error: str = '',
execution_id: str | None = None,
node: str = '',
):
try:
cfg = self.manager.telemetry_config
if not cfg or cfg.get('disable_telemetry', False) or not cfg.get('url'):
return
if family not in {'platform_event', 'event_route', 'runner', 'platform_api'}:
if family not in {'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}:
return
if mode not in MODES or outcome not in OUTCOMES:
return
# Only code-defined identifiers are accepted; never pass user values.
if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner)):
if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner, node)):
return
identity = workspace_identity(context)
state = trace_mod.current()
if state is not None:
self._record_trace_stage(
state,
identity=identity,
family=family,
operation=operation,
mode=mode,
adapter=adapter,
runner=runner,
outcome=outcome,
synthetic=synthetic,
)
key = (
identity['instance_id'],
identity['workspace_uuid'],
family,
operation,
mode,
adapter,
runner,
outcome,
bool(synthetic),
if state is None:
# Cross-task work has no inherited trace; resolve the owning
# execution through the registry instead.
lookup = str(execution_id or '').strip() or current_execution_id()
if lookup:
state = self.registry.get(lookup)
if state is None:
# Every observation belongs to exactly one execution chain.
return
try:
identity = workspace_identity(context)
except Exception:
# A cross-task caller may only carry the Workspace id; reuse the
# identity already bound to the owning trace.
identity = state.identity or {}
if not identity.get('instance_id') or not identity.get('workspace_uuid'):
return
self._record_trace_stage(
state,
identity=identity,
family=family,
operation=operation,
mode=mode,
adapter=adapter,
runner=runner,
outcome=outcome,
synthetic=synthetic,
error=_bounded_error(error),
node=node,
)
row = self.pending.get(key)
if row is None:
if len(self.pending) >= MAX_KEYS:
self.dropped += 1
return
row = {'count': 0, 'first_seen': datetime.now(timezone.utc).isoformat()}
self.pending[key] = row
row['count'] = min(row['count'] + 1, 2147483647)
row['last_seen'] = datetime.now(timezone.utc).isoformat()
self._ensure_loop()
except Exception:
# Observability must never change execution behavior.
return
def _record_trace_stage(
self, state: TraceState, *, identity, family, operation, mode, adapter, runner, outcome, synthetic
self,
state: TraceState,
*,
identity,
family,
operation,
mode,
adapter,
runner,
outcome,
synthetic,
error: str = '',
node: str = '',
) -> None:
if state.abandoned:
return
if trace_mod.current() is state:
# Recorded by the task that owns the step: its step stack is exact.
# A step's own record sits under its enclosing step, never itself.
parent = trace_mod.current_parent()
if node and parent == node:
parent = trace_mod.enclosing_parent()
else:
# A cross-task observation (plugin/RPC) cannot see the step
# contextvar; attach it to the innermost step the owning task has
# open on the trace.
parent = state.open_nodes[-1] if state.open_nodes else ''
if state.trace_id not in self.traces:
# Never buffer traces this configuration would discard anyway.
if self.trace_mode() == 'off' or len(self.traces) >= MAX_TRACES:
self.dropped += 1
state.abandoned = True
self._unregister_state(state)
return
self.traces[state.trace_id] = state
self.trace_deadlines[state.trace_id] = time.monotonic() + TRACE_TTL_SECONDS
@@ -149,50 +234,79 @@ class ExecutionCounters:
runner=runner,
outcome=outcome,
synthetic=bool(synthetic),
error=error,
node=node,
parent=parent,
)
def _ensure_loop(self) -> None:
if self.task is None or self.task.done():
self.task = asyncio.create_task(self._loop())
# ------------------------------------------------------------------ traces
# --------------------------------------------------------------- registry
def close_trace(self, state: TraceState, reason: str = 'event_done') -> None:
"""Emit one trace payload when it matches the configured sample."""
def register_trace(self, state: TraceState, execution_id) -> None:
"""Alias one more execution identity onto an in-flight trace."""
try:
if self.traces.pop(state.trace_id, None) is None:
exec_id = str(execution_id or '').strip()
if not exec_id or state.abandoned:
return
self.trace_deadlines.pop(state.trace_id, None)
if not state.stages or not self._trace_emitted(state):
return
payload = self._build_trace_payload(state, reason)
if payload is not None:
self._dispatch_payload(payload)
if exec_id not in self.registry and len(self.registry) >= MAX_REGISTRY_KEYS:
oldest = next(iter(self.registry))
self.registry.pop(oldest, None)
self.registry_deadlines.pop(oldest, None)
self.registry[exec_id] = state
self.registry_deadlines[exec_id] = time.monotonic() + TRACE_TTL_SECONDS
except Exception:
return
def _dispatch_payload(self, payload: dict) -> None:
"""Hand one built payload to the telemetry manager from this sync context.
def _unregister_state(self, state: TraceState) -> None:
for exec_id in [key for key, value in self.registry.items() if value is state]:
self.registry.pop(exec_id, None)
self.registry_deadlines.pop(exec_id, None)
TelemetryManager.start_send_task is a coroutine, so calling it without
scheduling dropped every trace closed here. Stand-ins used by tests and
manual tools schedule synchronously, hence the coroutine check.
"""
result = self.manager.start_send_task(payload)
if not asyncio.iscoroutine(result):
return
def _sweep_registry(self, now: float) -> None:
"""Drop aliases whose owning execution never closed (leaked bindings)."""
for exec_id in [key for key, deadline in self.registry_deadlines.items() if deadline <= now]:
self.registry.pop(exec_id, None)
self.registry_deadlines.pop(exec_id, None)
# ------------------------------------------------------------------ traces
def close_trace(self, state: TraceState, reason: str = 'event_done', error: str = '') -> None:
"""Buffer one complete execution record when it matches the sample."""
try:
asyncio.get_running_loop().create_task(result)
except RuntimeError:
result.close()
logger = getattr(getattr(self.manager, 'ap', None), 'logger', None)
if logger is not None:
logger.debug('Execution trace payload dropped: no running event loop')
if state.closed:
return
state.closed = True
registered = self.traces.pop(state.trace_id, None) is not None
self.trace_deadlines.pop(state.trace_id, None)
self._unregister_state(state)
if not registered and not state.failure_reason:
return
if (not state.stages and not state.failure_reason) or not self._trace_emitted(state):
return
record = self._build_record(state, reason, error)
if record is not None:
self._enqueue_record(record)
except Exception:
return
def _enqueue_record(self, record: dict) -> None:
if len(self.records) >= MAX_BUFFERED_RECORDS:
self.records.pop(0)
self.dropped += 1
self.records.append(record)
self._ensure_loop()
def _trace_emitted(self, state: TraceState) -> bool:
mode = self.trace_mode()
if mode == 'off':
return False
# A chain that failed must always be uploaded, even when no stage was
# recorded before it broke: the reason is the whole point of the trace.
if state.failure_reason:
return True
if mode == 'all':
return True
# A failed, cancelled or timed-out stage is always worth keeping.
@@ -208,31 +322,69 @@ class ExecutionCounters:
except ValueError:
return False
def _build_trace_payload(self, state: TraceState, reason: str) -> dict | None:
def _build_record(self, state: TraceState, reason: str, error: str = '') -> dict | None:
from ..utils import constants
identity = state.identity
if not identity.get('instance_id') or not identity.get('workspace_uuid'):
return None
observations = [{**stage, 'trace_id': state.trace_id, 'count': 1} for stage in state.stages]
ended_at = datetime.now(timezone.utc).isoformat()
stages = state.stages
observations = []
for stage in stages:
observation = {
'seq': stage['seq'],
'family': stage['family'],
'operation': stage['operation'],
'mode': stage['mode'],
'adapter': stage['adapter'],
'runner': stage['runner'],
'outcome': stage['outcome'],
'synthetic': stage['synthetic'],
'error': stage['error'],
'route_ref': stage['route_ref'],
'run_id': stage['run_id'],
'first_seen': stage['first_seen'],
'last_seen': stage['last_seen'],
'trace_id': state.trace_id,
}
# Workflow position is optional: records outside any step omit both
# keys entirely, so their payloads stay byte-identical.
if stage.get('node'):
observation['node'] = stage['node']
if stage.get('parent'):
observation['parent'] = stage['parent']
observations.append(observation)
return {
'event_type': 'feature_execution',
# One payload per trace: lookup uses the indexed query_id column.
# The execution identity: one record == one chain == one Space row.
'query_id': state.trace_id,
'instance_id': identity['instance_id'],
'workspace_uuid': identity['workspace_uuid'],
'version': constants.semantic_version,
'edition': constants.edition,
'timestamp': datetime.now(timezone.utc).isoformat(),
'timestamp': ended_at,
'trusted': True,
# First failure detail in this chain, so Space can show why it broke.
'error': _bounded_error(error)
or state.failure_reason
or next((stage['error'] for stage in stages if stage['error']), ''),
# Legacy-compatible columns Space already maps; best effort.
'duration_ms': _duration_ms(state.started_at, ended_at),
'model_name': state.model_name or '',
'adapter': state.adapter or next((stage['adapter'] for stage in stages if stage['adapter']), ''),
'runner': state.runner or next((stage['runner'] for stage in stages if stage['runner']), ''),
'runner_category': state.runner_category or '',
'pipeline_plugins': state.pipeline_plugins,
'features': {
'schema': 1,
'observations': observations,
'trace': {
'closed_by': reason,
'started_at': state.started_at,
'ended_at': datetime.now(timezone.utc).isoformat(),
'route_ref': next((s['route_ref'] for s in state.stages if s['route_ref']), ''),
'run_id': next((s['run_id'] for s in state.stages if s['run_id']), ''),
'ended_at': ended_at,
'route_ref': next((stage['route_ref'] for stage in stages if stage['route_ref']), ''),
'run_id': next((stage['run_id'] for stage in stages if stage['run_id']), ''),
'outcome': state.outcome(),
'dropped_stages': state.dropped_stages,
},
@@ -248,56 +400,36 @@ class ExecutionCounters:
# ----------------------------------------------------------------- flushing
async def _loop(self):
interval = self.flush_seconds()
last_flush = time.monotonic()
while self.pending or self.traces:
# Traces expire on a short TTL; counters keep their minute cadence.
await asyncio.sleep(1 if self.traces else FLUSH_SECONDS)
while self.records or self.traces or self.registry:
# Traces expire on a short TTL; records keep their own cadence.
await asyncio.sleep(1 if (self.traces or self.registry) else interval)
now = time.monotonic()
self._sweep_traces(now)
if self.pending and now - last_flush >= FLUSH_SECONDS:
self._sweep_registry(now)
if self.records and now - last_flush >= interval:
last_flush = now
await self.flush()
self.task = None
async def flush(self):
from ..utils import constants
# Remove only one bounded batch; subsequent windows drain the remainder.
# Drain one tenant per minute, at most one request, round-robin by insertion.
if not self.pending:
if not self.records:
return
tenant = next(iter(self.pending))[:2]
keys = [key for key in self.pending if key[:2] == tenant][:MAX_BATCH]
groups: dict[tuple[str, str], list[dict]] = {}
for key in keys:
row = self.pending.pop(key)
instance, workspace, family, operation, mode, adapter, runner, outcome, synthetic = key
groups.setdefault((instance, workspace), []).append(
{
**row,
'family': family,
'operation': operation,
'mode': mode,
'adapter': adapter,
'runner': runner,
'outcome': outcome,
'synthetic': synthetic,
}
)
for (instance, workspace), observations in groups.items():
payload = {
'event_type': 'feature_execution',
'query_id': str(uuid4()),
'instance_id': instance,
'workspace_uuid': workspace,
'version': constants.semantic_version,
'edition': constants.edition,
'timestamp': datetime.now(timezone.utc).isoformat(),
'features': {'schema': 1, 'observations': observations},
}
if not await self.manager.send(payload):
await asyncio.sleep(1)
await self.manager.send(payload)
batch: list[dict] = []
size = 0
while self.records and len(batch) < MAX_RECORDS_PER_FLUSH:
encoded = len(json.dumps(self.records[0], ensure_ascii=False, default=str))
if batch and size + encoded > MAX_FLUSH_BYTES:
break
batch.append(self.records.pop(0))
size += encoded
if not batch:
return
payload = {'records': batch}
if not await self.manager.send(payload):
await asyncio.sleep(1)
await self.manager.send(payload)
async def shutdown(self):
if self.task is not None:
@@ -305,47 +437,93 @@ class ExecutionCounters:
await asyncio.gather(self.task, return_exceptions=True)
self.task = None
try:
await asyncio.wait_for(self.flush(), timeout=2)
deadline = time.monotonic() + 5
while self.records and time.monotonic() < deadline:
remaining = len(self.records)
await asyncio.wait_for(self.flush(), timeout=2)
if len(self.records) >= remaining:
break
except (Exception, asyncio.CancelledError):
pass
self.pending.clear()
# Traces of in-flight events cannot be completed during shutdown.
self.records.clear()
# Executions still in flight cannot be completed during shutdown.
self.dropped += len(self.traces)
self.dropped_traces += len(self.traces)
self.traces.clear()
self.trace_deadlines.clear()
self.registry.clear()
self.registry_deadlines.clear()
def _execution_counters(ap):
counters = getattr(getattr(ap, 'telemetry', None), 'execution', None)
return counters if isinstance(counters, ExecutionCounters) else None
def record(ap, context, **observation):
"""Best-effort bridge usable with optional telemetry and test doubles."""
try:
counters = getattr(getattr(ap, 'telemetry', None), 'execution', None)
if isinstance(counters, ExecutionCounters):
counters = _execution_counters(ap)
if counters is not None:
counters.record(context, **observation)
except Exception:
pass
def close_trace(ap, state, reason: str = 'event_done') -> None:
def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None:
"""Best-effort trace close usable with optional telemetry and test doubles."""
try:
counters = getattr(getattr(ap, 'telemetry', None), 'execution', None)
if isinstance(counters, ExecutionCounters):
counters.close_trace(state, reason)
counters = _execution_counters(ap)
if counters is not None:
counters.close_trace(state, reason, error)
except Exception:
pass
def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding:
"""Bind an execution trace and alias its identity for cross-task stages."""
binding = trace_mod.bind(execution_id)
if execution_id:
try:
counters = _execution_counters(ap)
if counters is not None:
counters.register_trace(binding.state, execution_id)
except Exception:
pass
return binding
@contextlib.contextmanager
def ingress(ap, reason: str = 'event_done'):
"""Trace one ingress boundary; only the owner of the trace closes it.
def ingress(ap, reason: str = 'event_done', context=None, execution_id: str | None = None):
"""Trace one execution boundary; only the owner of the trace closes it.
Nested boundaries (an event routed into a Pipeline, a Runner invoked from a
dispatch) reuse the in-flight trace instead of starting a second one.
dispatch) reuse the in-flight trace instead of starting a second one, and
only alias their own execution id onto it.
"""
binding = trace_mod.bind()
binding = trace_mod.bind(execution_id)
if binding.created and context is not None:
try:
binding.state.identity = workspace_identity(context)
except Exception:
pass
if execution_id and not binding.state.abandoned:
try:
counters = _execution_counters(ap)
if counters is not None:
counters.register_trace(binding.state, execution_id)
except Exception:
pass
failure = ''
try:
yield binding
except BaseException as exc:
# The boundary itself failed: keep the reason on the trace so the uploaded
# payload explains the failure instead of reporting a bare outcome.
if binding.created:
failure = f'{type(exc).__name__}: {exc}'
binding.state.mark_failure(failure)
raise
finally:
if trace_mod.unbind_root(binding):
close_trace(ap, binding.state, reason)
close_trace(ap, binding.state, reason, failure)
src/langbot/pkg/telemetry/platform.py
+24 -2
View File
@@ -11,6 +11,21 @@ from .execution import record
processing_mode: ContextVar[str] = ContextVar('telemetry_processing_mode', default='none')
def result_failure_detail(result) -> str:
"""Pull a bounded human-readable reason out of a failed platform API result."""
raw = getattr(result, 'raw', result)
if isinstance(raw, dict):
for key in ('error', 'message', 'msg', 'detail', 'reason'):
value = raw.get(key)
if isinstance(value, str) and value.strip():
return value
if isinstance(value, dict):
for inner in ('message', 'msg', 'detail'):
if isinstance(value.get(inner), str) and value[inner].strip():
return value[inner]
return ''
def result_outcome(result):
# Empty returns do not prove a remote operation succeeded.
if result is None:
@@ -69,18 +84,24 @@ def observe_adapter(ap, context, adapter):
if action in declared:
observed_operation = action
outcome = 'unknown'
error_detail = ''
try:
result = await method(*args, **kwargs)
outcome = result_outcome(result)
if outcome != 'success':
error_detail = result_failure_detail(result)
return result
except asyncio.CancelledError:
outcome = 'cancelled'
error_detail = 'cancelled'
raise
except TimeoutError:
except TimeoutError as exc:
outcome = 'timeout'
error_detail = str(exc) or 'timeout'
raise
except Exception:
except Exception as exc:
outcome = 'failed'
error_detail = str(exc)
raise
finally:
record(
@@ -91,6 +112,7 @@ def observe_adapter(ap, context, adapter):
adapter=adapter.__class__.__name__,
mode=processing_mode.get(),
outcome=outcome,
error=error_detail,
)
return wrapped
src/langbot/pkg/telemetry/trace.py
+136 -30
View File
@@ -1,9 +1,11 @@
"""Content-free per-event execution traces.
"""Content-free per-execution traces.
One trace identity is minted per inbound platform event and survives until the
ingress handler returns. Stage records appended under it describe how that one
event was routed and processed, using only code-defined identifiers. Nothing
here is aggregated: a trace is a bounded, ordered sequence for one event.
One trace identity is one execution identity: the id a Pipeline query, inbound
platform event or Agent run is known by, so a stored chain can be joined with
the Space view of that same execution. Stage records appended under it describe
how that one execution was routed and processed, using only code-defined
identifiers. Nothing here is aggregated: a trace is a bounded, ordered sequence
for one execution.
Three bounds keep memory independent of traffic volume:
@@ -12,7 +14,9 @@ Three bounds keep memory independent of traffic volume:
* a wall-clock TTL, enforced by the sender's sweep.
The identity itself lives in a ContextVar so that asynchronous work spawned
while handling one event inherits it, mirroring ``telemetry.platform``.
while handling one execution inherits it, mirroring ``telemetry.platform``.
Cross-task stages (plugin/RPC work with no inherited context) are resolved by
the owner's execution-id registry instead.
"""
from __future__ import annotations
@@ -28,6 +32,10 @@ MAX_STAGES = 32
_current: contextvars.ContextVar['TraceState | None'] = contextvars.ContextVar('telemetry_trace', default=None)
_route: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_route', default='')
_run: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_run', default='')
# Open workflow step nodes, innermost last. Records emitted inside a step carry
# the innermost node as their ``parent``; the step's own record carries the one
# below it (its enclosing step), never itself.
_node: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar('telemetry_trace_node', default=())
def _now() -> str:
@@ -35,29 +43,55 @@ def _now() -> str:
class TraceState:
"""Bounded stage buffer for exactly one event."""
"""Bounded stage buffer for exactly one execution."""
__slots__ = (
'trace_id',
'execution_id',
'started_at',
'stages',
'dropped_stages',
'synthetic',
'sequence',
'node_sequence',
'open_nodes',
'identity',
'abandoned',
'failure_reason',
'closed',
'adapter',
'runner',
'runner_category',
'model_name',
'pipeline_plugins',
)
def __init__(self) -> None:
self.trace_id = str(uuid4())
def __init__(self, execution_id: str | None = None) -> None:
# The execution identity is the trace identity so a stored chain can be
# joined to the Space view of that same Pipeline query / event / run.
self.execution_id = str(execution_id).strip() if execution_id else ''
self.trace_id = self.execution_id or str(uuid4())
self.started_at = _now()
self.stages: list[dict[str, typing.Any]] = []
self.dropped_stages = 0
self.synthetic = False
self.sequence = 0
# Node ids number the workflow steps of this one execution ("n1", "n2"...).
self.node_sequence = 0
# Mirror of the open step stack, readable from any task: cross-task
# observations (plugin/RPC) resolve their parent through this.
self.open_nodes: tuple[str, ...] = ()
self.identity: dict[str, str] = {}
# Set when the sender refused to buffer this trace: stop appending.
self.abandoned = False
self.failure_reason = ''
self.closed = False
# Execution-scoped record fields attached by the owning lane when known.
self.adapter = ''
self.runner = ''
self.runner_category = ''
self.model_name = ''
self.pipeline_plugins: typing.Any = None
def append(
self,
@@ -69,34 +103,62 @@ class TraceState:
runner: str,
outcome: str,
synthetic: bool,
error: str = '',
node: str = '',
parent: str = '',
) -> None:
if len(self.stages) >= MAX_STAGES:
self.dropped_stages += 1
return
seen = _now()
self.stages.append(
{
'family': family,
'operation': operation,
'mode': mode,
'adapter': adapter,
'runner': runner,
'outcome': outcome,
'synthetic': bool(synthetic),
'seq': self.sequence,
'route_ref': _route.get(),
'run_id': _run.get(),
'first_seen': seen,
'last_seen': seen,
}
)
entry = {
'family': family,
'operation': operation,
'mode': mode,
'adapter': adapter,
'runner': runner,
'outcome': outcome,
'synthetic': bool(synthetic),
'seq': self.sequence,
'route_ref': _route.get(),
'run_id': _run.get(),
'first_seen': seen,
'last_seen': seen,
'error': error,
}
# Only workflow steps carry these; legacy records stay byte-identical.
if node:
entry['node'] = node
if parent:
entry['parent'] = parent
self.stages.append(entry)
self.sequence += 1
if synthetic:
self.synthetic = True
def allocate_node(self) -> str:
"""Mint the next step-node id for this execution ("n1", "n2", ...)."""
self.node_sequence += 1
return f'n{self.node_sequence}'
def mark_failure(self, reason: str) -> None:
"""Remember why this chain broke so the payload can explain itself."""
if not self.failure_reason:
self.failure_reason = reason
def outcome(self) -> str:
"""Terminal outcome of the last recorded stage, for space-side display."""
return self.stages[-1]['outcome'] if self.stages else 'unknown'
"""Terminal outcome of this chain, for space-side display.
A chain that carries a failure reason never reports ``success``: the
reason is the point of the trace.
"""
if self.stages:
last = self.stages[-1]['outcome']
else:
last = 'unknown'
if self.failure_reason and last in ('success', 'skipped', 'unknown'):
return 'failed'
return last
class TraceBinding(typing.NamedTuple):
@@ -105,12 +167,16 @@ class TraceBinding(typing.NamedTuple):
token: typing.Any
def bind() -> TraceBinding:
"""Start a trace unless one is already in flight in this context."""
def bind(execution_id: str | None = None) -> TraceBinding:
"""Start a trace unless one is already in flight in this context.
The execution id becomes the trace id only when this call starts the trace;
a nested boundary keeps the enclosing execution's identity.
"""
existing = _current.get()
if existing is not None:
return TraceBinding(existing, False, None)
state = TraceState()
state = TraceState(execution_id)
return TraceBinding(state, True, _current.set(state))
@@ -136,6 +202,46 @@ def current() -> TraceState | None:
return _current.get()
def current_parent() -> str:
"""The step node that records emitted right now attach to ('' when none)."""
stack = _node.get()
return stack[-1] if stack else ''
def enclosing_parent() -> str:
"""The step enclosing the innermost open one; '' when it is a root step."""
stack = _node.get()
return stack[-2] if len(stack) > 1 else ''
@contextlib.contextmanager
def stage_scope() -> typing.Iterator[str]:
"""Open one workflow step node for the duration of the block.
Allocates a node id unique within the execution, publishes it as the current
parent so that nested observations recorded inside the block attach to it,
and yields that id so the step's own records can be stamped with ``node=``.
Outside any trace nothing is allocated and the empty id is yielded.
"""
state = _current.get()
if state is None:
yield ''
return
node = state.allocate_node()
previous_open = state.open_nodes
state.open_nodes = previous_open + (node,)
token = _node.set(_node.get() + (node,))
try:
yield node
finally:
state.open_nodes = previous_open
try:
_node.reset(token)
except (ValueError, RuntimeError):
# A token from another context must never break execution.
pass
def set_run(run_id: str) -> typing.Any:
return _run.set(run_id)