mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-24 17:35:19 +08:00
feat(telemetry): add isolated beta quality diagnostics and release identity
This commit is contained in:
dadachann
@@ -0,0 +1,83 @@
|
||||
"""Packaged capability catalog and projections from existing runtime declarations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from . import diagnostic_privacy as privacy
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def catalog():
|
||||
result = {}
|
||||
base = Path(__file__).resolve().parents[1] / 'platform'
|
||||
for manifest in sorted((base / 'adapters').glob('*/manifest.yaml')):
|
||||
data = yaml.safe_load(manifest.read_text())
|
||||
spec = data.get('spec', {})
|
||||
name = data['metadata']['name']
|
||||
privacy.code_value('adapter', name)
|
||||
events = spec.get('supported_events', [])
|
||||
apis = spec.get('supported_apis', {})
|
||||
apis = apis if isinstance(apis, list) else [v for items in apis.values() for v in items]
|
||||
for event in events:
|
||||
privacy.code_value('platform_event_type', event)
|
||||
for operation in apis:
|
||||
privacy.code_value('operation', operation)
|
||||
for api in spec.get('platform_specific_apis', []):
|
||||
privacy.code_value('operation', api['action'])
|
||||
result[manifest.parent.name] = {'adapter': name, 'events': events, 'apis': apis}
|
||||
return result
|
||||
|
||||
|
||||
def adapter_fields(adapter):
|
||||
module = type(adapter).__module__.split('.')
|
||||
entry = next((entry for directory, entry in catalog().items() if directory in module), None)
|
||||
return {'adapter': entry['adapter']} if entry else {}
|
||||
|
||||
|
||||
def _snapshot_bot(bot):
|
||||
from . import diagnostics
|
||||
|
||||
manager = getattr(bot.ap, 'diagnostics', None)
|
||||
if not isinstance(manager, diagnostics.DiagnosticsManager) or not manager.enabled:
|
||||
return
|
||||
fields = adapter_fields(bot.adapter)
|
||||
if not fields:
|
||||
return
|
||||
fields['workspace_uuid'] = bot.execution_context.workspace_uuid
|
||||
entry = next(e for e in catalog().values() if e['adapter'] == fields['adapter'])
|
||||
for capability_type, method, declared in (
|
||||
('event', 'get_supported_events', entry['events']),
|
||||
('api', 'get_supported_apis', entry['apis']),
|
||||
):
|
||||
try:
|
||||
supported = set(getattr(bot.adapter, method)() or [])
|
||||
except Exception:
|
||||
supported = set()
|
||||
for name in declared:
|
||||
manager.emit(
|
||||
'capability',
|
||||
name if capability_type == 'api' else 'platform.receive',
|
||||
'succeeded',
|
||||
source='platform',
|
||||
stage='snapshot',
|
||||
**fields,
|
||||
platform_event_type=name if capability_type == 'event' else '',
|
||||
attributes={
|
||||
'capability_type': capability_type,
|
||||
'capability_name': name,
|
||||
'supported': name in supported,
|
||||
'configured': True,
|
||||
'available': True,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def snapshot_bot(bot):
|
||||
try:
|
||||
_snapshot_bot(bot)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Closed, content-free projection for Beta diagnostics; never serialize payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import builtins
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
from uuid import UUID
|
||||
|
||||
KINDS = frozenset('lifecycle event route run api delivery interaction capability transport summary'.split())
|
||||
SOURCES = frozenset(
|
||||
'platform pipeline agent event_processor webui_debug http websocket mcp plugin runtime startup internal synthetic'.split()
|
||||
)
|
||||
OUTCOMES = frozenset('started succeeded failed cancelled timeout skipped rejected waiting partial unknown'.split())
|
||||
PROCESSORS = frozenset(('', 'pipeline', 'agent', 'event_processor'))
|
||||
# These sets are populated ONLY by source-code decorators / packaged manifests,
|
||||
# never by requests, installed third-party plugin declarations or configuration.
|
||||
VOCABULARY: dict[str, set[str]] = {
|
||||
'operation': {'diagnostics.transport', 'startup', 'route.primary', 'route.subscription', 'runner.run'},
|
||||
'stage': {'execute', 'prepare', 'convert', 'dispatch', 'accepted', 'ack', 'resume', 'shutdown', 'snapshot'},
|
||||
'adapter': set(),
|
||||
'platform_event_type': set(),
|
||||
'reason_code': set(
|
||||
'runner_failed response_error route_not_found processor_incompatible processor_not_found discarded not_matched matched delivered waiting interaction_rejected generator_closed transport_loss'.split()
|
||||
),
|
||||
'capability_type': {'event', 'api', 'processor'},
|
||||
'tool_category': {'native', 'plugin', 'mcp', 'skill', 'platform', 'unknown'},
|
||||
'transport': {'stdio', 'websocket', 'http', 'unknown'},
|
||||
'runner_usage': {'agent', 'event'},
|
||||
'os': {'linux', 'darwin', 'windows'},
|
||||
'arch': {'x86_64', 'aarch64', 'arm64', 'amd64'},
|
||||
'database': {'sqlite', 'postgresql'},
|
||||
'edition': {'community', 'cloud', 'enterprise'},
|
||||
}
|
||||
BOOLS = frozenset('stream synthetic configured available previous_session_unclean recovered supported'.split())
|
||||
NUMBERS = frozenset(
|
||||
'attempts successes failures cancellations timeouts partial unknown generated queued acked dropped retried failed queue_size capacity result_count input_tokens output_tokens'.split()
|
||||
)
|
||||
VERSIONS = frozenset('sdk_version plugin_version runner_version python_version'.split())
|
||||
|
||||
|
||||
def code_value(field: str, value: str) -> str:
|
||||
"""Register a literal from trusted Core source, not a runtime string."""
|
||||
VOCABULARY.setdefault(field, set()).add(value)
|
||||
return value
|
||||
|
||||
|
||||
def category(field: str, value) -> str:
|
||||
return value if type(value) is str and value in VOCABULARY.get(field, ()) else ''
|
||||
|
||||
|
||||
def opaque(value) -> str:
|
||||
if not isinstance(value, str):
|
||||
return ''
|
||||
try:
|
||||
return str(UUID(value))
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return ''
|
||||
|
||||
|
||||
def version_string(value) -> str:
|
||||
# Public package versions only; do not permit arbitrary PEP440 local labels.
|
||||
return value if isinstance(value, str) and re.fullmatch(r'[0-9][0-9.abrcdevpost-]{0,47}', value) else ''
|
||||
|
||||
|
||||
def attributes(values) -> dict:
|
||||
if not isinstance(values, dict):
|
||||
return {}
|
||||
result = {}
|
||||
for key, value in values.items():
|
||||
if key in BOOLS and type(value) is bool:
|
||||
result[key] = value
|
||||
elif key in NUMBERS and type(value) is int and math.isfinite(value) and 0 <= value <= 1_000_000:
|
||||
result[key] = value
|
||||
elif key in VERSIONS and version_string(value):
|
||||
result[key] = value
|
||||
elif key in VOCABULARY and key not in {'operation', 'stage', 'adapter', 'platform_event_type', 'reason_code'}:
|
||||
if category(key, value):
|
||||
result[key] = value
|
||||
elif key == 'capability_name' and any(category(f, value) for f in ('operation', 'platform_event_type')):
|
||||
result[key] = value
|
||||
elif key == 'capability_name' and value in PROCESSORS - {''}:
|
||||
result[key] = value
|
||||
elif key in ('event_types', 'api_operations', 'processor_types') and isinstance(value, (tuple, list)):
|
||||
allowed = (
|
||||
VOCABULARY['platform_event_type']
|
||||
if key == 'event_types'
|
||||
else VOCABULARY['operation']
|
||||
if key == 'api_operations'
|
||||
else PROCESSORS
|
||||
)
|
||||
result[key] = [v for v in value[:128] if type(v) is str and v in allowed]
|
||||
elif (
|
||||
key in ('source_revision', 'target_revision')
|
||||
and isinstance(value, str)
|
||||
and re.fullmatch('[a-f0-9]{40}', value)
|
||||
):
|
||||
result[key] = value
|
||||
elif key == 'latency_buckets' and isinstance(value, (list, tuple)):
|
||||
result[key] = [v for v in value[:128] if type(v) is int and 0 <= v <= 1_000_000]
|
||||
return result
|
||||
|
||||
|
||||
def error_fields(error: BaseException, operation: str, stage: str) -> dict:
|
||||
cls = type(error)
|
||||
# Third party exception names may be dynamically constructed from input.
|
||||
name = cls.__name__ if getattr(builtins, cls.__name__, None) is cls else 'Exception'
|
||||
if isinstance(error, TimeoutError):
|
||||
name = 'TimeoutError'
|
||||
# No message, traceback, filename, line number, or locals are inspected.
|
||||
fingerprint = hashlib.sha256(f'{operation}|{stage}|{cls.__module__}|{cls.__qualname__}'.encode()).hexdigest()
|
||||
return {'error_type': name, 'error_fingerprint': fingerprint}
|
||||
@@ -0,0 +1,399 @@
|
||||
"""Bounded, memory-only Beta diagnostics transport, isolated from usage telemetry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import platform
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from packaging.version import Version, InvalidVersion
|
||||
|
||||
from . import diagnostic_privacy as privacy
|
||||
|
||||
|
||||
class DiagnosticsManager:
|
||||
"""All producers are synchronous; only this owned worker performs I/O.
|
||||
|
||||
Retention: 2048 events / 15 minutes, 5 attempts, 8 KiB per event. An in-flight
|
||||
event stays in the same bounded queue until explicitly ACKed. No disk spool
|
||||
means abrupt process loss cannot be counted after restart (documented).
|
||||
"""
|
||||
|
||||
def __init__(self, ap, *, version=None, instance_id=None, capacity=2048, marker_path=None):
|
||||
from ..utils import constants
|
||||
from .diagnostic_catalog import catalog
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
catalog()
|
||||
self.ap = ap
|
||||
self.marker_path = Path(marker_path) if marker_path is not None else None
|
||||
self._session_id = str(uuid4())
|
||||
self._marker_active = False
|
||||
raw_version = version or importlib.metadata.version('langbot')
|
||||
try:
|
||||
parsed = Version(raw_version)
|
||||
self.version = str(parsed)
|
||||
self.beta = bool(parsed.pre and parsed.pre[0] == 'b' and not parsed.local)
|
||||
except InvalidVersion:
|
||||
self.version, self.beta = '', False
|
||||
self.instance_id = instance_id or constants.instance_id
|
||||
try:
|
||||
from langbot._build_info import CORE_REVISION
|
||||
except ImportError:
|
||||
CORE_REVISION = ''
|
||||
candidate = os.getenv('LANGBOT_BUILD_REVISION', CORE_REVISION)
|
||||
self.revision = candidate.lower() if re.fullmatch('[a-fA-F0-9]{40}', candidate or '') else ''
|
||||
try:
|
||||
self.sdk_version = privacy.version_string(importlib.metadata.version('langbot-plugin'))
|
||||
except importlib.metadata.PackageNotFoundError:
|
||||
self.sdk_version = ''
|
||||
self.capacity = max(1, min(capacity, 2048))
|
||||
self.pending: list[dict] = []
|
||||
self.counters = Counter()
|
||||
self._reported = Counter()
|
||||
self._last_retry = False
|
||||
self.max_attempts = 5
|
||||
self.retention_seconds = 900
|
||||
self.request_timeout = 10
|
||||
self.client: httpx.AsyncClient | None = None
|
||||
self._attempts: dict[str, int] = {}
|
||||
self._born: dict[str, float] = {}
|
||||
self._worker = None
|
||||
self._request = None
|
||||
self._closing = False
|
||||
self._wake = asyncio.Event()
|
||||
self._flush_lock = asyncio.Lock()
|
||||
|
||||
async def start_session(self):
|
||||
"""One bounded, content-free marker per process; never per-event disk I/O."""
|
||||
if not self.enabled:
|
||||
await self._clear_marker()
|
||||
return
|
||||
previous = False
|
||||
if self.marker_path is not None:
|
||||
|
||||
def mark():
|
||||
previous = self.marker_path.exists()
|
||||
self.marker_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.marker_path.write_text(self._session_id)
|
||||
return previous
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
previous = await asyncio.to_thread(mark)
|
||||
self._marker_active = True
|
||||
privacy.code_value('operation', 'startup.session')
|
||||
self.emit(
|
||||
'lifecycle',
|
||||
'startup.session',
|
||||
'started',
|
||||
source='startup',
|
||||
stage='snapshot',
|
||||
attributes={
|
||||
'previous_session_unclean': previous,
|
||||
'os': platform.system().lower(),
|
||||
'arch': platform.machine().lower(),
|
||||
'python_version': '.'.join(map(str, sys.version_info[:3])),
|
||||
'database': self.ap.instance_config.data.get('database', {}).get('use', ''),
|
||||
'edition': self.ap.instance_config.data.get('system', {}).get('edition', ''),
|
||||
},
|
||||
)
|
||||
|
||||
async def _clear_marker(self):
|
||||
if self.marker_path is None:
|
||||
return
|
||||
|
||||
def remove():
|
||||
if self.marker_path.exists():
|
||||
with self.marker_path.open() as handle:
|
||||
owner = handle.read(64)
|
||||
if owner == self._session_id or not self._marker_active:
|
||||
self.marker_path.unlink(missing_ok=True)
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
await asyncio.to_thread(remove)
|
||||
self._marker_active = False
|
||||
|
||||
@property
|
||||
def enabled(self):
|
||||
config = getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('space', {})
|
||||
return (
|
||||
self.beta
|
||||
and not self._closing
|
||||
and not config.get('disable_telemetry', False)
|
||||
and not config.get('disable_beta_diagnostics', False)
|
||||
and bool(config.get('url'))
|
||||
)
|
||||
|
||||
def clear(self):
|
||||
self.pending.clear()
|
||||
self._attempts.clear()
|
||||
self._born.clear()
|
||||
if self._request is not None and not self._request.done():
|
||||
self._request.cancel()
|
||||
|
||||
def emit(
|
||||
self,
|
||||
kind,
|
||||
operation,
|
||||
outcome,
|
||||
*,
|
||||
source='internal',
|
||||
workspace_uuid='',
|
||||
attributes=None,
|
||||
stage='execute',
|
||||
adapter='',
|
||||
processor_type='',
|
||||
platform_event_type='',
|
||||
reason_code='',
|
||||
duration_ms=0,
|
||||
error=None,
|
||||
trace_id='',
|
||||
span_id='',
|
||||
parent_span_id='',
|
||||
run_id='',
|
||||
**ignored,
|
||||
):
|
||||
"""Project known scalar fields; failures in diagnostics never escape."""
|
||||
try:
|
||||
if not self.enabled:
|
||||
self.clear()
|
||||
return
|
||||
if kind not in privacy.KINDS or outcome not in privacy.OUTCOMES:
|
||||
return
|
||||
self.counters['generated'] += 1
|
||||
event = {
|
||||
'schema_version': 1,
|
||||
'event_id': str(uuid4()),
|
||||
'kind': kind,
|
||||
'instance_id': self.instance_id,
|
||||
'workspace_uuid': privacy.opaque(workspace_uuid),
|
||||
'core_version': self.version,
|
||||
'core_revision': self.revision,
|
||||
'sdk_version': self.sdk_version,
|
||||
'release_channel': 'beta',
|
||||
'source': source if source in privacy.SOURCES else 'internal',
|
||||
'operation': privacy.category('operation', operation),
|
||||
'stage': privacy.category('stage', stage),
|
||||
'outcome': outcome,
|
||||
'reason_code': privacy.category('reason_code', reason_code),
|
||||
'adapter': privacy.category('adapter', adapter),
|
||||
'processor_type': processor_type if processor_type in privacy.PROCESSORS else '',
|
||||
'platform_event_type': privacy.category('platform_event_type', platform_event_type),
|
||||
'occurred_at': datetime.now(timezone.utc).isoformat(),
|
||||
'count': 1,
|
||||
'sample_rate': 1,
|
||||
'duration_ms': max(0, min(float(duration_ms), 86400000)),
|
||||
'attributes': privacy.attributes(attributes),
|
||||
}
|
||||
for key, value in (
|
||||
('trace_id', trace_id),
|
||||
('span_id', span_id),
|
||||
('parent_span_id', parent_span_id),
|
||||
('run_id', run_id),
|
||||
):
|
||||
if privacy.opaque(value):
|
||||
event[key] = privacy.opaque(value)
|
||||
if error is not None:
|
||||
event.update(privacy.error_fields(error, event['operation'], event['stage']))
|
||||
encoded = json.dumps(event, allow_nan=False).encode()
|
||||
if (
|
||||
not isinstance(self.instance_id, str)
|
||||
or not 0 < len(self.instance_id) <= 128
|
||||
or len(encoded) > 8192
|
||||
or len(self.pending) >= self.capacity
|
||||
):
|
||||
self.counters['dropped'] += 1
|
||||
return
|
||||
self.pending.append(event)
|
||||
self._born[event['event_id']] = time.monotonic()
|
||||
self.counters['queued'] += 1
|
||||
self._wake.set()
|
||||
except Exception:
|
||||
self.counters['dropped'] += 1
|
||||
|
||||
async def credentials(self, workspace_uuid):
|
||||
# Resolve each Workspace independently in the background. No OSS secret.
|
||||
token = os.getenv('LANGBOT_TELEMETRY_INGEST_TOKEN', '').strip()
|
||||
if token:
|
||||
return {'X-LangBot-Telemetry-Token': token}
|
||||
if not workspace_uuid:
|
||||
return {}
|
||||
users = getattr(self.ap, 'user_service', None)
|
||||
space = getattr(self.ap, 'space_service', None)
|
||||
if users is None or space is None:
|
||||
return {}
|
||||
owner = await users.get_workspace_owner(workspace_uuid)
|
||||
email = getattr(owner, 'user', None)
|
||||
token = await space.get_valid_access_token(email) if email else None
|
||||
return {'Authorization': f'Bearer {token}'} if token else {}
|
||||
|
||||
def _remove(self, ids):
|
||||
self.pending[:] = [e for e in self.pending if e['event_id'] not in ids]
|
||||
for event_id in ids:
|
||||
self._attempts.pop(event_id, None)
|
||||
self._born.pop(event_id, None)
|
||||
|
||||
async def flush_once(self):
|
||||
async with self._flush_lock:
|
||||
if not self.enabled:
|
||||
self.clear()
|
||||
return
|
||||
expired = {
|
||||
e['event_id']
|
||||
for e in self.pending
|
||||
if time.monotonic() - self._born.get(e['event_id'], 0) > self.retention_seconds
|
||||
}
|
||||
self.counters['dropped'] += len(expired)
|
||||
self._remove(expired)
|
||||
if not self.pending:
|
||||
return
|
||||
# Do not authenticate a mixed-Workspace batch with one owner's token.
|
||||
workspace = self.pending[0]['workspace_uuid']
|
||||
batch, size = [], 64
|
||||
for event in self.pending:
|
||||
n = len(json.dumps(event).encode()) + 2
|
||||
if event['workspace_uuid'] != workspace:
|
||||
continue
|
||||
if len(batch) == 50 or size + n > 256 * 1024:
|
||||
break
|
||||
batch.append(event)
|
||||
size += n
|
||||
ids = {e['event_id'] for e in batch}
|
||||
for event_id in ids:
|
||||
self._attempts[event_id] = self._attempts.get(event_id, 0) + 1
|
||||
acked, rejected = set(), set()
|
||||
permanent = False
|
||||
try:
|
||||
async with asyncio.timeout(self.request_timeout):
|
||||
headers = {}
|
||||
try:
|
||||
async with asyncio.timeout(min(1.0, self.request_timeout / 3)):
|
||||
headers = await self.credentials(workspace)
|
||||
except Exception:
|
||||
pass
|
||||
if not self.enabled:
|
||||
self.clear()
|
||||
return
|
||||
if self.client is None:
|
||||
self.client = httpx.AsyncClient(timeout=self.request_timeout, follow_redirects=False)
|
||||
url = (
|
||||
self.ap.instance_config.data['space']['url'].rstrip('/') + '/api/v1/telemetry/diagnostics/batch'
|
||||
)
|
||||
# Stream the response to bound malicious/old server responses too.
|
||||
async with self.client.stream(
|
||||
'POST', url, json={'schema_version': 1, 'events': batch}, headers=headers
|
||||
) as response:
|
||||
body = bytearray()
|
||||
async for chunk in response.aiter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > 65536:
|
||||
raise ValueError('diagnostics_response_limit')
|
||||
if response.status_code == 200:
|
||||
payload = json.loads(body)
|
||||
if payload.get('code') == 200 and isinstance(payload.get('data'), dict):
|
||||
data = payload['data']
|
||||
acked = {x for x in data.get('accepted_event_ids', []) if isinstance(x, str)} & ids
|
||||
rejected = {
|
||||
x.get('event_id')
|
||||
for x in data.get('rejected', [])
|
||||
if isinstance(x, dict) and x.get('code') == 'invalid_event'
|
||||
} & ids
|
||||
elif isinstance(payload.get('code'), int):
|
||||
permanent = 400 <= payload['code'] < 500 and payload['code'] != 429
|
||||
else:
|
||||
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
if not self.enabled:
|
||||
self.clear()
|
||||
return
|
||||
rejected -= acked
|
||||
remaining = ids - acked - rejected
|
||||
if remaining:
|
||||
self.counters['failed'] += 1
|
||||
exhausted = {i for i in remaining if permanent or self._attempts.get(i, 0) >= self.max_attempts}
|
||||
self.counters['acked'] += len(acked)
|
||||
self.counters['dropped'] += len(rejected | exhausted)
|
||||
self._last_retry = bool(remaining - exhausted)
|
||||
self.counters['retried'] += len(remaining - exhausted)
|
||||
self._remove(acked | rejected | exhausted)
|
||||
|
||||
def report_transport(self, stage='snapshot'):
|
||||
# Snapshot interval deltas before enqueue; retries retain this event ID.
|
||||
snapshot = self.counters.copy()
|
||||
delta = snapshot - self._reported
|
||||
before = self.counters['queued']
|
||||
self.emit(
|
||||
'transport',
|
||||
'diagnostics.transport',
|
||||
'partial' if delta['dropped'] else 'succeeded',
|
||||
stage=stage,
|
||||
attributes={**dict(delta), 'queue_size': len(self.pending), 'capacity': self.capacity},
|
||||
)
|
||||
if self.counters['queued'] > before:
|
||||
self._reported = snapshot
|
||||
|
||||
def start(self):
|
||||
if self.beta and self._worker is None:
|
||||
self._worker = asyncio.create_task(self._loop(), name='beta-diagnostics')
|
||||
|
||||
async def _loop(self):
|
||||
next_send, next_summary, backoff = 0.0, time.monotonic() + 60, 0.0
|
||||
try:
|
||||
while not self._closing:
|
||||
if not self.enabled:
|
||||
self.clear()
|
||||
if self._marker_active:
|
||||
await self._clear_marker()
|
||||
elif time.monotonic() >= next_summary:
|
||||
self.report_transport()
|
||||
next_summary = time.monotonic() + 60
|
||||
if self._request is not None and self._request.done():
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
self._request.result()
|
||||
self._request = None
|
||||
backoff = min(30, max(1, backoff * 2)) if self._last_retry else 0
|
||||
next_send = time.monotonic() + random.uniform(0.5, 1.5) * backoff
|
||||
if self.enabled and self.pending and self._request is None and time.monotonic() >= next_send:
|
||||
self._request = asyncio.create_task(self.flush_once(), name='beta-diagnostics-batch')
|
||||
self._wake.clear()
|
||||
with contextlib.suppress(TimeoutError):
|
||||
await asyncio.wait_for(self._wake.wait(), 0.1)
|
||||
finally:
|
||||
if self._request is not None:
|
||||
self._request.cancel()
|
||||
await asyncio.gather(self._request, return_exceptions=True)
|
||||
|
||||
async def shutdown(self, drain_timeout=2):
|
||||
if self._worker is not None:
|
||||
self._worker.cancel()
|
||||
await asyncio.gather(self._worker, return_exceptions=True)
|
||||
self._worker = None
|
||||
if self.enabled and drain_timeout > 0:
|
||||
self.report_transport(stage='shutdown')
|
||||
with contextlib.suppress(Exception):
|
||||
async with asyncio.timeout(drain_timeout):
|
||||
while self.pending:
|
||||
await self.flush_once()
|
||||
if self.pending:
|
||||
await asyncio.sleep(0.1)
|
||||
self.counters['dropped'] += len(self.pending)
|
||||
self._closing = True
|
||||
self.clear()
|
||||
await self._clear_marker()
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
@@ -0,0 +1,461 @@
|
||||
"""Explicit diagnostic boundaries, preserving coroutine and generator semantics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import contextvars
|
||||
import functools
|
||||
import inspect
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from .diagnostic_transport import DiagnosticsManager # noqa: F401
|
||||
from . import diagnostic_privacy as privacy
|
||||
|
||||
_CURRENT = contextvars.ContextVar('beta_diagnostic_span', default=None)
|
||||
|
||||
|
||||
def current_span():
|
||||
return _CURRENT.get()
|
||||
|
||||
|
||||
def set_outcome(outcome, *, reason_code=''):
|
||||
span = current_span()
|
||||
if span is not None:
|
||||
span.outcome = outcome if outcome in privacy.OUTCOMES else 'unknown'
|
||||
span.fields['reason_code'] = privacy.category('reason_code', reason_code)
|
||||
|
||||
|
||||
def annotate(**fields):
|
||||
"""Trusted hook metadata is still projected at the transport boundary."""
|
||||
span = current_span()
|
||||
if span is not None:
|
||||
span.fields.update(fields)
|
||||
|
||||
|
||||
def _owner_app(owner):
|
||||
"""An explicit owner (even absent/disabled) is an inheritance barrier."""
|
||||
if owner is None:
|
||||
return None, False
|
||||
if hasattr(owner, 'ap'):
|
||||
return owner.ap, True
|
||||
if hasattr(owner, 'diagnostics') or hasattr(owner, 'instance_config'):
|
||||
return owner, True
|
||||
for name in ('requester', 'logger', 'adapter'):
|
||||
nested = getattr(owner, name, None)
|
||||
if nested is not None and nested is not owner:
|
||||
app, explicit = _owner_app(nested)
|
||||
if explicit:
|
||||
return app, True
|
||||
return None, False
|
||||
|
||||
|
||||
def _manager(owner):
|
||||
app, _ = _owner_app(owner)
|
||||
manager = getattr(app, 'diagnostics', None)
|
||||
if manager is None or not callable(getattr(manager, 'emit', None)) or not getattr(manager, 'enabled', False):
|
||||
return None
|
||||
# Honor the shared Span/management producer interface without inheriting a
|
||||
# different producer just because this application's producer is absent.
|
||||
config = getattr(getattr(app, 'instance_config', None), 'data', {}).get('space', {})
|
||||
if config.get('disable_telemetry', False) or config.get('disable_beta_diagnostics', False):
|
||||
return None
|
||||
# Direct manager holders (e.g. ReplyStreamSession) are intentional. An app
|
||||
# must never borrow another app's manager, policy, or credential resolver.
|
||||
if hasattr(app, 'instance_config') and getattr(manager, 'ap', app) is not app:
|
||||
return None
|
||||
return manager
|
||||
|
||||
|
||||
def _owner_context(owner):
|
||||
context = getattr(owner, 'execution_context', None)
|
||||
if context is not None:
|
||||
return context
|
||||
for name in ('requester', 'logger', 'adapter'):
|
||||
nested = getattr(owner, name, None)
|
||||
if nested is not None and nested is not owner:
|
||||
context = _owner_context(nested)
|
||||
if context is not None:
|
||||
return context
|
||||
return None
|
||||
|
||||
|
||||
def _execution_context(owner, bound):
|
||||
context = bound.get('execution_context') or _owner_context(owner)
|
||||
adapter_context = bound.get('adapter_context')
|
||||
if isinstance(adapter_context, dict):
|
||||
context = adapter_context.get('_execution_context') or context
|
||||
query = bound.get('query')
|
||||
if query is not None:
|
||||
context = getattr(query, '_execution_context', None) or context
|
||||
return context
|
||||
|
||||
|
||||
def _context_matches(manager, owner, context):
|
||||
from ..api.http.context import ExecutionContext
|
||||
|
||||
if not isinstance(context, ExecutionContext):
|
||||
return True
|
||||
instance = getattr(getattr(manager.ap, 'workspace_service', None), 'instance_uuid', manager.instance_id)
|
||||
if instance != context.instance_uuid:
|
||||
return False
|
||||
owned = _owner_context(owner)
|
||||
return not isinstance(owned, ExecutionContext) or (
|
||||
owned.instance_uuid,
|
||||
owned.workspace_uuid,
|
||||
owned.placement_generation,
|
||||
) == (context.instance_uuid, context.workspace_uuid, context.placement_generation)
|
||||
|
||||
|
||||
def _context_fields(owner, bound):
|
||||
context = _execution_context(owner, bound)
|
||||
query = bound.get('query')
|
||||
from .diagnostic_catalog import adapter_fields
|
||||
|
||||
adapter = getattr(owner, 'adapter', None) or owner
|
||||
fields = adapter_fields(adapter)
|
||||
# ExecutionContext is constructed/validated by the existing auth boundary;
|
||||
# do not infer Workspace identity from arbitrary payload dicts or event IDs.
|
||||
if context is not None:
|
||||
from ..api.http.context import ExecutionContext
|
||||
|
||||
if isinstance(context, ExecutionContext):
|
||||
fields['workspace_uuid'] = context.workspace_uuid
|
||||
saved = getattr(query, '_diagnostic_context', None) if query is not None else None
|
||||
if isinstance(saved, dict) and saved.get('workspace_uuid') == context.workspace_uuid:
|
||||
fields.update(saved)
|
||||
binding = bound.get('binding')
|
||||
if binding is not None:
|
||||
fields['processor_type'] = getattr(binding, 'processor_type', '')
|
||||
event = bound.get('event')
|
||||
if event is not None:
|
||||
fields['platform_event_type'] = getattr(event, 'event_type', None) or getattr(event, 'type', '')
|
||||
delivery = getattr(event, 'delivery', None)
|
||||
if getattr(delivery, 'surface', None) == 'webui':
|
||||
fields['source'] = 'webui_debug'
|
||||
fields['attributes'] = {'synthetic': True}
|
||||
if getattr(owner, 'mock', False) is True:
|
||||
fields['source'] = 'webui_debug'
|
||||
fields['attributes'] = {'synthetic': True}
|
||||
return fields
|
||||
|
||||
|
||||
class Span:
|
||||
def __init__(self, manager, kind, operation, fields):
|
||||
parent = current_span()
|
||||
self.manager = manager
|
||||
self.kind = kind
|
||||
self.operation = operation
|
||||
self.fields = dict(fields)
|
||||
if (
|
||||
parent
|
||||
and parent.fields.get('workspace_uuid')
|
||||
and self.fields.get('workspace_uuid')
|
||||
and parent.fields['workspace_uuid'] != self.fields['workspace_uuid']
|
||||
):
|
||||
parent = None
|
||||
if self.fields.get('operation'):
|
||||
self.operation = privacy.category('operation', self.fields.pop('operation')) or operation
|
||||
self.outcome = None
|
||||
self.finished = False
|
||||
self.started = time.monotonic()
|
||||
self.fields['trace_id'] = (
|
||||
parent.fields['trace_id']
|
||||
if parent and parent.manager is manager
|
||||
else (privacy.opaque(self.fields.get('trace_id')) or str(uuid4()))
|
||||
)
|
||||
self.fields['span_id'] = str(uuid4())
|
||||
if parent and parent.manager is manager:
|
||||
self.fields['parent_span_id'] = parent.fields['span_id']
|
||||
for key in ('workspace_uuid', 'adapter', 'processor_type', 'platform_event_type', 'run_id'):
|
||||
if not self.fields.get(key) and parent.fields.get(key):
|
||||
self.fields[key] = parent.fields[key]
|
||||
if parent.fields.get('source') in ('webui_debug', 'synthetic'):
|
||||
self.fields['source'] = parent.fields['source']
|
||||
self.fields['attributes'] = {**self.fields.get('attributes', {}), 'synthetic': True}
|
||||
self.emit('started')
|
||||
|
||||
def emit(self, outcome, **extra):
|
||||
if self.manager is not None:
|
||||
self.manager.emit(self.kind, self.operation, outcome, **{**self.fields, **extra})
|
||||
|
||||
@contextlib.contextmanager
|
||||
def activate(self):
|
||||
token = _CURRENT.set(self)
|
||||
try:
|
||||
yield self
|
||||
finally:
|
||||
_CURRENT.reset(token)
|
||||
|
||||
def finish(self, error=None):
|
||||
if self.finished:
|
||||
return
|
||||
self.finished = True
|
||||
error = error if error is not None else self.fields.pop('error', None)
|
||||
if isinstance(error, (asyncio.CancelledError, GeneratorExit)):
|
||||
outcome = 'cancelled'
|
||||
elif isinstance(error, TimeoutError):
|
||||
outcome = 'timeout'
|
||||
elif error is not None:
|
||||
outcome = self.outcome if self.outcome in ('timeout', 'partial', 'rejected') else 'failed'
|
||||
else:
|
||||
outcome = self.outcome or 'succeeded'
|
||||
self.emit(outcome, error=error, duration_ms=(time.monotonic() - self.started) * 1000)
|
||||
|
||||
|
||||
def result_outcome(value):
|
||||
"""Inspect only the SDK response's status, not arbitrary result contents."""
|
||||
from langbot_plugin.api.entities.builtin.platform.events import EBAEvent
|
||||
from langbot_plugin.runtime.io.handler import ActionResponse
|
||||
|
||||
span = current_span()
|
||||
if span is not None and span.fields.get('stage') == 'convert':
|
||||
if isinstance(value, EBAEvent):
|
||||
annotate(platform_event_type=value.type)
|
||||
elif value is None:
|
||||
set_outcome('skipped', reason_code='not_matched')
|
||||
if isinstance(value, ActionResponse):
|
||||
if value.code != 0:
|
||||
set_outcome('failed', reason_code='response_error')
|
||||
|
||||
|
||||
def observe(kind, operation, *, source='internal', stage='execute', ap=None, fields=None):
|
||||
"""Explicit boundary with a stable/off fast path and transparent generators."""
|
||||
privacy.code_value('operation', operation)
|
||||
privacy.code_value('stage', stage)
|
||||
|
||||
def decorate(fn):
|
||||
signature = inspect.signature(fn)
|
||||
|
||||
def span_for(args, kwargs):
|
||||
try:
|
||||
bound = signature.bind_partial(*args, **kwargs).arguments
|
||||
owner = bound.get(next(iter(signature.parameters), ''))
|
||||
manager_owner = (ap() if callable(ap) else ap) if ap is not None else owner
|
||||
manager = _manager(manager_owner)
|
||||
_, explicit = _owner_app(manager_owner)
|
||||
context = _execution_context(owner, bound)
|
||||
parent = current_span()
|
||||
if manager is None and not explicit and ap is None and parent is not None:
|
||||
# Stateless converters may inherit, but a different explicit
|
||||
# Workspace must not select a parent's credentials/policy.
|
||||
if context is not None and getattr(context, 'workspace_uuid', None) != parent.fields.get(
|
||||
'workspace_uuid'
|
||||
):
|
||||
return None
|
||||
manager = parent.manager
|
||||
if manager is None or not manager.enabled:
|
||||
return None
|
||||
if not _context_matches(manager, owner, context):
|
||||
return None
|
||||
metadata = {'source': source, 'stage': stage}
|
||||
metadata.update(_context_fields(owner, bound))
|
||||
from .diagnostic_catalog import catalog
|
||||
|
||||
for directory, entry in catalog().items():
|
||||
if directory in fn.__module__.split('.') and '.platform.' in fn.__module__:
|
||||
metadata['adapter'] = entry['adapter']
|
||||
break
|
||||
if fields:
|
||||
extra = fields(bound)
|
||||
extra['attributes'] = {**metadata.get('attributes', {}), **extra.get('attributes', {})}
|
||||
metadata.update(extra)
|
||||
return Span(manager, kind, operation, metadata)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def inspect_result(value):
|
||||
with contextlib.suppress(Exception):
|
||||
result_outcome(value)
|
||||
|
||||
def finish(span, error=None):
|
||||
if span is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
span.finish(error)
|
||||
|
||||
if inspect.isasyncgenfunction(fn):
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
class ObservedGenerator(AsyncGenerator):
|
||||
"""Delegate each native protocol operation, without extra close.
|
||||
|
||||
A yield-based proxy cannot distinguish athrow(GeneratorExit)
|
||||
(which may yield) from aclose() (which must reject a yield).
|
||||
Let the native generator implement that distinction and retain
|
||||
its own primary/cleanup exception and cancellation semantics.
|
||||
"""
|
||||
|
||||
def __init__(self, args, kwargs):
|
||||
self.gen = fn(*args, **kwargs)
|
||||
self.args, self.kwargs = args, kwargs
|
||||
self.span = None
|
||||
self.started = False
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self.gen, name)
|
||||
|
||||
async def _advance(self, method, *values):
|
||||
if not self.started:
|
||||
self.started = True
|
||||
self.span = span_for(self.args, self.kwargs)
|
||||
token = _CURRENT.set(self.span)
|
||||
try:
|
||||
value = await method(*values)
|
||||
if self.span is not None and method != self.gen.aclose:
|
||||
inspect_result(value)
|
||||
if method == self.gen.aclose:
|
||||
finish(self.span, GeneratorExit())
|
||||
return value
|
||||
except StopAsyncIteration:
|
||||
finish(self.span)
|
||||
raise
|
||||
except BaseException as exc:
|
||||
# Rejected protocol calls (e.g. concurrent asend, a
|
||||
# yielded GeneratorExit) need not terminate the stream.
|
||||
if self.gen.ag_frame is None:
|
||||
finish(self.span, exc)
|
||||
raise
|
||||
finally:
|
||||
_CURRENT.reset(token)
|
||||
|
||||
def __anext__(self):
|
||||
return self._advance(self.gen.__anext__)
|
||||
|
||||
def asend(self, value):
|
||||
return self._advance(self.gen.asend, value)
|
||||
|
||||
def athrow(self, *values):
|
||||
return self._advance(self.gen.athrow, *values)
|
||||
|
||||
def aclose(self):
|
||||
return self._advance(self.gen.aclose)
|
||||
|
||||
@functools.wraps(fn)
|
||||
def stream(*args, **kwargs):
|
||||
return ObservedGenerator(args, kwargs)
|
||||
|
||||
return stream
|
||||
|
||||
@functools.wraps(fn)
|
||||
async def call(*args, **kwargs):
|
||||
span = span_for(args, kwargs)
|
||||
if span is None:
|
||||
token = _CURRENT.set(None)
|
||||
try:
|
||||
return await fn(*args, **kwargs)
|
||||
finally:
|
||||
_CURRENT.reset(token)
|
||||
try:
|
||||
with span.activate():
|
||||
value = await fn(*args, **kwargs)
|
||||
inspect_result(value)
|
||||
finish(span)
|
||||
return value
|
||||
except BaseException as exc:
|
||||
finish(span, exc)
|
||||
raise
|
||||
|
||||
return call
|
||||
|
||||
return decorate
|
||||
|
||||
|
||||
def event(owner, kind, operation, outcome, **fields):
|
||||
"""Emit a point-in-time fact from an existing state transition."""
|
||||
try:
|
||||
manager = _manager(owner)
|
||||
_, explicit = _owner_app(owner)
|
||||
parent = current_span()
|
||||
context = _owner_context(owner)
|
||||
if manager is None and not explicit and parent:
|
||||
if context is not None and getattr(context, 'workspace_uuid', None) != parent.fields.get('workspace_uuid'):
|
||||
return
|
||||
manager = parent.manager
|
||||
if manager is not None and _context_matches(manager, owner, context):
|
||||
inherited = dict(parent.fields) if parent and parent.manager is manager else {}
|
||||
if context is not None and getattr(context, 'workspace_uuid', None) != inherited.get('workspace_uuid'):
|
||||
inherited = _context_fields(owner, {})
|
||||
inherited.update(fields)
|
||||
manager.emit(kind, operation, outcome, **inherited)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def capture_context():
|
||||
span = current_span()
|
||||
if span is None or not span.manager.enabled:
|
||||
return None
|
||||
result = {
|
||||
k: span.fields[k]
|
||||
for k in ('trace_id', 'workspace_uuid', 'run_id', 'adapter', 'processor_type', 'platform_event_type', 'source')
|
||||
if k in span.fields
|
||||
}
|
||||
result['parent_span_id'] = span.fields['span_id']
|
||||
return result
|
||||
|
||||
|
||||
def link_context(saved):
|
||||
"""Link only after the existing Host run/installation validator accepted it."""
|
||||
span = current_span()
|
||||
if span is None or not isinstance(saved, dict):
|
||||
return
|
||||
if span.fields.get('workspace_uuid') and span.fields['workspace_uuid'] != saved.get('workspace_uuid'):
|
||||
return
|
||||
for key in ('trace_id', 'parent_span_id', 'workspace_uuid', 'run_id'):
|
||||
if privacy.opaque(saved.get(key)):
|
||||
span.fields[key] = saved[key]
|
||||
for key in ('adapter', 'platform_event_type'):
|
||||
if privacy.category(key, saved.get(key)):
|
||||
span.fields[key] = saved[key]
|
||||
if saved.get('processor_type') in privacy.PROCESSORS:
|
||||
span.fields['processor_type'] = saved['processor_type']
|
||||
if saved.get('source') in ('webui_debug', 'synthetic'):
|
||||
span.fields['source'] = saved['source']
|
||||
span.fields['attributes'] = {**span.fields.get('attributes', {}), 'synthetic': True}
|
||||
|
||||
|
||||
def declare_runner(descriptor):
|
||||
"""Allow only identifiers from a validated installed public Runner manifest."""
|
||||
import re
|
||||
|
||||
pairs = {'plugin_id': descriptor.get_plugin_id(), 'runner_id': descriptor.id}
|
||||
for field, value in pairs.items():
|
||||
allowed = privacy.VOCABULARY.setdefault(field, set())
|
||||
if (
|
||||
isinstance(value, str)
|
||||
and len(value) <= 128
|
||||
and len(allowed) < 1024
|
||||
and re.fullmatch(r'(?:plugin:)?[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)?', value)
|
||||
):
|
||||
allowed.add(value)
|
||||
|
||||
|
||||
def runner_metadata(owner, descriptor, processor_type):
|
||||
manager = _manager(owner)
|
||||
if manager is None or not manager.enabled:
|
||||
return
|
||||
metadata = {
|
||||
'plugin_id': descriptor.get_plugin_id(),
|
||||
'runner_id': descriptor.id,
|
||||
'plugin_version': getattr(descriptor, 'plugin_version', ''),
|
||||
'runner_usage': 'event' if processor_type == 'event_processor' else 'agent',
|
||||
}
|
||||
annotate(attributes=metadata)
|
||||
event(
|
||||
owner,
|
||||
'capability',
|
||||
'runner.run',
|
||||
'succeeded',
|
||||
source='runtime',
|
||||
stage='snapshot',
|
||||
processor_type=processor_type,
|
||||
attributes={
|
||||
**metadata,
|
||||
'capability_type': 'processor',
|
||||
'capability_name': processor_type,
|
||||
'supported': True,
|
||||
'configured': True,
|
||||
'available': True,
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user