xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-10-21 16:05:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(telemetry): add isolated beta quality diagnostics and release identity

Browse Source
This commit is contained in:
dadachann
2026-09-15 09:35:01 +00:00
parent 91d6de8858
commit f8123ead0a
89 changed files with 4205 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/langbot/pkg/agent/runner/interaction_manager.py
+9
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from langbot.pkg.telemetry import diagnostics
import json
import time
import typing
@@ -91,6 +93,7 @@ class InteractionManager:
self.ap = ap
self.store = store or InteractionStore(ap.persistence_mgr.get_db_engine())
@diagnostics.observe('interaction', 'interaction.request', source='agent', stage='execute')
async def handle_result(
self,
*,
@@ -198,14 +201,17 @@ class InteractionManager:
except Exception as exc:
await self.store.mark_delivery_failed(run_id, request.interaction_id, str(exc))
raise
diagnostics.set_outcome('waiting', reason_code='waiting')
return True
@diagnostics.observe('interaction', 'interaction.acknowledge', source='platform', stage='ack')
async def acknowledge_submission(self, record: dict[str, typing.Any], adapter: typing.Any) -> None:
"""Best-effort transition of submitted controls into a read-only state."""
delivery_result = record.get('delivery_result')
if not isinstance(delivery_result, dict) or not self._supports_platform_api(
adapter, INTERACTION_ACKNOWLEDGE_API
):
diagnostics.set_outcome('skipped')
return
try:
delivery_result = await adapter.call_platform_api(
@@ -223,6 +229,8 @@ class InteractionManager:
delivery_result,
)
except Exception as exc:
diagnostics.set_outcome('failed', reason_code='response_error')
diagnostics.annotate(error=exc)
self._warning(f'Failed to acknowledge interaction submission: {exc}')
async def _find_update_target(
@@ -273,6 +281,7 @@ class InteractionManager:
f'interaction field {field.id} has duplicate option values',
)
@diagnostics.observe('interaction', 'interaction.consume', source='platform', stage='dispatch')
async def consume_callback(
self,
*,
src/langbot/pkg/agent/runner/orchestrator.py
+131 -111
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import time
import contextlib
import typing
from langbot_plugin.api.entities.builtin.provider import message as provider_message
@@ -10,6 +11,7 @@ from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
from langbot_plugin.entities.io.actions.enums import PluginToRuntimeAction
from ...telemetry import diagnostics as diagnostics
from .reply_stream import ReplyStreamSession
from ...core import app
from ...api.http.context import ExecutionContext
@@ -79,6 +81,7 @@ class AgentRunOrchestrator:
self.journal = AgentRunJournal(ap)
self._session_registry = get_session_registry()
@diagnostics.observe('run', 'runner.run', source='agent', stage='prepare')
async def run(
self,
event: AgentEventEnvelope,
@@ -105,6 +108,7 @@ class AgentRunOrchestrator:
bound_plugins,
)
diagnostics.runner_metadata(self.ap, descriptor, binding.processor_type)
usage = 'event' if binding.processor_type == 'event_processor' else 'agent'
if usage not in descriptor.usages:
raise ValueError(f'The selected Runner does not support {usage} usage')
@@ -158,6 +162,7 @@ class AgentRunOrchestrator:
state_context = build_state_context(event, binding, descriptor)
run_id = context['run_id']
diagnostics.annotate(run_id=run_id, stage='execute')
context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any(
tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform'
for tool in resources.get('tools', [])
@@ -168,6 +173,7 @@ class AgentRunOrchestrator:
source=(adapter_context or {}).get('_platform_event')
or getattr((adapter_context or {}).get('_query'), 'message_event', None),
)
reply_streams.diagnostics = getattr(self.ap, 'diagnostics', None)
available_apis = context.get('context', {}).get('available_apis')
run_authorization = {
'runner_id': descriptor.id,
@@ -236,125 +242,134 @@ class AgentRunOrchestrator:
event_log_id=event_log_id,
)
async for result_dict in self.invoker.invoke(descriptor, context):
result_dict = dict(result_dict)
sequence = result_dict.get('sequence')
if sequence is not None:
try:
sequence_int = int(sequence)
except (TypeError, ValueError):
self.ap.logger.warning(f'Runner {descriptor.id} returned invalid result sequence: {sequence}')
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
else:
if sequence_int in seen_sequences:
async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results:
async for result_dict in results:
result_dict = dict(result_dict)
sequence = result_dict.get('sequence')
if sequence is not None:
try:
sequence_int = int(sequence)
except (TypeError, ValueError):
self.ap.logger.warning(
f'Runner {descriptor.id} returned duplicate result sequence '
f'{sequence_int} for run {run_id}; dropping duplicate'
)
continue
if sequence_int <= 0:
self.ap.logger.warning(
f'Runner {descriptor.id} returned non-positive result sequence '
f'{sequence_int} for run {run_id}'
f'Runner {descriptor.id} returned invalid result sequence: {sequence}'
)
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
elif last_sequence and sequence_int != last_sequence + 1:
self.ap.logger.warning(
f'Runner {descriptor.id} result sequence gap or out-of-order '
f'for run {run_id}: previous={last_sequence}, current={sequence_int}'
)
else:
if sequence_int in seen_sequences:
self.ap.logger.warning(
f'Runner {descriptor.id} returned duplicate result sequence '
f'{sequence_int} for run {run_id}; dropping duplicate'
)
continue
if sequence_int <= 0:
self.ap.logger.warning(
f'Runner {descriptor.id} returned non-positive result sequence '
f'{sequence_int} for run {run_id}'
)
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
elif last_sequence and sequence_int != last_sequence + 1:
self.ap.logger.warning(
f'Runner {descriptor.id} result sequence gap or out-of-order '
f'for run {run_id}: previous={last_sequence}, current={sequence_int}'
)
seen_sequences.add(sequence_int)
last_sequence = max(last_sequence, sequence_int)
else:
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
seen_sequences.add(sequence_int)
last_sequence = max(last_sequence, sequence_int)
else:
sequence_int = last_sequence + 1
result_dict['sequence'] = sequence_int
seen_sequences.add(sequence_int)
last_sequence = sequence_int
last_sequence = sequence_int
result_type = result_dict.get('type')
if result_type and not self.result_normalizer.validate_payload(
result_type,
result_dict.get('data', {}),
descriptor,
):
continue
await self.journal.append_run_result(
result_dict=result_dict,
run_id=run_id,
sequence=sequence_int,
)
# Trusted Host observers receive validated events before message-only normalization.
result_observer = (adapter_context or {}).get('_result_observer')
if result_observer is not None:
await result_observer(result_dict)
if result_type == 'state.updated':
await self.journal.handle_state_updated_event(
result_dict,
event,
binding,
result_type = result_dict.get('type')
if result_type and not self.result_normalizer.validate_payload(
result_type,
result_dict.get('data', {}),
descriptor,
run_id=run_id,
)
await self.result_normalizer.normalize(result_dict, descriptor)
continue
if result_type == 'action.requested':
if await self.interaction_manager.handle_result(
result_dict=result_dict,
event=event,
binding=binding,
descriptor=descriptor,
run_id=run_id,
adapter_context=adapter_context,
):
continue
if result_type == 'run.completed':
terminal_status = 'completed'
terminal_reason = (
result_dict.get('data', {}).get('finish_reason')
if isinstance(result_dict.get('data'), dict)
else None
)
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
elif result_type == 'run.failed':
terminal_status = 'failed'
data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {}
terminal_reason = data.get('error') or data.get('code')
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
has_completed_message = result_type == 'message.completed' or (
result_type == 'run.completed'
and isinstance(result_dict.get('data'), dict)
and bool(result_dict['data'].get('message'))
)
if has_completed_message and event.conversation_id and not assistant_transcript_written:
await self.journal.write_assistant_transcript(
await self.journal.append_run_result(
result_dict=result_dict,
event=event,
run_id=run_id,
runner_id=descriptor.id,
sequence=sequence_int,
)
assistant_transcript_written = True
result = await self.result_normalizer.normalize(result_dict, descriptor)
if result is not None:
yield result
# Trusted Host observers receive validated events before message-only normalization.
result_observer = (adapter_context or {}).get('_result_observer')
if result_observer is not None:
await result_observer(result_dict)
run_snapshot = await self.journal.get_run(run_id)
if run_snapshot and run_snapshot.get('cancel_requested_at') is not None:
terminal_status = 'cancelled'
terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested'
break
if result_type == 'state.updated':
await self.journal.handle_state_updated_event(
result_dict,
event,
binding,
descriptor,
run_id=run_id,
)
await self.result_normalizer.normalize(result_dict, descriptor)
continue
if result_type == 'action.requested':
if await self.interaction_manager.handle_result(
result_dict=result_dict,
event=event,
binding=binding,
descriptor=descriptor,
run_id=run_id,
adapter_context=adapter_context,
):
continue
if result_type == 'run.completed':
terminal_status = 'completed'
terminal_reason = (
result_dict.get('data', {}).get('finish_reason')
if isinstance(result_dict.get('data'), dict)
else None
)
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
elif result_type == 'run.failed':
terminal_status = 'failed'
data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {}
terminal_reason = data.get('error') or data.get('code')
usage = result_dict.get('usage')
if isinstance(usage, dict):
terminal_usage = usage
has_completed_message = result_type == 'message.completed' or (
result_type == 'run.completed'
and isinstance(result_dict.get('data'), dict)
and bool(result_dict['data'].get('message'))
)
if has_completed_message and event.conversation_id and not assistant_transcript_written:
await self.journal.write_assistant_transcript(
result_dict=result_dict,
event=event,
run_id=run_id,
runner_id=descriptor.id,
)
assistant_transcript_written = True
result = await self.result_normalizer.normalize(result_dict, descriptor)
if result is not None:
yield result
run_snapshot = await self.journal.get_run(run_id)
if run_snapshot and run_snapshot.get('cancel_requested_at') is not None:
terminal_status = 'cancelled'
terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested'
break
diagnostics.set_outcome(
{'completed': 'succeeded', 'failed': 'failed', 'cancelled': 'cancelled'}.get(
terminal_status, 'succeeded'
),
reason_code='runner_failed' if terminal_status == 'failed' else '',
)
await self.journal.finalize_run(
run_id=run_id,
status=terminal_status or 'completed',
@@ -362,6 +377,7 @@ class AgentRunOrchestrator:
usage=terminal_usage,
)
except Exception as exc:
diagnostics.set_outcome('timeout' if self._is_deadline_exhausted(context) else 'failed')
failed_usage = terminal_usage
await self.journal.finalize_run(
run_id=run_id,
@@ -387,6 +403,7 @@ class AgentRunOrchestrator:
exc_info=True,
)
@diagnostics.observe('lifecycle', 'runner.query_prepare', source='pipeline', stage='prepare')
async def run_from_query(
self,
query: pipeline_query.Query,
@@ -403,13 +420,16 @@ class AgentRunOrchestrator:
# Materialize inbound attachments into sandbox before running
await self._materialize_inbound_attachments(query, plan.event)
async for result in self.run(
plan.event,
plan.binding,
bound_plugins=plan.bound_plugins,
adapter_context=adapter_context,
):
yield result
async with contextlib.aclosing(
self.run(
plan.event,
plan.binding,
bound_plugins=plan.bound_plugins,
adapter_context=adapter_context,
)
) as results:
async for result in results:
yield result
async def _materialize_inbound_attachments(
self,
src/langbot/pkg/agent/runner/platform_tools.py
+4
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from ...telemetry import diagnostics
import copy
import fnmatch
import typing
@@ -667,6 +669,7 @@ def resolve_platform_api_call(session, bot_uuid, action, params, context_tool=No
raise ValueError(f'Platform API {action} or its target is not authorized for this run')
@diagnostics.observe('api', 'host.platform_tool', source='agent')
async def execute_platform_tool(
ap: typing.Any,
execution_context: typing.Any,
@@ -686,6 +689,7 @@ async def execute_platform_tool(
normalized = _event_params(definition, context, normalized)
# This flag is frozen by the Host from the synthetic debug envelope, not tool arguments.
if delivery.get('surface') == 'webui' and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True:
diagnostics.annotate(source='webui_debug', attributes={'synthetic': True})
result = _execute_mock_platform_tool(definition, context, normalized)
if message_chain is not None:
result['parameters']['message'] = message_chain.model_dump(mode='json')
src/langbot/pkg/agent/runner/registry.py
+7 -1
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from ...telemetry import diagnostics
import typing
import asyncio
@@ -119,7 +121,7 @@ class RunnerRegistry:
typed_manifest = RunnerManifest.model_validate(manifest)
config_schema = [item.model_dump(mode='json') for item in typed_manifest.config_schema]
return RunnerDescriptor(
descriptor = RunnerDescriptor(
id=runner_id,
component_kind=typed_manifest.component_kind,
usages=typed_manifest.usages,
@@ -136,6 +138,10 @@ class RunnerRegistry:
permissions=typed_manifest.permissions,
raw_manifest=manifest,
)
manager = getattr(self.ap, 'diagnostics', None)
if isinstance(manager, diagnostics.DiagnosticsManager) and manager.enabled:
diagnostics.declare_runner(descriptor)
return descriptor
async def refresh(self, context: TenantContext) -> None:
"""Refresh runner cache.
src/langbot/pkg/agent/runner/reply_stream.py
+5
View File
@@ -7,6 +7,8 @@ from dataclasses import dataclass, field
from typing import Literal
from uuid import UUID, uuid4
from ...telemetry import diagnostics
import pydantic
from langbot_plugin.api.entities.builtin.platform import events, message
from langbot_plugin.api.entities.builtin.provider import message as provider_message
@@ -54,6 +56,9 @@ class ReplyStreamSession:
self._closed = False
self._active: set[asyncio.Task] = set()
@diagnostics.observe(
'delivery', 'reply_stream.apply', source='agent', fields=lambda b: {'attributes': {'stream': True}}
)
async def apply(self, request: ReplyStreamRequest) -> dict:
task = asyncio.create_task(self._apply(request))
self._active.add(task)
src/langbot/pkg/agent/runner/session_registry.py
+3
View File
@@ -2,6 +2,8 @@
from __future__ import annotations
from ...telemetry import diagnostics
import asyncio
import copy
import typing
@@ -170,6 +172,7 @@ class AgentRunSessionRegistry:
'query_id': query_id,
'execution_query': execution_query,
'reply_streams': reply_streams,
'_diagnostic_context': diagnostics.capture_context(),
'plugin_identity': plugin_identity,
'authorization': authorization,
'status': {