From f8123ead0a4d2c7e6f6898d968ccdab2ef9f0b76 Mon Sep 17 00:00:00 2001 From: dadachann <185672915+dadachann@users.noreply.github.com> Date: Tue, 15 Sep 2026 09:35:01 +0000 Subject: [PATCH] feat(telemetry): add isolated beta quality diagnostics and release identity --- .dockerignore | 6 + .github/workflows/build-dev-image.yaml | 2 + .github/workflows/build-docker-image.yml | 4 +- .../workflows/build-release-artifacts.yaml | 3 + .github/workflows/publish-to-pypi.yml | 1 + Dockerfile | 3 + pyproject.toml | 2 +- scripts/stamp_build_revision.py | 23 + src/langbot/_build_info.py | 3 + .../pkg/agent/runner/interaction_manager.py | 9 + src/langbot/pkg/agent/runner/orchestrator.py | 242 ++++----- .../pkg/agent/runner/platform_tools.py | 4 + src/langbot/pkg/agent/runner/registry.py | 8 +- src/langbot/pkg/agent/runner/reply_stream.py | 5 + .../pkg/agent/runner/session_registry.py | 3 + src/langbot/pkg/api/http/controller/group.py | 12 +- .../controller/groups/agent_debug_stream.py | 4 + .../http/controller/groups/pipelines/embed.py | 58 ++- .../groups/pipelines/websocket_chat.py | 58 ++- src/langbot/pkg/api/http/service/agent.py | 3 + src/langbot/pkg/api/management_diagnostics.py | 225 +++++++++ src/langbot/pkg/api/mcp/mount.py | 21 +- src/langbot/pkg/api/mcp/server.py | 13 +- src/langbot/pkg/box/connector.py | 3 + src/langbot/pkg/core/app.py | 4 + src/langbot/pkg/core/boot.py | 29 +- src/langbot/pkg/persistence/mgr.py | 4 + src/langbot/pkg/pipeline/pipelinemgr.py | 8 + src/langbot/pkg/pipeline/pool.py | 3 + .../platform/adapters/aiocqhttp/adapter.py | 5 + .../platform/adapters/aiocqhttp/api_impl.py | 22 + .../adapters/aiocqhttp/event_converter.py | 4 + .../pkg/platform/adapters/dingtalk/adapter.py | 8 + .../platform/adapters/dingtalk/api_impl.py | 11 + .../adapters/dingtalk/event_converter.py | 4 + .../pkg/platform/adapters/discord/adapter.py | 148 ++++++ .../pkg/platform/adapters/discord/api_impl.py | 15 + .../adapters/discord/event_converter.py | 4 + .../pkg/platform/adapters/kook/adapter.py | 8 +- .../pkg/platform/adapters/kook/api_impl.py | 20 + .../platform/adapters/kook/event_converter.py | 4 + .../pkg/platform/adapters/lark/adapter.py | 18 + .../pkg/platform/adapters/lark/api_impl.py | 7 + .../platform/adapters/lark/event_converter.py | 4 + .../adapters/officialaccount/adapter.py | 6 + .../adapters/officialaccount/api_impl.py | 14 + .../officialaccount/event_converter.py | 4 + .../platform/adapters/qqofficial/adapter.py | 8 + .../platform/adapters/qqofficial/api_impl.py | 17 + .../adapters/qqofficial/event_converter.py | 4 + .../pkg/platform/adapters/slack/adapter.py | 6 + .../pkg/platform/adapters/slack/api_impl.py | 14 + .../adapters/slack/event_converter.py | 4 + .../pkg/platform/adapters/telegram/adapter.py | 17 + .../platform/adapters/telegram/api_impl.py | 15 + .../adapters/telegram/event_converter.py | 4 + .../pkg/platform/adapters/wecom/adapter.py | 6 + .../pkg/platform/adapters/wecom/api_impl.py | 12 + .../adapters/wecom/event_converter.py | 4 + .../pkg/platform/adapters/wecombot/adapter.py | 8 + .../platform/adapters/wecombot/api_impl.py | 17 + .../adapters/wecombot/event_converter.py | 4 + .../pkg/platform/adapters/wecomcs/adapter.py | 6 + .../pkg/platform/adapters/wecomcs/api_impl.py | 12 + .../adapters/wecomcs/event_converter.py | 4 + src/langbot/pkg/platform/botmgr.py | 38 ++ src/langbot/pkg/plugin/agent_run_support.py | 3 + src/langbot/pkg/plugin/connector.py | 7 + src/langbot/pkg/plugin/handler.py | 16 +- .../pkg/provider/modelmgr/requester.py | 6 + src/langbot/pkg/provider/tools/toolmgr.py | 3 + src/langbot/pkg/rag/service/runtime.py | 7 + .../pkg/telemetry/diagnostic_catalog.py | 83 ++++ .../pkg/telemetry/diagnostic_privacy.py | 113 +++++ .../pkg/telemetry/diagnostic_transport.py | 399 +++++++++++++++ src/langbot/pkg/telemetry/diagnostics.py | 461 ++++++++++++++++++ src/langbot/templates/config.yaml | 4 + .../agent/test_orchestrator_integration.py | 53 ++ .../api/test_diagnostics_management_http.py | 194 ++++++++ .../test_diagnostics_management_inventory.py | 47 ++ .../test_diagnostics_management_isolation.py | 431 ++++++++++++++++ .../test_diagnostics_management_lifecycle.py | 163 +++++++ .../test_diagnostics_management_transports.py | 173 +++++++ .../telemetry/test_beta_diagnostic_hooks.py | 143 ++++++ ...test_beta_diagnostic_review_regressions.py | 405 +++++++++++++++ .../telemetry/test_beta_diagnostics.py | 350 +++++++++++++ tests/unit_tests/test_build_revision_stamp.py | 30 ++ uv.lock | 2 +- .../unit/processor-detail-workbench.test.mjs | 18 +- 89 files changed, 4205 insertions(+), 175 deletions(-) create mode 100644 scripts/stamp_build_revision.py create mode 100644 src/langbot/_build_info.py create mode 100644 src/langbot/pkg/api/management_diagnostics.py create mode 100644 src/langbot/pkg/telemetry/diagnostic_catalog.py create mode 100644 src/langbot/pkg/telemetry/diagnostic_privacy.py create mode 100644 src/langbot/pkg/telemetry/diagnostic_transport.py create mode 100644 src/langbot/pkg/telemetry/diagnostics.py create mode 100644 tests/unit_tests/api/test_diagnostics_management_http.py create mode 100644 tests/unit_tests/api/test_diagnostics_management_inventory.py create mode 100644 tests/unit_tests/api/test_diagnostics_management_isolation.py create mode 100644 tests/unit_tests/api/test_diagnostics_management_lifecycle.py create mode 100644 tests/unit_tests/api/test_diagnostics_management_transports.py create mode 100644 tests/unit_tests/telemetry/test_beta_diagnostic_hooks.py create mode 100644 tests/unit_tests/telemetry/test_beta_diagnostic_review_regressions.py create mode 100644 tests/unit_tests/telemetry/test_beta_diagnostics.py create mode 100644 tests/unit_tests/test_build_revision_stamp.py diff --git a/.dockerignore b/.dockerignore index d18057c9b..24924a4bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,10 @@ .github +.git +**/.git +__pycache__ +**/__pycache__ +*.pyc +**/*.pyc .venv .vscode .data diff --git a/.github/workflows/build-dev-image.yaml b/.github/workflows/build-dev-image.yaml index 623a9bf99..ad7bc310b 100644 --- a/.github/workflows/build-dev-image.yaml +++ b/.github/workflows/build-dev-image.yaml @@ -40,6 +40,8 @@ jobs: with: context: . push: true + build-args: | + LANGBOT_BUILD_REVISION=${{ github.sha }} tags: | rockchin/langbot:${{ steps.image.outputs.branch_tag }} rockchin/langbot:${{ steps.image.outputs.sha_tag }} diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 4575c4fba..9ef4506b8 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -41,7 +41,7 @@ jobs: run: docker buildx create --name mybuilder --use - name: Build for Release # only relase, exlude pre-release if: ${{ github.event.release.prerelease == false }} - run: docker buildx build --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} -t rockchin/langbot:latest . --push + run: docker buildx build --build-arg LANGBOT_BUILD_REVISION=${{ github.sha }} --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} -t rockchin/langbot:latest . --push - name: Build for Pre-release # no update for latest tag if: ${{ github.event.release.prerelease == true }} - run: docker buildx build --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} . --push \ No newline at end of file + run: docker buildx build --build-arg LANGBOT_BUILD_REVISION=${{ github.sha }} --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} . --push \ No newline at end of file diff --git a/.github/workflows/build-release-artifacts.yaml b/.github/workflows/build-release-artifacts.yaml index ed36fae92..5f92a3c64 100644 --- a/.github/workflows/build-release-artifacts.yaml +++ b/.github/workflows/build-release-artifacts.yaml @@ -31,6 +31,9 @@ jobs: echo ::set-output name=version::${GITHUB_REF} fi + - name: Stamp build revision + run: python3 scripts/stamp_build_revision.py + - name: Make Temp Directory run: | mkdir -p /tmp/langbot_build_web diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 9db2b57d6..57f12cfd7 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -70,6 +70,7 @@ jobs: - name: Build package run: | + python3 scripts/stamp_build_revision.py uv build - name: Publish to PyPI diff --git a/Dockerfile b/Dockerfile index 1d1351c72..3768a5802 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,9 @@ WORKDIR /app COPY . . +ARG LANGBOT_BUILD_REVISION +RUN if [ -n "$LANGBOT_BUILD_REVISION" ]; then python3 scripts/stamp_build_revision.py --revision "$LANGBOT_BUILD_REVISION"; fi + COPY --from=node /app/web/dist ./web/dist # nsjail binary built in the dedicated stage above. Self-contained sandbox diff --git a/pyproject.toml b/pyproject.toml index e4dbbe73a..4d85e7b51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "langbot" -version = "4.11.0-beta.1" +version = "4.11.0-beta.2" description = "Production-grade platform for building agentic IM bots" readme = "README.md" license-files = ["LICENSE"] diff --git a/scripts/stamp_build_revision.py b/scripts/stamp_build_revision.py new file mode 100644 index 000000000..1db6c2969 --- /dev/null +++ b/scripts/stamp_build_revision.py @@ -0,0 +1,23 @@ +"""Stamp the exact source revision into distributable artifacts (build time only).""" + +import argparse +from pathlib import Path +import re +import subprocess + + +def stamp(root: Path, revision: str | None = None) -> str: + if revision is None: + revision = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD'], cwd=root, text=True).strip() + if not re.fullmatch(r'[0-9a-f]{40}', revision): + raise ValueError('Build revision must be a full lowercase Git SHA') + target = root / 'src/langbot/_build_info.py' + target.write_text(f'"""Exact source revision stamped at artifact build time."""\n\nCORE_REVISION = {revision!r}\n') + return revision + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--revision') + args = parser.parse_args() + print(stamp(Path(__file__).resolve().parents[1], args.revision)) diff --git a/src/langbot/_build_info.py b/src/langbot/_build_info.py new file mode 100644 index 000000000..767661706 --- /dev/null +++ b/src/langbot/_build_info.py @@ -0,0 +1,3 @@ +"""Build identity; release workflows replace the revision before packaging.""" + +CORE_REVISION = '' diff --git a/src/langbot/pkg/agent/runner/interaction_manager.py b/src/langbot/pkg/agent/runner/interaction_manager.py index 0578f12cb..85f19a3e3 100644 --- a/src/langbot/pkg/agent/runner/interaction_manager.py +++ b/src/langbot/pkg/agent/runner/interaction_manager.py @@ -2,6 +2,8 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import json import time import typing @@ -91,6 +93,7 @@ class InteractionManager: self.ap = ap self.store = store or InteractionStore(ap.persistence_mgr.get_db_engine()) + @diagnostics.observe('interaction', 'interaction.request', source='agent', stage='execute') async def handle_result( self, *, @@ -198,14 +201,17 @@ class InteractionManager: except Exception as exc: await self.store.mark_delivery_failed(run_id, request.interaction_id, str(exc)) raise + diagnostics.set_outcome('waiting', reason_code='waiting') return True + @diagnostics.observe('interaction', 'interaction.acknowledge', source='platform', stage='ack') async def acknowledge_submission(self, record: dict[str, typing.Any], adapter: typing.Any) -> None: """Best-effort transition of submitted controls into a read-only state.""" delivery_result = record.get('delivery_result') if not isinstance(delivery_result, dict) or not self._supports_platform_api( adapter, INTERACTION_ACKNOWLEDGE_API ): + diagnostics.set_outcome('skipped') return try: delivery_result = await adapter.call_platform_api( @@ -223,6 +229,8 @@ class InteractionManager: delivery_result, ) except Exception as exc: + diagnostics.set_outcome('failed', reason_code='response_error') + diagnostics.annotate(error=exc) self._warning(f'Failed to acknowledge interaction submission: {exc}') async def _find_update_target( @@ -273,6 +281,7 @@ class InteractionManager: f'interaction field {field.id} has duplicate option values', ) + @diagnostics.observe('interaction', 'interaction.consume', source='platform', stage='dispatch') async def consume_callback( self, *, diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index 3b182f4cb..76348c25d 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +import contextlib import typing from langbot_plugin.api.entities.builtin.provider import message as provider_message @@ -10,6 +11,7 @@ from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query from langbot_plugin.entities.io.actions.enums import PluginToRuntimeAction +from ...telemetry import diagnostics as diagnostics from .reply_stream import ReplyStreamSession from ...core import app from ...api.http.context import ExecutionContext @@ -79,6 +81,7 @@ class AgentRunOrchestrator: self.journal = AgentRunJournal(ap) self._session_registry = get_session_registry() + @diagnostics.observe('run', 'runner.run', source='agent', stage='prepare') async def run( self, event: AgentEventEnvelope, @@ -105,6 +108,7 @@ class AgentRunOrchestrator: bound_plugins, ) + diagnostics.runner_metadata(self.ap, descriptor, binding.processor_type) usage = 'event' if binding.processor_type == 'event_processor' else 'agent' if usage not in descriptor.usages: raise ValueError(f'The selected Runner does not support {usage} usage') @@ -158,6 +162,7 @@ class AgentRunOrchestrator: state_context = build_state_context(event, binding, descriptor) run_id = context['run_id'] + diagnostics.annotate(run_id=run_id, stage='execute') context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any( tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform' for tool in resources.get('tools', []) @@ -168,6 +173,7 @@ class AgentRunOrchestrator: source=(adapter_context or {}).get('_platform_event') or getattr((adapter_context or {}).get('_query'), 'message_event', None), ) + reply_streams.diagnostics = getattr(self.ap, 'diagnostics', None) available_apis = context.get('context', {}).get('available_apis') run_authorization = { 'runner_id': descriptor.id, @@ -236,125 +242,134 @@ class AgentRunOrchestrator: event_log_id=event_log_id, ) - async for result_dict in self.invoker.invoke(descriptor, context): - result_dict = dict(result_dict) - sequence = result_dict.get('sequence') - if sequence is not None: - try: - sequence_int = int(sequence) - except (TypeError, ValueError): - self.ap.logger.warning(f'Runner {descriptor.id} returned invalid result sequence: {sequence}') - sequence_int = last_sequence + 1 - result_dict['sequence'] = sequence_int - else: - if sequence_int in seen_sequences: + async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results: + async for result_dict in results: + result_dict = dict(result_dict) + sequence = result_dict.get('sequence') + if sequence is not None: + try: + sequence_int = int(sequence) + except (TypeError, ValueError): self.ap.logger.warning( - f'Runner {descriptor.id} returned duplicate result sequence ' - f'{sequence_int} for run {run_id}; dropping duplicate' - ) - continue - if sequence_int <= 0: - self.ap.logger.warning( - f'Runner {descriptor.id} returned non-positive result sequence ' - f'{sequence_int} for run {run_id}' + f'Runner {descriptor.id} returned invalid result sequence: {sequence}' ) sequence_int = last_sequence + 1 result_dict['sequence'] = sequence_int - elif last_sequence and sequence_int != last_sequence + 1: - self.ap.logger.warning( - f'Runner {descriptor.id} result sequence gap or out-of-order ' - f'for run {run_id}: previous={last_sequence}, current={sequence_int}' - ) + else: + if sequence_int in seen_sequences: + self.ap.logger.warning( + f'Runner {descriptor.id} returned duplicate result sequence ' + f'{sequence_int} for run {run_id}; dropping duplicate' + ) + continue + if sequence_int <= 0: + self.ap.logger.warning( + f'Runner {descriptor.id} returned non-positive result sequence ' + f'{sequence_int} for run {run_id}' + ) + sequence_int = last_sequence + 1 + result_dict['sequence'] = sequence_int + elif last_sequence and sequence_int != last_sequence + 1: + self.ap.logger.warning( + f'Runner {descriptor.id} result sequence gap or out-of-order ' + f'for run {run_id}: previous={last_sequence}, current={sequence_int}' + ) + seen_sequences.add(sequence_int) + last_sequence = max(last_sequence, sequence_int) + else: + sequence_int = last_sequence + 1 + result_dict['sequence'] = sequence_int seen_sequences.add(sequence_int) - last_sequence = max(last_sequence, sequence_int) - else: - sequence_int = last_sequence + 1 - result_dict['sequence'] = sequence_int - seen_sequences.add(sequence_int) - last_sequence = sequence_int + last_sequence = sequence_int - result_type = result_dict.get('type') - if result_type and not self.result_normalizer.validate_payload( - result_type, - result_dict.get('data', {}), - descriptor, - ): - continue - - await self.journal.append_run_result( - result_dict=result_dict, - run_id=run_id, - sequence=sequence_int, - ) - - # Trusted Host observers receive validated events before message-only normalization. - result_observer = (adapter_context or {}).get('_result_observer') - if result_observer is not None: - await result_observer(result_dict) - - if result_type == 'state.updated': - await self.journal.handle_state_updated_event( - result_dict, - event, - binding, + result_type = result_dict.get('type') + if result_type and not self.result_normalizer.validate_payload( + result_type, + result_dict.get('data', {}), descriptor, - run_id=run_id, - ) - await self.result_normalizer.normalize(result_dict, descriptor) - continue - - if result_type == 'action.requested': - if await self.interaction_manager.handle_result( - result_dict=result_dict, - event=event, - binding=binding, - descriptor=descriptor, - run_id=run_id, - adapter_context=adapter_context, ): continue - if result_type == 'run.completed': - terminal_status = 'completed' - terminal_reason = ( - result_dict.get('data', {}).get('finish_reason') - if isinstance(result_dict.get('data'), dict) - else None - ) - usage = result_dict.get('usage') - if isinstance(usage, dict): - terminal_usage = usage - elif result_type == 'run.failed': - terminal_status = 'failed' - data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {} - terminal_reason = data.get('error') or data.get('code') - usage = result_dict.get('usage') - if isinstance(usage, dict): - terminal_usage = usage - - has_completed_message = result_type == 'message.completed' or ( - result_type == 'run.completed' - and isinstance(result_dict.get('data'), dict) - and bool(result_dict['data'].get('message')) - ) - if has_completed_message and event.conversation_id and not assistant_transcript_written: - await self.journal.write_assistant_transcript( + await self.journal.append_run_result( result_dict=result_dict, - event=event, run_id=run_id, - runner_id=descriptor.id, + sequence=sequence_int, ) - assistant_transcript_written = True - result = await self.result_normalizer.normalize(result_dict, descriptor) - if result is not None: - yield result + # Trusted Host observers receive validated events before message-only normalization. + result_observer = (adapter_context or {}).get('_result_observer') + if result_observer is not None: + await result_observer(result_dict) - run_snapshot = await self.journal.get_run(run_id) - if run_snapshot and run_snapshot.get('cancel_requested_at') is not None: - terminal_status = 'cancelled' - terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested' - break + if result_type == 'state.updated': + await self.journal.handle_state_updated_event( + result_dict, + event, + binding, + descriptor, + run_id=run_id, + ) + await self.result_normalizer.normalize(result_dict, descriptor) + continue + + if result_type == 'action.requested': + if await self.interaction_manager.handle_result( + result_dict=result_dict, + event=event, + binding=binding, + descriptor=descriptor, + run_id=run_id, + adapter_context=adapter_context, + ): + continue + + if result_type == 'run.completed': + terminal_status = 'completed' + terminal_reason = ( + result_dict.get('data', {}).get('finish_reason') + if isinstance(result_dict.get('data'), dict) + else None + ) + usage = result_dict.get('usage') + if isinstance(usage, dict): + terminal_usage = usage + elif result_type == 'run.failed': + terminal_status = 'failed' + data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {} + terminal_reason = data.get('error') or data.get('code') + usage = result_dict.get('usage') + if isinstance(usage, dict): + terminal_usage = usage + + has_completed_message = result_type == 'message.completed' or ( + result_type == 'run.completed' + and isinstance(result_dict.get('data'), dict) + and bool(result_dict['data'].get('message')) + ) + if has_completed_message and event.conversation_id and not assistant_transcript_written: + await self.journal.write_assistant_transcript( + result_dict=result_dict, + event=event, + run_id=run_id, + runner_id=descriptor.id, + ) + assistant_transcript_written = True + + result = await self.result_normalizer.normalize(result_dict, descriptor) + if result is not None: + yield result + + run_snapshot = await self.journal.get_run(run_id) + if run_snapshot and run_snapshot.get('cancel_requested_at') is not None: + terminal_status = 'cancelled' + terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested' + break + diagnostics.set_outcome( + {'completed': 'succeeded', 'failed': 'failed', 'cancelled': 'cancelled'}.get( + terminal_status, 'succeeded' + ), + reason_code='runner_failed' if terminal_status == 'failed' else '', + ) await self.journal.finalize_run( run_id=run_id, status=terminal_status or 'completed', @@ -362,6 +377,7 @@ class AgentRunOrchestrator: usage=terminal_usage, ) except Exception as exc: + diagnostics.set_outcome('timeout' if self._is_deadline_exhausted(context) else 'failed') failed_usage = terminal_usage await self.journal.finalize_run( run_id=run_id, @@ -387,6 +403,7 @@ class AgentRunOrchestrator: exc_info=True, ) + @diagnostics.observe('lifecycle', 'runner.query_prepare', source='pipeline', stage='prepare') async def run_from_query( self, query: pipeline_query.Query, @@ -403,13 +420,16 @@ class AgentRunOrchestrator: # Materialize inbound attachments into sandbox before running await self._materialize_inbound_attachments(query, plan.event) - async for result in self.run( - plan.event, - plan.binding, - bound_plugins=plan.bound_plugins, - adapter_context=adapter_context, - ): - yield result + async with contextlib.aclosing( + self.run( + plan.event, + plan.binding, + bound_plugins=plan.bound_plugins, + adapter_context=adapter_context, + ) + ) as results: + async for result in results: + yield result async def _materialize_inbound_attachments( self, diff --git a/src/langbot/pkg/agent/runner/platform_tools.py b/src/langbot/pkg/agent/runner/platform_tools.py index c3ba18458..eb7347a01 100644 --- a/src/langbot/pkg/agent/runner/platform_tools.py +++ b/src/langbot/pkg/agent/runner/platform_tools.py @@ -2,6 +2,8 @@ from __future__ import annotations +from ...telemetry import diagnostics + import copy import fnmatch import typing @@ -667,6 +669,7 @@ def resolve_platform_api_call(session, bot_uuid, action, params, context_tool=No raise ValueError(f'Platform API {action} or its target is not authorized for this run') +@diagnostics.observe('api', 'host.platform_tool', source='agent') async def execute_platform_tool( ap: typing.Any, execution_context: typing.Any, @@ -686,6 +689,7 @@ async def execute_platform_tool( normalized = _event_params(definition, context, normalized) # This flag is frozen by the Host from the synthetic debug envelope, not tool arguments. if delivery.get('surface') == 'webui' and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True: + diagnostics.annotate(source='webui_debug', attributes={'synthetic': True}) result = _execute_mock_platform_tool(definition, context, normalized) if message_chain is not None: result['parameters']['message'] = message_chain.model_dump(mode='json') diff --git a/src/langbot/pkg/agent/runner/registry.py b/src/langbot/pkg/agent/runner/registry.py index d00064688..91877e5e7 100644 --- a/src/langbot/pkg/agent/runner/registry.py +++ b/src/langbot/pkg/agent/runner/registry.py @@ -2,6 +2,8 @@ from __future__ import annotations +from ...telemetry import diagnostics + import typing import asyncio @@ -119,7 +121,7 @@ class RunnerRegistry: typed_manifest = RunnerManifest.model_validate(manifest) config_schema = [item.model_dump(mode='json') for item in typed_manifest.config_schema] - return RunnerDescriptor( + descriptor = RunnerDescriptor( id=runner_id, component_kind=typed_manifest.component_kind, usages=typed_manifest.usages, @@ -136,6 +138,10 @@ class RunnerRegistry: permissions=typed_manifest.permissions, raw_manifest=manifest, ) + manager = getattr(self.ap, 'diagnostics', None) + if isinstance(manager, diagnostics.DiagnosticsManager) and manager.enabled: + diagnostics.declare_runner(descriptor) + return descriptor async def refresh(self, context: TenantContext) -> None: """Refresh runner cache. diff --git a/src/langbot/pkg/agent/runner/reply_stream.py b/src/langbot/pkg/agent/runner/reply_stream.py index aa2f3500f..789038ff3 100644 --- a/src/langbot/pkg/agent/runner/reply_stream.py +++ b/src/langbot/pkg/agent/runner/reply_stream.py @@ -7,6 +7,8 @@ from dataclasses import dataclass, field from typing import Literal from uuid import UUID, uuid4 +from ...telemetry import diagnostics + import pydantic from langbot_plugin.api.entities.builtin.platform import events, message from langbot_plugin.api.entities.builtin.provider import message as provider_message @@ -54,6 +56,9 @@ class ReplyStreamSession: self._closed = False self._active: set[asyncio.Task] = set() + @diagnostics.observe( + 'delivery', 'reply_stream.apply', source='agent', fields=lambda b: {'attributes': {'stream': True}} + ) async def apply(self, request: ReplyStreamRequest) -> dict: task = asyncio.create_task(self._apply(request)) self._active.add(task) diff --git a/src/langbot/pkg/agent/runner/session_registry.py b/src/langbot/pkg/agent/runner/session_registry.py index e7c12716b..d7e2910d8 100644 --- a/src/langbot/pkg/agent/runner/session_registry.py +++ b/src/langbot/pkg/agent/runner/session_registry.py @@ -2,6 +2,8 @@ from __future__ import annotations +from ...telemetry import diagnostics + import asyncio import copy import typing @@ -170,6 +172,7 @@ class AgentRunSessionRegistry: 'query_id': query_id, 'execution_query': execution_query, 'reply_streams': reply_streams, + '_diagnostic_context': diagnostics.capture_context(), 'plugin_identity': plugin_identity, 'authorization': authorization, 'status': { diff --git a/src/langbot/pkg/api/http/controller/group.py b/src/langbot/pkg/api/http/controller/group.py index 6459ecec7..73a31bb5b 100644 --- a/src/langbot/pkg/api/http/controller/group.py +++ b/src/langbot/pkg/api/http/controller/group.py @@ -26,6 +26,7 @@ from ..authz import ( ) from ..context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext from ....cloud.support_admin import SupportAdminSessionError +from ... import management_diagnostics as diagnostics if typing.TYPE_CHECKING: from ....core.app import Application @@ -223,6 +224,7 @@ class RouterGroup(abc.ABC): try: if request_context is not None: + diagnostics.workspace(request_context) with bounded_executor.blocking_work_scope(request_context.workspace_uuid): persistence_mgr = getattr( self.ap, @@ -274,7 +276,14 @@ class RouterGroup(abc.ABC): ) return self.internal_error_response(request_id) - new_f = handler_error + # Observe outside authentication, using the registered Core handler + # identity rather than the URL (which can contain user identifiers). + new_f = diagnostics.observe( + diagnostics.operation_id('http', f, rule=rule, methods=options.get('methods')), + source='http', + ap=self.ap, + http=True, + )(handler_error) # Quart/Flask requires a unique endpoint name even when the same URL # intentionally has separate handlers for different HTTP methods. # Include the method set so CRUD routes can declare distinct @@ -561,6 +570,7 @@ class RouterGroup(abc.ABC): def fail(self, code: int | str, msg: str) -> quart.Response: """Return an error response""" + diagnostics.outcome('failed') return quart.jsonify( { 'code': code, diff --git a/src/langbot/pkg/api/http/controller/groups/agent_debug_stream.py b/src/langbot/pkg/api/http/controller/groups/agent_debug_stream.py index 295293c1a..272f7bd92 100644 --- a/src/langbot/pkg/api/http/controller/groups/agent_debug_stream.py +++ b/src/langbot/pkg/api/http/controller/groups/agent_debug_stream.py @@ -7,6 +7,7 @@ import contextlib import json import quart +from .... import management_diagnostics as diagnostics from .....agent.runner.errors import ( RunnerError, @@ -29,6 +30,9 @@ def debug_stream_response(service, context, agent_uuid: str, payload: dict) -> q result = await service.debug_agent(context, agent_uuid, payload, on_result=on_result) await queue.put({'kind': 'completed', 'data': result}) except Exception as exc: + # The stream still uses HTTP 200 when execution returns an + # error frame. Mark the inherited request, not its contents. + diagnostics.outcome('failed') if isinstance(exc, RunnerExecutionError): code, message = exc.error_code or 'runner_execution_failed', exc.message elif isinstance(exc, RunnerNotFoundError): diff --git a/src/langbot/pkg/api/http/controller/groups/pipelines/embed.py b/src/langbot/pkg/api/http/controller/groups/pipelines/embed.py index c00fb129e..7f37e8069 100644 --- a/src/langbot/pkg/api/http/controller/groups/pipelines/embed.py +++ b/src/langbot/pkg/api/http/controller/groups/pipelines/embed.py @@ -21,6 +21,7 @@ import httpx import quart from ... import group +from ..... import management_diagnostics as diagnostics from ......utils import httpclient, paths from ......platform.sources.websocket_manager import WebSocketScope, is_valid_session_id, ws_connection_manager from .websocket_chat import create_scoped_duplex_tasks, wait_for_duplex_tasks @@ -327,20 +328,24 @@ class EmbedRouterGroup(group.RouterGroup): # -- Embed WebSocket endpoint ---------------------------------------- @self.quart_app.websocket(self.path + '//ws/connect') + @diagnostics.observe('websocket.embed.session', source='websocket', ap=self.ap) async def embed_websocket_connect(bot_uuid: str): """WebSocket connection for embed widget, keyed by bot_uuid.""" await quart.websocket.accept() if not _is_valid_uuid(bot_uuid): await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'})) + diagnostics.outcome('rejected') return runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid) if runtime_bot is None: await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'})) + diagnostics.outcome('rejected') return session_type = quart.websocket.args.get('session_type', 'person') if session_type not in ['person', 'group']: + diagnostics.outcome('rejected') await quart.websocket.send( json.dumps({'type': 'error', 'message': 'session_type must be person or group'}) ) @@ -349,6 +354,7 @@ class EmbedRouterGroup(group.RouterGroup): session_id = quart.websocket.args.get('session_id', '') if not is_valid_session_id(session_id): await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Valid session_id is required'})) + diagnostics.outcome('rejected') return try: @@ -356,13 +362,16 @@ class EmbedRouterGroup(group.RouterGroup): await self._assert_execution_active(runtime_bot) except Exception: await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'})) + diagnostics.outcome('rejected') return + diagnostics.workspace(runtime_bot.execution_context) try: proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context) websocket_adapter = proxy_bot.adapter if not websocket_adapter: await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'})) + diagnostics.outcome('rejected') return connection = await ws_connection_manager.add_connection( @@ -420,11 +429,13 @@ class EmbedRouterGroup(group.RouterGroup): try: await wait_for_duplex_tasks(receive_task, send_task) except Exception as e: + diagnostics.outcome('failed') logger.error(f'Embed WebSocket task error: {e}') finally: await ws_connection_manager.remove_connection(connection.connection_id) except Exception as e: + diagnostics.outcome('failed') logger.error(f'Embed WebSocket connection error: {e}', exc_info=True) try: await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'})) @@ -439,28 +450,35 @@ class EmbedRouterGroup(group.RouterGroup): message = await quart.websocket.receive() await ws_connection_manager.update_activity(connection.connection_id) - try: - data = await asyncio.to_thread(json.loads, message) - message_type = data.get('type', 'message') + with diagnostics.scope(self.ap, 'websocket.embed.message', source='websocket'): + diagnostics.workspace(getattr(owner_bot, 'execution_context', None)) + try: + data = await asyncio.to_thread(json.loads, message) + message_type = data.get('type', 'message') - if message_type == 'ping': - await connection.send_queue.put( - {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()} - ) - elif message_type == 'message': - try: - current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid) - except Exception: - await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'}) + if message_type == 'ping': + await connection.send_queue.put( + {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()} + ) + elif message_type == 'message': + try: + current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid) + except Exception: + diagnostics.outcome('rejected') + await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'}) + break + await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot) + elif message_type == 'disconnect': break - await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot) - elif message_type == 'disconnect': - break + else: + diagnostics.outcome('skipped') - except json.JSONDecodeError: - await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'}) + except json.JSONDecodeError: + diagnostics.outcome('rejected') + await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'}) except Exception as e: + diagnostics.outcome('failed') logger.error(f'Embed receive error: {e}', exc_info=True) finally: connection.is_active = False @@ -476,11 +494,13 @@ class EmbedRouterGroup(group.RouterGroup): message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0) if message is None: break - encoded = await asyncio.to_thread(json.dumps, message) - await quart.websocket.send(encoded) + with diagnostics.scope(self.ap, 'websocket.embed.send', source='websocket'): + encoded = await asyncio.to_thread(json.dumps, message) + await quart.websocket.send(encoded) except asyncio.TimeoutError: continue except Exception as e: + diagnostics.outcome('failed') logger.error(f'Embed send error: {e}', exc_info=True) finally: connection.is_active = False diff --git a/src/langbot/pkg/api/http/controller/groups/pipelines/websocket_chat.py b/src/langbot/pkg/api/http/controller/groups/pipelines/websocket_chat.py index 056217348..cc9582719 100644 --- a/src/langbot/pkg/api/http/controller/groups/pipelines/websocket_chat.py +++ b/src/langbot/pkg/api/http/controller/groups/pipelines/websocket_chat.py @@ -14,6 +14,7 @@ import quart from ....authz import Permission, permissions_for_role, require_permission from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext from ... import group +from ..... import management_diagnostics as diagnostics from ......core.task_boundary import run_in_workspace_uow from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager from ......utils import bounded_executor @@ -210,6 +211,7 @@ class WebSocketChatRouterGroup(group.RouterGroup): async def initialize(self) -> None: @self.quart_app.websocket(self.path + '/connect') + @diagnostics.observe('websocket.dashboard.session', source='websocket', ap=self.ap) async def websocket_connect(pipeline_uuid: str): """Open one authenticated dashboard debug connection.""" @@ -217,11 +219,14 @@ class WebSocketChatRouterGroup(group.RouterGroup): try: request_context, token = await self._authenticate_websocket() except Exception: + diagnostics.outcome('rejected') await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'})) return + diagnostics.workspace(request_context) session_type = quart.websocket.args.get('session_type', 'person') if session_type not in ['person', 'group']: + diagnostics.outcome('rejected') await quart.websocket.send( json.dumps({'type': 'error', 'message': 'session_type must be person or group'}) ) @@ -230,6 +235,7 @@ class WebSocketChatRouterGroup(group.RouterGroup): try: websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid) if websocket_adapter is None: + diagnostics.outcome('rejected') await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Pipeline not found'})) return @@ -288,11 +294,13 @@ class WebSocketChatRouterGroup(group.RouterGroup): try: await wait_for_duplex_tasks(receive_task, send_task) except Exception as exc: + diagnostics.outcome('failed') logger.error(f'WebSocket task execution error: {exc}') finally: await ws_connection_manager.remove_connection(connection.connection_id) except Exception: + diagnostics.outcome('failed') logger.error('Dashboard WebSocket connection error', exc_info=True) try: await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'})) @@ -410,28 +418,34 @@ class WebSocketChatRouterGroup(group.RouterGroup): message = await quart.websocket.receive() await ws_connection_manager.update_activity(connection.connection_id) - try: - data = await asyncio.to_thread(json.loads, message) - message_type = data.get('type', 'message') - if message_type == 'ping': - await connection.send_queue.put( - {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()} - ) - elif message_type == 'message': - try: - request_context = await self._revalidate_websocket_authorization(request_context, token) - except Exception: - await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'}) + with diagnostics.scope(self.ap, 'websocket.dashboard.message', source='websocket'): + diagnostics.workspace(request_context) + try: + data = await asyncio.to_thread(json.loads, message) + message_type = data.get('type', 'message') + if message_type == 'ping': + await connection.send_queue.put( + {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()} + ) + elif message_type == 'message': + try: + request_context = await self._revalidate_websocket_authorization(request_context, token) + except Exception: + diagnostics.outcome('rejected') + await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'}) + break + await websocket_adapter.handle_websocket_message(connection, data) + elif message_type == 'disconnect': break - await websocket_adapter.handle_websocket_message(connection, data) - elif message_type == 'disconnect': - break - else: - logger.warning(f'Unknown WebSocket message type: {message_type}') - except json.JSONDecodeError: - await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'}) + else: + diagnostics.outcome('skipped') + logger.warning(f'Unknown WebSocket message type: {message_type}') + except json.JSONDecodeError: + diagnostics.outcome('rejected') + await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'}) except Exception: + diagnostics.outcome('failed') logger.error('Dashboard WebSocket receive error', exc_info=True) finally: connection.is_active = False @@ -447,11 +461,13 @@ class WebSocketChatRouterGroup(group.RouterGroup): message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0) if message is None: break - encoded = await asyncio.to_thread(json.dumps, message) - await quart.websocket.send(encoded) + with diagnostics.scope(self.ap, 'websocket.dashboard.send', source='websocket'): + encoded = await asyncio.to_thread(json.dumps, message) + await quart.websocket.send(encoded) except asyncio.TimeoutError: continue except Exception: + diagnostics.outcome('failed') logger.error('Dashboard WebSocket send error', exc_info=True) finally: connection.is_active = False diff --git a/src/langbot/pkg/api/http/service/agent.py b/src/langbot/pkg/api/http/service/agent.py index 36abd6645..4c90177c8 100644 --- a/src/langbot/pkg/api/http/service/agent.py +++ b/src/langbot/pkg/api/http/service/agent.py @@ -6,6 +6,7 @@ import fnmatch import time import uuid import typing +from ... import management_diagnostics as diagnostics import sqlalchemy from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext @@ -130,6 +131,7 @@ class AgentService: return None + @diagnostics.observe('http.agent.debug_agent', source='webui_debug') async def debug_agent( self, context: RequestContext, @@ -144,6 +146,7 @@ class AgentService: delivers outputs to a real platform, and supports both message and non-message event envelopes. """ + diagnostics.workspace(context) agent = await self.get_agent(context, agent_uuid) if agent is None or agent.get('kind') not in {AGENT_KIND_AGENT, AGENT_KIND_EVENT_PROCESSOR}: raise ValueError('Agent not found') diff --git a/src/langbot/pkg/api/management_diagnostics.py b/src/langbot/pkg/api/management_diagnostics.py new file mode 100644 index 000000000..a69f37366 --- /dev/null +++ b/src/langbot/pkg/api/management_diagnostics.py @@ -0,0 +1,225 @@ +"""Content-free management boundaries. Diagnostics never participate in I/O. + +Only source-code identities and status categories enter these helpers. Never +pass a URL, request, frame, token, tool argument, or serialized result to them. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import functools +import re + +import quart +from quart.wrappers.response import IterableBody, ResponseBody + +from ..telemetry import diagnostics as d +from ..telemetry import diagnostic_privacy as privacy + +_ACTIVE_BOUNDARY = contextvars.ContextVar('management_diagnostic_boundary', default=None) + + +def operation_id(source, fn, *, rule='', methods=()): + """Called at registration with a Core function, never a client tool name.""" + module = fn.__module__.split('.groups.', 1)[-1] + if source != 'http': + module = '' + name = fn.__name__ + if source == 'http' and name == '_': + # Many Core routes use the anonymous function name `_`. Disambiguate + # using ONLY the source-declared template and methods at registration. + # This is never quart.request.path, url_rule, endpoint or request.method. + template = re.sub(r'[^A-Za-z0-9_.:-]+', '.', rule).strip('.') or 'root' + name = '.'.join((*sorted(methods or ('GET',)), template)) + return '.'.join(part for part in (source, module, name) if part)[:128] + + +def outcome(value, reason_code='response_error'): + try: + d.set_outcome(value, reason_code=reason_code) + except Exception: + pass + + +def workspace(context): + """Annotate only this boundary's owned span with a matching trusted context.""" + try: + from .http.context import ExecutionContext, RequestContext + + if not isinstance(context, (ExecutionContext, RequestContext)): + return + boundary = _ACTIVE_BOUNDARY.get() + span = d.current_span() + if boundary is None or span is None or span is not boundary.span: + return + if d._manager(boundary.ap) is not span.manager: + return + instance = getattr( + getattr(boundary.ap, 'workspace_service', None), + 'instance_uuid', + getattr(span.manager, 'instance_id', None), + ) + # Structural recorders need not expose instance metadata. Real managers + # do, and must never receive another instance's Workspace annotation. + if instance is not None and instance != context.instance_uuid: + return + d.annotate(workspace_uuid=context.workspace_uuid) + except Exception: + pass + + +class Boundary: + def __init__(self, ap, operation, source): + self.ap = ap + self.span = None + try: + manager = d._manager(ap) + if manager is not None: + fields = {'source': source, 'stage': 'execute'} + if source == 'webui_debug': + fields['attributes'] = {'synthetic': True} + self.span = d.Span(manager, 'api', operation, fields) + except Exception: + pass + + @contextlib.contextmanager + def activate(self): + # None is an explicit inheritance barrier, not a no-op: disabled or + # broken B work must not borrow enabled A's span through ContextVars. + token = d._CURRENT.set(self.span) + boundary_token = _ACTIVE_BOUNDARY.set(self) + try: + yield + finally: + _ACTIVE_BOUNDARY.reset(boundary_token) + d._CURRENT.reset(token) + + def finish(self, error=None): + if self.span is not None: + try: + self.span.finish(error) + except Exception: + pass + + +class _ObservedBody(ResponseBody): + """Carry the HTTP parent through iteration, not generator suspension.""" + + def __init__(self, body, boundary): + self.body = body + self.boundary = boundary + self.iterator = None + self.exhausted = False + + async def __aenter__(self): + try: + with self.boundary.activate(): + entered = await self.body.__aenter__() + self.iterator = entered.__aiter__() + return self + except BaseException as exc: + self.boundary.finish(exc) + raise + + async def __aexit__(self, exc_type, exc_value, tb): + try: + with self.boundary.activate(): + result = await self.body.__aexit__(exc_type, exc_value, tb) + except BaseException as exc: + self.boundary.finish(exc) + raise + finally: + self.boundary.finish(exc_value if exc_value is not None else (None if self.exhausted else GeneratorExit())) + return result + + async def __aiter__(self): + try: + if self.iterator is None: + with self.boundary.activate(): + self.iterator = self.body.__aiter__() + while True: + try: + with self.boundary.activate(): + value = await anext(self.iterator) + except StopAsyncIteration: + self.exhausted = True + self.boundary.finish() + return + yield value + except BaseException as exc: + self.boundary.finish(exc) + raise + + +def _response_status(value): + """Inspect response metadata only; never read or deserialize a body.""" + response = value[0] if isinstance(value, tuple) else value + status = response.status_code if isinstance(response, quart.Response) else 200 + if isinstance(value, tuple) and len(value) > 1 and type(value[1]) is int: + status = value[1] + if status in (401, 403): + outcome('rejected') + elif status >= 400: + outcome('failed') + elif isinstance(response, dict) and 'code' in response and response['code'] != 0: + outcome('failed') + return response + + +@contextlib.contextmanager +def scope(ap, operation, *, source): + """A fixed operation around an existing non-generator statement block.""" + try: + privacy.code_value('operation', operation) + except Exception: + pass + boundary = Boundary(ap, operation, source) + try: + with boundary.activate(): + yield + except BaseException as exc: + boundary.finish(exc) + raise + finally: + boundary.finish() + + +def observe(operation, *, source, ap=None, http=False): + # The operation is supplied by Core registration code, not request data. + try: + privacy.code_value('operation', operation) + except Exception: + pass + + def decorator(fn): + @functools.wraps(fn) + async def wrapped(*args, **kwargs): + try: + owner = (ap() if callable(ap) else ap) if ap is not None else getattr(args[0], 'ap', None) + except Exception: + owner = None + boundary = Boundary(owner, operation, source) + streaming = False + try: + with boundary.activate(): + value = await fn(*args, **kwargs) + if http: + try: + response = _response_status(value) + if isinstance(response, quart.Response) and isinstance(response.response, IterableBody): + response.response = _ObservedBody(response.response, boundary) + streaming = True + except Exception: + pass + return value + except BaseException as exc: + boundary.finish(exc) + raise + finally: + if not streaming: + boundary.finish() + + return wrapped + + return decorator diff --git a/src/langbot/pkg/api/mcp/mount.py b/src/langbot/pkg/api/mcp/mount.py index 0856c1dab..6a07bd2e4 100644 --- a/src/langbot/pkg/api/mcp/mount.py +++ b/src/langbot/pkg/api/mcp/mount.py @@ -24,6 +24,7 @@ import uuid from ..http.context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext from .context import bind_request_context, reset_request_context from .server import LangBotMCPServer +from .. import management_diagnostics as diagnostics if typing.TYPE_CHECKING: from ...core import app as app_module @@ -85,13 +86,8 @@ class MCPMount: authenticate_api_key = self.ap.apikey_service.authenticate_api_key is_mcp_path = self._is_mcp_path - async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def] - # Pass through non-HTTP scopes (lifespan, websocket) to Quart so its - # own startup/shutdown and websocket routes keep working. - if scope['type'] != 'http' or not is_mcp_path(scope.get('path', '')): - await quart_asgi(scope, receive, send) - return - + @diagnostics.observe('mcp.request', source='mcp', ap=self.ap) + async def dispatch_mcp(scope, receive, send): # Authenticate MCP HTTP requests with a LangBot API key. api_key = _extract_api_key(scope.get('headers', [])) identity = None @@ -100,6 +96,7 @@ class MCPMount: identity = await authenticate_api_key(api_key) if identity is None: + diagnostics.outcome('rejected') await send( { 'type': 'http.response.start', @@ -126,6 +123,7 @@ class MCPMount: entitlement = await resolver.resolve(identity.workspace_uuid) entitlement_revision = entitlement.entitlement_revision except Exception: + diagnostics.outcome('rejected') await send( { 'type': 'http.response.start', @@ -153,6 +151,7 @@ class MCPMount: ), entitlement_revision=entitlement_revision, ) + diagnostics.workspace(request_context) tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None) if not callable(tenant_scope): raise RuntimeError('MCP request persistence scope is unavailable') @@ -165,4 +164,12 @@ class MCPMount: finally: reset_request_context(token) + async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def] + # Non-MCP traffic keeps its existing Quart boundaries. In + # particular, never derive an operation from an ASGI request path. + if scope['type'] != 'http' or not is_mcp_path(scope.get('path', '')): + await quart_asgi(scope, receive, send) + return + await dispatch_mcp(scope, receive, send) + return dispatcher diff --git a/src/langbot/pkg/api/mcp/server.py b/src/langbot/pkg/api/mcp/server.py index 3c282bd5c..88a6e17c7 100644 --- a/src/langbot/pkg/api/mcp/server.py +++ b/src/langbot/pkg/api/mcp/server.py @@ -24,6 +24,7 @@ from mcp.server.fastmcp import FastMCP from ..http.authz import Permission, require_permission from .context import get_request_context +from .. import management_diagnostics as diagnostics if typing.TYPE_CHECKING: from ...core import app as app_module @@ -52,6 +53,7 @@ def _dump(value: typing.Any) -> str: def _authorized(permission: Permission): context = get_request_context() require_permission(context, permission) + diagnostics.workspace(context) return context @@ -76,7 +78,7 @@ class LangBotMCPServer: # ------------------------------------------------------------------ # def _register_tools(self) -> None: ap = self.ap - mcp = self.mcp + mcp = self # ----- System (read-only) -------------------------------------- # @mcp.tool(description='Get basic LangBot system/runtime information (version, edition).') @@ -322,6 +324,15 @@ class LangBotMCPServer: context = _authorized(Permission.RESOURCE_VIEW) return _dump(await ap.skill_service.get_skill(context, skill_name)) + def tool(self, **options): + """Register a tool boundary before its authorization and service call.""" + + def register(fn): + observed = diagnostics.observe(diagnostics.operation_id('mcp', fn), source='mcp', ap=self.ap)(fn) + return self.mcp.tool(**options)(observed) + + return register + # ------------------------------------------------------------------ # # ASGI app # ------------------------------------------------------------------ # diff --git a/src/langbot/pkg/box/connector.py b/src/langbot/pkg/box/connector.py index 90a16deda..21bd97ae4 100644 --- a/src/langbot/pkg/box/connector.py +++ b/src/langbot/pkg/box/connector.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import contextlib import json @@ -162,6 +164,7 @@ class BoxRuntimeConnector(ManagedRuntimeConnector): def _uses_websocket(self) -> bool: return self.uses_websocket() + @diagnostics.observe('lifecycle', 'box.initialize', source='runtime', stage='execute') async def initialize(self) -> None: async with self._lifecycle_lock: if self._closing: diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 72f9ee03a..547179230 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -597,6 +597,10 @@ class Application: if self.telemetry is not None: with contextlib.suppress(Exception): await self.telemetry.shutdown() + diagnostics_manager = getattr(self, 'diagnostics', None) + if diagnostics_manager is not None: + with contextlib.suppress(Exception): + await diagnostics_manager.shutdown() if self.vector_db_mgr is not None: with contextlib.suppress(Exception): await self.vector_db_mgr.shutdown() diff --git a/src/langbot/pkg/core/boot.py b/src/langbot/pkg/core/boot.py index 1b15e9922..270610392 100644 --- a/src/langbot/pkg/core/boot.py +++ b/src/langbot/pkg/core/boot.py @@ -7,6 +7,7 @@ import os from . import app from . import stage +from ..telemetry import diagnostics from ..utils import constants, importutil # Import startup stage implementation to register @@ -39,7 +40,33 @@ async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application: stage_cls = stage.preregistered_stages[stage_name] stage_inst = stage_cls() - await stage_inst.run(ap) + if stage_name == 'GenKeysStage': + # Optional diagnostics must not make startup depend on package + # metadata, session markers, or its background transport. + ap.diagnostics = None + space_config = ap.instance_config.data.get('space', {}) + if not space_config.get('disable_telemetry', False) and not space_config.get( + 'disable_beta_diagnostics', False + ): + manager = None + try: + manager = diagnostics.DiagnosticsManager(ap, marker_path='data/labels/beta_diagnostics_session') + await manager.start_session() + manager.start() + ap.diagnostics = manager + except BaseException as exc: + if manager is not None: + # Cleanup faults cannot replace the startup fault. + try: + await manager.shutdown(drain_timeout=0) + except asyncio.CancelledError: + if not isinstance(exc, asyncio.CancelledError): + raise + except Exception: + pass + if not isinstance(exc, Exception): + raise + await diagnostics.observe('lifecycle', 'startup.' + stage_name, source='startup', ap=ap)(stage_inst.run)(ap) await ap.initialize() except BaseException: diff --git a/src/langbot/pkg/persistence/mgr.py b/src/langbot/pkg/persistence/mgr.py index f6ee713c8..2ff1c4bc6 100644 --- a/src/langbot/pkg/persistence/mgr.py +++ b/src/langbot/pkg/persistence/mgr.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import datetime import enum import sqlite3 @@ -153,6 +155,7 @@ class PersistenceManager: default=None, ) + @diagnostics.observe('lifecycle', 'persistence.initialize', source='startup', stage='execute') async def initialize(self): database_type = self.ap.instance_config.data.get('database', {}).get('use', 'sqlite') self.ap.logger.info(f'Initializing database type: {database_type}...') @@ -1648,6 +1651,7 @@ class PersistenceManager: # ================================= + @diagnostics.observe('lifecycle', 'persistence.run_alembic_migrations', source='startup', stage='execute') async def _run_alembic_migrations(self, target_revision: str = 'head'): """Run the supported Alembic-based 4.x migrations.""" from . import alembic_runner diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index 0f7b4d956..b069eea7e 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -1,5 +1,7 @@ from __future__ import annotations +from ..telemetry import diagnostics + import dataclasses import typing import traceback @@ -178,6 +180,7 @@ class RuntimePipeline: raise WorkspaceInvariantError('RuntimePipeline instance does not match the active Workspace binding') return execution_context + @diagnostics.observe('run', 'pipeline.run', source='pipeline', fields=lambda b: {'processor_type': 'pipeline'}) async def run(self, query: pipeline_query.Query): if ( query.instance_uuid != self.execution_context.instance_uuid @@ -254,6 +257,7 @@ class RuntimePipeline: if result.console_notice: self.ap.logger.info(result.console_notice) if result.error_notice: + diagnostics.set_outcome('failed', reason_code='response_error') self.ap.logger.error(result.error_notice) # Mark query as having error query.variables['_monitoring_has_error'] = True @@ -426,6 +430,7 @@ class RuntimePipeline: await self._assert_execution_active(query) if event_ctx.is_prevented_default(): + diagnostics.set_outcome('skipped', reason_code='discarded') self.ap.logger.debug( f'MessageReceived event prevented default for query {query.query_id}, pipeline={pipeline_name}' ) @@ -468,8 +473,11 @@ class RuntimePipeline: self.ap.logger.error(f'Failed to record query response: {e}') except WorkspaceError as e: + diagnostics.set_outcome('rejected', reason_code='processor_incompatible') self.ap.logger.info(f'Dropped query {query.query_id} because its Workspace execution binding is stale: {e}') except Exception as e: + diagnostics.set_outcome('failed', reason_code='response_error') + diagnostics.annotate(error=e) inst_name = query.current_stage_name if query.current_stage_name else 'unknown' self.ap.logger.error(f'Error processing query {query.query_id} stage={inst_name} : {e}') self.ap.logger.error(f'Traceback: {traceback.format_exc()}') diff --git a/src/langbot/pkg/pipeline/pool.py b/src/langbot/pkg/pipeline/pool.py index ebd3c96c2..08971d79c 100644 --- a/src/langbot/pkg/pipeline/pool.py +++ b/src/langbot/pkg/pipeline/pool.py @@ -1,5 +1,7 @@ from __future__ import annotations +from ..telemetry import diagnostics + import asyncio import dataclasses import inspect @@ -305,6 +307,7 @@ class QueryPool: ) object.__setattr__(query, 'query_uuid', query_uuid) object.__setattr__(query, '_execution_context', execution_context) + object.__setattr__(query, '_diagnostic_context', diagnostics.capture_context()) self.queries.append(query) self.cached_queries[(execution_context.workspace_uuid, query_uuid)] = query diff --git a/src/langbot/pkg/platform/adapters/aiocqhttp/adapter.py b/src/langbot/pkg/platform/adapters/aiocqhttp/adapter.py index 7d97cac81..952818dac 100644 --- a/src/langbot/pkg/platform/adapters/aiocqhttp/adapter.py +++ b/src/langbot/pkg/platform/adapters/aiocqhttp/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import traceback import typing @@ -94,6 +96,7 @@ class AiocqhttpAdapter(AiocqhttpAPIMixin, abstract_platform_adapter.AbstractPlat 'call_platform_api', ] + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: handler = PLATFORM_API_MAP.get(action) if handler is None: @@ -146,6 +149,7 @@ class AiocqhttpAdapter(AiocqhttpAPIMixin, abstract_platform_adapter.AbstractPlat await self.logger.info(f'WebSocket connection established, bot id: {self.bot_account_id}') await self._dispatch_native_event(event) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: aiocqhttp.Event): self.bot_account_id = str(getattr(event, 'self_id', '') or self.bot_account_id) if getattr(event, 'type', None) == 'message' and str(getattr(event, 'user_id', '')) == self.bot_account_id: @@ -163,6 +167,7 @@ class AiocqhttpAdapter(AiocqhttpAPIMixin, abstract_platform_adapter.AbstractPlat except Exception: await self.logger.error(f'Error in aiocqhttp native event: {traceback.format_exc()}') + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _dispatch_native_event(self, event: aiocqhttp.Event): eba_event = await self.event_converter.target2yiri(event, self.bot, self.bot_account_id, self._lookup) if eba_event: diff --git a/src/langbot/pkg/platform/adapters/aiocqhttp/api_impl.py b/src/langbot/pkg/platform/adapters/aiocqhttp/api_impl.py index f4c315909..75c4b49ba 100644 --- a/src/langbot/pkg/platform/adapters/aiocqhttp/api_impl.py +++ b/src/langbot/pkg/platform/adapters/aiocqhttp/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import aiocqhttp @@ -15,6 +17,7 @@ from langbot_plugin.api.entities.builtin.platform.errors import NotSupportedErro class AiocqhttpAPIMixin: bot: aiocqhttp.CQHttp + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -35,6 +38,7 @@ class AiocqhttpAPIMixin: raise ValueError(f'Unsupported aiocqhttp target_type: {target_type}') return platform_events.MessageResult(message_id=(raw or {}).get('message_id'), raw=raw or {}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -49,6 +53,7 @@ class AiocqhttpAPIMixin: raw = await self.bot.send(message_source.source_platform_object, aiocq_msg) return platform_events.MessageResult(message_id=(raw or {}).get('message_id'), raw=raw or {}) + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -57,6 +62,7 @@ class AiocqhttpAPIMixin: ) -> None: await self.bot.delete_msg(message_id=int(message_id)) + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -75,6 +81,7 @@ class AiocqhttpAPIMixin: raise ValueError(f'Unsupported aiocqhttp to_chat_type: {to_chat_type}') return platform_events.MessageResult(message_id=(raw or {}).get('message_id'), raw=raw or {}) + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -100,6 +107,7 @@ class AiocqhttpAPIMixin: ) return await AiocqhttpEventConverter.message_to_eba(event, self.bot) + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: raw = await self.bot.get_group_info(group_id=int(group_id)) return platform_entities.UserGroup( @@ -108,6 +116,7 @@ class AiocqhttpAPIMixin: member_count=raw.get('member_count'), ) + @diagnostics.observe('api', 'get_group_list', source='platform', stage='accepted') async def get_group_list(self) -> list[platform_entities.UserGroup]: raw_list = await self.bot.get_group_list() return [ @@ -119,6 +128,7 @@ class AiocqhttpAPIMixin: for item in raw_list ] + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], @@ -126,6 +136,7 @@ class AiocqhttpAPIMixin: raw_list = await self.bot.get_group_member_list(group_id=int(group_id)) return [self._member_to_entity(item, group_id) for item in raw_list] + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -134,9 +145,11 @@ class AiocqhttpAPIMixin: raw = await self.bot.get_group_member_info(group_id=int(group_id), user_id=int(user_id), no_cache=True) return self._member_to_entity(raw, group_id) + @diagnostics.observe('api', 'set_group_name', source='platform', stage='accepted') async def set_group_name(self, group_id: typing.Union[int, str], name: str) -> None: await self.bot.set_group_name(group_id=int(group_id), group_name=name) + @diagnostics.observe('api', 'mute_member', source='platform', stage='accepted') async def mute_member( self, group_id: typing.Union[int, str], @@ -145,15 +158,19 @@ class AiocqhttpAPIMixin: ) -> None: await self.bot.set_group_ban(group_id=int(group_id), user_id=int(user_id), duration=int(duration)) + @diagnostics.observe('api', 'unmute_member', source='platform', stage='accepted') async def unmute_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str]) -> None: await self.bot.set_group_ban(group_id=int(group_id), user_id=int(user_id), duration=0) + @diagnostics.observe('api', 'kick_member', source='platform', stage='accepted') async def kick_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str]) -> None: await self.bot.set_group_kick(group_id=int(group_id), user_id=int(user_id), reject_add_request=False) + @diagnostics.observe('api', 'leave_group', source='platform', stage='accepted') async def leave_group(self, group_id: typing.Union[int, str]) -> None: await self.bot.set_group_leave(group_id=int(group_id), is_dismiss=False) + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: raw = await self.bot.get_stranger_info(user_id=int(user_id), no_cache=True) return platform_entities.User( @@ -162,6 +179,7 @@ class AiocqhttpAPIMixin: avatar_url=raw.get('avatar_url'), ) + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: raw_list = await self.bot.get_friend_list() return [ @@ -173,6 +191,7 @@ class AiocqhttpAPIMixin: for item in raw_list ] + @diagnostics.observe('api', 'approve_friend_request', source='platform', stage='accepted') async def approve_friend_request( self, request_id: typing.Union[int, str], @@ -181,12 +200,15 @@ class AiocqhttpAPIMixin: ) -> None: await self.bot.set_friend_add_request(flag=str(request_id), approve=approve, remark=remark or '') + @diagnostics.observe('api', 'approve_group_invite', source='platform', stage='accepted') async def approve_group_invite(self, request_id: typing.Union[int, str], approve: bool = True) -> None: await self.bot.set_group_add_request(flag=str(request_id), sub_type='invite', approve=approve, reason='') + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') diff --git a/src/langbot/pkg/platform/adapters/aiocqhttp/event_converter.py b/src/langbot/pkg/platform/adapters/aiocqhttp/event_converter.py index d2c950c5f..cf809c513 100644 --- a/src/langbot/pkg/platform/adapters/aiocqhttp/event_converter.py +++ b/src/langbot/pkg/platform/adapters/aiocqhttp/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import aiocqhttp @@ -21,6 +23,7 @@ class AiocqhttpEventConverter(abstract_platform_adapter.AbstractEventConverter): return getattr(event, 'source_platform_object', None) @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri( event: aiocqhttp.Event, bot: aiocqhttp.CQHttp | None = None, @@ -39,6 +42,7 @@ class AiocqhttpEventConverter(abstract_platform_adapter.AbstractEventConverter): return None @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( event: aiocqhttp.Event, bot: aiocqhttp.CQHttp | None = None, diff --git a/src/langbot/pkg/platform/adapters/dingtalk/adapter.py b/src/langbot/pkg/platform/adapters/dingtalk/adapter.py index 230959719..632802f62 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/adapter.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import traceback import typing @@ -133,6 +135,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo def _plain_message(text: str) -> platform_message.MessageChain: return platform_message.MessageChain([platform_message.Plain(text=text)]) + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -149,6 +152,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo raise ValueError(f'Unsupported dingtalk target_type: {target_type}') return platform_events.MessageResult(raw=raw if isinstance(raw, dict) else {'result': raw}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -165,6 +169,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo raw=raw if isinstance(raw, dict) else {'result': raw}, ) + @diagnostics.observe('api', 'reply_message_chunk', source='platform', stage='accepted') async def reply_message_chunk( self, message_source: platform_events.MessageEvent, @@ -188,6 +193,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo if is_final and bot_message.tool_calls is None: self.card_instance_id_dict.pop(message_id) + @diagnostics.observe('api', 'create_message_card', source='platform', stage='accepted') async def create_message_card(self, message_id, event): while len(self.card_instance_id_dict) >= 1000: self.card_instance_id_dict.pop(next(iter(self.card_instance_id_dict)), None) @@ -205,6 +211,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo async def is_stream_output_supported(self) -> bool: return bool(self.config.get('enable-stream-reply', False)) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: if action == 'interaction.request' and action in self.get_supported_apis(): return await send_interaction(self, params) @@ -260,6 +267,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo DingTalkCardCallbackHandler(self), ) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: DingTalkEvent): try: interaction_event = interaction_event_from_native(event, self.interaction_callback_contexts) diff --git a/src/langbot/pkg/platform/adapters/dingtalk/api_impl.py b/src/langbot/pkg/platform/adapters/dingtalk/api_impl.py index 6db28ef4d..358e7a29d 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/api_impl.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.libs.dingtalk_api.api import DingTalkClient @@ -14,6 +16,7 @@ class DingTalkAPIMixin: _user_cache: dict[str, platform_entities.User] _group_cache: dict[str, platform_entities.UserGroup] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -25,18 +28,22 @@ class DingTalkAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: return self._group_cache.get(str(group_id)) or platform_entities.UserGroup(id=group_id, name='') + @diagnostics.observe('api', 'get_group_list', source='platform', stage='accepted') async def get_group_list(self) -> list[platform_entities.UserGroup]: return list(self._group_cache.values()) + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], ) -> list[platform_entities.UserGroupMember]: raise NotSupportedError('get_group_member_list') + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -52,14 +59,18 @@ class DingTalkAPIMixin: display_name=user.nickname, ) + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: return self._user_cache.get(str(user_id)) or platform_entities.User(id=user_id, nickname='') + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: return await self.bot.get_file_url(file_id) diff --git a/src/langbot/pkg/platform/adapters/dingtalk/event_converter.py b/src/langbot/pkg/platform/adapters/dingtalk/event_converter.py index 51e7db193..b3b23da2c 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/event_converter.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.libs.dingtalk_api.dingtalkevent import DingTalkEvent @@ -16,6 +18,7 @@ class DingTalkEventConverter(abstract_platform_adapter.AbstractEventConverter): return getattr(event, 'source_platform_object', None) @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(event: DingTalkEvent, bot_name: str) -> platform_events.Event | None: if event.conversation in {'FriendMessage', 'GroupMessage'}: return await DingTalkEventConverter.message_to_eba(event, bot_name) @@ -27,6 +30,7 @@ class DingTalkEventConverter(abstract_platform_adapter.AbstractEventConverter): return DingTalkEventConverter.platform_specific(event, f'message.{event.conversation or "unknown"}') @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( event: DingTalkEvent, bot_name: str, diff --git a/src/langbot/pkg/platform/adapters/discord/adapter.py b/src/langbot/pkg/platform/adapters/discord/adapter.py index 8ace25ba7..6691defa4 100644 --- a/src/langbot/pkg/platform/adapters/discord/adapter.py +++ b/src/langbot/pkg/platform/adapters/discord/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import os import time import traceback @@ -45,10 +47,30 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform adapter_self = self class LangBotDiscordClient(discord.Client): + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_ready(self: discord.Client): adapter_self.bot_account_id = str(self.user.id) if self.user else '' await adapter_self.logger.info(f'Discord adapter running as {self.user}') + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_message(self: discord.Client, message: discord.Message): if self.user and message.author.id == self.user.id: return @@ -72,6 +94,16 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform except Exception: await adapter_self.logger.error(f'Error in discord on_message: {traceback.format_exc()}') + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_interaction(self: discord.Client, interaction: discord.Interaction): custom_id = (interaction.data or {}).get('custom_id') if isinstance(interaction.data, dict) else None try: @@ -91,16 +123,46 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform except Exception: await adapter_self.logger.error(f'Error in Discord interaction callback: {traceback.format_exc()}') + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_message_edit(self: discord.Client, before: discord.Message, after: discord.Message): await adapter_self._dispatch_gateway_tuple( 'message_edit', (before, after), self.user.id if self.user else None ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_message_delete(self: discord.Client, message: discord.Message): await adapter_self._dispatch_gateway_tuple( 'message_delete', message, self.user.id if self.user else None ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_raw_message_delete(self: discord.Client, payload: discord.RawMessageDeleteEvent): await adapter_self._dispatch_gateway_tuple( 'raw_message_delete', @@ -108,6 +170,16 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform self.user.id if self.user else None, ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_reaction_add( self: discord.Client, reaction: discord.Reaction, user: discord.User | discord.Member ): @@ -117,6 +189,16 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform 'reaction_add', (reaction, user), self.user.id if self.user else None ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_reaction_remove( self: discord.Client, reaction: discord.Reaction, user: discord.User | discord.Member ): @@ -126,6 +208,16 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform 'reaction_remove', (reaction, user), self.user.id if self.user else None ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_raw_reaction_add(self: discord.Client, payload: discord.RawReactionActionEvent): if self.user and payload.user_id == self.user.id: return @@ -135,6 +227,16 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform self.user.id if self.user else None, ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_raw_reaction_remove(self: discord.Client, payload: discord.RawReactionActionEvent): if self.user and payload.user_id == self.user.id: return @@ -144,15 +246,55 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform self.user.id if self.user else None, ) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_member_join(self: discord.Client, member: discord.Member): await adapter_self._dispatch_gateway_tuple('member_join', member, self.user.id if self.user else None) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_member_remove(self: discord.Client, member: discord.Member): await adapter_self._dispatch_gateway_tuple('member_remove', member, self.user.id if self.user else None) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_guild_join(self: discord.Client, guild: discord.Guild): await adapter_self._dispatch_gateway_tuple('guild_join', guild, self.user.id if self.user else None) + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_guild_remove(self: discord.Client, guild: discord.Guild): await adapter_self._dispatch_gateway_tuple('guild_remove', guild, self.user.id if self.user else None) @@ -210,6 +352,7 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform def get_interaction_capabilities(self) -> dict[str, typing.Any]: return interaction_delivery_capabilities() + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain): content, files = await self.message_converter.yiri2target(message) channel = await self._get_channel(target_id) @@ -219,6 +362,7 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform sent = await channel.send(**kwargs) return platform_events.MessageResult(message_id=sent.id, raw={'message_id': sent.id}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -236,6 +380,7 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform sent = await message_source.source_platform_object.channel.send(**kwargs) return platform_events.MessageResult(message_id=sent.id, raw={'message_id': sent.id}) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _dispatch_gateway_tuple(self, kind: str, payload, bot_user_id: int | None): try: event = await self.event_converter.target2yiri((kind, payload), bot_user_id) @@ -269,6 +414,7 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform ): self.listeners.pop(event_type, None) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: if action == 'interaction.request': return await send_interaction(self, params) @@ -290,6 +436,7 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform async def is_stream_output_supported(self) -> bool: return True + @diagnostics.observe('api', 'create_message_card', source='platform', stage='accepted') async def create_message_card(self, message_id: str, event: platform_events.MessageEvent) -> bool: """Set up a stream context for progressive editing. @@ -309,6 +456,7 @@ class DiscordAdapter(DiscordAPIMixin, abstract_platform_adapter.AbstractPlatform } return True + @diagnostics.observe('api', 'reply_message_chunk', source='platform', stage='accepted') async def reply_message_chunk( self, message_source: platform_events.MessageEvent, diff --git a/src/langbot/pkg/platform/adapters/discord/api_impl.py b/src/langbot/pkg/platform/adapters/discord/api_impl.py index 767ae9a27..4c4ee9ecc 100644 --- a/src/langbot/pkg/platform/adapters/discord/api_impl.py +++ b/src/langbot/pkg/platform/adapters/discord/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import datetime import typing @@ -15,6 +17,7 @@ from langbot_plugin.api.entities.builtin.platform import message as platform_mes class DiscordAPIMixin: bot: discord.Client + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -31,6 +34,7 @@ class DiscordAPIMixin: return await message.edit(content=content) + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -41,6 +45,7 @@ class DiscordAPIMixin: message = await channel.fetch_message(int(message_id)) await message.delete() + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -56,10 +61,12 @@ class DiscordAPIMixin: sent = await to_channel.send(content=message.content, files=files) return platform_events.MessageResult(message_id=sent.id, raw={'message_id': sent.id}) + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: guild = await self._get_guild(group_id) return DiscordEventConverter.group_from_guild(guild) + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], @@ -68,6 +75,7 @@ class DiscordAPIMixin: members = guild.members or [member async for member in guild.fetch_members(limit=None)] return [self._member_to_entity(member) for member in members] + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -77,18 +85,22 @@ class DiscordAPIMixin: member = guild.get_member(int(user_id)) or await guild.fetch_member(int(user_id)) return self._member_to_entity(member) + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: user = self.bot.get_user(int(user_id)) or await self.bot.fetch_user(int(user_id)) return DiscordEventConverter.user_from_author(user) + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: from langbot_plugin.api.entities.builtin.platform.errors import NotSupportedError raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: return file_id + @diagnostics.observe('api', 'mute_member', source='platform', stage='accepted') async def mute_member( self, group_id: typing.Union[int, str], @@ -102,6 +114,7 @@ class DiscordAPIMixin: until = datetime.datetime.now(datetime.UTC) + datetime.timedelta(seconds=duration) await member.timeout(until, reason='LangBot Omni mute_member') + @diagnostics.observe('api', 'unmute_member', source='platform', stage='accepted') async def unmute_member( self, group_id: typing.Union[int, str], @@ -111,6 +124,7 @@ class DiscordAPIMixin: member = guild.get_member(int(user_id)) or await guild.fetch_member(int(user_id)) await member.timeout(None, reason='LangBot Omni unmute_member') + @diagnostics.observe('api', 'kick_member', source='platform', stage='accepted') async def kick_member( self, group_id: typing.Union[int, str], @@ -120,6 +134,7 @@ class DiscordAPIMixin: member = guild.get_member(int(user_id)) or await guild.fetch_member(int(user_id)) await member.kick(reason='LangBot Omni kick_member') + @diagnostics.observe('api', 'leave_group', source='platform', stage='accepted') async def leave_group(self, group_id: typing.Union[int, str]) -> None: guild = await self._get_guild(group_id) await guild.leave() diff --git a/src/langbot/pkg/platform/adapters/discord/event_converter.py b/src/langbot/pkg/platform/adapters/discord/event_converter.py index d34839725..b590b2f50 100644 --- a/src/langbot/pkg/platform/adapters/discord/event_converter.py +++ b/src/langbot/pkg/platform/adapters/discord/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import discord @@ -16,6 +18,7 @@ class DiscordEventConverter(abstract_platform_adapter.AbstractEventConverter): raise NotImplementedError @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(event: typing.Any, bot_user_id: int | None = None) -> platform_events.Event | None: if isinstance(event, discord.Message): return await DiscordEventConverter.message_to_eba(event) @@ -238,6 +241,7 @@ class DiscordEventConverter(abstract_platform_adapter.AbstractEventConverter): ) @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy(message: discord.Message) -> platform_events.FriendMessage | platform_events.GroupMessage: message_chain = await DiscordMessageConverter.target2yiri(message) if isinstance(message.channel, discord.DMChannel): diff --git a/src/langbot/pkg/platform/adapters/kook/adapter.py b/src/langbot/pkg/platform/adapters/kook/adapter.py index b89bcb3f9..61258b99d 100644 --- a/src/langbot/pkg/platform/adapters/kook/adapter.py +++ b/src/langbot/pkg/platform/adapters/kook/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + from langbot.pkg.platform.sources.kook import _decode_gateway_message import asyncio @@ -99,6 +101,7 @@ class KookAdapter(KookAPIMixin, BasePlatformAdapter): 'call_platform_api', ] + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: handler = PLATFORM_API_MAP.get(action) if handler is None: @@ -168,6 +171,7 @@ class KookAdapter(KookAPIMixin, BasePlatformAdapter): self.session_id = str(data.get('session_id') or '') await self.logger.info(f'KOOK WebSocket HELLO received, session_id: {self.session_id}') + @diagnostics.observe('event', 'platform.receive', source='platform', stage='accepted') async def _handle_event(self, data: dict, sn: int): self.current_sn = max(self.current_sn, sn) @@ -192,7 +196,9 @@ class KookAdapter(KookAPIMixin, BasePlatformAdapter): if eba_event: self._cache_event(eba_event) await self._dispatch_eba_event(eba_event) - except Exception: + except Exception as exc: + diagnostics.annotate(error=exc) + diagnostics.set_outcome('failed') await self.logger.error(f'Error handling KOOK event: {traceback.format_exc()}') async def _dispatch_eba_event(self, event: platform_events.EBAEvent): diff --git a/src/langbot/pkg/platform/adapters/kook/api_impl.py b/src/langbot/pkg/platform/adapters/kook/api_impl.py index 1d48b36d8..6170479b0 100644 --- a/src/langbot/pkg/platform/adapters/kook/api_impl.py +++ b/src/langbot/pkg/platform/adapters/kook/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.pkg.platform.adapters.kook.message_converter import KookMessageConverter @@ -14,6 +16,7 @@ class KookAPIMixin: _user_cache: dict[str, platform_entities.User] _group_cache: dict[str, platform_entities.UserGroup] + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -34,6 +37,7 @@ class KookAPIMixin: data = raw.get('data') or {} return platform_events.MessageResult(message_id=data.get('msg_id'), raw=raw) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -73,6 +77,7 @@ class KookAPIMixin: data = raw.get('data') or {} return platform_events.MessageResult(message_id=data.get('msg_id'), raw=raw) + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -84,6 +89,7 @@ class KookAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: cached = self._group_cache.get(str(group_id)) if cached: @@ -96,15 +102,18 @@ class KookAPIMixin: member_count=data.get('user_count'), ) + @diagnostics.observe('api', 'get_group_list', source='platform', stage='accepted') async def get_group_list(self) -> list[platform_entities.UserGroup]: return list(self._group_cache.values()) + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], ) -> list[platform_entities.UserGroupMember]: raise NotSupportedError('get_group_member_list') + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -128,6 +137,7 @@ class KookAPIMixin: display_name=user.nickname, ) + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: cached = self._user_cache.get(str(user_id)) if cached: @@ -142,18 +152,22 @@ class KookAPIMixin: is_bot=bool(data.get('bot', False)), ) + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: data = {'file': file_data} raw = await self._request('POST', '/asset/create', data=data, filename=filename) result = raw.get('data') or {} return str(result.get('url') or result.get('id') or '') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: return file_id + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -163,6 +177,7 @@ class KookAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -172,6 +187,7 @@ class KookAPIMixin: endpoint = '/message/delete' if str(chat_type).lower() in {'group', 'channel'} else '/direct-message/delete' await self._request('POST', endpoint, json={'msg_id': str(message_id)}) + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -185,6 +201,7 @@ class KookAPIMixin: raise NotSupportedError('forward_message:message_not_cached') return await self.send_message(to_chat_type, str(to_chat_id), cached.message_chain) + @diagnostics.observe('api', 'mute_member', source='platform', stage='accepted') async def mute_member( self, group_id: typing.Union[int, str], @@ -193,6 +210,7 @@ class KookAPIMixin: ) -> None: raise NotSupportedError('mute_member') + @diagnostics.observe('api', 'unmute_member', source='platform', stage='accepted') async def unmute_member( self, group_id: typing.Union[int, str], @@ -200,6 +218,7 @@ class KookAPIMixin: ) -> None: raise NotSupportedError('unmute_member') + @diagnostics.observe('api', 'kick_member', source='platform', stage='accepted') async def kick_member( self, group_id: typing.Union[int, str], @@ -207,5 +226,6 @@ class KookAPIMixin: ) -> None: raise NotSupportedError('kick_member') + @diagnostics.observe('api', 'leave_group', source='platform', stage='accepted') async def leave_group(self, group_id: typing.Union[int, str]) -> None: raise NotSupportedError('leave_group') diff --git a/src/langbot/pkg/platform/adapters/kook/event_converter.py b/src/langbot/pkg/platform/adapters/kook/event_converter.py index 5c7f8f4f2..34a2bbd19 100644 --- a/src/langbot/pkg/platform/adapters/kook/event_converter.py +++ b/src/langbot/pkg/platform/adapters/kook/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import time import langbot_plugin.api.definition.abstract.platform.adapter as abstract_platform_adapter @@ -16,6 +18,7 @@ class KookEventConverter(abstract_platform_adapter.AbstractEventConverter): raise NotImplementedError @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(kook_event: dict, bot_account_id: str = '') -> platform_events.Event | None: event_type = int(kook_event.get('type', 0) or 0) channel_type = kook_event.get('channel_type') @@ -55,6 +58,7 @@ class KookEventConverter(abstract_platform_adapter.AbstractEventConverter): ) @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( kook_event: dict, bot_account_id: str = '' ) -> platform_events.FriendMessage | platform_events.GroupMessage: diff --git a/src/langbot/pkg/platform/adapters/lark/adapter.py b/src/langbot/pkg/platform/adapters/lark/adapter.py index 776c0b7b0..0ce7839c6 100644 --- a/src/langbot/pkg/platform/adapters/lark/adapter.py +++ b/src/langbot/pkg/platform/adapters/lark/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + from langbot.pkg.platform.sources.lark import ( LarkAdapter as LegacyLarkAdapter, NonBlockingLarkWSClient, @@ -186,6 +188,16 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte self.request_app_ticket() def _build_event_handler(self): + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def on_message(event: lark_oapi.im.v1.P2ImMessageReceiveV1): await self._handle_message_event(event) @@ -326,6 +338,7 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte self.request_tenant_access_token(tenant_key) return self.tenant_access_tokens.get(tenant_key, {}).get('token') + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -359,6 +372,7 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte message_id=message_ids[-1] if message_ids else '', raw={'message_ids': message_ids} ) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -435,6 +449,7 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte while len(self.pending_monitoring_msg) > 1000: self.pending_monitoring_msg.pop(next(iter(self.pending_monitoring_msg)), None) + @diagnostics.observe('api', 'create_message_card', source='platform', stage='accepted') async def create_message_card(self, message_id, event) -> bool: card_id = await self.create_card_id(message_id) content = {'type': 'card', 'data': {'card_id': card_id, 'template_variable': {'content': 'Thinking...'}}} @@ -539,6 +554,7 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte raise RuntimeError(f'Lark card update failed: {response.code} {response.msg}') self.closed_streaming_cards.add(card_id) + @diagnostics.observe('api', 'reply_message_chunk', source='platform', stage='accepted') async def reply_message_chunk( self, message_source: platform_events.MessageEvent, @@ -589,6 +605,7 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte self.card_last_update_dict.pop(card_id, None) self.closed_streaming_cards.discard(card_id) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: if action == 'interaction.request': return await send_interaction(self, params) @@ -733,6 +750,7 @@ class LarkAdapter(LarkAPIMixin, abstract_platform_adapter.AbstractPlatformAdapte async def is_muted(self, group_id: int | None = None) -> bool: return False + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_message_event(self, event: lark_oapi.im.v1.P2ImMessageReceiveV1): try: if platform_events.FriendMessage in self.listeners or platform_events.GroupMessage in self.listeners: diff --git a/src/langbot/pkg/platform/adapters/lark/api_impl.py b/src/langbot/pkg/platform/adapters/lark/api_impl.py index ae2ff23d5..fb122a74a 100644 --- a/src/langbot/pkg/platform/adapters/lark/api_impl.py +++ b/src/langbot/pkg/platform/adapters/lark/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from lark_oapi.api.im.v1 import GetChatRequest, GetMessageRequest @@ -17,6 +19,7 @@ class LarkAPIMixin: _user_cache: dict[str, platform_entities.User] _group_cache: dict[str, platform_entities.UserGroup] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -52,6 +55,7 @@ class LarkAPIMixin: self._message_cache[str(message_id)] = event return event + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: cached = self._group_cache.get(str(group_id)) if cached: @@ -71,6 +75,7 @@ class LarkAPIMixin: self._group_cache[str(group.id)] = group return group + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -79,12 +84,14 @@ class LarkAPIMixin: user = self._user_cache.get(str(user_id)) or platform_entities.User(id=user_id) return platform_entities.UserGroupMember(user=user, group_id=group_id, role=platform_entities.MemberRole.MEMBER) + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: cached = self._user_cache.get(str(user_id)) if cached: return cached return platform_entities.User(id=user_id) + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: if str(file_id).startswith('file://'): return str(file_id) diff --git a/src/langbot/pkg/platform/adapters/lark/event_converter.py b/src/langbot/pkg/platform/adapters/lark/event_converter.py index d76a03ab8..6cfd590c7 100644 --- a/src/langbot/pkg/platform/adapters/lark/event_converter.py +++ b/src/langbot/pkg/platform/adapters/lark/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import time import typing @@ -24,6 +26,7 @@ class LarkEventConverter(abstract_platform_adapter.AbstractEventConverter): return getattr(event, 'source_platform_object', None) @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri( event: lark_oapi.im.v1.P2ImMessageReceiveV1, api_client: lark_oapi.Client, @@ -31,6 +34,7 @@ class LarkEventConverter(abstract_platform_adapter.AbstractEventConverter): return await LarkEventConverter.message_to_eba(event, api_client) @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( event: lark_oapi.im.v1.P2ImMessageReceiveV1, api_client: lark_oapi.Client, diff --git a/src/langbot/pkg/platform/adapters/officialaccount/adapter.py b/src/langbot/pkg/platform/adapters/officialaccount/adapter.py index 124230aad..6d5b49973 100644 --- a/src/langbot/pkg/platform/adapters/officialaccount/adapter.py +++ b/src/langbot/pkg/platform/adapters/officialaccount/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import traceback import typing @@ -94,6 +96,7 @@ class OfficialAccountAdapter(OfficialAccountAPIMixin, abstract_platform_adapter. 'call_platform_api', ] + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -102,6 +105,7 @@ class OfficialAccountAdapter(OfficialAccountAPIMixin, abstract_platform_adapter. ) -> platform_events.MessageResult: raise NotSupportedError('send_message:official_account_requires_inbound_webhook_reply') + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -118,6 +122,7 @@ class OfficialAccountAdapter(OfficialAccountAPIMixin, abstract_platform_adapter. await self.bot.set_message(source.message_id, content) return platform_events.MessageResult(message_id=source.message_id, raw={'queued': True}) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: handler = PLATFORM_API_MAP.get(action) if handler is None: @@ -170,6 +175,7 @@ class OfficialAccountAdapter(OfficialAccountAPIMixin, abstract_platform_adapter. for msg_type in ('text', 'image', 'voice', 'event'): self.bot.on_message(msg_type)(self._handle_native_event) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: OAEvent): self.bot_account_id = event.receiver_id or self.bot_account_id try: diff --git a/src/langbot/pkg/platform/adapters/officialaccount/api_impl.py b/src/langbot/pkg/platform/adapters/officialaccount/api_impl.py index 377986c64..4468cb660 100644 --- a/src/langbot/pkg/platform/adapters/officialaccount/api_impl.py +++ b/src/langbot/pkg/platform/adapters/officialaccount/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot_plugin.api.entities.builtin.platform import entities as platform_entities @@ -12,6 +14,7 @@ class OfficialAccountAPIMixin: _message_cache: dict[str, platform_events.MessageReceivedEvent] _user_cache: dict[str, platform_entities.User] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -23,15 +26,18 @@ class OfficialAccountAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: user = self._user_cache.get(str(user_id)) if user is None: raise NotSupportedError('get_user_info:not_cached') return user + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -41,6 +47,7 @@ class OfficialAccountAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -49,6 +56,7 @@ class OfficialAccountAPIMixin: ) -> None: raise NotSupportedError('delete_message') + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -59,24 +67,30 @@ class OfficialAccountAPIMixin: ) -> platform_events.MessageResult: raise NotSupportedError('forward_message') + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: raise NotSupportedError('get_group_info') + @diagnostics.observe('api', 'get_group_list', source='platform', stage='accepted') async def get_group_list(self) -> list[platform_entities.UserGroup]: raise NotSupportedError('get_group_list') + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], ) -> list[platform_entities.UserGroupMember]: raise NotSupportedError('get_group_member_list') + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], diff --git a/src/langbot/pkg/platform/adapters/officialaccount/event_converter.py b/src/langbot/pkg/platform/adapters/officialaccount/event_converter.py index 5523a55f8..dec3a2834 100644 --- a/src/langbot/pkg/platform/adapters/officialaccount/event_converter.py +++ b/src/langbot/pkg/platform/adapters/officialaccount/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import time import typing @@ -16,6 +18,7 @@ class OfficialAccountEventConverter(abstract_platform_adapter.AbstractEventConve async def yiri2target(event: platform_events.Event) -> typing.Any: return getattr(event, 'source_platform_object', None) + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy(self, event: OAEvent) -> platform_events.FriendMessage | None: eba_event = await self.target2yiri(event) if not isinstance(eba_event, platform_events.MessageReceivedEvent): @@ -31,6 +34,7 @@ class OfficialAccountEventConverter(abstract_platform_adapter.AbstractEventConve source_platform_object=event, ) + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(self, event: OAEvent) -> platform_events.Event | None: if event.type in {'text', 'image', 'voice'}: return await self.message_to_eba(event) diff --git a/src/langbot/pkg/platform/adapters/qqofficial/adapter.py b/src/langbot/pkg/platform/adapters/qqofficial/adapter.py index 7703a5935..da0c1547f 100644 --- a/src/langbot/pkg/platform/adapters/qqofficial/adapter.py +++ b/src/langbot/pkg/platform/adapters/qqofficial/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import time import traceback @@ -132,6 +134,7 @@ class QQOfficialAdapter(QQOfficialAPIMixin, abstract_platform_adapter.AbstractPl def _plain_message(text: str) -> platform_message.MessageChain: return platform_message.MessageChain([platform_message.Plain(text=text)]) + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -143,6 +146,7 @@ class QQOfficialAdapter(QQOfficialAPIMixin, abstract_platform_adapter.AbstractPl ) return platform_events.MessageResult(raw={'results': raw}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -161,6 +165,7 @@ class QQOfficialAdapter(QQOfficialAPIMixin, abstract_platform_adapter.AbstractPl ) return platform_events.MessageResult(message_id=source.d_id or source.id, raw={'results': raw}) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: if action == 'interaction.request': return await send_interaction(self, params) @@ -258,6 +263,7 @@ class QQOfficialAdapter(QQOfficialAPIMixin, abstract_platform_adapter.AbstractPl async def is_stream_output_supported(self) -> bool: return bool(self.config.get('enable-stream-reply') or self.config.get('enable_stream_reply')) + @diagnostics.observe('api', 'create_message_card', source='platform', stage='accepted') async def create_message_card(self, message_id: str, event: platform_events.MessageEvent) -> bool: source = event.source_platform_object if not isinstance(source, QQOfficialEvent) or source.t != 'C2C_MESSAGE_CREATE': @@ -277,6 +283,7 @@ class QQOfficialAdapter(QQOfficialAPIMixin, abstract_platform_adapter.AbstractPl self._stream_ctx_ts[message_id] = time.time() return True + @diagnostics.observe('api', 'reply_message_chunk', source='platform', stage='accepted') async def reply_message_chunk( self, message_source: platform_events.MessageEvent, @@ -364,6 +371,7 @@ class QQOfficialAdapter(QQOfficialAPIMixin, abstract_platform_adapter.AbstractPl if event is not None: await self._dispatch_eba_event(event) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: QQOfficialEvent): self.bot_account_id = self.config.get('appid', self.bot_account_id) try: diff --git a/src/langbot/pkg/platform/adapters/qqofficial/api_impl.py b/src/langbot/pkg/platform/adapters/qqofficial/api_impl.py index 59ed22956..e64570138 100644 --- a/src/langbot/pkg/platform/adapters/qqofficial/api_impl.py +++ b/src/langbot/pkg/platform/adapters/qqofficial/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.pkg.platform.adapters.qqofficial.errors import NotSupportedError @@ -14,6 +16,7 @@ class QQOfficialAPIMixin: _group_cache: dict[str, platform_entities.UserGroup] _member_cache: dict[tuple[str, str], platform_entities.UserGroupMember] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -25,21 +28,25 @@ class QQOfficialAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: user = self._user_cache.get(str(user_id)) if user is None: raise NotSupportedError('get_user_info:not_cached') return user + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: group = self._group_cache.get(str(group_id)) if group is None: raise NotSupportedError('get_group_info:not_cached') return group + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -50,6 +57,7 @@ class QQOfficialAPIMixin: raise NotSupportedError('get_group_member_info:not_cached') return member + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], @@ -58,6 +66,7 @@ class QQOfficialAPIMixin: member for (cached_group_id, _), member in self._member_cache.items() if cached_group_id == str(group_id) ] + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -67,6 +76,7 @@ class QQOfficialAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -75,6 +85,7 @@ class QQOfficialAPIMixin: ) -> None: raise NotSupportedError('delete_message') + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -85,20 +96,26 @@ class QQOfficialAPIMixin: ) -> platform_events.MessageResult: raise NotSupportedError('forward_message') + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') + @diagnostics.observe('api', 'mute_member', source='platform', stage='accepted') async def mute_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str], duration: int = 0): raise NotSupportedError('mute_member') + @diagnostics.observe('api', 'unmute_member', source='platform', stage='accepted') async def unmute_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str]): raise NotSupportedError('unmute_member') + @diagnostics.observe('api', 'kick_member', source='platform', stage='accepted') async def kick_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str]): raise NotSupportedError('kick_member') + @diagnostics.observe('api', 'leave_group', source='platform', stage='accepted') async def leave_group(self, group_id: typing.Union[int, str]): raise NotSupportedError('leave_group') diff --git a/src/langbot/pkg/platform/adapters/qqofficial/event_converter.py b/src/langbot/pkg/platform/adapters/qqofficial/event_converter.py index 66b550418..130061ce5 100644 --- a/src/langbot/pkg/platform/adapters/qqofficial/event_converter.py +++ b/src/langbot/pkg/platform/adapters/qqofficial/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import datetime import time import typing @@ -31,6 +33,7 @@ class QQOfficialEventConverter(abstract_platform_adapter.AbstractEventConverter) async def yiri2target(event: platform_events.Event) -> typing.Any: return getattr(event, 'source_platform_object', None) + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( self, event: QQOfficialEvent ) -> platform_events.FriendMessage | platform_events.GroupMessage | None: @@ -65,6 +68,7 @@ class QQOfficialEventConverter(abstract_platform_adapter.AbstractEventConverter) source_platform_object=event, ) + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(self, event: QQOfficialEvent) -> platform_events.Event: if event.t in MESSAGE_EVENT_TYPES: return await self.message_to_eba(event) diff --git a/src/langbot/pkg/platform/adapters/slack/adapter.py b/src/langbot/pkg/platform/adapters/slack/adapter.py index c2ee91a8d..c6e6d5db4 100644 --- a/src/langbot/pkg/platform/adapters/slack/adapter.py +++ b/src/langbot/pkg/platform/adapters/slack/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import traceback import typing @@ -90,6 +92,7 @@ class SlackAdapter(SlackAPIMixin, abstract_platform_adapter.AbstractPlatformAdap 'call_platform_api', ] + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -100,6 +103,7 @@ class SlackAdapter(SlackAPIMixin, abstract_platform_adapter.AbstractPlatformAdap raw = await self._send_text(str(target_type), str(target_id), content) return platform_events.MessageResult(raw=raw) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -114,6 +118,7 @@ class SlackAdapter(SlackAPIMixin, abstract_platform_adapter.AbstractPlatformAdap raw = await self._send_text(target_type, target_id, await SlackMessageConverter.yiri2target(message)) return platform_events.MessageResult(message_id=source.message_id, raw=raw) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: handler = PLATFORM_API_MAP.get(action) if handler is None: @@ -162,6 +167,7 @@ class SlackAdapter(SlackAPIMixin, abstract_platform_adapter.AbstractPlatformAdap for msg_type in ('im', 'channel'): self.bot.on_message(msg_type)(self._handle_native_event) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: SlackEvent): try: if platform_events.FriendMessage in self.listeners or platform_events.GroupMessage in self.listeners: diff --git a/src/langbot/pkg/platform/adapters/slack/api_impl.py b/src/langbot/pkg/platform/adapters/slack/api_impl.py index 500738b15..269444415 100644 --- a/src/langbot/pkg/platform/adapters/slack/api_impl.py +++ b/src/langbot/pkg/platform/adapters/slack/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.pkg.platform.adapters.slack.errors import NotSupportedError @@ -14,6 +16,7 @@ class SlackAPIMixin: _group_cache: dict[str, platform_entities.UserGroup] _member_cache: dict[tuple[str, str], platform_entities.UserGroupMember] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -25,24 +28,29 @@ class SlackAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: user = self._user_cache.get(str(user_id)) if user is None: raise NotSupportedError('get_user_info:not_cached') return user + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: group = self._group_cache.get(str(group_id)) if group is None: raise NotSupportedError('get_group_info:not_cached') return group + @diagnostics.observe('api', 'get_group_list', source='platform', stage='accepted') async def get_group_list(self) -> list[platform_entities.UserGroup]: return list(self._group_cache.values()) + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], @@ -51,6 +59,7 @@ class SlackAPIMixin: member for (cached_group_id, _), member in self._member_cache.items() if cached_group_id == str(group_id) ] + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -61,6 +70,7 @@ class SlackAPIMixin: raise NotSupportedError('get_group_member_info:not_cached') return member + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -70,6 +80,7 @@ class SlackAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -78,6 +89,7 @@ class SlackAPIMixin: ) -> None: raise NotSupportedError('delete_message') + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -88,8 +100,10 @@ class SlackAPIMixin: ) -> platform_events.MessageResult: raise NotSupportedError('forward_message') + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') diff --git a/src/langbot/pkg/platform/adapters/slack/event_converter.py b/src/langbot/pkg/platform/adapters/slack/event_converter.py index 6ebd48d91..8e7d4cc3c 100644 --- a/src/langbot/pkg/platform/adapters/slack/event_converter.py +++ b/src/langbot/pkg/platform/adapters/slack/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import time import typing @@ -19,6 +21,7 @@ class SlackEventConverter(abstract_platform_adapter.AbstractEventConverter): async def yiri2target(event: platform_events.Event) -> typing.Any: return getattr(event, 'source_platform_object', None) + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( self, event: SlackEvent ) -> platform_events.FriendMessage | platform_events.GroupMessage | None: @@ -53,6 +56,7 @@ class SlackEventConverter(abstract_platform_adapter.AbstractEventConverter): source_platform_object=event, ) + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(self, event: SlackEvent) -> platform_events.Event: if event.type in {'im', 'channel'}: return await self.message_to_eba(event) diff --git a/src/langbot/pkg/platform/adapters/telegram/adapter.py b/src/langbot/pkg/platform/adapters/telegram/adapter.py index 7debca097..0ea343f88 100644 --- a/src/langbot/pkg/platform/adapters/telegram/adapter.py +++ b/src/langbot/pkg/platform/adapters/telegram/adapter.py @@ -6,6 +6,8 @@ Preserves all existing functionality (messaging, streaming output, markdown card from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import traceback @@ -68,6 +70,16 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo arbitrary_types_allowed = True def __init__(self, config: dict, logger: abstract_platform_logger.AbstractEventLogger): + @diagnostics.observe( + 'event', + 'platform.native_callback', + source='platform', + stage='convert', + ap=lambda: getattr(logger, 'ap', None), + fields=lambda b: { + 'workspace_uuid': getattr(getattr(logger, 'execution_context', None), 'workspace_uuid', '') + }, + ) async def telegram_callback(update: Update, context: ContextTypes.DEFAULT_TYPE): if ( not update.message @@ -206,6 +218,7 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo # ---- Message Send / Reply (preserving original logic) ---- + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain): components = await TelegramMessageConverter.yiri2target(message, self.bot) @@ -240,6 +253,7 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo args['document'] = telegram.InputFile(doc, filename=filename) await self.bot.send_document(**args) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -317,6 +331,7 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo cleaned = text.replace('\u200b', '').replace('\u200c', '').replace('\u200d', '').replace('\ufeff', '').strip() return cleaned == '' + @diagnostics.observe('api', 'create_message_card', source='platform', stage='accepted') async def create_message_card(self, message_id, event): assert isinstance(event.source_platform_object, Update) update = event.source_platform_object @@ -331,6 +346,7 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo return True + @diagnostics.observe('api', 'reply_message_chunk', source='platform', stage='accepted') async def reply_message_chunk( self, message_source: platform_events.MessageEvent, @@ -492,6 +508,7 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo # ---- Pass-through API ---- + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api( self, action: str, diff --git a/src/langbot/pkg/platform/adapters/telegram/api_impl.py b/src/langbot/pkg/platform/adapters/telegram/api_impl.py index f058d54e6..f9aa7de40 100644 --- a/src/langbot/pkg/platform/adapters/telegram/api_impl.py +++ b/src/langbot/pkg/platform/adapters/telegram/api_impl.py @@ -5,6 +5,8 @@ Implements optional API methods defined in AbstractPlatformAdapter. from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import telegram @@ -25,6 +27,7 @@ class TelegramAPIMixin: bot: telegram.Bot + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -52,6 +55,7 @@ class TelegramAPIMixin: await self.bot.edit_message_text(**args) return + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -61,6 +65,7 @@ class TelegramAPIMixin: """Delete / recall a message.""" await self.bot.delete_message(chat_id=chat_id, message_id=message_id) + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -80,6 +85,7 @@ class TelegramAPIMixin: raw={'message_id': result.message_id}, ) + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info( self, group_id: typing.Union[int, str], @@ -100,6 +106,7 @@ class TelegramAPIMixin: except Exception: return None + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], @@ -134,6 +141,7 @@ class TelegramAPIMixin: ) return members + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -160,6 +168,7 @@ class TelegramAPIMixin: display_name=member.custom_title if hasattr(member, 'custom_title') else None, ) + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info( self, user_id: typing.Union[int, str], @@ -172,6 +181,7 @@ class TelegramAPIMixin: username=chat.username, ) + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file( self, file_data: bytes, @@ -186,6 +196,7 @@ class TelegramAPIMixin: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url( self, file_id: str, @@ -194,6 +205,7 @@ class TelegramAPIMixin: file = await self.bot.get_file(file_id) return file.file_path + @diagnostics.observe('api', 'mute_member', source='platform', stage='accepted') async def mute_member( self, group_id: typing.Union[int, str], @@ -213,6 +225,7 @@ class TelegramAPIMixin: kwargs['until_date'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=duration) await self.bot.restrict_chat_member(**kwargs) + @diagnostics.observe('api', 'unmute_member', source='platform', stage='accepted') async def unmute_member( self, group_id: typing.Union[int, str], @@ -236,6 +249,7 @@ class TelegramAPIMixin: permissions=permissions, ) + @diagnostics.observe('api', 'kick_member', source='platform', stage='accepted') async def kick_member( self, group_id: typing.Union[int, str], @@ -244,6 +258,7 @@ class TelegramAPIMixin: """Kick a member from the group.""" await self.bot.ban_chat_member(chat_id=group_id, user_id=user_id) + @diagnostics.observe('api', 'leave_group', source='platform', stage='accepted') async def leave_group( self, group_id: typing.Union[int, str], diff --git a/src/langbot/pkg/platform/adapters/telegram/event_converter.py b/src/langbot/pkg/platform/adapters/telegram/event_converter.py index 53a717c48..af2828175 100644 --- a/src/langbot/pkg/platform/adapters/telegram/event_converter.py +++ b/src/langbot/pkg/platform/adapters/telegram/event_converter.py @@ -5,6 +5,8 @@ Converts all Telegram Update types to unified EBA events, not just messages. from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import telegram @@ -54,6 +56,7 @@ class TelegramEventConverter(abstract_platform_adapter.AbstractEventConverter): return None @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri( update: Update, bot: telegram.Bot, @@ -384,6 +387,7 @@ class LegacyEventConverter(abstract_platform_adapter.AbstractEventConverter): return event.source_platform_object @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(event: Update, bot: telegram.Bot, bot_account_id: str): """Convert to legacy format (FriendMessage / GroupMessage).""" import langbot_plugin.api.entities.builtin.platform.events as legacy_events diff --git a/src/langbot/pkg/platform/adapters/wecom/adapter.py b/src/langbot/pkg/platform/adapters/wecom/adapter.py index 7096e8d80..8c54cc7c0 100644 --- a/src/langbot/pkg/platform/adapters/wecom/adapter.py +++ b/src/langbot/pkg/platform/adapters/wecom/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import traceback import typing @@ -90,6 +92,7 @@ class WecomAdapter(WecomAPIMixin, abstract_platform_adapter.AbstractPlatformAdap 'call_platform_api', ] + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -106,6 +109,7 @@ class WecomAdapter(WecomAPIMixin, abstract_platform_adapter.AbstractPlatformAdap raw_results.append(await self._send_content(user_id, agent_id, content)) return platform_events.MessageResult(raw={'results': raw_results}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -121,6 +125,7 @@ class WecomAdapter(WecomAPIMixin, abstract_platform_adapter.AbstractPlatformAdap raw_results.append(await self._send_content(wecom_event.user_id, int(wecom_event.agent_id), content)) return platform_events.MessageResult(message_id=wecom_event.message_id, raw={'results': raw_results}) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: handler = PLATFORM_API_MAP.get(action) if handler is None: @@ -174,6 +179,7 @@ class WecomAdapter(WecomAPIMixin, abstract_platform_adapter.AbstractPlatformAdap self.bot.on_message('text')(on_message) self.bot.on_message('image')(on_message) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: WecomEvent): self.bot_account_id = event.receiver_id or self.bot_account_id try: diff --git a/src/langbot/pkg/platform/adapters/wecom/api_impl.py b/src/langbot/pkg/platform/adapters/wecom/api_impl.py index 07ec2784d..b768a1259 100644 --- a/src/langbot/pkg/platform/adapters/wecom/api_impl.py +++ b/src/langbot/pkg/platform/adapters/wecom/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.libs.wecom_api.api import WecomClient @@ -14,6 +16,7 @@ class WecomAPIMixin: _message_cache: dict[str, platform_events.MessageReceivedEvent] _user_cache: dict[str, platform_entities.User] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -25,6 +28,7 @@ class WecomAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: cached = self._user_cache.get(str(user_id)) if cached is not None: @@ -36,24 +40,30 @@ class WecomAPIMixin: username=info.get('alias') or info.get('userid') or None, ) + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: raise NotSupportedError('get_group_info') + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], ) -> list[platform_entities.UserGroupMember]: raise NotSupportedError('get_group_member_list') + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -61,6 +71,7 @@ class WecomAPIMixin: ) -> platform_entities.UserGroupMember: raise NotSupportedError('get_group_member_info') + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -70,6 +81,7 @@ class WecomAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, diff --git a/src/langbot/pkg/platform/adapters/wecom/event_converter.py b/src/langbot/pkg/platform/adapters/wecom/event_converter.py index 94c43fd78..41d301ae6 100644 --- a/src/langbot/pkg/platform/adapters/wecom/event_converter.py +++ b/src/langbot/pkg/platform/adapters/wecom/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.libs.wecom_api.api import WecomClient @@ -17,6 +19,7 @@ class WecomEventConverter(abstract_platform_adapter.AbstractEventConverter): return getattr(event, 'source_platform_object', None) @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy(event: WecomEvent, bot: WecomClient | None = None) -> platform_events.FriendMessage | None: eba_event = await WecomEventConverter.target2yiri(event, bot) if hasattr(eba_event, 'to_legacy_event'): @@ -36,6 +39,7 @@ class WecomEventConverter(abstract_platform_adapter.AbstractEventConverter): return None @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(event: WecomEvent, bot: WecomClient | None = None) -> platform_events.Event | None: if event.type in {'text', 'image'}: return await WecomEventConverter.message_to_eba(event, bot) diff --git a/src/langbot/pkg/platform/adapters/wecombot/adapter.py b/src/langbot/pkg/platform/adapters/wecombot/adapter.py index 38fb5e8df..416af30a2 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/adapter.py +++ b/src/langbot/pkg/platform/adapters/wecombot/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + from langbot.pkg.platform.sources.wecombot import WecomBotAdapter as LegacyWecomBotAdapter import asyncio @@ -133,6 +135,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo _iter_media_components = staticmethod(LegacyWecomBotAdapter._iter_media_components) _send_media = staticmethod(LegacyWecomBotAdapter._send_media) + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -148,6 +151,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo raw = await self.bot.send_message(str(target_id), content) return platform_events.MessageResult(raw={'result': raw}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -169,6 +173,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo raw = await self.bot.set_message(event.message_id, content) return platform_events.MessageResult(message_id=event.message_id, raw={'result': raw}) + @diagnostics.observe('api', 'reply_message_chunk', source='platform', stage='accepted') async def reply_message_chunk( self, message_source: platform_events.MessageEvent, @@ -196,6 +201,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo async def is_stream_output_supported(self) -> bool: return self.config.get('enable-stream-reply', True) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: if action == 'interaction.request' and 'interaction.request' in self.get_supported_apis(): return await send_interaction(self, params) @@ -271,6 +277,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo self.bot.on_message('event')(self._handle_native_event) self.bot.on_message('template_card_event')(self._handle_interaction_event) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_interaction_event(self, event: WecomBotEvent): try: interaction_event = interaction_event_from_native(event) @@ -279,6 +286,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo except Exception: await self.logger.error(f'Error in WeComBot interaction callback: {traceback.format_exc()}') + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: WecomBotEvent): try: if platform_events.FriendMessage in self.listeners or platform_events.GroupMessage in self.listeners: diff --git a/src/langbot/pkg/platform/adapters/wecombot/api_impl.py b/src/langbot/pkg/platform/adapters/wecombot/api_impl.py index 15758fa5b..cbd34ba38 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/api_impl.py +++ b/src/langbot/pkg/platform/adapters/wecombot/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot_plugin.api.entities.builtin.platform import entities as platform_entities @@ -14,6 +16,7 @@ class WecomBotAPIMixin: _group_cache: dict[str, platform_entities.UserGroup] _member_cache: dict[tuple[str, str], platform_entities.UserGroupMember] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -25,21 +28,25 @@ class WecomBotAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: cached = self._user_cache.get(str(user_id)) if cached is None: raise NotSupportedError('get_user_info:not_cached') return cached + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: cached = self._group_cache.get(str(group_id)) if cached is None: raise NotSupportedError('get_group_info:not_cached') return cached + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -50,6 +57,7 @@ class WecomBotAPIMixin: raise NotSupportedError('get_group_member_info:not_cached') return cached + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], @@ -58,12 +66,15 @@ class WecomBotAPIMixin: member for (cached_group_id, _), member in self._member_cache.items() if cached_group_id == str(group_id) ] + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -73,6 +84,7 @@ class WecomBotAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, @@ -81,6 +93,7 @@ class WecomBotAPIMixin: ) -> None: raise NotSupportedError('delete_message') + @diagnostics.observe('api', 'forward_message', source='platform', stage='accepted') async def forward_message( self, from_chat_type: str, @@ -91,14 +104,18 @@ class WecomBotAPIMixin: ) -> platform_events.MessageResult: raise NotSupportedError('forward_message') + @diagnostics.observe('api', 'mute_member', source='platform', stage='accepted') async def mute_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str], duration: int = 0): raise NotSupportedError('mute_member') + @diagnostics.observe('api', 'unmute_member', source='platform', stage='accepted') async def unmute_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str]): raise NotSupportedError('unmute_member') + @diagnostics.observe('api', 'kick_member', source='platform', stage='accepted') async def kick_member(self, group_id: typing.Union[int, str], user_id: typing.Union[int, str]): raise NotSupportedError('kick_member') + @diagnostics.observe('api', 'leave_group', source='platform', stage='accepted') async def leave_group(self, group_id: typing.Union[int, str]): raise NotSupportedError('leave_group') diff --git a/src/langbot/pkg/platform/adapters/wecombot/event_converter.py b/src/langbot/pkg/platform/adapters/wecombot/event_converter.py index 31fefb559..207cfd71c 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/event_converter.py +++ b/src/langbot/pkg/platform/adapters/wecombot/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import time import typing @@ -19,6 +21,7 @@ class WecomBotEventConverter(abstract_platform_adapter.AbstractEventConverter): async def yiri2target(event: platform_events.Event) -> typing.Any: return getattr(event, 'source_platform_object', None) + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( self, event: WecomBotEvent ) -> platform_events.FriendMessage | platform_events.GroupMessage | None: @@ -49,6 +52,7 @@ class WecomBotEventConverter(abstract_platform_adapter.AbstractEventConverter): source_platform_object=event, ) + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(self, event: WecomBotEvent) -> platform_events.Event: if event.type in {'single', 'group'} and event.msgtype != 'event': return await self.message_to_eba(event) diff --git a/src/langbot/pkg/platform/adapters/wecomcs/adapter.py b/src/langbot/pkg/platform/adapters/wecomcs/adapter.py index 2fb12ef7f..4fd9f40bb 100644 --- a/src/langbot/pkg/platform/adapters/wecomcs/adapter.py +++ b/src/langbot/pkg/platform/adapters/wecomcs/adapter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import time import traceback @@ -92,6 +94,7 @@ class WecomCSAdapter(WecomCSAPIMixin, abstract_platform_adapter.AbstractPlatform 'call_platform_api', ] + @diagnostics.observe('api', 'send_message', source='platform', stage='accepted') async def send_message( self, target_type: str, @@ -110,6 +113,7 @@ class WecomCSAdapter(WecomCSAPIMixin, abstract_platform_adapter.AbstractPlatform ) return platform_events.MessageResult(raw={'results': raw_results}) + @diagnostics.observe('api', 'reply_message', source='platform', stage='accepted') async def reply_message( self, message_source: platform_events.MessageEvent, @@ -132,6 +136,7 @@ class WecomCSAdapter(WecomCSAPIMixin, abstract_platform_adapter.AbstractPlatform ) return platform_events.MessageResult(message_id=wecom_event.message_id, raw={'results': raw_results}) + @diagnostics.observe('api', 'call_platform_api', source='platform', stage='accepted') async def call_platform_api(self, action: str, params: dict = {}) -> dict: handler = PLATFORM_API_MAP.get(action) if handler is None: @@ -186,6 +191,7 @@ class WecomCSAdapter(WecomCSAPIMixin, abstract_platform_adapter.AbstractPlatform for msg_type in ('text', 'image', 'file', 'voice'): self.bot.on_message(msg_type)(on_message) + @diagnostics.observe('event', 'platform.native_receive', source='platform', stage='convert') async def _handle_native_event(self, event: WecomCSEvent): self.bot_account_id = event.receiver_id or self.bot_account_id try: diff --git a/src/langbot/pkg/platform/adapters/wecomcs/api_impl.py b/src/langbot/pkg/platform/adapters/wecomcs/api_impl.py index 3fca63275..d14565b2a 100644 --- a/src/langbot/pkg/platform/adapters/wecomcs/api_impl.py +++ b/src/langbot/pkg/platform/adapters/wecomcs/api_impl.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.libs.wecom_customer_service_api.api import WecomCSClient @@ -14,6 +16,7 @@ class WecomCSAPIMixin: _message_cache: dict[str, platform_events.MessageReceivedEvent] _user_cache: dict[str, platform_entities.User] + @diagnostics.observe('api', 'get_message', source='platform', stage='accepted') async def get_message( self, chat_type: str, @@ -25,6 +28,7 @@ class WecomCSAPIMixin: raise NotSupportedError('get_message:message_not_cached') return event + @diagnostics.observe('api', 'get_user_info', source='platform', stage='accepted') async def get_user_info(self, user_id: typing.Union[int, str]) -> platform_entities.User: cached = self._user_cache.get(str(user_id)) if cached is not None: @@ -39,24 +43,30 @@ class WecomCSAPIMixin: username=info.get('external_userid') or None, ) + @diagnostics.observe('api', 'get_friend_list', source='platform', stage='accepted') async def get_friend_list(self) -> list[platform_entities.User]: return list(self._user_cache.values()) + @diagnostics.observe('api', 'upload_file', source='platform', stage='accepted') async def upload_file(self, file_data: bytes, filename: str) -> str: raise NotSupportedError('upload_file') + @diagnostics.observe('api', 'get_file_url', source='platform', stage='accepted') async def get_file_url(self, file_id: str) -> str: raise NotSupportedError('get_file_url') + @diagnostics.observe('api', 'get_group_info', source='platform', stage='accepted') async def get_group_info(self, group_id: typing.Union[int, str]) -> platform_entities.UserGroup: raise NotSupportedError('get_group_info') + @diagnostics.observe('api', 'get_group_member_list', source='platform', stage='accepted') async def get_group_member_list( self, group_id: typing.Union[int, str], ) -> list[platform_entities.UserGroupMember]: raise NotSupportedError('get_group_member_list') + @diagnostics.observe('api', 'get_group_member_info', source='platform', stage='accepted') async def get_group_member_info( self, group_id: typing.Union[int, str], @@ -64,6 +74,7 @@ class WecomCSAPIMixin: ) -> platform_entities.UserGroupMember: raise NotSupportedError('get_group_member_info') + @diagnostics.observe('api', 'edit_message', source='platform', stage='accepted') async def edit_message( self, chat_type: str, @@ -73,6 +84,7 @@ class WecomCSAPIMixin: ) -> None: raise NotSupportedError('edit_message') + @diagnostics.observe('api', 'delete_message', source='platform', stage='accepted') async def delete_message( self, chat_type: str, diff --git a/src/langbot/pkg/platform/adapters/wecomcs/event_converter.py b/src/langbot/pkg/platform/adapters/wecomcs/event_converter.py index 3dc56c68e..64201513b 100644 --- a/src/langbot/pkg/platform/adapters/wecomcs/event_converter.py +++ b/src/langbot/pkg/platform/adapters/wecomcs/event_converter.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing from langbot.libs.wecom_customer_service_api.api import WecomCSClient @@ -17,6 +19,7 @@ class WecomCSEventConverter(abstract_platform_adapter.AbstractEventConverter): return getattr(event, 'source_platform_object', None) @staticmethod + @diagnostics.observe('event', 'platform.target2legacy', source='platform', stage='convert') async def target2legacy( event: WecomCSEvent, bot: WecomCSClient | None = None ) -> platform_events.FriendMessage | None: @@ -26,6 +29,7 @@ class WecomCSEventConverter(abstract_platform_adapter.AbstractEventConverter): return None @staticmethod + @diagnostics.observe('event', 'platform.target2yiri', source='platform', stage='convert') async def target2yiri(event: WecomCSEvent, bot: WecomCSClient | None = None) -> platform_events.Event | None: if event.type in {'text', 'image', 'file', 'voice'}: return await WecomCSEventConverter.message_to_eba(event, bot) diff --git a/src/langbot/pkg/platform/botmgr.py b/src/langbot/pkg/platform/botmgr.py index 7dee7ac83..85d7bc0a3 100644 --- a/src/langbot/pkg/platform/botmgr.py +++ b/src/langbot/pkg/platform/botmgr.py @@ -1,5 +1,7 @@ from __future__ import annotations +from ..telemetry import diagnostics + import asyncio import contextlib import dataclasses @@ -368,6 +370,22 @@ class RuntimeBot: ) -> dict[str, typing.Any]: """Record structured event routing state while preserving the human log.""" binding = binding or {} + diagnostics.event( + self, + 'route', + 'route.primary', + { + 'not_matched': 'skipped', + 'discarded': 'skipped', + 'matched': 'started', + 'delivered': 'succeeded', + 'failed': 'failed', + }.get(status, 'unknown'), + stage='dispatch', + platform_event_type=event_type, + processor_type=target_type or binding.get('target_type', ''), + reason_code=failure_code or status, + ) metadata = { 'kind': 'event_route_trace', 'event_type': event_type, @@ -380,6 +398,16 @@ class RuntimeBot: 'reason': reason or text, 'run_id': run_id, } + diagnostics.set_outcome( + { + 'not_matched': 'skipped', + 'discarded': 'skipped', + 'matched': 'started', + 'delivered': 'succeeded', + 'failed': 'failed', + }.get(status, 'unknown'), + reason_code=failure_code or status, + ) log_method = getattr(self.logger, level, self.logger.info) await log_method(text, metadata=metadata) return metadata @@ -835,6 +863,7 @@ class RuntimeBot: processor_id=agent.get('uuid'), ) + @diagnostics.observe('event', 'platform.receive', source='platform', stage='dispatch') async def _handle_platform_event( self, event: platform_events.EBAEvent, @@ -864,6 +893,7 @@ class RuntimeBot: if isinstance(result, BaseException): await self.logger.error(f'Event delivery failed: {result}') + @diagnostics.observe('route', 'route.subscription', source='platform', stage='dispatch') async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid): event_type = event.type event_binding = { @@ -882,6 +912,7 @@ class RuntimeBot: # Resolve the installed declaration each time, including after plugin updates. patterns = descriptor.supported_event_patterns if not patterns or not self._agent_supports_event_type(patterns, event_type): + diagnostics.set_outcome('skipped', reason_code='not_matched') return agent = {**agent, 'supported_event_patterns': patterns} return await self._dispatch_eba_event_to_processor(event, adapter, event_binding, agent) @@ -898,6 +929,7 @@ class RuntimeBot: text=f'Plugin processor {processor_uuid} failed: {exc}', ) + @diagnostics.observe('route', 'route.dispatch', source='platform', stage='dispatch') async def _dispatch_eba_event_to_processor( self, event: platform_events.EBAEvent, @@ -1165,6 +1197,7 @@ class RuntimeBot: except Exception as e: await self.logger.error(f'Failed to record discarded message: {e}') + @diagnostics.observe('event', 'platform.legacy_receive', source='platform', stage='dispatch') async def _handle_legacy_message_event( self, event: platform_events.FriendMessage | platform_events.GroupMessage, @@ -1250,6 +1283,7 @@ class RuntimeBot: execution_context=self.execution_context, ) + @diagnostics.observe('interaction', 'interaction.submit', source='platform', stage='dispatch') async def _handle_interaction_submission( self, event: platform_events.PlatformSpecificEvent, @@ -1358,6 +1392,7 @@ class RuntimeBot: execution_context=self.execution_context, ) + @diagnostics.observe('interaction', 'interaction.resume', source='platform', stage='resume') async def _resume_agent_interaction( self, record: dict[str, typing.Any], @@ -1512,6 +1547,9 @@ class RuntimeBot: return await self._handle_platform_event(self._legacy_message_to_eba_event(event, adapter), adapter) + from ..telemetry.diagnostic_catalog import snapshot_bot + + snapshot_bot(self) get_supported_events = getattr(self.adapter, 'get_supported_events', None) supported_events: list[str] = [] if callable(get_supported_events): diff --git a/src/langbot/pkg/plugin/agent_run_support.py b/src/langbot/pkg/plugin/agent_run_support.py index 78070087e..4bd770d20 100644 --- a/src/langbot/pkg/plugin/agent_run_support.py +++ b/src/langbot/pkg/plugin/agent_run_support.py @@ -2,6 +2,8 @@ from __future__ import annotations +from ..telemetry import diagnostics + from typing import Any, Union import json import time @@ -302,6 +304,7 @@ async def _validate_agent_run_session( if not available_apis.get(api_capability, False) and not has_admin_permission: return None, handler.ActionResponse.error(message=f'{api_name} access not authorized') + diagnostics.link_context(session.get('_diagnostic_context')) return session, None diff --git a/src/langbot/pkg/plugin/connector.py b/src/langbot/pkg/plugin/connector.py index a2eebbb52..ee17c0200 100644 --- a/src/langbot/pkg/plugin/connector.py +++ b/src/langbot/pkg/plugin/connector.py @@ -1,6 +1,8 @@ # For connect to plugin runtime. from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import asyncio import contextlib import contextvars @@ -668,6 +670,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): } ) + @diagnostics.observe('lifecycle', 'runtime.prepare_connected_runtime', source='runtime', stage='execute') async def _prepare_connected_runtime(self) -> None: """Handshake follow-up: pin OSS compatibility, then replay authority.""" @@ -853,6 +856,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self.schedule_reconnect() failures = 0 + @diagnostics.observe('lifecycle', 'runtime.initialize', source='runtime', stage='execute') async def initialize(self): if not self.is_enable_plugin: self.ap.logger.info('Plugin system is disabled.') @@ -1705,6 +1709,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): ) return plugin_package, latest_version + @diagnostics.observe('lifecycle', 'runtime.install_plugin', source='runtime', stage='execute') async def install_plugin( self, install_source: PluginInstallSource, @@ -1820,6 +1825,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): task_context.set_current_action('plugin updated' if operation == 'upgrade' else 'plugin installed') task_context.metadata['progress_percent'] = 100 + @diagnostics.observe('lifecycle', 'runtime.upgrade_plugin', source='runtime', stage='execute') async def upgrade_plugin( self, plugin_author: str, @@ -1846,6 +1852,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): ) return {} + @diagnostics.observe('lifecycle', 'runtime.delete_plugin', source='runtime', stage='execute') async def delete_plugin( self, plugin_author: str, diff --git a/src/langbot/pkg/plugin/handler.py b/src/langbot/pkg/plugin/handler.py index ddc784acb..a2d0e7a35 100644 --- a/src/langbot/pkg/plugin/handler.py +++ b/src/langbot/pkg/plugin/handler.py @@ -1,5 +1,7 @@ from __future__ import annotations +from ..telemetry import diagnostics + import asyncio import inspect import typing @@ -639,6 +641,7 @@ class RuntimeConnectionHandler(handler.Handler): avoids reserving one pooled connection across provider and network waits. """ + diagnostics.annotate(workspace_uuid=action_context.workspace_uuid) persistence_mgr = getattr(self.ap, 'persistence_mgr', None) if persistence_mgr is None: yield @@ -687,10 +690,13 @@ class RuntimeConnectionHandler(handler.Handler): return if trusted_plugin_identity is not None: safe_data['caller_plugin_identity'] = trusted_plugin_identity - async for response in _action_handler(safe_data): - yield response + async with contextlib.aclosing(_action_handler(safe_data)) as responses: + async for response in responses: + yield response - self.actions[action_name] = secured_stream_action + self.actions[action_name] = diagnostics.observe( + 'api', 'host.' + action_name, source='plugin', ap=self.ap + )(secured_stream_action) continue async def secured_action( @@ -719,7 +725,9 @@ class RuntimeConnectionHandler(handler.Handler): response = await response return response - self.actions[action_name] = secured_action + self.actions[action_name] = diagnostics.observe('api', 'host.' + action_name, source='plugin', ap=self.ap)( + secured_action + ) async def _get_plugin_setting( self, diff --git a/src/langbot/pkg/provider/modelmgr/requester.py b/src/langbot/pkg/provider/modelmgr/requester.py index 069c9981e..331cb6fa7 100644 --- a/src/langbot/pkg/provider/modelmgr/requester.py +++ b/src/langbot/pkg/provider/modelmgr/requester.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import abc import typing import time @@ -98,6 +100,7 @@ class RuntimeProvider: raise WorkspaceInvariantError('LLM invocation requires an ExecutionContext when query is absent') return execution_context + @diagnostics.observe('api', 'model.invoke_llm', source='agent') async def invoke_llm( self, query: pipeline_query.Query | None, @@ -182,6 +185,7 @@ class RuntimeProvider: except Exception as monitor_err: self.requester.ap.logger.error(f'[Monitoring] Failed to record LLM call: {monitor_err}') + @diagnostics.observe('api', 'model.invoke_llm_stream', source='agent') async def invoke_llm_stream( self, query: pipeline_query.Query | None, @@ -264,6 +268,7 @@ class RuntimeProvider: except Exception as monitor_err: self.requester.ap.logger.error(f'[Monitoring] Failed to record LLM stream call: {monitor_err}') + @diagnostics.observe('api', 'model.invoke_embedding', source='agent') async def invoke_embedding( self, model: RuntimeEmbeddingModel, @@ -331,6 +336,7 @@ class RuntimeProvider: except Exception as monitor_err: self.requester.ap.logger.error(f'[Monitoring] Failed to record embedding call: {monitor_err}') + @diagnostics.observe('api', 'model.invoke_rerank', source='agent') async def invoke_rerank( self, model: RuntimeRerankModel, diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index b09dff9d8..0dda17b56 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import typing import time import inspect @@ -473,6 +475,7 @@ class ToolManager: ) return result + @diagnostics.observe('api', 'tool.execute', source='agent', stage='execute') async def execute_func_call( self, name: str, diff --git a/src/langbot/pkg/rag/service/runtime.py b/src/langbot/pkg/rag/service/runtime.py index a685dc521..b8b814f27 100644 --- a/src/langbot/pkg/rag/service/runtime.py +++ b/src/langbot/pkg/rag/service/runtime.py @@ -1,5 +1,7 @@ from __future__ import annotations +from langbot.pkg.telemetry import diagnostics + import posixpath import re from typing import TYPE_CHECKING, Any @@ -71,6 +73,7 @@ class RAGRuntimeService: raise WorkspaceNotFoundError('Knowledge base not found') return kb_uuid + @diagnostics.observe('api', 'knowledge.vector_upsert', source='agent') async def vector_upsert( self, execution_context: ExecutionContext, @@ -98,6 +101,7 @@ class RAGRuntimeService: documents=documents, ) + @diagnostics.observe('api', 'knowledge.vector_search', source='agent') async def vector_search( self, execution_context: ExecutionContext, @@ -122,6 +126,7 @@ class RAGRuntimeService: vector_weight=vector_weight, ) + @diagnostics.observe('api', 'knowledge.vector_delete', source='agent') async def vector_delete( self, execution_context: ExecutionContext, @@ -158,6 +163,7 @@ class RAGRuntimeService: ) return count + @diagnostics.observe('api', 'knowledge.vector_list', source='agent') async def vector_list( self, execution_context: ExecutionContext, @@ -186,6 +192,7 @@ class RAGRuntimeService: offset=offset, ) + @diagnostics.observe('api', 'knowledge.get_file_stream', source='agent') async def get_file_stream( self, execution_context: ExecutionContext, diff --git a/src/langbot/pkg/telemetry/diagnostic_catalog.py b/src/langbot/pkg/telemetry/diagnostic_catalog.py new file mode 100644 index 000000000..3130af636 --- /dev/null +++ b/src/langbot/pkg/telemetry/diagnostic_catalog.py @@ -0,0 +1,83 @@ +"""Packaged capability catalog and projections from existing runtime declarations.""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +import yaml + +from . import diagnostic_privacy as privacy + + +@lru_cache(maxsize=1) +def catalog(): + result = {} + base = Path(__file__).resolve().parents[1] / 'platform' + for manifest in sorted((base / 'adapters').glob('*/manifest.yaml')): + data = yaml.safe_load(manifest.read_text()) + spec = data.get('spec', {}) + name = data['metadata']['name'] + privacy.code_value('adapter', name) + events = spec.get('supported_events', []) + apis = spec.get('supported_apis', {}) + apis = apis if isinstance(apis, list) else [v for items in apis.values() for v in items] + for event in events: + privacy.code_value('platform_event_type', event) + for operation in apis: + privacy.code_value('operation', operation) + for api in spec.get('platform_specific_apis', []): + privacy.code_value('operation', api['action']) + result[manifest.parent.name] = {'adapter': name, 'events': events, 'apis': apis} + return result + + +def adapter_fields(adapter): + module = type(adapter).__module__.split('.') + entry = next((entry for directory, entry in catalog().items() if directory in module), None) + return {'adapter': entry['adapter']} if entry else {} + + +def _snapshot_bot(bot): + from . import diagnostics + + manager = getattr(bot.ap, 'diagnostics', None) + if not isinstance(manager, diagnostics.DiagnosticsManager) or not manager.enabled: + return + fields = adapter_fields(bot.adapter) + if not fields: + return + fields['workspace_uuid'] = bot.execution_context.workspace_uuid + entry = next(e for e in catalog().values() if e['adapter'] == fields['adapter']) + for capability_type, method, declared in ( + ('event', 'get_supported_events', entry['events']), + ('api', 'get_supported_apis', entry['apis']), + ): + try: + supported = set(getattr(bot.adapter, method)() or []) + except Exception: + supported = set() + for name in declared: + manager.emit( + 'capability', + name if capability_type == 'api' else 'platform.receive', + 'succeeded', + source='platform', + stage='snapshot', + **fields, + platform_event_type=name if capability_type == 'event' else '', + attributes={ + 'capability_type': capability_type, + 'capability_name': name, + 'supported': name in supported, + 'configured': True, + 'available': True, + }, + ) + + +def snapshot_bot(bot): + try: + _snapshot_bot(bot) + except Exception: + pass diff --git a/src/langbot/pkg/telemetry/diagnostic_privacy.py b/src/langbot/pkg/telemetry/diagnostic_privacy.py new file mode 100644 index 000000000..3b8b4a8ba --- /dev/null +++ b/src/langbot/pkg/telemetry/diagnostic_privacy.py @@ -0,0 +1,113 @@ +"""Closed, content-free projection for Beta diagnostics; never serialize payloads.""" + +from __future__ import annotations + +import builtins +import hashlib +import math +import re +from uuid import UUID + +KINDS = frozenset('lifecycle event route run api delivery interaction capability transport summary'.split()) +SOURCES = frozenset( + 'platform pipeline agent event_processor webui_debug http websocket mcp plugin runtime startup internal synthetic'.split() +) +OUTCOMES = frozenset('started succeeded failed cancelled timeout skipped rejected waiting partial unknown'.split()) +PROCESSORS = frozenset(('', 'pipeline', 'agent', 'event_processor')) +# These sets are populated ONLY by source-code decorators / packaged manifests, +# never by requests, installed third-party plugin declarations or configuration. +VOCABULARY: dict[str, set[str]] = { + 'operation': {'diagnostics.transport', 'startup', 'route.primary', 'route.subscription', 'runner.run'}, + 'stage': {'execute', 'prepare', 'convert', 'dispatch', 'accepted', 'ack', 'resume', 'shutdown', 'snapshot'}, + 'adapter': set(), + 'platform_event_type': set(), + 'reason_code': set( + 'runner_failed response_error route_not_found processor_incompatible processor_not_found discarded not_matched matched delivered waiting interaction_rejected generator_closed transport_loss'.split() + ), + 'capability_type': {'event', 'api', 'processor'}, + 'tool_category': {'native', 'plugin', 'mcp', 'skill', 'platform', 'unknown'}, + 'transport': {'stdio', 'websocket', 'http', 'unknown'}, + 'runner_usage': {'agent', 'event'}, + 'os': {'linux', 'darwin', 'windows'}, + 'arch': {'x86_64', 'aarch64', 'arm64', 'amd64'}, + 'database': {'sqlite', 'postgresql'}, + 'edition': {'community', 'cloud', 'enterprise'}, +} +BOOLS = frozenset('stream synthetic configured available previous_session_unclean recovered supported'.split()) +NUMBERS = frozenset( + 'attempts successes failures cancellations timeouts partial unknown generated queued acked dropped retried failed queue_size capacity result_count input_tokens output_tokens'.split() +) +VERSIONS = frozenset('sdk_version plugin_version runner_version python_version'.split()) + + +def code_value(field: str, value: str) -> str: + """Register a literal from trusted Core source, not a runtime string.""" + VOCABULARY.setdefault(field, set()).add(value) + return value + + +def category(field: str, value) -> str: + return value if type(value) is str and value in VOCABULARY.get(field, ()) else '' + + +def opaque(value) -> str: + if not isinstance(value, str): + return '' + try: + return str(UUID(value)) + except (ValueError, TypeError, AttributeError): + return '' + + +def version_string(value) -> str: + # Public package versions only; do not permit arbitrary PEP440 local labels. + return value if isinstance(value, str) and re.fullmatch(r'[0-9][0-9.abrcdevpost-]{0,47}', value) else '' + + +def attributes(values) -> dict: + if not isinstance(values, dict): + return {} + result = {} + for key, value in values.items(): + if key in BOOLS and type(value) is bool: + result[key] = value + elif key in NUMBERS and type(value) is int and math.isfinite(value) and 0 <= value <= 1_000_000: + result[key] = value + elif key in VERSIONS and version_string(value): + result[key] = value + elif key in VOCABULARY and key not in {'operation', 'stage', 'adapter', 'platform_event_type', 'reason_code'}: + if category(key, value): + result[key] = value + elif key == 'capability_name' and any(category(f, value) for f in ('operation', 'platform_event_type')): + result[key] = value + elif key == 'capability_name' and value in PROCESSORS - {''}: + result[key] = value + elif key in ('event_types', 'api_operations', 'processor_types') and isinstance(value, (tuple, list)): + allowed = ( + VOCABULARY['platform_event_type'] + if key == 'event_types' + else VOCABULARY['operation'] + if key == 'api_operations' + else PROCESSORS + ) + result[key] = [v for v in value[:128] if type(v) is str and v in allowed] + elif ( + key in ('source_revision', 'target_revision') + and isinstance(value, str) + and re.fullmatch('[a-f0-9]{40}', value) + ): + result[key] = value + elif key == 'latency_buckets' and isinstance(value, (list, tuple)): + result[key] = [v for v in value[:128] if type(v) is int and 0 <= v <= 1_000_000] + return result + + +def error_fields(error: BaseException, operation: str, stage: str) -> dict: + cls = type(error) + # Third party exception names may be dynamically constructed from input. + name = cls.__name__ if getattr(builtins, cls.__name__, None) is cls else 'Exception' + if isinstance(error, TimeoutError): + name = 'TimeoutError' + # No message, traceback, filename, line number, or locals are inspected. + fingerprint = hashlib.sha256(f'{operation}|{stage}|{cls.__module__}|{cls.__qualname__}'.encode()).hexdigest() + return {'error_type': name, 'error_fingerprint': fingerprint} diff --git a/src/langbot/pkg/telemetry/diagnostic_transport.py b/src/langbot/pkg/telemetry/diagnostic_transport.py new file mode 100644 index 000000000..4220bd1a1 --- /dev/null +++ b/src/langbot/pkg/telemetry/diagnostic_transport.py @@ -0,0 +1,399 @@ +"""Bounded, memory-only Beta diagnostics transport, isolated from usage telemetry.""" + +from __future__ import annotations + +import asyncio +import contextlib +import importlib.metadata +import json +import os +import random +import re +import time +import platform +import sys +from pathlib import Path +from collections import Counter +from datetime import datetime, timezone +from uuid import uuid4 + +import httpx +from packaging.version import Version, InvalidVersion + +from . import diagnostic_privacy as privacy + + +class DiagnosticsManager: + """All producers are synchronous; only this owned worker performs I/O. + + Retention: 2048 events / 15 minutes, 5 attempts, 8 KiB per event. An in-flight + event stays in the same bounded queue until explicitly ACKed. No disk spool + means abrupt process loss cannot be counted after restart (documented). + """ + + def __init__(self, ap, *, version=None, instance_id=None, capacity=2048, marker_path=None): + from ..utils import constants + from .diagnostic_catalog import catalog + + with contextlib.suppress(Exception): + catalog() + self.ap = ap + self.marker_path = Path(marker_path) if marker_path is not None else None + self._session_id = str(uuid4()) + self._marker_active = False + raw_version = version or importlib.metadata.version('langbot') + try: + parsed = Version(raw_version) + self.version = str(parsed) + self.beta = bool(parsed.pre and parsed.pre[0] == 'b' and not parsed.local) + except InvalidVersion: + self.version, self.beta = '', False + self.instance_id = instance_id or constants.instance_id + try: + from langbot._build_info import CORE_REVISION + except ImportError: + CORE_REVISION = '' + candidate = os.getenv('LANGBOT_BUILD_REVISION', CORE_REVISION) + self.revision = candidate.lower() if re.fullmatch('[a-fA-F0-9]{40}', candidate or '') else '' + try: + self.sdk_version = privacy.version_string(importlib.metadata.version('langbot-plugin')) + except importlib.metadata.PackageNotFoundError: + self.sdk_version = '' + self.capacity = max(1, min(capacity, 2048)) + self.pending: list[dict] = [] + self.counters = Counter() + self._reported = Counter() + self._last_retry = False + self.max_attempts = 5 + self.retention_seconds = 900 + self.request_timeout = 10 + self.client: httpx.AsyncClient | None = None + self._attempts: dict[str, int] = {} + self._born: dict[str, float] = {} + self._worker = None + self._request = None + self._closing = False + self._wake = asyncio.Event() + self._flush_lock = asyncio.Lock() + + async def start_session(self): + """One bounded, content-free marker per process; never per-event disk I/O.""" + if not self.enabled: + await self._clear_marker() + return + previous = False + if self.marker_path is not None: + + def mark(): + previous = self.marker_path.exists() + self.marker_path.parent.mkdir(parents=True, exist_ok=True) + self.marker_path.write_text(self._session_id) + return previous + + with contextlib.suppress(Exception): + previous = await asyncio.to_thread(mark) + self._marker_active = True + privacy.code_value('operation', 'startup.session') + self.emit( + 'lifecycle', + 'startup.session', + 'started', + source='startup', + stage='snapshot', + attributes={ + 'previous_session_unclean': previous, + 'os': platform.system().lower(), + 'arch': platform.machine().lower(), + 'python_version': '.'.join(map(str, sys.version_info[:3])), + 'database': self.ap.instance_config.data.get('database', {}).get('use', ''), + 'edition': self.ap.instance_config.data.get('system', {}).get('edition', ''), + }, + ) + + async def _clear_marker(self): + if self.marker_path is None: + return + + def remove(): + if self.marker_path.exists(): + with self.marker_path.open() as handle: + owner = handle.read(64) + if owner == self._session_id or not self._marker_active: + self.marker_path.unlink(missing_ok=True) + + with contextlib.suppress(Exception): + await asyncio.to_thread(remove) + self._marker_active = False + + @property + def enabled(self): + config = getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('space', {}) + return ( + self.beta + and not self._closing + and not config.get('disable_telemetry', False) + and not config.get('disable_beta_diagnostics', False) + and bool(config.get('url')) + ) + + def clear(self): + self.pending.clear() + self._attempts.clear() + self._born.clear() + if self._request is not None and not self._request.done(): + self._request.cancel() + + def emit( + self, + kind, + operation, + outcome, + *, + source='internal', + workspace_uuid='', + attributes=None, + stage='execute', + adapter='', + processor_type='', + platform_event_type='', + reason_code='', + duration_ms=0, + error=None, + trace_id='', + span_id='', + parent_span_id='', + run_id='', + **ignored, + ): + """Project known scalar fields; failures in diagnostics never escape.""" + try: + if not self.enabled: + self.clear() + return + if kind not in privacy.KINDS or outcome not in privacy.OUTCOMES: + return + self.counters['generated'] += 1 + event = { + 'schema_version': 1, + 'event_id': str(uuid4()), + 'kind': kind, + 'instance_id': self.instance_id, + 'workspace_uuid': privacy.opaque(workspace_uuid), + 'core_version': self.version, + 'core_revision': self.revision, + 'sdk_version': self.sdk_version, + 'release_channel': 'beta', + 'source': source if source in privacy.SOURCES else 'internal', + 'operation': privacy.category('operation', operation), + 'stage': privacy.category('stage', stage), + 'outcome': outcome, + 'reason_code': privacy.category('reason_code', reason_code), + 'adapter': privacy.category('adapter', adapter), + 'processor_type': processor_type if processor_type in privacy.PROCESSORS else '', + 'platform_event_type': privacy.category('platform_event_type', platform_event_type), + 'occurred_at': datetime.now(timezone.utc).isoformat(), + 'count': 1, + 'sample_rate': 1, + 'duration_ms': max(0, min(float(duration_ms), 86400000)), + 'attributes': privacy.attributes(attributes), + } + for key, value in ( + ('trace_id', trace_id), + ('span_id', span_id), + ('parent_span_id', parent_span_id), + ('run_id', run_id), + ): + if privacy.opaque(value): + event[key] = privacy.opaque(value) + if error is not None: + event.update(privacy.error_fields(error, event['operation'], event['stage'])) + encoded = json.dumps(event, allow_nan=False).encode() + if ( + not isinstance(self.instance_id, str) + or not 0 < len(self.instance_id) <= 128 + or len(encoded) > 8192 + or len(self.pending) >= self.capacity + ): + self.counters['dropped'] += 1 + return + self.pending.append(event) + self._born[event['event_id']] = time.monotonic() + self.counters['queued'] += 1 + self._wake.set() + except Exception: + self.counters['dropped'] += 1 + + async def credentials(self, workspace_uuid): + # Resolve each Workspace independently in the background. No OSS secret. + token = os.getenv('LANGBOT_TELEMETRY_INGEST_TOKEN', '').strip() + if token: + return {'X-LangBot-Telemetry-Token': token} + if not workspace_uuid: + return {} + users = getattr(self.ap, 'user_service', None) + space = getattr(self.ap, 'space_service', None) + if users is None or space is None: + return {} + owner = await users.get_workspace_owner(workspace_uuid) + email = getattr(owner, 'user', None) + token = await space.get_valid_access_token(email) if email else None + return {'Authorization': f'Bearer {token}'} if token else {} + + def _remove(self, ids): + self.pending[:] = [e for e in self.pending if e['event_id'] not in ids] + for event_id in ids: + self._attempts.pop(event_id, None) + self._born.pop(event_id, None) + + async def flush_once(self): + async with self._flush_lock: + if not self.enabled: + self.clear() + return + expired = { + e['event_id'] + for e in self.pending + if time.monotonic() - self._born.get(e['event_id'], 0) > self.retention_seconds + } + self.counters['dropped'] += len(expired) + self._remove(expired) + if not self.pending: + return + # Do not authenticate a mixed-Workspace batch with one owner's token. + workspace = self.pending[0]['workspace_uuid'] + batch, size = [], 64 + for event in self.pending: + n = len(json.dumps(event).encode()) + 2 + if event['workspace_uuid'] != workspace: + continue + if len(batch) == 50 or size + n > 256 * 1024: + break + batch.append(event) + size += n + ids = {e['event_id'] for e in batch} + for event_id in ids: + self._attempts[event_id] = self._attempts.get(event_id, 0) + 1 + acked, rejected = set(), set() + permanent = False + try: + async with asyncio.timeout(self.request_timeout): + headers = {} + try: + async with asyncio.timeout(min(1.0, self.request_timeout / 3)): + headers = await self.credentials(workspace) + except Exception: + pass + if not self.enabled: + self.clear() + return + if self.client is None: + self.client = httpx.AsyncClient(timeout=self.request_timeout, follow_redirects=False) + url = ( + self.ap.instance_config.data['space']['url'].rstrip('/') + '/api/v1/telemetry/diagnostics/batch' + ) + # Stream the response to bound malicious/old server responses too. + async with self.client.stream( + 'POST', url, json={'schema_version': 1, 'events': batch}, headers=headers + ) as response: + body = bytearray() + async for chunk in response.aiter_bytes(): + body.extend(chunk) + if len(body) > 65536: + raise ValueError('diagnostics_response_limit') + if response.status_code == 200: + payload = json.loads(body) + if payload.get('code') == 200 and isinstance(payload.get('data'), dict): + data = payload['data'] + acked = {x for x in data.get('accepted_event_ids', []) if isinstance(x, str)} & ids + rejected = { + x.get('event_id') + for x in data.get('rejected', []) + if isinstance(x, dict) and x.get('code') == 'invalid_event' + } & ids + elif isinstance(payload.get('code'), int): + permanent = 400 <= payload['code'] < 500 and payload['code'] != 429 + else: + permanent = 400 <= response.status_code < 500 and response.status_code != 429 + except asyncio.CancelledError: + raise + except Exception: + pass + if not self.enabled: + self.clear() + return + rejected -= acked + remaining = ids - acked - rejected + if remaining: + self.counters['failed'] += 1 + exhausted = {i for i in remaining if permanent or self._attempts.get(i, 0) >= self.max_attempts} + self.counters['acked'] += len(acked) + self.counters['dropped'] += len(rejected | exhausted) + self._last_retry = bool(remaining - exhausted) + self.counters['retried'] += len(remaining - exhausted) + self._remove(acked | rejected | exhausted) + + def report_transport(self, stage='snapshot'): + # Snapshot interval deltas before enqueue; retries retain this event ID. + snapshot = self.counters.copy() + delta = snapshot - self._reported + before = self.counters['queued'] + self.emit( + 'transport', + 'diagnostics.transport', + 'partial' if delta['dropped'] else 'succeeded', + stage=stage, + attributes={**dict(delta), 'queue_size': len(self.pending), 'capacity': self.capacity}, + ) + if self.counters['queued'] > before: + self._reported = snapshot + + def start(self): + if self.beta and self._worker is None: + self._worker = asyncio.create_task(self._loop(), name='beta-diagnostics') + + async def _loop(self): + next_send, next_summary, backoff = 0.0, time.monotonic() + 60, 0.0 + try: + while not self._closing: + if not self.enabled: + self.clear() + if self._marker_active: + await self._clear_marker() + elif time.monotonic() >= next_summary: + self.report_transport() + next_summary = time.monotonic() + 60 + if self._request is not None and self._request.done(): + with contextlib.suppress(asyncio.CancelledError, Exception): + self._request.result() + self._request = None + backoff = min(30, max(1, backoff * 2)) if self._last_retry else 0 + next_send = time.monotonic() + random.uniform(0.5, 1.5) * backoff + if self.enabled and self.pending and self._request is None and time.monotonic() >= next_send: + self._request = asyncio.create_task(self.flush_once(), name='beta-diagnostics-batch') + self._wake.clear() + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self._wake.wait(), 0.1) + finally: + if self._request is not None: + self._request.cancel() + await asyncio.gather(self._request, return_exceptions=True) + + async def shutdown(self, drain_timeout=2): + if self._worker is not None: + self._worker.cancel() + await asyncio.gather(self._worker, return_exceptions=True) + self._worker = None + if self.enabled and drain_timeout > 0: + self.report_transport(stage='shutdown') + with contextlib.suppress(Exception): + async with asyncio.timeout(drain_timeout): + while self.pending: + await self.flush_once() + if self.pending: + await asyncio.sleep(0.1) + self.counters['dropped'] += len(self.pending) + self._closing = True + self.clear() + await self._clear_marker() + if self.client is not None: + await self.client.aclose() diff --git a/src/langbot/pkg/telemetry/diagnostics.py b/src/langbot/pkg/telemetry/diagnostics.py new file mode 100644 index 000000000..179c6f73c --- /dev/null +++ b/src/langbot/pkg/telemetry/diagnostics.py @@ -0,0 +1,461 @@ +"""Explicit diagnostic boundaries, preserving coroutine and generator semantics.""" + +from __future__ import annotations + +import asyncio +import contextlib +import contextvars +import functools +import inspect +import time +from uuid import uuid4 + +from .diagnostic_transport import DiagnosticsManager # noqa: F401 +from . import diagnostic_privacy as privacy + +_CURRENT = contextvars.ContextVar('beta_diagnostic_span', default=None) + + +def current_span(): + return _CURRENT.get() + + +def set_outcome(outcome, *, reason_code=''): + span = current_span() + if span is not None: + span.outcome = outcome if outcome in privacy.OUTCOMES else 'unknown' + span.fields['reason_code'] = privacy.category('reason_code', reason_code) + + +def annotate(**fields): + """Trusted hook metadata is still projected at the transport boundary.""" + span = current_span() + if span is not None: + span.fields.update(fields) + + +def _owner_app(owner): + """An explicit owner (even absent/disabled) is an inheritance barrier.""" + if owner is None: + return None, False + if hasattr(owner, 'ap'): + return owner.ap, True + if hasattr(owner, 'diagnostics') or hasattr(owner, 'instance_config'): + return owner, True + for name in ('requester', 'logger', 'adapter'): + nested = getattr(owner, name, None) + if nested is not None and nested is not owner: + app, explicit = _owner_app(nested) + if explicit: + return app, True + return None, False + + +def _manager(owner): + app, _ = _owner_app(owner) + manager = getattr(app, 'diagnostics', None) + if manager is None or not callable(getattr(manager, 'emit', None)) or not getattr(manager, 'enabled', False): + return None + # Honor the shared Span/management producer interface without inheriting a + # different producer just because this application's producer is absent. + config = getattr(getattr(app, 'instance_config', None), 'data', {}).get('space', {}) + if config.get('disable_telemetry', False) or config.get('disable_beta_diagnostics', False): + return None + # Direct manager holders (e.g. ReplyStreamSession) are intentional. An app + # must never borrow another app's manager, policy, or credential resolver. + if hasattr(app, 'instance_config') and getattr(manager, 'ap', app) is not app: + return None + return manager + + +def _owner_context(owner): + context = getattr(owner, 'execution_context', None) + if context is not None: + return context + for name in ('requester', 'logger', 'adapter'): + nested = getattr(owner, name, None) + if nested is not None and nested is not owner: + context = _owner_context(nested) + if context is not None: + return context + return None + + +def _execution_context(owner, bound): + context = bound.get('execution_context') or _owner_context(owner) + adapter_context = bound.get('adapter_context') + if isinstance(adapter_context, dict): + context = adapter_context.get('_execution_context') or context + query = bound.get('query') + if query is not None: + context = getattr(query, '_execution_context', None) or context + return context + + +def _context_matches(manager, owner, context): + from ..api.http.context import ExecutionContext + + if not isinstance(context, ExecutionContext): + return True + instance = getattr(getattr(manager.ap, 'workspace_service', None), 'instance_uuid', manager.instance_id) + if instance != context.instance_uuid: + return False + owned = _owner_context(owner) + return not isinstance(owned, ExecutionContext) or ( + owned.instance_uuid, + owned.workspace_uuid, + owned.placement_generation, + ) == (context.instance_uuid, context.workspace_uuid, context.placement_generation) + + +def _context_fields(owner, bound): + context = _execution_context(owner, bound) + query = bound.get('query') + from .diagnostic_catalog import adapter_fields + + adapter = getattr(owner, 'adapter', None) or owner + fields = adapter_fields(adapter) + # ExecutionContext is constructed/validated by the existing auth boundary; + # do not infer Workspace identity from arbitrary payload dicts or event IDs. + if context is not None: + from ..api.http.context import ExecutionContext + + if isinstance(context, ExecutionContext): + fields['workspace_uuid'] = context.workspace_uuid + saved = getattr(query, '_diagnostic_context', None) if query is not None else None + if isinstance(saved, dict) and saved.get('workspace_uuid') == context.workspace_uuid: + fields.update(saved) + binding = bound.get('binding') + if binding is not None: + fields['processor_type'] = getattr(binding, 'processor_type', '') + event = bound.get('event') + if event is not None: + fields['platform_event_type'] = getattr(event, 'event_type', None) or getattr(event, 'type', '') + delivery = getattr(event, 'delivery', None) + if getattr(delivery, 'surface', None) == 'webui': + fields['source'] = 'webui_debug' + fields['attributes'] = {'synthetic': True} + if getattr(owner, 'mock', False) is True: + fields['source'] = 'webui_debug' + fields['attributes'] = {'synthetic': True} + return fields + + +class Span: + def __init__(self, manager, kind, operation, fields): + parent = current_span() + self.manager = manager + self.kind = kind + self.operation = operation + self.fields = dict(fields) + if ( + parent + and parent.fields.get('workspace_uuid') + and self.fields.get('workspace_uuid') + and parent.fields['workspace_uuid'] != self.fields['workspace_uuid'] + ): + parent = None + if self.fields.get('operation'): + self.operation = privacy.category('operation', self.fields.pop('operation')) or operation + self.outcome = None + self.finished = False + self.started = time.monotonic() + self.fields['trace_id'] = ( + parent.fields['trace_id'] + if parent and parent.manager is manager + else (privacy.opaque(self.fields.get('trace_id')) or str(uuid4())) + ) + self.fields['span_id'] = str(uuid4()) + if parent and parent.manager is manager: + self.fields['parent_span_id'] = parent.fields['span_id'] + for key in ('workspace_uuid', 'adapter', 'processor_type', 'platform_event_type', 'run_id'): + if not self.fields.get(key) and parent.fields.get(key): + self.fields[key] = parent.fields[key] + if parent.fields.get('source') in ('webui_debug', 'synthetic'): + self.fields['source'] = parent.fields['source'] + self.fields['attributes'] = {**self.fields.get('attributes', {}), 'synthetic': True} + self.emit('started') + + def emit(self, outcome, **extra): + if self.manager is not None: + self.manager.emit(self.kind, self.operation, outcome, **{**self.fields, **extra}) + + @contextlib.contextmanager + def activate(self): + token = _CURRENT.set(self) + try: + yield self + finally: + _CURRENT.reset(token) + + def finish(self, error=None): + if self.finished: + return + self.finished = True + error = error if error is not None else self.fields.pop('error', None) + if isinstance(error, (asyncio.CancelledError, GeneratorExit)): + outcome = 'cancelled' + elif isinstance(error, TimeoutError): + outcome = 'timeout' + elif error is not None: + outcome = self.outcome if self.outcome in ('timeout', 'partial', 'rejected') else 'failed' + else: + outcome = self.outcome or 'succeeded' + self.emit(outcome, error=error, duration_ms=(time.monotonic() - self.started) * 1000) + + +def result_outcome(value): + """Inspect only the SDK response's status, not arbitrary result contents.""" + from langbot_plugin.api.entities.builtin.platform.events import EBAEvent + from langbot_plugin.runtime.io.handler import ActionResponse + + span = current_span() + if span is not None and span.fields.get('stage') == 'convert': + if isinstance(value, EBAEvent): + annotate(platform_event_type=value.type) + elif value is None: + set_outcome('skipped', reason_code='not_matched') + if isinstance(value, ActionResponse): + if value.code != 0: + set_outcome('failed', reason_code='response_error') + + +def observe(kind, operation, *, source='internal', stage='execute', ap=None, fields=None): + """Explicit boundary with a stable/off fast path and transparent generators.""" + privacy.code_value('operation', operation) + privacy.code_value('stage', stage) + + def decorate(fn): + signature = inspect.signature(fn) + + def span_for(args, kwargs): + try: + bound = signature.bind_partial(*args, **kwargs).arguments + owner = bound.get(next(iter(signature.parameters), '')) + manager_owner = (ap() if callable(ap) else ap) if ap is not None else owner + manager = _manager(manager_owner) + _, explicit = _owner_app(manager_owner) + context = _execution_context(owner, bound) + parent = current_span() + if manager is None and not explicit and ap is None and parent is not None: + # Stateless converters may inherit, but a different explicit + # Workspace must not select a parent's credentials/policy. + if context is not None and getattr(context, 'workspace_uuid', None) != parent.fields.get( + 'workspace_uuid' + ): + return None + manager = parent.manager + if manager is None or not manager.enabled: + return None + if not _context_matches(manager, owner, context): + return None + metadata = {'source': source, 'stage': stage} + metadata.update(_context_fields(owner, bound)) + from .diagnostic_catalog import catalog + + for directory, entry in catalog().items(): + if directory in fn.__module__.split('.') and '.platform.' in fn.__module__: + metadata['adapter'] = entry['adapter'] + break + if fields: + extra = fields(bound) + extra['attributes'] = {**metadata.get('attributes', {}), **extra.get('attributes', {})} + metadata.update(extra) + return Span(manager, kind, operation, metadata) + except Exception: + return None + + def inspect_result(value): + with contextlib.suppress(Exception): + result_outcome(value) + + def finish(span, error=None): + if span is not None: + with contextlib.suppress(Exception): + span.finish(error) + + if inspect.isasyncgenfunction(fn): + from collections.abc import AsyncGenerator + + class ObservedGenerator(AsyncGenerator): + """Delegate each native protocol operation, without extra close. + + A yield-based proxy cannot distinguish athrow(GeneratorExit) + (which may yield) from aclose() (which must reject a yield). + Let the native generator implement that distinction and retain + its own primary/cleanup exception and cancellation semantics. + """ + + def __init__(self, args, kwargs): + self.gen = fn(*args, **kwargs) + self.args, self.kwargs = args, kwargs + self.span = None + self.started = False + + def __getattr__(self, name): + return getattr(self.gen, name) + + async def _advance(self, method, *values): + if not self.started: + self.started = True + self.span = span_for(self.args, self.kwargs) + token = _CURRENT.set(self.span) + try: + value = await method(*values) + if self.span is not None and method != self.gen.aclose: + inspect_result(value) + if method == self.gen.aclose: + finish(self.span, GeneratorExit()) + return value + except StopAsyncIteration: + finish(self.span) + raise + except BaseException as exc: + # Rejected protocol calls (e.g. concurrent asend, a + # yielded GeneratorExit) need not terminate the stream. + if self.gen.ag_frame is None: + finish(self.span, exc) + raise + finally: + _CURRENT.reset(token) + + def __anext__(self): + return self._advance(self.gen.__anext__) + + def asend(self, value): + return self._advance(self.gen.asend, value) + + def athrow(self, *values): + return self._advance(self.gen.athrow, *values) + + def aclose(self): + return self._advance(self.gen.aclose) + + @functools.wraps(fn) + def stream(*args, **kwargs): + return ObservedGenerator(args, kwargs) + + return stream + + @functools.wraps(fn) + async def call(*args, **kwargs): + span = span_for(args, kwargs) + if span is None: + token = _CURRENT.set(None) + try: + return await fn(*args, **kwargs) + finally: + _CURRENT.reset(token) + try: + with span.activate(): + value = await fn(*args, **kwargs) + inspect_result(value) + finish(span) + return value + except BaseException as exc: + finish(span, exc) + raise + + return call + + return decorate + + +def event(owner, kind, operation, outcome, **fields): + """Emit a point-in-time fact from an existing state transition.""" + try: + manager = _manager(owner) + _, explicit = _owner_app(owner) + parent = current_span() + context = _owner_context(owner) + if manager is None and not explicit and parent: + if context is not None and getattr(context, 'workspace_uuid', None) != parent.fields.get('workspace_uuid'): + return + manager = parent.manager + if manager is not None and _context_matches(manager, owner, context): + inherited = dict(parent.fields) if parent and parent.manager is manager else {} + if context is not None and getattr(context, 'workspace_uuid', None) != inherited.get('workspace_uuid'): + inherited = _context_fields(owner, {}) + inherited.update(fields) + manager.emit(kind, operation, outcome, **inherited) + except Exception: + pass + + +def capture_context(): + span = current_span() + if span is None or not span.manager.enabled: + return None + result = { + k: span.fields[k] + for k in ('trace_id', 'workspace_uuid', 'run_id', 'adapter', 'processor_type', 'platform_event_type', 'source') + if k in span.fields + } + result['parent_span_id'] = span.fields['span_id'] + return result + + +def link_context(saved): + """Link only after the existing Host run/installation validator accepted it.""" + span = current_span() + if span is None or not isinstance(saved, dict): + return + if span.fields.get('workspace_uuid') and span.fields['workspace_uuid'] != saved.get('workspace_uuid'): + return + for key in ('trace_id', 'parent_span_id', 'workspace_uuid', 'run_id'): + if privacy.opaque(saved.get(key)): + span.fields[key] = saved[key] + for key in ('adapter', 'platform_event_type'): + if privacy.category(key, saved.get(key)): + span.fields[key] = saved[key] + if saved.get('processor_type') in privacy.PROCESSORS: + span.fields['processor_type'] = saved['processor_type'] + if saved.get('source') in ('webui_debug', 'synthetic'): + span.fields['source'] = saved['source'] + span.fields['attributes'] = {**span.fields.get('attributes', {}), 'synthetic': True} + + +def declare_runner(descriptor): + """Allow only identifiers from a validated installed public Runner manifest.""" + import re + + pairs = {'plugin_id': descriptor.get_plugin_id(), 'runner_id': descriptor.id} + for field, value in pairs.items(): + allowed = privacy.VOCABULARY.setdefault(field, set()) + if ( + isinstance(value, str) + and len(value) <= 128 + and len(allowed) < 1024 + and re.fullmatch(r'(?:plugin:)?[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+(?:/[A-Za-z0-9_.-]+)?', value) + ): + allowed.add(value) + + +def runner_metadata(owner, descriptor, processor_type): + manager = _manager(owner) + if manager is None or not manager.enabled: + return + metadata = { + 'plugin_id': descriptor.get_plugin_id(), + 'runner_id': descriptor.id, + 'plugin_version': getattr(descriptor, 'plugin_version', ''), + 'runner_usage': 'event' if processor_type == 'event_processor' else 'agent', + } + annotate(attributes=metadata) + event( + owner, + 'capability', + 'runner.run', + 'succeeded', + source='runtime', + stage='snapshot', + processor_type=processor_type, + attributes={ + **metadata, + 'capability_type': 'processor', + 'capability_name': processor_type, + 'supported': True, + 'configured': True, + 'available': True, + }, + ) diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index a4a97963d..f53864094 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -416,4 +416,8 @@ space: # OAuth authorization page URL (user will be redirected here) oauth_authorize_url: 'https://space.langbot.app/auth/authorize' disable_models_service: false + # Master OFF for usage telemetry and Beta diagnostics. disable_telemetry: false + # Beta builds only: opt out of content-free quality diagnostics separately. + # Stable, alpha and RC builds never enable this diagnostics producer. + disable_beta_diagnostics: false diff --git a/tests/unit_tests/agent/test_orchestrator_integration.py b/tests/unit_tests/agent/test_orchestrator_integration.py index ad55fbd70..2bac63dbb 100644 --- a/tests/unit_tests/agent/test_orchestrator_integration.py +++ b/tests/unit_tests/agent/test_orchestrator_integration.py @@ -1473,3 +1473,56 @@ async def test_synthetic_event_query_exposes_trusted_workspace_to_tools(clean_ag app.skill_mgr.get_skills = lambda scope: received.append(scope) or {} get_visible_skills(app, synthetic) assert received[0].workspace_uuid == context.workspace_uuid + + +@pytest.mark.asyncio +async def test_beta_diagnostics_close_releases_real_orchestrator_session(clean_agent_state): + from langbot.pkg.telemetry.diagnostics import DiagnosticsManager + + plugin_connector = FakePluginConnector( + results=[{'type': 'message.completed', 'data': {'message': {'role': 'assistant', 'content': 'CANARY'}}}] + ) + ap = FakeApplication(plugin_connector, clean_agent_state) + ap.instance_config = types.SimpleNamespace(data={'space': {'url': 'https://example.invalid'}}) + ap.diagnostics = DiagnosticsManager(ap, version='4.11.0b2', instance_id='instance-test') + orchestrator = AgentRunOrchestrator(ap, FakeRegistry(make_descriptor())) + gen = orchestrator.run_from_query(make_query()) + assert (await anext(gen)).content == 'CANARY' + run_id = plugin_connector.contexts[0]['run_id'] + assert await get_session_registry().get(run_id) is not None + await gen.aclose() + assert await get_session_registry().get(run_id) is None + records = [e for e in ap.diagnostics.pending if e['operation'] == 'runner.run'] + assert records[-1]['outcome'] == 'cancelled' + import json + + assert 'CANARY' not in json.dumps(ap.diagnostics.pending) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('terminal', ['run.completed', 'run.failed']) +async def test_beta_diagnostics_real_runner_terminal(clean_agent_state, terminal): + from langbot.pkg.telemetry.diagnostics import DiagnosticsManager + + plugin_connector = FakePluginConnector( + results=[ + { + 'type': terminal, + 'data': {'finish_reason': 'stop'} if terminal == 'run.completed' else {'error': 'CANARY'}, + } + ] + ) + ap = FakeApplication(plugin_connector, clean_agent_state) + ap.instance_config = types.SimpleNamespace(data={'space': {'url': 'https://example.invalid'}}) + ap.diagnostics = DiagnosticsManager(ap, version='4.11.0b2', instance_id='instance-test') + orchestrator = AgentRunOrchestrator(ap, FakeRegistry(make_descriptor())) + try: + _ = [v async for v in orchestrator.run_from_query(make_query())] + except RunnerExecutionError: + assert terminal == 'run.failed' + records = [e for e in ap.diagnostics.pending if e['operation'] == 'runner.run'] + assert records[-1]['outcome'] == ('succeeded' if terminal == 'run.completed' else 'failed') + assert records[-1]['run_id'] == plugin_connector.contexts[0]['run_id'] + import json + + assert 'CANARY' not in json.dumps(ap.diagnostics.pending) diff --git a/tests/unit_tests/api/test_diagnostics_management_http.py b/tests/unit_tests/api/test_diagnostics_management_http.py new file mode 100644 index 000000000..5c7186d6c --- /dev/null +++ b/tests/unit_tests/api/test_diagnostics_management_http.py @@ -0,0 +1,194 @@ +"""Real Quart registration boundaries with content-canary payloads.""" + +import asyncio +import json +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +import quart + +from langbot.pkg.api.http.controller.group import AuthType, RouterGroup +from langbot.pkg.telemetry import diagnostics as d + + +class Recorder: + enabled = True + + def __init__(self): + self.events = [] + + def emit(self, kind, operation, outcome, **fields): + # A real transport projects the exception class, never its message. + error = fields.pop('error', None) + if error is not None: + fields['error_type'] = type(error).__name__ + self.events.append(dict(kind=kind, operation=operation, outcome=outcome, **fields)) + + +class Routes(RouterGroup): + path = '/management' + name = 'management' + + async def initialize(self): + @self.route('/ok/', auth_type=AuthType.NONE) + async def success(identifier): + self.ap.seen.append(d.current_span()) + return self.success({'secret': identifier}) + + @self.route('/business', auth_type=AuthType.NONE) + async def business(): + return self.fail('private-error-code', 'private-error-message') + + @self.route('/auth') + async def authenticated(): + raise AssertionError('must not run') + + @self.route('/error', auth_type=AuthType.NONE) + async def error(): + raise ValueError('private-exception-message') + + @self.route('/cancel', auth_type=AuthType.NONE) + async def cancel(): + raise asyncio.CancelledError('private-cancel-message') + + @self.route('/stream', auth_type=AuthType.NONE) + async def stream(): + async def body(): + self.ap.seen.append(d.current_span()) + yield b'private-stream-chunk' + self.ap.seen.append(d.current_span()) + + return quart.Response(body()) + + +async def setup(manager=True): + app = quart.Quart(__name__) + ap = SimpleNamespace(seen=[]) + if manager is True: + manager = Recorder() + if manager is not None: + ap.diagnostics = manager + routes = Routes(ap, app) + await routes.initialize() + return app, ap, routes + + +@pytest.mark.asyncio +async def test_http_success_uses_code_identity_not_path_or_payload(): + app, ap, _ = await setup() + response = await app.test_client().get( + '/management/ok/private-id?token=private-query', headers={'Authorization': 'private-token'} + ) + assert (await response.get_json())['data']['secret'] == 'private-id' + assert [e['outcome'] for e in ap.diagnostics.events] == ['started', 'succeeded'] + event = ap.diagnostics.events[-1] + assert event['source'] == 'http' + assert re.fullmatch(r'[A-Za-z_][A-Za-z0-9_.:-]{0,127}', event['operation']) + assert ap.seen[0] is not None + assert d.current_span() is None + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'path,status,outcome', [('business', 200, 'failed'), ('auth', 401, 'rejected'), ('error', 500, 'failed')] +) +async def test_http_business_auth_and_exception_outcomes(path, status, outcome): + app, ap, _ = await setup() + response = await app.test_client().get('/management/' + path) + assert response.status_code == status + assert ap.diagnostics.events[-1]['outcome'] == outcome + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +async def test_http_cancellation_propagates(): + app, ap, _ = await setup() + async with app.test_request_context('/management/cancel'): + with pytest.raises(asyncio.CancelledError): + await app.full_dispatch_request() + assert ap.diagnostics.events[-1]['outcome'] == 'cancelled' + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_http_auth_cancellation_not_reinterpreted_as_api_key(): + app, ap, routes = await setup() + routes._authenticate_support_admin = AsyncMock(return_value=None) + routes._authenticate_account = AsyncMock(side_effect=asyncio.CancelledError()) + async with app.test_request_context('/management/auth', headers={'Authorization': 'Bearer private-token'}): + with pytest.raises(asyncio.CancelledError): + await app.full_dispatch_request() + assert ap.diagnostics.events[-1]['outcome'] == 'cancelled' + + +@pytest.mark.asyncio +async def test_http_stream_has_parent_context_without_consumer_leak(): + app, ap, _ = await setup() + async with app.test_request_context('/management/stream'): + response = await app.full_dispatch_request() + assert [e['outcome'] for e in ap.diagnostics.events] == ['started'] + assert d.current_span() is None + async with response.response as body: + iterator = body.__aiter__() + assert await anext(iterator) == b'private-stream-chunk' + assert d.current_span() is None + assert ap.seen[-1] is not None + with pytest.raises(StopAsyncIteration): + await anext(iterator) + assert ap.seen[0] is ap.seen[1] + assert ap.diagnostics.events[-1]['outcome'] == 'succeeded' + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'manager', [None, SimpleNamespace(enabled=False, emit=lambda *a, **kw: pytest.fail('disabled emission'))] +) +async def test_absent_disabled_manager_preserves_result_without_context(manager): + app, ap, _ = await setup(manager) + response = await app.test_client().get('/management/ok/private-id') + assert response.status_code == 200 + assert ap.seen == [None] + + +@pytest.mark.asyncio +async def test_broken_diagnostic_emit_never_masks_operation(): + class Broken(Recorder): + def emit(self, *args, **kwargs): + raise RuntimeError('diagnostics broken') + + app, _, _ = await setup(Broken()) + response = await app.test_client().get('/management/ok/private-id') + assert response.status_code == 200 + async with app.test_request_context('/management/cancel'): + with pytest.raises(asyncio.CancelledError): + await app.full_dispatch_request() + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_anonymous_handlers_have_distinct_stable_code_operations(): + app, ap, routes = await setup() + + @routes.route('/items/', auth_type=AuthType.NONE, methods=['GET']) + async def _(identifier): + return routes.success() + + @routes.route('/items/', auth_type=AuthType.NONE, methods=['POST']) + async def _(identifier): + return routes.success() + + @routes.route('/other', auth_type=AuthType.NONE) + async def _(): + return routes.success() + + await app.test_client().get('/management/items/private-id') + await app.test_client().post('/management/items/private-id') + await app.test_client().get('/management/other') + operations = [e['operation'] for e in ap.diagnostics.events if e['outcome'] == 'started'] + assert len(set(operations)) == 3 + assert all(re.fullmatch(r'[A-Za-z_][A-Za-z0-9_.:-]{0,127}', op) for op in operations) + assert 'private-' not in json.dumps(ap.diagnostics.events) diff --git a/tests/unit_tests/api/test_diagnostics_management_inventory.py b/tests/unit_tests/api/test_diagnostics_management_inventory.py new file mode 100644 index 000000000..7a97fd175 --- /dev/null +++ b/tests/unit_tests/api/test_diagnostics_management_inventory.py @@ -0,0 +1,47 @@ +"""Verify identities across every source-declared management registration.""" + +import ast +from pathlib import Path +import re +from types import SimpleNamespace + +from langbot.pkg.api.management_diagnostics import operation_id + + +def test_every_http_registration_has_a_distinct_wire_safe_operation(): + root = Path(__file__).resolve().parents[3] / 'src/langbot/pkg/api/http/controller/groups' + identities = [] + for path in root.rglob('*.py'): + module = 'langbot.pkg.api.http.controller.groups.' + '.'.join(path.relative_to(root).with_suffix('').parts) + for cls in ast.walk(ast.parse(path.read_text())): + if not isinstance(cls, ast.ClassDef): + continue + prefix = '' + for decorator in cls.decorator_list: + if ( + isinstance(decorator, ast.Call) + and isinstance(decorator.func, ast.Attribute) + and decorator.func.attr == 'group_class' + ): + prefix = ast.literal_eval(decorator.args[1]) + for fn in ast.walk(cls): + if not isinstance(fn, ast.AsyncFunctionDef): + continue + for decorator in fn.decorator_list: + if not ( + isinstance(decorator, ast.Call) + and isinstance(decorator.func, ast.Attribute) + and decorator.func.attr == 'route' + ): + continue + rule = prefix + ast.literal_eval(decorator.args[0]) + methods = next( + (ast.literal_eval(k.value) for k in decorator.keywords if k.arg == 'methods'), ['GET'] + ) + operation = operation_id( + 'http', SimpleNamespace(__module__=module, __name__=fn.name), rule=rule, methods=methods + ) + assert re.fullmatch(r'[A-Za-z_][A-Za-z0-9_.:-]{0,127}', operation), operation + identities.append(operation) + assert len(identities) >= 200 # Guard against accidentally scanning an empty/subset tree. + assert len(set(identities)) == len(identities) diff --git a/tests/unit_tests/api/test_diagnostics_management_isolation.py b/tests/unit_tests/api/test_diagnostics_management_isolation.py new file mode 100644 index 000000000..c55eb7bb2 --- /dev/null +++ b/tests/unit_tests/api/test_diagnostics_management_isolation.py @@ -0,0 +1,431 @@ +"""CORE-DIAG-6: management ownership barriers under a foreign live span.""" + +import asyncio +from types import SimpleNamespace as NS +from unittest.mock import AsyncMock, patch +from uuid import uuid4 + +import pytest +import quart + +from langbot.pkg.api import management_diagnostics as md +from langbot.pkg.api.http.context import ( + ExecutionContext, + PrincipalContext, + PrincipalType, + RequestContext, + WorkspaceContext, +) +from langbot.pkg.api.http.controller.group import AuthType +from langbot.pkg.api.http.controller.groups.pipelines.embed import EmbedRouterGroup +from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import WebSocketChatRouterGroup +from langbot.pkg.api.http.service.agent import AgentService +from langbot.pkg.api.mcp.server import LangBotMCPServer +from langbot.pkg.telemetry import diagnostics as d +from tests.unit_tests.api.test_diagnostics_management_http import Recorder, Routes + + +MODES = ['enabled', 'disabled', 'absent', 'mismatched', 'policy_off', 'beta_off', 'broken'] + + +def app(identity): + ap = NS(instance_config=NS(data={'space': {'url': 'https://example.invalid'}})) + ap.diagnostics = d.DiagnosticsManager(ap, version='4.11.0b2', instance_id=identity) + return ap + + +def owners(mode): + a, b = app('instance-A'), app('instance-B') + b_manager = b.diagnostics + if mode == 'disabled': + b.instance_config.data['space']['disable_telemetry'] = True + b.diagnostics = d.DiagnosticsManager(b, version='4.11.0b2', instance_id='instance-B') + b_manager = b.diagnostics + elif mode == 'absent': + del b.diagnostics + elif mode == 'mismatched': + b.diagnostics = a.diagnostics + elif mode in {'policy_off', 'beta_off'}: + # Also test an attached structural producer whose enabled flag stays true. + b.diagnostics = Recorder() + b.instance_config.data['space']['disable_telemetry' if mode == 'policy_off' else 'disable_beta_diagnostics'] = ( + True + ) + elif mode == 'broken': + b.diagnostics = NS(enabled=True, emit=lambda *a, **kw: (_ for _ in ()).throw(RuntimeError('broken'))) + ca = ExecutionContext('instance-A', str(uuid4()), 1) + cb = ExecutionContext('instance-B', str(uuid4()), 1) + d.privacy.code_value('operation', 'http.isolation.parent') + parent = d.Span(a.diagnostics, 'api', 'http.isolation.parent', {'workspace_uuid': ca.workspace_uuid}) + return a, b, b_manager, ca, cb, parent + + +@d.observe('event', 'platform.target2yiri', source='platform', stage='convert') +async def converter(native): + return native + + +async def nested(b, cb, seen): + md.workspace(cb) + seen.append(d.current_span()) + with md.scope(b, 'websocket.isolation.inner', source='websocket'): + md.workspace(cb) + assert await converter(42) == 42 + # Both an inner management boundary and an ownerless converter must be safe. + assert await converter(43) == 43 + return 44 + + +def assert_isolated(a, b, b_manager, ca, cb, parent, mode): + assert parent.fields['workspace_uuid'] == ca.workspace_uuid + assert parent.outcome is None + assert len(a.diagnostics.pending) == 1 # Only the caller's own start event. + assert not b_manager.pending if mode != 'enabled' else b_manager.pending + if isinstance(getattr(b, 'diagnostics', None), Recorder): + assert b.diagnostics.events == [] + if mode == 'enabled': + assert all(e['instance_id'] == 'instance-B' for e in b_manager.pending) + assert all(e['trace_id'] != parent.fields['trace_id'] for e in b_manager.pending) + assert all(e['workspace_uuid'] != ca.workspace_uuid for e in b_manager.pending) + assert any(e['workspace_uuid'] == cb.workspace_uuid for e in b_manager.pending) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +@pytest.mark.parametrize('shape', ['decorator', 'scope']) +async def test_nested_management_masks_foreign_parent_and_restores_caller(mode, shape): + a, b, manager, ca, cb, parent = owners(mode) + seen = [] + + @md.observe('http.isolation.endpoint', source='http', ap=b) + async def endpoint(): + return await nested(b, cb, seen) + + with parent.activate(): + if shape == 'decorator': + assert await endpoint() == 44 + else: + with md.scope(b, 'websocket.isolation.outer', source='websocket'): + assert await nested(b, cb, seen) == 44 + assert d.current_span() is parent + assert d.current_span() is None + assert (seen[0] is not None) == (mode == 'enabled') + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +async def test_actual_agent_debug_never_mutates_foreign_workspace(mode): + a, b, manager, ca, cb, parent = owners(mode) + service = AgentService(b) + seen = [] + + async def missing(*args): + await nested(b, cb, seen) + return None + + service.get_agent = missing + with parent.activate(): + with pytest.raises(ValueError, match='^Agent not found$'): + await service.debug_agent(cb, 'private-agent', {}) + assert d.current_span() is parent + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +@pytest.mark.parametrize('termination', ['exhaust', 'close', 'error', 'cancel']) +async def test_real_http_body_keeps_barrier_during_advancement_and_cleanup(mode, termination): + a, b, manager, ca, cb, parent = owners(mode) + web = quart.Quart(__name__) + routes = Routes(b, web) + seen, closed = [], [] + failure = asyncio.CancelledError('private-cancel') if termination == 'cancel' else ValueError('private-error') + + @routes.route('/isolated', auth_type=AuthType.NONE) + async def endpoint(): + await nested(b, cb, seen) + + async def stream(): + try: + await nested(b, cb, seen) + yield b'one' + if termination in {'cancel', 'error'}: + raise failure + await nested(b, cb, seen) + finally: + await nested(b, cb, seen) + closed.append(True) + + return quart.Response(stream()) + + # Construct with no caller parent; the later body consumer has app A's span. + async with web.test_request_context('/management/isolated'): + response = await web.full_dispatch_request() + with parent.activate(): + try: + async with response.response as body: + iterator = body.__aiter__() + assert await anext(iterator) == b'one' + assert d.current_span() is parent + if termination != 'close': + with pytest.raises((StopAsyncIteration, type(failure))) as caught: + await anext(iterator) + if termination in {'cancel', 'error'}: + assert caught.value is failure + else: + assert isinstance(caught.value, StopAsyncIteration) + finally: + assert d.current_span() is parent + assert closed == [True] + assert all((span is not None) == (mode == 'enabled') for span in seen) + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +async def test_real_mcp_registered_tool_masks_foreign_parent(mode): + a, b, manager, ca, cb, parent = owners(mode) + seen = [] + + async def get_bot(*args, **kwargs): + return {'value': await nested(b, cb, seen)} + + b.bot_service = NS(get_bot=get_bot) + server = LangBotMCPServer(b) + with parent.activate(), patch('langbot.pkg.api.mcp.server._authorized', return_value=cb): + assert await server.mcp.call_tool('get_bot', {'bot_uuid': 'private-bot'}) + assert d.current_span() is parent + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +@pytest.mark.parametrize('group_class', [WebSocketChatRouterGroup, EmbedRouterGroup]) +async def test_real_websocket_receive_masks_foreign_parent(mode, group_class): + a, b, manager, ca, cb, parent = owners(mode) + group = group_class(b, quart.Quart(__name__)) + connection = NS(is_active=True, connection_id='private-id', send_queue=asyncio.Queue()) + seen = [] + + async def handle(*args, **kwargs): + await nested(b, cb, seen) + + adapter = NS(handle_websocket_message=handle) + if group_class is WebSocketChatRouterGroup: + group._revalidate_websocket_authorization = AsyncMock(return_value=cb) + else: + group._resolve_connected_bot = AsyncMock(return_value=NS(execution_context=cb)) + + async def receive(): + connection.is_active = False + return '{"type":"message","text":"private-prompt"}' + + with ( + parent.activate(), + patch('quart.websocket', NS(receive=receive)), + patch( + 'langbot.pkg.api.http.controller.groups.pipelines.websocket_chat.ws_connection_manager.update_activity', + new=AsyncMock(), + ), + ): + await group._handle_receive(connection, adapter, NS(execution_context=cb), 'private-token') + assert d.current_span() is parent + assert seen + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.parametrize('kind', ['execution', 'request']) +def test_workspace_rejects_foreign_instance_or_foreign_active_span(kind): + a, b, _, ca, cb, parent = owners('enabled') + context = cb + if kind == 'request': + context = RequestContext( + cb.instance_uuid, + 1, + 'request', + 'api_key', + PrincipalContext(PrincipalType.API_KEY), + WorkspaceContext(cb.workspace_uuid, None, None, frozenset()), + ) + with md.scope(a, 'http.isolation.annotation', source='http'): + span = d.current_span() + md.workspace(context) + assert 'workspace_uuid' not in span.fields + md.workspace(ca) + assert span.fields['workspace_uuid'] == ca.workspace_uuid + with parent.activate(): + before = dict(parent.fields) + md.workspace(ca) + md.workspace(context) + assert parent.fields == before + + +def test_structural_recorder_workspace_and_scope_exception_identity(): + recorder = Recorder() + ap = NS(diagnostics=recorder) + ctx = ExecutionContext('instance', str(uuid4()), 1) + error = ValueError('private-error') + with pytest.raises(ValueError) as caught: + with md.scope(ap, 'http.isolation.recorder', source='http'): + md.workspace(ctx) + assert d.current_span().fields['workspace_uuid'] == ctx.workspace_uuid + raise error + assert caught.value is error + assert recorder.events[-1]['outcome'] == 'failed' + assert d.current_span() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES[1:]) +@pytest.mark.parametrize('termination', ['success', 'error', 'cancel']) +async def test_nested_off_boundary_restores_outer_annotation_and_error(mode, termination): + a, b, manager, ca, cb, parent = owners(mode) + error = asyncio.CancelledError('private-cancel') if termination == 'cancel' else ValueError('private-error') + + @md.observe('http.isolation.off', source='http', ap=b) + async def endpoint(): + await nested(b, cb, []) + md.outcome('failed') + if termination != 'success': + raise error + return 42 + + with parent.activate(): + with md.scope(a, 'http.isolation.outer', source='http'): + outer = d.current_span() + if termination == 'success': + assert await endpoint() == 42 + else: + with pytest.raises(type(error)) as caught: + await endpoint() + assert caught.value is error + assert d.current_span() is outer + md.workspace(ca) + assert outer.fields['workspace_uuid'] == ca.workspace_uuid + assert outer.outcome is None + assert d.current_span() is parent + assert not manager.pending + assert [e['operation'] for e in a.diagnostics.pending] == [ + 'http.isolation.parent', + 'http.isolation.outer', + 'http.isolation.outer', + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +@pytest.mark.parametrize('group_class', [WebSocketChatRouterGroup, EmbedRouterGroup]) +async def test_websocket_send_cancel_restores_foreign_parent(mode, group_class): + a, b, manager, ca, cb, parent = owners(mode) + group = group_class(b, quart.Quart(__name__)) + connection = NS(is_active=False, send_queue=asyncio.Queue()) + await connection.send_queue.put({'text': 'private-answer'}) + error = asyncio.CancelledError('private-cancel') + + async def send(payload): + await nested(b, cb, []) + raise error + + with parent.activate(), patch('quart.websocket', NS(send=send)): + with pytest.raises(asyncio.CancelledError) as caught: + await group._handle_send(connection) + assert caught.value is error + assert d.current_span() is parent + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', MODES) +async def test_sync_http_body_exhaustion_has_owned_context_without_consumer_leak(mode): + a, b, manager, ca, cb, parent = owners(mode) + seen, closed = [], [] + + def stream(): + try: + for value in (b'one', b'two'): + md.workspace(cb) + seen.append(d.current_span()) + yield value + finally: + seen.append(d.current_span()) + closed.append(True) + + @md.observe('http.isolation.sync_body', source='http', ap=b, http=True) + async def endpoint(): + return quart.Response(stream()) + + response = await endpoint() + with parent.activate(): + async with response.response as body: + chunks = [] + async for chunk in body: + assert d.current_span() is parent + chunks.append(chunk) + assert d.current_span() is parent + assert chunks == [b'one', b'two'] + assert closed == [True] + assert all((span is not None) == (mode == 'enabled') for span in seen) + assert_isolated(a, b, manager, ca, cb, parent, mode) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('mode', ['enabled', 'disabled', 'absent', 'mismatched']) +@pytest.mark.parametrize('failure_stage', ['enter', 'iterator', 'advance', 'exit', None]) +async def test_custom_http_body_protocol_and_cancellation_identity(mode, failure_stage): + from quart.wrappers.response import IterableBody + + a, b, manager, ca, cb, parent = owners(mode) + error = asyncio.CancelledError('private-body-cancel') + seen = [] + + def visit(stage): + md.workspace(cb) + seen.append((stage, d.current_span())) + if stage == failure_stage: + raise error + + class Body(IterableBody): + def __init__(self): + pass + + async def __aenter__(self): + visit('enter') + return self + + def __aiter__(self): + visit('iterator') + return self + + async def __anext__(self): + visit('advance') + raise StopAsyncIteration + + async def __aexit__(self, *args): + visit('exit') + + @md.observe('http.isolation.body_protocol', source='http', ap=b, http=True) + async def endpoint(): + response = quart.Response() + response.response = Body() + return response + + response = await endpoint() + + async def consume(): + async with response.response as body: + assert d.current_span() is parent + async for _ in body: + pytest.fail('empty body yielded') + + with parent.activate(): + if failure_stage is None: + await consume() + else: + with pytest.raises(asyncio.CancelledError) as caught: + await consume() + assert caught.value is error + assert d.current_span() is parent + assert all((span is not None) == (mode == 'enabled') for _, span in seen) + assert [s for s, _ in seen].count('exit') == (0 if failure_stage in {'enter', 'iterator'} else 1) + assert_isolated(a, b, manager, ca, cb, parent, mode) diff --git a/tests/unit_tests/api/test_diagnostics_management_lifecycle.py b/tests/unit_tests/api/test_diagnostics_management_lifecycle.py new file mode 100644 index 000000000..df68d44bf --- /dev/null +++ b/tests/unit_tests/api/test_diagnostics_management_lifecycle.py @@ -0,0 +1,163 @@ +"""HTTP streaming termination, fail-open behavior and real producer privacy.""" + +import asyncio +import json +import re +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +import quart + +from langbot.pkg.agent.runner import errors as runner_errors +from langbot.pkg.api import management_diagnostics as md +from langbot.pkg.api.http.controller.group import AuthType +from langbot.pkg.api.http.controller.groups.agent_debug_stream import debug_stream_response +from langbot.pkg.api.mcp.mount import MCPMount +from langbot.pkg.telemetry import diagnostics as d +from tests.unit_tests.api.test_diagnostics_management_http import Recorder, setup + + +@pytest.mark.asyncio +@pytest.mark.parametrize('termination', ['close', 'cancel', 'error']) +async def test_stream_termination_preserves_cleanup_and_error(termination): + app, ap, routes = await setup() + closed = [] + + @routes.route('/end', auth_type=AuthType.NONE) + async def end(): + async def body(): + try: + yield b'one' + if termination == 'error': + raise ValueError('private-error') + if termination == 'cancel': + raise asyncio.CancelledError('private-cancel') + yield b'two' + finally: + closed.append(d.current_span()) + + return quart.Response(body()) + + async with app.test_request_context('/management/end'): + response = await app.full_dispatch_request() + if termination == 'close': + async with response.response as body: + assert await anext(body.__aiter__()) == b'one' + else: + error = ValueError if termination == 'error' else asyncio.CancelledError + with pytest.raises(error): + async with response.response as body: + iterator = body.__aiter__() + assert await anext(iterator) == b'one' + await anext(iterator) + assert len(closed) == 1 and closed[0] is not None + expected = 'failed' if termination == 'error' else 'cancelled' + assert [e['outcome'] for e in ap.diagnostics.events] == ['started', expected] + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_debug_ndjson_error_marks_http_business_failure(): + app, ap, routes = await setup() + service = SimpleNamespace(debug_agent=AsyncMock(side_effect=runner_errors.RunnerNotFoundError('private-error'))) + + @routes.route('/ndjson', auth_type=AuthType.NONE) + async def ndjson(): + return debug_stream_response(service, object(), 'private-agent', {'text': 'private-text'}) + + response = await app.test_client().get('/management/ndjson') + assert response.status_code == 200 + assert json.loads(await response.get_data())['kind'] == 'error' + assert ap.diagnostics.events[-1]['outcome'] == 'failed' + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +async def test_terminal_diagnostic_failure_preserves_success_and_stream(): + class BrokenFinish(Recorder): + def emit(self, kind, operation, outcome, **fields): + if outcome != 'started': + raise RuntimeError('private-diagnostic-error') + super().emit(kind, operation, outcome, **fields) + + app, _, _ = await setup(BrokenFinish()) + response = await app.test_client().get('/management/stream') + assert await response.get_data() == b'private-stream-chunk' + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_real_manager_status_privacy_and_no_network_in_request(): + app, ap, _ = await setup(None) + ap.instance_config = SimpleNamespace(data={'space': {'url': 'https://example.invalid'}}) + ap.diagnostics = d.DiagnosticsManager(ap, version='4.11.0-beta.2', instance_id='instance-test', capacity=20) + ap.diagnostics.credentials = AsyncMock(side_effect=AssertionError('must not await credentials')) + for path in ['ok/private-identifier', 'business', 'auth', 'cancel']: + async with app.test_request_context('/management/' + path): + try: + await app.full_dispatch_request() + except asyncio.CancelledError: + pass + events = list(ap.diagnostics.pending) + assert len(events) == 8 + assert {e['outcome'] for e in events} == {'started', 'succeeded', 'failed', 'rejected', 'cancelled'} + assert all(re.fullmatch(r'[A-Za-z_][A-Za-z0-9_.:-]{0,127}', e['operation']) for e in events) + assert 'private-' not in json.dumps(events) + ap.diagnostics.credentials.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'config,version', + [ + ({}, '4.11.0'), + ({'disable_telemetry': True}, '4.11.0-beta.2'), + ({'disable_beta_diagnostics': True}, '4.11.0-beta.2'), + ], +) +async def test_real_manager_disabled_modes_do_not_install_context(config, version): + app, ap, _ = await setup(None) + ap.instance_config = SimpleNamespace(data={'space': config}) + ap.diagnostics = d.DiagnosticsManager(ap, version=version, instance_id='instance-test') + response = await app.test_client().get('/management/ok/private-id') + assert response.status_code == 200 + assert ap.seen == [None] + assert not ap.diagnostics.pending + + +@pytest.mark.asyncio +async def test_mcp_mount_observes_auth_without_headers_or_body(): + ap = SimpleNamespace( + diagnostics=Recorder(), apikey_service=SimpleNamespace(authenticate_api_key=AsyncMock(return_value=None)) + ) + mount = MCPMount(ap) + send = AsyncMock() + fallback = AsyncMock() + await mount.wrap(fallback)( + {'type': 'http', 'path': '/mcp/private-path', 'headers': [(b'x-api-key', b'private-token')]}, AsyncMock(), send + ) + assert send.call_args_list[0].args[0]['status'] == 401 + assert [e['outcome'] for e in ap.diagnostics.events] == ['started', 'rejected'] + assert ap.diagnostics.events[-1]['operation'] == 'mcp.request' + assert 'private-' not in json.dumps(ap.diagnostics.events) + fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_debug_boundary_source_inherits_without_duplicate_run(): + recorder = Recorder() + ap = SimpleNamespace(diagnostics=recorder) + + @md.observe('http.test.debug', source='webui_debug', ap=ap) + async def debug(): + @d.observe('run', 'runner.run', source='agent', ap=ap) + async def run(): + return 'private-content' + + return await run() + + assert await debug() == 'private-content' + assert len([e for e in recorder.events if e['kind'] == 'run' and e['outcome'] == 'started']) == 1 + assert all(e['source'] == 'webui_debug' and e['attributes']['synthetic'] for e in recorder.events) + assert 'private-' not in json.dumps(recorder.events) diff --git a/tests/unit_tests/api/test_diagnostics_management_transports.py b/tests/unit_tests/api/test_diagnostics_management_transports.py new file mode 100644 index 000000000..4b78cd91a --- /dev/null +++ b/tests/unit_tests/api/test_diagnostics_management_transports.py @@ -0,0 +1,173 @@ +"""Management MCP, WebSocket and debug execution boundaries.""" + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +import quart + +from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import WebSocketChatRouterGroup +from langbot.pkg.api.http.controller.groups.pipelines.embed import EmbedRouterGroup +from langbot.pkg.api.mcp.server import LangBotMCPServer +from langbot.pkg.telemetry import diagnostics as d +from tests.unit_tests.api.test_diagnostics_management_http import Recorder + + +@pytest.mark.asyncio +async def test_mcp_real_registered_tool_has_source_and_never_content(): + recorder = Recorder() + seen = [] + + async def get_bot(*args, **kwargs): + seen.append(d.current_span()) + return {'secret': 'private-result'} + + ap = SimpleNamespace(diagnostics=recorder, bot_service=SimpleNamespace(get_bot=get_bot)) + server = LangBotMCPServer(ap) + with patch('langbot.pkg.api.mcp.server._authorized', return_value=object()): + result = await server.mcp.call_tool('get_bot', {'bot_uuid': 'private-bot'}) + assert result + assert [e['outcome'] for e in recorder.events] == ['started', 'succeeded'] + assert recorder.events[-1]['operation'] == 'mcp.get_bot' + assert recorder.events[-1]['source'] == 'mcp' + assert seen[0] is not None + assert 'private-' not in json.dumps(recorder.events) + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_mcp_tool_permission_rejection_is_observed(): + server = LangBotMCPServer(SimpleNamespace(diagnostics=Recorder())) + with pytest.raises(Exception): + await server.mcp.call_tool('list_bots', {}) + assert server.ap.diagnostics.events[-1]['outcome'] in {'rejected', 'failed'} + assert server.ap.diagnostics.events[-1]['source'] == 'mcp' + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'group_class,path', + [ + (WebSocketChatRouterGroup, '/api/v1/pipelines/private-pipeline/ws/connect'), + (EmbedRouterGroup, '/api/v1/embed/11111111-1111-4111-8111-111111111111/ws/connect?session_id=private-session'), + ], +) +async def test_real_websocket_session_auth_rejection_is_content_free(group_class, path): + app = quart.Quart(__name__) + ap = SimpleNamespace(diagnostics=Recorder()) + group = group_class(ap, app) + group._authenticate_websocket = AsyncMock(side_effect=ValueError('private-token')) + if group_class is EmbedRouterGroup: + group._resolve_bot = AsyncMock(return_value=(object(), 'private-pipeline')) + await group.initialize() + async with app.test_client().websocket(path) as socket: + frame = json.loads(await socket.receive()) + assert frame['type'] == 'error' + terminal = [e for e in ap.diagnostics.events if e['outcome'] != 'started'] + assert terminal + assert terminal[-1]['outcome'] == 'rejected' + assert terminal[-1]['source'] == 'websocket' + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('group_class', [WebSocketChatRouterGroup, EmbedRouterGroup]) +async def test_websocket_received_message_boundaries_do_not_capture_frames(group_class): + app = quart.Quart(__name__) + ap = SimpleNamespace(diagnostics=Recorder()) + group = group_class(ap, app) + connection = SimpleNamespace(is_active=True, connection_id='private-id', send_queue=asyncio.Queue()) + adapter = SimpleNamespace(handle_websocket_message=AsyncMock()) + if group_class is WebSocketChatRouterGroup: + group._revalidate_websocket_authorization = AsyncMock(return_value=object()) + args = (connection, adapter, object(), 'private-token') + else: + group._resolve_connected_bot = AsyncMock(return_value=object()) + args = (connection, adapter, object(), 'private-pipeline') + + async def receive(): + connection.is_active = False + return json.dumps({'type': 'message', 'text': 'private-prompt'}) + + with ( + patch('quart.websocket', SimpleNamespace(receive=receive)), + patch( + 'langbot.pkg.api.http.controller.groups.pipelines.websocket_chat.ws_connection_manager.update_activity', + new=AsyncMock(), + ), + ): + await group._handle_receive(*args) + adapter.handle_websocket_message.assert_awaited_once() + assert [e['outcome'] for e in ap.diagnostics.events] == ['started', 'succeeded'] + assert ap.diagnostics.events[-1]['operation'].endswith('.message') + assert 'private-' not in json.dumps(ap.diagnostics.events) + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_debug_service_marks_synthetic_source_before_validation(): + from langbot.pkg.api.http.service.agent import AgentService + + ap = SimpleNamespace(diagnostics=Recorder()) + service = AgentService(ap) + seen = [] + + async def get_agent(*args): + seen.append(d.current_span()) + return None + + service.get_agent = get_agent + with pytest.raises(ValueError): + await service.debug_agent(object(), 'private-id', {'text': 'private-prompt'}) + assert seen[0].fields['source'] == 'webui_debug' + assert ap.diagnostics.events[-1]['operation'] == 'http.agent.debug_agent' + assert ap.diagnostics.events[-1]['source'] == 'webui_debug' + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('group_class', [WebSocketChatRouterGroup, EmbedRouterGroup]) +@pytest.mark.parametrize( + 'frame,expected', [('private-invalid-json', 'rejected'), ('{"type":"private-unknown"}', 'skipped')] +) +async def test_websocket_invalid_frames_have_finite_outcomes(group_class, frame, expected): + app = quart.Quart(__name__) + ap = SimpleNamespace(diagnostics=Recorder()) + group = group_class(ap, app) + connection = SimpleNamespace(is_active=True, connection_id='private-id', send_queue=asyncio.Queue()) + + async def receive(): + connection.is_active = False + return frame + + with ( + patch('quart.websocket', SimpleNamespace(receive=receive)), + patch( + 'langbot.pkg.api.http.controller.groups.pipelines.websocket_chat.ws_connection_manager.update_activity', + new=AsyncMock(), + ), + ): + await group._handle_receive(connection, object(), object(), 'private-token') + assert ap.diagnostics.events[-1]['outcome'] == expected + assert 'private-' not in json.dumps(ap.diagnostics.events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('group_class', [WebSocketChatRouterGroup, EmbedRouterGroup]) +async def test_websocket_send_boundary_preserves_payload_and_cancellation(group_class): + app = quart.Quart(__name__) + ap = SimpleNamespace(diagnostics=Recorder()) + group = group_class(ap, app) + connection = SimpleNamespace(is_active=False, send_queue=asyncio.Queue()) + await connection.send_queue.put({'text': 'private-answer'}) + send = AsyncMock(side_effect=asyncio.CancelledError('private-error')) + with patch('quart.websocket', SimpleNamespace(send=send)): + with pytest.raises(asyncio.CancelledError): + await group._handle_send(connection) + assert json.loads(send.call_args.args[0]) == {'text': 'private-answer'} + assert ap.diagnostics.events[-1]['outcome'] == 'cancelled' + assert ap.diagnostics.events[-1]['operation'].endswith('.send') + assert 'private-' not in json.dumps(ap.diagnostics.events) + assert d.current_span() is None diff --git a/tests/unit_tests/telemetry/test_beta_diagnostic_hooks.py b/tests/unit_tests/telemetry/test_beta_diagnostic_hooks.py new file mode 100644 index 000000000..e4ce1e93e --- /dev/null +++ b/tests/unit_tests/telemetry/test_beta_diagnostic_hooks.py @@ -0,0 +1,143 @@ +"""Execute real Core boundaries with content canaries and early failures.""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock +from uuid import uuid4 + +import pytest + +from langbot.pkg.telemetry import diagnostics as d +from langbot.pkg.api.http.context import ExecutionContext +from langbot.pkg.agent.runner.orchestrator import AgentRunOrchestrator +from langbot.pkg.agent.runner.reply_stream import ReplyStreamSession, ReplyStreamRequest + + +def make_ap(): + ap = SimpleNamespace(instance_config=SimpleNamespace(data={'space': {'url': 'https://example.invalid'}})) + ap.persistence_mgr = SimpleNamespace(get_db_engine=lambda: None) + ap.diagnostics = d.DiagnosticsManager(ap, version='4.11.0b2', instance_id='instance-test') + return ap + + +@pytest.mark.asyncio +@pytest.mark.parametrize('processor', ['pipeline', 'agent', 'event_processor']) +async def test_real_orchestrator_prepare_failure(processor): + ap = make_ap() + registry = SimpleNamespace(get=AsyncMock(side_effect=ValueError('CANARY private runner URL'))) + orchestrator = AgentRunOrchestrator(ap, registry) + context = ExecutionContext(instance_uuid='instance-test', workspace_uuid=str(uuid4()), placement_generation=1) + event = SimpleNamespace(workspace_id=context.workspace_uuid, event_type='message.received') + binding = SimpleNamespace(runner_id='CANARY', processor_type=processor) + with pytest.raises(ValueError, match='CANARY'): + await anext(orchestrator.run(event, binding, adapter_context={'_execution_context': context})) + records = ap.diagnostics.pending + assert [r['outcome'] for r in records] == ['started', 'failed'] + assert records[-1]['stage'] == 'prepare' + assert records[-1]['workspace_uuid'] == context.workspace_uuid + assert records[-1]['processor_type'] == processor + assert 'CANARY' not in json.dumps(records) + + +@pytest.mark.asyncio +async def test_real_reply_stream_mock_is_not_platform_success(): + ap = make_ap() + event = SimpleNamespace( + delivery=SimpleNamespace(reply_target={}, surface='webui', platform_capabilities={'debug_mock': True}) + ) + session = ReplyStreamSession(event) + # Real runtime supplies the manager at construction from the orchestrator. + session.diagnostics = ap.diagnostics + result = await session.apply(ReplyStreamRequest(stream_id=uuid4(), operation='finish', text='CANARY user reply')) + assert result['mock'] is True and result['text'] == 'CANARY user reply' + assert ap.diagnostics.pending[-1]['source'] == 'webui_debug' + assert ap.diagnostics.pending[-1]['attributes']['synthetic'] is True + assert 'CANARY' not in json.dumps(ap.diagnostics.pending) + + +@pytest.mark.asyncio +async def test_real_bot_route_projects_status_not_reason(): + from langbot.pkg.platform.botmgr import RuntimeBot + + ap = make_ap() + bot = object.__new__(RuntimeBot) + bot.ap = ap + bot.logger = SimpleNamespace(info=AsyncMock()) + await bot._record_event_route_trace( + event_type='message.received', + status='not_matched', + text='CANARY secret', + reason='CANARY', + failure_code='route_not_found', + ) + records = ap.diagnostics.pending + assert records[-1]['outcome'] == 'skipped' + assert records[-1]['reason_code'] == 'route_not_found' + assert 'CANARY' not in json.dumps(records) + + +@pytest.mark.asyncio +async def test_real_telegram_conversion_failure_before_bot_manager(monkeypatch): + from langbot.pkg.platform.adapters.telegram.adapter import TelegramAdapter + from langbot.pkg.platform.adapters.telegram.event_converter import TelegramEventConverter + + ap = make_ap() + context = ExecutionContext(instance_uuid='instance-test', workspace_uuid=str(uuid4()), placement_generation=1) + from langbot.pkg.platform.logger import EventLogger + + logger = EventLogger('test', ap, context, 'test') + logger.error = AsyncMock() + logger.warning = AsyncMock() + adapter = TelegramAdapter({'token': '123456:ABCDEFGHIJKLMNOPQRSTUVWXYZ_123456789'}, logger) + adapter.listeners = {} + monkeypatch.setattr( + TelegramEventConverter, '_convert_message', AsyncMock(side_effect=ValueError('CANARY conversion')) + ) + update = SimpleNamespace( + message=SimpleNamespace(from_user=SimpleNamespace(is_bot=False), text='CANARY text'), + edited_message=None, + chat_member=None, + my_chat_member=None, + callback_query=None, + message_reaction=None, + ) + callback = adapter.application.handlers[0][0].callback + await callback(update, None) + records = [e for e in ap.diagnostics.pending if e['stage'] == 'convert' and e['outcome'] == 'failed'] + assert records and records[-1]['adapter'] == 'telegram-omni' + assert records[-1]['workspace_uuid'] == context.workspace_uuid + assert 'CANARY' not in json.dumps(ap.diagnostics.pending) + + +@pytest.mark.asyncio +async def test_real_interaction_ack_skip_and_failure(): + from langbot.pkg.agent.runner.interaction_manager import InteractionManager + + ap = make_ap() + interactions = InteractionManager(ap, store=SimpleNamespace(record_delivery_success=AsyncMock())) + await interactions.acknowledge_submission({}, SimpleNamespace(get_supported_apis=lambda: [])) + assert ap.diagnostics.pending[-1]['outcome'] == 'skipped' + ap.diagnostics.pending.clear() + ap.logger = SimpleNamespace(warning=lambda *a: None) + adapter = SimpleNamespace( + get_supported_apis=lambda: ['interaction.acknowledge'], + call_platform_api=AsyncMock(side_effect=ValueError('CANARY ack')), + ) + await interactions.acknowledge_submission({'delivery_result': {'secret': 'CANARY'}}, adapter) + assert ap.diagnostics.pending[-1]['outcome'] == 'failed' + assert 'CANARY' not in json.dumps(ap.diagnostics.pending) + + +def test_actual_adapter_capability_snapshot(): + from langbot.pkg.platform.adapters.telegram.adapter import TelegramAdapter + from langbot.pkg.telemetry.diagnostic_catalog import snapshot_bot + + ap = make_ap() + context = ExecutionContext(instance_uuid='instance-test', workspace_uuid=str(uuid4()), placement_generation=1) + adapter = TelegramAdapter.model_construct(config={}, listeners={}) + snapshot_bot(SimpleNamespace(ap=ap, adapter=adapter, execution_context=context)) + records = ap.diagnostics.pending + assert records and all(e['kind'] == 'capability' for e in records) + api_rows = [e for e in records if e['attributes']['capability_type'] == 'api'] + assert any(e['operation'] == 'send_message' and e['attributes']['supported'] for e in api_rows) + assert all(e['workspace_uuid'] == context.workspace_uuid for e in records) diff --git a/tests/unit_tests/telemetry/test_beta_diagnostic_review_regressions.py b/tests/unit_tests/telemetry/test_beta_diagnostic_review_regressions.py new file mode 100644 index 000000000..67deb287b --- /dev/null +++ b/tests/unit_tests/telemetry/test_beta_diagnostic_review_regressions.py @@ -0,0 +1,405 @@ +"""Independent-review regressions through real ownership and lifecycle boundaries.""" + +import asyncio +import importlib.metadata +import json +from collections.abc import AsyncGenerator +from types import SimpleNamespace as NS +from unittest.mock import AsyncMock, Mock +from uuid import uuid4 + +import pytest + +from langbot.pkg.api.http.context import ExecutionContext +from langbot.pkg.provider.modelmgr.requester import RuntimeProvider +from langbot.pkg.telemetry import diagnostics as d + + +def make_ap(identity='instance-test', disabled=False): + ap = NS(instance_config=NS(data={'space': {'url': 'https://example.invalid', 'disable_telemetry': disabled}})) + ap.diagnostics = d.DiagnosticsManager(ap, version='4.11.0b2', instance_id=identity) + return ap + + +def context(identity='instance-test'): + return ExecutionContext(instance_uuid=identity, workspace_uuid=str(uuid4()), placement_generation=1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('disabled', [True, False]) +async def test_real_runtime_provider_cannot_use_other_app_manager(disabled): + a, b = make_ap('instance-A'), make_ap('instance-B', disabled) + ca, cb = context('instance-A'), context('instance-B') + requester = NS(ap=b, invoke_llm=AsyncMock(return_value='business result')) + provider = RuntimeProvider(cb, NS(workspace_uuid=cb.workspace_uuid), None, requester) + model = NS(execution_context=cb, provider=provider) + parent = d.Span(a.diagnostics, 'api', 'review.parent', {'workspace_uuid': ca.workspace_uuid}) + with parent.activate(): + assert await provider.invoke_llm(None, model, [], execution_context=cb) == 'business result' + assert not [e for e in a.diagnostics.pending if e['operation'] == 'model.invoke_llm'] + assert len(b.diagnostics.pending) == (0 if disabled else 2) + for event in b.diagnostics.pending: + assert event['instance_id'] == 'instance-B' + assert event['workspace_uuid'] == cb.workspace_uuid + assert event['trace_id'] != parent.fields['trace_id'] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('shape', ['app', 'ap', 'logger', 'requester', 'adapter', 'direct', 'absent']) +async def test_explicit_disabled_owner_blocks_parent_even_in_ownerless_children(shape): + a, b = make_ap('instance-A'), make_ap('instance-B', True) + shapes = { + 'app': b, + 'ap': NS(ap=b), + 'logger': NS(logger=NS(ap=b)), + 'requester': NS(requester=NS(ap=b)), + 'adapter': NS(adapter=NS(logger=NS(ap=b))), + 'direct': NS(diagnostics=b.diagnostics), + 'absent': NS(ap=None), + } + + @d.observe('api', 'review.child') + async def child(data): + d.event(data, 'api', 'review.point', 'succeeded') + return 42 + + @d.observe('api', 'review.owner') + async def call(owner): + return await child({}) + + @d.observe('run', 'review.stream') + async def stream(owner): + yield await child({}) + + parent = d.Span(a.diagnostics, 'api', 'review.parent', {}) + with parent.activate(): + assert await call(shapes[shape]) == 42 + assert [x async for x in stream(shapes[shape])] == [42] + d.event(shapes[shape], 'api', 'review.point', 'succeeded') + assert len(a.diagnostics.pending) == 1 + assert not b.diagnostics.pending + + +@pytest.mark.asyncio +async def test_context_cannot_select_other_instance_or_owned_workspace(): + ap = make_ap() + ca, cb = context(), context('instance-other') + + @d.observe('api', 'review.context') + async def call(owner, execution_context): + return 42 + + owner = NS(ap=ap, execution_context=ca) + with d.Span(ap.diagnostics, 'api', 'review.parent', {'workspace_uuid': ca.workspace_uuid}).activate(): + assert await call(owner, cb) == 42 + assert await call(owner, context()) == 42 + assert not [e for e in ap.diagnostics.pending if e['operation'] == 'review.context'] + # A misplaced manager attachment is not an authoritative manager for B. + other = make_ap('instance-other') + other.diagnostics = ap.diagnostics + assert await call(NS(ap=other), cb) == 42 + assert not [e for e in ap.diagnostics.pending if e['operation'] == 'review.context'] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('disabled', [False, True]) +@pytest.mark.parametrize('action', ['aclose', 'athrow_exit', 'athrow_value', 'athrow_cancel']) +async def test_cleanup_exception_identity_matches_native(disabled, action): + ap = make_ap(disabled=disabled) + cleanup_error = ValueError('CANARY cleanup') + + async def original(owner): + try: + yield 1 + finally: + raise cleanup_error + + for fn in (original, d.observe('run', 'review.cleanup')(original)): + gen = fn(ap) + assert isinstance(gen, AsyncGenerator) + assert await anext(gen) == 1 + with pytest.raises(ValueError) as caught: + if action == 'aclose': + await gen.aclose() + else: + error = { + 'athrow_exit': GeneratorExit(), + 'athrow_value': KeyError('business'), + 'athrow_cancel': asyncio.CancelledError('cancel'), + }[action] + await gen.athrow(error) + assert caught.value is cleanup_error + assert d.current_span() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize('disabled', [False, True]) +async def test_athrow_generator_exit_can_yield_and_close_remains_native(disabled): + ap = make_ap(disabled=disabled) + + async def original(owner): + try: + yield 1 + except GeneratorExit: + yield 2 + yield 3 + + for fn in (original, d.observe('run', 'review.exit')(original)): + gen = fn(ap) + assert await anext(gen) == 1 + assert await gen.athrow(GeneratorExit()) == 2 + assert await anext(gen) == 3 + await gen.aclose() + gen = fn(ap) + assert await anext(gen) == 1 + with pytest.raises(RuntimeError, match='ignored GeneratorExit'): + await gen.aclose() + # The native generator is still suspended after the refused close. + assert await anext(gen) == 3 + with pytest.raises(StopAsyncIteration): + await anext(gen) + await gen.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('disabled', [False, True]) +async def test_generator_send_throw_cancellation_and_primary_exception(disabled): + ap = make_ap(disabled=disabled) + entered = asyncio.Event() + primary = ValueError('CANARY business') + + async def original(owner): + value = yield 1 + try: + yield value + except KeyError: + yield 3 + entered.set() + await asyncio.Event().wait() + + for fn in (original, d.observe('run', 'review.protocol')(original)): + gen = fn(ap) + assert await anext(gen) == 1 + assert await gen.asend(7) == 7 + assert await gen.athrow(KeyError('throw')) == 3 + entered.clear() + task = asyncio.create_task(anext(gen)) + await entered.wait() + task.cancel('native cancellation') + with pytest.raises(asyncio.CancelledError, match='native cancellation'): + await task + await gen.aclose() + + async def failing(owner): + yield 1 + raise primary + + gen = d.observe('run', 'review.primary')(failing)(ap) + assert await anext(gen) == 1 + with pytest.raises(ValueError) as caught: + await anext(gen) + assert caught.value is primary + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_real_kook_native_entry_conversion_failure_has_trusted_workspace(): + from langbot.pkg.platform.adapters.kook.adapter import KookAdapter + from langbot.pkg.platform.logger import EventLogger + + ap, ctx = make_ap(), context() + logger = EventLogger('test', ap, ctx, 'test') + logger.error = AsyncMock() + adapter = KookAdapter({'token': 'test-placeholder'}, logger) + # Invalid native timestamp fails inside the real static converter. + await adapter._handle_event({'type': 255, 'msg_timestamp': 'CANARY invalid', 'workspace_uuid': str(uuid4())}, 1) + logger.error.assert_awaited_once() + failures = [e for e in ap.diagnostics.pending if e['outcome'] == 'failed'] + assert any(e['stage'] == 'convert' for e in failures) + assert any(e['operation'] == 'platform.receive' for e in failures) + assert all(e['workspace_uuid'] == ctx.workspace_uuid for e in ap.diagnostics.pending) + assert all(e['adapter'] == 'kook-omni' for e in ap.diagnostics.pending) + assert 'CANARY' not in json.dumps(ap.diagnostics.pending) + + +@pytest.fixture +def boot_stages(monkeypatch): + from langbot.pkg.core import boot + + stages_run = [] + + class Stage: + async def run(self, app): + stages_run.append(app) + + # Earlier registry tests clear this shared dictionary; cached imports do not + # re-register stages. Own the registry per test and restore it on teardown. + monkeypatch.setattr(boot.stage, 'preregistered_stages', {'LoadConfigStage': Stage, 'GenKeysStage': Stage}) + return stages_run + + +@pytest.mark.asyncio +@pytest.mark.parametrize('failure', ['constructor', 'metadata', 'session', 'start']) +@pytest.mark.parametrize('disabled', [False, True]) +async def test_real_make_app_optional_initialization_fail_open(monkeypatch, boot_stages, failure, disabled): + from langbot.pkg.core import boot + + ap = NS( + instance_config=NS(data={'space': {'url': 'https://example.invalid', 'disable_telemetry': disabled}}), + initialize=AsyncMock(), + shutdown=AsyncMock(), + ) + stages_run = boot_stages + monkeypatch.setattr(boot.app, 'Application', lambda: ap) + monkeypatch.setattr(boot, 'stage_order', ['LoadConfigStage', 'GenKeysStage']) + error = RuntimeError('optional diagnostics') + if failure == 'constructor': + monkeypatch.setattr(boot.diagnostics, 'DiagnosticsManager', Mock(side_effect=error)) + elif failure == 'metadata': + monkeypatch.setattr( + importlib.metadata, 'version', Mock(side_effect=importlib.metadata.PackageNotFoundError('langbot')) + ) + else: + manager = NS( + start_session=AsyncMock(), start=Mock(), shutdown=AsyncMock(side_effect=RuntimeError('optional cleanup')) + ) + getattr(manager, 'start_session' if failure == 'session' else 'start').side_effect = error + monkeypatch.setattr(boot.diagnostics, 'DiagnosticsManager', Mock(return_value=manager)) + assert await boot.make_app(asyncio.get_running_loop()) is ap + assert len(stages_run) == 2 + ap.initialize.assert_awaited_once() + ap.shutdown.assert_not_awaited() + assert getattr(ap, 'diagnostics', None) is None + + +@pytest.mark.asyncio +async def test_real_make_app_session_cancellation_and_shutdown_failure_preserve_primary(monkeypatch, boot_stages): + from langbot.pkg.core import boot + + cancelled = asyncio.CancelledError('genuine cancellation') + ap = NS( + instance_config=NS(data={'space': {'url': 'https://example.invalid'}}), + initialize=AsyncMock(), + shutdown=AsyncMock(side_effect=RuntimeError('shutdown error')), + ) + manager = NS(start_session=AsyncMock(side_effect=cancelled), start=Mock(), shutdown=AsyncMock()) + monkeypatch.setattr(boot.app, 'Application', lambda: ap) + monkeypatch.setattr(boot, 'stage_order', ['GenKeysStage']) + monkeypatch.setattr(boot.diagnostics, 'DiagnosticsManager', Mock(return_value=manager)) + with pytest.raises(asyncio.CancelledError) as caught: + await boot.make_app(asyncio.get_running_loop()) + assert caught.value is cancelled + ap.initialize.assert_not_awaited() + ap.shutdown.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_real_make_app_business_failure_not_masked_by_optional_shutdown(monkeypatch, boot_stages): + from langbot.pkg.core import boot + + primary = ValueError('business startup') + ap = boot.app.Application() + ap.instance_config = NS(data={'space': {'url': 'https://example.invalid'}}) + ap.initialize = AsyncMock(side_effect=primary) + manager = make_ap().diagnostics + manager.start_session = AsyncMock() + manager.start = Mock() + manager.shutdown = AsyncMock(side_effect=RuntimeError('optional shutdown')) + monkeypatch.setattr(boot.app, 'Application', lambda: ap) + monkeypatch.setattr(boot.diagnostics, 'DiagnosticsManager', lambda *a, **kw: manager) + monkeypatch.setattr(boot, 'stage_order', ['GenKeysStage']) + + with pytest.raises(ValueError) as caught: + await boot.make_app(asyncio.get_running_loop()) + assert caught.value is primary + manager.shutdown.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('disabled', [False, True]) +async def test_protocol_rejected_calls_and_unstarted_throw_match_native(disabled): + ap = make_ap(disabled=disabled) + + async def original(owner): + try: + yield 1 + except GeneratorExit: + yield 2 + yield 3 + + async def record(fn, actions): + gen = fn(ap) + results = [] + for name, values in actions: + try: + results.append(('value', await getattr(gen, name)(*values))) + except BaseException as exc: + results.append((type(exc).__name__, str(exc))) + # Fully exhaust any generator left suspended by a refused close. + try: + while True: + await anext(gen) + except (StopAsyncIteration, GeneratorExit): + pass + return results + + wrapped = d.observe('run', 'review.protocol_matrix')(original) + for actions in ( + [('aclose', ())], + [('athrow', (GeneratorExit(),))], + [('athrow', (ValueError('unstarted'),))], + [('asend', (7,)), ('__anext__', ()), ('athrow', (ValueError('primary'),))], + [('__anext__', ()), ('aclose', ()), ('__anext__', ()), ('aclose', ())], + [('__anext__', ()), ('athrow', (GeneratorExit(),)), ('__anext__', ()), ('aclose', ())], + ): + assert await record(wrapped, actions) == await record(original, actions) + + +@pytest.mark.asyncio +@pytest.mark.parametrize('disabled', [False, True]) +async def test_native_task_cancel_cleanup_error_wins_without_extra_close(disabled): + ap = make_ap(disabled=disabled) + error = ValueError('native cleanup') + entered = asyncio.Event() + + async def original(owner): + yield 1 + try: + entered.set() + await asyncio.Event().wait() + finally: + raise error + + for fn in (original, d.observe('run', 'review.cancel_cleanup')(original)): + gen = fn(ap) + await anext(gen) + entered.clear() + task = asyncio.create_task(anext(gen)) + await entered.wait() + task.cancel() + with pytest.raises(ValueError) as caught: + await task + assert caught.value is error + await gen.aclose() + + +@pytest.mark.asyncio +async def test_sdk_b2_handler_consumes_observed_async_generator(tmp_path): + from langbot_plugin.runtime.io.handler import Handler, ActionResponse + from langbot_plugin.entities.io.resp import ChunkStatus + + ap = make_ap() + handler = Handler(NS(), file_storage_dir=str(tmp_path)) + handler._send_message = AsyncMock() + + @d.observe('api', 'host.review_stream', ap=ap) + async def stream(data): + yield ActionResponse.success({'business': 'unchanged'}) + + handler.actions['review_stream'] = stream + await handler._handle_action({'seq_id': 1, 'action': 'review_stream', 'data': {}}) + responses = [call.args[0] for call in handler._send_message.await_args_list] + assert len(responses) == 2 + assert responses[0].data == {'business': 'unchanged'} + assert [response.chunk_status for response in responses] == [ChunkStatus.CONTINUE, ChunkStatus.END] + assert [e['outcome'] for e in ap.diagnostics.pending] == ['started', 'succeeded'] diff --git a/tests/unit_tests/telemetry/test_beta_diagnostics.py b/tests/unit_tests/telemetry/test_beta_diagnostics.py new file mode 100644 index 000000000..a73d8d8a9 --- /dev/null +++ b/tests/unit_tests/telemetry/test_beta_diagnostics.py @@ -0,0 +1,350 @@ +"""Content-free diagnostics contract and lifecycle regression tests.""" + +import asyncio +import json +from types import SimpleNamespace +from uuid import uuid4 + +import httpx +import pytest + +from langbot.pkg.telemetry import diagnostics as d + + +def manager(version='4.11.0-beta.2', **config): + ap = SimpleNamespace(instance_config=SimpleNamespace(data={'space': {'url': 'https://example.invalid', **config}})) + ap.diagnostics = d.DiagnosticsManager(ap, version=version, instance_id='instance-test', capacity=4) + return ap.diagnostics + + +def test_release_gate_and_privacy(): + assert not manager('4.11.0').enabled + assert manager().enabled + assert manager('4.11.0b2').enabled + assert not manager(disable_telemetry=True).enabled + assert not manager(disable_beta_diagnostics=True).enabled + m = manager() + m.emit( + 'api', + 'test.operation', + 'failed', + attributes={'prompt': 'CANARY', 'plugin_id': 'CANARY', 'attempts': 1}, + error=ValueError('CANARY https://secret/token'), + workspace_uuid=str(uuid4()), + ) + payload = m.pending[0] + assert 'CANARY' not in json.dumps(payload) + assert payload['attributes'] == {'attempts': 1} + assert payload['error_type'] == 'ValueError' + assert payload['instance_id'] != payload['workspace_uuid'] + assert payload['sample_rate'] == 1 + + +@pytest.mark.asyncio +async def test_disabled_boundary_skips_all_projection(monkeypatch): + m = manager('4.11.0') + + def broken(*args, **kwargs): + raise AssertionError('diagnostic machinery ran while disabled') + + monkeypatch.setattr(d, 'Span', broken) + monkeypatch.setattr(d, 'result_outcome', broken) + + class Service: + ap = m.ap + + @d.observe('api', 'test.disabled', fields=broken) + async def call(self): + return 42 + + @d.observe('run', 'test.disabled', fields=broken) + async def stream(self): + yield 42 + + assert await Service().call() == 42 + assert [v async for v in Service().stream()] == [42] + assert not m.pending + + +def test_bounds_and_disable_clear(): + m = manager() + for _ in range(8): + m.emit('api', 'test.operation', 'succeeded') + assert len(m.pending) == 4 + assert m.counters['dropped'] == 4 + m.ap.instance_config.data['space']['disable_telemetry'] = True + m.emit('api', 'test.operation', 'succeeded') + assert not m.pending + + +@pytest.mark.asyncio +async def test_partial_ack_retries_same_identity_and_drops_rejected(): + m = manager() + for _ in range(3): + m.emit('api', 'test.operation', 'succeeded') + ids = [e['event_id'] for e in m.pending] + requests = [] + + async def handler(req): + requests.append(json.loads(req.content)) + return httpx.Response( + 200, + json={ + 'code': 200, + 'data': {'accepted_event_ids': [ids[0]], 'rejected': [{'event_id': ids[1], 'code': 'invalid_event'}]}, + }, + ) + + m.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + await m.flush_once() + assert [e['event_id'] for e in m.pending] == [ids[2]] + assert m.counters['acked'] == 1 + assert m.counters['dropped'] == 1 + assert requests[0]['schema_version'] == 1 + await m.shutdown(drain_timeout=0) + + +@pytest.mark.asyncio +async def test_outage_finite_retry_and_slow_credentials(): + m = manager() + m.max_attempts = 2 + m.emit('run', 'test.operation', 'started') + calls = [] + + async def handler(req): + calls.append(req) + return httpx.Response(503) + + m.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + await m.flush_once() + await m.flush_once() + assert not m.pending + assert m.counters['dropped'] == 1 + assert len(calls) == 2 + + async def credentials(workspace): + await asyncio.sleep(60) + + m.credentials = credentials + m.request_timeout = 0.01 + m.emit('api', 'test.operation', 'succeeded') + await asyncio.wait_for(m.flush_once(), 0.2) + await m.shutdown(drain_timeout=0) + + +@pytest.mark.asyncio +async def test_inflight_disable_cancels_and_clears(): + m = manager() + entered = asyncio.Event() + + async def handler(req): + entered.set() + await asyncio.sleep(60) + + m.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + m.start() + m.emit('api', 'test.operation', 'succeeded') + await asyncio.wait_for(entered.wait(), 1) + m.ap.instance_config.data['space']['disable_beta_diagnostics'] = True + await asyncio.sleep(0.3) + assert not m.pending + await asyncio.wait_for(m.shutdown(drain_timeout=0), 0.5) + + +@pytest.mark.asyncio +async def test_observe_returns_errors_and_cancellation(): + m = manager() + + class Service: + ap = m.ap + + @d.observe('api', 'test.operation') + async def call(self, error=None): + if error: + raise error + return {'secret': 'CANARY'} + + s = Service() + assert await s.call() == {'secret': 'CANARY'} + with pytest.raises(ValueError): + await s.call(ValueError('CANARY')) + m.pending.clear() + with pytest.raises(asyncio.CancelledError): + await s.call(asyncio.CancelledError()) + assert m.pending[-1]['outcome'] == 'cancelled' + assert 'CANARY' not in json.dumps(m.pending) + + +@pytest.mark.asyncio +async def test_generator_send_throw_close_and_context_isolation(): + m = manager() + m.capacity = 30 + closed = [] + + class Service: + ap = m.ap + + @d.observe('run', 'test.operation') + async def stream(self): + try: + value = yield 1 + try: + yield value + except ValueError: + yield 3 + finally: + closed.append(True) + + gen = Service().stream() + assert await anext(gen) == 1 + assert d.current_span() is None + assert await gen.asend(7) == 7 + assert await gen.athrow(ValueError('CANARY')) == 3 + await gen.aclose() + assert closed == [True] + assert m.pending[-1]['outcome'] == 'cancelled' + assert d.current_span() is None + + +@pytest.mark.asyncio +async def test_generator_early_failure_and_explicit_terminal(): + m = manager() + + class Service: + ap = m.ap + + @d.observe('run', 'test.operation') + async def stream(self, fail): + if fail: + raise ValueError('prepare CANARY') + d.set_outcome('failed', reason_code='runner_failed') + yield 1 + + with pytest.raises(ValueError): + await anext(Service().stream(True)) + assert m.pending[-1]['outcome'] == 'failed' + m.pending.clear() + assert [v async for v in Service().stream(False)] == [1] + assert m.pending[-1]['outcome'] == 'failed' + + +@pytest.mark.asyncio +async def test_projection_fault_does_not_replace_business_return(monkeypatch): + m = manager() + + def broken(*args, **kwargs): + raise RuntimeError('CANARY projection') + + monkeypatch.setattr(d, 'result_outcome', broken) + + class Service: + ap = m.ap + + @d.observe('api', 'test.projection_fault') + async def call(self): + return 42 + + assert await Service().call() == 42 + assert m.pending[-1]['outcome'] == 'succeeded' + + +@pytest.mark.asyncio +async def test_workspace_batches_and_credential_failure_are_anonymous(): + m = manager() + workspaces = [str(uuid4()), str(uuid4())] + for workspace in workspaces: + m.emit('api', 'test.operation', 'succeeded', workspace_uuid=workspace) + requests = [] + + async def broken(workspace): + raise RuntimeError('CANARY credentials') + + m.credentials = broken + + async def handler(request): + data = json.loads(request.content) + requests.append((data, dict(request.headers))) + return httpx.Response( + 200, + json={'code': 200, 'data': {'accepted_event_ids': [e['event_id'] for e in data['events']], 'rejected': []}}, + ) + + m.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + await m.flush_once() + await m.flush_once() + assert len(requests) == 2 + assert [request[0]['events'][0]['workspace_uuid'] for request in requests] == workspaces + assert all('authorization' not in headers for _, headers in requests) + assert not m.pending + await m.shutdown(drain_timeout=0) + + +@pytest.mark.asyncio +async def test_summary_is_interval_delta_and_ack_replay_is_idempotent(): + m = manager() + m.capacity = 20 + m.emit('api', 'test.operation', 'succeeded') + m.report_transport() + first = m.pending[-1] + m.report_transport() + second = m.pending[-1] + assert first['attributes']['generated'] == 1 + assert second['attributes']['generated'] == 1 # Only first summary itself. + assert first['event_id'] != second['event_id'] + ids = [e['event_id'] for e in m.pending] + seen = [] + + async def handler(request): + body = json.loads(request.content) + seen.append([e['event_id'] for e in body['events']]) + return ( + httpx.Response(503) + if len(seen) == 1 + else httpx.Response(200, json={'code': 200, 'data': {'accepted_event_ids': ids + ids, 'rejected': []}}) + ) + + m.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + await m.flush_once() + await m.flush_once() + assert seen == [ids, ids] + assert m.counters['acked'] == len(ids) + assert not m.pending + await m.shutdown(drain_timeout=0) + + +@pytest.mark.asyncio +async def test_retention_and_bounded_shutdown(): + m = manager() + m.emit('api', 'test.operation', 'succeeded') + m.retention_seconds = -1 + await m.flush_once() + assert not m.pending and m.counters['dropped'] == 1 + m.retention_seconds = 900 + + async def handler(request): + await asyncio.sleep(60) + return httpx.Response(503) + + m.client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + m.emit('api', 'test.operation', 'succeeded') + await asyncio.wait_for(m.shutdown(drain_timeout=0.01), 0.5) + assert not m.pending and m.client.is_closed + + +@pytest.mark.asyncio +async def test_session_marker_restart_and_disable(tmp_path): + marker = tmp_path / 'session.json' + m = manager() + m.marker_path = marker + await m.start_session() + assert marker.exists() + recovered = manager() + recovered.marker_path = marker + await recovered.start_session() + assert recovered.pending[-1]['attributes']['previous_session_unclean'] is True + recovered.ap.instance_config.data['space']['disable_beta_diagnostics'] = True + recovered.start() + await asyncio.sleep(0.2) + assert not marker.exists() + await recovered.shutdown(drain_timeout=0) + await m.shutdown(drain_timeout=0) diff --git a/tests/unit_tests/test_build_revision_stamp.py b/tests/unit_tests/test_build_revision_stamp.py new file mode 100644 index 000000000..abbf195c2 --- /dev/null +++ b/tests/unit_tests/test_build_revision_stamp.py @@ -0,0 +1,30 @@ +"""Release artifacts must identify their actual source, not a branch label.""" + +import ast +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('stamp_build_revision', ROOT / 'scripts/stamp_build_revision.py') +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_stamp_accepts_full_revision(tmp_path): + target = tmp_path / 'src/langbot/_build_info.py' + target.parent.mkdir(parents=True) + revision = 'a1' * 20 + assert MODULE.stamp(tmp_path, revision) == revision + assignments = [node for node in ast.parse(target.read_text()).body if isinstance(node, ast.Assign)] + assert len(assignments) == 1 + assert ast.literal_eval(assignments[0].value) == revision + + +@pytest.mark.parametrize('revision', ['main', 'abc123', 'A' * 40, 'a' * 39, 'a' * 41, '../secret', 'a' * 40 + '\n']) +def test_stamp_rejects_non_revision_before_write(tmp_path, revision): + with pytest.raises(ValueError): + MODULE.stamp(tmp_path, revision) + assert not (tmp_path / 'src').exists() diff --git a/uv.lock b/uv.lock index d9cec6c6a..fc4e3b7c7 100644 --- a/uv.lock +++ b/uv.lock @@ -1999,7 +1999,7 @@ wheels = [ [[package]] name = "langbot" -version = "4.11.0b1" +version = "4.11.0b2" source = { editable = "." } dependencies = [ { name = "aiocqhttp" }, diff --git a/web/tests/unit/processor-detail-workbench.test.mjs b/web/tests/unit/processor-detail-workbench.test.mjs index ffcb1198b..5ff6068a9 100644 --- a/web/tests/unit/processor-detail-workbench.test.mjs +++ b/web/tests/unit/processor-detail-workbench.test.mjs @@ -72,15 +72,25 @@ test('processor forms expose their primary orchestration flow horizontally', () 'src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx', ); - assert.match( - agentForm, - /name: 'runner'[\s\S]*name: 'runner_config'[\s\S]*name: 'events_and_tools'/, + const agentSections = agentForm.slice( + agentForm.indexOf('const primarySections:'), + agentForm.indexOf('const runnerStatus ='), ); + assert.deepEqual( + [...agentSections.matchAll(/name: '([^']+)'/g)].map((match) => match[1]), + ['runner', 'events_and_tools'], + ); + const runnerPanel = agentForm.slice( + agentForm.indexOf("{activeSection === 'runner' &&"), + agentForm.indexOf("{activeSection === 'events_and_tools' &&"), + ); + assert.match(runnerPanel, /renderDynamicStage\(runnerSelectorStage\)/); + assert.match(runnerPanel, /renderDynamicStage\(activeRunnerStage\)/); assert.match( pipelineForm, /const primarySectionNames = \['trigger', 'ai', 'output'\]/, ); - assert.match(agentForm, /]*grid-cols-3/); + assert.match(agentForm, /]*grid-cols-2/); assert.match(pipelineForm, /]*grid-cols-3/); assert.doesNotMatch(agentForm, /