diff --git a/docs/API_KEY_AUTH.md b/docs/API_KEY_AUTH.md index 3eac9eb10..256da5f45 100644 --- a/docs/API_KEY_AUTH.md +++ b/docs/API_KEY_AUTH.md @@ -360,3 +360,42 @@ curl -X POST \ - API-key-enabled endpoints use the same resource shapes as the web UI - No need to learn different API paths - use the existing API documentation with API key authentication - API keys never select a Workspace from a request header; their persisted binding is authoritative + +## Workspace assistant browser session + +The Workspace assistant endpoints under `/api/v1/assistant` are account-session +features, not API-key/MCP automation tools. They require a Workspace-scoped user +token and `runtime.operate` for recommendation and turns. Conversation reads +also enforce account ownership. + +- `GET /recommended-model` resolves and validates the recommended tool-capable + model when the panel opens. An explicit `model_uuid` in a turn always takes + precedence; it never silently falls back if unavailable. +- `POST /conversations/{id}/turn/stream` accepts the same revision, text/model, + or approval decision as `/turn`, and returns NDJSON. `snapshot` frames contain + the public conversation; `text` frames contain the cumulative text for the + current model invocation; `phase` frames identify the model round or running + tool. A snapshot replaces transient text/tool progress. `completed` contains + the final conversation (including approval or failed states); `error` is a + transport/admission failure. Blank lines are heartbeats. +- `GET /conversations` lists the latest 100 conversations owned by the current + account in this Workspace. `GET /conversations/{id}` includes transient + `progress` (cumulative text, round and running tool) while execution is active. +- Disconnecting a streaming response only detaches the browser. The backend owns + the task until completion, approval, timeout or explicit stop. Slow or absent + consumers never block execution; reconnect using the conversation snapshot. +- `POST /conversations/{id}/stop` accepts `{ "revision": N }` for the active turn. + It enforces ownership and revision, cancels execution and records `stopped`. + Stopping cannot roll back tool effects and clients must never replay a write + automatically. Writes still require confirmation of saved arguments. +- Background execution lives in the current backend process, with at most four + simultaneous turns and the existing 120-second turn limit. This is not a durable + job queue: backend restarts do not resume interrupted tasks. + +The original JSON `/turn` endpoint remains compatible for existing callers. + +The assistant recommendation uses Space `/api/v1/models/assistant/recommendation`, +which selects only from the assistant pool using fresh successful health checks. +An unavailable recommendation returns 503 without falling back to general chat rankings. +The browser checks Workspace model providers before mounting the assistant; an empty +provider list shows Account sign-in and model settings actions instead of a composer. diff --git a/packaging/fnos/manifest b/packaging/fnos/manifest index a05b5be1a..500a93ff2 100644 --- a/packaging/fnos/manifest +++ b/packaging/fnos/manifest @@ -1,5 +1,5 @@ appname=langbot -version=4.10.11 +version=4.11.0-rc.1 display_name=LangBot desc=基于 LLM 的多平台智能对话机器人，支持 QQ、微信、飞书、钉钉、Telegram 等十余种即时通讯平台，内置 Web 管理界面和 AI Agent 能力。 platform=all diff --git a/pyproject.toml b/pyproject.toml index 9d9b80fe8..5319a8aaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "langbot" -version = "4.11.0-beta.3" +version = "4.11.0-rc.1" description = "Production-grade platform for building agentic IM bots" readme = "README.md" license-files = ["LICENSE"] @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin @ https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl#sha256=b2f05302ca25b374ff8c094794573696b75a7d1a7731a0fe2eebbc71796e8e0d", + "langbot-plugin==0.7.11rc1", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/skills/skills/langbot-mcp-ops/SKILL.md b/skills/skills/langbot-mcp-ops/SKILL.md index d96e56f2f..2647573b4 100644 --- a/skills/skills/langbot-mcp-ops/SKILL.md +++ b/skills/skills/langbot-mcp-ops/SKILL.md @@ -174,3 +174,22 @@ Legacy EventListener plugins remain in the Pipeline lifecycle. `list_processor_runs` includes `created_at_ms`, `started_at_ms`, and `finished_at_ms`: Host lifecycle times in epoch milliseconds. Use the start and finish times for elapsed processing time; select a run and call `get_processor_run_events` for its logs and action results. These times are not internal plugin profiling data. + +## Unified execution monitoring + +Use `get_monitoring_executions` for the execution list and +its legacy `pipeline_ids` parameter to filter any processor kind (Agent, +Pipeline or event processor); the summary uses the same processor scope. Use +`get_monitoring_execution_detail` with `source=auto` to resolve a run, +message or event identifier outside the current list page. The detail exposes +`inputs`, `outputs` (generated content), `deliveries` (recorded platform sends), +`events`, `llm_calls`, `tool_calls`, `errors`, `related`, and `conversation` in +`pages`. Follow each section's `has_more` and `next_offset` independently. +Conversation history supplies context; historical messages without explicit +links must not be asserted to belong to the selected execution. Events without +a processor run use `source=event`. All lookups remain Workspace-scoped. + +Monitoring record filters accept `mode` (`all`, `real`, `debug`) and +`execution_statuses` (normalized execution statuses). These select the owning +execution, not the individual model/tool call outcome. Calls without a recorded +execution link are excluded when an execution filter is active. diff --git a/src/langbot/pkg/agent/runner/config_schema.py b/src/langbot/pkg/agent/runner/config_schema.py index aa7d13f7e..1fad3193f 100644 --- a/src/langbot/pkg/agent/runner/config_schema.py +++ b/src/langbot/pkg/agent/runner/config_schema.py @@ -16,6 +16,65 @@ KB_SELECTOR_TYPES = {'knowledge-base-multi-selector'} PROMPT_EDITOR_TYPES = {'prompt-editor'} NONE_SENTINELS = {'', '__none__', '__none'} +# Selector field types that pick a Host resource. The Runner config form is +# plugin-declared and open-ended (DynamicFormItemSchema allows extra types), so +# this is a registry, not a fixed list: a new selector type is one line here and +# is then reported by telemetry without touching the extraction call sites. +RESOURCE_SELECTOR_TYPES: dict[str, str] = { + 'knowledge-base-multi-selector': 'knowledge_bases', + 'tool-multi-selector': 'tools', + 'mcp-server-multi-selector': 'mcp_servers', + 'skill-multi-selector': 'skills', +} + + +def declared_resource_selectors(descriptor: RunnerDescriptor | None) -> dict[str, list[str]]: + """Map each selected resource kind to the schema field names that select it. + + Only the descriptor is read, so this reports what a Runner *can* select even + when the current configuration leaves the field empty. + """ + declared: dict[str, list[str]] = {} + if descriptor is None: + return declared + for item in descriptor.config_schema or []: + if not isinstance(item, dict): + continue + kind = RESOURCE_SELECTOR_TYPES.get(normalize_schema_item_type(item.get('type'))) + field_name = item.get('name') + if not kind or not isinstance(field_name, str) or not field_name: + continue + declared.setdefault(kind, []) + if field_name not in declared[kind]: + declared[kind].append(field_name) + return declared + + +def extract_selected_resources( + descriptor: RunnerDescriptor | None, + runner_config: dict[str, typing.Any] | None, +) -> dict[str, list[str]]: + """Extract the resource ids selected through schema-declared selector fields. + + Field names are never hard-coded: they come from the Runner's own config + schema, so a form that renames or adds a selector keeps working. + """ + selected: dict[str, list[str]] = {} + if not isinstance(runner_config, dict): + return selected + for kind, field_names in declared_resource_selectors(descriptor).items(): + values: list[str] = [] + for field_name in field_names: + value = runner_config.get(field_name) + if isinstance(value, str): + if value not in NONE_SENTINELS: + values.append(value) + elif isinstance(value, (list, tuple)): + values.extend(item for item in value if isinstance(item, str) and item not in NONE_SENTINELS) + if values: + selected[kind] = list(dict.fromkeys(values)) + return selected + def normalize_schema_item_type(item_type: typing.Any) -> typing.Any: """Normalize legacy/frontend DynamicForm aliases to protocol field types.""" diff --git a/src/langbot/pkg/agent/runner/event_log_store.py b/src/langbot/pkg/agent/runner/event_log_store.py index 1dbd8a8b2..10b3a1860 100644 --- a/src/langbot/pkg/agent/runner/event_log_store.py +++ b/src/langbot/pkg/agent/runner/event_log_store.py @@ -128,7 +128,33 @@ class EventLogStore: created_at=as_naive_utc(_utc_now()), ) session.add(event) - await session.commit() + try: + await session.commit() + except sqlalchemy.exc.IntegrityError: + await session.rollback() + existing = ( + (await session.execute(sqlalchemy.select(EventLog).where(EventLog.event_id == event_id))) + .scalars() + .first() + ) + if existing is None or existing.workspace_id != workspace_id or existing.bot_id != bot_id: + raise + # Complete the ingress snapshot once a runner claims it. A + # second subscriber must not replace another run's identity. + for field, value in { + 'run_id': run_id, + 'runner_id': runner_id, + 'conversation_id': conversation_id, + 'thread_id': thread_id, + 'actor_id': actor_id, + 'actor_name': actor_name, + }.items(): + if getattr(existing, field) is None and value is not None: + setattr(existing, field, value) + if input_json and ('contents' in input_json or 'attachments' in input_json): + existing.input_json = json.dumps(input_json) + existing.input_summary = input_summary + await session.commit() return event_id diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index e207e743a..74f862d79 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -37,7 +37,9 @@ from .run_journal import AgentRunJournal from .session_registry import AgentRunSessionRegistry, get_session_registry from .state_scope import build_state_context from ...provider.tools.loaders import skill as skill_loader +from ...telemetry import resources as telemetry_resources from ...telemetry import trace as trace_mod +from ...telemetry.execution import bind_trace as bind_execution_trace from ...telemetry.execution import close_trace as close_execution_trace from ...telemetry.execution import record as record_execution @@ -141,6 +143,22 @@ class AgentRunOrchestrator: binding=binding, descriptor=descriptor, ) + # Telemetry: report what this run was granted (never what it used). The + # materialized resource list is authoritative for tools/skills/models; + # the Runner's own config form is authoritative for its selectors. + try: + from . import config_schema as runner_config_schema + + capabilities = getattr(descriptor, 'capabilities', None) + telemetry_resources.note_agent( + runner_id=descriptor.id, + resources=resources, + selectors=runner_config_schema.extract_selected_resources(descriptor, binding.runner_config), + capabilities=capabilities.model_dump() if hasattr(capabilities, 'model_dump') else None, + tools_all=getattr(binding.resource_policy, 'allow_all_tools', None), + ) + except Exception: + pass context = await self.context_builder.build_context_from_event( event=execution_event, @@ -176,6 +194,21 @@ class AgentRunOrchestrator: state_context = build_state_context(event, binding, descriptor) run_id = context['run_id'] + + # Monitoring attributes every call to the processor that ran it and to the + # record that owns it; a Runner has to claim both itself, since no + # Pipeline did. Runs before the Runner is invoked and before the run + # context is handed to it, so its own variables stay private. + from ...pipeline import monitoring_helper + + await monitoring_helper.prepare_monitoring_identity( + self.ap, + execution_query, + processor_type=binding.processor_type, + processor_id=binding.processor_id, + runner_id=binding.runner_id, + execution_record_id=run_id, + ) context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any( tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform' for tool in resources.get('tools', []) @@ -213,13 +246,16 @@ class AgentRunOrchestrator: terminal_reason: str | None = None terminal_usage: dict[str, typing.Any] | None = None execution_outcome = 'unknown' + execution_error = '' # A run reached without a platform ingress (WebUI debug, service API) # owns its own chain; a run inside an ingress reuses that chain. trace_binding: trace_mod.TraceBinding | None = None run_token: typing.Any = None + # Workflow step node for this run; '' until the step scope is opened. + node = '' try: - trace_binding = trace_mod.bind() + trace_binding = bind_execution_trace(self.ap, run_id) run_token = trace_mod.set_run(run_id) await self.journal.create_run( event=event, @@ -228,6 +264,11 @@ class AgentRunOrchestrator: context=context, authorization=run_authorization, ) + monitoring_id = (execution_query.variables or {}).get('_monitoring_message_id') + if monitoring_id and monitoring_id != run_id: + await self.ap.monitoring_service.link_execution_message( + get_query_execution_context(execution_query), monitoring_id, run_id, event.event_id + ) await self._session_registry.register( run_id=run_id, runner_id=descriptor.id, @@ -261,139 +302,149 @@ class AgentRunOrchestrator: event_log_id=event_log_id, ) - async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results: - async for result_dict in results: - result_dict = dict(result_dict) - sequence = result_dict.get('sequence') - if sequence is not None: - try: - sequence_int = int(sequence) - except (TypeError, ValueError): - self.ap.logger.warning( - f'Runner {descriptor.id} returned invalid result sequence: {sequence}' - ) - sequence_int = last_sequence + 1 - result_dict['sequence'] = sequence_int - else: - if sequence_int in seen_sequences: + with trace_mod.stage_scope() as node: + async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results: + async for result_dict in results: + result_dict = dict(result_dict) + sequence = result_dict.get('sequence') + if sequence is not None: + try: + sequence_int = int(sequence) + except (TypeError, ValueError): self.ap.logger.warning( - f'Runner {descriptor.id} returned duplicate result sequence ' - f'{sequence_int} for run {run_id}; dropping duplicate' - ) - continue - if sequence_int <= 0: - self.ap.logger.warning( - f'Runner {descriptor.id} returned non-positive result sequence ' - f'{sequence_int} for run {run_id}' + f'Runner {descriptor.id} returned invalid result sequence: {sequence}' ) sequence_int = last_sequence + 1 result_dict['sequence'] = sequence_int - elif last_sequence and sequence_int != last_sequence + 1: - self.ap.logger.warning( - f'Runner {descriptor.id} result sequence gap or out-of-order ' - f'for run {run_id}: previous={last_sequence}, current={sequence_int}' - ) + else: + if sequence_int in seen_sequences: + self.ap.logger.warning( + f'Runner {descriptor.id} returned duplicate result sequence ' + f'{sequence_int} for run {run_id}; dropping duplicate' + ) + continue + if sequence_int <= 0: + self.ap.logger.warning( + f'Runner {descriptor.id} returned non-positive result sequence ' + f'{sequence_int} for run {run_id}' + ) + sequence_int = last_sequence + 1 + result_dict['sequence'] = sequence_int + elif last_sequence and sequence_int != last_sequence + 1: + self.ap.logger.warning( + f'Runner {descriptor.id} result sequence gap or out-of-order ' + f'for run {run_id}: previous={last_sequence}, current={sequence_int}' + ) + seen_sequences.add(sequence_int) + last_sequence = max(last_sequence, sequence_int) + else: + sequence_int = last_sequence + 1 + result_dict['sequence'] = sequence_int seen_sequences.add(sequence_int) - last_sequence = max(last_sequence, sequence_int) - else: - sequence_int = last_sequence + 1 - result_dict['sequence'] = sequence_int - seen_sequences.add(sequence_int) - last_sequence = sequence_int + last_sequence = sequence_int - result_type = result_dict.get('type') - if result_type and not self.result_normalizer.validate_payload( - result_type, - result_dict.get('data', {}), - descriptor, - ): - continue - - await self.journal.append_run_result( - result_dict=result_dict, - run_id=run_id, - sequence=sequence_int, - ) - - # Trusted Host observers receive validated events before message-only normalization. - result_observer = (adapter_context or {}).get('_result_observer') - if result_observer is not None: - await result_observer(result_dict) - - if result_type == 'state.updated': - await self.journal.handle_state_updated_event( - result_dict, - event, - binding, + result_type = result_dict.get('type') + if result_type and not self.result_normalizer.validate_payload( + result_type, + result_dict.get('data', {}), descriptor, - run_id=run_id, - ) - await self.result_normalizer.normalize(result_dict, descriptor) - continue - - if result_type == 'action.requested': - if await self.interaction_manager.handle_result( - result_dict=result_dict, - event=event, - binding=binding, - descriptor=descriptor, - run_id=run_id, - adapter_context=adapter_context, ): continue - if result_type == 'run.completed': - terminal_status = 'completed' - terminal_reason = ( - result_dict.get('data', {}).get('finish_reason') - if isinstance(result_dict.get('data'), dict) - else None - ) - usage = result_dict.get('usage') - if isinstance(usage, dict): - terminal_usage = usage - elif result_type == 'run.failed': - terminal_status = 'failed' - data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {} - terminal_reason = data.get('error') or data.get('code') - usage = result_dict.get('usage') - if isinstance(usage, dict): - terminal_usage = usage - - has_completed_message = result_type == 'message.completed' or ( - result_type == 'run.completed' - and isinstance(result_dict.get('data'), dict) - and bool(result_dict['data'].get('message')) - ) - if has_completed_message and event.conversation_id and not assistant_transcript_written: - await self.journal.write_assistant_transcript( + journal_started = time.monotonic() + await self.journal.append_run_result( result_dict=result_dict, - event=event, run_id=run_id, - runner_id=descriptor.id, + sequence=sequence_int, ) - assistant_transcript_written = True + journal_ms = (time.monotonic() - journal_started) * 1000 - result = await self.result_normalizer.normalize(result_dict, descriptor) - file_ids = result_dict.get('data', {}).get('file_ids', []) - if file_ids: - if result_type != 'message.completed' or result is None: - raise ValueError('Files must be attached to a completed message') - if binding.processor_type != 'pipeline': - raise ValueError('Agent files must be sent explicitly through the reply API') - from ...box.runner import exported_message, binding_for + # Trusted Host observers receive validated events before message-only normalization. + result_observer = (adapter_context or {}).get('_result_observer') + observer_started = time.monotonic() + if result_observer is not None: + await result_observer(result_dict) + observer_ms = (time.monotonic() - observer_started) * 1000 + if journal_ms + observer_ms >= 100: + self.ap.logger.debug( + f'Slow runner event consumer run={run_id} type={result_type} ' + f'journal_ms={journal_ms:.1f} observer_ms={observer_ms:.1f}' + ) - async with binding_for(execution_query).lock: - result.attachments = exported_message(execution_query, file_ids, consume=True) + if result_type == 'state.updated': + await self.journal.handle_state_updated_event( + result_dict, + event, + binding, + descriptor, + run_id=run_id, + ) + await self.result_normalizer.normalize(result_dict, descriptor) + continue - if result is not None: - yield result + if result_type == 'action.requested': + if await self.interaction_manager.handle_result( + result_dict=result_dict, + event=event, + binding=binding, + descriptor=descriptor, + run_id=run_id, + adapter_context=adapter_context, + ): + continue - run_snapshot = await self.journal.get_run(run_id) - if run_snapshot and run_snapshot.get('cancel_requested_at') is not None: - terminal_status = 'cancelled' - terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested' - break + if result_type == 'run.completed': + terminal_status = 'completed' + terminal_reason = ( + result_dict.get('data', {}).get('finish_reason') + if isinstance(result_dict.get('data'), dict) + else None + ) + usage = result_dict.get('usage') + if isinstance(usage, dict): + terminal_usage = usage + elif result_type == 'run.failed': + terminal_status = 'failed' + data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {} + terminal_reason = data.get('error') or data.get('code') + usage = result_dict.get('usage') + if isinstance(usage, dict): + terminal_usage = usage + + has_completed_message = result_type == 'message.completed' or ( + result_type == 'run.completed' + and isinstance(result_dict.get('data'), dict) + and bool(result_dict['data'].get('message')) + ) + if has_completed_message and event.conversation_id and not assistant_transcript_written: + await self.journal.write_assistant_transcript( + result_dict=result_dict, + event=event, + run_id=run_id, + runner_id=descriptor.id, + ) + assistant_transcript_written = True + + result = await self.result_normalizer.normalize(result_dict, descriptor) + file_ids = result_dict.get('data', {}).get('file_ids', []) + if file_ids: + if result_type != 'message.completed' or result is None: + raise ValueError('Files must be attached to a completed message') + if binding.processor_type != 'pipeline': + raise ValueError('Agent files must be sent explicitly through the reply API') + from ...box.runner import exported_message, binding_for + + async with binding_for(execution_query).lock: + result.attachments = exported_message(execution_query, file_ids, consume=True) + + if result is not None: + yield result + + run_snapshot = await self.journal.get_run(run_id) + if run_snapshot and run_snapshot.get('cancel_requested_at') is not None: + terminal_status = 'cancelled' + terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested' + break await self.journal.finalize_run( run_id=run_id, status=terminal_status or 'completed', @@ -403,11 +454,21 @@ class AgentRunOrchestrator: execution_outcome = {'completed': 'success', 'failed': 'failed', 'cancelled': 'cancelled'}.get( terminal_status or '', 'unknown' ) - except asyncio.CancelledError: + execution_error = '' if execution_outcome == 'success' else (terminal_reason or execution_outcome) + except (asyncio.CancelledError, GeneratorExit): execution_outcome = 'cancelled' + execution_error = 'cancelled' + await self.journal.finalize_run( + run_id=run_id, + status='cancelled', + status_reason='consumer_cancelled', + usage=terminal_usage, + ) raise except Exception as exc: execution_outcome = 'timeout' if self._is_deadline_exhausted(context) else 'failed' + # A chain that broke must always carry a reason, even for a bare exception. + execution_error = str(exc) or type(exc).__name__ failed_usage = terminal_usage await self.journal.finalize_run( run_id=run_id, @@ -426,6 +487,8 @@ class AgentRunOrchestrator: runner=descriptor.id, outcome=execution_outcome, synthetic=event.source == 'webui', + error=execution_error, + node=node, ) trace_mod.reset_run(run_token) if trace_binding is not None and trace_mod.unbind_root(trace_binding): diff --git a/src/langbot/pkg/agent/runner/platform_tools.py b/src/langbot/pkg/agent/runner/platform_tools.py index 5599c4953..6591542ed 100644 --- a/src/langbot/pkg/agent/runner/platform_tools.py +++ b/src/langbot/pkg/agent/runner/platform_tools.py @@ -684,49 +684,104 @@ async def execute_platform_tool( normalized = _normalize_platform_params(definition, parameters) if definition.scope == 'event': normalized = _event_params(definition, context, normalized) - # This flag is frozen by the Host from the synthetic debug envelope, not tool arguments. - if delivery.get('surface') == 'webui' and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True: - result = _execute_mock_platform_tool(definition, context, normalized) - if message_chain is not None: - result['parameters']['message'] = message_chain.model_dump(mode='json') - from ...telemetry.execution import record + # Plugin/RPC actions run outside the ingress context, so bind the owning + # execution id here: nested adapter observations then join its trace. + from ...telemetry.execution import record, reset_execution_id, set_execution_id - record( - ap, - execution_context, - family='platform_api', - operation=definition.api, - mode=authorization.get('processor_type', 'none'), - synthetic=True, - outcome='success', - ) - return result - bot_id = authorization.get('bot_id') - if not bot_id: - raise ValueError('This run is not associated with a platform bot') - bot = await ap.platform_mgr.get_bot_by_uuid(execution_context, bot_id) - if bot is None: - raise ValueError(f'Bot {bot_id} is not running') - if definition.api not in set(bot.adapter.get_supported_apis() or []): - raise ValueError(f'Platform API {definition.api} is no longer supported by bot {bot_id}') - api_func = getattr(bot.adapter, definition.api, None) - if not callable(api_func): - raise ValueError(f'Platform API {definition.api} is declared but not implemented') - if definition.api == 'send_message': - normalized = { - 'target_type': _require_string(normalized, 'target_type'), - 'target_id': _require_string(normalized, 'target_id'), - 'message': message_chain - if message_chain is not None - else platform_message.MessageChain([platform_message.Plain(text=_require_string(normalized, 'text'))]), - } - from ...telemetry.platform import processing_mode - - token = processing_mode.set(authorization.get('processor_type', 'none')) + execution_id = str(session.get('run_id') or '').strip() + execution_token = set_execution_id(execution_id) try: - return await api_func(**normalized) + # This flag is frozen by the Host from the synthetic debug envelope, not tool arguments. + mock = ( + delivery.get('surface') == 'webui' + and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True + ) + if mock: + outcome = 'unknown' + error_detail = '' + try: + result = _execute_mock_platform_tool(definition, context, normalized) + if message_chain is not None: + result['parameters']['message'] = message_chain.model_dump(mode='json') + outcome = 'success' + except Exception as exc: + outcome = 'failed' + error_detail = str(exc) + raise + finally: + record( + ap, + execution_context, + family='platform_api', + operation=definition.api, + mode=authorization.get('processor_type', 'none'), + synthetic=True, + outcome=outcome, + error=error_detail, + execution_id=execution_id or None, + ) + return result + bot_id = authorization.get('bot_id') + if not bot_id: + raise ValueError('This run is not associated with a platform bot') + bot = await ap.platform_mgr.get_bot_by_uuid(execution_context, bot_id) + if bot is None: + raise ValueError(f'Bot {bot_id} is not running') + if definition.api not in set(bot.adapter.get_supported_apis() or []): + raise ValueError(f'Platform API {definition.api} is no longer supported by bot {bot_id}') + api_func = getattr(bot.adapter, definition.api, None) + if not callable(api_func): + raise ValueError(f'Platform API {definition.api} is declared but not implemented') + if definition.api == 'send_message': + normalized = { + 'target_type': _require_string(normalized, 'target_type'), + 'target_id': _require_string(normalized, 'target_id'), + 'message': message_chain + if message_chain is not None + else platform_message.MessageChain([platform_message.Plain(text=_require_string(normalized, 'text'))]), + } + from ...telemetry.platform import processing_mode + + token = processing_mode.set(authorization.get('processor_type', 'none')) + delivery_error = None + try: + return await api_func(**normalized) + except Exception as exc: + delivery_error = str(exc) + raise + finally: + processing_mode.reset(token) + if definition.api == 'send_message' and getattr(ap, 'monitoring_service', None) is not None: + try: + import json + + chain = normalized.get('message') + content = chain.model_dump(mode='json') if hasattr(chain, 'model_dump') else str(chain or '') + await ap.monitoring_service.record_message( + execution_context, + bot_id=bot_id, + bot_name=str(getattr(getattr(bot, 'bot_entity', None), 'name', bot_id)), + pipeline_id=str(authorization.get('processor_id') or ''), + pipeline_name=str(authorization.get('processor_id') or authorization.get('runner_id') or ''), + message_content=json.dumps(content, ensure_ascii=False), + session_id=str(authorization.get('conversation_id') or ''), + role='assistant', + status='error' if delivery_error else 'success', + level='error' if delivery_error else 'info', + run_id=execution_id, + parent_message_id=execution_id, + variables=json.dumps( + { + 'delivery_error': delivery_error, + 'target_type': normalized.get('target_type'), + 'target_id': normalized.get('target_id'), + } + ), + ) + except Exception as exc: + ap.logger.warning(f'Failed to record platform delivery: {exc}') finally: - processing_mode.reset(token) + reset_execution_id(execution_token) def _execute_mock_platform_tool( diff --git a/src/langbot/pkg/agent/runner/run_journal.py b/src/langbot/pkg/agent/runner/run_journal.py index dd2bfbf27..3a0217f89 100644 --- a/src/langbot/pkg/agent/runner/run_journal.py +++ b/src/langbot/pkg/agent/runner/run_journal.py @@ -68,6 +68,26 @@ class AgentRunJournal: def _sanitize_attachments(cls, attachments: typing.Iterable[typing.Any]) -> list[dict[str, typing.Any]]: return [cls._sanitize_attachment_ref(attachment) for attachment in attachments] + @classmethod + def _sanitize_event_data(cls, value: typing.Any) -> typing.Any: + """Retain arbitrary event fields without copying inline binary payloads.""" + if hasattr(value, 'model_dump'): + value = value.model_dump(mode='json') + if isinstance(value, list): + return [cls._sanitize_event_data(item) for item in value] + if not isinstance(value, dict): + return value + result = {} + for key, item in value.items(): + if key in ('image_base64', 'file_base64', 'base64') and item: + result[key] = None + result['content_redacted'] = True + elif key == 'attachments' and isinstance(item, list): + result[key] = [cls._sanitize_event_data(cls._sanitize_attachment_ref(ref)) for ref in item] + else: + result[key] = cls._sanitize_event_data(item) + return result + async def create_run( self, *, @@ -78,6 +98,9 @@ class AgentRunJournal: authorization: dict[str, typing.Any], ) -> dict[str, typing.Any]: """Create the Host-owned run ledger record.""" + from ...telemetry import trace + + trace_state = trace.current() runtime = context.get('runtime') if isinstance(context, dict) else {} return await self._get_run_ledger_store().create_run( run_id=context['run_id'], @@ -93,17 +116,21 @@ class AgentRunJournal: authorization=authorization, metadata={ 'event_type': event.event_type, + 'ingress_event_id': trace_state.event_id if trace_state else event.event_id, 'source': event.source, 'processor_id': binding.processor_id, 'processor_type': binding.processor_type, **( - {'input_event': event.data, 'delivery': event.delivery.model_dump(mode='json')} + { + 'input_event': self._sanitize_event_data(event.data), + 'delivery': event.delivery.model_dump(mode='json'), + } if binding.processor_type == 'event_processor' else {} ), **( { - **({'input_event': event.data} if event.event_type != 'message.received' else {}), + 'input_event': self._sanitize_event_data(event.data), 'input': { 'text': context.get('input', {}).get('text', ''), 'contents': self._sanitize_contents(context.get('input', {}).get('contents', [])), @@ -260,7 +287,7 @@ class AgentRunJournal: event_time=( datetime.datetime.fromtimestamp(event.event_time, datetime.timezone.utc) if event.event_time else None ), - metadata=metadata, + metadata={**(metadata or {}), 'input_event': self._sanitize_event_data(event.data)}, ) async def write_user_transcript( diff --git a/src/langbot/pkg/api/http/controller/groups/assistant.py b/src/langbot/pkg/api/http/controller/groups/assistant.py index f902c6f44..7684db2e1 100644 --- a/src/langbot/pkg/api/http/controller/groups/assistant.py +++ b/src/langbot/pkg/api/http/controller/groups/assistant.py @@ -1,5 +1,8 @@ """Web-session-only assistant endpoints; resource tools use the existing service layer.""" +import asyncio +import json + import quart from uuid import UUID from pydantic import BaseModel, ConfigDict, Field, ValidationError @@ -10,6 +13,11 @@ from ...context import RequestContext from ...service.assistant import AssistantError, AssistantService +class StopInput(BaseModel): + model_config = ConfigDict(extra='forbid') + revision: int = Field(ge=0, strict=True) + + class TurnInput(BaseModel): model_config = ConfigDict(extra='forbid') revision: int = Field(ge=0, strict=True) @@ -23,6 +31,27 @@ class AssistantRouterGroup(group.RouterGroup): async def initialize(self): service = AssistantService(self.ap) + @self.route('/conversations', methods=['GET'], permission=Permission.RESOURCE_VIEW) + async def list_conversations(request_context: RequestContext): + return self.success(data={'conversations': await service.list_conversations(request_context)}) + + @self.route('/conversations//stop', methods=['POST'], permission=Permission.RUNTIME_OPERATE) + async def stop(conversation_id: str, request_context: RequestContext): + try: + body = StopInput.model_validate(await quart.request.get_json()) + return self.success(data=await service.stop(request_context, conversation_id, body.revision)) + except ValidationError: + return self.http_status(400, 'invalid_input', 'Invalid stop request') + except AssistantError as exc: + return self.http_status(exc.status, exc.code, exc.code) + + @self.route('/recommended-model', methods=['GET'], permission=Permission.RUNTIME_OPERATE) + async def recommended_model(request_context: RequestContext): + try: + return self.success(data=await service.recommended_model(request_context)) + except AssistantError as exc: + return self.http_status(exc.status, exc.code, exc.code) + @self.route('/conversations', methods=['POST'], permission=Permission.RUNTIME_OPERATE) async def create(request_context: RequestContext): try: @@ -33,7 +62,7 @@ class AssistantRouterGroup(group.RouterGroup): @self.route('/conversations/', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get(conversation_id: str, request_context: RequestContext): try: - return self.success(data=service.public_view(await service.get(request_context, conversation_id))) + return self.success(data=await service.progress(request_context, conversation_id)) except AssistantError as exc: return self.http_status(exc.status, exc.code, exc.code) @@ -56,3 +85,43 @@ class AssistantRouterGroup(group.RouterGroup): return self.http_status(400, 'invalid_input', 'Invalid assistant request') except AssistantError as exc: return self.http_status(exc.status, exc.code, exc.code) + + @self.route( + '/conversations//turn/stream', methods=['POST'], permission=Permission.RUNTIME_OPERATE + ) + async def stream_turn(conversation_id: str, request_context: RequestContext): + try: + body = TurnInput.model_validate(await quart.request.get_json()) + if (body.text is None) == (body.approved is None) or (body.text is not None and not body.text.strip()): + return self.http_status(400, 'invalid_input', 'Provide text or an approval decision') + await service.get(request_context, conversation_id) + except ValidationError: + return self.http_status(400, 'invalid_input', 'Invalid assistant request') + except AssistantError as exc: + return self.http_status(exc.status, exc.code, exc.code) + try: + return assistant_stream_response(service, request_context, conversation_id, body) + except AssistantError as exc: + return self.http_status(exc.status, exc.code, exc.code) + + +def assistant_stream_response(service, context, conversation_id, body): + """Detach execution from the subscriber; only an explicit stop cancels a run.""" + queue = service.start_run(context, conversation_id, body) + + async def stream(): + while True: + try: + frame = await asyncio.wait_for(queue.get(), timeout=15) + except TimeoutError: + yield '\n' + continue + yield json.dumps(frame, ensure_ascii=False) + '\n' + if frame['kind'] in {'completed', 'error'}: + break + + response = quart.Response(stream(), content_type='application/x-ndjson; charset=utf-8') + response.timeout = None + response.headers['Cache-Control'] = 'no-store' + response.headers['X-Accel-Buffering'] = 'no' + return response diff --git a/src/langbot/pkg/api/http/controller/groups/monitoring.py b/src/langbot/pkg/api/http/controller/groups/monitoring.py index 01a901743..8df374b98 100644 --- a/src/langbot/pkg/api/http/controller/groups/monitoring.py +++ b/src/langbot/pkg/api/http/controller/groups/monitoring.py @@ -50,6 +50,54 @@ class MonitoringRouterGroup(group.RouterGroup): return self.success(data=metrics) + @self.route('/executions', methods=['GET'], permission=Permission.RESOURCE_VIEW) + async def get_executions(request_context: RequestContext) -> str: + """Unified execution list across agent runs and pipeline queries.""" + bot_ids = quart.request.args.getlist('botId') + pipeline_ids = quart.request.args.getlist('pipelineId') + agent_ids = quart.request.args.getlist('agentId') + statuses = quart.request.args.getlist('status') + source = quart.request.args.get('source', 'all') + mode = quart.request.args.get('mode', 'all') + start_time = parse_iso_datetime(quart.request.args.get('startTime')) + end_time = parse_iso_datetime(quart.request.args.get('endTime')) + + result = await self.ap.monitoring_service.get_executions( + request_context, + bot_ids=bot_ids if bot_ids else None, + pipeline_ids=pipeline_ids if pipeline_ids else None, + agent_ids=agent_ids if agent_ids else None, + statuses=statuses if statuses else None, + source=source, + mode=mode, + start_time=start_time, + end_time=end_time, + limit=quart.request.args.get('limit', 50), + offset=quart.request.args.get('offset', 0), + ) + + return self.success(data=result) + + @self.route( + '/executions//', + methods=['GET'], + permission=Permission.RESOURCE_VIEW, + ) + async def get_execution_detail(source: str, execution_id: str, request_context: RequestContext) -> str: + """Full trace of a single execution (agent run or pipeline query).""" + try: + detail = await self.ap.monitoring_service.get_execution_detail( + request_context, + source, + execution_id, + section=quart.request.args.get('section'), + offset=quart.request.args.get('offset', 0), + limit=quart.request.args.get('limit', 100), + ) + except ValueError as exc: + return self.fail(404, str(exc)) + return self.success(data=detail) + @self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_token_statistics(request_context: RequestContext) -> str: """Get detailed token usage statistics (summary, per-model, timeseries).""" @@ -66,6 +114,8 @@ class MonitoringRouterGroup(group.RouterGroup): stats = await self.ap.monitoring_service.get_token_statistics( request_context, + execution_statuses=quart.request.args.getlist('status') or None, + mode=quart.request.args.get('mode', 'all'), bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -327,6 +377,10 @@ class MonitoringRouterGroup(group.RouterGroup): @self.route('/data', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_all_data(request_context: RequestContext) -> str: """Get all monitoring data in a single request""" + execution_filters = { + 'execution_statuses': quart.request.args.getlist('status') or None, + 'mode': quart.request.args.get('mode', 'all'), + } # Parse query parameters bot_ids = quart.request.args.getlist('botId') pipeline_ids = quart.request.args.getlist('pipelineId') @@ -341,6 +395,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get overview metrics overview = await self.ap.monitoring_service.get_overview_metrics( request_context, + **execution_filters, bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -350,6 +405,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get messages messages, messages_total = await self.ap.monitoring_service.get_messages( request_context, + **execution_filters, bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -361,6 +417,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get LLM calls llm_calls, llm_calls_total = await self.ap.monitoring_service.get_llm_calls( request_context, + **execution_filters, bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -372,6 +429,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get tool calls tool_calls, tool_calls_total = await self.ap.monitoring_service.get_tool_calls( request_context, + **execution_filters, bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -383,6 +441,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get sessions sessions, sessions_total = await self.ap.monitoring_service.get_sessions( request_context, + **execution_filters, bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -395,6 +454,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get errors errors, errors_total = await self.ap.monitoring_service.get_errors( request_context, + **execution_filters, bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -406,6 +466,7 @@ class MonitoringRouterGroup(group.RouterGroup): # Get embedding calls embedding_calls, embedding_calls_total = await self.ap.monitoring_service.get_embedding_calls( request_context, + **execution_filters, start_time=start_time, end_time=end_time, limit=limit, @@ -417,6 +478,7 @@ class MonitoringRouterGroup(group.RouterGroup): 'traffic': await get_traffic_series( self.ap, request_context, + **execution_filters, bot_ids=bot_ids or None, pipeline_ids=pipeline_ids or None, start_time=start_time, @@ -639,7 +701,7 @@ class MonitoringRouterGroup(group.RouterGroup): 'platform', ] else: - return self.error(message=f'Invalid export type: {export_type}', code=400) + return self.fail(400, f'Invalid export type: {export_type}') # Generate CSV content with UTF-8 BOM for Excel compatibility import io @@ -684,6 +746,8 @@ class MonitoringRouterGroup(group.RouterGroup): stats = await self.ap.monitoring_service.get_feedback_stats( request_context, + execution_statuses=quart.request.args.getlist('status') or None, + mode=quart.request.args.get('mode', 'all'), bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, start_time=start_time, @@ -713,6 +777,8 @@ class MonitoringRouterGroup(group.RouterGroup): feedback_list, total = await self.ap.monitoring_service.get_feedback_list( request_context, + execution_statuses=quart.request.args.getlist('status') or None, + mode=quart.request.args.get('mode', 'all'), bot_ids=bot_ids if bot_ids else None, pipeline_ids=pipeline_ids if pipeline_ids else None, feedback_type=feedback_type, diff --git a/src/langbot/pkg/api/http/controller/groups/plugins.py b/src/langbot/pkg/api/http/controller/groups/plugins.py index 4bce79869..950a8954d 100644 --- a/src/langbot/pkg/api/http/controller/groups/plugins.py +++ b/src/langbot/pkg/api/http/controller/groups/plugins.py @@ -165,6 +165,16 @@ except Exception: pass +def _is_plugin_not_installed(exc: BaseException) -> bool: + """True when the connector reports a plugin absent from the resolved Workspace. + + The connector raises a plain ``ValueError`` for this case, so resource routes + answer 404 instead of surfacing a 500 the WebUI would have to swallow. + """ + + return 'is not installed in this workspace' in str(exc).lower() + + def _normalize_plugin_asset_path(filepath: str) -> str | None: filepath = filepath.replace('\\', '/') if filepath.startswith('/'): @@ -596,7 +606,15 @@ class PluginsRouterGroup(group.RouterGroup): request_context: RequestContext, ) -> quart.Response: await self._require_authenticated_plugin_runtime_context(request_context) - icon_data = await self.ap.plugin_connector.get_plugin_icon(author, plugin_name) + try: + icon_data = await self.ap.plugin_connector.get_plugin_icon(author, plugin_name) + except ValueError as exc: + # A plugin that is not installed for this Workspace is a missing + # resource, not a server fault: the WebUI asks for icons by plugin + # identity and must be able to fall back silently. + if _is_plugin_not_installed(exc): + return quart.Response('Icon not found', status=404) + raise icon_bytes = await asyncio.to_thread(base64.b64decode, icon_data['plugin_icon_base64']) return quart.Response(icon_bytes, mimetype=icon_data['mime_type']) diff --git a/src/langbot/pkg/api/http/controller/main.py b/src/langbot/pkg/api/http/controller/main.py index e19552c65..dfe8a685a 100644 --- a/src/langbot/pkg/api/http/controller/main.py +++ b/src/langbot/pkg/api/http/controller/main.py @@ -174,9 +174,21 @@ class HTTPController: frontend_path = paths.get_frontend_path() + async def frontend_index(): + from .social_preview import cloud_preview_html + + response = await quart.send_from_directory( + frontend_path, 'index.html', mimetype='text/html', conditional=False + ) + if quart.request.host.split(':', 1)[0].lower() == 'cloud.langbot.app': + response.set_data(cloud_preview_html(await response.get_data(as_text=True), quart.request.host)) + response.headers.pop('ETag', None) + response.headers.pop('Last-Modified', None) + return response + @self.quart_app.route('/') async def index(): - response = await quart.send_from_directory(frontend_path, 'index.html', mimetype='text/html') + response = await frontend_index() response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' response.headers['Pragma'] = 'no-cache' response.headers['Expires'] = '0' @@ -207,7 +219,7 @@ class HTTPController: return response # Fallback to index.html for SPA client-side routing - response = await quart.send_from_directory(frontend_path, 'index.html', mimetype='text/html') + response = await frontend_index() response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' response.headers['Pragma'] = 'no-cache' response.headers['Expires'] = '0' diff --git a/src/langbot/pkg/api/http/controller/social_preview.py b/src/langbot/pkg/api/http/controller/social_preview.py new file mode 100644 index 000000000..03e530ee0 --- /dev/null +++ b/src/langbot/pkg/api/http/controller/social_preview.py @@ -0,0 +1,20 @@ +"""Crawler-visible preview for the hosted Cloud frontend only.""" + +CLOUD_IMAGE = 'https://cloud.langbot.app/social/cloud-v3.png' + + +def cloud_preview_html(html: str, hostname: str) -> str: + if hostname.split(':', 1)[0].lower() != 'cloud.langbot.app': + return html + tags = ( + '' + '' + '' + f'' + '' + '' + '' + '' + f'' + ) + return html.replace('', tags + '', 1) diff --git a/src/langbot/pkg/api/http/service/agent.py b/src/langbot/pkg/api/http/service/agent.py index 260b1f2df..932d436fd 100644 --- a/src/langbot/pkg/api/http/service/agent.py +++ b/src/langbot/pkg/api/http/service/agent.py @@ -1,5 +1,7 @@ from __future__ import annotations +from contextlib import aclosing + import datetime import copy import fnmatch @@ -348,25 +350,45 @@ class AgentService: execution_events.append(visible_result) final_text = '' - async for output in self.ap.agent_run_orchestrator.run( - event, - binding, - adapter_context={ - '_query': execution_query, - '_execution_context': execution_context, - '_result_observer': observe_result, - }, + # A debug run has no platform event: the synthetic event envelope is the + # execution's inbound boundary, so the chain is opened here with the real + # event id and marked as a debug WebUI/API execution. The Runner inside + # then joins this chain instead of becoming a trace origin of its own. + from ....telemetry.execution import ingress as execution_ingress + + principal_type = str(getattr(getattr(context, 'principal', None), 'principal_type', '') or '') + origin = 'api' if principal_type == 'api_key' else 'webui' + with execution_ingress( + self.ap, + 'event_done', + execution_context, + event_id, + debug=True, + origin=origin, + synthetic_event=event_type, ): - output_text = self._provider_output_to_text(output) - if output_text: - final_text = output_text - output_items.append( - { - 'kind': output.__class__.__name__, - 'role': str(getattr(output, 'role', '') or ''), - 'text': output_text, - } - ) + async with aclosing( + self.ap.agent_run_orchestrator.run( + event, + binding, + adapter_context={ + '_query': execution_query, + '_execution_context': execution_context, + '_result_observer': observe_result, + }, + ) + ) as owned_stream: + async for output in owned_stream: + output_text = self._provider_output_to_text(output) + if output_text: + final_text = output_text + output_items.append( + { + 'kind': output.__class__.__name__, + 'role': str(getattr(output, 'role', '') or ''), + 'text': output_text, + } + ) return { 'event_id': event_id, diff --git a/src/langbot/pkg/api/http/service/assistant.py b/src/langbot/pkg/api/http/service/assistant.py index 74e7073ae..033b53750 100644 --- a/src/langbot/pkg/api/http/service/assistant.py +++ b/src/langbot/pkg/api/http/service/assistant.py @@ -1,6 +1,7 @@ """In-process management assistant using the existing model and resource services.""" import asyncio +import contextlib import json import uuid @@ -44,6 +45,187 @@ class AssistantService: self.ap = ap # ponytail: per-process admission; shared quotas if multiple workers need a global ceiling. self._slots = asyncio.Semaphore(4) + self._runs = {} + + async def list_conversations(self, context): + self._scope(context, '') + result = await self.ap.persistence_mgr.execute_async( + sa.select(Conversation) + .where( + Conversation.workspace_uuid == context.workspace_uuid, + Conversation.account_uuid == context.account_uuid, + ) + .order_by(Conversation.updated_at.desc(), Conversation.uuid) + .limit(100) + ) + return [ + dict( + uuid=row['uuid'], + status=row['status'], + revision=row['revision'], + title=next( + ( + m.get('content', '')[:80] + for m in row['messages'] + if m['role'] == 'user' and isinstance(m.get('content'), str) + ), + '', + ), + ) + for row in result.mappings() + ] + + async def progress(self, context, conversation_id): + conversation = await self.get(context, conversation_id) + run = self._runs.get(conversation_id) + if run: + return {**(run['snapshot'] or self.public_view(conversation)), 'progress': run['progress']} + return self.public_view(conversation) + + def start_run(self, context, conversation_id, body): + """Own execution independently of any HTTP subscriber; never block on a slow browser.""" + if conversation_id in self._runs: + raise AssistantError('stale_turn') + if len(self._runs) >= 4: + raise AssistantError('busy', 429) + queue = asyncio.Queue(maxsize=32) + run = { + 'queue': queue, + 'snapshot': None, + 'progress': {}, + 'stopping': False, + 'revision': body.revision + 1, + 'started': asyncio.Event(), + } + self._runs[conversation_id] = run + + async def emit(frame): + if frame['kind'] == 'snapshot': + run['snapshot'] = frame['data'] + run['progress'] = {} + elif frame['kind'] == 'text': + run['progress']['text'] = frame['data']['text'] + elif frame['kind'] == 'phase': + run['progress'].update(frame['data']) + if queue.full(): + queue.get_nowait() + queue.put_nowait(frame) + + async def execute(): + run['started'].set() + tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None) + scope = tenant_scope(context.workspace_uuid) if callable(tenant_scope) else contextlib.nullcontext() + try: + async with scope: + try: + conversation = await self.turn( + context, + conversation_id, + body.revision, + body.text, + body.approved, + str(body.model_uuid) if body.model_uuid else None, + on_event=emit, + ) + except asyncio.CancelledError: + conversation = await self.get(context, conversation_id) + # Preserve interrupted tool outcomes; stopping cannot undo an external write. + if run['stopping'] and ( + conversation['error'] == 'result_unknown' + or (conversation['status'] == 'running' and conversation['revision'] == run['revision']) + ): + await self._save(context, conversation, 'failed', 'stopped') + await emit({'kind': 'completed', 'data': self.public_view(conversation)}) + except AssistantError as exc: + await emit({'kind': 'error', 'code': exc.code}) + except Exception: + self.ap.logger.exception('Assistant background run failed') + await emit({'kind': 'error', 'code': 'turn_failed'}) + finally: + self._runs.pop(conversation_id, None) + + run['task'] = asyncio.create_task(execute()) + return queue + + async def stop(self, context, conversation_id, revision): + require_permission(context, Permission.RUNTIME_OPERATE) + conversation = await self.get(context, conversation_id) + run = self._runs.get(conversation_id) + if run and run['revision'] != revision: + raise AssistantError('stale_turn') + if not run: + if conversation['revision'] != revision: + raise AssistantError('stale_turn') + return self.public_view(conversation) + await run['started'].wait() + if not run['stopping']: + run['stopping'] = True + run['task'].cancel() + await asyncio.shield(run['task']) + return self.public_view(await self.get(context, conversation_id)) + + async def recommended_model(self, context): + require_permission(context, Permission.RUNTIME_OPERATE) + try: + recommended = await self.ap.space_service.get_recommended_assistant_model(context) + model = await self.ap.model_mgr.get_model_by_uuid( + ExecutionContext.from_request(context), recommended['uuid'] + ) + if 'func_call' not in (model.model_entity.abilities or []): + raise ValueError('Recommended model does not support tools') + return {'uuid': recommended['uuid'], 'name': model.model_entity.name} + except Exception as exc: + raise AssistantError('model_unavailable', 503) from exc + + async def _invoke(self, model, kwargs, on_event): + if on_event is None: + return await model.provider.invoke_llm(**kwargs) + message = {'role': 'assistant', 'content': ''} + calls = {} + finished = False + async for chunk in model.provider.invoke_llm_stream(**kwargs): + finished = finished or chunk.is_final + content = chunk.content or '' + if isinstance(content, list): + content = ''.join(item.text or '' for item in content if item.type == 'text') + if chunk.all_content is not None: + message['content'] = chunk.all_content + else: + message['content'] += content + for field in ('resp_message_id', 'provider_specific_fields'): + value = getattr(chunk, field, None) + if value is not None: + if field == 'provider_specific_fields': + message[field] = {**message.get(field, {}), **value} + else: + message[field] = value + for call in chunk.tool_calls or []: + value = call.model_dump(mode='json') + if call.id in calls: + saved = calls[call.id] + saved['function']['arguments'] += call.function.arguments or '' + if call.function.name: + saved['function']['name'] = call.function.name + if value.get('provider_specific_fields'): + saved['provider_specific_fields'] = { + **(saved.get('provider_specific_fields') or {}), + **value['provider_specific_fields'], + } + else: + calls[call.id] = value + if len(calls) > 8 or len(json.dumps(message, ensure_ascii=False)) + len(json.dumps(calls)) > 64000: + raise AssistantError('response_too_large') + if content or chunk.all_content is not None: + await on_event({'kind': 'text', 'data': {'text': message['content']}}) + if not finished: + raise AssistantError('stream_interrupted') + if calls: + message['tool_calls'] = list(calls.values()) + return Message.model_validate(message) + + async def _snapshot(self, conversation, on_event): + if on_event is not None: + await on_event({'kind': 'snapshot', 'data': self.public_view(conversation)}) @staticmethod def _scope(context, conversation_id): @@ -150,7 +332,7 @@ class AssistantService: raise AssistantError('stale_turn') conversation.update(status=status, error=error) - async def turn(self, context, conversation_id, revision, text=None, approved=None, model_uuid=None): + async def turn(self, context, conversation_id, revision, text=None, approved=None, model_uuid=None, on_event=None): require_permission(context, Permission.RUNTIME_OPERATE) if model_uuid is not None and text is None: raise AssistantError('invalid_input', 400) @@ -201,7 +383,7 @@ class AssistantService: await self._save(context, conversation, 'running') try: if not conversation['model_uuid']: - recommended = await self.ap.space_service.get_recommended_chat_model(context) + recommended = await self.recommended_model(context) conversation['model_uuid'] = recommended['uuid'] execution = ExecutionContext.from_request(context) model = selected_model or await self.ap.model_mgr.get_model_by_uuid( @@ -215,22 +397,30 @@ class AssistantService: return conversation if approved is not None: calls = self._calls(conversation['messages'][-1]) - await self._execute(context, conversation, calls, approved) - for _ in range(8): - response = await model.provider.invoke_llm( - query=None, - model=model, - messages=[Message(role='system', content=SYSTEM_PROMPT)] - + [Message.model_validate(message) for message in conversation['messages']], - funcs=tool_definitions(context), - extra_args=model.model_entity.extra_args or {}, - remove_think=True, - execution_context=execution, + await self._execute(context, conversation, calls, approved, on_event) + for round_index in range(8): + await self._snapshot(conversation, on_event) + if on_event is not None: + await on_event({'kind': 'phase', 'data': {'phase': 'thinking', 'round': round_index + 1}}) + response = await self._invoke( + model, + dict( + query=None, + model=model, + messages=[Message(role='system', content=SYSTEM_PROMPT)] + + [Message.model_validate(message) for message in conversation['messages']], + funcs=tool_definitions(context), + extra_args=model.model_entity.extra_args or {}, + remove_think=True, + execution_context=execution, + ), + on_event, ) message = response.model_dump(mode='json') if len(json.dumps(message, ensure_ascii=False)) > 64000: raise AssistantError('response_too_large') conversation['messages'].append(message) + await self._snapshot(conversation, on_event) calls = self._calls(message) if not calls: await self._save(context, conversation, 'ready') @@ -249,9 +439,12 @@ class AssistantService: if any(TOOLS[call['function']['name']][2] for call in calls): await self._save(context, conversation, 'approval') return conversation - await self._execute(context, conversation, calls, True) + await self._execute(context, conversation, calls, True, on_event) await self._save(context, conversation, 'failed', 'round_limit') except asyncio.CancelledError: + partial = self._runs.get(conversation_id, {}).get('progress', {}).get('text') + if partial: + conversation['messages'].append(Message(role='assistant', content=partial).model_dump(mode='json')) await asyncio.shield(self._save(context, conversation, 'failed', 'result_unknown')) raise except TimeoutError: @@ -280,7 +473,7 @@ class AssistantService: ).model_dump(mode='json') ) - async def _execute(self, context, conversation, calls, approved): + async def _execute(self, context, conversation, calls, approved, on_event=None): # Validate the complete saved batch before any write, including after approval. if approved: for call in calls: @@ -293,6 +486,19 @@ class AssistantService: else: # Persist before effects. A crash leaves a running/unknown operation, never a replayable approval. await self._save(context, conversation, 'running') + if on_event is not None: + await on_event( + { + 'kind': 'phase', + 'data': { + 'phase': 'tool', + 'tool': { + 'name': func['name'], + 'arguments': redact_secrets(json.loads(func['arguments'] or '{}')), + }, + }, + } + ) try: result = await execute_tool(self.ap, context, func['name'], json.loads(func['arguments'] or '{}')) except Exception as exc: @@ -308,3 +514,4 @@ class AssistantService: raise AssistantError('tool_failed') self._append_result(conversation, call, result) await self._save(context, conversation, 'running') + await self._snapshot(conversation, on_event) diff --git a/src/langbot/pkg/api/http/service/monitoring.py b/src/langbot/pkg/api/http/service/monitoring.py index 0bae983a6..653be4c93 100644 --- a/src/langbot/pkg/api/http/service/monitoring.py +++ b/src/langbot/pkg/api/http/service/monitoring.py @@ -10,10 +10,15 @@ from sqlalchemy.dialects import postgresql as postgresql_dialect from sqlalchemy.dialects import sqlite as sqlite_dialect from ....core import app +from ....entity.persistence import agent as persistence_agent from ....entity.persistence import monitoring as persistence_monitoring +from ....entity.persistence import agent_run as persistence_agent_run +from ....entity.persistence import agent_interaction as persistence_agent_interaction +from ....entity.persistence.event_log import EventLog from ..authz import WorkspaceRequiredError from ..context import ExecutionContext from .tenant import TenantContext, require_workspace_uuid +from .monitoring_execution_details import ExecutionDetailsMixin _DEFAULT_MONITORING_PAGE_ROWS = 1000 @@ -29,6 +34,76 @@ _HARD_MAX_MONITORING_OFFSET = 10000000 _DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4 _HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100 +# Canonical execution status groups shared by the agent ledger and pipeline query +# records so both object types can be summarised with one vocabulary. +_AGENT_STATUS_GROUP = { + 'completed': 'completed', + 'failed': 'failed', + 'timeout': 'failed', + 'cancelled': 'cancelled', + 'created': 'queued', + 'queued': 'queued', + 'claimed': 'running', + 'running': 'running', +} +_PIPELINE_STATUS_GROUP = { + 'success': 'completed', + 'error': 'failed', + 'pending': 'running', + 'discarded': 'ignored', + 'not_matched': 'ignored', +} + + +def _status_group(status: str, source: str) -> str: + table = _AGENT_STATUS_GROUP if source == 'agent' else _PIPELINE_STATUS_GROUP + return table.get(status, status) + + +def _epoch_ms(value: datetime.datetime | None) -> int | None: + """Epoch milliseconds for a DB timestamp, treating naive values as UTC.""" + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + return int(round(value.timestamp() * 1000)) + + +def _percentile_ms(values: list[float], fraction: float) -> int | None: + """Nearest-rank percentile (ms) over a non-empty, prefix-sorted list.""" + if not values: + return None + rank = max(1, min(len(values), int(round(fraction * len(values))))) + return int(round(values[rank - 1])) + + +def _collect_text(node, out: list[str], depth: int = 0) -> None: + """Best-effort extraction of human text from a serialised message chain.""" + if depth > 6: + return + if isinstance(node, dict): + for key, value in node.items(): + if key == 'text' and isinstance(value, str): + out.append(value) + elif isinstance(value, (dict, list)): + _collect_text(value, out, depth + 1) + elif isinstance(node, list): + for item in node: + _collect_text(item, out, depth + 1) + + +def _message_preview(raw: str | None, limit: int = 160) -> str: + if not raw: + return '' + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + return raw[:limit] + parts: list[str] = [] + _collect_text(parsed, parts) + text = ' '.join(part.strip() for part in parts if part.strip()) + return (text or raw)[:limit] + def _normalize_user_id(value: str | int | None) -> str | None: """Convert numeric platform IDs before binding a VARCHAR with asyncpg. @@ -58,7 +133,7 @@ def _workspace_transaction(method): return wrapped -class MonitoringService: +class MonitoringService(ExecutionDetailsMixin): """Monitoring service""" ap: app.Application @@ -462,9 +537,17 @@ class MonitoringService: runner_name: str | None = None, variables: str | None = None, role: str = 'user', + event_id: str | None = None, + run_id: str | None = None, + parent_message_id: str | None = None, ) -> str: """Record a message""" workspace_uuid = self._require_write_context(context) + from ....telemetry import trace + + trace_state = trace.current() + event_id = event_id or (trace_state.event_id if trace_state else None) + run_id = run_id or trace.current_run() or None message_id = str(uuid.uuid4()) message_content = self._sanitize_message_content(message_content) message_data = { @@ -485,6 +568,9 @@ class MonitoringService: 'runner_name': runner_name, 'variables': variables, 'role': role, + 'event_id': event_id, + 'run_id': run_id, + 'parent_message_id': parent_message_id, } await self.ap.persistence_mgr.execute_async( @@ -806,6 +892,8 @@ class MonitoringService: pipeline_ids: list[str] | None = None, start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> dict: """Get overview metrics""" workspace_uuid = require_workspace_uuid(context) @@ -815,15 +903,47 @@ class MonitoringService: embedding_conditions = [persistence_monitoring.MonitoringEmbeddingCall.workspace_uuid == workspace_uuid] session_conditions = [persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid] + message_conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringMessage, workspace_uuid, execution_statuses, mode + ) + ) + llm_conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, execution_statuses, mode + ) + ) + session_conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringSession, workspace_uuid, execution_statuses, mode + ) + ) + embedding_conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringEmbeddingCall, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: message_conditions.append(persistence_monitoring.MonitoringMessage.bot_id.in_(bot_ids)) llm_conditions.append(persistence_monitoring.MonitoringLLMCall.bot_id.in_(bot_ids)) session_conditions.append(persistence_monitoring.MonitoringSession.bot_id.in_(bot_ids)) if pipeline_ids: - message_conditions.append(persistence_monitoring.MonitoringMessage.pipeline_id.in_(pipeline_ids)) - llm_conditions.append(persistence_monitoring.MonitoringLLMCall.pipeline_id.in_(pipeline_ids)) - session_conditions.append(persistence_monitoring.MonitoringSession.pipeline_id.in_(pipeline_ids)) + message_conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringMessage, workspace_uuid, pipeline_ids + ) + ) + llm_conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, pipeline_ids + ) + ) + session_conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringSession, workspace_uuid, pipeline_ids + ) + ) if start_time: message_conditions.append(persistence_monitoring.MonitoringMessage.timestamp >= start_time) @@ -885,6 +1005,10 @@ class MonitoringService: active_sessions_result = await self.ap.persistence_mgr.execute_async(active_session_query) active_sessions = active_sessions_result.scalar() or 0 + # Execution-first KPI values (runs + pipeline queries, latency, token + # coverage) are served by `get_executions`, which applies the execution + # view filters. This endpoint stays the message/traffic rollup so the + # dashboard does not compute the same rollup twice per page load. return { 'total_messages': total_messages, 'llm_calls': llm_calls, @@ -894,6 +1018,627 @@ class MonitoringService: 'active_sessions': active_sessions, } + # ========== Execution view ========== + + @staticmethod + def _debug_run_condition(model): + return sqlalchemy.or_( + model.binding_id.like('debug:%'), + model.conversation_id.like('debug:%'), + model.bot_id == 'websocket-proxy-bot', + ) + + @staticmethod + def _debug_pipeline_condition(model): + # Dashboard/API pipeline debug uses this reserved proxy identity. + # Real Web Page bots have their own bot UUID, even on the same adapter. + return sqlalchemy.func.coalesce(model.bot_id == 'websocket-proxy-bot', False) + + def _monitoring_execution_condition(self, model, workspace_uuid, statuses=None, mode='all'): + """Filter telemetry by its owning execution, not by the call's outcome.""" + if not statuses and mode == 'all': + return sqlalchemy.true() + Run = persistence_agent_run.AgentRun + Message = persistence_monitoring.MonitoringMessage + roots = sqlalchemy.orm.aliased(Message) + run_conditions = [Run.workspace_id == workspace_uuid] + if statuses: + run_conditions.append(Run.status.in_(self._raw_agent_statuses(statuses))) + debug = self._debug_run_condition(Run) + if mode != 'all': + run_conditions.append(debug if mode == 'debug' else sqlalchemy.not_(sqlalchemy.func.coalesce(debug, False))) + runs = sqlalchemy.select(Run.run_id).where(*run_conditions) + root_conditions = [roots.workspace_uuid == workspace_uuid, roots.role != 'assistant'] + if statuses: + root_conditions.append(roots.status.in_(self._raw_pipeline_statuses(statuses))) + if mode != 'all': + root_debug = self._debug_pipeline_condition(roots) + root_conditions.append(root_debug if mode == 'debug' else sqlalchemy.not_(root_debug)) + root_ids = sqlalchemy.select(roots.id).where(*root_conditions) + linked = sqlalchemy.or_( + Message.id.in_(root_ids), Message.parent_message_id.in_(root_ids), Message.run_id.in_(runs) + ) + message_ids = sqlalchemy.select(Message.id).where(Message.workspace_uuid == workspace_uuid, linked) + if model is Message: + return linked + if hasattr(model, 'message_id'): + return sqlalchemy.or_(model.message_id.in_(runs), model.message_id.in_(message_ids)) + if model is persistence_monitoring.MonitoringSession: + return sqlalchemy.or_( + model.session_id.in_(sqlalchemy.select(roots.session_id).where(*root_conditions)), + model.session_id.in_(sqlalchemy.select(Run.conversation_id).where(*run_conditions)), + ) + return sqlalchemy.false() + + def _processor_runs(self, workspace_uuid, processor_ids): + Run = persistence_agent_run.AgentRun + return sqlalchemy.select(Run.run_id).where( + Run.workspace_id == workspace_uuid, + sqlalchemy.func.coalesce(self._execution_json_text(Run.metadata_json, 'processor_id'), Run.agent_id).in_( + processor_ids + ), + ) + + def _monitoring_processor_condition(self, model, workspace_uuid, processor_ids): + direct = model.pipeline_id.in_(processor_ids) + if hasattr(model, 'message_id'): + Message = persistence_monitoring.MonitoringMessage + return sqlalchemy.or_( + direct, + model.message_id.in_(self._processor_runs(workspace_uuid, processor_ids)), + model.message_id.in_( + sqlalchemy.select(Message.id).where( + Message.workspace_uuid == workspace_uuid, Message.pipeline_id.in_(processor_ids) + ) + ), + ) + return direct + + def _agent_run_conditions( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + agent_ids: list[str] | None = None, + processor_ids: list[str] | None = None, + statuses: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + debug: bool | None = None, + ) -> list: + AgentRun = persistence_agent_run.AgentRun + conditions = [ + AgentRun.workspace_id == workspace_uuid, + ~sqlalchemy.exists( + sqlalchemy.select(persistence_monitoring.MonitoringMessage.id).where( + persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid, + persistence_monitoring.MonitoringMessage.run_id == AgentRun.run_id, + persistence_monitoring.MonitoringMessage.role != 'assistant', + ) + ), + ] + if bot_ids: + conditions.append(AgentRun.bot_id.in_(bot_ids)) + if agent_ids: + conditions.append(AgentRun.agent_id.in_(agent_ids)) + if processor_ids: + conditions.append(AgentRun.run_id.in_(self._processor_runs(workspace_uuid, processor_ids))) + if statuses is not None: + conditions.append(AgentRun.status.in_(statuses) if statuses else sqlalchemy.false()) + if start_time: + conditions.append(AgentRun.created_at >= start_time) + if end_time: + conditions.append(AgentRun.created_at <= end_time) + if debug is not None: + debug_condition = self._debug_run_condition(AgentRun) + conditions.append( + debug_condition if debug else sqlalchemy.not_(sqlalchemy.func.coalesce(debug_condition, False)) + ) + return conditions + + def _pipeline_execution_conditions( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + statuses: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + debug: bool | None = None, + ) -> list: + MonitoringMessage = persistence_monitoring.MonitoringMessage + conditions = [ + MonitoringMessage.workspace_uuid == workspace_uuid, + # Assistant replies are outputs of a query, not an execution of their own. + sqlalchemy.or_(MonitoringMessage.role.is_(None), MonitoringMessage.role != 'assistant'), + ] + if bot_ids: + conditions.append(MonitoringMessage.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(self._monitoring_processor_condition(MonitoringMessage, workspace_uuid, pipeline_ids)) + if statuses is not None: + conditions.append(MonitoringMessage.status.in_(statuses) if statuses else sqlalchemy.false()) + if start_time: + conditions.append(MonitoringMessage.timestamp >= start_time) + if end_time: + conditions.append(MonitoringMessage.timestamp <= end_time) + if debug is not None: + debug_condition = self._debug_pipeline_condition(MonitoringMessage) + conditions.append(debug_condition if debug else sqlalchemy.not_(debug_condition)) + return conditions + + @staticmethod + def _raw_agent_statuses(groups: list[str] | None) -> list[str] | None: + """Raw agent statuses for a group filter; empty means "no group applies".""" + if not groups: + return None + raw: list[str] = [] + for group in groups: + raw.extend(status for status, mapped in _AGENT_STATUS_GROUP.items() if mapped == group) + return raw + + @staticmethod + def _raw_pipeline_statuses(groups: list[str] | None) -> list[str] | None: + """Raw pipeline statuses for a group filter; empty means "no group applies".""" + if not groups: + return None + raw: list[str] = [] + for group in groups: + raw.extend(status for status, mapped in _PIPELINE_STATUS_GROUP.items() if mapped == group) + return raw + + @staticmethod + def _model_rows(result) -> list: + """Rows of one ``select(Model)`` executed through ``execute_async``. + + The Core keeps the connection result contract, where ``select(Model)`` + yields a *flat* row of the model's columns (``row.uuid``, ``row.status``) + rather than a one-item ORM row. ``.scalars()`` would therefore return the + first column - the id - instead of the row, so the rows are taken as-is. + """ + return list(result.all()) + + @staticmethod + def _deserialize_json(raw: str | None) -> dict: + if not raw: + return {} + try: + value = json.loads(raw) + except (TypeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + async def _agent_names(self, workspace_uuid: str, rows: list) -> dict[str, str]: + """The processor name of every Agent run in one page, in one query. + + A run stores the Agent it executed as a uuid; the list has to name the + processor, so the names are resolved for the whole page at once. + """ + agent_ids = {self._deserialize_json(row.metadata_json).get('processor_id') or row.agent_id for row in rows} - { + None, + '', + } + if not agent_ids: + return {} + Agent = persistence_agent.Agent + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(Agent.uuid, Agent.name).where( + sqlalchemy.and_( + Agent.workspace_uuid == workspace_uuid, + Agent.uuid.in_(agent_ids), + ) + ) + ) + return {row.uuid: row.name for row in self._model_rows(result)} + + def _serialize_agent_execution(self, row, agent_names: dict[str, str] | None = None) -> dict: + metadata = self._deserialize_json(row.metadata_json) + duration_ms = None + if row.started_at and row.finished_at: + duration_ms = int(round((row.finished_at - row.started_at).total_seconds() * 1000)) + created_at_ms = _epoch_ms(row.created_at) + started_at_ms = _epoch_ms(row.started_at) + finished_at_ms = _epoch_ms(row.finished_at) + is_debug = ( + str(row.binding_id or '').startswith('debug:') + or str(row.conversation_id or '').startswith('debug:') + or row.bot_id == 'websocket-proxy-bot' + ) + return { + 'source': 'agent', + 'id': row.run_id, + 'event_id': row.event_id, + 'status': row.status, + 'status_group': _status_group(row.status, 'agent'), + 'title': metadata.get('event_type') or row.queue_name or row.runner_id, + 'target_kind': metadata.get('processor_type') or ('agent' if row.agent_id else 'processor'), + 'target_id': metadata.get('processor_id') or row.agent_id or row.binding_id, + 'target_name': metadata.get('agent_name') + or metadata.get('target_name') + or (agent_names or {}).get(metadata.get('processor_id') or row.agent_id or ''), + 'bot_id': row.bot_id, + 'bot_name': metadata.get('bot_name'), + 'pipeline_id': None, + 'pipeline_name': None, + 'runner_id': row.runner_id, + 'conversation_id': row.conversation_id, + 'session_id': row.conversation_id, + 'created_at_ms': created_at_ms, + 'started_at_ms': started_at_ms, + 'finished_at_ms': finished_at_ms, + 'duration_ms': duration_ms, + 'usage': self._deserialize_json(row.usage_json) or None, + 'cost': self._deserialize_json(row.cost_json) or None, + 'queue_name': row.queue_name, + 'debug': is_debug, + 'has_error': row.status in {'failed', 'timeout'}, + 'status_reason': row.status_reason, + 'input_preview': _message_preview(json.dumps(metadata.get('input') or {}, ensure_ascii=False), 200), + } + + def _serialize_pipeline_execution(self, row) -> dict: + created_at_ms = _epoch_ms(row.timestamp) + return { + 'source': 'pipeline', + 'id': row.id, + 'event_id': row.event_id, + 'status': row.status, + 'status_group': _status_group(row.status, 'pipeline'), + 'title': _message_preview(row.message_content), + 'target_kind': 'pipeline', + 'target_id': row.pipeline_id, + 'target_name': row.pipeline_name, + 'bot_id': row.bot_id, + 'bot_name': row.bot_name, + 'pipeline_id': row.pipeline_id, + 'pipeline_name': row.pipeline_name, + 'runner_id': row.runner_name, + 'conversation_id': None, + 'session_id': row.session_id, + 'created_at_ms': created_at_ms, + 'started_at_ms': None, + 'finished_at_ms': None, + 'duration_ms': None, + 'usage': None, + 'cost': None, + 'queue_name': None, + 'debug': row.bot_id == 'websocket-proxy-bot', + 'has_error': row.status == 'error', + 'input_preview': _message_preview(row.message_content), + 'platform': row.platform, + 'user_id': row.user_id, + 'user_name': row.user_name, + } + + async def _get_token_coverage( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + execution_statuses: list[str] | None = None, + mode: str = 'all', + ) -> dict: + LLMCall = persistence_monitoring.MonitoringLLMCall + conditions = [LLMCall.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, execution_statuses, mode + ) + ) + if bot_ids: + conditions.append(LLMCall.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(self._monitoring_processor_condition(LLMCall, workspace_uuid, pipeline_ids)) + if start_time: + conditions.append(LLMCall.timestamp >= start_time) + if end_time: + conditions.append(LLMCall.timestamp <= end_time) + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select( + sqlalchemy.func.count(LLMCall.id), + sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0), + sqlalchemy.func.sum(sqlalchemy.case((LLMCall.total_tokens > 0, 1), else_=0)), + ).where(sqlalchemy.and_(*conditions)) + ) + row = result.first() + calls, total, calls_with_usage = row if row else (0, 0, 0) + return { + 'total_tokens': int(total or 0), + 'calls': calls or 0, + 'calls_with_usage': calls_with_usage or 0, + } + + async def _get_execution_summary( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + statuses: list[str] | None = None, + mode: str = 'all', + ) -> dict: + AgentRun = persistence_agent_run.AgentRun + run_conditions = self._agent_run_conditions( + workspace_uuid, + bot_ids=bot_ids, + processor_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + statuses=self._raw_agent_statuses(statuses), + debug=True if mode == 'debug' else False if mode == 'real' else None, + ) + run_status_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun.status, sqlalchemy.func.count(AgentRun.id)) + .where(sqlalchemy.and_(*run_conditions)) + .group_by(AgentRun.status) + ) + agent_counts = {row[0]: row[1] for row in run_status_result.all()} + agent_total = sum(agent_counts.values()) + + debug_count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where( + sqlalchemy.and_( + *run_conditions, + self._debug_run_condition(AgentRun), + ) + ) + ) + debug_count = debug_count_result.scalar() or 0 + + # Percentiles are computed in Python, so the sample must stay bounded: + # take the most recent completed runs of the window and label the count + # so the UI can state how many runs the numbers rest on. + timing_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun.started_at, AgentRun.finished_at) + .where( + sqlalchemy.and_( + *run_conditions, + AgentRun.status == 'completed', + AgentRun.started_at.is_not(None), + AgentRun.finished_at.is_not(None), + ) + ) + .order_by(AgentRun.created_at.desc()) + .limit(self._detail_limit()) + ) + durations = sorted((row[1] - row[0]).total_seconds() * 1000 for row in timing_result.all() if row[0] and row[1]) + + pipeline_conditions = self._pipeline_execution_conditions( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + statuses=self._raw_pipeline_statuses(statuses), + start_time=start_time, + end_time=end_time, + debug=True if mode == 'debug' else False if mode == 'real' else None, + ) + pipeline_status_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select( + persistence_monitoring.MonitoringMessage.status, + sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id), + ) + .where(sqlalchemy.and_(*pipeline_conditions)) + .group_by(persistence_monitoring.MonitoringMessage.status) + ) + pipeline_counts = {row[0]: row[1] for row in pipeline_status_result.all()} + pipeline_total = sum(pipeline_counts.values()) + pipeline_debug_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id)).where( + *pipeline_conditions, self._debug_pipeline_condition(persistence_monitoring.MonitoringMessage) + ) + ) + debug_count += pipeline_debug_result.scalar() or 0 + + # Pending interactions are gated by the same bot/window scope as the runs + # so "waiting input" cannot drift from the rest of the card. + waiting_conditions = [ + persistence_agent_interaction.AgentInteraction.workspace_id == workspace_uuid, + persistence_agent_interaction.AgentInteraction.status == 'pending', + ] + if bot_ids: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.bot_id.in_(bot_ids)) + if pipeline_ids: + waiting_conditions.append( + persistence_agent_interaction.AgentInteraction.run_id.in_( + self._processor_runs(workspace_uuid, pipeline_ids) + ) + ) + if start_time: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at >= start_time) + if end_time: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at <= end_time) + waiting_conditions.append( + persistence_agent_interaction.AgentInteraction.run_id.in_( + sqlalchemy.select(AgentRun.run_id).where(*run_conditions) + ) + ) + waiting_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(persistence_agent_interaction.AgentInteraction.id)).where( + sqlalchemy.and_(*waiting_conditions) + ) + ) + waiting = waiting_result.scalar() or 0 + + by_status: dict[str, int] = {} + for status, count in agent_counts.items(): + group = _status_group(status, 'agent') + by_status[group] = by_status.get(group, 0) + count + for status, count in pipeline_counts.items(): + group = _status_group(status, 'pipeline') + by_status[group] = by_status.get(group, 0) + count + + completed = by_status.get('completed', 0) + failed = by_status.get('failed', 0) + denominator = completed + failed + total = agent_total + pipeline_total + return { + 'total': total, + 'by_source': {'agent': agent_total, 'pipeline': pipeline_total}, + 'by_status': by_status, + 'completed': completed, + 'failed': failed, + 'running': by_status.get('running', 0), + 'queued': by_status.get('queued', 0), + 'cancelled': by_status.get('cancelled', 0), + 'ignored': by_status.get('ignored', 0), + 'waiting': waiting, + 'success_rate': round(completed / denominator * 100, 2) if denominator > 0 else None, + 'denominator': denominator, + 'p50_duration_ms': _percentile_ms(durations, 0.5), + 'p95_duration_ms': _percentile_ms(durations, 0.95), + 'duration_sample': len(durations), + 'debug': debug_count, + 'real': total - debug_count, + } + + async def get_executions( + self, + context: TenantContext, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + agent_ids: list[str] | None = None, + statuses: list[str] | None = None, + source: str = 'all', + mode: str = 'all', + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + limit: int = 50, + offset: int = 0, + ) -> dict: + """List executions from both object types (agent runs and pipeline queries).""" + workspace_uuid = require_workspace_uuid(context) + limit, offset = self.normalize_page_window(limit, offset) + if source not in {'all', 'agent', 'pipeline', 'event'}: + source = 'all' + if mode not in {'all', 'real', 'debug'}: + mode = 'all' + debug = None if mode == 'all' else mode == 'debug' + branches = [] + AgentRun = persistence_agent_run.AgentRun + Message = persistence_monitoring.MonitoringMessage + if source in {'all', 'agent'}: + conditions = self._agent_run_conditions( + workspace_uuid, + bot_ids=bot_ids, + agent_ids=agent_ids, + processor_ids=pipeline_ids, + statuses=self._raw_agent_statuses(statuses), + start_time=start_time, + end_time=end_time, + debug=debug, + ) + branches.append( + sqlalchemy.select( + sqlalchemy.literal('agent').label('source'), + AgentRun.run_id.label('id'), + AgentRun.created_at.label('time'), + ).where(*conditions) + ) + if source in {'all', 'pipeline'} and (not agent_ids or pipeline_ids): + conditions = self._pipeline_execution_conditions( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + statuses=self._raw_pipeline_statuses(statuses), + start_time=start_time, + end_time=end_time, + debug=debug, + ) + branches.append( + sqlalchemy.select( + sqlalchemy.literal('pipeline').label('source'), + Message.id.label('id'), + Message.timestamp.label('time'), + ).where(*conditions) + ) + if source in {'all', 'event'} and mode != 'debug' and not pipeline_ids and not agent_ids: + conditions = self._unhandled_event_conditions(workspace_uuid, bot_ids, start_time, end_time) + if statuses: + conditions.append(self._execution_json_text(EventLog.metadata_json, 'status').in_(statuses)) + branches.append( + sqlalchemy.select( + sqlalchemy.literal('event').label('source'), + EventLog.event_id.label('id'), + EventLog.created_at.label('time'), + ).where(*conditions) + ) + total = 0 + page = [] + if branches: + combined = sqlalchemy.union_all(*branches).subquery() + count = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count()).select_from(combined) + ) + total = count.scalar() or 0 + refs = ( + await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(combined) + .order_by(combined.c.time.desc(), combined.c.source, combined.c.id) + .offset(offset) + .limit(limit) + ) + ).all() + by_key = {} + for kind, model, identity_column in [ + ('agent', AgentRun, AgentRun.run_id), + ('pipeline', Message, Message.id), + ('event', EventLog, EventLog.event_id), + ]: + ids = [r.id for r in refs if r.source == kind] + if not ids: + continue + workspace_column = model.workspace_uuid if kind == 'pipeline' else model.workspace_id + records = await self._execution_rows( + workspace_uuid, + sqlalchemy.select(model).where(workspace_column == workspace_uuid, identity_column.in_(ids)), + ) + names = await self._agent_names(workspace_uuid, records) if kind == 'agent' else {} + for record in records: + item = ( + self._serialize_agent_execution(record, names) + if kind == 'agent' + else self._serialize_pipeline_execution(record) + if kind == 'pipeline' + else self._serialize_event_execution(record) + ) + by_key[(kind, item['id'])] = item + page = [by_key[(ref.source, ref.id)] for ref in refs if (ref.source, ref.id) in by_key] + await self.enrich_execution_rows(workspace_uuid, page) + return { + 'items': page, + 'total': total, + 'limit': limit, + 'offset': offset, + 'has_more': total > offset + len(page), + 'summary': { + 'executions': await self._get_execution_summary( + workspace_uuid, + statuses=statuses, + mode=mode, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + ), + 'tokens': await self._get_token_coverage( + workspace_uuid, + execution_statuses=statuses, + mode=mode, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + ), + }, + } + async def get_token_statistics( self, context: TenantContext, @@ -902,6 +1647,8 @@ class MonitoringService: start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, bucket: str = 'hour', + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> dict: """Get detailed token usage statistics for production observability. @@ -920,10 +1667,15 @@ class MonitoringService: bucket = 'hour' conditions = [LLMCall.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(LLMCall.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(LLMCall.pipeline_id.in_(pipeline_ids)) + conditions.append(self._monitoring_processor_condition(LLMCall, workspace_uuid, pipeline_ids)) if start_time: conditions.append(LLMCall.timestamp >= start_time) if end_time: @@ -1104,16 +1856,27 @@ class MonitoringService: end_time: datetime.datetime | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get messages with filters""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringMessage, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringMessage.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringMessage.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringMessage, workspace_uuid, pipeline_ids + ) + ) if session_ids: conditions.append(persistence_monitoring.MonitoringMessage.session_id.in_(session_ids)) if start_time: @@ -1159,16 +1922,27 @@ class MonitoringService: end_time: datetime.datetime | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get LLM calls with filters""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringLLMCall.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringLLMCall.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringLLMCall.timestamp >= start_time) if end_time: @@ -1214,16 +1988,27 @@ class MonitoringService: end_time: datetime.datetime | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get tool calls with filters""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringToolCall.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringToolCall, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringToolCall.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringToolCall.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringToolCall, workspace_uuid, pipeline_ids + ) + ) if session_ids: conditions.append(persistence_monitoring.MonitoringToolCall.session_id.in_(session_ids)) if start_time: @@ -1267,12 +2052,19 @@ class MonitoringService: knowledge_base_id: str | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get embedding calls with filters""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringEmbeddingCall.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringEmbeddingCall, workspace_uuid, execution_statuses, mode + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringEmbeddingCall.timestamp >= start_time) if end_time: @@ -1321,16 +2113,27 @@ class MonitoringService: is_active: bool | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get sessions with filters""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringSession, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringSession.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringSession.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringSession, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringSession.last_activity >= start_time) if end_time: @@ -1385,16 +2188,27 @@ class MonitoringService: end_time: datetime.datetime | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get errors with filters""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringError.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringError, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringError.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringError.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringError, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringError.timestamp >= start_time) if end_time: @@ -1860,7 +2674,11 @@ class MonitoringService: if bot_ids: conditions.append(persistence_monitoring.MonitoringMessage.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringMessage.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringMessage, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringMessage.timestamp >= start_time) if end_time: @@ -1916,7 +2734,11 @@ class MonitoringService: if bot_ids: conditions.append(persistence_monitoring.MonitoringLLMCall.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringLLMCall.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringLLMCall, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringLLMCall.timestamp >= start_time) if end_time: @@ -2023,7 +2845,11 @@ class MonitoringService: if bot_ids: conditions.append(persistence_monitoring.MonitoringError.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringError.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringError, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringError.timestamp >= start_time) if end_time: @@ -2074,7 +2900,11 @@ class MonitoringService: if bot_ids: conditions.append(persistence_monitoring.MonitoringSession.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringSession.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringSession, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringSession.last_activity >= start_time) if end_time: @@ -2227,6 +3057,8 @@ class MonitoringService: pipeline_ids: list[str] | None = None, start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> dict: """Get feedback statistics. @@ -2236,10 +3068,19 @@ class MonitoringService: workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringFeedback.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringFeedback, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringFeedback.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringFeedback.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringFeedback, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringFeedback.timestamp >= start_time) if end_time: @@ -2320,16 +3161,27 @@ class MonitoringService: end_time: datetime.datetime | None = None, limit: int = 100, offset: int = 0, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> tuple[list[dict], int]: """Get feedback list with filters.""" limit, offset = self.normalize_page_window(limit, offset) workspace_uuid = require_workspace_uuid(context) conditions = [persistence_monitoring.MonitoringFeedback.workspace_uuid == workspace_uuid] + conditions.append( + self._monitoring_execution_condition( + persistence_monitoring.MonitoringFeedback, workspace_uuid, execution_statuses, mode + ) + ) if bot_ids: conditions.append(persistence_monitoring.MonitoringFeedback.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringFeedback.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringFeedback, workspace_uuid, pipeline_ids + ) + ) if feedback_type is not None: conditions.append(persistence_monitoring.MonitoringFeedback.feedback_type == feedback_type) if start_time: @@ -2382,7 +3234,11 @@ class MonitoringService: if bot_ids: conditions.append(persistence_monitoring.MonitoringFeedback.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(persistence_monitoring.MonitoringFeedback.pipeline_id.in_(pipeline_ids)) + conditions.append( + self._monitoring_processor_condition( + persistence_monitoring.MonitoringFeedback, workspace_uuid, pipeline_ids + ) + ) if start_time: conditions.append(persistence_monitoring.MonitoringFeedback.timestamp >= start_time) if end_time: diff --git a/src/langbot/pkg/api/http/service/monitoring_execution_details.py b/src/langbot/pkg/api/http/service/monitoring_execution_details.py new file mode 100644 index 000000000..1b5fa94f9 --- /dev/null +++ b/src/langbot/pkg/api/http/service/monitoring_execution_details.py @@ -0,0 +1,580 @@ +"""Tenant-scoped execution inputs, outputs, related runs and conversation history.""" + +from __future__ import annotations + +import datetime +import json + +import sqlalchemy as sa +from sqlalchemy.ext.asyncio import AsyncSession + +from ....entity.persistence.agent_run import AgentRun, AgentRunEvent +from ....entity.persistence.event_log import EventLog +from ....entity.persistence.transcript import Transcript +from ....entity.persistence import monitoring as models +from .tenant import require_workspace_uuid + + +def epoch(value): + if value is None: + return None + return int(value.replace(tzinfo=datetime.timezone.utc).timestamp() * 1000) + + +def parsed(value, fallback=None): + try: + return json.loads(value) if value else fallback + except (TypeError, ValueError): + return fallback + + +def event_content(content, event_data): + """Keep event-specific properties alongside the normalized message input.""" + if not isinstance(event_data, dict) or not event_data: + return content + if not isinstance(content, dict): + return event_data if not content else {**event_data, 'input': content} + merged = dict(event_data) + conflicts = {} + for key, value in content.items(): + if key not in merged: + merged[key] = value + elif merged[key] != value and value is not None and value != '' and value != [] and value != {}: + conflicts[key] = value + if conflicts: + # Never silently overwrite a custom field that shares an input name. + return {'event': merged, 'input': conflicts} + return merged + + +def event_item(row): + metadata = parsed(row.metadata_json, {}) + return { + 'id': row.event_id, + 'event_type': row.event_type, + 'source': row.source, + 'timestamp_ms': epoch(row.event_time or row.created_at), + 'actor_id': row.actor_id, + 'actor_name': row.actor_name, + 'conversation_id': row.conversation_id, + 'thread_id': row.thread_id, + 'content': event_content(parsed(row.input_json, None) or row.input_summary, metadata.get('input_event')), + 'metadata': metadata, + } + + +def message_item(row): + return { + 'id': row.id, + 'role': row.role or 'user', + 'timestamp_ms': epoch(row.timestamp), + 'content': parsed(row.message_content, None) or row.message_content, + 'actor_id': row.user_id, + 'actor_name': row.user_name, + 'status': row.status, + 'event_id': row.event_id, + 'run_id': row.run_id, + 'origin': 'delivery' if row.role == 'assistant' else 'input', + 'delivery': parsed(row.variables, {}) if row.role == 'assistant' else None, + } + + +def transcript_item(row): + return { + 'id': row.transcript_id, + 'role': row.role, + 'timestamp_ms': epoch(row.created_at), + 'content': parsed(row.content_json, None) or row.content, + 'attachments': parsed(row.attachment_refs_json, []), + 'event_id': row.event_id, + 'run_id': row.run_id, + 'origin': 'generated' if row.role == 'assistant' else 'input', + } + + +class ExecutionDetailsMixin: + def _execution_json_text(self, column, key): + # SQLite JSON functions accept TEXT directly; PostgreSQL operators + # require an explicit JSON cast for the existing text-backed journals. + if self.ap.persistence_mgr.get_db_engine().dialect.name == 'sqlite': + expression = sa.type_coerce(column, sa.JSON) + else: + expression = sa.cast(column, sa.JSON) + return expression[key].as_string() + + async def link_execution_message(self, context, message_id, run_id, event_id): + workspace = self._require_write_context(context) + statement = ( + sa.update(models.MonitoringMessage) + .where( + models.MonitoringMessage.workspace_uuid == workspace, + models.MonitoringMessage.id == message_id, + ) + .values(run_id=run_id, event_id=sa.func.coalesce(models.MonitoringMessage.event_id, event_id)) + ) + tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None) + if callable(tenant_uow): + async with tenant_uow(workspace) as uow: + await uow.session.execute(statement) + else: + await self.ap.persistence_mgr.execute_async(statement) + + async def record_ingress_event( + self, + context, + *, + event_id, + event_type, + bot_id, + bot_name, + platform, + payload, + actor_id=None, + actor_name=None, + status='running', + routes=None, + ): + """Persist entry facts even when routing never creates a processor run.""" + from ....agent.runner.event_log_store import EventLogStore + from .monitoring import _message_preview + + workspace = self._require_write_context(context) + store = EventLogStore(self.ap.persistence_mgr.get_db_engine()) + # Reuse the message sanitizer (including inline-media handling) instead + # of storing an unbounded adapter object in the event journal. + content = parsed(self._sanitize_message_content(json.dumps(payload, ensure_ascii=False, default=str)), {}) + summary = _message_preview(json.dumps(content, ensure_ascii=False), 1000) if content else event_type + await store.append_event( + event_id=event_id, + event_type=event_type, + source='platform', + workspace_id=workspace, + bot_id=bot_id, + actor_id=actor_id, + actor_name=actor_name, + actor_type='user' if actor_id else 'system', + input_summary=summary, + input_json=content, + metadata={'status': status, 'bot_name': bot_name, 'platform': platform}, + ) + if routes is not None: + safe_routes = [] + for route in routes: + if isinstance(route, BaseException): + safe_routes.append({'status': 'failed', 'reason': str(route)}) + elif isinstance(route, dict): + safe_routes.append( + { + key: route.get(key) + for key in ( + 'status', + 'reason', + 'target_type', + 'target_uuid', + 'run_id', + 'failure_code', + ) + } + ) + metadata = {'status': status, 'bot_name': bot_name, 'platform': platform, 'routes': safe_routes} + async with AsyncSession(self.ap.persistence_mgr.get_db_engine()) as session: + await session.execute( + sa.update(EventLog) + .where( + EventLog.workspace_id == workspace, + EventLog.event_id == event_id, + ) + .values(metadata_json=json.dumps(metadata, ensure_ascii=False)) + ) + await session.commit() + + async def _execution_query(self, workspace, statement): + tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None) + if callable(tenant_uow): + async with tenant_uow(workspace) as uow: + return await uow.session.execute(statement) + async with AsyncSession(self.ap.persistence_mgr.get_db_engine()) as session: + return await session.execute(statement) + + async def _execution_rows(self, workspace, statement): + return list((await self._execution_query(workspace, statement)).scalars().all()) + + async def _execution_page(self, workspace, statement, convert, offset, limit): + rows = await self._execution_rows(workspace, statement.offset(offset).limit(limit + 1)) + return { + 'items': [convert(row) for row in rows[:limit]], + 'has_more': len(rows) > limit, + 'next_offset': offset + min(len(rows), limit), + } + + def _unhandled_event_conditions(self, workspace, bot_ids=None, start_time=None, end_time=None): + # Absorbed steering events belong to an existing run, too. + conditions = [ + EventLog.workspace_id == workspace, + ~sa.exists( + sa.select(AgentRun.id).where( + AgentRun.workspace_id == workspace, + sa.or_(AgentRun.event_id == EventLog.event_id, AgentRun.run_id == EventLog.run_id), + ) + ), + ~sa.exists( + sa.select(models.MonitoringMessage.id).where( + models.MonitoringMessage.workspace_uuid == workspace, + models.MonitoringMessage.event_id == EventLog.event_id, + ) + ), + ] + if bot_ids: + conditions.append(EventLog.bot_id.in_(bot_ids)) + if start_time: + conditions.append(EventLog.created_at >= start_time) + if end_time: + conditions.append(EventLog.created_at <= end_time) + return conditions + + def _serialize_event_execution(self, row): + metadata = parsed(row.metadata_json, {}) + status = metadata.get('status', 'ignored') + return { + 'source': 'event', + 'id': row.event_id, + 'event_id': row.event_id, + 'status': status, + 'status_group': status, + 'title': row.event_type, + 'input_preview': row.input_summary or '', + 'target_kind': 'event', + 'target_id': None, + 'target_name': None, + 'bot_id': row.bot_id, + 'bot_name': metadata.get('bot_name'), + 'platform': metadata.get('platform') or row.source, + 'user_id': row.actor_id, + 'user_name': row.actor_name, + 'conversation_id': row.conversation_id, + 'session_id': row.conversation_id, + 'created_at_ms': epoch(row.created_at), + 'started_at_ms': None, + 'finished_at_ms': None, + 'duration_ms': None, + 'usage': None, + 'cost': None, + 'queue_name': None, + 'pipeline_id': None, + 'pipeline_name': None, + 'runner_id': None, + 'debug': row.source == 'debug', + 'has_error': status == 'failed', + 'status_reason': metadata.get('reason'), + } + + async def enrich_execution_rows(self, workspace, items): + ids = {item['event_id'] for item in items if item.get('event_id')} + if not ids: + return + events = await self._execution_rows( + workspace, + sa.select(EventLog).where( + EventLog.workspace_id == workspace, + EventLog.event_id.in_(ids), + ), + ) + by_id = {row.event_id: row for row in events} + for item in items: + event = by_id.get(item.get('event_id')) + if event: + meta = parsed(event.metadata_json, {}) + item.update( + event_type=event.event_type, + input_preview=event.input_summary or item.get('input_preview') or '', + user_id=event.actor_id, + user_name=event.actor_name, + platform=meta.get('platform') or event.source, + bot_name=item.get('bot_name') or meta.get('bot_name'), + ) + + async def get_execution_detail(self, context, source, execution_id, *, section=None, offset=0, limit=100): + workspace = require_workspace_uuid(context) + limit, offset = self.normalize_page_window(limit, offset) + limit = min(limit, 100) + sections = { + 'inputs', + 'outputs', + 'deliveries', + 'conversation', + 'related', + 'events', + 'llm_calls', + 'tool_calls', + 'errors', + } + if section is not None and section not in sections: + raise ValueError('Unknown execution section') + + run = message = event = None + if source in {'agent', 'auto'}: + rows = await self._execution_rows( + workspace, + sa.select(AgentRun).where( + AgentRun.workspace_id == workspace, + AgentRun.run_id == execution_id, + ), + ) + run = rows[0] if rows else None + if run is None and source in {'pipeline', 'auto'}: + rows = await self._execution_rows( + workspace, + sa.select(models.MonitoringMessage).where( + models.MonitoringMessage.workspace_uuid == workspace, + models.MonitoringMessage.id == execution_id, + ), + ) + message = rows[0] if rows else None + if message and message.parent_message_id: + return await self.get_execution_detail( + context, 'auto', message.parent_message_id, section=section, offset=offset, limit=limit + ) + if run is None and message is None and source in {'event', 'auto'}: + rows = await self._execution_rows( + workspace, + sa.select(EventLog).where( + EventLog.workspace_id == workspace, + EventLog.event_id == execution_id, + ), + ) + event = rows[0] if rows else None + if run is None and message is None and event is None: + raise ValueError('Execution not found') + + if message and message.run_id: + linked = await self._execution_rows( + workspace, + sa.select(AgentRun).where(AgentRun.workspace_id == workspace, AgentRun.run_id == message.run_id), + ) + if linked: + linked_run = linked[0] + else: + linked_run = None + else: + linked_run = None + + if run: + names = await self._agent_names(workspace, [run]) + row = self._serialize_agent_execution(run, names) + event_id, bot, conversation, thread = run.event_id, run.bot_id, run.conversation_id, run.thread_id + run_id = run.run_id + elif message: + row = self._serialize_pipeline_execution(message) + event_id, bot, conversation, thread = message.event_id, message.bot_id, message.session_id, None + run_id = message.run_id + if linked_run: + row['duration_ms'] = ( + int((linked_run.finished_at - linked_run.started_at).total_seconds() * 1000) + if linked_run.started_at and linked_run.finished_at + else None + ) + row['usage'] = parsed(linked_run.usage_json) + row['status_reason'] = linked_run.status_reason + else: + row = self._serialize_event_execution(event) + event_id, bot, conversation, thread = event.event_id, event.bot_id, event.conversation_id, event.thread_id + run_id = event.run_id + await self.enrich_execution_rows(workspace, [row]) + + ledger = run or linked_run + ingress_id = parsed(ledger.metadata_json, {}).get('ingress_event_id') if ledger else None + event_conditions = [ + EventLog.workspace_id == workspace, + sa.or_( + EventLog.event_id == ingress_id if ingress_id else sa.false(), + EventLog.event_id == ledger.event_id if ledger else sa.false(), + EventLog.event_id == event_id if event_id else sa.false(), + EventLog.run_id == run_id if run_id else sa.false(), + ), + ] + event_ids = sa.select(EventLog.event_id).where(*event_conditions) + # All joins use explicit identities. Session/time matching is reserved + # for the visibly separate conversation history section. + message_scope = [models.MonitoringMessage.workspace_uuid == workspace] + related_message = sa.or_( + models.MonitoringMessage.id == message.id if message else sa.false(), + models.MonitoringMessage.parent_message_id == (message.id if message else run_id) + if message or run_id + else sa.false(), + models.MonitoringMessage.run_id == run_id if run_id else sa.false(), + ) + transcript_scope = [Transcript.workspace_id == workspace, Transcript.bot_id == bot] + transcripts = sa.select(Transcript).where( + *transcript_scope, + sa.or_( + Transcript.run_id == run_id if run_id else sa.false(), + sa.and_(Transcript.role == 'user', Transcript.event_id.in_(event_ids)), + ), + ) + call_ids = sa.select(models.MonitoringMessage.id).where(*message_scope, related_message) + statements = { + 'inputs': (sa.select(EventLog).where(*event_conditions).order_by(EventLog.id.asc()), event_item), + 'outputs': (transcripts.where(Transcript.role != 'user').order_by(Transcript.seq.asc()), transcript_item), + 'deliveries': ( + sa.select(models.MonitoringMessage) + .where(*message_scope, related_message, models.MonitoringMessage.role == 'assistant') + .order_by(models.MonitoringMessage.timestamp.asc(), models.MonitoringMessage.id.asc()), + message_item, + ), + 'events': ( + sa.select(AgentRunEvent) + .where(AgentRunEvent.run_id == run_id if run_id else sa.false()) + .order_by(AgentRunEvent.sequence.asc()), + lambda r: { + 'id': r.id, + 'sequence': r.sequence, + 'type': r.type, + 'data': parsed(r.data_json, {}), + 'timestamp_ms': epoch(r.created_at), + }, + ), + } + for name, model in [ + ('llm_calls', models.MonitoringLLMCall), + ('tool_calls', models.MonitoringToolCall), + ('errors', models.MonitoringError), + ]: + statements[name] = ( + sa.select(model) + .where( + model.workspace_uuid == workspace, + sa.or_( + model.message_id.in_(call_ids), + model.message_id == run_id if run_id else sa.false(), + ), + ) + .order_by(model.timestamp.asc(), model.id.asc()), + lambda r, model=model: self.ap.persistence_mgr.serialize_model(model, r), + ) + if message: + statements['conversation'] = ( + sa.select(models.MonitoringMessage) + .where( + *message_scope, + models.MonitoringMessage.bot_id == bot, + models.MonitoringMessage.session_id == conversation, + ) + .order_by(models.MonitoringMessage.timestamp.desc(), models.MonitoringMessage.id.desc()), + message_item, + ) + else: + statements['conversation'] = ( + sa.select(Transcript) + .where( + *transcript_scope, + Transcript.conversation_id == conversation if conversation else sa.false(), + Transcript.thread_id == thread, + ) + .order_by(Transcript.seq.desc()), + transcript_item, + ) + + pages = {} + for name, (query, convert) in statements.items(): + if section is None or section == name: + pages[name] = await self._execution_page(workspace, query, convert, offset, limit) + if run: + meta = parsed(run.metadata_json, {}) + for item in pages.get('inputs', {}).get('items', []): + # Older event logs omitted data, but the root run may retain it. + # Do not apply the root snapshot to later steering events. + if item['id'] == run.event_id and not item.get('metadata', {}).get('input_event'): + item['content'] = event_content(item['content'], meta.get('input_event')) + if message and (section is None or section == 'inputs'): + # The original rich message is authoritative even when the event + # log is absent or only retained a compact event summary. + if offset == 0: + pages['inputs']['items'].insert(0, message_item(message)) + if ( + run + and not pages.get('inputs', {}).get('items') + and offset == 0 + and (section is None or section == 'inputs') + ): + meta = parsed(run.metadata_json, {}) + content = event_content(meta.get('input'), meta.get('input_event')) + if content: + pages['inputs']['items'] = [ + {'id': run.event_id or run.run_id, 'content': content, 'timestamp_ms': epoch(run.created_at)} + ] + + if section is None or section == 'related': + # Union of ledger executions and legacy pipeline roots. SQL paging + # avoids silently dropping siblings on highly fanned-out events. + related_runs = sa.select( + sa.literal('agent').label('source'), AgentRun.run_id.label('id'), AgentRun.created_at.label('time') + ).where( + AgentRun.workspace_id == workspace, + sa.or_( + AgentRun.event_id.in_(event_ids), + self._execution_json_text(AgentRun.metadata_json, 'ingress_event_id').in_(event_ids), + ), + AgentRun.run_id != (run_id or ''), + ~sa.exists().where( + models.MonitoringMessage.workspace_uuid == workspace, + models.MonitoringMessage.run_id == AgentRun.run_id, + models.MonitoringMessage.role != 'assistant', + ), + ) + related_messages = sa.select( + sa.literal('pipeline'), models.MonitoringMessage.id, models.MonitoringMessage.timestamp + ).where( + *message_scope, + models.MonitoringMessage.event_id.in_(event_ids), + models.MonitoringMessage.role != 'assistant', + models.MonitoringMessage.id != (message.id if message else ''), + ) + union = sa.union_all(related_runs, related_messages).subquery() + refs = ( + await self._execution_query( + workspace, + sa.select(union).order_by(union.c.time, union.c.source, union.c.id).offset(offset).limit(limit + 1), + ) + ).all() + related = [] + for ref in refs[:limit]: + if ref.source == 'agent': + records = await self._execution_rows( + workspace, + sa.select(AgentRun).where(AgentRun.workspace_id == workspace, AgentRun.run_id == ref.id), + ) + related.append( + self._serialize_agent_execution(records[0], await self._agent_names(workspace, records)) + ) + else: + records = await self._execution_rows( + workspace, + sa.select(models.MonitoringMessage).where( + *message_scope, models.MonitoringMessage.id == ref.id + ), + ) + related.append(self._serialize_pipeline_execution(records[0])) + pages['related'] = { + 'items': related, + 'has_more': len(refs) > limit, + 'next_offset': offset + min(len(refs), limit), + } + + result = { + 'source': row['source'], + 'row': row, + 'pages': pages, + 'legacy_context': bool(message and not message.event_id), + } + if run: + # Do not expose runtime authorization, leases or claim tokens. + result['run'] = {'run_id': run.run_id, 'status': run.status, 'status_reason': run.status_reason} + if message: + result['message'] = self.ap.persistence_mgr.serialize_model(models.MonitoringMessage, message) + # Backwards-compatible fields for existing API consumers. + for key in ('events', 'llm_calls', 'tool_calls', 'errors'): + if key in pages: + result[key] = pages[key]['items'] + result['has_more'] = pages.get('events', {}).get('has_more', False) + result['next_cursor'] = pages.get('events', {}).get('next_offset') + return result diff --git a/src/langbot/pkg/api/http/service/monitoring_traffic.py b/src/langbot/pkg/api/http/service/monitoring_traffic.py index 702d34e01..c8469c488 100644 --- a/src/langbot/pkg/api/http/service/monitoring_traffic.py +++ b/src/langbot/pkg/api/http/service/monitoring_traffic.py @@ -24,9 +24,14 @@ async def get_traffic_series( pipeline_ids: list[str] | None = None, start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, + execution_statuses: list[str] | None = None, + mode: str = 'all', ) -> dict: """Count all matching records in UTC buckets, returning at most 1000 points.""" workspace_uuid = require_workspace_uuid(context) + from .monitoring import MonitoringService + + service = MonitoringService(ap) bucket = 'hour' if start_time and end_time and end_time - start_time <= datetime.timedelta(days=7) else 'day' step = datetime.timedelta(hours=1) if bucket == 'hour' else datetime.timedelta(days=1) postgres = ap.persistence_mgr.get_db_engine().dialect.name == 'postgresql' @@ -40,10 +45,11 @@ async def get_traffic_series( pattern = '%Y-%m-%dT%H:00:00' if bucket == 'hour' else '%Y-%m-%dT00:00:00' time_bucket = sqlalchemy.func.strftime(pattern, timestamp) conditions = [model.workspace_uuid == workspace_uuid] + conditions.append(service._monitoring_execution_condition(model, workspace_uuid, execution_statuses, mode)) if bot_ids: conditions.append(model.bot_id.in_(bot_ids)) if pipeline_ids: - conditions.append(model.pipeline_id.in_(pipeline_ids)) + conditions.append(service._monitoring_processor_condition(model, workspace_uuid, pipeline_ids)) if start_time is not None: conditions.append(timestamp >= start_time) if end_time is not None: diff --git a/src/langbot/pkg/api/http/service/space.py b/src/langbot/pkg/api/http/service/space.py index b07f4f1ed..c95713f56 100644 --- a/src/langbot/pkg/api/http/service/space.py +++ b/src/langbot/pkg/api/http/service/space.py @@ -290,7 +290,23 @@ class SpaceService: selection = await self.get_model_selection('chat') if not selection: raise ValueError('No recommended chat model is available') - recommended = selection[0] + return await self._resolve_recommended_model(context, selection[0]) + + async def get_recommended_assistant_model(self, context: typing.Any) -> dict: + """Resolve the dedicated, health-filtered assistant recommendation without fallback.""" + space_url = self._get_space_config()['url'] + session = httpclient.get_session() + async with session.get(f'{space_url}/api/v1/models/assistant/recommendation') as response: + if response.status != 200: + raise ValueError('No recommended assistant model is available') + payload = await httpclient.read_json_limited(response) + if payload.get('code') != 0: + raise ValueError('No recommended assistant model is available') + recommended = SpaceModel.model_validate(payload['data']['model']) + return await self._resolve_recommended_model(context, recommended) + + async def _resolve_recommended_model(self, context: typing.Any, recommended: SpaceModel) -> dict: + """Map a public catalog model to this Workspace, refreshing the catalog once.""" async def find_local_model(): result = await self.ap.persistence_mgr.execute_async( diff --git a/src/langbot/pkg/api/mcp/server.py b/src/langbot/pkg/api/mcp/server.py index 58cc9be71..104532f5d 100644 --- a/src/langbot/pkg/api/mcp/server.py +++ b/src/langbot/pkg/api/mcp/server.py @@ -165,7 +165,7 @@ class LangBotMCPServer: return _dump(ap.box_service.get_recent_errors(context)) @mcp.tool( - description='Read bounded Workspace runtime records. Filters follow the monitoring service: bot_ids, pipeline_ids, session_ids, start_time, end_time, knowledge_base_id, user_query, is_active as supported by the record kind.' + description='Read bounded Workspace runtime records. Filters follow the monitoring service: bot_ids, pipeline_ids, session_ids, start_time, end_time, mode (all/real/debug), execution_statuses (owning execution status groups), knowledge_base_id, user_query, is_active as supported by the record kind.' ) async def get_monitoring_records( kind: typing.Literal['messages', 'llm_calls', 'tool_calls', 'embedding_calls', 'sessions', 'errors'], @@ -176,7 +176,7 @@ class LangBotMCPServer: import datetime context = _authorized(Permission.RESOURCE_VIEW) - allowed = {'start_time', 'end_time'} + allowed = {'start_time', 'end_time', 'mode', 'execution_statuses'} if kind != 'embedding_calls': allowed |= {'bot_ids', 'pipeline_ids'} if kind in {'messages', 'tool_calls'}: @@ -207,6 +207,52 @@ class LangBotMCPServer: return _dump(await ap.monitoring_service.get_message_details(context, identifier)) return _dump(await ap.monitoring_service.get_session_analysis(context, identifier)) + @mcp.tool( + description='List Workspace executions, including events that did not start a processor. The legacy pipeline_ids filter accepts Agent, Pipeline and event processor IDs. Returns bounded rows and summary metrics.' + ) + async def get_monitoring_executions( + limit: int = 50, + offset: int = 0, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + statuses: list[str] | None = None, + mode: typing.Literal['all', 'real', 'debug'] = 'all', + ) -> str: + context = _authorized(Permission.RESOURCE_VIEW) + return _dump( + await ap.monitoring_service.get_executions( + context, + limit=limit, + offset=offset, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + statuses=statuses, + mode=mode, + ) + ) + + @mcp.tool( + description='Read execution inputs, generated outputs, delivery records, calls, errors, related executions and conversation history. Each section is independently paginated; follow pages[section].next_offset when has_more is true.' + ) + async def get_monitoring_execution_detail( + source: typing.Literal['auto', 'agent', 'pipeline', 'event'], + identifier: str, + section: str | None = None, + offset: int = 0, + limit: int = 100, + ) -> str: + context = _authorized(Permission.RESOURCE_VIEW) + return _dump( + await ap.monitoring_service.get_execution_detail( + context, + source, + identifier, + section=section, + offset=offset, + limit=limit, + ) + ) + # ----- Bots ---------------------------------------------------- # @mcp.tool(description='List all messaging-platform bots. Secrets are redacted.') async def list_bots() -> str: diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index 4f51b15aa..a18ebd914 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -20,7 +20,6 @@ from langbot_plugin.box.security import BOX_SHARED_WORKSPACE_PROBE_PREFIX from .admission import SandboxAdmissionController, require_cloud_admission_policy from .connector import BoxRuntimeConnector, _get_box_config from . import secure_fs -from ..telemetry import features as telemetry_features from ..utils import httpclient from ..api.http.context import ExecutionContext from ..api.http.service.tenant import TenantContext, require_workspace_uuid @@ -636,7 +635,6 @@ class BoxService: f'query_id={query.query_id} ' f'summary={json.dumps(self._summarize_result(result), ensure_ascii=False)}' ) - telemetry_features.increment(query, 'sandbox', 'execs') return self._serialize_result(result) def resolve_box_session_id(self, query: pipeline_query.Query) -> str: @@ -2011,7 +2009,6 @@ class BoxService: # ── Observability ───────────────────────────────────────────────── def _record_error(self, exc: Exception, query: pipeline_query.Query): - telemetry_features.increment(query, 'sandbox', 'errors') self._recent_errors.append( { 'timestamp': _dt.datetime.now(_UTC).isoformat(), diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 5a4120240..cf3ecd13a 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -304,6 +304,19 @@ class Application: if callable(database_snapshot): database_stats = database_snapshot() + runtime_ops_stats = {} + runtime_ops_reporter = getattr(self.plugin_connector, 'runtime_ops_reporter', None) + if runtime_ops_reporter is not None: + runtime_ops_stats = runtime_ops_reporter.get_stats() + + telemetry_stats = {} + telemetry_health = getattr(self.telemetry, 'health', None) + if callable(telemetry_health): + try: + telemetry_stats = telemetry_health() + except Exception: + telemetry_stats = {} + return { 'asyncio_tasks': asyncio_tasks, 'event_loop': self.event_loop_monitor.snapshot(), @@ -314,7 +327,8 @@ class Application: 'query_pool': query_pool_stats, 'models': model_stats, 'runtimes': runtime_stats, - 'telemetry_tasks': len(getattr(self.telemetry, 'send_tasks', ())), + 'runtime_ops': runtime_ops_stats, + 'telemetry': telemetry_stats, } async def initialize(self): diff --git a/src/langbot/pkg/entity/persistence/monitoring.py b/src/langbot/pkg/entity/persistence/monitoring.py index 1d2e8ed83..68d68d867 100644 --- a/src/langbot/pkg/entity/persistence/monitoring.py +++ b/src/langbot/pkg/entity/persistence/monitoring.py @@ -29,6 +29,9 @@ class MonitoringMessage(Base): runner_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # Runner name for this query variables = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # Query variables as JSON string role = sqlalchemy.Column(sqlalchemy.String(50), nullable=True, default='user') # user, assistant + event_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) + run_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) + parent_message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) __table_args__ = ( sqlalchemy.Index('ix_monitoring_messages_workspace_timestamp', 'workspace_uuid', 'timestamp'), diff --git a/src/langbot/pkg/persistence/alembic/versions/0034_monitoring_execution_links.py b/src/langbot/pkg/persistence/alembic/versions/0034_monitoring_execution_links.py new file mode 100644 index 000000000..4cc720d07 --- /dev/null +++ b/src/langbot/pkg/persistence/alembic/versions/0034_monitoring_execution_links.py @@ -0,0 +1,36 @@ +"""Link monitoring inputs and deliveries to events and runs.""" + +from alembic import op +import sqlalchemy as sa + +revision = '0034_monitoring_execution_links' +down_revision = '0033_operation_traceability' +branch_labels = None +depends_on = None + + +def upgrade(): + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table('monitoring_messages'): + return + columns = {c['name'] for c in inspector.get_columns('monitoring_messages')} + indexes = {i['name'] for i in inspector.get_indexes('monitoring_messages')} + for name in ('event_id', 'run_id', 'parent_message_id'): + if name not in columns: + op.add_column('monitoring_messages', sa.Column(name, sa.String(255), nullable=True)) + index = f'ix_monitoring_messages_{name}' + if index not in indexes: + op.create_index(index, 'monitoring_messages', [name]) + + +def downgrade(): + inspector = sa.inspect(op.get_bind()) + if not inspector.has_table('monitoring_messages'): + return + columns = {c['name'] for c in inspector.get_columns('monitoring_messages')} + indexes = {i['name'] for i in inspector.get_indexes('monitoring_messages')} + for name in ('parent_message_id', 'run_id', 'event_id'): + if f'ix_monitoring_messages_{name}' in indexes: + op.drop_index(f'ix_monitoring_messages_{name}', table_name='monitoring_messages') + if name in columns: + op.drop_column('monitoring_messages', name) diff --git a/src/langbot/pkg/persistence/tenant_uow.py b/src/langbot/pkg/persistence/tenant_uow.py index 0eacc85c5..6eaf860c3 100644 --- a/src/langbot/pkg/persistence/tenant_uow.py +++ b/src/langbot/pkg/persistence/tenant_uow.py @@ -483,9 +483,16 @@ def _validate_scoped_statement_call(args: tuple[typing.Any, ...], kwargs: dict[s and getattr(getattr(source, 'table', None), 'name', None) == 'legacy_pipelines' and source.name in {'config', 'stages', 'extensions_preferences'} ) - if not pipeline_json_cas: + monitoring_metadata = ( + type(element.type) is sqlalchemy.JSON + and isinstance(source, sqlalchemy.Column) + and type(source.type) is sqlalchemy.Text + and getattr(getattr(source, 'table', None), 'name', None) in {'event_log', 'agent_run'} + and source.name == 'metadata_json' + ) + if not (pipeline_json_cas or monitoring_metadata): raise ScopedSessionTransactionError( - 'TenantUnitOfWork only allows trusted pgvector and Pipeline JSON CAS casts' + 'TenantUnitOfWork only allows trusted pgvector, Pipeline JSON CAS and monitoring metadata casts' ) if isinstance(element, sqlalchemy.sql.functions.FunctionElement): diff --git a/src/langbot/pkg/pipeline/monitoring_helper.py b/src/langbot/pkg/pipeline/monitoring_helper.py index 15ae38f47..3fae71d9c 100644 --- a/src/langbot/pkg/pipeline/monitoring_helper.py +++ b/src/langbot/pkg/pipeline/monitoring_helper.py @@ -18,6 +18,103 @@ if typing.TYPE_CHECKING: from .pool import get_query_execution_context +async def resolve_processor_name( + ap: app.Application, + workspace_uuid: str, + processor_type: str, + processor_id: str | None, + runner_id: str = '', +) -> str: + """Display name of the processor kind that handled a run. + + Monitoring attributed every call to a Pipeline, so an Agent or a plugin event + processor recorded "Unknown" for the entity that actually ran. Each kind is + resolved from the table that owns it, falling back to the identity the + binding carried. + """ + import sqlalchemy + + from ..entity.persistence import agent as persistence_agent + from ..entity.persistence import pipeline as persistence_pipeline + + identifier = str(processor_id or '').strip() + if identifier: + try: + model = persistence_pipeline.LegacyPipeline if processor_type == 'pipeline' else persistence_agent.Agent + result = await ap.persistence_mgr.execute_async( + sqlalchemy.select(model.name).where( + sqlalchemy.and_( + model.uuid == identifier, + model.workspace_uuid == workspace_uuid, + ) + ) + ) + name = str(result.scalar() or '').strip() + if name: + return name + except Exception as e: + ap.logger.warning(f'Failed to resolve monitoring processor name: {e}') + # A deleted Agent or an unregistered plugin processor still names the + # component that ran, never a bare "Unknown". + return identifier or runner_id or processor_type + + +async def prepare_monitoring_identity( + ap: app.Application, + query: pipeline_query.Query, + *, + processor_type: str, + processor_id: str | None, + runner_id: str = '', + execution_record_id: str = '', +) -> None: + """Pin the monitoring identity of a runner-owned execution. + + A Pipeline pins these variables when it starts a query; a Runner started by + an Agent or a plugin event processor has to pin them itself, otherwise every + LLM/tool call it makes is attributed to "Unknown". An enclosing Pipeline + already named the run, so its label is never overwritten. + + ``_monitoring_message_id`` is the execution record the calls belong to: the + Pipeline's message, or the Agent run's id. Both are the id the executions + list shows, which is what lets a call jump back to its record. + """ + try: + variables = getattr(query, 'variables', None) + if not isinstance(variables, dict): + return + if variables.get('_monitoring_pipeline_name'): + return + + bot_name = 'WebChat' + bot_uuid = str(getattr(query, 'bot_uuid', '') or '') + workspace_uuid = str(getattr(query, 'workspace_uuid', '') or '') + if bot_uuid and workspace_uuid: + try: + bot = await ap.bot_service.get_bot( + workspace_uuid, + bot_uuid, + include_secret=False, + ) + if bot: + bot_name = bot.get('name', bot_name) + except Exception as e: + ap.logger.warning(f'Failed to resolve monitoring bot name: {e}') + + variables['_monitoring_bot_name'] = bot_name + variables['_monitoring_pipeline_name'] = await resolve_processor_name( + ap, + workspace_uuid, + processor_type, + processor_id, + runner_id, + ) + if execution_record_id: + variables['_monitoring_message_id'] = execution_record_id + except Exception as e: + ap.logger.error(f'Failed to prepare monitoring identity: {e}') + + class MonitoringHelper: """Helper class for monitoring operations""" @@ -199,6 +296,7 @@ class MonitoringHelper: user_name=sender_name, runner_name=runner_name, role='assistant', + parent_message_id=(query.variables or {}).get('_monitoring_message_id'), ) except Exception as e: ap.logger.error(f'Failed to record query response: {e}') diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index a737594d4..ad45b4c50 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -3,6 +3,7 @@ from __future__ import annotations import dataclasses import typing import traceback +import asyncio import sqlalchemy @@ -193,6 +194,19 @@ class RuntimePipeline: query.variables['_pipeline_bound_mcp_servers'] = self.bound_mcp_servers query.variables['_pipeline_mcp_resource_attachments'] = self.mcp_resource_attachments query.variables['_pipeline_mcp_resource_agent_read_enabled'] = self.mcp_resource_agent_read_enabled + # Telemetry: the bindings this lane runs with. None means "all enabled", + # which the *_all flags make explicit in the chain record. + try: + from ..telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline( + plugins=self.bound_plugins, + plugins_all=self.enable_all_plugins, + mcp_servers=self.bound_mcp_servers, + mcp_all=self.enable_all_mcp_servers, + ) + except Exception: + pass # Record query start for monitoring try: @@ -257,6 +271,9 @@ class RuntimePipeline: self.ap.logger.error(result.error_notice) # Mark query as having error query.variables['_monitoring_has_error'] = True + # The lane reports failures as a value instead of raising, so record the + # reason here: without it the uploaded trace would not explain the break. + self._record_lane_failure(query, str(result.error_notice)) # Record error to monitoring system try: await self._assert_execution_active(query) @@ -366,16 +383,103 @@ class RuntimePipeline: i += 1 + @staticmethod + def _synthetic_origin(query: pipeline_query.Query) -> str: + """Origin of a lane that starts without an inbound platform event.""" + try: + variables = getattr(query, 'variables', None) + declared = str(variables.get('_telemetry_origin') or '').strip() if isinstance(variables, dict) else '' + except Exception: + declared = '' + if declared in ('webui', 'api'): + return declared + adapter = getattr(query, 'adapter', None) + if adapter is not None and adapter.__class__.__name__ == 'WebSocketAdapter': + # WebChat and the CLI diagnostic surface both speak through the + # WebSocket proxy adapter; the CLI marks itself in its variables. + try: + variables = getattr(query, 'variables', None) + if isinstance(variables, dict) and '_cli_run_status' in variables: + return 'api' + except Exception: + pass + return 'webui' + return 'api' + + @staticmethod + def _synthetic_event_type(query: pipeline_query.Query) -> str: + """Event type of the virtual inbound event a synthetic lane stands for.""" + event = getattr(query, 'message_event', None) + event_type = str(getattr(event, 'type', None) or '').strip() + return event_type[:160] or 'message.received' + async def process_query(self, query: pipeline_query.Query): + from ..telemetry import trace as trace_mod from ..telemetry.execution import ingress + from ..telemetry.execution import record as record_execution from ..telemetry.platform import processing_mode + from ..telemetry.trace import stage_scope token = processing_mode.set('pipeline') + # The Workspace-scoped opaque query uuid is the execution identity Space + # shows for this lane; it is a stable UUID for every pooled query. + execution_id = str(getattr(query, 'query_uuid', '') or '').strip() or str(query.query_id) + # A lane with no inbound platform event in flight (WebChat, WebUI debug, + # HTTP pipeline run) synthesizes its virtual inbound event here, so every + # execution starts at an event boundary and the Runner is never the trace + # origin. A lane running under a platform route nests inside that ingress. + synthesizing = trace_mod.current() is None try: - # Callers without a platform event (Webchat, HTTP API) still get one - # trace for the whole Pipeline lane; nested calls reuse the trace. - with ingress(self.ap, 'pipeline_done'): - return await self._process_query(query) + with ingress( + self.ap, + 'pipeline_done', + getattr(self, 'execution_context', None), + execution_id=execution_id, + debug=synthesizing, + origin=self._synthetic_origin(query) if synthesizing else 'platform', + synthetic_event=self._synthetic_event_type(query) if synthesizing else '', + ): + # The lane is its own workflow step: acknowledgements and replies it + # sends directly, and the runner it drives, hang under this node. + # When the lane runs under a platform route the node nests there. + with stage_scope() as node: + lane_outcome = 'success' + lane_error = '' + try: + return await self._process_query(query) + except asyncio.CancelledError: + lane_outcome = 'cancelled' + lane_error = 'cancelled' + raise + except BaseException as exc: + lane_outcome = 'failed' + lane_error = str(exc) or type(exc).__name__ + raise + finally: + try: + variables = getattr(query, 'variables', None) or {} + lane_has_error = bool(variables.get('_monitoring_has_error')) + except Exception: + lane_has_error = False + if lane_outcome == 'success' and lane_has_error: + # The lane reported the failure as a value, not an exception. + lane_outcome = 'failed' + config = getattr(query, 'pipeline_config', None) + try: + runner_id = (RunnerConfigResolver.resolve_runner_id(config) or '') if config else '' + except Exception: + runner_id = '' + record_execution( + self.ap, + getattr(self, 'execution_context', None), + family='pipeline', + operation='run', + mode='pipeline', + runner=runner_id, + outcome=lane_outcome, + error=lane_error, + node=node, + ) finally: processing_mode.reset(token) @@ -490,6 +594,9 @@ class RuntimePipeline: inst_name = query.current_stage_name if query.current_stage_name else 'unknown' self.ap.logger.error(f'Error processing query {query.query_id} stage={inst_name} : {e}') self.ap.logger.error(f'Traceback: {traceback.format_exc()}') + # The lane itself broke: land the reason on the execution trace so the + # chain is uploaded even though the error never became a StageProcessResult. + self._record_lane_failure(query, str(e) or type(e).__name__) # Record query error try: @@ -513,6 +620,40 @@ class RuntimePipeline: self.ap.logger.debug(f'Query {query.query_id} processed') await self.ap.query_pool.remove_query(query) + def _record_lane_failure(self, query: pipeline_query.Query, reason: str) -> None: + """Land one failed ``runner/execute`` stage on the owning execution trace.""" + try: + from ..telemetry.execution import record as record_execution + from ..telemetry.trace import current as current_trace + from ..telemetry.trace import stage_scope + + detail = reason or 'pipeline_lane_failed' + state = current_trace() + # The runner orchestrator already lands its own failed ``runner/execute`` + # node when the break happens inside a runner; in that case only carry the + # reason over instead of repeating the node on the chain. + already_recorded = bool(state is not None and state.runner_failed) + if not already_recorded: + # The lane failure is its own workflow step, hanging under whatever + # step routed into it (or a root when nothing did). + with stage_scope() as node: + record_execution( + self.ap, + getattr(self, 'execution_context', None), + family='runner', + operation='execute', + mode='pipeline', + runner=(RunnerConfigResolver.resolve_runner_id(query.pipeline_config) or ''), + outcome='failed', + error=detail, + node=node, + ) + state = current_trace() + if state is not None: + state.mark_failure(detail) + except Exception: + pass + class PipelineManager: """流水线管理器""" diff --git a/src/langbot/pkg/pipeline/preproc/preproc.py b/src/langbot/pkg/pipeline/preproc/preproc.py index 67db3b1a4..2a0878170 100644 --- a/src/langbot/pkg/pipeline/preproc/preproc.py +++ b/src/langbot/pkg/pipeline/preproc/preproc.py @@ -108,6 +108,12 @@ class PreProcessor(stage.PipelineStage): ) catalog = ResourcePolicyProjector.filter_tools(catalog, policy) ToolManager.bind_query_tool_sources(query, catalog) + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline(tools=catalog, tools_all=policy.allow_all_tools) + except Exception: + pass return ToolManager.tools_from_catalog(catalog) def _runner_accepts_multimodal_input(self, descriptor: RunnerDescriptor | None) -> bool: @@ -393,6 +399,18 @@ class PreProcessor(stage.PipelineStage): descriptor, runner_config, ) + # Telemetry: the knowledge bases this lane may retrieve from, plus the + # schema-declared selector fields that selected them. + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline( + knowledge_bases=query.variables['_knowledge_base_uuids'], + runner_id=descriptor.id if descriptor is not None else '', + selectors=config_schema.extract_selected_resources(descriptor, runner_config), + ) + except Exception: + pass # Emit PromptPreProcessing before the runner receives the query. @@ -427,4 +445,10 @@ class PreProcessor(stage.PipelineStage): bound_skills = extensions_prefs['skills'] query.variables['_pipeline_bound_skills'] = bound_skills + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline(skills=bound_skills) + except Exception: + pass return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/pipeline/process/handlers/chat.py b/src/langbot/pkg/pipeline/process/handlers/chat.py index 2fc7b0f82..2cd681fa1 100644 --- a/src/langbot/pkg/pipeline/process/handlers/chat.py +++ b/src/langbot/pkg/pipeline/process/handlers/chat.py @@ -3,8 +3,6 @@ from __future__ import annotations import uuid import typing import traceback -import time -from datetime import datetime from .. import handler @@ -14,9 +12,7 @@ from ... import plugin_diagnostics import langbot_plugin.api.entities.events as events from ....agent.runner.config_resolver import RunnerConfigResolver from ....agent.runner import config_schema -from ....utils import constants, runner as runner_utils -from ....telemetry import features as telemetry_features -from ....telemetry.identity import workspace_identity +from ....utils import runner as runner_utils import langbot_plugin.api.entities.builtin.provider.session as provider_session import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query import langbot_plugin.api.entities.builtin.provider.message as provider_message @@ -115,9 +111,6 @@ class ChatMessageHandler(handler.MessageHandler): text_length = 0 runner = None try: - # Mark start time for telemetry - start_ts = time.time() - try_claim_steering = getattr( self.ap.agent_run_orchestrator, 'try_claim_steering_from_query', @@ -258,69 +251,48 @@ class ChatMessageHandler(handler.MessageHandler): debug_notice=traceback.format_exc(), ) finally: - # Telemetry reporting + # Telemetry: the per-execution record is built when the owning + # trace closes, so only attach the execution-scoped fields this + # lane knows here. try: - end_ts = time.time() - duration_ms = None - if 'start_ts' in locals(): - duration_ms = int((end_ts - start_ts) * 1000) + from ....telemetry.trace import current as current_trace + state = current_trace() + if state is not None: + adapter_name = query.adapter.__class__.__name__ if hasattr(query, 'adapter') else '' + runner_name = self.ap.agent_run_orchestrator.resolve_runner_id_for_telemetry(query) + state.adapter = state.adapter or adapter_name + state.runner = state.runner or (runner_name or '') + state.runner_category = state.runner_category or runner_utils.get_runner_category_from_runner( + runner_name, None, query.pipeline_config + ) + model_name = '' + try: + if getattr(query, 'use_llm_model_uuid', None): + m = await self.ap.model_mgr.get_model_by_uuid( + get_query_execution_context(query), + query.use_llm_model_uuid, + ) + if m and getattr(m, 'model_entity', None): + model_name = getattr(m.model_entity, 'name', '') or '' + except Exception: + model_name = '' + state.model_name = state.model_name or model_name + if state.pipeline_plugins is None: + state.pipeline_plugins = query.variables.get('_pipeline_bound_plugins', None) + except Exception as ex: + self.ap.logger.warning(f'Failed to attach execution telemetry fields: {ex}') + + # Trigger survey events on successful non-WebSocket responses + try: adapter_name = query.adapter.__class__.__name__ if hasattr(query, 'adapter') else None - - # Use orchestrator to resolve runner ID for telemetry - runner_name = self.ap.agent_run_orchestrator.resolve_runner_id_for_telemetry(query) - - # Model name if available - model_name = None - try: - if getattr(query, 'use_llm_model_uuid', None): - m = await self.ap.model_mgr.get_model_by_uuid( - get_query_execution_context(query), - query.use_llm_model_uuid, - ) - if m and getattr(m, 'model_entity', None): - model_name = getattr(m.model_entity, 'name', None) - except Exception: - model_name = None - - pipeline_plugins = query.variables.get('_pipeline_bound_plugins', None) - - runner_category = runner_utils.get_runner_category_from_runner( - runner_name, None, query.pipeline_config - ) - - # Feature usage collected during query processing (tool calls, - # knowledge base usage, sandbox executions, activated skills, ...) - features = telemetry_features.collect_features(query) - - payload = { - 'event_type': 'query', - 'query_id': query.query_id, - 'adapter': adapter_name, - 'runner': runner_name, - 'runner_category': runner_category, - 'duration_ms': duration_ms, - 'model_name': model_name, - 'version': constants.semantic_version, - **workspace_identity(get_query_execution_context(query)), - 'runtime_instance_id': constants.instance_id, - 'edition': constants.edition, - 'pipeline_plugins': pipeline_plugins, - 'features': features, - 'error': locals().get('error_info', None), - 'timestamp': datetime.utcnow().isoformat(), - } - - await self.ap.telemetry.start_send_task(payload) - - # Trigger survey events on successful non-WebSocket responses if not locals().get('error_info') and adapter_name and 'WebSocket' not in adapter_name: if self.ap.survey: await self.ap.survey.trigger_event('first_bot_response_success') # Counts toward the bot_response_success_100 milestone event await self.ap.survey.record_bot_response_success() except Exception as ex: - self.ap.logger.warning(f'Failed to send telemetry: {ex}') + self.ap.logger.warning(f'Failed to trigger survey event: {ex}') async def _ensure_conversation_for_history( self, diff --git a/src/langbot/pkg/platform/botmgr.py b/src/langbot/pkg/platform/botmgr.py index d4430f55d..291acd1db 100644 --- a/src/langbot/pkg/platform/botmgr.py +++ b/src/langbot/pkg/platform/botmgr.py @@ -396,20 +396,29 @@ class RuntimeBot: } log_method = getattr(self.logger, level, self.logger.info) await log_method(text, metadata=metadata) - if status in {'delivered', 'failed', 'discarded', 'not_matched'}: + if status in {'delivered', 'failed'}: from ..telemetry import trace as trace_mod from ..telemetry.execution import record + state = trace_mod.current() + if status == 'failed' and state is not None and state.reportable and not trace_mod.reporting_suppressed(): + # The root event can finish successfully after a delivery failed. + # Preserve that failure for closure reporting in failures mode. + state.mark_failure(failure_code or 'route_failed') with trace_mod.scope(route_ref=self._route_ref(binding, target_type, target_uuid)): - record( - getattr(self, 'ap', None), - getattr(self, 'execution_context', None), - family='event_route', - operation=event_type, - adapter=type(getattr(self, 'adapter', None)).__name__, - mode=target_type if target_type in {'pipeline', 'agent', 'event_processor'} else 'none', - outcome={'delivered': 'success', 'failed': 'failed'}.get(status, 'skipped'), - ) + # The routing decision is its own workflow step; the Pipeline or + # processor it delivers to hangs under this node. + with trace_mod.stage_scope() as node: + record( + getattr(self, 'ap', None), + getattr(self, 'execution_context', None), + family='event_route', + operation=event_type, + adapter=type(getattr(self, 'adapter', None)).__name__, + mode=target_type if target_type in {'pipeline', 'agent', 'event_processor'} else 'none', + outcome={'delivered': 'success', 'failed': 'failed'}.get(status, 'skipped'), + node=node, + ) return metadata def get_pipeline_target_for_event_type(self, event_type: str = 'message.received') -> str | None: @@ -736,12 +745,19 @@ class RuntimeBot: self, event: platform_events.EBAEvent, adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, + execution_id: str | None = None, ) -> AgentEventEnvelope: event_type = getattr(event, 'type', None) or event.__class__.__name__ event_time = getattr(event, 'timestamp', None) or time.time() - event_id = ( - getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}' - ) + if execution_id: + # Reuse the ingress execution identity so the run view, the journaled + # event and the uploaded chain all share one id. + envelope_event_id = str(execution_id) + prefix = f'platform:{self.bot_entity.uuid}:' + event_id = envelope_event_id[len(prefix) :] if envelope_event_id.startswith(prefix) else envelope_event_id + else: + event_id = self._platform_event_raw_id(event) + envelope_event_id = f'platform:{self.bot_entity.uuid}:{event_id}' target_type, target_id, target_metadata = self._infer_reply_target(event) supported_apis = self._get_adapter_supported_apis(adapter) @@ -773,7 +789,7 @@ class RuntimeBot: delivery_data['interactions'] = interaction_capabilities return AgentEventEnvelope( - event_id=f'platform:{self.bot_entity.uuid}:{event_id}', + event_id=envelope_event_id, event_type=event_type, event_time=int(event_time) if isinstance(event_time, (int, float)) else None, source='platform', @@ -863,6 +879,21 @@ class RuntimeBot: processor_id=agent.get('uuid'), ) + def _platform_event_execution_id(self, event: platform_events.EBAEvent) -> str: + """Space-visible execution identity for one inbound platform event. + + Matches the ``AgentEventEnvelope.event_id`` the same event produces, so + the stored chain and the Space run view share one identity. + """ + return f'platform:{self.bot_entity.uuid}:{self._platform_event_raw_id(event)}' + + @staticmethod + def _platform_event_raw_id(event: platform_events.EBAEvent) -> str: + event_type = getattr(event, 'type', None) or event.__class__.__name__ + return str( + getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}' + ) + async def _handle_platform_event( self, event: platform_events.EBAEvent, @@ -870,51 +901,142 @@ class RuntimeBot: ) -> None: # One inbound event owns one execution trace; every stage recorded while # it is handled (routing, runner, platform API calls) joins that trace. + from ..telemetry import trace as trace_mod from ..telemetry.execution import ingress event.bot_uuid = self.bot_entity.uuid - with ingress(getattr(self, 'ap', None), 'event_done'): - await self._handle_platform_event_body(event, adapter) + execution_id = self._platform_event_execution_id(event) + with ingress( + getattr(self, 'ap', None), + 'event_done', + getattr(self, 'execution_context', None), + execution_id, + require_route=True, + ): + # The inbound event is the root workflow step: everything recorded + # while handling it hangs under this node. + with trace_mod.stage_scope() as node: + await self._handle_platform_event_body(event, adapter, execution_id, node) - async def _handle_platform_event_body( - self, - event: platform_events.EBAEvent, - adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, - ) -> None: + @staticmethod + def _summarize_event_delivery(results: list[typing.Any]) -> tuple[str, str]: + outcome = 'success' + error_detail = '' + for result in results: + if isinstance(result, BaseException): + outcome = 'failed' + error_detail = error_detail or f'{type(result).__name__}: {result}' + return outcome, error_detail + + def _record_platform_event(self, event, adapter, outcome: str, error_detail: str, node: str = '') -> None: from ..telemetry.execution import record record( getattr(self, 'ap', None), getattr(self, 'execution_context', None), family='platform_event', - operation=event.type, + operation=str(getattr(event, 'type', None) or event.__class__.__name__), adapter=adapter.__class__.__name__, - outcome='success', + outcome=outcome, + error=error_detail, + node=node, ) - await self._record_adapter_event(event, adapter) - primary = ( - self._handle_interaction_submission(event, adapter) - if isinstance(event, platform_events.PlatformSpecificEvent) and event.action == 'interaction.submitted' - else self._dispatch_eba_event_to_processor(event, adapter) - ) - subscriptions = self._get_event_bindings_from_value(getattr(self.bot_entity, 'plugin_processors', [])) - seen = set() - tasks = [primary] - for subscription in subscriptions: - processor_uuid = subscription.get('processor_uuid') - if not processor_uuid or processor_uuid in seen or not subscription.get('enabled', True): - continue - seen.add(processor_uuid) - tasks.append(self._dispatch_plugin_subscription(event, adapter, processor_uuid)) - # Start all deliveries together. A slow or failed subscriber cannot block - # another subscriber or the primary route from receiving the event. - results = await asyncio.gather(*tasks, return_exceptions=True) + async def _persist_monitoring_ingress(self, event, adapter, execution_id, status='running', routes=None): + service = getattr(getattr(self, 'ap', None), 'monitoring_service', None) + if service is None or not execution_id: + return + try: + sender = getattr(event, 'sender', None) + await service.record_ingress_event( + self.execution_context, + event_id=execution_id, + event_type=getattr(event, 'type', None) or event.__class__.__name__, + bot_id=self.bot_entity.uuid, + bot_name=self.bot_entity.name, + platform=getattr(self.bot_entity, 'adapter', None) or adapter.__class__.__name__, + payload={ + key: value + for key, value in self._safe_model_dump(event).items() + if key != 'source_platform_object' and not key.startswith('_') + }, + actor_id=self._get_entity_id(sender), + actor_name=self._get_entity_name(sender), + status=status, + routes=routes, + ) + except Exception as exc: + await self.logger.warning(f'Failed to persist monitoring ingress: {exc}') + + async def _handle_platform_event_body( + self, + event: platform_events.EBAEvent, + adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, + execution_id: str | None = None, + node: str = '', + ) -> None: + results: list[typing.Any] = [] + try: + await self._record_adapter_event(event, adapter) + await self._persist_monitoring_ingress(event, adapter, execution_id) + + primary_handler = ( + self._handle_interaction_submission + if isinstance(event, platform_events.PlatformSpecificEvent) and event.action == 'interaction.submitted' + else self._dispatch_eba_event_to_processor + ) + subscriptions = self._get_event_bindings_from_value(getattr(self.bot_entity, 'plugin_processors', [])) + seen = set() + tasks = [self._run_event_delivery(primary_handler, event, adapter, execution_id=execution_id)] + for subscription in subscriptions: + processor_uuid = subscription.get('processor_uuid') + if not processor_uuid or processor_uuid in seen or not subscription.get('enabled', True): + continue + seen.add(processor_uuid) + tasks.append( + self._run_event_delivery( + self._dispatch_plugin_subscription, event, adapter, processor_uuid, execution_id + ) + ) + # Start all deliveries together. A slow or failed subscriber cannot block + # another subscriber or the primary route from receiving the event. + results = await asyncio.gather(*tasks, return_exceptions=True) + except BaseException as exc: + self._record_platform_event(event, adapter, 'failed', f'{type(exc).__name__}: {exc}', node) + await self._persist_monitoring_ingress(event, adapter, execution_id, 'failed', [exc]) + raise + + # Recorded after handling so the inbound stage carries the real outcome + # instead of a hardcoded success. + outcome, error_detail = self._summarize_event_delivery(results) for result in results: if isinstance(result, BaseException): - await self.logger.error(f'Event delivery failed: {result}') + try: + await self.logger.error(f'Event delivery failed: {result}') + except Exception: + pass + self._record_platform_event(event, adapter, outcome, error_detail, node) + await self._persist_monitoring_ingress( + event, + adapter, + execution_id, + {'success': 'completed', 'skipped': 'ignored'}.get(outcome, 'failed'), + results, + ) - async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid): + async def _run_event_delivery(self, handler, *args, **kwargs): + # gather creates child tasks. Each delivery needs its own database scope; + # copying the listener's ContextVar does not transfer task ownership. + persistence = getattr(getattr(self, 'ap', None), 'persistence_mgr', None) + if getattr(getattr(persistence, 'mode', None), 'value', None) == 'cloud_runtime': + tenant_scope = getattr(persistence, 'tenant_scope', None) + if not callable(tenant_scope): + raise RuntimeError('Cloud event deliveries require a tenant scope') + async with tenant_scope(self.workspace_uuid): + return await handler(*args, **kwargs) + return await handler(*args, **kwargs) + + async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid, execution_id=None): event_type = event.type event_binding = { 'id': f'plugin_processor:{processor_uuid}', @@ -933,20 +1055,27 @@ class RuntimeBot: patterns = descriptor.supported_event_patterns if not patterns or not self._agent_supports_event_type(patterns, event_type): return - agent = {**agent, 'supported_event_patterns': patterns} - return await self._dispatch_eba_event_to_processor(event, adapter, event_binding, agent) except Exception as exc: - return await self._record_event_route_trace( - event_type=event_type, - status='failed', - level='error', - binding=event_binding, - target_type='event_processor', - target_uuid=processor_uuid, - failure_code='runner_failed', - reason=str(exc), - text=f'Plugin processor {processor_uuid} failed: {exc}', - ) + from ..telemetry import trace as trace_mod + + # No matching declaration was resolved: log locally without + # reporting a failed delivery for every unrelated incoming event. + with trace_mod.suppress_reporting(): + return await self._record_event_route_trace( + event_type=event_type, + status='failed', + level='error', + binding=event_binding, + target_type='event_processor', + target_uuid=processor_uuid, + failure_code='runner_failed', + reason=str(exc), + text=f'Plugin processor {processor_uuid} failed: {exc}', + ) + agent = {**agent, 'supported_event_patterns': patterns} + return await self._dispatch_eba_event_to_processor( + event, adapter, event_binding, agent, execution_id=execution_id + ) async def _dispatch_eba_event_to_processor( self, @@ -954,6 +1083,7 @@ class RuntimeBot: adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, event_binding: dict | None = None, agent: dict | None = None, + execution_id: str | None = None, ) -> dict[str, typing.Any]: event_type = getattr(event, 'type', None) or event.__class__.__name__ @@ -969,6 +1099,12 @@ class RuntimeBot: ) target_type = event_binding.get('target_type') + from ..telemetry import trace as trace_mod + + if target_type != 'discard': + # Matching precedes target lookup/execution, so genuine delivery + # failures remain observable even when the target is unavailable. + trace_mod.admit() await self._record_event_route_trace( event_type=event_type, status='matched', @@ -981,7 +1117,7 @@ class RuntimeBot: if isinstance(event, platform_events.MessageReceivedEvent): from ..telemetry import trace as trace_mod - with trace_mod.scope(route_ref=self._route_ref(event_binding)): + with trace_mod.suppress_reporting(), trace_mod.scope(route_ref=self._route_ref(event_binding)): await self._dispatch_eba_message_to_pipeline( event, adapter, @@ -1101,9 +1237,13 @@ class RuntimeBot: text=f'Event {event_type} target agent has no runner: {target_uuid}', ) - envelope = self._eba_event_to_agent_envelope(event, adapter) + envelope = self._eba_event_to_agent_envelope(event, adapter, execution_id) if target_type == 'event_processor': envelope.data = event.model_dump(mode='json', exclude={'source_platform_object', 'legacy_event'}) + if isinstance(event, platform_events.MessageReceivedEvent): + # Use the SDK wire format for Source timestamps and nested chains. + # Pydantic's enclosing event serializer bypasses model_dump overrides. + envelope.data['message_chain'] = event.message_chain.model_dump() from ..telemetry import trace as trace_mod try: @@ -1318,6 +1458,7 @@ class RuntimeBot: self, event: platform_events.PlatformSpecificEvent, adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, + execution_id: str | None = None, ) -> None: """Consume a platform interaction callback and resume its original processor.""" data = event.data if isinstance(event.data, dict) else {} @@ -1343,13 +1484,18 @@ class RuntimeBot: conversation_id=conversation_id, actor_id=actor_id, ) + if record['processor_type'] not in {'agent', 'pipeline'}: + raise ValueError(f'Unsupported interaction processor type: {record["processor_type"]}') + from ..telemetry import trace as trace_mod + + # Only a validated callback resolving an original processor is eligible. + # Admit before acknowledgement and target lookup to preserve failures. + trace_mod.admit() await self.ap.agent_run_orchestrator.interaction_manager.acknowledge_submission(record, adapter) if record['processor_type'] == 'agent': - await self._resume_agent_interaction(record, event, adapter, actor_id) + await self._resume_agent_interaction(record, event, adapter, actor_id, execution_id) return - if record['processor_type'] != 'pipeline': - raise ValueError(f'Unsupported interaction processor type: {record["processor_type"]}') pipeline = await self.ap.pipeline_service.get_pipeline( self.execution_context, @@ -1428,6 +1574,7 @@ class RuntimeBot: event: platform_events.PlatformSpecificEvent, adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, actor_id: str | None, + execution_id: str | None = None, ) -> None: """Resume the exact independent Agent that produced an interaction.""" agent = await self.ap.agent_service.get_agent( @@ -1480,7 +1627,7 @@ class RuntimeBot: delivery_data['interactions'] = interaction_capabilities envelope = AgentEventEnvelope( - event_id=f'interaction:{record["id"]}:{uuid.uuid4()}', + event_id=execution_id or f'interaction:{record["id"]}:{uuid.uuid4()}', event_type='interaction.submitted', event_time=int(event.timestamp) if event.timestamp else int(time.time()), source='platform', diff --git a/src/langbot/pkg/plugin/connector.py b/src/langbot/pkg/plugin/connector.py index dc9aa9bec..905fc459c 100644 --- a/src/langbot/pkg/plugin/connector.py +++ b/src/langbot/pkg/plugin/connector.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import contextlib import contextvars +import datetime import hashlib import json import math @@ -21,6 +22,7 @@ from langbot_plugin.api.entities.builtin.pipeline.query import provider_session from ..core import app from . import handler +from . import runtime_ops from .errors import ( PluginRuntimeNotConnectedError, PluginInstallationFailedError, @@ -54,7 +56,9 @@ from langbot_plugin.api.entities.builtin.command import ( errors as command_errors, ) from langbot_plugin.runtime.plugin.mgr import PluginInstallSource +from langbot_plugin.runtime.io.handler import SHARED_WORKER_FILE_STORAGE_DIR from langbot_plugin.runtime.security import ( + PLUGIN_FILE_STORAGE_DIR_ENV, PLUGIN_RUNTIME_CONTROL_TOKEN_ENV, PLUGIN_RUNTIME_CONTROL_TOKEN_HEADER, validate_runtime_secret, @@ -97,6 +101,12 @@ _HEARTBEAT_FAILURE_THRESHOLD = 3 _RECONNECT_MAX_DELAY_SEC = 60.0 +def _utc_now_iso() -> str: + """Return the current UTC time as a second-precision RFC 3339 string.""" + + return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + + async def _read_httpx_response_limited( response: httpx.Response, *, @@ -223,6 +233,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self.runtime_profile: typing.Literal['oss_dev', 'shared'] = ( 'shared' if getattr(getattr(ap, 'deployment', None), 'mode', 'oss') == 'cloud' else 'oss_dev' ) + self._align_plugin_file_transfer_root() self.runtime_identity: RuntimeIdentity | None = None self._runtime_id = self._build_runtime_id() self.worker_policy: PluginWorkerPolicy | None = None @@ -239,6 +250,32 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self._reconnect_task: asyncio.Task | None = None self._generation = 0 self._connected = asyncio.Event() + self.runtime_ops_reporter = runtime_ops.RuntimeOpsReporter(ap) + self._reconcile_summary: dict[str, Any] = { + 'last_started_at': None, + 'last_duration_ms': None, + 'last_ok': None, + 'failed_installations': 0, + 'missing_artifacts': 0, + } + + def _align_plugin_file_transfer_root(self) -> None: + """Use the Runtime's plugin file-transfer root when running shared. + + The SDK binds its transfer root to one directory inode with mode 0700 and + picks the default per process: a shared worker uses + ``SHARED_WORKER_FILE_STORAGE_DIR``, every other process falls back to + ``data/temp/lbp``. A host talking to a shared Runtime must use the same root, + otherwise plugin icons and assets die with "Invalid file transfer + capability" (the icon route then answers 500/404 instead of the image). + An explicit operator value always wins. + """ + + if self.runtime_profile != 'shared': + return + if os.environ.get(PLUGIN_FILE_STORAGE_DIR_ENV): + return + os.environ[PLUGIN_FILE_STORAGE_DIR_ENV] = SHARED_WORKER_FILE_STORAGE_DIR @staticmethod def _build_runtime_id() -> str: @@ -618,6 +655,10 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): 'installation_uuid': installation_uuid, 'error_code': error_code, 'message': message, + # When the failure actually happened. The ops reporter must publish this + # instead of its own sampling time, otherwise every sample re-stamps the + # same failure with the newest report time. + 'failed_at': _utc_now_iso(), } self._installation_failures[installation_uuid] = failure self.ap.logger.error( @@ -659,9 +700,15 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): 'installation_uuid': installation_uuid, 'error_code': error_code, 'message': message, + # See _raise_apply_failure: never let the reporter substitute its own + # sampling time for the moment the failure was observed. + 'failed_at': _utc_now_iso(), } failures[installation_uuid] = failure - if self._installation_failures.get(installation_uuid) != failure: + previous = self._installation_failures.get(installation_uuid) or {} + if {key: value for key, value in previous.items() if key != 'failed_at'} != { + key: value for key, value in failure.items() if key != 'failed_at' + }: self.ap.logger.error( 'Plugin installation %s failed during reconcile [%s]: %s', installation_uuid, @@ -674,6 +721,21 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self._installation_failures.pop(installation_uuid, None) self._installation_failures.update(failures) + def _record_reconcile_outcome(self, started_at: str, duration_ms: float, result: dict[str, Any]) -> None: + """Expose the last reconcile outcome for the runtime ops sample.""" + + failed = result.get('failed_installations') + missing = result.get('missing_artifacts') + failed_installations = len(failed) if isinstance(failed, list) else 0 + missing_artifacts = len(missing) if isinstance(missing, list) else 0 + self._reconcile_summary = { + 'last_started_at': started_at, + 'last_duration_ms': int(max(duration_ms, 0.0)), + 'last_ok': failed_installations == 0 and missing_artifacts == 0, + 'failed_installations': failed_installations, + 'missing_artifacts': missing_artifacts, + } + async def _repair_reconcile_missing_artifacts( self, desired_states: dict[str, PluginInstallationDesiredState], @@ -754,12 +816,19 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): reconcile_timeout_seconds = max( 300.0, self._runtime_connect_timeout(self.ap.instance_config.data.get('plugin', {})) ) + reconcile_started_at = _utc_now_iso() + reconcile_started_monotonic = time.monotonic() result = await runtime_handler.reconcile_plugin_installations( tuple(self._known_desired_states.values()), timeout=reconcile_timeout_seconds, ) await self._repair_reconcile_missing_artifacts(self._known_desired_states, result) self._record_reconcile_failures(self._known_desired_states, result) + self._record_reconcile_outcome( + reconcile_started_at, + (time.monotonic() - reconcile_started_monotonic) * 1000.0, + result, + ) async def reconcile_projected_workspaces( self, @@ -775,6 +844,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): runtime_handler = self._runtime_handler() started_at = time.monotonic() + started_at_iso = _utc_now_iso() async with self._state_lock: all_states: dict[str, PluginInstallationDesiredState] = {} workspace_installations: dict[str, set[str]] = {} @@ -801,6 +871,11 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): ) await self._repair_reconcile_missing_artifacts(all_states, result) self._record_reconcile_failures(all_states, result) + self._record_reconcile_outcome( + started_at_iso, + (time.monotonic() - started_at) * 1000.0, + result, + ) for installation_uuid, previous in tuple(self._known_desired_states.items()): if installation_uuid not in all_states: runtime_handler.unregister_installation_binding(previous.binding) @@ -1068,6 +1143,8 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): if self.heartbeat_task is None or self.heartbeat_task.done(): self.heartbeat_task = asyncio.create_task(self.heartbeat_loop()) + self.runtime_ops_reporter.start() + def schedule_reconnect(self) -> None: if self._closing or not self.is_enable_plugin: return @@ -1130,6 +1207,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): await asyncio.gather(self.heartbeat_task, return_exceptions=True) self.heartbeat_task = None await self._stop_transport() + await self.runtime_ops_reporter.stop() await self._close_managed_subprocess() @staticmethod diff --git a/src/langbot/pkg/plugin/handler.py b/src/langbot/pkg/plugin/handler.py index c8d12c340..5b2356802 100644 --- a/src/langbot/pkg/plugin/handler.py +++ b/src/langbot/pkg/plugin/handler.py @@ -1,5 +1,7 @@ from __future__ import annotations +from contextlib import aclosing + import asyncio import inspect import typing @@ -1582,22 +1584,25 @@ class RuntimeConnectionHandler(handler.Handler): remove_think = _resolve_remove_think(data, query) effective_funcs = funcs_obj if 'func_call' in (llm_model.model_entity.abilities or []) else [] - async for chunk in llm_model.provider.invoke_llm_stream( - query=query, - model=llm_model, - messages=messages_obj, - funcs=effective_funcs, - extra_args=effective_extra_args, - remove_think=remove_think, - execution_context=execution_context, - ): - if chunk is None: - continue - yield handler.ActionResponse.success( - data={ - 'chunk': chunk.model_dump(), - }, + async with aclosing( + llm_model.provider.invoke_llm_stream( + query=query, + model=llm_model, + messages=messages_obj, + funcs=effective_funcs, + extra_args=effective_extra_args, + remove_think=remove_think, + execution_context=execution_context, ) + ) as owned_stream: + async for chunk in owned_stream: + if chunk is None: + continue + yield handler.ActionResponse.success( + data={ + 'chunk': chunk.model_dump(), + }, + ) usage = _pop_query_llm_usage(query) if usage is not None: yield handler.ActionResponse.success( diff --git a/src/langbot/pkg/plugin/runtime_ops.py b/src/langbot/pkg/plugin/runtime_ops.py new file mode 100644 index 000000000..0c96820a0 --- /dev/null +++ b/src/langbot/pkg/plugin/runtime_ops.py @@ -0,0 +1,390 @@ +"""Cloud Edition runtime ops sample reporter. + +Aggregates this instance's plugin Runtime health (probed from the Runtime +``/healthz`` endpoint) together with the connector's own desired-state and +failure registries, then pushes one compact sample to the Space control +plane where the internal ops dashboard stores and renders it. + +This is a Cloud-only ops channel, deliberately separate from product +telemetry so ``space.disable_telemetry`` cannot blind operations. Sending is +best-effort: a failed probe or upload never raises out of the reporter. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import datetime +import math +import os +import typing +from urllib.parse import urlsplit, urlunsplit + +import httpx + +from ..utils import constants, httpclient + +_CONTROL_PLANE_TOKEN_ENV = 'LANGBOT_SPACE_CONTROL_PLANE_TOKEN' +_RUNTIME_OPS_PATH = '/api/v1/cloud/v2/internal/runtime-ops' +_DEFAULT_INTERVAL_SECONDS = 30.0 +_DEFAULT_TIMEOUT_SECONDS = 10.0 +_DEFAULT_MAX_FAILURES = 5 +_MAX_FAILURE_MESSAGE_CHARS = 512 + + +def _utc_now_iso() -> str: + """Return the current UTC time as a second-precision RFC 3339 string.""" + + return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + + +def _coerce_bool(value: typing.Any, default: bool) -> bool: + return value if isinstance(value, bool) else default + + +def _coerce_positive_float(value: typing.Any, default: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return default + value = float(value) + if not math.isfinite(value) or value <= 0: + return default + return value + + +def _coerce_positive_int(value: typing.Any, default: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + return default + return value + + +def _sanitize_failure_message(value: typing.Any) -> str: + """Flatten and bound one failure message for the wire. + + Mirrors the connector's own sanitization so ops never receives control + characters, unbounded payloads, or newline-injected log content. + """ + + candidate = str(value or '').replace('\r', ' ').replace('\n', ' ').strip() + return candidate[:_MAX_FAILURE_MESSAGE_CHARS] + + +class RuntimeOpsReporter: + """Collect and push one runtime ops sample per interval. + + The reporter never raises out of its send loop; failures are counted and + surfaced through :meth:`get_stats`. + """ + + def __init__(self, ap: typing.Any): + self.ap = ap + instance_data = getattr(getattr(ap, 'instance_config', None), 'data', None) + if not isinstance(instance_data, dict): + instance_data = {} + plugin_config = instance_data.get('plugin', {}) + if not isinstance(plugin_config, dict): + plugin_config = {} + config = plugin_config.get('runtime_ops', {}) + if not isinstance(config, dict): + config = {} + self.config = config + + deployment_mode = getattr(getattr(ap, 'deployment', None), 'mode', 'oss') + cloud_edition = deployment_mode == 'cloud' + # Cloud-only feature: an explicit flag may turn it off, but a + # non-cloud deployment is never enabled. + self.enabled = cloud_edition and _coerce_bool(config.get('enabled'), True) + self.interval_seconds = _coerce_positive_float(config.get('interval_seconds'), _DEFAULT_INTERVAL_SECONDS) + self.timeout_seconds = _coerce_positive_float(config.get('timeout_seconds'), _DEFAULT_TIMEOUT_SECONDS) + self.max_failures = _coerce_positive_int(config.get('max_failures'), _DEFAULT_MAX_FAILURES) + + self.health_url = self._derive_health_url(plugin_config.get('runtime_ws_url')) + self.space_url = str(instance_data.get('space', {}).get('url', '') or '').rstrip('/') + self._token = str(os.environ.get(_CONTROL_PLANE_TOKEN_ENV, '') or '').strip() + + if self.enabled and not self.health_url: + self._log_info('Plugin runtime ops reporter is disabled: no plugin runtime WebSocket URL is configured') + self.enabled = False + if self.enabled and not self.space_url: + self._log_info('Plugin runtime ops reporter is disabled: no Space URL is configured') + self.enabled = False + + self._task: asyncio.Task | None = None + self._client: httpx.AsyncClient | None = None + self._sent_total = 0 + self._dropped_total = 0 + self._last_error: str | None = None + + def _log_info(self, message: str) -> None: + logger = getattr(self.ap, 'logger', None) + info = getattr(logger, 'info', None) + if callable(info): + info(message) + + @staticmethod + def _derive_health_url(runtime_ws_url: typing.Any) -> str | None: + """Derive the Runtime ``/healthz`` URL from the control WebSocket URL.""" + + if not isinstance(runtime_ws_url, str): + return None + candidate = runtime_ws_url.strip() + if not candidate: + return None + try: + parts = urlsplit(candidate) + port = parts.port + except ValueError: + return None + if parts.scheme not in ('ws', 'wss') or not parts.hostname: + return None + scheme = 'https' if parts.scheme == 'wss' else 'http' + netloc = parts.hostname if port is None else f'{parts.hostname}:{port}' + return urlunsplit((scheme, netloc, '/healthz', '', '')) + + def start(self) -> None: + """Start the background sample loop once the control link is up.""" + + if not self.enabled: + return + if self._task is not None and not self._task.done(): + return + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + """Cancel the sample loop and release the HTTP client.""" + + task = self._task + self._task = None + if task is not None and task is not asyncio.current_task(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + client = self._client + self._client = None + if client is not None and not client.is_closed: + with contextlib.suppress(Exception): + await client.aclose() + + def get_stats(self) -> dict[str, typing.Any]: + """Return in-memory counters for the instance liveness surface.""" + + return { + 'sent_total': self._sent_total, + 'dropped_total': self._dropped_total, + 'last_error': self._last_error, + } + + @contextlib.asynccontextmanager + async def _client_context(self): + if self._client is None or self._client.is_closed: + self._client = httpx.AsyncClient( + timeout=httpx.Timeout(self.timeout_seconds), + event_hooks=httpclient.httpx_response_limit_hooks(), + ) + yield self._client + + async def _run(self) -> None: + while True: + try: + await self._collect_and_send() + except asyncio.CancelledError: + raise + except Exception as exc: # never let the ops channel crash the connector + self._last_error = _sanitize_failure_message(f'sample cycle failed: {exc}') + await asyncio.sleep(self.interval_seconds) + + async def _collect_and_send(self) -> None: + health = await self._fetch_health() + if health is None: + self._dropped_total += 1 + return + await self._post(self.build_payload(health)) + + async def _fetch_health(self) -> dict[str, typing.Any] | None: + """Probe the Runtime ``/healthz`` endpoint, returning its verbatim JSON.""" + + try: + async with self._client_context() as client: + response = await asyncio.wait_for( + client.get(self.health_url), + timeout=self.timeout_seconds, + ) + if response.status_code >= 400: + self._last_error = f'health probe returned status {response.status_code}' + return None + payload = await httpclient.parse_json_response(response) + if not isinstance(payload, dict): + self._last_error = 'health probe returned a non-object payload' + return None + self._last_error = None + return payload + except asyncio.TimeoutError: + self._last_error = 'health probe timed out' + return None + except Exception as exc: + self._last_error = _sanitize_failure_message(f'health probe failed: {exc}') + return None + + async def _post(self, payload: dict[str, typing.Any]) -> None: + """Best-effort POST of one sample; never raises.""" + + url = f'{self.space_url}{_RUNTIME_OPS_PATH}' + headers = {'Authorization': f'Bearer {self._token}'} + try: + async with self._client_context() as client: + response = await asyncio.wait_for( + client.post(url, json=payload, headers=headers), + timeout=self.timeout_seconds, + ) + if response.status_code >= 400: + body = await httpclient.response_text(response, max_chars=200) + self._dropped_total += 1 + self._last_error = f'POST {url} returned status {response.status_code} - {body}' + return + self._sent_total += 1 + self._last_error = None + except asyncio.TimeoutError: + self._dropped_total += 1 + self._last_error = f'POST {url} timed out' + except Exception as exc: + self._dropped_total += 1 + self._last_error = _sanitize_failure_message(f'POST {url} failed: {exc}') + + def build_payload(self, health: dict[str, typing.Any]) -> dict[str, typing.Any]: + """Assemble one ops sample in the frozen Core -> Space contract shape.""" + + connector = getattr(self.ap, 'plugin_connector', None) + instance_data = getattr(getattr(self.ap, 'instance_config', None), 'data', None) + if not isinstance(instance_data, dict): + instance_data = {} + system_config = instance_data.get('system', {}) + if not isinstance(system_config, dict): + system_config = {} + instance_uuid = str(system_config.get('instance_id', '') or '') or str(constants.instance_id or '') + edition = str(system_config.get('edition', 'oss') or 'oss') + return { + 'event': 'runtime_ops_sample', + 'kind': 'aggregate', + 'instance_uuid': instance_uuid, + 'generated_at': _utc_now_iso(), + 'core': { + 'version': constants.semantic_version, + 'edition': edition, + 'connected': self._runtime_connected(connector), + 'runtime_profile': getattr(connector, 'runtime_profile', 'oss_dev'), + 'worker_policy': self._worker_policy(connector), + 'desired_states': self._desired_states(connector), + 'reconcile': self._reconcile_summary(connector), + }, + 'health': health, + 'failures': self._failures(connector), + } + + @staticmethod + def _runtime_connected(connector: typing.Any) -> bool: + available = getattr(connector, '_runtime_available', None) + if not callable(available): + return False + try: + return bool(available()) + except Exception: + return False + + @staticmethod + def _worker_policy(connector: typing.Any) -> dict[str, typing.Any] | None: + policy = getattr(connector, 'worker_policy', None) + if policy is None: + return None + model_dump = getattr(policy, 'model_dump', None) + if callable(model_dump): + return model_dump() + if isinstance(policy, dict): + return dict(policy) + return None + + @staticmethod + def _desired_states(connector: typing.Any) -> dict[str, typing.Any]: + states = getattr(connector, '_known_desired_states', None) + if not isinstance(states, dict): + states = {} + enabled = 0 + by_mode: dict[str, int] = {} + for state in states.values(): + if bool(getattr(state, 'enabled', False)): + enabled += 1 + mode = getattr(getattr(state, 'execution_mode', None), 'value', None) + mode = str(mode) if isinstance(mode, str) and mode else 'dedicated' + by_mode[mode] = by_mode.get(mode, 0) + 1 + return {'total': len(states), 'enabled': enabled, 'by_mode': by_mode} + + @staticmethod + def _reconcile_summary(connector: typing.Any) -> dict[str, typing.Any]: + summary = getattr(connector, '_reconcile_summary', None) + if isinstance(summary, dict): + return dict(summary) + return { + 'last_started_at': None, + 'last_duration_ms': None, + 'last_ok': None, + 'failed_installations': 0, + 'missing_artifacts': 0, + } + + def _failures(self, connector: typing.Any) -> list[dict[str, typing.Any]]: + records = getattr(connector, '_installation_failures', None) + if not isinstance(records, dict): + return [] + states = getattr(connector, '_known_desired_states', None) + if not isinstance(states, dict): + states = {} + identities = self._plugin_identities(connector) + observed_at = _utc_now_iso() + failures: list[dict[str, typing.Any]] = [] + for installation_uuid in list(records)[: self.max_failures]: + record = records.get(installation_uuid) + if not isinstance(record, dict): + continue + state = states.get(installation_uuid) + binding = getattr(state, 'binding', None) + mode = getattr(getattr(state, 'execution_mode', None), 'value', None) + mode = str(mode) if isinstance(mode, str) and mode else 'dedicated' + plugin_author, plugin_name = identities.get(str(installation_uuid), ('', '')) + failures.append( + { + 'workspace_uuid': str(getattr(binding, 'workspace_uuid', '') or ''), + 'installation_uuid': str(installation_uuid), + 'plugin_author': plugin_author, + 'plugin_name': plugin_name, + 'execution_mode': mode, + 'error_code': str(record.get('error_code', '') or ''), + 'message': _sanitize_failure_message(record.get('message')), + # The moment the failure was observed by the connector. Falling back to + # the sampling time would make a stale failure look freshly broken on + # every report. + 'observed_at': str(record.get('failed_at') or observed_at), + } + ) + return failures + + @staticmethod + def _plugin_identities(connector: typing.Any) -> dict[str, tuple[str, str]]: + """Read the connector handler's installation identity registry. + + The Runtime connection handler binds every desired installation to + its plugin identity; the connector's failure records themselves only + carry the installation UUID, so the reporter joins them here. + """ + + handler = getattr(connector, 'handler', None) + bindings = getattr(handler, '_installation_bindings', None) + if not isinstance(bindings, dict): + return {} + identities: dict[str, tuple[str, str]] = {} + for installation_uuid, entry in bindings.items(): + try: + identity = entry[1] + except (TypeError, IndexError, KeyError): + continue + identities[str(installation_uuid)] = ( + str(getattr(identity, 'plugin_author', '') or ''), + str(getattr(identity, 'plugin_name', '') or ''), + ) + return identities diff --git a/src/langbot/pkg/provider/modelmgr/requester.py b/src/langbot/pkg/provider/modelmgr/requester.py index 069c9981e..99f6e885d 100644 --- a/src/langbot/pkg/provider/modelmgr/requester.py +++ b/src/langbot/pkg/provider/modelmgr/requester.py @@ -1,6 +1,9 @@ from __future__ import annotations +from contextlib import aclosing + import abc +import asyncio import typing import time @@ -204,15 +207,18 @@ class RuntimeProvider: try: # Stream the response - async for chunk in self.requester.invoke_llm_stream( - query=query, - model=model, - messages=messages, - funcs=funcs, - extra_args=extra_args, - remove_think=remove_think, - ): - yield chunk + async with aclosing( + self.requester.invoke_llm_stream( + query=query, + model=model, + messages=messages, + funcs=funcs, + extra_args=extra_args, + remove_think=remove_think, + ) + ) as owned_stream: + async for chunk in owned_stream: + yield chunk # Extract usage from stream if available (stored by LiteLLM requester) if query: if query.variables is None: @@ -223,6 +229,10 @@ class RuntimeProvider: input_tokens = usage_info.get('prompt_tokens', 0) output_tokens = usage_info.get('completion_tokens', 0) del query.variables[STREAM_USAGE_QUERY_VARIABLE] + except (asyncio.CancelledError, GeneratorExit): + status = 'error' + error_message = 'Streaming generation cancelled by consumer' + raise except Exception as e: status = 'error' error_message = str(e) diff --git a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py index 0bbaa1db5..23a69bea8 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py +++ b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py @@ -3,6 +3,7 @@ from __future__ import annotations import typing +import asyncio import litellm from litellm import acompletion, aembedding, arerank @@ -1245,6 +1246,7 @@ class LiteLLMRequester(requester.ProviderAPIRequester): reasoning_started = False reasoning_closed = False thinking_blocks_state: list[dict[str, typing.Any]] = [] + response = None try: response = await acompletion(**args) @@ -1372,6 +1374,16 @@ class LiteLLMRequester(requester.ProviderAPIRequester): except Exception as e: self._handle_litellm_error(e) + finally: + # Closing a downstream async generator must release the provider's + # HTTP stream immediately, including cancellation while at yield. + close = getattr(response, 'aclose', None) + if callable(close): + try: + await asyncio.wait_for(close(), timeout=5.0) + except Exception as exc: + self.ap.logger.debug(f'Failed to close LLM stream: {type(exc).__name__}') + async def invoke_embedding( self, model: requester.RuntimeEmbeddingModel, diff --git a/src/langbot/pkg/provider/tools/loaders/mcp.py b/src/langbot/pkg/provider/tools/loaders/mcp.py index 48501292b..7b5a83afd 100644 --- a/src/langbot/pkg/provider/tools/loaders/mcp.py +++ b/src/langbot/pkg/provider/tools/loaders/mcp.py @@ -1464,9 +1464,6 @@ class RuntimeMCPSession: if query is None: return try: - from langbot.pkg.telemetry import features as telemetry_features - - telemetry_features.increment(query, 'mcp_resource_reads', envelope.get('source') or 'unknown') query.variables.setdefault(MCP_RESOURCE_TRACE_QUERY_KEY, []).append( { 'server_name': envelope.get('server_name'), diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index b09dff9d8..97e554f42 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -2,6 +2,7 @@ from __future__ import annotations import typing import time +import asyncio import inspect from typing import TYPE_CHECKING @@ -448,6 +449,19 @@ class ToolManager: start_time = time.perf_counter() try: result = await invoke() + except asyncio.CancelledError: + duration_ms = int((time.perf_counter() - start_time) * 1000) + await self._record_tool_call( + name=name, + source=source, + parameters=parameters, + query=query, + duration_ms=duration_ms, + status='error', + error_message='cancelled', + ) + self._trace_tool_call(source=source, name=name, query=query, outcome='cancelled', error='cancelled') + raise except Exception as e: duration_ms = int((time.perf_counter() - start_time) * 1000) await self._record_tool_call( @@ -459,6 +473,14 @@ class ToolManager: status='error', error_message=str(e), ) + outcome = 'timeout' if isinstance(e, TimeoutError) else 'failed' + self._trace_tool_call( + source=source, + name=name, + query=query, + outcome=outcome, + error=str(e) or type(e).__name__, + ) raise duration_ms = int((time.perf_counter() - start_time) * 1000) @@ -471,8 +493,60 @@ class ToolManager: status='success', result=result, ) + self._trace_tool_call(source=source, name=name, query=query, outcome='success') return result + def _trace_tool_call( + self, + *, + source: str, + name: str, + query: pipeline_query.Query, + outcome: str, + error: str = '', + ) -> None: + """Report one tool invocation as its own step of the execution chain. + + Every tool call a Run makes - sandbox, skill, plugin or MCP - is the + observable action of that Run, so the chain shows it under the step that + issued it instead of collapsing the Run into a single opaque node. + + An Agent's tool call is executed for the Runtime's RPC task, which never + inherited the chain context. The Run id the caller already carries is the + alias that task's record resolves the owning execution through. + """ + # Imported here: telemetry pulls the application, and the tool manager is + # constructed while the application is still assembling. + from ...telemetry.execution import record as record_execution + from ...telemetry.platform import processing_mode + + try: + record_execution( + self.ap, + get_query_execution_context(query), + execution_id=self._trace_execution_id(query), + family='tool', + operation=name, + adapter=source, + mode=processing_mode.get(), + outcome=outcome, + error=error, + ) + except Exception as e: + self.ap.logger.warning(f'Failed to record tool call trace: {e}') + + @staticmethod + def _trace_execution_id(query: pipeline_query.Query) -> str: + """The execution chain this call belongs to, when the caller knows it. + + ``''`` lets the emitter fall back to the task's own trace context, which + is what a Pipeline-lane tool call has. + """ + session = getattr(query, '_agent_run_session', None) + if not isinstance(session, dict): + return '' + return str(session.get('run_id') or '').strip() + async def execute_func_call( self, name: str, @@ -480,8 +554,6 @@ class ToolManager: query: pipeline_query.Query, source_ref: ToolSourceRef | None = None, ) -> typing.Any: - from langbot.pkg.telemetry import features as telemetry_features - source_ref = source_ref or self.get_query_tool_source(query, name) if source_ref is not None: execution_context = get_query_execution_context(query) @@ -534,7 +606,6 @@ class ToolManager: return await loader.invoke_tool(name, parameters, query, source_id=source_id) return await loader.invoke_tool(name, parameters, query) - telemetry_features.increment(query, 'tool_calls', telemetry_source) return await self._invoke_tool_with_monitoring( source=telemetry_source, name=name, @@ -547,7 +618,6 @@ class ToolManager: await self._bind_plugin_workspace(execution_context) sandbox_available = await self._workspace_sandbox_available(execution_context) if sandbox_available and await self.native_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'native') return await self._invoke_tool_with_monitoring( source='native', name=name, @@ -556,7 +626,6 @@ class ToolManager: invoke=lambda: self.native_tool_loader.invoke_tool(name, parameters, query), ) if await self.plugin_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'plugin') return await self._invoke_tool_with_monitoring( source='plugin', name=name, @@ -565,7 +634,6 @@ class ToolManager: invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query), ) if await self.mcp_tool_loader.has_tool(execution_context, name): - telemetry_features.increment(query, 'tool_calls', 'mcp') return await self._invoke_tool_with_monitoring( source='mcp', name=name, @@ -574,7 +642,6 @@ class ToolManager: invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query), ) if sandbox_available and await self.skill_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'skill') return await self._invoke_tool_with_monitoring( source='skill', name=name, diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index ff5ef28ff..d914deadd 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -1,58 +1,163 @@ -"""Bounded, content-free execution counters for the existing telemetry sender. +"""Point-wise execution records for the existing telemetry sender. -This module reports observations only. Coverage catalogs and acceptance rules -belong to Space. Aggregation keys include both immutable execution identities. +One execution is reported as a chain of independent records that all carry the +same ``event_id``: one ``execution_node`` per workflow node, emitted the moment +that node completes, plus exactly one ``execution_chain`` emitted when the +execution closes. The sender never assembles a trace locally, so there is no +per-trace buffer, no TTL and no per-process trace cap; memory is bounded by the +outbound record buffer, the flush batch size and the small cross-task registry. -Two shapes share one payload type: +Platform events are reportable only after a business route, matching plugin +subscription or validated interaction callback admits the execution. Pending +events enqueue neither nodes nor closures, even on failure or in Cloud mode. +Independent API/WebUI executions remain eligible without a Bot route. -* window counters, aggregated by ``(family, operation, mode, adapter, runner, - outcome, synthetic)`` — unchanged coverage reporting; -* per-event traces, one bounded payload per traced event, keyed by - ``query_id = trace_id`` so Space can fetch a whole chain by primary identity. - -Traces are additive: they never alter the counters, and both are bounded in -memory, batch size and send concurrency. +Sampling is decided exactly once per event, at ingress, from a stable hash of +the execution identity, so every record of one execution carries the same +decision. A chain that broke is always emitted even when sampling excluded it, +which is why a chain may arrive with zero or partial nodes. """ from __future__ import annotations import asyncio import contextlib +import contextvars +import json +import re import time +import zlib from datetime import datetime, timezone -from uuid import uuid4 from . import trace as trace_mod from .identity import workspace_identity +from .resources import snapshot as resources_snapshot +from .resources import begin as resources_begin +from .resources import end as resources_end from .trace import TraceState -MAX_KEYS = 512 -MAX_BATCH = 32 -FLUSH_SECONDS = 60 MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'}) OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'}) +FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api', 'tool'}) +ORIGINS = frozenset({'platform', 'webui', 'api'}) +CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'}) +CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'}) -# Trace bounds: stages per trace, traces buffered per process, trace lifetime. -MAX_TRACES = 64 -TRACE_TTL_SECONDS = 120 +SCHEMA = 2 TRACE_MODES = frozenset({'off', 'failures', 'sampled', 'all'}) -DEFAULT_TRACE_MODE = 'sampled' +DEFAULT_TRACE_MODE = 'all' DEFAULT_TRACE_SAMPLE = 20 +# Sampling bounds: outbound record buffer, one flush batch, flush cadence. +MAX_BUFFERED_RECORDS = 512 +MAX_RECORDS_PER_FLUSH = 128 +MAX_FLUSH_BYTES = 200 * 1024 +DEFAULT_FLUSH_SECONDS = 180 +MAX_FLUSH_ATTEMPTS = 3 +FLUSH_RETRY_BACKOFF_SECONDS = 0.2 + +# Cross-task registry: execution id -> owning execution, bounded and time-limited. +MAX_REGISTRY_KEYS = 256 +REGISTRY_TTL_SECONDS = 300 + +# Failure details are operator-facing: keep the failure class and a short +# machine-ish prefix, never the free text a user or a model produced. +ERROR_LIMIT = 200 +ERROR_PREFIX_CHARS = 120 +ERROR_PREFIX_TOKENS = 12 +_ERROR_CLASS = re.compile(r'[A-Za-z_][\w.]*') + +# Set by cross-task call sites (plugin/RPC actions) that run without the +# ingress context but know which execution they belong to. +_execution_id: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_execution_id', default='') + + +def set_execution_id(execution_id: str) -> contextvars.Token: + return _execution_id.set(str(execution_id or '').strip()) + + +def reset_execution_id(token: contextvars.Token | None) -> None: + if token is None: + return + try: + _execution_id.reset(token) + except (ValueError, RuntimeError): + # A token from another context must never break execution. + pass + + +def current_execution_id() -> str: + return _execution_id.get() + + +def _short_prefix(text: str) -> str: + """Keep at most a short machine-ish prefix of one failure message.""" + if len(text) > ERROR_PREFIX_CHARS: + text = text[:ERROR_PREFIX_CHARS] + parts = text.split(' ') + if len(parts) > ERROR_PREFIX_TOKENS: + text = ' '.join(parts[:ERROR_PREFIX_TOKENS]) + return text + + +def _bounded_error(value, limit: int = ERROR_LIMIT) -> str: + """Bound and redact one failure detail before it leaves the process. + + The failure class and a short machine-ish prefix survive; control characters + are stripped, whitespace is collapsed and the result is capped, so a noisy + failure cannot smuggle user or model content into the payload. + """ + if value is None: + return '' + text = ''.join(ch if ch.isprintable() else ' ' for ch in str(value)) + text = ' '.join(text.split()) + if not text: + return '' + head, separator, rest = text.partition(':') + head = head.strip() + if separator and len(head) <= 80 and _ERROR_CLASS.fullmatch(head): + prefix = _short_prefix(rest.strip()) + text = f'{head}: {prefix}' if prefix else head + else: + text = _short_prefix(text) + return text[:limit] + + +def _duration_ms(started_at: str, ended_at: str) -> int: + """Wall-clock duration of one execution; 0 when the interval is unusable.""" + try: + delta = datetime.fromisoformat(ended_at) - datetime.fromisoformat(started_at) + return max(int(delta.total_seconds() * 1000), 0) + except Exception: + return 0 + class ExecutionCounters: + """Point-wise execution reporting: one record per node, one per closure.""" + def __init__(self, manager): self.manager = manager - self.pending: dict[tuple, dict] = {} + self.records: list[dict] = [] self.task: asyncio.Task | None = None + # Exposed health: records dropped or accepted by the sender. self.dropped = 0 - self.traces: dict[str, TraceState] = {} - self.trace_deadlines: dict[str, float] = {} - self.dropped_traces = 0 + self.sent = 0 + # Execution identity -> owning execution, for nodes recorded by tasks + # that no longer share the ingress context (plugin/RPC platform calls). + # It resolves identity and the sampling decision only; node payloads are + # never accumulated here. + self.registry: dict[str, TraceState] = {} + self.registry_deadlines: dict[str, float] = {} # ------------------------------------------------------------------ config def trace_mode(self) -> str: + # Cloud activity counts must not be computed from sampled successes. + # Telemetry opt-out is still enforced before recording observations. + from ..utils import constants + + if constants.edition == 'cloud': + return 'all' mode = str(self.manager.telemetry_config.get('execution_trace', DEFAULT_TRACE_MODE) or '').strip().lower() return mode if mode in TRACE_MODES else DEFAULT_TRACE_MODE @@ -63,6 +168,70 @@ class ExecutionCounters: sample = DEFAULT_TRACE_SAMPLE return sample if 1 <= sample <= 100000 else DEFAULT_TRACE_SAMPLE + def flush_seconds(self) -> int: + try: + seconds = int(self.manager.telemetry_config.get('telemetry_flush_seconds', DEFAULT_FLUSH_SECONDS)) + except (TypeError, ValueError): + seconds = DEFAULT_FLUSH_SECONDS + return seconds if 1 <= seconds <= 86400 else DEFAULT_FLUSH_SECONDS + + def health(self) -> dict: + """Real telemetry health for the instance liveness snapshot.""" + return { + 'records_buffered': len(self.records), + 'records_sent': self.sent, + 'records_dropped': self.dropped, + 'executions_in_flight': len({id(state) for state in self.registry.values()}), + } + + # ----------------------------------------------------------------- sampling + + def _sampling_decision(self, event_id: str) -> tuple[str, int, bool]: + """Decide once, deterministically, whether this event is reported. + + The identity is hashed instead of being parsed, so every id shape + (``platform:...``, uuids, debug ids) samples the same way. + """ + mode = self.trace_mode() + if mode == 'sampled': + denominator = self.trace_sample() + try: + sampled_in = zlib.crc32(str(event_id).encode('utf-8')) % denominator == 0 + except Exception: + sampled_in = False + return mode, denominator, sampled_in + # 'off' reports nothing, 'failures' keeps nodes out but still reports a + # broken chain, 'all' reports everything; denominator is 1 for all three. + return mode, 1, mode == 'all' + + def configure( + self, + state: TraceState, + context=None, + *, + debug: bool = False, + origin: str = 'platform', + identity=None, + ) -> None: + """Bind the execution identity, the sampling decision and the flags.""" + if state.configured: + return + state.configured = True + state.debug = bool(debug) + state.origin = origin if origin in ORIGINS else 'platform' + mode, denominator, sampled_in = self._sampling_decision(state.event_id) + state.mode = mode + state.denominator = denominator + # Debug chains are always emitted (unless reporting is off entirely). + state.emit_nodes = mode != 'off' and (state.debug or sampled_in) + if identity is not None: + state.identity = dict(identity) + elif context is not None: + try: + state.identity = workspace_identity(context) + except Exception: + pass + # --------------------------------------------------------------- recording def record( @@ -76,228 +245,257 @@ class ExecutionCounters: runner: str = '', outcome: str = 'unknown', synthetic: bool = False, + error: str = '', + execution_id: str | None = None, + node: str = '', ): try: - cfg = self.manager.telemetry_config - if not cfg or cfg.get('disable_telemetry', False) or not cfg.get('url'): + if not self._enabled() or trace_mod.reporting_suppressed(): return - if family not in {'platform_event', 'event_route', 'runner', 'platform_api'}: - return - if mode not in MODES or outcome not in OUTCOMES: + if family not in FAMILIES or mode not in MODES or outcome not in OUTCOMES: return # Only code-defined identifiers are accepted; never pass user values. - if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner)): + if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner, node)): return - identity = workspace_identity(context) state = trace_mod.current() - if state is not None: - self._record_trace_stage( - state, - identity=identity, - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=synthetic, - ) - key = ( - identity['instance_id'], - identity['workspace_uuid'], - family, - operation, - mode, - adapter, - runner, - outcome, - bool(synthetic), - ) - row = self.pending.get(key) - if row is None: - if len(self.pending) >= MAX_KEYS: - self.dropped += 1 - return - row = {'count': 0, 'first_seen': datetime.now(timezone.utc).isoformat()} - self.pending[key] = row - row['count'] = min(row['count'] + 1, 2147483647) - row['last_seen'] = datetime.now(timezone.utc).isoformat() - self._ensure_loop() - except Exception: - # Observability must never change execution behavior. - return - - def _record_trace_stage( - self, state: TraceState, *, identity, family, operation, mode, adapter, runner, outcome, synthetic - ) -> None: - if state.abandoned: - return - if state.trace_id not in self.traces: - # Never buffer traces this configuration would discard anyway. - if self.trace_mode() == 'off' or len(self.traces) >= MAX_TRACES: - self.dropped += 1 - state.abandoned = True + if state is None: + # Cross-task work has no inherited context; resolve the owning + # execution through the registry. It owns no step here, so the + # record attaches to the innermost step the chain has open - + # the step that caused this work. + lookup = str(execution_id or '').strip() or current_execution_id() + if lookup: + state = self._resolve(lookup) + if state is None or state.closed or not state.reportable: + # Every observation belongs to exactly one execution chain. + return + self.configure(state) + try: + identity = workspace_identity(context) + except Exception: + # A cross-task caller may only carry the Workspace id; reuse the + # identity already bound to the owning execution. + identity = state.identity or {} + if not identity.get('instance_id') or not identity.get('workspace_uuid'): return - self.traces[state.trace_id] = state - self.trace_deadlines[state.trace_id] = time.monotonic() + TRACE_TTL_SECONDS state.identity = identity - state.append( - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=bool(synthetic), - ) - def _ensure_loop(self) -> None: - if self.task is None or self.task.done(): - self.task = asyncio.create_task(self._loop()) - - # ------------------------------------------------------------------ traces - - def close_trace(self, state: TraceState, reason: str = 'event_done') -> None: - """Emit one trace payload when it matches the configured sample.""" - try: - if self.traces.pop(state.trace_id, None) is None: + ended_at = datetime.now(timezone.utc).isoformat() + node_id, seq, started_at, parent = trace_mod.resolve_node(state, str(node or ''), ended_at) + root = parent == '' and not state.root_seen + if root: + state.root_seen = True + bounded_error = _bounded_error(error) + state.note_node(outcome=outcome, adapter=adapter, runner=runner, family=family, error=bounded_error) + if not state.emit_nodes: + # Sampling excluded this execution: the node is never buffered, + # only counted so the terminal chain can explain the gap. + state.dropped_nodes += 1 return - self.trace_deadlines.pop(state.trace_id, None) - if not state.stages or not self._trace_emitted(state): - return - payload = self._build_trace_payload(state, reason) - if payload is not None: - self._dispatch_payload(payload) - except Exception: - return - - def _dispatch_payload(self, payload: dict) -> None: - """Hand one built payload to the telemetry manager from this sync context. - - TelemetryManager.start_send_task is a coroutine, so calling it without - scheduling dropped every trace closed here. Stand-ins used by tests and - manual tools schedule synchronously, hence the coroutine check. - """ - result = self.manager.start_send_task(payload) - if not asyncio.iscoroutine(result): - return - try: - asyncio.get_running_loop().create_task(result) - except RuntimeError: - result.close() - logger = getattr(getattr(self.manager, 'ap', None), 'logger', None) - if logger is not None: - logger.debug('Execution trace payload dropped: no running event loop') - - def _trace_emitted(self, state: TraceState) -> bool: - mode = self.trace_mode() - if mode == 'off': - return False - if mode == 'all': - return True - # A failed, cancelled or timed-out stage is always worth keeping. - if any(stage['outcome'] not in ('success', 'skipped') for stage in state.stages): - return True - # WebUI debug runs are deliberately traced; they stay flagged synthetic. - if state.synthetic: - return True - if mode == 'failures': - return False - try: - return int(state.trace_id[:8], 16) % self.trace_sample() == 0 - except ValueError: - return False - - def _build_trace_payload(self, state: TraceState, reason: str) -> dict | None: - from ..utils import constants - - identity = state.identity - if not identity.get('instance_id') or not identity.get('workspace_uuid'): - return None - observations = [{**stage, 'trace_id': state.trace_id, 'count': 1} for stage in state.stages] - return { - 'event_type': 'feature_execution', - # One payload per trace: lookup uses the indexed query_id column. - 'query_id': state.trace_id, - 'instance_id': identity['instance_id'], - 'workspace_uuid': identity['workspace_uuid'], - 'version': constants.semantic_version, - 'edition': constants.edition, - 'timestamp': datetime.now(timezone.utc).isoformat(), - 'features': { - 'schema': 1, - 'observations': observations, - 'trace': { - 'closed_by': reason, - 'started_at': state.started_at, - 'ended_at': datetime.now(timezone.utc).isoformat(), - 'route_ref': next((s['route_ref'] for s in state.stages if s['route_ref']), ''), - 'run_id': next((s['run_id'] for s in state.stages if s['run_id']), ''), - 'outcome': state.outcome(), - 'dropped_stages': state.dropped_stages, - }, - }, - } - - def _sweep_traces(self, now: float) -> None: - for trace_id in [key for key, deadline in self.trace_deadlines.items() if deadline <= now]: - state = self.traces.get(trace_id) - if state is not None: - self.close_trace(state, 'ttl') - - # ----------------------------------------------------------------- flushing - - async def _loop(self): - last_flush = time.monotonic() - while self.pending or self.traces: - # Traces expire on a short TTL; counters keep their minute cadence. - await asyncio.sleep(1 if self.traces else FLUSH_SECONDS) - now = time.monotonic() - self._sweep_traces(now) - if self.pending and now - last_flush >= FLUSH_SECONDS: - last_flush = now - await self.flush() - self.task = None - - async def flush(self): - from ..utils import constants - - # Remove only one bounded batch; subsequent windows drain the remainder. - # Drain one tenant per minute, at most one request, round-robin by insertion. - if not self.pending: - return - tenant = next(iter(self.pending))[:2] - keys = [key for key in self.pending if key[:2] == tenant][:MAX_BATCH] - groups: dict[tuple[str, str], list[dict]] = {} - for key in keys: - row = self.pending.pop(key) - instance, workspace, family, operation, mode, adapter, runner, outcome, synthetic = key - groups.setdefault((instance, workspace), []).append( + self._enqueue_record( { - **row, + 'event_type': 'execution_node', + 'schema': SCHEMA, + **self._common_fields(state, ended_at), + 'node_id': node_id, + 'parent_node_id': parent, + 'root': root, + 'seq': seq, 'family': family, 'operation': operation, 'mode': mode, 'adapter': adapter, 'runner': runner, 'outcome': outcome, - 'synthetic': synthetic, + 'error': bounded_error, + 'route_ref': trace_mod.current_route(), + 'run_id': trace_mod.current_run(), + 'synthetic': bool(synthetic), + 'started_at': started_at, + 'ended_at': ended_at, } ) - for (instance, workspace), observations in groups.items(): - payload = { - 'event_type': 'feature_execution', - 'query_id': str(uuid4()), - 'instance_id': instance, - 'workspace_uuid': workspace, - 'version': constants.semantic_version, - 'edition': constants.edition, - 'timestamp': datetime.now(timezone.utc).isoformat(), - 'features': {'schema': 1, 'observations': observations}, - } - if not await self.manager.send(payload): - await asyncio.sleep(1) - await self.manager.send(payload) + except Exception: + # Observability must never change execution behavior. + return + + def _common_fields(self, state: TraceState, timestamp: str) -> dict: + identity = state.identity + from ..utils import constants + + return { + 'event_id': state.event_id, + 'instance_id': identity['instance_id'], + 'workspace_uuid': identity['workspace_uuid'], + 'version': constants.semantic_version, + 'edition': constants.edition, + 'timestamp': timestamp, + 'debug': bool(state.debug), + 'sample': {'mode': state.mode, 'denominator': state.denominator}, + } + + def _enabled(self) -> bool: + try: + cfg = self.manager.telemetry_config + except Exception: + return False + return bool(cfg) and not cfg.get('disable_telemetry', False) and bool(cfg.get('url')) + + def _ensure_loop(self) -> None: + if self.task is None or self.task.done(): + self.task = asyncio.create_task(self._loop()) + + # --------------------------------------------------------------- registry + + def register_trace(self, state: TraceState, execution_id) -> None: + """Alias one more execution identity onto an in-flight execution.""" + try: + exec_id = str(execution_id or '').strip() + if not exec_id: + return + self.configure(state) + now = time.monotonic() + self._sweep_registry(now) + if exec_id not in self.registry and len(self.registry) >= MAX_REGISTRY_KEYS: + oldest = next(iter(self.registry)) + self.registry.pop(oldest, None) + self.registry_deadlines.pop(oldest, None) + self.registry[exec_id] = state + self.registry_deadlines[exec_id] = now + REGISTRY_TTL_SECONDS + except Exception: + return + + def _resolve(self, execution_id: str) -> TraceState | None: + self._sweep_registry(time.monotonic()) + return self.registry.get(execution_id) + + def _unregister_state(self, state: TraceState) -> None: + for exec_id in [key for key, value in self.registry.items() if value is state]: + self.registry.pop(exec_id, None) + self.registry_deadlines.pop(exec_id, None) + + def _sweep_registry(self, now: float) -> None: + """Drop aliases whose owning execution never closed (leaked bindings).""" + for exec_id in [key for key, deadline in self.registry_deadlines.items() if deadline <= now]: + self.registry.pop(exec_id, None) + self.registry_deadlines.pop(exec_id, None) + + # ------------------------------------------------------------------ chains + + def close_trace(self, state: TraceState, reason: str = 'event_done', error: str = '') -> None: + """Emit the one terminal ``execution_chain`` record of this execution.""" + try: + if state.closed: + return + state.closed = True + self._unregister_state(state) + if not self._enabled(): + # Nothing can be uploaded; never grow the buffer for a record + # the sender would refuse to send. + return + self.configure(state) + if not self._chain_emitted(state): + return + self._enqueue_record(self._build_chain_record(state, reason, error)) + except Exception: + return + finally: + # The snapshot was read while building the chain record; stop + # collecting so a late sibling task cannot leak into the next chain. + resources_end() + + def _chain_emitted(self, state: TraceState) -> bool: + if not state.reportable or state.mode == 'off': + return False + if state.emit_nodes or state.debug: + return True + # A chain that broke is always uploaded, even when sampling excluded it. + return bool(state.failure_reason) or state.outcome() in CHAIN_FAILURES + + def _build_chain_record(self, state: TraceState, reason: str, error: str = '') -> dict: + ended_at = datetime.now(timezone.utc).isoformat() + record = { + 'event_type': 'execution_chain', + 'schema': SCHEMA, + **self._common_fields(state, ended_at), + 'closed_by': reason if reason in CLOSED_BY else 'event_done', + 'outcome': state.outcome(), + # First failure detail in this chain, so Space can show why it broke. + 'error': _bounded_error(error) + or _bounded_error(state.failure_reason) + or _bounded_error(state.first_error) + or '', + 'duration_ms': _duration_ms(state.started_at, ended_at), + 'stage_count': state.stage_count, + 'dropped_nodes': state.dropped_nodes, + 'origin': state.origin, + 'model_name': state.model_name or '', + 'adapter': state.adapter or state.node_adapter, + 'runner': state.runner or state.node_runner, + 'runner_category': state.runner_category or '', + 'pipeline_plugins': state.pipeline_plugins, + } + # Bound resources of this execution, when any lane reported them. The + # key is omitted entirely when nothing was noted, so the payload stays + # byte-identical for executions with no bindings. + resources = resources_snapshot() + if resources: + record['resources'] = resources + return record + + def _enqueue_record(self, record: dict) -> None: + if len(self.records) >= MAX_BUFFERED_RECORDS: + self.records.pop(0) + self.dropped += 1 + self.records.append(record) + self._ensure_loop() + + # ----------------------------------------------------------------- flushing + + async def _loop(self): + interval = self.flush_seconds() + try: + # The outbound loop exists only to flush the record buffer. + while self.records: + await asyncio.sleep(interval) + await self.flush() + finally: + self.task = None + + async def flush(self) -> bool: + """Upload the buffered prefix; records stay buffered until accepted.""" + if not self.records: + return True + batch: list[dict] = [] + size = 0 + for record in self.records: + if len(batch) >= MAX_RECORDS_PER_FLUSH: + break + encoded = len(json.dumps(record, ensure_ascii=False, default=str)) + if batch and size + encoded > MAX_FLUSH_BYTES: + break + batch.append(record) + size += encoded + if not batch: + return True + payload = {'records': batch} + for attempt in range(MAX_FLUSH_ATTEMPTS): + try: + if await self.manager.send(payload): + # Peek-then-send: nothing is removed until the sender + # accepted the exact batch it was given. + del self.records[: len(batch)] + self.sent += len(batch) + return True + except asyncio.CancelledError: + raise + except Exception: + pass + if attempt + 1 < MAX_FLUSH_ATTEMPTS: + await asyncio.sleep(FLUSH_RETRY_BACKOFF_SECONDS * (2**attempt)) + # The batch stays buffered for the next cycle; a permanently unaccepted + # record is dropped by the buffer bound above and counted there. + return False async def shutdown(self): if self.task is not None: @@ -305,47 +503,123 @@ class ExecutionCounters: await asyncio.gather(self.task, return_exceptions=True) self.task = None try: - await asyncio.wait_for(self.flush(), timeout=2) + deadline = time.monotonic() + 5 + while self.records and time.monotonic() < deadline: + remaining = len(self.records) + await asyncio.wait_for(self.flush(), timeout=2) + if len(self.records) >= remaining: + break except (Exception, asyncio.CancelledError): pass - self.pending.clear() - # Traces of in-flight events cannot be completed during shutdown. - self.dropped += len(self.traces) - self.dropped_traces += len(self.traces) - self.traces.clear() - self.trace_deadlines.clear() + self.dropped += len(self.records) + self.records.clear() + self.registry.clear() + self.registry_deadlines.clear() + + +def _execution_counters(ap): + counters = getattr(getattr(ap, 'telemetry', None), 'execution', None) + return counters if isinstance(counters, ExecutionCounters) else None def record(ap, context, **observation): """Best-effort bridge usable with optional telemetry and test doubles.""" try: - counters = getattr(getattr(ap, 'telemetry', None), 'execution', None) - if isinstance(counters, ExecutionCounters): + counters = _execution_counters(ap) + if counters is not None: counters.record(context, **observation) except Exception: pass -def close_trace(ap, state, reason: str = 'event_done') -> None: - """Best-effort trace close usable with optional telemetry and test doubles.""" +def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None: + """Best-effort chain close usable with optional telemetry and test doubles.""" try: - counters = getattr(getattr(ap, 'telemetry', None), 'execution', None) - if isinstance(counters, ExecutionCounters): - counters.close_trace(state, reason) + counters = _execution_counters(ap) + if counters is not None: + counters.close_trace(state, reason, error) except Exception: pass +def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding: + """Bind an execution and alias its identity for cross-task nodes.""" + binding = trace_mod.bind(execution_id) + if binding.created: + resources_begin() + if execution_id: + try: + counters = _execution_counters(ap) + if counters is not None: + counters.register_trace(binding.state, execution_id) + except Exception: + pass + return binding + + @contextlib.contextmanager -def ingress(ap, reason: str = 'event_done'): - """Trace one ingress boundary; only the owner of the trace closes it. +def ingress( + ap, + reason: str = 'event_done', + context=None, + execution_id: str | None = None, + *, + debug: bool = False, + origin: str = 'platform', + synthetic_event: str = '', + require_route: bool = False, +): + """Open one execution boundary; only the owner of the chain closes it. + + ``require_route`` starts a platform event without reporting permission; + dispatch calls ``trace.admit`` when a delivery matches. Nested entrypoints + must reuse that decision instead of implicitly admitting the parent. Nested boundaries (an event routed into a Pipeline, a Runner invoked from a - dispatch) reuse the in-flight trace instead of starting a second one. + dispatch) reuse the in-flight chain instead of starting a second one, and + only alias their own execution id onto it. ``debug``/``origin`` apply when + this boundary starts the chain: a WebUI/API run without a real platform + event synthesizes its virtual inbound event here, so the chain has a real + identity and a node for the event it stands for. """ - binding = trace_mod.bind() + binding = trace_mod.bind(execution_id) + state = binding.state + if binding.created: + if require_route: + state.reportable = False + resources_begin() + counters = _execution_counters(ap) + if counters is not None: + if binding.created: + counters.configure(state, context, debug=debug, origin=origin) + if execution_id: + counters.register_trace(state, execution_id) + scope = None + if binding.created and synthetic_event: + scope = trace_mod.stage_scope() + node = scope.__enter__() + record( + ap, + context, + family='platform_event', + operation=str(synthetic_event)[:160], + outcome='success', + synthetic=True, + node=node, + ) + failure = '' try: yield binding + except BaseException as exc: + # The boundary itself failed: keep the reason on the chain so the + # uploaded payload explains the failure instead of reporting a bare + # outcome, even when sampling had excluded this execution. + if binding.created: + failure = f'{type(exc).__name__}: {exc}' + state.mark_failure(failure) + raise finally: + if scope is not None: + scope.__exit__(None, None, None) if trace_mod.unbind_root(binding): - close_trace(ap, binding.state, reason) + close_trace(ap, state, reason, failure) diff --git a/src/langbot/pkg/telemetry/features.py b/src/langbot/pkg/telemetry/features.py deleted file mode 100644 index ab66f9884..000000000 --- a/src/langbot/pkg/telemetry/features.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Per-query telemetry feature counters. - -Collects anonymous, content-free usage signals (tool call counts, knowledge -base usage, sandbox executions, ...) into ``query.variables`` during query -processing. The chat handler reads the accumulated dict when building the -telemetry payload and ships it as the ``features`` JSON object. - -Every helper here is defensive: telemetry must NEVER break the pipeline, so -all mutations are wrapped and failures are silently ignored. -""" - -from __future__ import annotations - -import typing - -if typing.TYPE_CHECKING: - import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query - - -FEATURES_KEY = '_telemetry_features' - - -def get_features(query: pipeline_query.Query) -> dict: - """Return the mutable features dict for this query, creating it if needed.""" - try: - return query.variables.setdefault(FEATURES_KEY, {}) - except Exception: - return {} - - -def increment(query: pipeline_query.Query, group: str, key: str | None = None, amount: int = 1) -> None: - """Increment a counter. - - ``increment(q, 'sandbox', 'execs')`` -> features['sandbox']['execs'] += 1 - ``increment(q, 'tool_call_rounds')`` -> features['tool_call_rounds'] += 1 - """ - try: - features = get_features(query) - if key is None: - features[group] = int(features.get(group, 0)) + amount - else: - nested = features.setdefault(group, {}) - if isinstance(nested, dict): - nested[key] = int(nested.get(key, 0)) + amount - except Exception: - pass - - -def set_value(query: pipeline_query.Query, group: str, value: typing.Any) -> None: - """Set a feature value (overwrites).""" - try: - get_features(query)[group] = value - except Exception: - pass - - -def collect_features(query: pipeline_query.Query) -> dict: - """Build the final ``features`` object for the telemetry payload. - - Combines the counters accumulated during processing with end-of-query - snapshots (activated skills, bound MCP servers). Returns a plain dict - that must be JSON-serializable; non-serializable values are dropped. - """ - features: dict = {} - try: - accumulated = query.variables.get(FEATURES_KEY) - if isinstance(accumulated, dict): - features.update(accumulated) - except Exception: - pass - - # Activated skills (names only, registered by the activate tool) - try: - activated = query.variables.get('_activated_skills', {}) - if isinstance(activated, dict) and activated: - features['activated_skills'] = sorted(activated.keys()) - except Exception: - pass - - # MCP servers bound to the pipeline (names only; None means "all enabled") - try: - bound_mcp = query.variables.get('_pipeline_bound_mcp_servers', None) - if bound_mcp is not None: - features['mcp_servers'] = list(bound_mcp) - except Exception: - pass - - # Drop anything that is not JSON-serializable - import json - - try: - json.dumps(features) - return features - except Exception: - safe: dict = {} - for k, v in features.items(): - try: - json.dumps({k: v}) - safe[k] = v - except Exception: - continue - return safe diff --git a/src/langbot/pkg/telemetry/heartbeat.py b/src/langbot/pkg/telemetry/heartbeat.py index 45dcbf268..7deb20400 100644 --- a/src/langbot/pkg/telemetry/heartbeat.py +++ b/src/langbot/pkg/telemetry/heartbeat.py @@ -13,6 +13,7 @@ credentials. from __future__ import annotations import asyncio +import time import typing from datetime import datetime, timezone @@ -36,10 +37,66 @@ class WorkspaceResourceSnapshot(typing.TypedDict): mcp_server_count: int extension_count: int skill_count: int + agent_count: int + event_processor_count: int + plugin_processor_count: int + event_binding_count: int adapters: list[str] execution_generation: int +# Agent rows are configuration, not runtime registries, so a Workspace count +# needs one scoped query. The heartbeat runs every five minutes on Cloud; a +# one-hour cache keeps the cost at one bounded query per Workspace per hour +# instead of one per cycle. Unavailable counts stay -1, never a fake zero. +_AGENT_COUNT_CACHE_TTL_SECONDS = 3600 +_agent_count_cache: dict[str, tuple[float, dict[str, int]]] = {} + + +async def _workspace_agent_kind_counts(ap: core_app.Application, workspace_uuid: str) -> dict[str, int]: + """Count one Workspace's Agents per ``kind``; empty when unavailable.""" + if not workspace_uuid: + return {} + now = time.monotonic() + cached = _agent_count_cache.get(workspace_uuid) + if cached is not None and cached[0] > now: + return dict(cached[1]) + counts: dict[str, int] = {} + try: + from ..entity.persistence import agent as persistence_agent + + statement = sqlalchemy.select(persistence_agent.Agent.kind, sqlalchemy.func.count()).group_by( + persistence_agent.Agent.kind + ) + tenant_uow = getattr(ap.persistence_mgr, 'tenant_uow', None) + if callable(tenant_uow): + async with tenant_uow(workspace_uuid): + result = await ap.persistence_mgr.execute_async(statement) + else: + result = await ap.persistence_mgr.execute_async(statement) + for kind, count in result.all(): + counts[str(kind or 'agent')] = int(count or 0) + except Exception: + counts = {} + _agent_count_cache[workspace_uuid] = (now + _AGENT_COUNT_CACHE_TTL_SECONDS, dict(counts)) + return counts + + +def _bot_processor_counts(bots) -> tuple[int, int]: + """Count plugin-processor subscriptions and event bindings across bots.""" + processors = 0 + bindings = 0 + for bot in bots: + entity = getattr(bot, 'bot_entity', bot) + declared_processors = getattr(entity, 'plugin_processors', None) + if isinstance(declared_processors, list): + processors += len(declared_processors) + declared_bindings = getattr(entity, 'event_bindings', None) + if isinstance(declared_bindings, list): + bindings += len(declared_bindings) + return processors, bindings + + async def _count( ap: core_app.Application, table, @@ -81,6 +138,10 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - 'mcp_server_count': 0, 'extension_count': 0, 'skill_count': 0, + 'agent_count': -1, + 'event_processor_count': -1, + 'plugin_processor_count': 0, + 'event_binding_count': 0, 'adapters': [], 'execution_generation': binding.placement_generation, } @@ -91,6 +152,9 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - for key, bot in getattr(ap.platform_mgr, '_bots_by_key', {}).items(): if len(key) >= 2 and key[1] in counts: counts[key[1]]['bot_count'] += 1 + processors, bindings = _bot_processor_counts([bot]) + counts[key[1]]['plugin_processor_count'] += processors + counts[key[1]]['event_binding_count'] += bindings adapter = getattr(bot, 'adapter', None) if adapter is not None and getattr(bot, 'enable', False): adapter_sets[key[1]].add(adapter.__class__.__name__) @@ -113,6 +177,10 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - for workspace_uuid, resource in counts.items(): resource['extension_count'] = resource['plugin_count'] + resource['mcp_server_count'] resource['adapters'] = sorted(adapter_sets[workspace_uuid]) + agent_counts = await _workspace_agent_kind_counts(ap, workspace_uuid) + if agent_counts: + resource['agent_count'] = int(agent_counts.get('agent', 0)) + resource['event_processor_count'] = int(agent_counts.get('event_processor', 0)) return list(counts.values()) @@ -163,6 +231,9 @@ async def build_heartbeat_payload( features['bot_count'] = len(platform_mgr.bots) adapters = sorted({bot.adapter.__class__.__name__ for bot in enabled_bots if getattr(bot, 'adapter', None)}) features['adapters'] = adapters + processors, bindings = _bot_processor_counts(platform_mgr.bots) + features['plugin_processor_count'] = processors + features['event_binding_count'] = bindings except Exception: pass @@ -189,10 +260,10 @@ async def build_heartbeat_payload( cloud_counter=lambda: len(ap.platform_mgr._bots_by_key), ) - # Plugin count (from plugin runtime) + # Scoped Cloud counts are already available; never issue tenantless RPCs. try: plugin_connector = getattr(ap, 'plugin_connector', None) - if plugin_connector is not None: + if workspace_resource is None and plugin_connector is not None: plugins = await plugin_connector.list_plugins() features['plugin_count'] = len(plugins) except Exception: @@ -206,6 +277,13 @@ async def build_heartbeat_payload( except Exception: pass + # Agent processors: configuration rows, so one scoped count per Workspace. + # The Cloud snapshot supplies these already and overrides below. + if workspace_resource is None: + agent_counts = await _workspace_agent_kind_counts(ap, workspace_uuid) + features['agent_count'] = int(agent_counts.get('agent', 0)) if agent_counts else -1 + features['event_processor_count'] = int(agent_counts.get('event_processor', 0)) if agent_counts else -1 + if workspace_resource is not None: features.update({key: value for key, value in workspace_resource.items() if key != 'workspace_uuid'}) @@ -271,4 +349,8 @@ async def heartbeat_loop(ap: core_app.Application) -> None: ap.logger.debug(f'Telemetry heartbeat failed: {e}') except Exception: pass - await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS) + # Cloud dashboards need fresh scoped snapshots, including Workspaces + # discovered after startup. A transient startup/projection failure must + # not suppress synchronization for an entire day. + cloud_runtime = getattr(getattr(ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime' + await asyncio.sleep(300 if cloud_runtime else HEARTBEAT_INTERVAL_SECONDS) diff --git a/src/langbot/pkg/telemetry/platform.py b/src/langbot/pkg/telemetry/platform.py index a7e636bcf..0842d466b 100644 --- a/src/langbot/pkg/telemetry/platform.py +++ b/src/langbot/pkg/telemetry/platform.py @@ -11,6 +11,21 @@ from .execution import record processing_mode: ContextVar[str] = ContextVar('telemetry_processing_mode', default='none') +def result_failure_detail(result) -> str: + """Pull a bounded human-readable reason out of a failed platform API result.""" + raw = getattr(result, 'raw', result) + if isinstance(raw, dict): + for key in ('error', 'message', 'msg', 'detail', 'reason'): + value = raw.get(key) + if isinstance(value, str) and value.strip(): + return value + if isinstance(value, dict): + for inner in ('message', 'msg', 'detail'): + if isinstance(value.get(inner), str) and value[inner].strip(): + return value[inner] + return '' + + def result_outcome(result): # Empty returns do not prove a remote operation succeeded. if result is None: @@ -69,18 +84,24 @@ def observe_adapter(ap, context, adapter): if action in declared: observed_operation = action outcome = 'unknown' + error_detail = '' try: result = await method(*args, **kwargs) outcome = result_outcome(result) + if outcome != 'success': + error_detail = result_failure_detail(result) return result except asyncio.CancelledError: outcome = 'cancelled' + error_detail = 'cancelled' raise - except TimeoutError: + except TimeoutError as exc: outcome = 'timeout' + error_detail = str(exc) or 'timeout' raise - except Exception: + except Exception as exc: outcome = 'failed' + error_detail = str(exc) raise finally: record( @@ -91,9 +112,12 @@ def observe_adapter(ap, context, adapter): adapter=adapter.__class__.__name__, mode=processing_mode.get(), outcome=outcome, + error=error_detail, ) return wrapped - setattr(adapter, name, make_wrapper(original, name)) - setattr(adapter, '_execution_observed', True) + # Adapters are Pydantic models: method instrumentation is not a model + # field assignment. Keep wrappers instance-local, never on the class. + object.__setattr__(adapter, name, make_wrapper(original, name)) + object.__setattr__(adapter, '_execution_observed', True) diff --git a/src/langbot/pkg/telemetry/resources.py b/src/langbot/pkg/telemetry/resources.py new file mode 100644 index 000000000..5164e090e --- /dev/null +++ b/src/langbot/pkg/telemetry/resources.py @@ -0,0 +1,244 @@ +"""Bound-resource reporting for one execution. + +Answers "what was this execution given", never "what did it use": the ids and +names of the Plugins, MCP servers, knowledge bases, tools, skills and models +that were bound or granted to a Pipeline lane or an Agent run. Nothing here is +user content - only code-defined identifiers (uuids, plugin names, tool names) +and booleans. + +Two sources feed one execution-scoped snapshot: + +* the Host policy actually granted to the run (``ResourcePolicy`` / + materialized ``AgentResources``), which is authoritative for tools, skills and + platform APIs; +* the Runner's own configuration form (``DynamicFormItemSchema``), which is + where knowledge bases and models are selected. Selector field types are read + through :mod:`langbot.pkg.agent.runner.config_schema`, so a Runner that + declares a new selector type is picked up without a change here. + +The snapshot is attached to the terminal ``execution_chain`` record. It is +bounded twice - per list and in total - and it is best effort: telemetry must +never change execution behaviour, so every entry point swallows failures. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import json +import typing + +MAX_ITEMS_PER_LIST = 64 +MAX_NAME_CHARS = 160 +MAX_RESOURCES_BYTES = 8 * 1024 + +_resources: contextvars.ContextVar[dict[str, typing.Any] | None] = contextvars.ContextVar( + 'telemetry_resources', default=None +) + + +def begin() -> None: + """Start collecting for one execution (called when a chain is opened).""" + _resources.set({}) + + +def end() -> None: + """Stop collecting; the snapshot has already been read.""" + _resources.set(None) + + +def _bucket(section: str) -> dict[str, typing.Any] | None: + root = _resources.get() + if root is None: + return None + bucket = root.get(section) + if not isinstance(bucket, dict): + bucket = {} + root[section] = bucket + return bucket + + +def _names(values: typing.Any) -> list[str]: + """Bound one list of code-defined identifiers, preserving source order.""" + if not isinstance(values, (list, tuple)): + return [] + names: list[str] = [] + seen: set[str] = set() + for value in values: + name = '' + if isinstance(value, str): + name = value + elif isinstance(value, dict): + name = str(value.get('name') or value.get('kb_id') or value.get('tool_name') or '') + if not name: + continue + name = name[:MAX_NAME_CHARS] + if name in seen: + continue + seen.add(name) + names.append(name) + if len(names) >= MAX_ITEMS_PER_LIST: + break + return names + + +def _tool_refs(values: typing.Any) -> list[dict[str, str]]: + """Bound one list of tool grants, keeping their Host source.""" + if not isinstance(values, (list, tuple)): + return [] + refs: list[dict[str, str]] = [] + seen: set[str] = set() + for value in values: + if isinstance(value, str): + name, source, source_id = value, '', '' + elif isinstance(value, dict): + name = str(value.get('tool_name') or value.get('name') or '') + source = str(value.get('tool_type') or value.get('source') or '') + raw_id = value.get('source_id') + source_id = raw_id if isinstance(raw_id, str) else '' + else: + continue + name = name[:MAX_NAME_CHARS] + if not name or name in seen: + continue + seen.add(name) + ref = {'name': name} + if source: + ref['source'] = source[:64] + if source_id: + ref['source_id'] = source_id[:MAX_NAME_CHARS] + refs.append(ref) + if len(refs) >= MAX_ITEMS_PER_LIST: + break + return refs + + +def _set(bucket: dict[str, typing.Any] | None, key: str, value: typing.Any) -> None: + if bucket is None or value in (None, [], {}, ''): + return + bucket[key] = value + + +def note_pipeline( + *, + plugins: typing.Any = None, + plugins_all: typing.Any = None, + mcp_servers: typing.Any = None, + mcp_all: typing.Any = None, + knowledge_bases: typing.Any = None, + skills: typing.Any = None, + tools: typing.Any = None, + tools_all: typing.Any = None, + runner_id: str = '', + selectors: typing.Any = None, +) -> None: + """Record the resources one Pipeline lane is bound to. + + ``plugins``/``mcp_servers`` accept ``None`` to mean "all enabled", matching + the binding model; the ``*_all`` flags make that explicit in the payload. + """ + try: + bucket = _bucket('pipeline') + if bucket is None: + return + _set(bucket, 'plugins', _names(plugins)) + _set(bucket, 'mcp_servers', _names(mcp_servers)) + _set(bucket, 'knowledge_bases', _names(knowledge_bases)) + _set(bucket, 'skills', _names(skills)) + _set(bucket, 'tools', _tool_refs(tools)) + _set(bucket, 'runner_id', str(runner_id or '')[:MAX_NAME_CHARS]) + if isinstance(selectors, dict): + cleaned = { + str(kind)[:64]: _names(fields) + for kind, fields in selectors.items() + if isinstance(fields, (list, tuple)) and fields + } + _set(bucket, 'selectors', cleaned) + if plugins_all is not None: + bucket['plugins_all'] = bool(plugins_all) + if mcp_all is not None: + bucket['mcp_all'] = bool(mcp_all) + if tools_all is not None: + bucket['tools_all'] = bool(tools_all) + except Exception: + return + + +def note_agent( + *, + runner_id: str = '', + resources: typing.Any = None, + selectors: typing.Any = None, + capabilities: typing.Any = None, + tools_all: typing.Any = None, +) -> None: + """Record the resources one Agent run was granted. + + ``resources`` is the materialized ``AgentResources`` mapping the Host + actually handed to the Runner (tools carry their source), ``selectors`` + names the Runner config-form fields that selected a resource, and + ``capabilities`` records which resource categories the Runner declared. + """ + try: + bucket = _bucket('agent') + if bucket is None: + return + _set(bucket, 'runner_id', str(runner_id or '')[:MAX_NAME_CHARS]) + if isinstance(resources, dict): + _set(bucket, 'knowledge_bases', _names(resources.get('knowledge_bases'))) + _set(bucket, 'skills', _names(resources.get('skills'))) + _set(bucket, 'models', _names(resources.get('models'))) + _set(bucket, 'tools', _tool_refs(resources.get('tools'))) + _set(bucket, 'platform_apis', _names(resources.get('platform_tools'))) + if isinstance(selectors, dict): + cleaned = { + str(kind)[:64]: _names(fields) + for kind, fields in selectors.items() + if isinstance(fields, (list, tuple)) and fields + } + _set(bucket, 'selectors', cleaned) + if isinstance(capabilities, dict): + declared = {key: bool(value) for key, value in capabilities.items() if value} + _set(bucket, 'capabilities', declared) + if tools_all is not None: + bucket['tools_all'] = bool(tools_all) + except Exception: + return + + +def snapshot() -> dict[str, typing.Any] | None: + """Return the bounded snapshot of this execution; None when nothing was noted.""" + try: + root = _resources.get() + if not isinstance(root, dict) or not root: + return None + + def encoded_size(value: typing.Any) -> int: + return len(json.dumps(value, ensure_ascii=False, default=str).encode('utf-8')) + + if encoded_size(root) <= MAX_RESOURCES_BYTES: + return root + # Over budget: shed the widest detail (per-tool grants) first, then fall + # back to a marker, so an oversized binding can never block the chain + # record or bloat the upload. + reduced = json.loads(json.dumps(root, ensure_ascii=False, default=str)) + for section in ('pipeline', 'agent'): + bucket = reduced.get(section) + if isinstance(bucket, dict): + bucket.pop('tools', None) + reduced['truncated'] = True + if encoded_size(reduced) <= MAX_RESOURCES_BYTES: + return reduced + return {'truncated': True} + except Exception: + return None + + +@contextlib.contextmanager +def collected() -> typing.Iterator[None]: + """Bind a collection scope, for callers outside an execution boundary.""" + token = _resources.set({}) + try: + yield + finally: + _resources.reset(token) diff --git a/src/langbot/pkg/telemetry/telemetry.py b/src/langbot/pkg/telemetry/telemetry.py index c36b35e3c..5c9a41d22 100644 --- a/src/langbot/pkg/telemetry/telemetry.py +++ b/src/langbot/pkg/telemetry/telemetry.py @@ -12,9 +12,6 @@ from ..utils import httpclient from .execution import ExecutionCounters -_MAX_INFLIGHT_TELEMETRY_TASKS = 8 - - class TelemetryManager: """TelemetryManager handles sending telemetry for a given application instance. @@ -27,36 +24,18 @@ class TelemetryManager: self.ap = ap self.telemetry_config: dict[str, typing.Any] = {} - self.send_tasks: list[asyncio.Task] = [] self._client: httpx.AsyncClient | None = None self.execution = ExecutionCounters(self) async def initialize(self): self.telemetry_config = self.ap.instance_config.data.get('space', {}) - async def start_send_task(self, payload: dict): - self.send_tasks = [task for task in self.send_tasks if not task.done()] - if len(self.send_tasks) >= _MAX_INFLIGHT_TELEMETRY_TASKS: - self.ap.logger.debug('Telemetry queue is full; dropping best-effort event') - return - task = asyncio.create_task(self.send(payload)) - self.send_tasks.append(task) - task.add_done_callback(self._send_task_done) - - def _send_task_done(self, task: asyncio.Task) -> None: - try: - self.send_tasks.remove(task) - except ValueError: - pass + def health(self) -> dict: + """Telemetry health for the instance liveness snapshot.""" + return self.execution.health() async def shutdown(self) -> None: await self.execution.shutdown() - tasks = list(self.send_tasks) - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - self.send_tasks.clear() if self._client is not None: await self._client.aclose() self._client = None @@ -95,45 +74,51 @@ class TelemetryManager: url = server.rstrip('/') + '/api/v1/telemetry' try: - # Sanitize payload so string fields are strings and not nulls + # Sanitize payload so string fields are strings and not nulls. + # A record batch is an envelope of whole execution records; it + # carries no top-level record fields, so only a single-record + # payload gets the legacy defaults. sanitized = dict(payload) - if 'query_id' in sanitized: - try: - sanitized['query_id'] = '' if sanitized['query_id'] is None else str(sanitized['query_id']) - except Exception: - sanitized['query_id'] = str(sanitized.get('query_id', '')) + if isinstance(sanitized.get('records'), list): + sanitized['records'] = [item for item in sanitized['records'] if isinstance(item, dict)] + else: + if 'query_id' in sanitized: + try: + sanitized['query_id'] = '' if sanitized['query_id'] is None else str(sanitized['query_id']) + except Exception: + sanitized['query_id'] = str(sanitized.get('query_id', '')) - for sfield in ( - 'adapter', - 'runner', - 'runner_category', - 'model_name', - 'version', - 'edition', - 'error', - 'timestamp', - 'event_type', - ): - if sfield not in sanitized: - continue - v = sanitized.get(sfield) - sanitized[sfield] = '' if v is None else str(v) + for sfield in ( + 'adapter', + 'runner', + 'runner_category', + 'model_name', + 'version', + 'edition', + 'error', + 'timestamp', + 'event_type', + ): + if sfield not in sanitized: + continue + v = sanitized.get(sfield) + sanitized[sfield] = '' if v is None else str(v) - # event_type defaults to 'query' for backward compatibility - if not sanitized.get('event_type'): - sanitized['event_type'] = 'query' + # event_type defaults to 'query' for backward compatibility + if not sanitized.get('event_type'): + sanitized['event_type'] = 'query' - # features must be a JSON object - if 'features' in sanitized and not isinstance(sanitized['features'], dict): - sanitized['features'] = {} + # features must be a JSON object + if 'features' in sanitized and not isinstance(sanitized['features'], dict): + sanitized['features'] = {} - if 'duration_ms' in sanitized: - try: - sanitized['duration_ms'] = ( - int(sanitized['duration_ms']) if sanitized['duration_ms'] is not None else 0 - ) - except Exception: - sanitized['duration_ms'] = 0 + if 'duration_ms' in sanitized: + try: + sanitized['duration_ms'] = ( + int(sanitized['duration_ms']) if sanitized['duration_ms'] is not None else 0 + ) + except Exception: + sanitized['duration_ms'] = 0 async with self._client_context() as client: try: diff --git a/src/langbot/pkg/telemetry/trace.py b/src/langbot/pkg/telemetry/trace.py index a239c707e..ddc5f5d05 100644 --- a/src/langbot/pkg/telemetry/trace.py +++ b/src/langbot/pkg/telemetry/trace.py @@ -1,18 +1,27 @@ -"""Content-free per-event execution traces. +"""Per-execution telemetry context for the v2 execution chain. -One trace identity is minted per inbound platform event and survives until the -ingress handler returns. Stage records appended under it describe how that one -event was routed and processed, using only code-defined identifiers. Nothing -here is aggregated: a trace is a bounded, ordered sequence for one event. +One execution identity (a Pipeline query, inbound platform event or Agent run) +is one chain. The chain is not assembled here: every workflow node is emitted +by ``telemetry.execution`` the moment it completes, carrying its own identity +and the identity of the execution it belongs to. This module only holds the +small execution-scoped context the emitter needs: -Three bounds keep memory independent of traffic volume: +* the execution state (identity, the once-per-event sampling decision, the + execution-scoped display fields); +* the routing/run identity pinned around the block that produced a node; +* the open-step stack, which lives in a ContextVar holding *one task's own + immutable tuple*. Child tasks inherit a snapshot at creation; no shared + mutable stack exists, so sibling tasks can never corrupt each other's parent + links. -* ``MAX_STAGES`` stages per trace (further stages are counted, not kept); -* ``MAX_TRACES`` traces in flight, enforced by the sender that owns the buffer; -* a wall-clock TTL, enforced by the sender's sweep. - -The identity itself lives in a ContextVar so that asynchronous work spawned -while handling one event inherits it, mirroring ``telemetry.platform``. +Cross-task callers (plugin/RPC work with no inherited context) resolve their +execution through the owner's execution-id registry instead, which only +carries the execution state - never node payloads. Such a caller owns no step +of its own, yet its work is still caused by a step of the chain: the sandbox +tool a Runner invoked, the platform API a plugin called back over RPC. The +execution state therefore also keeps the ordered set of steps any task +currently has open, and a record emitted with no open step of its own attaches +to the innermost one of those instead of becoming a root of the chain. """ from __future__ import annotations @@ -23,80 +32,154 @@ import typing from datetime import datetime, timezone from uuid import uuid4 -MAX_STAGES = 32 - _current: contextvars.ContextVar['TraceState | None'] = contextvars.ContextVar('telemetry_trace', default=None) _route: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_route', default='') _run: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_run', default='') +_reporting_suppressed: contextvars.ContextVar[bool] = contextvars.ContextVar( + 'telemetry_reporting_suppressed', default=False +) +# Open workflow step frames of the current task, innermost last. The tuple is +# never mutated in place: opening a step publishes a new tuple on this task's +# context only, and closing it restores the previous one. +_node: contextvars.ContextVar[tuple['NodeFrame', ...]] = contextvars.ContextVar('telemetry_trace_node', default=()) def _now() -> str: return datetime.now(timezone.utc).isoformat() +class NodeFrame(typing.NamedTuple): + """One workflow step open in the task that opened it.""" + + node_id: str + seq: int + started_at: str + + class TraceState: - """Bounded stage buffer for exactly one event.""" + """The small per-execution context; node payloads never accumulate here.""" __slots__ = ( - 'trace_id', - 'started_at', - 'stages', - 'dropped_stages', - 'synthetic', - 'sequence', + 'event_id', 'identity', - 'abandoned', + 'configured', + 'reportable', + 'mode', + 'denominator', + 'emit_nodes', + 'debug', + 'origin', + 'started_at', + 'node_seq', + 'root_seen', + 'stage_count', + 'dropped_nodes', + 'last_outcome', + 'first_error', + 'failure_reason', + 'runner_failed', + 'closed', + 'adapter', + 'runner', + 'runner_category', + 'model_name', + 'pipeline_plugins', + 'node_adapter', + 'node_runner', + 'open_frames', ) - def __init__(self) -> None: - self.trace_id = str(uuid4()) - self.started_at = _now() - self.stages: list[dict[str, typing.Any]] = [] - self.dropped_stages = 0 - self.synthetic = False - self.sequence = 0 + def __init__(self, execution_id: str | None = None) -> None: + # The execution identity every record of this chain carries. + self.event_id = str(execution_id).strip() if execution_id else str(uuid4()) self.identity: dict[str, str] = {} - # Set when the sender refused to buffer this trace: stop appending. - self.abandoned = False + self.configured = False + # Platform ingress starts pending; an actual route admits the shared + # execution before work starts. Independent API/debug runs are eligible. + self.reportable = not reporting_suppressed() + # Sampling, decided exactly once per event (at ingress or first touch). + self.mode = 'all' + self.denominator = 1 + self.emit_nodes = False + self.debug = False + self.origin = 'platform' + self.started_at = _now() + self.node_seq = 0 + self.root_seen = False + self.stage_count = 0 + self.dropped_nodes = 0 + self.last_outcome = '' + self.first_error = '' + self.failure_reason = '' + # Set when a runner node already reported a non-success outcome. + self.runner_failed = False + self.closed = False + # Execution-scoped record fields attached by the owning lane when known. + self.adapter = '' + self.runner = '' + self.runner_category = '' + self.model_name = '' + self.pipeline_plugins: typing.Any = None + # First adapter/runner seen on a node, used for the chain record. + self.node_adapter = '' + self.node_runner = '' + # Steps currently open anywhere in this chain, in the order they were + # opened. A record emitted by a task that owns no step of its own + # attaches to the innermost entry here. + self.open_frames: dict[str, NodeFrame] = {} - def append( + def allocate_seq(self) -> int: + """Mint the next chain-local node order.""" + seq = self.node_seq + self.node_seq += 1 + return seq + + def open_lane(self) -> str: + """The innermost step open in this chain, whichever task opened it.""" + frames = list(self.open_frames) + return frames[-1] if frames else '' + + def open_step(self, frame: NodeFrame) -> None: + self.open_frames[frame.node_id] = frame + + def close_step(self, frame: NodeFrame) -> None: + self.open_frames.pop(frame.node_id, None) + + def note_node( self, *, - family: str, - operation: str, - mode: str, - adapter: str, - runner: str, outcome: str, - synthetic: bool, + adapter: str = '', + runner: str = '', + family: str = '', + error: str = '', ) -> None: - if len(self.stages) >= MAX_STAGES: - self.dropped_stages += 1 - return - seen = _now() - self.stages.append( - { - 'family': family, - 'operation': operation, - 'mode': mode, - 'adapter': adapter, - 'runner': runner, - 'outcome': outcome, - 'synthetic': bool(synthetic), - 'seq': self.sequence, - 'route_ref': _route.get(), - 'run_id': _run.get(), - 'first_seen': seen, - 'last_seen': seen, - } - ) - self.sequence += 1 - if synthetic: - self.synthetic = True + """Remember only what the terminal chain record needs from a node.""" + self.stage_count += 1 + self.last_outcome = outcome or 'unknown' + if adapter and not self.node_adapter: + self.node_adapter = adapter + if runner and not self.node_runner: + self.node_runner = runner + if family == 'runner' and outcome not in ('success', 'skipped'): + # The lane records its own failed runner step only when the runner + # orchestrator has not already landed one. + self.runner_failed = True + if outcome in ('failed', 'cancelled', 'timeout') and error and not self.first_error: + # The chain must explain itself even when the break was reported as + # a failed node rather than as an exception at the boundary. + self.first_error = error + + def mark_failure(self, reason: str) -> None: + """Remember why this chain broke so the payload can explain itself.""" + if not self.failure_reason: + self.failure_reason = reason def outcome(self) -> str: - """Terminal outcome of the last recorded stage, for space-side display.""" - return self.stages[-1]['outcome'] if self.stages else 'unknown' + """Terminal outcome of this chain; a broken chain never reports success.""" + if self.failure_reason and self.last_outcome in ('', 'success', 'skipped', 'unknown'): + return 'failed' + return self.last_outcome or 'unknown' class TraceBinding(typing.NamedTuple): @@ -105,17 +188,21 @@ class TraceBinding(typing.NamedTuple): token: typing.Any -def bind() -> TraceBinding: - """Start a trace unless one is already in flight in this context.""" +def bind(execution_id: str | None = None) -> TraceBinding: + """Start a chain unless one is already in flight in this context. + + The execution id becomes the chain identity only when this call starts it; + a nested boundary keeps the enclosing execution's identity. + """ existing = _current.get() if existing is not None: return TraceBinding(existing, False, None) - state = TraceState() + state = TraceState(execution_id) return TraceBinding(state, True, _current.set(state)) def unbind(binding: TraceBinding) -> bool: - """Detach this binding. Returns True when this caller owns the trace.""" + """Detach this binding. Returns True when this caller owns the chain.""" if binding.token is not None: try: _current.reset(binding.token) @@ -126,7 +213,7 @@ def unbind(binding: TraceBinding) -> bool: def unbind_root(binding: TraceBinding) -> bool: - """Detach only when this caller started the trace, else leave it in place.""" + """Detach only when this caller started the chain, else leave it in place.""" if not binding.created: return False return unbind(binding) @@ -136,6 +223,96 @@ def current() -> TraceState | None: return _current.get() +def reporting_suppressed() -> bool: + return _reporting_suppressed.get() + + +def admit() -> None: + """Admit a matched delivery without revoking any concurrent sibling's choice.""" + state = current() + if state is not None and not state.closed and not reporting_suppressed(): + state.reportable = True + + +@contextlib.contextmanager +def suppress_reporting(): + """Keep discarded branch side effects local even if a sibling is admitted.""" + token = _reporting_suppressed.set(True) + try: + yield + finally: + _reporting_suppressed.reset(token) + + +def node_stack() -> tuple[NodeFrame, ...]: + """This task's own open-step frames, innermost last.""" + return _node.get() + + +def current_parent() -> str: + """The step node that records emitted right now attach to ('' when none).""" + stack = _node.get() + return stack[-1].node_id if stack else '' + + +def enclosing_parent() -> str: + """The step enclosing the innermost open one; '' when it is a root step.""" + stack = _node.get() + return stack[-2].node_id if len(stack) > 1 else '' + + +def resolve_node( + state: TraceState, + node: str, + fallback_started_at: str, +) -> tuple[str, int, str, str]: + """Resolve one record to ``(node_id, seq, started_at, parent_node_id)``. + + A record stamped with a ``node`` the calling task has open reuses that + step's identity, order and start time; anything else (a plain observation, + or a node opened by another task) is a fresh node under the innermost step + open in this task. A task that inherited no step at all - plugin/RPC work + that only knows the execution - attaches to the innermost step the chain + has open, which is the step that caused the work. + """ + stack = _node.get() + if node: + for index in range(len(stack) - 1, -1, -1): + frame = stack[index] + if frame.node_id == node: + parent = stack[index - 1].node_id if index > 0 else '' + return frame.node_id, frame.seq, frame.started_at, parent + parent = stack[-1].node_id if stack else state.open_lane() + return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent + + +@contextlib.contextmanager +def stage_scope() -> typing.Iterator[str]: + """Open one workflow step node for the duration of the block. + + Allocates a globally unique node id and the chain-local order, publishes the + frame on this task's own stack so nested observations attach to it as their + parent, and yields the node id so the step's own record can be stamped with + ``node=``. Outside any execution nothing is allocated and '' is yielded. + """ + state = _current.get() + if state is None: + yield '' + return + frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now()) + token = _node.set(_node.get() + (frame,)) + state.open_step(frame) + try: + yield frame.node_id + finally: + state.close_step(frame) + try: + _node.reset(token) + except (ValueError, RuntimeError): + # A token from another context must never break execution. + pass + + def set_run(run_id: str) -> typing.Any: return _run.set(run_id) @@ -152,7 +329,7 @@ def reset_run(token: typing.Any) -> None: @contextlib.contextmanager def scope(*, route_ref: str | None = None) -> typing.Iterator[None]: - """Pin routing identity for stages recorded inside this block.""" + """Pin routing identity for nodes recorded inside this block.""" if not route_ref: yield return @@ -164,3 +341,11 @@ def scope(*, route_ref: str | None = None) -> typing.Iterator[None]: _route.reset(token) except (ValueError, RuntimeError): pass + + +def current_route() -> str: + return _route.get() + + +def current_run() -> str: + return _run.get() diff --git a/src/langbot/pkg/utils/bounded_executor.py b/src/langbot/pkg/utils/bounded_executor.py index 006714181..cc53beb6a 100644 --- a/src/langbot/pkg/utils/bounded_executor.py +++ b/src/langbot/pkg/utils/bounded_executor.py @@ -173,6 +173,13 @@ class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): max_workers=max_workers, thread_name_prefix=thread_name_prefix, ) + # asyncio DNS uses the default executor too. Keep a separately bounded + # resolver pool so long-lived SDK calls cannot starve control-plane DNS. + self._dns_executor = ( + BoundedThreadPoolExecutor(max_workers=2, max_pending=32, thread_name_prefix='langbot-dns') + if thread_name_prefix != 'langbot-dns' + else None + ) self.max_workers = max_workers self.max_pending = max_pending self.max_inflight_per_scope = max_inflight_per_scope @@ -194,6 +201,10 @@ class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): *args: Any, **kwargs: Any, ) -> concurrent.futures.Future: + import socket + + if self._dns_executor is not None and fn in (socket.getaddrinfo, socket.getnameinfo): + return self._dns_executor.submit(fn, *args, **kwargs) scope = current_blocking_work_scope() if not self._capacity.acquire(blocking=False): with self._stats_lock: @@ -246,6 +257,11 @@ class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): future.add_done_callback(complete) return future + def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None: + if self._dns_executor is not None: + self._dns_executor.shutdown(wait=wait, cancel_futures=cancel_futures) + super().shutdown(wait=wait, cancel_futures=cancel_futures) + def _release_scope_locked(self, scope: str | None) -> None: if scope is None: return diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 19c46ae35..856563e39 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -293,6 +293,20 @@ plugin: binary_storage: # Max bytes for a single plugin binary storage value max_value_bytes: 10485760 + runtime_ops: + # Cloud-only ops telemetry: aggregates this instance's plugin Runtime + # health, desired installations and reconcile failures and pushes them + # to the Space control plane for the internal ops dashboard. This is + # separate from product telemetry (space.disable_telemetry) and has no + # effect outside the cloud edition. Defaults to on for the cloud + # edition; set false to stop pushing samples. + enabled: true + # Seconds between samples pushed to Space. + interval_seconds: 30 + # Per-request timeout for the Runtime /healthz probe and the upload. + timeout_seconds: 10 + # Maximum number of failed installations included per sample. + max_failures: 5 runner: # Host-level admin permissions for trusted control plugins. These plugins # can use existing plugin action handlers to inspect or manage Runner @@ -430,7 +444,17 @@ space: # heartbeat and execution reporting; restart after changing this setting. disable_telemetry: false # Per-event execution traces: off | failures | sampled | all. - # 'sampled' keeps every failed run plus every N-th successful one - # (N is execution_trace_sample) and always keeps WebUI debug runs. - execution_trace: sampled + # Every eligible execution is reported by default: one record per workflow + # node plus one terminal chain record. 'failures' keeps only broken chains; + # 'sampled' keeps broken chains plus about 1/N of successful ones, selected + # by event hash (N is execution_trace_sample); 'off' reports no executions. + # Unmatched/discarded platform events are never reported unless a plugin + # subscription matches. Matched delivery failures and valid interaction + # resumptions remain eligible, as do independent API/WebUI executions. + # Cloud uses 'all' but still applies this route eligibility filter. + execution_trace: all execution_trace_sample: 20 + # How often buffered execution records are uploaded, in seconds. + # Complete execution records are batched as {"records": [...]}; records are + # removed from the buffer only once the sender accepted them. + telemetry_flush_seconds: 180 diff --git a/tests/unit_tests/agent/test_orchestrator_integration.py b/tests/unit_tests/agent/test_orchestrator_integration.py index 1404a1537..bd8cf61a1 100644 --- a/tests/unit_tests/agent/test_orchestrator_integration.py +++ b/tests/unit_tests/agent/test_orchestrator_integration.py @@ -80,6 +80,7 @@ class FakePluginConnector: self.results = results or [] self.error = error self.delay = delay + self.delay_entered = asyncio.Event() self.calls: list[dict] = [] self.contexts: list[dict] = [] self.sessions_during_run: list[dict | None] = [] @@ -100,6 +101,7 @@ class FakePluginConnector: for result in self.results: if self.delay: + self.delay_entered.set() await asyncio.sleep(self.delay) yield result @@ -600,6 +602,35 @@ async def test_orchestrator_stops_after_cancel_request(clean_agent_state): assert run['status_reason'] == 'user stopped' +@pytest.mark.asyncio +@pytest.mark.parametrize('cancel_while_waiting', [False, True]) +async def test_consumer_exit_finalizes_cancelled_ledger(clean_agent_state, cancel_while_waiting): + connector = FakePluginConnector( + results=[ + {'type': 'message.delta', 'data': {'chunk': {'role': 'assistant', 'content': 'first'}}}, + {'type': 'run.completed', 'data': {'finish_reason': 'stop'}}, + ] + ) + orchestrator = AgentRunOrchestrator(FakeApplication(connector, clean_agent_state), FakeRegistry(make_descriptor())) + plan = orchestrator.query_bridge.build_plan(make_query()) + stream = orchestrator.run(plan.event, plan.binding, adapter_context={'_execution_context': TEST_CONTEXT}) + await anext(stream) + if cancel_while_waiting: + connector.delay = 10 + pending = asyncio.create_task(anext(stream)) + # Cancel the provider wait, not an arbitrary journal DB operation. + await asyncio.wait_for(connector.delay_entered.wait(), timeout=2) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + else: + await stream.aclose() + run = await RunLedgerStore(clean_agent_state).get_run(connector.contexts[0]['run_id']) + assert run['status'] == 'cancelled' + assert run['status_reason'] == 'consumer_cancelled' + assert await get_session_registry().list_active_runs() == [] + + @pytest.mark.asyncio async def test_orchestrator_does_not_package_query_messages_into_context(clean_agent_state): """Host should not build an agent working-context window from query.messages.""" diff --git a/tests/unit_tests/agent/test_run_journal.py b/tests/unit_tests/agent/test_run_journal.py index 8908681ba..1ae134a01 100644 --- a/tests/unit_tests/agent/test_run_journal.py +++ b/tests/unit_tests/agent/test_run_journal.py @@ -52,7 +52,19 @@ async def test_agent_run_snapshot_records_identity_and_redacts_inline_attachment assert raw_input['contents'][0]['image_base64'] == 'large-image-data' assert raw_input['attachments'][0]['content'] == 'large-file-data' - if event_type != 'message.received': - assert saved['metadata']['input_event'] == event.data - else: - assert 'input_event' not in saved['metadata'] + assert saved['metadata']['input_event'] == event.data + + +def test_event_snapshot_preserves_custom_values_and_redacts_nested_binary(): + data = { + 'member': {'id': 'u1', 'active': False}, + 'count': 0, + 'custom': {'contents': [{'type': 'image_base64', 'image_base64': 'secret'}]}, + 'attachments': [{'name': 'file.txt', 'content': 'secret'}], + } + saved = AgentRunJournal._sanitize_event_data(data) + assert saved['member'] == data['member'] + assert saved['count'] == 0 + assert saved['custom']['contents'][0]['image_base64'] is None + assert saved['attachments'][0]['content'] is None + assert data['custom']['contents'][0]['image_base64'] == 'secret' diff --git a/tests/unit_tests/api/service/test_monitoring_executions.py b/tests/unit_tests/api/service/test_monitoring_executions.py new file mode 100644 index 000000000..ce3d79f34 --- /dev/null +++ b/tests/unit_tests/api/service/test_monitoring_executions.py @@ -0,0 +1,738 @@ +"""Execution-view tests for the monitoring service. + +Covers the unified agent-run + pipeline-query execution list, its rollup +summary, and single-execution traces. +""" + +from __future__ import annotations + +import datetime +from types import SimpleNamespace + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker + +from langbot.pkg.api.http.service.monitoring import MonitoringService +from langbot.pkg.entity.persistence import agent_interaction as persistence_interaction +from langbot.pkg.entity.persistence import agent_run as persistence_agent_run +from langbot.pkg.entity.persistence import monitoring as persistence_monitoring +from langbot.pkg.entity.persistence import workspace as persistence_workspace # noqa: F401 +from langbot.pkg.entity.persistence.base import Base + +WORKSPACE = 'workspace-1' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('has_log', [False, True]) +async def test_event_properties_survive_detail_snapshot_fallback(engine, service, has_log): + import json + from langbot.pkg.entity.persistence.event_log import EventLog + + data = {'member': {'id': 'u1', 'active': False}, 'count': 0, 'custom': {'reason': 'joined'}} + async with AsyncSession(engine) as session: + session.add( + persistence_agent_run.AgentRun( + run_id='custom-run', + event_id='custom-event', + workspace_id=WORKSPACE, + runner_id='runner', + status='completed', + created_at=_dt(10), + metadata_json=json.dumps( + { + 'event_type': 'group.member_joined', + 'input_event': data, + 'input': {'text': '', 'contents': [], 'attachments': []}, + } + ), + ) + ) + if has_log: + session.add( + EventLog( + event_id='custom-event', + event_type='group.member_joined', + source='webui', + workspace_id=WORKSPACE, + run_id='custom-run', + input_json='{"text":"","contents":[]}', + created_at=_dt(10), + ) + ) + session.add( + EventLog( + event_id='later-event', + event_type='message.received', + source='webui', + workspace_id=WORKSPACE, + run_id='custom-run', + input_json='{"text":"later"}', + created_at=_dt(10, 1), + ) + ) + await session.commit() + detail = await service.get_execution_detail(WORKSPACE, 'agent', 'custom-run') + items = detail['pages']['inputs']['items'] + assert items[0]['content']['member'] == data['member'] + assert items[0]['content']['count'] == 0 + assert items[0]['content']['custom'] == data['custom'] + if has_log: + assert items[1]['content'] == {'text': 'later'} + + +def test_event_content_preserves_conflicting_custom_input_fields(): + from langbot.pkg.api.http.service.monitoring_execution_details import event_content + + assert event_content({'text': 'normalized'}, {'text': {'custom': False}}) == { + 'event': {'text': {'custom': False}}, + 'input': {'text': 'normalized'}, + } + + +class FakePersistenceManager: + """Runs statements against a real in-memory SQLite engine.""" + + def __init__(self, engine): + self._engine = engine + self._factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + def get_db_engine(self): + return self._engine + + def tenant_uow(self, workspace_uuid): + from langbot.pkg.persistence.tenant_uow import TenantUnitOfWork + + return TenantUnitOfWork(self._engine, workspace_uuid) + + async def execute_async(self, statement): + # Match PersistenceManager's connection-level result shape. + async with self._engine.connect() as connection: + return await connection.execute(statement) + + @staticmethod + def serialize_model(model, data): + out = {} + for column in model.__table__.columns: + value = getattr(data, column.name) + out[column.name] = value.isoformat() if isinstance(value, datetime.datetime) else value + return out + + +@pytest.fixture +async def engine(tmp_path): + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "monitoring_executions.db"}') + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield engine + await engine.dispose() + + +@pytest.fixture +def service(engine): + ap = SimpleNamespace( + persistence_mgr=FakePersistenceManager(engine), + instance_config=None, + ) + return MonitoringService(ap) + + +def _dt(hour: int, minute: int = 0, second: int = 0) -> datetime.datetime: + return datetime.datetime(2026, 1, 1, hour, minute, second) + + +async def _seed(engine) -> None: + factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with factory() as session: + session.add_all( + [ + persistence_agent_run.AgentRun( + run_id='run-done', + event_id='evt-1', + agent_id='agent-1', + binding_id='agent_agent-1_default', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='completed', + created_at=_dt(10), + started_at=_dt(10, 0, 1), + finished_at=_dt(10, 0, 3), + usage_json='{"total_tokens": 42}', + ), + persistence_agent_run.AgentRun( + run_id='run-failed', + binding_id='agent_agent-1_default', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='failed', + created_at=_dt(10, 5), + ), + persistence_agent_run.AgentRun( + run_id='run-debug', + event_id='debug:agent-1:e1', + agent_id='agent-1', + binding_id='debug:agent-1:runner-1', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='completed', + created_at=_dt(10, 6), + started_at=_dt(10, 6, 0), + finished_at=_dt(10, 6, 1), + ), + persistence_agent_run.AgentRunEvent( + run_id='run-done', + sequence=1, + type='run.completed', + data_json='{"message": {"role": "assistant", "content": "hi"}}', + ), + persistence_interaction.AgentInteraction( + interaction_id='int-1', + run_id='run-done', + binding_id='agent_agent-1_default', + runner_id='runner-1', + processor_type='agent', + processor_id='agent-1', + workspace_id=WORKSPACE, + status='pending', + request_json='{}', + callback_token_hash='hash-1', + ), + ] + ) + for message in [ + dict( + id='msg-user', + timestamp=_dt(10, 2), + session_id='session-1', + message_content='{"chain": [{"type": "Plain", "text": "hello there"}]}', + status='success', + role='user', + ), + dict( + id='msg-assistant', + timestamp=_dt(10, 3), + session_id='session-1', + message_content='{"chain": [{"type": "Plain", "text": "reply"}]}', + status='success', + role='assistant', + ), + dict( + id='msg-discarded', + timestamp=_dt(10, 4), + session_id='session-2', + message_content='{"chain": [{"type": "Plain", "text": "ignored"}]}', + status='discarded', + role='user', + ), + ]: + session.add( + persistence_monitoring.MonitoringMessage( + workspace_uuid=WORKSPACE, + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + level='info', + **message, + ) + ) + session.add_all( + [ + persistence_monitoring.MonitoringLLMCall( + id='llm-1', + workspace_uuid=WORKSPACE, + timestamp=_dt(10, 2, 30), + model_name='gpt', + input_tokens=10, + output_tokens=20, + total_tokens=30, + duration=100, + status='success', + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + session_id='session-1', + message_id='msg-user', + ), + persistence_monitoring.MonitoringLLMCall( + id='llm-2', + workspace_uuid=WORKSPACE, + timestamp=_dt(10, 4, 30), + model_name='gpt', + input_tokens=0, + output_tokens=0, + total_tokens=0, + duration=50, + status='success', + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + session_id='session-2', + message_id='msg-discarded', + ), + ] + ) + session.add( + persistence_monitoring.MonitoringSession( + workspace_uuid=WORKSPACE, + bot_id='bot-1', + session_id='session-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + message_count=2, + start_time=_dt(10, 2), + last_activity=_dt(10, 5), + is_active=True, + ) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_executions_merge_and_order_both_sources(engine, service): + await _seed(engine) + + result = await service.get_executions(WORKSPACE, limit=50) + + assert result['total'] == 5 + ids = [(item['source'], item['id']) for item in result['items']] + # Assistant replies are outputs, not executions. + assert ('pipeline', 'msg-assistant') not in ids + # Newest first across both object types. + assert ids == [ + ('agent', 'run-debug'), + ('agent', 'run-failed'), + ('pipeline', 'msg-discarded'), + ('pipeline', 'msg-user'), + ('agent', 'run-done'), + ] + + by_id = {item['id']: item for item in result['items']} + assert result['summary']['executions']['total'] == 5 + assert result['summary']['executions']['denominator'] == 4 + assert result['summary']['tokens']['calls'] == 2 + assert result['summary']['tokens']['calls_with_usage'] == 1 + assert by_id['run-done']['status_group'] == 'completed' + assert by_id['run-failed']['status_group'] == 'failed' + assert by_id['run-debug']['debug'] is True + assert by_id['run-done']['debug'] is False + assert by_id['run-done']['usage'] == {'total_tokens': 42} + assert by_id['run-done']['duration_ms'] == 2000 + assert by_id['msg-user']['status_group'] == 'completed' + assert by_id['msg-discarded']['status_group'] == 'ignored' + assert by_id['msg-user']['title'] == 'hello there' + assert by_id['msg-user']['target_id'] == 'pipeline-1' + + +@pytest.mark.asyncio +async def test_executions_source_status_and_paging(engine, service): + await _seed(engine) + + only_agents = await service.get_executions(WORKSPACE, source='agent') + assert {item['source'] for item in only_agents['items']} == {'agent'} + assert only_agents['total'] == 3 + + pipeline_only = await service.get_executions(WORKSPACE, source='pipeline') + assert {item['source'] for item in pipeline_only['items']} == {'pipeline'} + assert pipeline_only['total'] == 2 + + # A single-object target selection narrows the union to that source. + by_pipeline = await service.get_executions(WORKSPACE, pipeline_ids=['pipeline-1']) + assert {item['source'] for item in by_pipeline['items']} == {'pipeline'} + + failed = await service.get_executions(WORKSPACE, statuses=['failed']) + assert [item['id'] for item in failed['items']] == ['run-failed'] + + # `ignored` only exists for pipeline queries: it must not widen the agent + # filter back to "every status". + ignored = await service.get_executions(WORKSPACE, statuses=['ignored']) + assert [item['id'] for item in ignored['items']] == ['msg-discarded'] + + debug_only = await service.get_executions(WORKSPACE, mode='debug') + assert [item['id'] for item in debug_only['items']] == ['run-debug'] + + first_page = await service.get_executions(WORKSPACE, limit=2) + assert len(first_page['items']) == 2 + assert first_page['has_more'] is True + second_page = await service.get_executions(WORKSPACE, limit=2, offset=2) + assert first_page['items'][0]['id'] != second_page['items'][0]['id'] + + +@pytest.mark.asyncio +async def test_execution_summary_uses_explicit_denominator(engine, service): + await _seed(engine) + + summary = (await service.get_executions(WORKSPACE))['summary'] + executions = summary['executions'] + + # completed = run-done + run-debug + msg-user; failed = run-failed. + assert executions['by_source'] == {'agent': 3, 'pipeline': 2} + assert executions['completed'] == 3 + assert executions['failed'] == 1 + assert executions['ignored'] == 1 + assert executions['denominator'] == 4 + assert executions['success_rate'] == 75.0 + assert executions['debug'] == 1 + assert executions['real'] == 4 + assert executions['waiting'] == 1 + assert executions['p50_duration_ms'] == 1000 + assert executions['p95_duration_ms'] == 2000 + # Only run-done and run-debug are completed runs with a start and a finish. + assert executions['duration_sample'] == 2 + + # Token coverage is explicit: llm-2 succeeded without usage. + assert summary['tokens'] == {'total_tokens': 30, 'calls': 2, 'calls_with_usage': 1} + + +@pytest.mark.asyncio +async def test_overview_metrics_keep_message_rollup(engine, service): + await _seed(engine) + + overview = await service.get_overview_metrics(WORKSPACE) + + assert overview['total_messages'] == 3 + assert overview['llm_calls'] == 2 + assert overview['model_calls'] == 2 + assert overview['active_sessions'] == 1 + + +@pytest.mark.asyncio +async def test_execution_detail_for_agent_and_pipeline(engine, service): + await _seed(engine) + + agent_detail = await service.get_execution_detail(WORKSPACE, 'agent', 'run-done') + assert agent_detail['run']['run_id'] == 'run-done' + assert agent_detail['events'][0]['type'] == 'run.completed' + + pipeline_detail = await service.get_execution_detail(WORKSPACE, 'pipeline', 'msg-user') + assert pipeline_detail['message']['id'] == 'msg-user' + assert [call['id'] for call in pipeline_detail['llm_calls']] == ['llm-1'] + assert pipeline_detail['tool_calls'] == [] + + with pytest.raises(ValueError): + await service.get_execution_detail(WORKSPACE, 'agent', 'missing-run') + + with pytest.raises(ValueError): + await service.get_execution_detail(WORKSPACE, 'pipeline', 'missing-message') + + +@pytest.mark.asyncio +async def test_explicit_deliveries_context_and_tenant_isolation(engine, service): + import sqlalchemy as sa + + await _seed(engine) + async with AsyncSession(engine) as session: + await session.execute( + sa.update(persistence_monitoring.MonitoringMessage) + .where(persistence_monitoring.MonitoringMessage.id == 'msg-assistant') + .values(parent_message_id='msg-user') + ) + await session.commit() + detail = await service.get_execution_detail(WORKSPACE, 'auto', 'msg-assistant') + assert detail['row']['id'] == 'msg-user' + assert [m['id'] for m in detail['pages']['deliveries']['items']] == ['msg-assistant'] + assert detail['pages']['inputs']['items'][0]['id'] == 'msg-user' + assert detail['pages']['conversation']['items'] + with pytest.raises(ValueError, match='not found'): + await service.get_execution_detail('another-workspace', 'auto', 'msg-assistant') + + +@pytest.mark.asyncio +async def test_legacy_replies_are_context_not_guessed_deliveries(engine, service): + await _seed(engine) + detail = await service.get_execution_detail(WORKSPACE, 'pipeline', 'msg-user') + assert detail['legacy_context'] is True + assert detail['pages']['deliveries']['items'] == [] + assert 'msg-assistant' in [m['id'] for m in detail['pages']['conversation']['items']] + + +@pytest.mark.asyncio +async def test_fanout_steering_orphans_and_paged_trace(engine, service): + import json + import sqlalchemy as sa + from langbot.pkg.entity.persistence.event_log import EventLog + from langbot.pkg.entity.persistence.transcript import Transcript + + await _seed(engine) + async with AsyncSession(engine) as session: + await session.execute( + sa.update(persistence_agent_run.AgentRun) + .where(persistence_agent_run.AgentRun.run_id == 'run-done') + .values( + conversation_id='shared', + thread_id='thread-1', + metadata_json=json.dumps({'processor_type': 'event_processor'}), + ) + ) + session.add_all( + [ + EventLog( + event_id='evt-1', + event_type='message.received', + source='platform', + workspace_id=WORKSPACE, + bot_id='bot-1', + input_json='{"text":"original input"}', + actor_name='Alice', + created_at=_dt(10), + ), + EventLog( + event_id='steer', + event_type='message.received', + source='platform', + workspace_id=WORKSPACE, + bot_id='bot-1', + run_id='run-done', + input_json='{"text":"additional input"}', + created_at=_dt(10, 1), + ), + EventLog( + event_id='unrouted', + event_type='group.member.joined', + source='platform', + workspace_id=WORKSPACE, + bot_id='bot-1', + metadata_json='{"status":"ignored"}', + created_at=_dt(10, 7), + ), + persistence_agent_run.AgentRun( + run_id='sibling', + event_id='evt-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + runner_id='plugin', + status='completed', + created_at=_dt(10), + ), + Transcript( + transcript_id='output', + event_id='generated', + conversation_id='shared', + thread_id='thread-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + role='assistant', + content='generated only', + run_id='run-done', + seq=1, + ), + Transcript( + transcript_id='other-thread', + event_id='other', + conversation_id='shared', + thread_id='thread-2', + workspace_id=WORKSPACE, + bot_id='bot-1', + role='user', + content='private other thread', + seq=2, + ), + ] + ) + session.add_all( + [ + persistence_agent_run.AgentRunEvent(run_id='run-done', sequence=i, type='message.delta', data_json='{}') + for i in range(2, 206) + ] + ) + await session.commit() + result = await service.get_executions(WORKSPACE) + assert 'unrouted' in [r['id'] for r in result['items']] + assert 'steer' not in [r['id'] for r in result['items']] + ignored = await service.get_executions(WORKSPACE, statuses=['ignored']) + assert 'unrouted' in [r['id'] for r in ignored['items']] + detail = await service.get_execution_detail(WORKSPACE, 'agent', 'run-done') + assert detail['row']['target_kind'] == 'event_processor' + assert detail['row']['user_name'] == 'Alice' + assert [i['id'] for i in detail['pages']['inputs']['items']] == ['evt-1', 'steer'] + assert detail['pages']['outputs']['items'][0]['content'] == 'generated only' + assert [i['id'] for i in detail['pages']['related']['items']] == ['sibling'] + assert 'other-thread' not in [i['id'] for i in detail['pages']['conversation']['items']] + assert len(detail['pages']['events']['items']) == 100 + assert detail['pages']['events']['has_more'] + final = await service.get_execution_detail(WORKSPACE, 'agent', 'run-done', section='events', offset=200) + assert len(final['pages']['events']['items']) == 5 + assert not final['pages']['events']['has_more'] + orphan = await service.get_execution_detail(WORKSPACE, 'event', 'unrouted') + assert orphan['pages']['deliveries']['items'] == [] + assert orphan['pages']['conversation']['items'] == [] + + +@pytest.mark.asyncio +async def test_pipeline_runner_is_not_counted_twice(engine, service): + import sqlalchemy as sa + + await _seed(engine) + async with AsyncSession(engine) as session: + await session.execute( + sa.update(persistence_monitoring.MonitoringMessage) + .where(persistence_monitoring.MonitoringMessage.id == 'msg-user') + .values(run_id='run-done') + ) + await session.commit() + result = await service.get_executions(WORKSPACE) + assert result['total'] == 4 + assert result['summary']['executions']['total'] == 4 + assert 'run-done' not in [r['id'] for r in result['items']] + detail = await service.get_execution_detail(WORKSPACE, 'pipeline', 'msg-user') + assert detail['row']['duration_ms'] == 2000 + assert detail['events'][0]['type'] == 'run.completed' + + +@pytest.mark.asyncio +async def test_event_log_fanout_is_idempotent_and_cannot_cross_tenants(engine): + from langbot.pkg.agent.runner.event_log_store import EventLogStore + + store = EventLogStore(engine) + await store.append_event('event-id', 'message.received', 'platform', workspace_id=WORKSPACE, bot_id='bot') + await store.append_event( + 'event-id', + 'message.received', + 'platform', + workspace_id=WORKSPACE, + bot_id='bot', + run_id='first-run', + input_json={'text': 'input', 'attachments': []}, + ) + await store.append_event( + 'event-id', 'message.received', 'platform', workspace_id=WORKSPACE, bot_id='bot', run_id='second-run' + ) + event = await store.get_event('event-id') + assert event['run_id'] == 'first-run' + assert event['input_json']['text'] == 'input' + with pytest.raises(Exception): + await store.append_event('event-id', 'message.received', 'platform', workspace_id='other', bot_id='bot') + + +@pytest.mark.asyncio +async def test_processor_filter_scopes_each_kind_and_legacy_calls(engine, service): + import json + + await _seed(engine) + async with AsyncSession(engine) as session: + session.add( + persistence_agent_run.AgentRun( + run_id='event-run', + workspace_id=WORKSPACE, + runner_id='event-runner', + status='completed', + created_at=_dt(11), + metadata_json=json.dumps({'processor_type': 'event_processor', 'processor_id': 'event-processor-1'}), + ) + ) + session.add( + persistence_monitoring.MonitoringLLMCall( + id='event-call', + bot_id='bot-1', + bot_name='Bot One', + pipeline_name='Event Processor', + session_id='session-event', + input_tokens=6, + output_tokens=6, + duration=1, + timestamp=_dt(11), + workspace_uuid=WORKSPACE, + model_name='model', + message_id='event-run', + pipeline_id='', + status='success', + total_tokens=12, + ) + ) + await session.commit() + for processor, expected in [ + ('agent-1', {'run-done', 'run-debug'}), + ('pipeline-1', {'msg-user', 'msg-discarded'}), + ('event-processor-1', {'event-run'}), + ('missing', set()), + ]: + result = await service.get_executions(WORKSPACE, pipeline_ids=[processor]) + assert {row['id'] for row in result['items']} == expected + assert result['total'] == result['summary']['executions']['total'] == len(expected) + calls, total = await service.get_llm_calls(WORKSPACE, pipeline_ids=['event-processor-1']) + assert total == 1 + assert [row['id'] for row in calls] == ['event-call'] + other = await service.get_executions('another-workspace', pipeline_ids=['event-processor-1']) + assert other['total'] == 0 + + +@pytest.mark.asyncio +async def test_execution_filters_also_scope_cards_calls_and_traffic(engine, service): + import sqlalchemy as sa + from langbot.pkg.api.http.service.monitoring_traffic import get_traffic_series + + await _seed(engine) + async with AsyncSession(engine) as session: + await session.execute( + sa.update(persistence_monitoring.MonitoringLLMCall) + .where(persistence_monitoring.MonitoringLLMCall.id == 'llm-1') + .values(message_id='run-failed', pipeline_id='') + ) + await session.commit() + failed = await service.get_executions(WORKSPACE, statuses=['failed']) + assert failed['total'] == failed['summary']['executions']['total'] == 1 + assert failed['summary']['executions']['waiting'] == 0 + calls, total = await service.get_llm_calls(WORKSPACE, execution_statuses=['failed']) + assert total == 1 and calls[0]['id'] == 'llm-1' + # A successful model call may belong to a failed execution. + assert calls[0]['status'] == 'success' + overview = await service.get_overview_metrics(WORKSPACE, execution_statuses=['failed']) + assert overview['llm_calls'] == 1 and overview['total_messages'] == 0 + traffic = await get_traffic_series(service.ap, WORKSPACE, execution_statuses=['failed']) + assert sum(p['llm_calls'] for p in traffic['points']) == 1 + assert sum(p['messages'] for p in traffic['points']) == 0 + + debug = await service.get_executions(WORKSPACE, mode='debug') + assert debug['total'] == debug['summary']['executions']['total'] == 1 + real = await service.get_executions(WORKSPACE, mode='real') + assert real['total'] == real['summary']['executions']['total'] == 4 + traffic = await get_traffic_series(service.ap, WORKSPACE, mode='debug') + assert sum(p['messages'] for p in traffic['points']) == 0 + + +@pytest.mark.asyncio +async def test_pipeline_debug_is_consistent_across_rows_rollups_and_telemetry(engine, service): + import sqlalchemy as sa + from langbot.pkg.api.http.service.monitoring_traffic import get_traffic_series + + await _seed(engine) + async with AsyncSession(engine) as session: + await session.execute( + sa.update(persistence_monitoring.MonitoringMessage) + .where(persistence_monitoring.MonitoringMessage.id == 'msg-user') + .values(bot_id='websocket-proxy-bot') + ) + await session.execute( + sa.update(persistence_monitoring.MonitoringLLMCall) + .where(persistence_monitoring.MonitoringLLMCall.id == 'llm-1') + .values(message_id='msg-user', pipeline_id='pipeline-1') + ) + await session.execute( + sa.update(persistence_agent_run.AgentRun) + .where(persistence_agent_run.AgentRun.run_id == 'run-done') + .values(bot_id='websocket-proxy-bot') + ) + await session.commit() + debug = await service.get_executions(WORKSPACE, mode='debug') + ids = {r['id'] for r in debug['items']} + assert ids == {'msg-user', 'run-done', 'run-debug'} + assert all(r['debug'] for r in debug['items']) + assert debug['summary']['executions']['total'] == 3 + assert debug['summary']['executions']['debug'] == 3 + real = await service.get_executions(WORKSPACE, mode='real') + assert not ids.intersection(r['id'] for r in real['items']) + assert real['summary']['executions']['debug'] == 0 + filtered = await service.get_executions(WORKSPACE, pipeline_ids=['pipeline-1'], mode='debug') + assert [r['id'] for r in filtered['items']] == ['msg-user'] + calls, total = await service.get_llm_calls(WORKSPACE, pipeline_ids=['pipeline-1'], mode='debug') + assert total == 1 and calls[0]['id'] == 'llm-1' + real_rows, real_calls = await service.get_llm_calls(WORKSPACE, pipeline_ids=['pipeline-1'], mode='real') + assert real_calls == 1 and real_rows[0]['id'] == 'llm-2' + traffic = await get_traffic_series(service.ap, WORKSPACE, mode='debug', pipeline_ids=['pipeline-1']) + assert sum(p['llm_calls'] for p in traffic['points']) == 1 diff --git a/tests/unit_tests/api/service/test_space_service.py b/tests/unit_tests/api/service/test_space_service.py index a05754af5..8dd1543e7 100644 --- a/tests/unit_tests/api/service/test_space_service.py +++ b/tests/unit_tests/api/service/test_space_service.py @@ -1025,3 +1025,45 @@ class TestSpaceServiceCreditsCache: assert result == 500 assert 'test@example.com' in service._credits_cache assert service._credits_cache['test@example.com'][0] == 500 + + +@pytest.mark.parametrize('status', [200, 503]) +async def test_assistant_recommendation_uses_dedicated_endpoint_without_fallback(status): + service = SpaceService(SimpleNamespace(instance_config=SimpleNamespace(data={}))) + service.get_model_selection = AsyncMock() + service._resolve_recommended_model = AsyncMock(return_value={'uuid': 'local', 'name': 'deepseek-v4-flash'}) + response = MagicMock(status=status) + _set_response_body( + response, + { + 'code': 0, + 'data': { + 'model': { + 'uuid': 'upstream', + 'model_id': 'deepseek-v4-flash', + 'category': 'chat', + 'provider': 'deepseek', + 'status': 'active', + } + }, + }, + ) + session = MagicMock() + session.get.return_value.__aenter__ = AsyncMock(return_value=response) + session.get.return_value.__aexit__ = AsyncMock(return_value=None) + context = object() + with patch('langbot.pkg.api.http.service.space.httpclient.get_session', return_value=session): + if status == 200: + assert await service.get_recommended_assistant_model(context) == { + 'uuid': 'local', + 'name': 'deepseek-v4-flash', + } + args = service._resolve_recommended_model.await_args.args + assert args[0] is context + assert args[1].model_id == 'deepseek-v4-flash' + else: + with pytest.raises(ValueError, match='No recommended assistant model'): + await service.get_recommended_assistant_model(context) + service._resolve_recommended_model.assert_not_awaited() + service.get_model_selection.assert_not_awaited() + session.get.assert_called_once_with('https://space.langbot.app/api/v1/models/assistant/recommendation') diff --git a/tests/unit_tests/api/test_management_assistant.py b/tests/unit_tests/api/test_management_assistant.py index a3f993e6b..8e6fc78b7 100644 --- a/tests/unit_tests/api/test_management_assistant.py +++ b/tests/unit_tests/api/test_management_assistant.py @@ -74,7 +74,7 @@ async def assistant(tmp_path): ap = SimpleNamespace( persistence_mgr=SimpleNamespace(execute_async=execute), logger=logging.getLogger('assistant-test'), - space_service=SimpleNamespace(get_recommended_chat_model=AsyncMock(return_value={'uuid': 'model'})), + space_service=SimpleNamespace(get_recommended_assistant_model=AsyncMock(return_value={'uuid': 'model'})), model_mgr=SimpleNamespace(get_model_by_uuid=AsyncMock(return_value=model)), pipeline_service=SimpleNamespace(create_pipeline=AsyncMock(return_value='created-pipeline')), ) @@ -234,3 +234,250 @@ async def test_model_switch_preserves_history_and_rejects_invalid_selection(assi ap.model_mgr.get_model_by_uuid.assert_awaited_with( next_provider.invoke_llm.call_args.kwargs['execution_context'], 'second' ) + + +@pytest.mark.asyncio +async def test_recommendation_resolved_before_any_turn(assistant): + service, ap, provider = assistant + assert await service.recommended_model(context()) == {'uuid': 'model', 'name': 'test-model'} + provider.invoke_llm.assert_not_awaited() + model = ap.model_mgr.get_model_by_uuid.return_value + model.model_entity.abilities = [] + with pytest.raises(AssistantError, match='model_unavailable'): + await service.recommended_model(context()) + + +@pytest.mark.asyncio +async def test_stream_selected_model_deltas_tools_and_approval(assistant): + from langbot_plugin.api.entities.builtin.provider.message import MessageChunk + import copy + + service, ap, old_provider = assistant + events = [] + inputs = [] + + async def stream(**kwargs): + inputs.append(kwargs) + if len(inputs) == 1: + yield MessageChunk(role='assistant', content='Create ') + yield MessageChunk(role='assistant', content='this?') + call = proposal().tool_calls[0].model_dump() + call['function']['arguments'] = '{"name":"Demo",' + yield MessageChunk(role='assistant', tool_calls=[call]) + call['function']['arguments'] = '"description":"Test draft"}' + yield MessageChunk( + role='assistant', tool_calls=[call], is_final=True, provider_specific_fields={'signature': 'kept'} + ) + else: + yield MessageChunk(role='assistant', content='Done') + yield MessageChunk(role='assistant', content='', all_content='Done', is_final=True) + + chosen = SimpleNamespace( + provider=SimpleNamespace(invoke_llm_stream=stream), + model_entity=SimpleNamespace(name='manual-model', abilities=['func_call'], extra_args={}), + ) + ap.model_mgr.get_model_by_uuid.return_value = chosen + + async def emit(event): + events.append(copy.deepcopy(event)) + + ctx = context() + cid = (await service.create(ctx))['uuid'] + pending = await service.turn(ctx, cid, 0, text='Create', model_uuid='manual', on_event=emit) + assert pending['status'] == 'approval' + assert pending['model_uuid'] == 'manual' + assert inputs[0]['model'] is chosen + ap.space_service.get_recommended_assistant_model.assert_not_awaited() + old_provider.invoke_llm.assert_not_awaited() + assert [e['data']['text'] for e in events if e['kind'] == 'text'] == ['Create ', 'Create this?'] + ap.pipeline_service.create_pipeline.assert_not_awaited() + events.clear() + complete = await service.turn(ctx, cid, 1, approved=True, on_event=emit) + assert complete['status'] == 'ready' + assert complete['messages'][-1]['content'] == 'Done' + ap.pipeline_service.create_pipeline.assert_awaited_once() + assert inputs[1]['messages'][2].provider_specific_fields == {'signature': 'kept'} + tool_start = next(i for i, e in enumerate(events) if e['kind'] == 'phase' and e['data']['phase'] == 'tool') + result = next( + i + for i, e in enumerate(events) + if e['kind'] == 'snapshot' and any(m['role'] == 'tool' for m in e['data']['messages']) + ) + assert tool_start < result + with pytest.raises(AssistantError, match='stale_turn'): + await service.turn(ctx, cid, 1, approved=True, on_event=emit) + + +@pytest.mark.asyncio +async def test_truncated_stream_never_executes_proposed_write(assistant): + from langbot_plugin.api.entities.builtin.provider.message import MessageChunk + + service, ap, provider = assistant + + async def stream(**kwargs): + yield MessageChunk(role='assistant', tool_calls=proposal().tool_calls) + + provider.invoke_llm_stream = stream + cid = (await service.create(context()))['uuid'] + result = await service.turn(context(), cid, 0, text='Create', on_event=AsyncMock()) + assert result['status'] == 'failed' + assert result['error'] == 'stream_interrupted' + ap.pipeline_service.create_pipeline.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_cancelled_stream_persists_non_replayable_state(assistant): + from langbot_plugin.api.entities.builtin.provider.message import MessageChunk + + service, ap, provider = assistant + started = asyncio.Event() + + async def stream(**kwargs): + yield MessageChunk(role='assistant', content='Hello') + started.set() + await asyncio.Event().wait() + + provider.invoke_llm_stream = stream + ctx = context() + cid = (await service.create(ctx))['uuid'] + task = asyncio.create_task(service.turn(ctx, cid, 0, text='Hello', on_event=AsyncMock())) + await asyncio.wait_for(started.wait(), 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + saved = await service.get(ctx, cid) + assert saved['status'] == 'failed' + assert saved['error'] == 'result_unknown' + with pytest.raises(AssistantError, match='stale_turn'): + await service.turn(ctx, cid, 0, text='Hello') + + +@pytest.mark.asyncio +async def test_http_stream_emits_before_provider_completion_and_restores_tenant(assistant): + import contextlib + import json + import quart + from langbot_plugin.api.entities.builtin.provider.message import MessageChunk + from langbot.pkg.api.http.controller.groups.assistant import assistant_stream_response, TurnInput + + service, ap, provider = assistant + released = asyncio.Event() + scoped = [] + + @contextlib.asynccontextmanager + async def tenant_scope(workspace): + scoped.append(workspace) + try: + yield + finally: + scoped.pop() + + async def stream(**kwargs): + assert scoped == ['workspace-a'] + yield MessageChunk(role='assistant', content='First') + await released.wait() + yield MessageChunk(role='assistant', content=' second', is_final=True) + + ap.persistence_mgr.tenant_scope = tenant_scope + provider.invoke_llm_stream = stream + ctx = context() + cid = (await service.create(ctx))['uuid'] + app = quart.Quart(__name__) + + @app.post('/stream') + async def route(): + return assistant_stream_response(service, ctx, cid, TurnInput(revision=0, text='Hi')) + + frames = [] + async with app.test_client().request('/stream', method='POST') as connection: + while not any(frame['kind'] == 'text' for frame in frames): + chunk = await asyncio.wait_for(connection.receive(), 5) + frames.extend(json.loads(line) for line in chunk.decode().splitlines() if line.strip()) + assert frames[-1]['data']['text'] == 'First' + assert not released.is_set() + released.set() + while not any(frame['kind'] == 'completed' for frame in frames): + chunk = await asyncio.wait_for(connection.receive(), 5) + frames.extend(json.loads(line) for line in chunk.decode().splitlines() if line.strip()) + assert frames[-1]['data']['messages'][-1]['content'] == 'First second' + assert scoped == [] + + +@pytest.mark.asyncio +async def test_background_run_survives_http_disconnect_and_finishes(assistant): + import json + import quart + from langbot_plugin.api.entities.builtin.provider.message import MessageChunk + from langbot.pkg.api.http.controller.groups.assistant import assistant_stream_response, TurnInput + + service, ap, provider = assistant + released = asyncio.Event() + + async def stream(**kwargs): + yield MessageChunk(role='assistant', content='Background') + await released.wait() + yield MessageChunk(role='assistant', content=' done', is_final=True) + + provider.invoke_llm_stream = stream + ctx = context() + cid = (await service.create(ctx))['uuid'] + app = quart.Quart(__name__) + + @app.post('/stream') + async def route(): + return assistant_stream_response(service, ctx, cid, TurnInput(revision=0, text='Hi')) + + async with app.test_client().request('/stream', method='POST') as connection: + while True: + chunk = await asyncio.wait_for(connection.receive(), 5) + if any(json.loads(line)['kind'] == 'text' for line in chunk.decode().splitlines() if line.strip()): + break + task = service._runs[cid]['task'] + await connection.disconnect() + assert not task.done() + assert (await service.progress(ctx, cid))['progress']['text'] == 'Background' + released.set() + await asyncio.wait_for(task, 5) + completed = await service.get(ctx, cid) + assert completed['status'] == 'ready' + assert completed['messages'][-1]['content'] == 'Background done' + + +@pytest.mark.asyncio +async def test_stop_is_private_revision_scoped_and_does_not_stop_other_chat(assistant): + from langbot_plugin.api.entities.builtin.provider.message import MessageChunk + from langbot.pkg.api.http.controller.groups.assistant import TurnInput + + service, ap, provider = assistant + released = asyncio.Event() + + async def stream(**kwargs): + yield MessageChunk(role='assistant', content='Partial') + await released.wait() + yield MessageChunk(role='assistant', content=' done', is_final=True) + + provider.invoke_llm_stream = stream + ctx = context() + ids = [(await service.create(ctx))['uuid'] for _ in range(2)] + tasks = [] + for cid in ids: + queue = service.start_run(ctx, cid, TurnInput(revision=0, text='Background task')) + tasks.append(service._runs[cid]['task']) + while (await asyncio.wait_for(queue.get(), 5))['kind'] != 'text': + pass + for other in (context(account='bob'), context(workspace='workspace-b')): + assert await service.list_conversations(other) == [] + with pytest.raises(AssistantError, match='conversation_not_found'): + await service.stop(other, ids[0], 1) + with pytest.raises(AssistantError, match='stale_turn'): + await service.stop(ctx, ids[0], 0) + result = await service.stop(ctx, ids[0], 1) + assert result['status'] == 'failed' + assert result['error'] == 'stopped' + assert result['messages'][-1]['content'] == 'Partial' + assert not tasks[1].done() + assert len(await service.list_conversations(ctx)) == 2 + assert (await service.stop(ctx, ids[0], 1))['error'] == 'stopped' + released.set() + await asyncio.wait_for(tasks[1], 5) + assert (await service.get(ctx, ids[1]))['status'] == 'ready' diff --git a/tests/unit_tests/persistence/test_tenant_uow.py b/tests/unit_tests/persistence/test_tenant_uow.py index 0c374f201..7b624d2ee 100644 --- a/tests/unit_tests/persistence/test_tenant_uow.py +++ b/tests/unit_tests/persistence/test_tenant_uow.py @@ -980,6 +980,18 @@ async def test_scoped_sql_structure_allows_only_the_production_vocabulary(statem _validate_scoped_statement_call((statement,), {}) +async def test_monitoring_json_casts_are_limited_to_journal_metadata() -> None: + from langbot.pkg.entity.persistence.agent_run import AgentRun + from langbot.pkg.entity.persistence.event_log import EventLog + + for column in (AgentRun.metadata_json, EventLog.metadata_json): + statement = sa.select(sa.cast(column, sa.JSON)['status'].as_string()) + _validate_scoped_statement_call((statement,), {}) + for column in (AgentRun.authorization_json, EventLog.input_json, sa.literal('{}')): + with pytest.raises(ScopedSessionTransactionError, match='casts'): + _validate_scoped_statement_call((sa.select(sa.cast(column, sa.JSON)),), {}) + + async def test_scoped_session_executes_sqlite_strftime() -> None: engine = create_async_engine('sqlite+aiosqlite:///:memory:') try: diff --git a/tests/unit_tests/platform/test_botmgr_execution_trace.py b/tests/unit_tests/platform/test_botmgr_execution_trace.py index 12f55fde6..2002f3045 100644 --- a/tests/unit_tests/platform/test_botmgr_execution_trace.py +++ b/tests/unit_tests/platform/test_botmgr_execution_trace.py @@ -1,9 +1,10 @@ -"""RuntimeBot ingress tracing: one inbound event owns one execution trace.""" +"""RuntimeBot ingress tracing: one inbound event owns one v2 execution chain.""" from __future__ import annotations from types import SimpleNamespace from unittest.mock import AsyncMock, Mock +import asyncio import pytest @@ -28,9 +29,6 @@ class FakeTelemetryManager: self.sent.append(payload) return True - def start_send_task(self, payload: dict) -> None: - self.sent.append(payload) - def make_bot(event_bindings: list[dict], config=None, agent=None): from langbot.pkg.platform.botmgr import RuntimeBot @@ -75,35 +73,43 @@ class FakeAdapter: return [] +async def flushed_records(counters, manager) -> list[dict]: + """Flush buffered telemetry and return everything the event emitted. + + Records are batched and uploaded by ``ExecutionCounters.flush()``; there is + no per-trace direct send any more. + """ + + await counters.flush() + assert manager.sent, 'expected the flushed telemetry batch' + return [record for payload in manager.sent for record in payload.get('records', [])] + + +def execution_nodes(records: list[dict]) -> list[dict]: + return [record for record in records if record.get('event_type') == 'execution_node'] + + +def execution_chains(records: list[dict]) -> list[dict]: + return [record for record in records if record.get('event_type') == 'execution_chain'] + + @pytest.mark.asyncio -async def test_route_miss_emits_one_closed_trace(): +async def test_route_miss_emits_nothing(): bot, manager, counters = make_bot([]) await bot._handle_platform_event(message_received_event(), FakeAdapter()) - assert len(manager.sent) == 1 - payload = manager.sent[0] - assert payload['event_type'] == 'feature_execution' - assert len(payload['query_id']) == 36 - assert payload['instance_id'] == 'instance-test' - assert payload['workspace_uuid'] == 'workspace-test' - features = payload['features'] - assert features['schema'] == 1 - assert [(row['family'], row['operation'], row['outcome']) for row in features['observations']] == [ - ('platform_event', 'message.received', 'success'), - ('event_route', 'message.received', 'skipped'), - ] - assert [row['seq'] for row in features['observations']] == [0, 1] - assert all(row['trace_id'] == payload['query_id'] for row in features['observations']) - assert all(row['adapter'] == 'FakeAdapter' for row in features['observations']) - assert features['trace']['closed_by'] == 'event_done' - # Window counters are still aggregated for coverage. - assert counters.pending + await counters.flush() + await counters.shutdown() + assert manager.sent == [] + assert counters.records == [] + assert counters.dropped == 0 + assert counters.registry == {} @pytest.mark.asyncio async def test_unavailable_route_target_carries_route_identity(): - bot, manager, _ = make_bot( + bot, manager, counters = make_bot( [ { 'id': 'agent-binding', @@ -117,18 +123,18 @@ async def test_unavailable_route_target_carries_route_identity(): await bot._handle_platform_event(message_received_event(), FakeAdapter()) - stages = manager.sent[0]['features']['observations'] - assert [(stage['family'], stage['outcome'], stage['mode']) for stage in stages] == [ - ('platform_event', 'success', 'none'), + nodes = execution_nodes(await flushed_records(counters, manager)) + assert [(node['family'], node['outcome'], node['mode']) for node in nodes] == [ ('event_route', 'failed', 'agent'), + ('platform_event', 'success', 'none'), ] - assert stages[1]['route_ref'] == 'agent:agent-1' - assert stages[0]['route_ref'] == '' + assert nodes[0]['route_ref'] == 'agent:agent-1' + assert nodes[1]['route_ref'] == '' @pytest.mark.asyncio -async def test_discarded_route_is_visible_without_user_values(): - bot, manager, _ = make_bot( +async def test_discarded_route_is_only_logged_locally(): + bot, manager, counters = make_bot( [ { 'id': 'discard-binding', @@ -141,10 +147,10 @@ async def test_discarded_route_is_visible_without_user_values(): await bot._handle_platform_event(SimpleNamespace(type='platform.member.joined'), FakeAdapter()) - stages = manager.sent[0]['features']['observations'] - assert [stage['outcome'] for stage in stages] == ['success', 'skipped'] - assert stages[1]['operation'] == 'platform.member.joined' - assert stages[1]['mode'] == 'none' + await counters.flush() + assert manager.sent == [] + assert counters.records == [] + assert bot.logger.info.await_args.kwargs['metadata']['status'] == 'discarded' @pytest.mark.asyncio @@ -154,33 +160,402 @@ async def test_telemetry_opt_out_emits_nothing(): await bot._handle_platform_event(message_received_event(), FakeAdapter()) assert manager.sent == [] - assert counters.pending == {} - assert counters.traces == {} + assert counters.records == [] + assert counters.registry == {} @pytest.mark.asyncio async def test_route_trace_scope_is_restored_after_dispatch(): from langbot.pkg.telemetry import trace as trace_mod - bot, _, _ = make_bot([]) + bot, _, counters = make_bot([]) assert trace_mod.current() is None await bot._handle_platform_event(message_received_event(), FakeAdapter()) assert trace_mod.current() is None - # A later record outside the ingress must not join the closed trace. + buffered = list(counters.records) + # A later record outside the ingress must not join the closed chain, and an + # observation without an owning chain is not buffered at all. from langbot.pkg.telemetry.execution import record record(bot.ap, TEST_CONTEXT, family='platform_api', operation='send_message', outcome='success') - assert list(bot.ap.telemetry.execution.pending.values())[-1]['count'] == 1 + assert counters.records == buffered + assert counters.registry == {} @pytest.mark.asyncio -async def test_adapter_call_without_ingress_still_counts(): +async def test_mock_adapter_route_miss_is_not_reported(): """Mock adapter objects used by other tests must not break the ingress path.""" - bot, manager, _ = make_bot([]) + bot, manager, counters = make_bot([]) adapter = Mock() await bot._handle_platform_event(message_received_event(), adapter) - assert manager.sent + await counters.flush() + assert manager.sent == [] + + +def business_route(**overrides): + return { + 'id': 'route', + 'enabled': True, + 'event_pattern': '*', + 'target_type': 'agent', + 'target_uuid': 'agent-1', + **overrides, + } + + +async def assert_no_upload(counters, manager): + assert counters.records == [] + assert counters.task is None + await counters.flush() + await counters.shutdown() + assert manager.sent == [] + assert counters.health() == { + 'records_buffered': 0, + 'records_sent': 0, + 'records_dropped': 0, + 'executions_in_flight': 0, + } + + +@pytest.mark.parametrize('edition', ['community', 'cloud']) +@pytest.mark.parametrize('mode', ['all', 'sampled', 'failures', 'off']) +@pytest.mark.parametrize( + 'routes', + [ + [], + [business_route(enabled=False)], + [business_route(event_pattern='group.member_joined')], + [business_route(filters=[{'field': 'chat_id', 'operator': 'eq', 'value': 'another-chat'}])], + [business_route(target_type='discard', priority=10), business_route(priority=1)], + ], +) +async def test_unrouted_events_never_upload_in_any_mode(monkeypatch, edition, mode, routes): + from langbot.pkg.utils import constants + + monkeypatch.setattr(constants, 'edition', edition) + bot, manager, counters = make_bot(routes, config={'url': 'https://space.example.test', 'execution_trace': mode}) + bot._dispatch_eba_message_to_pipeline = AsyncMock() + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + await assert_no_upload(counters, manager) + # Normal local route diagnostics still exist, including explicit discard. + assert bot.logger.info.await_args.kwargs['metadata']['status'] in {'not_matched', 'discarded'} + + +@pytest.mark.parametrize('mode', ['all', 'sampled', 'failures']) +async def test_matched_target_failure_is_retained_even_after_root_success(mode): + bot, manager, counters = make_bot( + [business_route()], + config={ + 'url': 'https://space.example.test', + 'execution_trace': mode, + 'execution_trace_sample': 100000, + }, + ) + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + records = await flushed_records(counters, manager) + (chain,) = execution_chains(records) + assert chain['outcome'] == 'failed' + assert chain['error'] == 'processor_not_found' + if mode == 'failures': + assert execution_nodes(records) == [] + await counters.shutdown() + + +def install_processor(bot, patterns=None): + bot.bot_entity.plugin_processors = [{'processor_uuid': 'processor', 'enabled': True}] + bot.ap.agent_service.get_agent = AsyncMock( + return_value={ + 'kind': 'event_processor', + 'component_ref': 'plugin:test/runner/default', + } + ) + bot.ap.runner_registry = SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + usages=['event'], + supported_event_patterns=['*'] if patterns is None else patterns, + ) + ) + ) + bot._agent_product_to_binding = Mock(return_value=SimpleNamespace()) + bot._eba_event_to_agent_envelope = Mock(return_value=SimpleNamespace()) + + async def run(*args, **kwargs): + from langbot.pkg.telemetry.execution import record + + record(bot.ap, TEST_CONTEXT, family='runner', operation='execute', outcome='success') + if False: + yield None + + bot.ap.agent_run_orchestrator = SimpleNamespace(run=run) + + +@pytest.mark.parametrize('primary', ['miss', 'discard']) +@pytest.mark.parametrize('primary_first', [True, False]) +async def test_matching_subscription_admits_chain_regardless_of_primary_order(primary, primary_first): + from langbot.pkg.telemetry.execution import record + + routes = [] if primary == 'miss' else [business_route(target_type='discard')] + bot, manager, counters = make_bot(routes) + install_processor(bot) + bot.bot_entity.plugin_processors *= 2 # Duplicate subscriptions still execute once. + original = bot._dispatch_eba_event_to_processor + first_done = asyncio.Event() + run_calls = [] + + async def ordered_dispatch(event, adapter, event_binding=None, agent=None, execution_id=None): + is_primary = event_binding is None + if is_primary != primary_first: + await first_done.wait() + result = await original(event, adapter, event_binding, agent, execution_id) + if not is_primary: + run_calls.append(result) + if is_primary == primary_first: + first_done.set() + return result + + async def discard_side_effect(*args, **kwargs): + # A discarded message can still trigger a webhook/adapter side effect. + # A concurrent admitted subscriber must not expose this branch's node. + record(bot.ap, TEST_CONTEXT, family='platform_api', operation='discard_side_effect', outcome='success') + await counters.flush() + + bot._dispatch_eba_event_to_processor = ordered_dispatch + bot._dispatch_eba_message_to_pipeline = discard_side_effect + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + records = await flushed_records(counters, manager) + (chain,) = execution_chains(records) + nodes = execution_nodes(records) + assert len(run_calls) == 1 + assert chain['stage_count'] == 3 + assert {node['family'] for node in nodes} == {'runner', 'event_route', 'platform_event'} + assert all(node['outcome'] == 'success' for node in nodes) + (root,) = [node for node in nodes if node['root']] + assert root['family'] == 'platform_event' + assert all(node['parent_node_id'] == root['node_id'] for node in nodes if not node['root']) + assert all(node['event_id'] == chain['event_id'] for node in nodes) + await counters.shutdown() + + +@pytest.mark.parametrize('kind', ['disabled', 'pattern_miss', 'no_patterns', 'missing', 'declaration_error']) +async def test_unmatched_subscription_cannot_admit_event(kind): + bot, manager, counters = make_bot([]) + install_processor(bot) + if kind == 'disabled': + bot.bot_entity.plugin_processors[0]['enabled'] = False + elif kind in {'pattern_miss', 'no_patterns'}: + install_processor(bot, [] if kind == 'no_patterns' else ['group.member_joined']) + elif kind == 'missing': + bot.ap.agent_service.get_agent.return_value = None + else: + bot.ap.runner_registry.get.side_effect = RuntimeError('unavailable') + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + await assert_no_upload(counters, manager) + + +@pytest.mark.parametrize('failure', [RuntimeError('runner failed'), TimeoutError('timeout'), asyncio.CancelledError()]) +async def test_matched_subscription_keeps_failed_timeout_and_cancelled_execution(failure): + bot, manager, counters = make_bot([], config={'url': 'https://space.example.test', 'execution_trace': 'failures'}) + install_processor(bot) + + async def run(*args, **kwargs): + raise failure + yield + + bot.ap.agent_run_orchestrator.run = run + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + (chain,) = execution_chains(await flushed_records(counters, manager)) + assert chain['outcome'] == 'failed' + assert chain['error'] + await counters.shutdown() + + +@pytest.mark.parametrize('processor', ['agent', 'pipeline', 'invalid', 'expired', 'duplicate']) +async def test_interaction_admission_requires_valid_callback(processor): + from langbot_plugin.api.entities.builtin.platform import events + + bot, manager, counters = make_bot([]) + callbacks = SimpleNamespace( + consume_callback=AsyncMock(return_value={'processor_type': processor, 'processor_id': 'target'}), + acknowledge_submission=AsyncMock(), + ) + if processor in {'expired', 'duplicate'}: + callbacks.consume_callback.side_effect = ValueError(processor) + bot.ap.agent_run_orchestrator = SimpleNamespace(interaction_manager=callbacks) + bot._resume_agent_interaction = AsyncMock(side_effect=ValueError('target unavailable')) + event = events.PlatformSpecificEvent(platform='test', action='interaction.submitted', data={}) + await bot._handle_platform_event(event, FakeAdapter()) + if processor in {'agent', 'pipeline'}: + (chain,) = execution_chains(await flushed_records(counters, manager)) + assert chain['outcome'] == 'failed' + callbacks.acknowledge_submission.assert_awaited_once() + await counters.shutdown() + else: + callbacks.acknowledge_submission.assert_not_awaited() + await assert_no_upload(counters, manager) + + +async def test_rejected_event_does_not_contaminate_next_matched_event_or_monitoring(): + from langbot.pkg.telemetry import trace + + bot, manager, counters = make_bot([]) + bot._persist_monitoring_ingress = AsyncMock() + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + assert bot._persist_monitoring_ingress.await_count == 2 + assert counters.records == [] + assert trace.current() is None + assert trace.node_stack() == () + assert counters.registry == {} + bot.bot_entity.event_bindings = [business_route()] + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + assert len(execution_chains(await flushed_records(counters, manager))) == 1 + await counters.shutdown() + + +async def test_pending_gate_blocks_nested_and_rpc_records_and_failure_fallback(): + import contextvars + from langbot.pkg.telemetry import execution, trace + + bot, manager, counters = make_bot([]) + with pytest.raises(RuntimeError): + with execution.ingress(bot.ap, context=TEST_CONTEXT, execution_id='pending', require_route=True, debug=True): + with execution.ingress(bot.ap, context=TEST_CONTEXT, debug=True): + execution.bind_trace(bot.ap, 'run-alias') + execution.record(bot.ap, TEST_CONTEXT, family='tool', operation='nested', outcome='failed') + + async def rpc(): + execution.record( + bot.ap, + TEST_CONTEXT, + execution_id='run-alias', + family='platform_api', + operation='rpc', + outcome='failed', + ) + await counters.flush() + + await contextvars.Context().run(asyncio.create_task, rpc()) + assert counters.records == [] + raise RuntimeError('failed before a route matched') + assert trace.current() is None + await assert_no_upload(counters, manager) + + +@pytest.mark.parametrize('origin', ['api', 'webui']) +async def test_independent_debug_entry_still_reports(origin): + from langbot.pkg.telemetry import execution + + bot, manager, counters = make_bot([], config={'url': 'https://space.example.test', 'execution_trace': 'failures'}) + with execution.ingress( + bot.ap, + context=TEST_CONTEXT, + execution_id='debug', + debug=True, + origin=origin, + synthetic_event='message.received', + ): + execution.record(bot.ap, TEST_CONTEXT, family='runner', operation='execute', outcome='success') + (chain,) = execution_chains(await flushed_records(counters, manager)) + assert chain['debug'] is True + assert chain['origin'] == origin + await counters.shutdown() + + +@pytest.mark.parametrize('edition', ['community', 'cloud']) +async def test_admission_does_not_override_telemetry_opt_out(monkeypatch, edition): + from langbot.pkg.utils import constants + + monkeypatch.setattr(constants, 'edition', edition) + bot, manager, counters = make_bot( + [business_route()], + config={ + 'url': 'https://space.example.test', + 'disable_telemetry': True, + }, + ) + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + await assert_no_upload(counters, manager) + + +async def test_unrelated_broken_subscription_does_not_poison_matched_delivery(): + bot, manager, counters = make_bot([business_route(target_type='pipeline')]) + bot._dispatch_eba_message_to_pipeline = AsyncMock() + bot.bot_entity.plugin_processors = [{'processor_uuid': 'missing'}] + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + records = await flushed_records(counters, manager) + (chain,) = execution_chains(records) + assert chain['outcome'] == 'success' + assert chain['error'] == '' + assert all(node['outcome'] == 'success' for node in execution_nodes(records)) + bot.logger.error.assert_awaited_once() + await counters.shutdown() + + +async def test_pending_event_cancelled_during_dispatch_never_uploads(): + bot, manager, counters = make_bot([]) + started = asyncio.Event() + + async def waiting_dispatch(*args, **kwargs): + started.set() + await asyncio.Event().wait() + + bot._dispatch_eba_event_to_processor = waiting_dispatch + task = asyncio.create_task(bot._handle_platform_event(message_received_event(), FakeAdapter())) + await started.wait() + await counters.flush() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await assert_no_upload(counters, manager) + + +async def test_retry_of_matched_batch_cannot_include_intervening_unrouted_event(): + bot, manager, counters = make_bot([business_route()]) + await bot._handle_platform_event(message_received_event(), FakeAdapter()) + expected = list(counters.records) + original_send = manager.send + manager.send = AsyncMock(return_value=False) + assert await counters.flush() is False + assert counters.records == expected + bot.bot_entity.event_bindings = [] + ignored = message_received_event() + ignored.message_id = 'ignored-message' + await bot._handle_platform_event(ignored, FakeAdapter()) + assert counters.records == expected + manager.send = original_send + await counters.shutdown() + sent = [record for batch in manager.sent for record in batch['records']] + assert sent == expected + assert counters.dropped == 0 + + +async def test_valid_interaction_success_keeps_acknowledgement_and_execution(): + from langbot.pkg.telemetry.execution import record + from langbot_plugin.api.entities.builtin.platform import events + + bot, manager, counters = make_bot([]) + + async def acknowledge(*args): + record(bot.ap, TEST_CONTEXT, family='platform_api', operation='ack', outcome='success') + + async def resume(*args): + record(bot.ap, TEST_CONTEXT, family='runner', operation='execute', outcome='success') + + bot.ap.agent_run_orchestrator = SimpleNamespace( + interaction_manager=SimpleNamespace( + consume_callback=AsyncMock(return_value={'processor_type': 'agent'}), + acknowledge_submission=acknowledge, + ) + ) + bot._resume_agent_interaction = resume + event = events.PlatformSpecificEvent(platform='test', action='interaction.submitted', data={}) + await bot._handle_platform_event(event, FakeAdapter()) + records = await flushed_records(counters, manager) + (chain,) = execution_chains(records) + assert chain['outcome'] == 'success' + assert {node['family'] for node in execution_nodes(records)} == {'platform_event', 'platform_api', 'runner'} + await counters.shutdown() diff --git a/tests/unit_tests/platform/test_plugin_subscriptions.py b/tests/unit_tests/platform/test_plugin_subscriptions.py index 44819f934..d074bf659 100644 --- a/tests/unit_tests/platform/test_plugin_subscriptions.py +++ b/tests/unit_tests/platform/test_plugin_subscriptions.py @@ -1,6 +1,7 @@ """Independent bot subscriptions must not change primary route delivery.""" import asyncio +import contextvars from types import SimpleNamespace from unittest.mock import AsyncMock @@ -8,6 +9,7 @@ import pytest from langbot_plugin.api.entities.builtin.platform.events import MemberJoinedEvent from langbot.pkg.platform.botmgr import RuntimeBot +from langbot.pkg.persistence.tenant_uow import PersistenceScope, PersistenceScopeBoundary def make_bot(subscriptions): @@ -33,14 +35,14 @@ async def test_slow_failed_and_duplicate_subscriptions_do_not_block_primary_rout started = [] release = asyncio.Event() - async def subscriber(event, adapter, uuid): + async def subscriber(event, adapter, uuid, execution_id=None): started.append(uuid) if uuid == 'slow': await release.wait() if uuid == 'failed': raise ValueError('plugin error') - async def primary(event, adapter): + async def primary(event, adapter, execution_id=None): started.append('primary') bot._dispatch_plugin_subscription = subscriber @@ -105,3 +107,46 @@ async def test_missing_processor_is_logged_without_unscoped_lookup(): await bot._dispatch_plugin_subscription(MemberJoinedEvent(), None, 'missing') bot.ap.agent_service.get_agent.assert_awaited_once_with('workspace', 'missing') assert bot._record_event_route_trace.await_args.kwargs['status'] == 'failed' + + +@pytest.mark.parametrize('primary_fails', [False, True]) +async def test_cloud_deliveries_own_their_scope_and_preserve_parent(primary_fails): + bot = make_bot([{'processor_uuid': 'plugin'}]) + bot.workspace_uuid = 'workspace' + active_scope = contextvars.ContextVar('test_event_scope', default=None) + active_transaction = contextvars.ContextVar('test_event_transaction', default=None) + + def tenant_scope(workspace): + return PersistenceScopeBoundary( + PersistenceScope.workspace(workspace), + active_scope=active_scope, + active_transaction=active_transaction, + ) + + bot.ap = SimpleNamespace( + persistence_mgr=SimpleNamespace(mode=SimpleNamespace(value='cloud_runtime'), tenant_scope=tenant_scope) + ) + seen = [] + + async def delivery(event, adapter, *args, **kwargs): + state = active_scope.get() + assert state.owner_task is asyncio.current_task() + assert state.scope == PersistenceScope.workspace('workspace') + seen.append(state) + await asyncio.sleep(0) + assert active_scope.get() is state + if primary_fails and not args: + raise ValueError('primary failed') + + bot._dispatch_eba_event_to_processor = delivery + bot._dispatch_plugin_subscription = delivery + async with tenant_scope('workspace'): + parent = active_scope.get() + await bot._handle_platform_event(MemberJoinedEvent(), None) + assert active_scope.get() is parent + assert parent.owner_task is asyncio.current_task() + assert active_scope.get() is None + assert len(seen) == 2 + assert seen[0] is not seen[1] + assert all(state is not parent and state.depth == 0 for state in seen) + assert bot.logger.error.await_count == int(primary_fails) diff --git a/tests/unit_tests/platform/test_routing_rules.py b/tests/unit_tests/platform/test_routing_rules.py index 1a339c936..ce93c6099 100644 --- a/tests/unit_tests/platform/test_routing_rules.py +++ b/tests/unit_tests/platform/test_routing_rules.py @@ -767,6 +767,7 @@ async def test_processor_outputs_require_explicit_platform_actions(kind, output_ """Draining runner results must not send text, duplicate replies, or stream cards.""" from langbot_plugin.api.entities.builtin.platform import entities, events, message from langbot_plugin.api.entities.builtin.provider import message as provider_message + from datetime import datetime from langbot.pkg.agent.runner.platform_tools import execute_platform_tool, freeze_platform_context @@ -783,6 +784,10 @@ async def test_processor_outputs_require_explicit_platform_actions(kind, output_ completed = [] async def run(envelope, binding, adapter_context): + if kind == 'event_processor': + restored = message.MessageChain.model_validate(envelope.data['message_chain']) + assert restored.source.id == 'source-1' + assert restored.source.time == datetime(2026, 10, 8, 12) assert envelope.delivery.supports_streaming is False assert binding.delivery_policy.enable_streaming is False assert binding.delivery_policy.enable_reply is False @@ -823,7 +828,9 @@ async def test_processor_outputs_require_explicit_platform_actions(kind, output_ ) event = events.MessageReceivedEvent( message_id='message-1', - message_chain=message.MessageChain([message.Plain(text='hello')]), + message_chain=message.MessageChain( + [message.Source(id='source-1', time=datetime(2026, 10, 8, 12)), message.Plain(text='hello')] + ), sender=entities.User(id='user-1', nickname='QA'), chat_type=entities.ChatType.PRIVATE, chat_id='user-1', diff --git a/tests/unit_tests/plugin/test_connector_reconcile.py b/tests/unit_tests/plugin/test_connector_reconcile.py index 8dbb409dc..0b54f9384 100644 --- a/tests/unit_tests/plugin/test_connector_reconcile.py +++ b/tests/unit_tests/plugin/test_connector_reconcile.py @@ -735,11 +735,13 @@ async def test_apply_dependency_failure_raises_stable_observable_error(): assert error.installation_uuid == setting.installation_uuid assert error.error_code == 'dependency_prepare_failed' assert '[dependency_prepare_failed]' in str(error) - assert connector._installation_failures[setting.installation_uuid] == { + recorded = connector._installation_failures[setting.installation_uuid] + assert {key: value for key, value in recorded.items() if key != 'failed_at'} == { 'installation_uuid': setting.installation_uuid, 'error_code': 'dependency_prepare_failed', 'message': 'Plugin dependency installer exited with code 1', } + assert str(recorded['failed_at']).endswith('Z') connector.ap.logger.error.assert_called_once() @@ -767,9 +769,12 @@ async def test_shared_reconcile_records_one_failure_without_blocking_other_state setting_a.installation_uuid, setting_b.installation_uuid, } - assert connector._installation_failures == { - setting_a.installation_uuid: failure, - } + recorded_failures = connector._installation_failures + assert { + uuid: {key: value for key, value in record.items() if key != 'failed_at'} + for uuid, record in recorded_failures.items() + } == {setting_a.installation_uuid: failure} + assert str(recorded_failures[setting_a.installation_uuid]['failed_at']).endswith('Z') assert set(connector._known_desired_states) == { setting_a.installation_uuid, setting_b.installation_uuid, diff --git a/tests/unit_tests/plugin/test_plugin_file_transfer_root.py b/tests/unit_tests/plugin/test_plugin_file_transfer_root.py new file mode 100644 index 000000000..fad2751ba --- /dev/null +++ b/tests/unit_tests/plugin/test_plugin_file_transfer_root.py @@ -0,0 +1,51 @@ +"""The host must share the Runtime's plugin file-transfer root in shared deployments. + +A mismatch leaves plugin icons and assets failing with +"Invalid file transfer capability" because the SDK binds one directory inode per +process (mode 0700) and picks the default from the runtime profile. +""" + +from __future__ import annotations + +import os +from importlib import import_module +from types import SimpleNamespace + +from langbot_plugin.runtime.io.handler import SHARED_WORKER_FILE_STORAGE_DIR +from langbot_plugin.runtime.security import PLUGIN_FILE_STORAGE_DIR_ENV + + +def connector_module(): + import_module('langbot.pkg.core.app') + return import_module('langbot.pkg.plugin.connector') + + +def align(profile: str) -> None: + connector = connector_module() + connector.PluginRuntimeConnector._align_plugin_file_transfer_root( + SimpleNamespace(runtime_profile=profile) + ) + + +def test_shared_profile_pins_the_runtime_transfer_root(monkeypatch): + monkeypatch.delenv(PLUGIN_FILE_STORAGE_DIR_ENV, raising=False) + + align('shared') + + assert os.environ[PLUGIN_FILE_STORAGE_DIR_ENV] == SHARED_WORKER_FILE_STORAGE_DIR + + +def test_explicit_operator_value_is_never_overridden(monkeypatch): + monkeypatch.setenv(PLUGIN_FILE_STORAGE_DIR_ENV, '/var/lib/custom-transfer') + + align('shared') + + assert os.environ[PLUGIN_FILE_STORAGE_DIR_ENV] == '/var/lib/custom-transfer' + + +def test_oss_profile_leaves_the_default_alone(monkeypatch): + monkeypatch.delenv(PLUGIN_FILE_STORAGE_DIR_ENV, raising=False) + + align('oss_dev') + + assert PLUGIN_FILE_STORAGE_DIR_ENV not in os.environ diff --git a/tests/unit_tests/plugin/test_runtime_ops_reporter.py b/tests/unit_tests/plugin/test_runtime_ops_reporter.py new file mode 100644 index 000000000..c5fbd4b5b --- /dev/null +++ b/tests/unit_tests/plugin/test_runtime_ops_reporter.py @@ -0,0 +1,327 @@ +"""Tests for the Cloud plugin runtime ops sample reporter. + +Covers the frozen Core -> Space sample contract (C2): +- Runtime /healthz URL derivation from the control WebSocket URL +- Disablement without a runtime WebSocket URL / outside the cloud edition +- Sample assembly from connector internals (desired states, failures, health) +- Best-effort upload semantics (non-2xx and transport failures never raise) +- The pushed payload never carries the runtime control token +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import Mock + +import httpx +import pytest + +from langbot.pkg.plugin.runtime_ops import RuntimeOpsReporter +from langbot_plugin.entities.io.context import PluginExecutionMode +from langbot_plugin.runtime.security import PLUGIN_RUNTIME_CONTROL_TOKEN_ENV + +_CONTROL_PLANE_TOKEN_ENV = 'LANGBOT_SPACE_CONTROL_PLANE_TOKEN' + +_DEFAULT_WS_URL = 'ws://langbot_plugin_runtime:5400/control/ws' + + +def make_policy() -> SimpleNamespace: + return SimpleNamespace(model_dump=lambda: {'max_workers': 32, 'max_cpus': 0.25}) + + +def make_connector( + *, + desired_states: dict | None = None, + failures: dict | None = None, + identities: dict | None = None, + connected: bool = True, +) -> SimpleNamespace: + return SimpleNamespace( + runtime_profile='shared', + worker_policy=make_policy(), + _known_desired_states=desired_states if desired_states is not None else {}, + _installation_failures=failures if failures is not None else {}, + _reconcile_summary={ + 'last_started_at': '2026-10-01T07:11:00Z', + 'last_duration_ms': 1234, + 'last_ok': True, + 'failed_installations': 0, + 'missing_artifacts': 0, + }, + handler=SimpleNamespace(_installation_bindings=identities if identities is not None else {}), + _runtime_available=lambda: connected, + ) + + +def make_app( + *, + runtime_ws_url: str | None = _DEFAULT_WS_URL, + runtime_ops: dict | None = None, + space_url: str = 'https://space.example', + deployment_mode: str = 'cloud', + edition: str = 'cloud', + connector: SimpleNamespace | None = None, +) -> SimpleNamespace: + plugin: dict = {} + if runtime_ws_url is not None: + plugin['runtime_ws_url'] = runtime_ws_url + if runtime_ops is not None: + plugin['runtime_ops'] = runtime_ops + return SimpleNamespace( + logger=Mock(), + instance_config=SimpleNamespace( + data={ + 'plugin': plugin, + 'space': {'url': space_url}, + 'system': {'instance_id': 'inst-1', 'edition': edition}, + } + ), + deployment=SimpleNamespace(mode=deployment_mode), + plugin_connector=connector if connector is not None else make_connector(), + ) + + +def make_desired_state(workspace_uuid: str, mode: PluginExecutionMode, *, enabled: bool = True) -> SimpleNamespace: + return SimpleNamespace( + enabled=enabled, + execution_mode=mode, + binding=SimpleNamespace(workspace_uuid=workspace_uuid), + ) + + +class TestHealthUrlDerivation: + @pytest.mark.parametrize( + ('ws_url', 'expected'), + [ + ('ws://langbot_plugin_runtime:5400/control/ws', 'http://langbot_plugin_runtime:5400/healthz'), + ('wss://runtime.example/control/ws', 'https://runtime.example/healthz'), + ('wss://runtime.example:8443/control/ws', 'https://runtime.example:8443/healthz'), + ('ws://127.0.0.1:5400', 'http://127.0.0.1:5400/healthz'), + (' ws://runtime.example:5400/control/ws ', 'http://runtime.example:5400/healthz'), + ], + ) + def test_derives_health_url(self, ws_url: str, expected: str): + reporter = RuntimeOpsReporter(make_app(runtime_ws_url=ws_url)) + + assert reporter.health_url == expected + + @pytest.mark.parametrize('ws_url', ['', None, 'http://runtime.example:5400', 'not a url']) + def test_rejects_non_websocket_url(self, ws_url): + reporter = RuntimeOpsReporter(make_app(runtime_ws_url=ws_url)) + + assert reporter.health_url is None + + +class TestEnablement: + def test_disabled_without_runtime_websocket_url(self): + reporter = RuntimeOpsReporter(make_app(runtime_ws_url=None)) + + assert reporter.enabled is False + + def test_disabled_on_non_cloud_deployment(self): + reporter = RuntimeOpsReporter(make_app(deployment_mode='oss')) + + assert reporter.enabled is False + + def test_explicit_disable_on_cloud(self): + reporter = RuntimeOpsReporter(make_app(runtime_ops={'enabled': False})) + + assert reporter.enabled is False + + def test_enabled_by_default_on_cloud(self): + reporter = RuntimeOpsReporter(make_app()) + + assert reporter.enabled is True + assert reporter.interval_seconds == 30.0 + assert reporter.timeout_seconds == 10.0 + assert reporter.max_failures == 5 + + +class TestSampleAssembly: + def test_build_payload_matches_contract(self): + desired = { + 'inst-dedicated': make_desired_state('ws-1', PluginExecutionMode.DEDICATED), + 'inst-shared-off': make_desired_state( + 'ws-2', PluginExecutionMode.SHARED_CERTIFIED, enabled=False + ), + 'inst-shared-on': make_desired_state('ws-3', PluginExecutionMode.SHARED_CERTIFIED), + } + failures = { + 'inst-dedicated': { + 'installation_uuid': 'inst-dedicated', + 'error_code': 'worker_launch_failed', + 'message': 'boom\nstack line', + }, + 'inst-shared-off': { + 'installation_uuid': 'inst-shared-off', + 'error_code': 'dependency_prepare_failed', + 'message': 'prepare failed', + }, + } + identities = { + 'inst-dedicated': ( + None, + SimpleNamespace(plugin_author='acme', plugin_name='dedicated-plugin'), + ), + } + connector = make_connector(desired_states=desired, failures=failures, identities=identities) + reporter = RuntimeOpsReporter(make_app(connector=connector)) + health = {'live': True, 'runtime': {'version': '0.7.6'}, 'shared_pool': {'workers': 12}} + + payload = reporter.build_payload(health) + + assert set(payload) == {'event', 'kind', 'instance_uuid', 'generated_at', 'core', 'health', 'failures'} + assert payload['event'] == 'runtime_ops_sample' + assert payload['kind'] == 'aggregate' + assert payload['instance_uuid'] == 'inst-1' + assert payload['generated_at'].endswith('Z') + # The health blob is forwarded verbatim. + assert payload['health'] is health + + core = payload['core'] + assert core['edition'] == 'cloud' + assert core['connected'] is True + assert core['runtime_profile'] == 'shared' + assert core['worker_policy'] == {'max_workers': 32, 'max_cpus': 0.25} + assert core['desired_states'] == { + 'total': 3, + 'enabled': 2, + 'by_mode': {'dedicated': 1, 'shared-runtime-v1': 2}, + } + assert core['reconcile'] == { + 'last_started_at': '2026-10-01T07:11:00Z', + 'last_duration_ms': 1234, + 'last_ok': True, + 'failed_installations': 0, + 'missing_artifacts': 0, + } + + assert len(payload['failures']) == 2 + first = payload['failures'][0] + assert first['workspace_uuid'] == 'ws-1' + assert first['installation_uuid'] == 'inst-dedicated' + assert first['plugin_author'] == 'acme' + assert first['plugin_name'] == 'dedicated-plugin' + assert first['execution_mode'] == 'dedicated' + assert first['error_code'] == 'worker_launch_failed' + assert first['message'] == 'boom stack line' + assert first['observed_at'].endswith('Z') + second = payload['failures'][1] + assert second['workspace_uuid'] == 'ws-2' + assert second['plugin_author'] == '' + assert second['plugin_name'] == '' + assert second['execution_mode'] == 'shared-runtime-v1' + + def test_failures_capped_by_max_failures(self): + failures = { + f'inst-{index}': { + 'installation_uuid': f'inst-{index}', + 'error_code': 'worker_launch_failed', + 'message': 'boom', + } + for index in range(8) + } + connector = make_connector(failures=failures) + reporter = RuntimeOpsReporter(make_app(runtime_ops={'max_failures': 3}, connector=connector)) + + payload = reporter.build_payload({'live': True}) + + assert len(payload['failures']) == 3 + + def test_disconnected_connector_reports_not_connected(self): + connector = make_connector(connected=False) + reporter = RuntimeOpsReporter(make_app(connector=connector)) + + payload = reporter.build_payload({'live': False}) + + assert payload['core']['connected'] is False + + +class TestUpload: + @pytest.mark.asyncio + async def test_successful_post_increments_sent(self): + reporter = RuntimeOpsReporter(make_app()) + reporter._client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={'code': 0})) + ) + + await reporter._post({'event': 'runtime_ops_sample'}) + + assert reporter.get_stats() == {'sent_total': 1, 'dropped_total': 0, 'last_error': None} + await reporter.stop() + + @pytest.mark.asyncio + async def test_non_2xx_increments_dropped_without_raising(self): + reporter = RuntimeOpsReporter(make_app()) + reporter._client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(503, text='unavailable')) + ) + + await reporter._post({'event': 'runtime_ops_sample'}) + + stats = reporter.get_stats() + assert stats['sent_total'] == 0 + assert stats['dropped_total'] == 1 + assert '503' in stats['last_error'] + await reporter.stop() + + @pytest.mark.asyncio + async def test_transport_exception_increments_dropped_without_raising(self): + def raise_connect_error(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError('connection refused') + + reporter = RuntimeOpsReporter(make_app()) + reporter._client = httpx.AsyncClient(transport=httpx.MockTransport(raise_connect_error)) + + await reporter._post({'event': 'runtime_ops_sample'}) + + stats = reporter.get_stats() + assert stats['dropped_total'] == 1 + assert 'connection refused' in stats['last_error'] + await reporter.stop() + + +class TestFailureTimestamps: + def test_publishes_when_the_failure_happened_not_when_it_was_sampled(self): + failures = { + 'inst-1': { + 'installation_uuid': 'inst-1', + 'error_code': 'dependency_prepare_failed', + 'message': 'prepare failed', + 'failed_at': '2026-10-01T07:11:00Z', + } + } + reporter = RuntimeOpsReporter(make_app(connector=make_connector(failures=failures))) + + payload = reporter.build_payload({'live': True}) + + entry = payload['failures'][0] + assert entry['observed_at'] == '2026-10-01T07:11:00Z' + assert entry['observed_at'] != payload['generated_at'] + + def test_legacy_records_without_a_timestamp_fall_back_to_the_sample_time(self): + failures = { + 'inst-2': { + 'installation_uuid': 'inst-2', + 'error_code': 'worker_launch_failed', + 'message': 'boom', + } + } + reporter = RuntimeOpsReporter(make_app(connector=make_connector(failures=failures))) + + payload = reporter.build_payload({'live': True}) + + assert payload['failures'][0]['observed_at'] == payload['generated_at'] + + +def test_payload_never_contains_a_control_token(monkeypatch): + monkeypatch.setenv(PLUGIN_RUNTIME_CONTROL_TOKEN_ENV, 'runtime-control-secret') + monkeypatch.setenv(_CONTROL_PLANE_TOKEN_ENV, 'space-control-plane-secret') + reporter = RuntimeOpsReporter(make_app()) + + payload = reporter.build_payload({'live': True}) + serialized = json.dumps(payload) + + assert 'runtime-control-secret' not in serialized + assert 'space-control-plane-secret' not in serialized diff --git a/tests/unit_tests/provider/test_litellmchat.py b/tests/unit_tests/provider/test_litellmchat.py index 490f7aa12..4c089816d 100644 --- a/tests/unit_tests/provider/test_litellmchat.py +++ b/tests/unit_tests/provider/test_litellmchat.py @@ -53,6 +53,37 @@ class MockRuntimeRerankModel: self.provider.token_mgr.get_token = Mock(return_value=api_key) +@pytest.mark.asyncio +async def test_closing_consumer_closes_provider_http_stream(): + import asyncio + from types import SimpleNamespace + + app = Mock() + app.tool_mgr.generate_tools_for_openai = AsyncMock(return_value=None) + requester = litellmchat.LiteLLMRequester(ap=app, config={}) + closed = asyncio.Event() + + async def provider(): + try: + yield SimpleNamespace( + choices=[ + SimpleNamespace( + delta=SimpleNamespace(model_dump=lambda: {'role': 'assistant', 'content': 'first'}), + finish_reason=None, + ) + ] + ) + await asyncio.Event().wait() + finally: + closed.set() + + with patch.object(litellmchat, 'acompletion', new=AsyncMock(side_effect=lambda **kw: provider())): + output = requester.invoke_llm_stream(query=None, model=MockRuntimeModel(), messages=[]) + assert (await anext(output)).content == 'first' + await output.aclose() + assert closed.is_set() + + class TestBuildLiteLLMModelName: """Test _build_litellm_model_name method""" diff --git a/tests/unit_tests/provider/test_mcp_resources.py b/tests/unit_tests/provider/test_mcp_resources.py index 0ba164e9a..97b136d56 100644 --- a/tests/unit_tests/provider/test_mcp_resources.py +++ b/tests/unit_tests/provider/test_mcp_resources.py @@ -24,7 +24,6 @@ from langbot.pkg.provider.tools.loaders.mcp import ( MCPToolCallTimeoutError, RuntimeMCPSession, ) -from langbot.pkg.telemetry import features as telemetry_features from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError @@ -298,10 +297,6 @@ async def test_read_resource_envelope_truncates_caches_and_records_trace(): traces = query.variables[MCP_RESOURCE_TRACE_QUERY_KEY] assert [trace['source'] for trace in traces] == ['ui_preview', 'agent_tool'] assert traces[1]['cache_hit'] is True - assert query.variables[telemetry_features.FEATURES_KEY]['mcp_resource_reads'] == { - 'ui_preview': 1, - 'agent_tool': 1, - } @pytest.mark.asyncio diff --git a/tests/unit_tests/provider/test_tool_manager.py b/tests/unit_tests/provider/test_tool_manager.py index d404400a4..0e81ae5e2 100644 --- a/tests/unit_tests/provider/test_tool_manager.py +++ b/tests/unit_tests/provider/test_tool_manager.py @@ -7,6 +7,8 @@ Tests cover: from __future__ import annotations +import asyncio +import contextvars from importlib import import_module from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -380,6 +382,127 @@ class TestToolManagerExecuteFuncCall: assert 'MCP resource agent reads are disabled' in record.await_args.kwargs['error_message'] +class _RecordingTelemetrySender: + """Telemetry sender stand-in that keeps every payload it was handed.""" + + def __init__(self): + self.telemetry_config = {'url': 'https://space.example.test', 'execution_trace': 'all'} + self.sent: list[dict] = [] + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +class TestToolCallTracing: + """Every tool call a Run makes is its own step of the execution chain.""" + + @pytest.fixture + def traced_app(self): + execution = import_module('langbot.pkg.telemetry.execution') + counters = execution.ExecutionCounters(_RecordingTelemetrySender()) + app = Mock() + app.logger = Mock() + app.telemetry = SimpleNamespace(execution=counters) + # Tool-call monitoring is asserted by the tests above; this path only + # needs the execution chain. + app.monitoring_service = None + + def _inert_loader(): + loader = Mock() + loader.has_tool = AsyncMock(return_value=False) + loader.invoke_tool = AsyncMock(return_value=None) + return loader + + app._native_loader = _inert_loader() + app._plugin_loader = _inert_loader() + app._mcp_loader = _inert_loader() + app._skill_loader = _inert_loader() + return app, counters + + @pytest.fixture + def sample_query(self): + query = Mock(spec=pipeline_query.Query) + query._execution_context = _CONTEXT + return query + + @staticmethod + def _manager(app): + """ToolManager wired to app-local inert loaders.""" + manager = get_toolmgr_module().ToolManager(app) + manager.native_tool_loader = app._native_loader + manager.plugin_tool_loader = app._plugin_loader + manager.mcp_tool_loader = app._mcp_loader + manager.skill_tool_loader = app._skill_loader + return manager + + @staticmethod + def _tool_nodes(counters): + return [record for record in counters.records if record['event_type'] == 'execution_node'] + + @pytest.mark.asyncio + async def test_a_tool_call_becomes_a_node_under_the_step_that_issued_it(self, traced_app, sample_query): + trace = import_module('langbot.pkg.telemetry.trace') + execution = import_module('langbot.pkg.telemetry.execution') + app, counters = traced_app + manager = self._manager(app) + + binding = execution.bind_trace(app, 'exec-tool') + # The Runtime's RPC task executes the call with no inherited trace + # context; the Run id on the query is what resolves the owning chain. + sample_query._agent_run_session = {'run_id': 'exec-tool'} + try: + with trace.stage_scope() as lane: + foreign = contextvars.Context() + task = asyncio.get_running_loop().create_task( + manager._invoke_tool_with_monitoring( + source='native', + name='exec', + parameters={'command': 'ls'}, + query=sample_query, + invoke=AsyncMock(return_value='sandbox output'), + ), + context=foreign, + ) + assert await task == 'sandbox output' + finally: + trace.unbind_root(binding) + + rows = self._tool_nodes(counters) + assert len(rows) == 1 + assert rows[0]['family'] == 'tool' + assert rows[0]['operation'] == 'exec' + assert rows[0]['adapter'] == 'native' + assert rows[0]['outcome'] == 'success' + assert rows[0]['parent_node_id'] == lane + + @pytest.mark.asyncio + async def test_a_failed_tool_call_carries_its_reason(self, traced_app, sample_query): + execution = import_module('langbot.pkg.telemetry.execution') + trace = import_module('langbot.pkg.telemetry.trace') + app, counters = traced_app + manager = self._manager(app) + + binding = execution.bind_trace(app, 'exec-tool-failed') + try: + with pytest.raises(RuntimeError): + await manager._invoke_tool_with_monitoring( + source='mcp', + name='read_resource', + parameters={}, + query=sample_query, + invoke=AsyncMock(side_effect=RuntimeError('server unreachable')), + ) + finally: + trace.unbind_root(binding) + + rows = self._tool_nodes(counters) + assert len(rows) == 1 + assert rows[0]['family'] == 'tool' + assert rows[0]['outcome'] == 'failed' + assert rows[0]['error'] == 'server unreachable' + + class TestToolManagerSourceResolution: @pytest.mark.asyncio @pytest.mark.parametrize('source', ['mcp', 'plugin']) diff --git a/tests/unit_tests/telemetry/test_chat_emitter.py b/tests/unit_tests/telemetry/test_chat_emitter.py new file mode 100644 index 000000000..0c66d3f83 --- /dev/null +++ b/tests/unit_tests/telemetry/test_chat_emitter.py @@ -0,0 +1,110 @@ +"""The Chat handler must not emit the legacy per-query telemetry payload. + +Per-execution telemetry is produced by the trace that owns the Pipeline lane; +the handler only annotates that trace with execution-scoped record fields. +""" + +from __future__ import annotations + +import types +from importlib import import_module +from unittest.mock import AsyncMock, Mock + +import langbot_plugin.api.entities.builtin.provider.session as provider_session + + +def get_modules(): + # Import the application package first: the handler package is wired into + # it during initialization, so importing the handler directly would re-enter + # the partially initialized pipeline package. + import_module('langbot.pkg.core.app') + return ( + import_module('langbot.pkg.pipeline.process.handlers.chat'), + import_module('langbot.pkg.telemetry.trace'), + ) + + +class FakeEvent: + """Stand-in for the plugin event models, which need a real Query.""" + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.event_name = 'person_normal_message_received' + + +class FakeEventContext: + event = types.SimpleNamespace(user_message_alter=None) + + def is_prevented_default(self) -> bool: + return False + + +async def empty_run_from_query(query): + if False: # pragma: no cover - async generator with no results + yield + + +def build_ap(): + ap = Mock() + ap.logger = Mock() + ap.plugin_connector = Mock() + ap.plugin_connector.emit_event = AsyncMock(return_value=FakeEventContext()) + ap.agent_run_orchestrator = Mock() + ap.agent_run_orchestrator.try_claim_steering_from_query = AsyncMock(return_value=False) + ap.agent_run_orchestrator.run_from_query = empty_run_from_query + ap.agent_run_orchestrator.resolve_runner_id_for_telemetry = Mock(return_value='runner-1') + ap.survey = Mock() + ap.survey.trigger_event = AsyncMock() + ap.survey.record_bot_response_success = AsyncMock() + ap.telemetry = Mock() + ap.telemetry.start_send_task = AsyncMock() + ap.telemetry.send = AsyncMock() + return ap + + +def build_query(): + query = Mock() + query.query_id = 7 + query.query_uuid = 'exec-uuid' + query.launcher_type = provider_session.LauncherTypes.PERSON + query.launcher_id = 'user-1' + query.sender_id = 'user-1' + query.message_chain = 'hello' + query.message_event = object() + query.variables = {'_pipeline_bound_plugins': ['plugin:a']} + query.pipeline_config = {'ai': {}, 'output': {'misc': {}}} + query.session = Mock(using_conversation=object()) + query.use_llm_model_uuid = None + query.adapter = Mock() + query.adapter.is_stream_output_supported = AsyncMock(return_value=False) + query.resp_messages = [] + return query + + +async def test_normal_query_sends_no_legacy_payload_and_annotates_the_trace(monkeypatch): + chat, trace_mod = get_modules() + monkeypatch.setattr(chat.events, 'PersonNormalMessageReceived', FakeEvent) + monkeypatch.setattr(chat.events, 'GroupNormalMessageReceived', FakeEvent) + + ap = build_ap() + handler = chat.ChatMessageHandler(ap) + query = build_query() + + binding = trace_mod.bind('exec-uuid') + try: + async for _ in handler.handle(query): + pass + finally: + trace_mod.unbind_root(binding) + + # The legacy `event_type='query'` emitter is gone. + ap.telemetry.start_send_task.assert_not_called() + ap.telemetry.send.assert_not_called() + # The survey trigger and its milestone bookkeeping still fire. + ap.survey.trigger_event.assert_awaited_once_with('first_bot_response_success') + ap.survey.record_bot_response_success.assert_awaited_once() + # Execution-scoped fields are attached to the owning trace instead. + assert binding.state.runner == 'runner-1' + assert binding.state.runner_category == 'unknown' + assert binding.state.pipeline_plugins == ['plugin:a'] + assert binding.state.model_name == '' diff --git a/tests/unit_tests/telemetry/test_cloud_sampling.py b/tests/unit_tests/telemetry/test_cloud_sampling.py new file mode 100644 index 000000000..ef4de5e2e --- /dev/null +++ b/tests/unit_tests/telemetry/test_cloud_sampling.py @@ -0,0 +1,37 @@ +from types import SimpleNamespace + +from langbot.pkg.telemetry.execution import ExecutionCounters +from langbot.pkg.telemetry.trace import TraceState +from langbot.pkg.utils import constants + + +def test_cloud_counts_are_not_sampled(monkeypatch): + monkeypatch.setattr(constants, 'edition', 'cloud') + counters = ExecutionCounters(SimpleNamespace(telemetry_config={'execution_trace': 'sampled'})) + assert counters.trace_mode() == 'all' + state = TraceState('00000001') + counters.configure(state) + assert state.emit_nodes is True + assert counters._chain_emitted(state) is True + + +def test_community_retains_sampling(monkeypatch): + monkeypatch.setattr(constants, 'edition', 'community') + counters = ExecutionCounters(SimpleNamespace(telemetry_config={'execution_trace': 'sampled'})) + assert counters.trace_mode() == 'sampled' + + +def test_dns_survives_saturated_executor(): + import socket + import threading + from langbot.pkg.utils.bounded_executor import BoundedThreadPoolExecutor + + gate = threading.Event() + executor = BoundedThreadPoolExecutor(max_workers=1, max_pending=0) + try: + blocked = executor.submit(gate.wait) + assert executor.submit(socket.getaddrinfo, '127.0.0.1', 443).result(timeout=2) + finally: + gate.set() + blocked.result(timeout=2) + executor.shutdown() diff --git a/tests/unit_tests/telemetry/test_features.py b/tests/unit_tests/telemetry/test_features.py deleted file mode 100644 index 6e19d0828..000000000 --- a/tests/unit_tests/telemetry/test_features.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Unit tests for telemetry feature counters (pkg/telemetry/features.py).""" - -from __future__ import annotations - -from importlib import import_module - - -def get_features_module(): - return import_module('langbot.pkg.telemetry.features') - - -class FakeQuery: - def __init__(self): - self.variables = {} - - -class TestIncrement: - def test_increment_nested_counter(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'tool_calls', 'native') - features.increment(q, 'tool_calls', 'native') - features.increment(q, 'tool_calls', 'mcp') - assert q.variables[features.FEATURES_KEY]['tool_calls'] == {'native': 2, 'mcp': 1} - - def test_increment_flat_counter(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'something') - features.increment(q, 'something', amount=2) - assert q.variables[features.FEATURES_KEY]['something'] == 3 - - def test_increment_never_raises_on_broken_query(self): - features = get_features_module() - - class Broken: - @property - def variables(self): - raise RuntimeError('boom') - - # Must not raise - features.increment(Broken(), 'tool_calls', 'native') - - def test_set_value(self): - features = get_features_module() - q = FakeQuery() - features.set_value(q, 'tool_call_rounds', 5) - assert q.variables[features.FEATURES_KEY]['tool_call_rounds'] == 5 - - -class TestCollectFeatures: - def test_collect_empty(self): - features = get_features_module() - q = FakeQuery() - assert features.collect_features(q) == {} - - def test_collect_combines_counters_and_snapshots(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'sandbox', 'execs') - features.set_value(q, 'kb', {'kb_count': 2, 'engine_plugins': ['builtin'], 'retrieved_entries': 7}) - q.variables['_activated_skills'] = {'pdf-tools': {}, 'a-skill': {}} - q.variables['_pipeline_bound_mcp_servers'] = ['srv1', 'srv2'] - - result = features.collect_features(q) - assert result['sandbox'] == {'execs': 1} - assert result['kb']['kb_count'] == 2 - assert result['activated_skills'] == ['a-skill', 'pdf-tools'] # sorted - assert result['mcp_servers'] == ['srv1', 'srv2'] - - def test_collect_omits_mcp_when_all_enabled(self): - """None means 'all enabled' and is not reported.""" - features = get_features_module() - q = FakeQuery() - q.variables['_pipeline_bound_mcp_servers'] = None - assert 'mcp_servers' not in features.collect_features(q) - - def test_collect_drops_non_json_serializable(self): - features = get_features_module() - q = FakeQuery() - features.set_value(q, 'good', 1) - features.set_value(q, 'bad', object()) - result = features.collect_features(q) - assert result == {'good': 1} - - def test_collect_is_json_serializable(self): - import json - - features = get_features_module() - q = FakeQuery() - features.increment(q, 'tool_calls', 'skill') - json.dumps(features.collect_features(q)) diff --git a/tests/unit_tests/telemetry/test_heartbeat.py b/tests/unit_tests/telemetry/test_heartbeat.py index 86b70094b..f559f1679 100644 --- a/tests/unit_tests/telemetry/test_heartbeat.py +++ b/tests/unit_tests/telemetry/test_heartbeat.py @@ -183,6 +183,7 @@ class TestBuildHeartbeatPayload: assert by_workspace['workspace-b']['execution_generation'] == 9 assert by_workspace['workspace-b']['adapters'] == ['WorkspaceBAdapter'] assert 'workspace_resources' not in by_workspace['workspace-a'] + ap.plugin_connector.list_plugins.assert_not_awaited() ap.persistence_mgr.execute_async.assert_not_awaited() ap.workspace_service.list_active_execution_bindings.assert_awaited_once() diff --git a/tests/unit_tests/telemetry/test_pipeline_lane.py b/tests/unit_tests/telemetry/test_pipeline_lane.py new file mode 100644 index 000000000..4b3e8625e --- /dev/null +++ b/tests/unit_tests/telemetry/test_pipeline_lane.py @@ -0,0 +1,288 @@ +"""The Pipeline lane must land its nodes on one v2 execution chain.""" + +from __future__ import annotations + +import asyncio +import types +from importlib import import_module + +import pytest + + +def get_modules(): + # Import the application package first; the pipeline package is wired into it. + import_module('langbot.pkg.core.app') + return ( + import_module('langbot.pkg.pipeline.pipelinemgr'), + import_module('langbot.pkg.telemetry.execution'), + ) + + +def get_trace(): + import_module('langbot.pkg.core.app') + return import_module('langbot.pkg.telemetry.trace') + + +class FakeManager: + def __init__(self): + self.telemetry_config = {'url': 'https://space.example.test', 'execution_trace': 'all'} + self.sent: list[dict] = [] + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +CONTEXT = types.SimpleNamespace(instance_uuid='instance-1', workspace_uuid='workspace-1') + + +def build_pipeline(pipelinemgr, ap): + pipeline = object.__new__(pipelinemgr.RuntimePipeline) + pipeline.ap = ap + pipeline.execution_context = CONTEXT + return pipeline + + +def test_lane_failure_marks_the_owned_chain_without_any_prior_stage(): + pipelinemgr, execution = get_modules() + manager = FakeManager() + counters = execution.ExecutionCounters(manager) + ap = types.SimpleNamespace(telemetry=types.SimpleNamespace(execution=counters)) + pipeline = build_pipeline(pipelinemgr, ap) + query = types.SimpleNamespace(pipeline_config={}) + + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-7'): + pipeline._record_lane_failure(query, 'sandbox exited 127') + + rows = [record for record in counters.records if record['event_type'] == 'execution_node'] + assert len(rows) == 1 + node = rows[0] + assert node['event_id'] == 'query-7' + assert node['error'] == 'sandbox exited 127' + assert (node['family'], node['operation'], node['outcome']) == ('runner', 'execute', 'failed') + chain = [record for record in counters.records if record['event_type'] == 'execution_chain'][0] + assert chain['event_id'] == 'query-7' + assert chain['outcome'] == 'failed' + assert chain['error'] == 'sandbox exited 127' + + +def test_lane_failure_does_not_duplicate_the_runner_node_the_orchestrator_recorded(): + """A break inside a runner is already a node; the lane only carries the reason over.""" + pipelinemgr, execution = get_modules() + manager = FakeManager() + counters = execution.ExecutionCounters(manager) + ap = types.SimpleNamespace(telemetry=types.SimpleNamespace(execution=counters)) + pipeline = build_pipeline(pipelinemgr, ap) + query = types.SimpleNamespace(pipeline_config={}) + + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-8'): + execution.record( + ap, + CONTEXT, + family='runner', + operation='execute', + mode='pipeline', + runner='plugin:langbot-team/LocalAgent/default', + outcome='failed', + error='No authorized model for local-agent', + ) + pipeline._record_lane_failure(query, 'No authorized model for local-agent') + + rows = [record for record in counters.records if record['event_type'] == 'execution_node'] + assert [(record['family'], record['operation'], record['outcome']) for record in rows] == [ + ('runner', 'execute', 'failed') + ] + chain = [record for record in counters.records if record['event_type'] == 'execution_chain'][-1] + assert chain['outcome'] == 'failed' + assert chain['error'] == 'No authorized model for local-agent' + + +def build_counters(): + manager = FakeManager() + counters = import_module('langbot.pkg.telemetry.execution').ExecutionCounters(manager) + ap = types.SimpleNamespace(telemetry=types.SimpleNamespace(execution=counters)) + return counters, ap + + +def build_lane(pipelinemgr, ap): + # ``process_query`` lives on the RuntimePipeline lane, not on PipelineManager. + lane = object.__new__(pipelinemgr.RuntimePipeline) + lane.ap = ap + lane.execution_context = CONTEXT + return lane + + +def build_query(**overrides): + query = types.SimpleNamespace(query_uuid='exec-lane', query_id=1, pipeline_config={}, variables={}) + for key, value in overrides.items(): + setattr(query, key, value) + return query + + +def lane_nodes(counters): + return [record for record in counters.records if record['event_type'] == 'execution_node'] + + +def lane_shapes(counters): + return [ + (record['family'], record['operation'], record['node_id'], record['parent_node_id']) + for record in lane_nodes(counters) + ] + + +async def stop_counters(counters): + """Cancel the telemetry flush task so no loop is left with pending work.""" + task = counters.task + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + counters.task = None + + +async def test_lane_step_owns_the_apis_and_runner_it_calls(): + pipelinemgr, execution = get_modules() + trace = get_trace() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + query = build_query() + seen = {} + + async def fake_process(_query): + execution.record( + ap, CONTEXT, family='platform_api', operation='send_message', + adapter='AiocqhttpAdapter', mode='pipeline', outcome='success', + ) + with trace.stage_scope() as run_node: + seen['run'] = run_node + execution.record( + ap, CONTEXT, family='runner', operation='execute', mode='pipeline', outcome='success', node=run_node + ) + # A reply issued after the runner still belongs to the lane. + execution.record( + ap, CONTEXT, family='platform_api', operation='reply_message', + adapter='AiocqhttpAdapter', mode='pipeline', outcome='success', + ) + + lane._process_query = fake_process + + async def scenario(): + await lane.process_query(query) + await stop_counters(counters) + + await scenario() + + shapes = lane_shapes(counters) + # The synthetic inbound event roots the chain; the lane step hangs under it. + assert [shape[0:2] for shape in shapes] == [ + ('platform_event', 'message.received'), + ('platform_api', 'send_message'), + ('runner', 'execute'), + ('platform_api', 'reply_message'), + ('pipeline', 'run'), + ] + event_node = shapes[0][2] + lane_node = shapes[-1][2] + assert shapes[0][3] == '' + assert shapes[1][3] == lane_node + assert shapes[2] == ('runner', 'execute', seen['run'], lane_node) + assert shapes[3][3] == lane_node + # The lane step is recorded last and nests under the synthetic event node. + assert shapes[4] == ('pipeline', 'run', lane_node, event_node) + + +async def test_lane_node_nests_under_the_platform_route_that_called_it(): + pipelinemgr, execution = get_modules() + trace = get_trace() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + query = build_query(query_uuid='exec-routed') + + async def fake_process(_query): + execution.record( + ap, CONTEXT, family='platform_api', operation='send_message', mode='pipeline', outcome='success' + ) + + lane._process_query = fake_process + + async def scenario(): + # A platform event owns the chain; the route step wraps the Pipeline lane. + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-routed'): + with trace.stage_scope() as event_node: + execution.record( + ap, CONTEXT, family='platform_event', operation='message.received', + adapter='AiocqhttpAdapter', outcome='success', node=event_node, + ) + with trace.stage_scope() as route_node: + execution.record( + ap, CONTEXT, family='event_route', operation='message.received', + adapter='AiocqhttpAdapter', outcome='success', node=route_node, + ) + await lane.process_query(query) + await stop_counters(counters) + return event_node, route_node + + event_node, route_node = await scenario() + + assert event_node != route_node + shapes = lane_shapes(counters) + assert shapes[0] == ('platform_event', 'message.received', event_node, '') + assert shapes[1] == ('event_route', 'message.received', route_node, event_node) + lane_node = shapes[-1][2] + # The API is called by the lane, so it hangs on the lane node. + assert shapes[2][0:2] == ('platform_api', 'send_message') + assert shapes[2][3] == lane_node and shapes[2][2] != lane_node + # The lane nests under the route step that called it. + assert shapes[3] == ('pipeline', 'run', lane_node, route_node) + + +async def test_broken_lane_records_a_failed_step_with_its_reason(): + pipelinemgr, execution = get_modules() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + + async def fake_process(_query): + raise RuntimeError('lane exploded') + + lane._process_query = fake_process + + async def scenario(): + try: + await lane.process_query(build_query()) + finally: + await stop_counters(counters) + + with pytest.raises(RuntimeError, match='lane exploded'): + await scenario() + + shapes = lane_shapes(counters) + family, operation, node, parent = shapes[-1] + row = lane_nodes(counters)[-1] + # The lane step nests under the synthetic inbound event that roots the chain. + assert (family, operation) == ('pipeline', 'run') + assert shapes[0] == ('platform_event', 'message.received', parent, '') + assert row['outcome'] == 'failed' + assert row['error'] == 'lane exploded' + + +async def test_lane_reported_failure_marks_the_step_failed(): + pipelinemgr, execution = get_modules() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + query = build_query(variables={'_monitoring_has_error': True}) + + async def fake_process(_query): + return None + + lane._process_query = fake_process + + async def scenario(): + await lane.process_query(query) + await stop_counters(counters) + + await scenario() + + row = lane_nodes(counters)[-1] + assert (row['family'], row['operation'], row['outcome']) == ('pipeline', 'run', 'failed') diff --git a/tests/unit_tests/telemetry/test_platform_model.py b/tests/unit_tests/telemetry/test_platform_model.py new file mode 100644 index 000000000..8bca383ea --- /dev/null +++ b/tests/unit_tests/telemetry/test_platform_model.py @@ -0,0 +1,27 @@ +from types import SimpleNamespace + +import pytest +from pydantic import BaseModel + +from langbot.pkg.telemetry.platform import observe_adapter + + +class ModelAdapter(BaseModel): + def get_supported_apis(self): + return ['send_message'] + + async def send_message(self): + return {'message_id': 'ok'} + + +@pytest.mark.asyncio +async def test_model_adapter_instrumentation_is_instance_local(): + first, second = ModelAdapter(), ModelAdapter() + ap = SimpleNamespace(telemetry=None) + observe_adapter(ap, None, first) + wrapped = first.send_message + observe_adapter(ap, None, first) + assert first.send_message is wrapped + assert not getattr(second, '_execution_observed', False) + assert await first.send_message() == {'message_id': 'ok'} + assert await second.send_message() == {'message_id': 'ok'} diff --git a/tests/unit_tests/telemetry/test_telemetry.py b/tests/unit_tests/telemetry/test_telemetry.py index 004009a68..87699a18d 100644 --- a/tests/unit_tests/telemetry/test_telemetry.py +++ b/tests/unit_tests/telemetry/test_telemetry.py @@ -6,7 +6,6 @@ Tests cover: - Early return conditions (disabled, empty config, no server) - URL construction (with actual URL verification) - HTTP request success/failure scenarios -- Source code bug: send_tasks should be instance variable """ from __future__ import annotations @@ -629,53 +628,3 @@ class TestAuthenticatedWorkspaceReporter: 'Authorization': 'Bearer refreshed-workspace-owner-token' } - -class TestStartSendTask: - """Tests for start_send_task() method.""" - - @pytest.mark.asyncio - async def test_start_send_task_creates_task(self): - """Test that start_send_task creates an asyncio task.""" - telemetry = get_telemetry_module() - mock_app = Mock() - mock_app.logger = Mock() - mock_app.instance_config = Mock() - mock_app.instance_config.data = {} - - manager = telemetry.TelemetryManager(mock_app) - manager.telemetry_config = {} - - await manager.start_send_task({'query_id': 'test'}) - - # Task should be added to send_tasks list - assert len(manager.send_tasks) >= 1 - - # Clean up the task - for task in manager.send_tasks: - if not task.done(): - task.cancel() - manager.send_tasks.clear() - - @pytest.mark.asyncio - async def test_start_send_task_multiple_tasks(self): - """Test that multiple tasks are tracked.""" - telemetry = get_telemetry_module() - mock_app = Mock() - mock_app.logger = Mock() - mock_app.instance_config = Mock() - mock_app.instance_config.data = {} - - manager = telemetry.TelemetryManager(mock_app) - manager.telemetry_config = {} - - await manager.start_send_task({'query_id': 'test1'}) - await manager.start_send_task({'query_id': 'test2'}) - await manager.start_send_task({'query_id': 'test3'}) - - assert len(manager.send_tasks) >= 3 - - # Clean up - for task in manager.send_tasks: - if not task.done(): - task.cancel() - manager.send_tasks.clear() diff --git a/tests/unit_tests/telemetry/test_trace.py b/tests/unit_tests/telemetry/test_trace.py index f19ab2668..381b0ee6b 100644 --- a/tests/unit_tests/telemetry/test_trace.py +++ b/tests/unit_tests/telemetry/test_trace.py @@ -1,10 +1,19 @@ -"""Unit tests for per-event execution traces (pkg/telemetry/trace.py + execution.py).""" +"""Unit tests for per-execution execution chains (pkg/telemetry/trace.py + execution.py). + +One execution is one chain of independent records: one ``execution_node`` per +workflow node, emitted the moment the node completes, plus exactly one +``execution_chain`` when the execution closes. Every record of a chain carries +the same ``event_id``. +""" from __future__ import annotations import asyncio +import contextvars +import json import time import types +from datetime import datetime, timedelta, timezone from importlib import import_module @@ -28,8 +37,21 @@ class FakeManager: self.sent.append(payload) return True - def start_send_task(self, payload: dict) -> None: + +class RejectingManager(FakeManager): + """Stand-in whose sender refuses every batch until told to accept them.""" + + def __init__(self, config=None): + super().__init__(config) + self.rejecting = True + self.attempts = 0 + + async def send(self, payload: dict) -> bool: + self.attempts += 1 + if self.rejecting: + return False self.sent.append(payload) + return True def make_counters(config=None): @@ -48,6 +70,19 @@ def trace_config(**overrides): return config +def records_sent(manager): + """Flatten the buffered-record batches a FakeManager received.""" + return [record for payload in manager.sent for record in payload.get('records', [])] + + +def nodes(counters): + return [record for record in counters.records if record.get('event_type') == 'execution_node'] + + +def chains(counters): + return [record for record in counters.records if record.get('event_type') == 'execution_chain'] + + CONTEXT = types.SimpleNamespace(instance_uuid='instance-1', workspace_uuid='workspace-1') STAGE = { @@ -58,6 +93,35 @@ STAGE = { 'outcome': 'success', } +NODE_KEYS = { + 'event_type', + 'schema', + 'event_id', + 'instance_id', + 'workspace_uuid', + 'version', + 'edition', + 'timestamp', + 'debug', + 'sample', + 'node_id', + 'parent_node_id', + 'root', + 'seq', + 'family', + 'operation', + 'mode', + 'adapter', + 'runner', + 'outcome', + 'error', + 'route_ref', + 'run_id', + 'synthetic', + 'started_at', + 'ended_at', +} + class TestTraceIdentity: def test_nested_bind_reuses_the_in_flight_trace(self): @@ -85,14 +149,35 @@ class TestTraceIdentity: # A repeated unbind of the same binding must never raise. assert trace.unbind_root(outer) is True + def test_event_id_is_the_execution_id(self): + trace, _ = get_modules() + binding = trace.bind('exec-1234') + try: + assert binding.created is True + assert binding.state.event_id == 'exec-1234' + finally: + trace.unbind_root(binding) + + def test_nested_boundary_keeps_the_enclosing_execution_id(self): + trace, _ = get_modules() + outer = trace.bind('event-1') + inner = trace.bind('query-9') + try: + assert inner.created is False + assert inner.state.event_id == 'event-1' + finally: + trace.unbind_root(inner) + trace.unbind_root(outer) + def test_route_and_run_are_only_pinned_inside_the_block(self): trace, _ = get_modules() + _, counters = make_counters(trace_config()) binding = trace.bind() token = trace.set_run('run-1') try: with trace.scope(route_ref='agent:agent-1'): - inside = trace.current() - inside.append( + counters.record( + CONTEXT, family='runner', operation='execute', mode='agent', @@ -101,8 +186,8 @@ class TestTraceIdentity: outcome='success', synthetic=False, ) - outside = trace.current() - outside.append( + counters.record( + CONTEXT, family='platform_api', operation='send_message', mode='agent', @@ -114,7 +199,7 @@ class TestTraceIdentity: finally: trace.reset_run(token) trace.unbind_root(binding) - assert [(s['route_ref'], s['run_id']) for s in binding.state.stages] == [ + assert [(row['route_ref'], row['run_id']) for row in nodes(counters)] == [ ('agent:agent-1', 'run-1'), ('', 'run-1'), ] @@ -123,74 +208,240 @@ class TestTraceIdentity: class TestTraceRecording: def test_stage_carries_sequence_identity_and_timestamps(self): trace, _ = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = trace.bind() try: counters.record(CONTEXT, **STAGE) counters.record(CONTEXT, **{**STAGE, 'family': 'runner', 'operation': 'execute', 'mode': 'pipeline'}) + counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - stages = binding.state.stages - assert [stage['seq'] for stage in stages] == [0, 1] - assert stages[0]['first_seen'] == stages[0]['last_seen'] - assert binding.state.identity == {'instance_id': 'instance-1', 'workspace_uuid': 'workspace-1'} - assert binding.state.outcome() == 'success' - - def test_stage_buffer_is_bounded_and_counts_overflow(self): - trace, _ = get_modules() - _, counters = make_counters() - binding = trace.bind() - try: - for _ in range(trace.MAX_STAGES + 5): - counters.record(CONTEXT, **STAGE) - finally: - trace.unbind_root(binding) - assert len(binding.state.stages) == trace.MAX_STAGES - assert binding.state.dropped_stages == 5 + rows = nodes(counters) + assert [row['seq'] for row in rows] == [0, 1] + # A plain observation starts and ends at the same instant. + assert rows[0]['started_at'] == rows[0]['ended_at'] + assert {(row['instance_id'], row['workspace_uuid']) for row in rows} == {('instance-1', 'workspace-1')} + chain = chains(counters)[0] + assert chain['outcome'] == 'success' + assert chain['stage_count'] == 2 def test_oversized_identifier_never_becomes_a_stage(self): trace, _ = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = trace.bind() try: counters.record(CONTEXT, **{**STAGE, 'operation': 'x' * 200}) counters.record(CONTEXT, **{**STAGE, 'family': 'not-a-family'}) finally: trace.unbind_root(binding) - assert binding.state.stages == [] + assert nodes(counters) == [] - def test_trace_limit_abandons_further_traces(self): - trace, execution = get_modules() + def test_observation_without_an_owning_execution_is_dropped(self): + _, _ = get_modules() _, counters = make_counters() - bindings = [] - for _ in range(execution.MAX_TRACES + 1): - binding = trace.bind() - bindings.append(binding) - counters.record(CONTEXT, **STAGE) - trace.unbind_root(binding) - assert trace.current() is None - assert len(counters.traces) == execution.MAX_TRACES - assert counters.dropped == 1 - assert bindings[-1].state.abandoned is True - assert bindings[-1].state.stages == [] + # An observation that belongs to no execution is discarded instead of + # being aggregated into a synthetic bucket. + counters.record(CONTEXT, **STAGE) + assert counters.records == [] + assert counters.registry == {} - def test_counters_keep_aggregating_while_tracing(self): + +class TestTraceNodeScope: + """Workflow step nodes: a step's own record and the observations it calls.""" + + @staticmethod + def _step(**overrides): + return { + 'family': 'runner', + 'operation': 'execute', + 'mode': 'agent', + 'runner': 'runner-1', + 'outcome': 'success', + **overrides, + } + + def test_nested_observation_carries_the_step_node_as_parent(self): trace, _ = get_modules() _, counters = make_counters() binding = trace.bind() try: - for _ in range(3): - counters.record(CONTEXT, **STAGE) + with trace.stage_scope() as step: + counters.record(CONTEXT, node=step, **self._step()) + counters.record(CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}) finally: trace.unbind_root(binding) - assert list(counters.pending.values())[0]['count'] == 3 + + step_row, api_row = nodes(counters) + assert step_row['node_id'] == step + assert step_row['parent_node_id'] == '' # a root step carries no parent + assert step_row['root'] is True + assert api_row['parent_node_id'] == step + assert api_row['root'] is False + assert api_row['node_id'] != step # observations are not step nodes + + def test_nested_step_records_its_enclosing_step_as_parent(self): + trace, _ = get_modules() + _, counters = make_counters() + binding = trace.bind() + try: + with trace.stage_scope() as outer: + counters.record(CONTEXT, node=outer, **self._step(mode='pipeline')) + with trace.stage_scope() as inner: + counters.record(CONTEXT, node=inner, **self._step()) + counters.record(CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}) + finally: + trace.unbind_root(binding) + + outer_row, inner_row, api_row = nodes(counters) + assert (outer_row['node_id'], outer_row['parent_node_id']) == (outer, '') + # The step's own record sits under its enclosing step, never under itself. + assert (inner_row['node_id'], inner_row['parent_node_id']) == (inner, outer) + assert api_row['parent_node_id'] == inner and api_row['node_id'] != inner + + def test_sequential_steps_get_distinct_ids_and_roots_have_no_parent(self): + trace, _ = get_modules() + _, counters = make_counters() + binding = trace.bind() + try: + with trace.stage_scope() as first: + counters.record(CONTEXT, node=first, **self._step()) + with trace.stage_scope() as second: + counters.record(CONTEXT, node=second, **self._step()) + finally: + trace.unbind_root(binding) + + assert first and second and first != second + rows = nodes(counters) + assert [row['node_id'] for row in rows] == [first, second] + assert [row['seq'] for row in rows] == [0, 1] + assert rows[0]['root'] is True + assert all(row['parent_node_id'] == '' for row in rows) + + def test_cross_task_observation_joins_the_owning_execution(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-cross') + try: + with trace.stage_scope() as lane: + counters.record( + CONTEXT, family='pipeline', operation='run', mode='pipeline', outcome='success', node=lane + ) + # A plugin/RPC observation arrives on a task that never had the + # trace context; it is resolved through the execution registry and + # still lands on the owning chain. + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-cross') + foreign.run( + counters.record, + CONTEXT, + family='platform_api', + operation='send_message', + mode='pipeline', + outcome='success', + ) + finally: + trace.unbind_root(binding) + + lane_row, foreign_row = nodes(counters) + assert lane_row['node_id'] == lane and lane_row['parent_node_id'] == '' and lane_row['root'] is True + # The cross-task node joins the same execution and hangs off the step the + # chain had open - the step that caused the work - instead of mirroring + # any task's own stack. + assert foreign_row['event_id'] == 'exec-cross' + assert foreign_row['parent_node_id'] == lane + assert foreign_row['root'] is False + + def test_cross_task_observation_attaches_to_the_innermost_open_step(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-nested') + try: + with trace.stage_scope() as outer: + counters.record(CONTEXT, **{**STAGE, 'node': outer}) + with trace.stage_scope() as inner: + counters.record(CONTEXT, **{**STAGE, 'node': inner}) + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-nested') + foreign.run(counters.record, CONTEXT, **{**STAGE, 'family': 'tool', 'operation': 'exec'}) + finally: + trace.unbind_root(binding) + + outer_row, inner_row, tool_row = nodes(counters) + assert outer_row['parent_node_id'] == '' + assert inner_row['parent_node_id'] == outer + # The lane that caused the cross-task work is the innermost one open. + assert tool_row['parent_node_id'] == inner + + def test_cross_task_observation_after_the_lane_closed_is_a_root(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-late') + try: + with trace.stage_scope() as lane: + counters.record(CONTEXT, **{**STAGE, 'node': lane}) + # A late callback arrives after the owning step returned: nothing is + # open anymore, so the node is a root of the chain rather than a + # child of a finished step. + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-late') + foreign.run(counters.record, CONTEXT, family='platform_api', operation='reply', mode='pipeline') + finally: + trace.unbind_root(binding) + + lane_row, late_row = nodes(counters) + assert late_row['parent_node_id'] == '' and late_row['root'] is False + assert lane_row['root'] is True + + def test_cross_task_observation_without_an_open_step_has_no_parent(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-bare') + try: + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-bare') + foreign.run( + counters.record, + CONTEXT, + family='platform_api', + operation='send_message', + mode='pipeline', + outcome='success', + ) + finally: + trace.unbind_root(binding) + + row = nodes(counters)[0] + assert row['parent_node_id'] == '' and row['root'] is True + + def test_scope_outside_any_trace_is_inert(self): + trace, _ = get_modules() + with trace.stage_scope() as node: + assert node == '' + assert trace.current_parent() == '' + + def test_records_outside_any_scope_are_root_nodes(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-legacy') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + + node = nodes(counters)[0] + assert set(node) == NODE_KEYS + assert node['node_id'] and node['parent_node_id'] == '' and node['root'] is True + assert node['seq'] == 0 + # Node placement lives on node records only; the chain carries none. + assert 'node_id' not in json.dumps(chains(counters)[0], sort_keys=True) class TestTraceDelivery: - def test_close_emits_one_payload_per_trace(self): + def test_close_emits_one_terminal_chain_record_per_execution(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) - binding = trace.bind() + binding = trace.bind('exec-1') try: counters.record(CONTEXT, **STAGE) counters.record( @@ -201,68 +452,82 @@ class TestTraceDelivery: finally: trace.unbind_root(binding) - assert len(manager.sent) == 1 - payload = manager.sent[0] - assert payload['event_type'] == 'feature_execution' - assert payload['query_id'] == binding.state.trace_id - assert len(payload['query_id']) == 36 - assert payload['instance_id'] == 'instance-1' - assert payload['workspace_uuid'] == 'workspace-1' - features = payload['features'] - assert features['schema'] == 1 - assert [row['count'] for row in features['observations']] == [1, 1] - assert [row['seq'] for row in features['observations']] == [0, 1] - assert [row['trace_id'] for row in features['observations']] == [binding.state.trace_id] * 2 - assert features['trace'] == { - 'closed_by': 'event_done', - 'started_at': binding.state.started_at, - 'ended_at': features['trace']['ended_at'], - 'route_ref': '', - 'run_id': '', - 'outcome': 'success', - 'dropped_stages': 0, - } - assert counters.traces == {} + # Closing buffers; nothing is sent before the flush cadence. + assert manager.sent == [] + assert len(nodes(counters)) == 2 + chain_rows = chains(counters) + assert len(chain_rows) == 1 + record = chain_rows[0] + assert record['schema'] == 2 + assert record['event_id'] == 'exec-1' + assert record['instance_id'] == 'instance-1' + assert record['workspace_uuid'] == 'workspace-1' + assert record['closed_by'] == 'event_done' + assert record['outcome'] == 'success' + assert record['stage_count'] == 2 + assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 + assert 'query_id' not in record and 'features' not in record def test_sampling_decision_is_deterministic_and_keeps_failures(self): trace, _ = get_modules() _, counters = make_counters(trace_config(execution_trace='sampled', execution_trace_sample=2)) - binding = trace.bind() - try: - binding.state.trace_id = '00000001-0000-4000-8000-000000000000' - counters.record(CONTEXT, **STAGE) - assert counters._trace_emitted(binding.state) is False - binding.state.trace_id = '00000000-0000-4000-8000-000000000000' - assert counters._trace_emitted(binding.state) is True - binding.state.trace_id = '00000001-0000-4000-8000-000000000001' - counters.record(CONTEXT, **{**STAGE, 'outcome': 'timeout'}) - assert counters._trace_emitted(binding.state) is True - finally: - trace.unbind_root(binding) + in_id = next(i for i in (f'sample-{n}' for n in range(64)) if counters._sampling_decision(i)[2]) + out_id = next(i for i in (f'sample-{n}' for n in range(64)) if not counters._sampling_decision(i)[2]) + # Same id, same answer: the decision is a pure function of the identity. + assert counters._sampling_decision(in_id) == counters._sampling_decision(in_id) - def test_failures_mode_keeps_failed_and_debug_traces_only(self): + excluded = trace.bind(out_id) + counters.record(CONTEXT, **STAGE) + assert excluded.state.emit_nodes is False + assert counters._chain_emitted(excluded.state) is False + counters.close_trace(excluded.state, 'event_done') + trace.unbind_root(excluded) + assert counters.records == [] + + included = trace.bind(in_id) + counters.record(CONTEXT, **STAGE) + assert included.state.emit_nodes is True + counters.close_trace(included.state, 'event_done') + trace.unbind_root(included) + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + + counters.records.clear() + broken = trace.bind(out_id) + counters.record(CONTEXT, **{**STAGE, 'outcome': 'timeout'}) + assert counters._chain_emitted(broken.state) is True + counters.close_trace(broken.state, 'event_done') + trace.unbind_root(broken) + assert [record['event_type'] for record in counters.records] == ['execution_chain'] + assert counters.records[0]['outcome'] == 'timeout' + + def test_failures_mode_keeps_failed_and_debug_chains_only(self): trace, _ = get_modules() - manager, counters = make_counters(trace_config(execution_trace='failures', execution_trace_sample=1000000)) + _, counters = make_counters(trace_config(execution_trace='failures')) - failed = trace.bind() + failed = trace.bind('exec-failed') counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) counters.close_trace(failed.state, 'event_done') trace.unbind_root(failed) - debug = trace.bind() - counters.record(CONTEXT, **{**STAGE, 'synthetic': True}) + debug = trace.bind('exec-debug') + counters.configure(debug.state, debug=True) + counters.record(CONTEXT, **STAGE) counters.close_trace(debug.state, 'event_done') trace.unbind_root(debug) - sampled_out = trace.bind() + sampled_out = trace.bind('exec-sampled') counters.record(CONTEXT, **STAGE) counters.close_trace(sampled_out.state, 'event_done') trace.unbind_root(sampled_out) - assert [payload['features']['observations'][0]['outcome'] for payload in manager.sent] == ['failed', 'success'] - assert manager.sent[1]['features']['observations'][0]['synthetic'] is True + assert [record['event_id'] for record in chains(counters)] == ['exec-failed', 'exec-debug'] + assert chains(counters)[0]['outcome'] == 'failed' + # Its node was sampled out, but the broken chain still explains itself. + assert chains(counters)[0]['dropped_nodes'] == 1 + assert chains(counters)[1]['debug'] is True + assert [record['event_id'] for record in nodes(counters)] == ['exec-debug'] - def test_off_mode_and_opt_out_emit_nothing(self): + def test_off_mode_and_opt_out_buffer_nothing(self): trace, _ = get_modules() manager, counters = make_counters(trace_config(execution_trace='off')) binding = trace.bind() @@ -272,6 +537,7 @@ class TestTraceDelivery: finally: trace.unbind_root(binding) assert manager.sent == [] + assert counters.records == [] disabled_manager, disabled = make_counters( {'url': 'https://space.example.test', 'disable_telemetry': True, 'execution_trace': 'all'} @@ -283,50 +549,85 @@ class TestTraceDelivery: finally: trace.unbind_root(binding) assert disabled_manager.sent == [] - assert disabled.pending == {} - assert disabled.traces == {} + assert disabled.records == [] + assert disabled.registry == {} - def test_ttl_sweep_closes_stale_trace(self): + async def test_shutdown_does_not_claim_an_unfinished_execution(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) - binding = trace.bind() - try: - counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - counters.trace_deadlines[binding.state.trace_id] = 0 - counters._sweep_traces(time.monotonic()) - finally: - trace.unbind_root(binding) - assert manager.sent[0]['features']['trace']['closed_by'] == 'ttl' - assert counters.traces == {} - assert counters.trace_deadlines == {} - - async def test_shutdown_drops_open_traces(self): - trace, _ = get_modules() - manager, counters = make_counters(trace_config()) - binding = trace.bind() + binding = trace.bind('exec-open') try: counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) await counters.shutdown() finally: trace.unbind_root(binding) - # Shutdown may still flush window counters; it must not claim a trace. - assert [payload for payload in manager.sent if 'trace' in payload['features']] == [] - assert counters.traces == {} - assert counters.dropped_traces == 1 + # The already-emitted node is flushed, but shutdown never fabricates a + # terminal chain for an execution that never completed. + assert [record['event_type'] for record in records_sent(manager)] == ['execution_node'] + assert counters.records == [] + assert chains(counters) == [] + assert counters.registry == {} - async def test_loop_sweeps_and_sends_expired_trace(self): - trace, _ = get_modules() + async def test_flush_sends_one_batch_of_complete_records(self): + _, execution = get_modules() manager, counters = make_counters(trace_config()) - binding = trace.bind() + ap = make_ap(counters) + for index in range(2): + with execution.ingress(ap, 'event_done', CONTEXT, execution_id=f'exec-{index}'): + counters.record(CONTEXT, **STAGE) + assert len(counters.records) == 4 + + await counters.flush() + assert len(manager.sent) == 1 + payload = manager.sent[0] + assert set(payload) == {'records'} + assert [record['event_type'] for record in payload['records']] == [ + 'execution_node', + 'execution_chain', + 'execution_node', + 'execution_chain', + ] + assert [record['event_id'] for record in payload['records']] == ['exec-0', 'exec-0', 'exec-1', 'exec-1'] + await counters.shutdown() + + async def test_flush_interval_is_honoured(self): + _, execution = get_modules() + manager, counters = make_counters(trace_config(telemetry_flush_seconds=1)) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-timed'): + counters.record(CONTEXT, **STAGE) + assert counters.flush_seconds() == 1 + assert manager.sent == [] + await asyncio.sleep(1.3) + sent = records_sent(manager) + assert [record['event_type'] for record in sent] == ['execution_node', 'execution_chain'] + assert {record['event_id'] for record in sent} == {'exec-timed'} + await counters.shutdown() + + def test_flush_interval_config_is_bounded(self): + _, counters = make_counters(trace_config()) + assert counters.flush_seconds() == 180 + counters.manager.telemetry_config = trace_config(telemetry_flush_seconds='nope') + assert counters.flush_seconds() == 180 + counters.manager.telemetry_config = trace_config(telemetry_flush_seconds=0) + assert counters.flush_seconds() == 180 + counters.manager.telemetry_config = trace_config(telemetry_flush_seconds=300) + assert counters.flush_seconds() == 300 + + def test_buffered_record_cap_drops_oldest(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + original = execution.MAX_BUFFERED_RECORDS + execution.MAX_BUFFERED_RECORDS = 3 try: - counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - counters.trace_deadlines[binding.state.trace_id] = 0 - await asyncio.sleep(1.2) - assert [payload['features']['trace']['closed_by'] for payload in manager.sent] == ['ttl'] - assert counters.traces == {} + for index in range(5): + binding = trace.bind(f'exec-{index}') + counters.record(CONTEXT, **STAGE) + trace.unbind_root(binding) finally: - await counters.shutdown() - trace.unbind_root(binding) + execution.MAX_BUFFERED_RECORDS = original + assert [record['event_id'] for record in nodes(counters)] == ['exec-2', 'exec-3', 'exec-4'] + assert counters.dropped == 2 class TestIngress: @@ -334,38 +635,380 @@ class TestIngress: _, execution = get_modules() manager, counters = make_counters(trace_config()) ap = make_ap(counters) - with execution.ingress(ap, 'event_done'): - with execution.ingress(ap, 'pipeline_done'): + with execution.ingress(ap, 'event_done', execution_id='exec-outer'): + with execution.ingress(ap, 'pipeline_done', execution_id='exec-inner'): counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - assert len(manager.sent) == 1 - assert manager.sent[0]['features']['trace']['closed_by'] == 'event_done' + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'exec-outer' for record in counters.records) + assert chains(counters)[0]['closed_by'] == 'event_done' + assert manager.sent == [] def test_ingress_without_telemetry_is_a_no_op(self): _, execution = get_modules() - with execution.ingress(types.SimpleNamespace(), 'event_done'): + with execution.ingress(types.SimpleNamespace(), 'event_done', execution_id='exec-x'): pass + def test_ingress_registers_every_execution_alias(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', execution_id='event-1') as outer: + with execution.ingress(ap, 'pipeline_done', execution_id='query-1') as inner: + assert inner.state is outer.state + assert counters.registry == {} + # Closing the owning trace unregisters every alias it held. + assert counters.registry_deadlines == {} -class AsyncSendManager(FakeManager): - """Stand-in whose start_send_task is a coroutine, like TelemetryManager.""" - async def start_send_task(self, payload: dict) -> None: - self.sent.append(payload) +class TestCrossTaskAttribution: + def test_registry_joins_nodes_from_a_task_without_a_trace(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'run-1') + trace.unbind_root(binding) + # The plugin/RPC task has no inherited trace context. + assert trace.current() is None + + counters.record(CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}) + counters.record( + CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}, execution_id='run-1' + ) + rows = nodes(counters) + assert len(rows) == 1 + assert rows[0]['operation'] == 'send_message' and rows[0]['event_id'] == 'run-1' + + counters.close_trace(binding.state, 'event_done') + assert chains(counters)[0]['event_id'] == 'run-1' + + def test_registry_aliases_a_nested_execution(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='event-1'): + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-1'): + pass + assert set(counters.registry) == {'event-1', 'query-1'} + counters.record(CONTEXT, **STAGE, execution_id='query-1') + rows = nodes(counters) + assert len(rows) == 1 and rows[0]['event_id'] == 'event-1' + + def test_contextvar_execution_id_is_used_when_no_kwarg(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'run-2') + trace.unbind_root(binding) + token = execution.set_execution_id('run-2') + try: + counters.record(CONTEXT, **STAGE) + finally: + execution.reset_execution_id(token) + rows = nodes(counters) + assert len(rows) == 1 and rows[0]['event_id'] == 'run-2' + + def test_registry_alias_is_swept_when_never_closed(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'run-3') + trace.unbind_root(binding) + counters.registry_deadlines['run-3'] = 0 + counters._sweep_registry(time.monotonic()) + assert counters.registry == {} + assert counters.registry_deadlines == {} class TestTraceDispatch: - async def test_close_trace_schedules_the_coroutine_send(self): - trace, execution = get_modules() + async def test_flush_awaits_the_manager_send(self): + _, execution = get_modules() manager = AsyncSendManager(trace_config()) counters = execution.ExecutionCounters(manager) - binding = trace.bind() + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-async'): + counters.record(CONTEXT, **STAGE) + await counters.flush() + sent = records_sent(manager) + assert [record['event_type'] for record in sent] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'exec-async' for record in sent) + await counters.shutdown() + + +class AsyncSendManager(FakeManager): + """Stand-in whose send is a coroutine, like TelemetryManager.""" + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +class TestTraceFailureDetail: + def test_failed_stage_carries_the_reason(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-fail') + try: + counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}, error='boom: sandbox exited 127') + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + assert nodes(counters)[0]['error'] == 'boom: sandbox exited 127' + # The detail lives on the node; the chain carries the terminal outcome. + assert chains(counters)[0]['outcome'] == 'failed' + + def test_successful_stage_reports_no_error(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-ok') try: counters.record(CONTEXT, **STAGE) counters.close_trace(binding.state, 'event_done') - # The payload is only delivered once the scheduled task runs. - assert manager.sent == [] - await asyncio.sleep(0) finally: trace.unbind_root(binding) - assert len(manager.sent) == 1 - assert manager.sent[0]['event_type'] == 'feature_execution' + assert nodes(counters)[0]['error'] == '' + assert chains(counters)[0]['error'] == '' + + def test_error_detail_is_bounded_and_single_line(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-noisy') + try: + counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}, error='x' * 1000 + '\n\tboom') + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + detail = chains(counters)[0]['error'] + assert len(detail) <= 200 + assert '\n' not in detail and '\t' not in detail + + +class TestIngressFailureDetail: + def test_ingress_records_the_failure_reason(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + raised = False + try: + with execution.ingress(ap, 'event_done', execution_id='exec-boom'): + counters.record(CONTEXT, **STAGE) + raise RuntimeError('dispatch exploded') + except RuntimeError: + raised = True + assert raised is True + chain = chains(counters)[0] + assert chain['error'].startswith('RuntimeError: dispatch exploded') + assert chain['outcome'] == 'failed' + assert chain['closed_by'] == 'event_done' + + def test_successful_ingress_still_reports_success(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', execution_id='exec-fine'): + counters.record(CONTEXT, **STAGE) + chain = chains(counters)[0] + assert chain['error'] == '' + assert chain['outcome'] == 'success' + + +class TestFailureOnlyTrace: + def test_failure_without_stages_still_uploads(self): + _, execution = get_modules() + # failures mode: nodes are not reported, but a broken chain always is. + _, counters = make_counters({'url': 'https://space.example.test', 'execution_trace': 'failures'}) + ap = make_ap(counters) + try: + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='exec-empty'): + raise RuntimeError('pipeline exploded before any stage') + except RuntimeError: + pass + rows = chains(counters) + assert len(rows) == 1 + record = rows[0] + assert record['error'].startswith('RuntimeError: pipeline exploded') + assert record['outcome'] == 'failed' + assert record['stage_count'] == 0 + assert nodes(counters) == [] + + def test_cancelled_and_timeout_chains_report_their_outcome(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + for outcome, reason in (('cancelled', 'run cancelled by operator'), ('timeout', 'deadline exhausted')): + binding = trace.bind(f'exec-{outcome}') + try: + counters.record(CONTEXT, **{**STAGE, 'outcome': outcome}, error=reason) + counters.close_trace(binding.state, 'runner_done') + finally: + trace.unbind_root(binding) + record = chains(counters)[-1] + assert record['event_id'] == f'exec-{outcome}' + assert record['outcome'] == outcome + assert nodes(counters)[-1]['error'] == reason + + +class TestLegacyRecordFields: + def test_chain_carries_execution_scoped_columns(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-legacy') + binding.state.started_at = (datetime.now(timezone.utc) - timedelta(seconds=2)).isoformat() + binding.state.model_name = 'gpt-4o' + binding.state.pipeline_plugins = ['plugin:a'] + binding.state.runner_category = 'cloud' + try: + counters.record(CONTEXT, **STAGE) + counters.record( + CONTEXT, **{**STAGE, 'family': 'runner', 'operation': 'execute', 'mode': 'agent', 'runner': 'runner-1'} + ) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + record = chains(counters)[0] + assert record['model_name'] == 'gpt-4o' + assert record['pipeline_plugins'] == ['plugin:a'] + assert record['runner_category'] == 'cloud' + assert record['adapter'] == 'AiocqhttpAdapter' + assert record['runner'] == 'runner-1' + assert record['duration_ms'] >= 1000 + + def test_legacy_string_fields_are_never_null(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-blank') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + record = chains(counters)[0] + assert record['model_name'] == '' + assert record['runner'] == '' + assert record['runner_category'] == '' + assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 + assert record['pipeline_plugins'] is None + + +class TestNoTraceTtl: + def test_nodes_more_than_two_minutes_apart_still_emit(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-long') + try: + counters.record(CONTEXT, **STAGE) + # A gap far beyond the old 120s TTL must not sweep the chain: there is + # no local trace deadline any more, so both nodes and the terminal + # chain are still emitted. + binding.state.started_at = (datetime.now(timezone.utc) - timedelta(seconds=200)).isoformat() + counters.record(CONTEXT, **{**STAGE, 'operation': 'still.alive'}) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + rows = nodes(counters) + assert [record['seq'] for record in rows] == [0, 1] + assert [record['operation'] for record in rows] == ['message.received', 'still.alive'] + chain = chains(counters)[0] + assert chain['stage_count'] == 2 + assert chain['duration_ms'] >= 120_000 + + +class TestFlushDelivery: + async def test_rejected_batch_stays_buffered_and_is_retried(self): + trace, execution = get_modules() + manager = RejectingManager(trace_config()) + counters = execution.ExecutionCounters(manager) + binding = trace.bind('exec-retry') + try: + counters.record(CONTEXT, **STAGE) + assert await counters.flush() is False + assert manager.attempts == execution.MAX_FLUSH_ATTEMPTS + # The record was never lost: it is still buffered and unsent. + assert len(counters.records) == 1 and manager.sent == [] + manager.rejecting = False + assert await counters.flush() is True + finally: + trace.unbind_root(binding) + assert counters.records == [] + assert [record['event_id'] for record in records_sent(manager)] == ['exec-retry'] + await counters.shutdown() + + async def test_health_reports_buffered_sent_and_dropped(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'exec-health') + try: + counters.record(CONTEXT, **STAGE) + health = counters.health() + assert health['records_buffered'] == 1 + assert health['records_sent'] == 0 + assert health['records_dropped'] == 0 + assert health['executions_in_flight'] == 1 + await counters.flush() + health = counters.health() + assert health['records_buffered'] == 0 + assert health['records_sent'] == 1 + finally: + trace.unbind_root(binding) + + # Overflowing the outbound buffer is surfaced as dropped records. + original = execution.MAX_BUFFERED_RECORDS + execution.MAX_BUFFERED_RECORDS = 1 + try: + for index in range(3): + inner = trace.bind(f'exec-drop-{index}') + counters.record(CONTEXT, **STAGE) + trace.unbind_root(inner) + finally: + execution.MAX_BUFFERED_RECORDS = original + health = counters.health() + assert health['records_dropped'] == 2 + assert health['records_buffered'] == 1 + await counters.shutdown() + + +class TestSiblingTaskIsolation: + async def test_sibling_stage_scopes_do_not_corrupt_parent_links(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + opened = {} + + async def work(name): + with trace.stage_scope() as node: + opened[name] = node + counters.record(CONTEXT, **{**STAGE, 'operation': name, 'node': node}) + + async def scenario(): + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-siblings'): + await asyncio.gather(work('alpha'), work('beta')) + snapshot = list(counters.records) + await counters.shutdown() + return snapshot + + records = await scenario() + rows = {record['operation']: record for record in records if record['event_type'] == 'execution_node'} + assert opened['alpha'] != opened['beta'] + # Each sibling's step is its own root: neither becomes the other's parent. + assert rows['alpha']['node_id'] == opened['alpha'] + assert rows['beta']['node_id'] == opened['beta'] + assert rows['alpha']['parent_node_id'] == '' + assert rows['beta']['parent_node_id'] == '' + assert [record['event_type'] for record in records].count('execution_chain') == 1 + + +class TestSamplingShape: + def test_platform_prefixed_event_id_is_emitted_at_default_config(self): + trace, _ = get_modules() + _, counters = make_counters() # no execution_trace override -> default mode + assert counters.trace_mode() == 'all' + # The identity is hashed, not parsed: a platform-prefixed id samples the + # same way any other id shape does. + assert counters._sampling_decision('platform:bot-1:message-1') == ('all', 1, True) + binding = trace.bind('platform:bot-1:message-1') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'platform:bot-1:message-1' for record in counters.records) diff --git a/tests/unit_tests/test_social_preview.py b/tests/unit_tests/test_social_preview.py new file mode 100644 index 000000000..c71ec6341 --- /dev/null +++ b/tests/unit_tests/test_social_preview.py @@ -0,0 +1,28 @@ +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('social_preview', ROOT / 'src/langbot/pkg/api/http/controller/social_preview.py') +assert spec is not None and spec.loader is not None +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class SocialPreviewTest(unittest.TestCase): + def test_cloud_has_crawler_visible_image(self): + html = (ROOT / 'web/index.html').read_text() + result = module.cloud_preview_html(html, 'cloud.langbot.app') + self.assertIn(module.CLOUD_IMAGE, result) + self.assertIn('summary_large_image', result) + self.assertEqual(result.count('property="og:image"'), 1) + self.assertTrue((ROOT / 'web/public/social/cloud-v3.png').is_file()) + + def test_self_hosted_is_unchanged(self): + html = 'LangBot' + for host in ['localhost:5300', 'example.com', 'cloud.langbot.app.evil.com']: + self.assertEqual(module.cloud_preview_html(html, host), html) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit_tests/test_telemetry.py b/tests/unit_tests/test_telemetry.py deleted file mode 100644 index ed3994923..000000000 --- a/tests/unit_tests/test_telemetry.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -import asyncio - -import pytest - -from langbot.pkg.telemetry.telemetry import TelemetryManager - - -@pytest.mark.asyncio -async def test_send_tasks_are_scoped_to_manager_instance(monkeypatch): - async def fake_send(self, payload): - return payload - - monkeypatch.setattr(TelemetryManager, 'send', fake_send) - - first = TelemetryManager(SimpleNamespace()) - second = TelemetryManager(SimpleNamespace()) - - assert first.send_tasks is not second.send_tasks - - await first.start_send_task({'event': 'first'}) - task = first.send_tasks[0] - await task - await asyncio.sleep(0) - - assert first.send_tasks == [] - assert second.send_tasks == [] diff --git a/uv.lock b/uv.lock index 6661723ba..fcc862510 100644 --- a/uv.lock +++ b/uv.lock @@ -2059,7 +2059,7 @@ wheels = [ [[package]] name = "langbot" -version = "4.11.0b3" +version = "4.11.0rc1" source = { editable = "." } dependencies = [ { name = "aiocqhttp" }, @@ -2185,7 +2185,7 @@ requires-dist = [ { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" }, - { name = "langbot-plugin", url = "https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl" }, + { name = "langbot-plugin", specifier = "==0.7.11rc1" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2255,8 +2255,8 @@ dev = [ [[package]] name = "langbot-plugin" -version = "0.7.6" -source = { url = "https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl" } +version = "0.7.11rc1" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, @@ -2276,29 +2276,9 @@ dependencies = [ { name = "watchdog" }, { name = "websockets" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/77/e4/882ed1fd15067e3ca778da2850833ad61716741a6f63d3328028496536b8/langbot_plugin-0.7.11rc1.tar.gz", hash = "sha256:2ef8ba6f1998da365c7ade53573b8cd65f1a660f772ed97385ca445f2b0faa58", size = 690128, upload-time = "2026-10-08T07:49:33.789Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl", hash = "sha256:b2f05302ca25b374ff8c094794573696b75a7d1a7731a0fe2eebbc71796e8e0d" }, -] - -[package.metadata] -requires-dist = [ - { name = "aiofiles", specifier = ">=24.1.0" }, - { name = "aiohttp", specifier = ">=3.9.0" }, - { name = "dotenv", specifier = ">=0.9.9" }, - { name = "e2b", specifier = ">=2.15" }, - { name = "httpx", specifier = ">=0.28.1" }, - { name = "jinja2", specifier = ">=3.1.6" }, - { name = "packaging", specifier = ">=24.0" }, - { name = "pip", specifier = ">=25.2" }, - { name = "pydantic", specifier = ">=2.11.5" }, - { name = "pydantic-settings", specifier = ">=2.10.1" }, - { name = "pytest", specifier = ">=8.4.0" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "textual", specifier = ">=3.2.0" }, - { name = "types-aiofiles", specifier = ">=24.1.0.20250516" }, - { name = "types-pyyaml", specifier = ">=6.0.12.20250516" }, - { name = "watchdog", specifier = ">=6.0.0" }, - { name = "websockets", specifier = ">=15.0.1" }, + { url = "https://files.pythonhosted.org/packages/57/84/21f04d8441c47c910a345a90d5ca5ecc1222269bb951f015f404996b7a36/langbot_plugin-0.7.11rc1-py3-none-any.whl", hash = "sha256:73227613ef827cfcccb04c2535034139ce5315c54dcd32ac01fcbca2150b7d13", size = 442833, upload-time = "2026-10-08T07:49:32.305Z" }, ] [[package]] diff --git a/web/package.json b/web/package.json index 554594214..358f146c6 100644 --- a/web/package.json +++ b/web/package.json @@ -55,6 +55,7 @@ "@radix-ui/react-toggle": "^1.1.8", "@radix-ui/react-toggle-group": "^1.1.9", "@radix-ui/react-tooltip": "^1.2.7", + "@remixicon/react": "^4.9.0", "@simplewebauthn/browser": "^14.0.0", "@tailwindcss/postcss": "^4.1.5", "@tanstack/react-table": "^8.21.3", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index fd0c6c5f6..a861a5328 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -93,6 +93,9 @@ dependencies: '@radix-ui/react-tooltip': specifier: ^1.2.7 version: 1.2.8(@types/react-dom@19.2.3)(@types/react@19.2.10)(react-dom@19.2.1)(react@19.2.1) + '@remixicon/react': + specifier: ^4.9.0 + version: 4.9.0(react@19.2.1) '@simplewebauthn/browser': specifier: ^14.0.0 version: 14.0.0 @@ -2921,6 +2924,14 @@ packages: resolution: {integrity: sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==} dev: false + /@remixicon/react@4.9.0(react@19.2.1): + resolution: {integrity: sha512-5/jLDD4DtKxH2B4QVXTobvV1C2uL8ab9D5yAYNtFt+w80O0Ys1xFOrspqROL3fjrZi+7ElFUWE37hBfaAl6U+Q==} + peerDependencies: + react: '>=18.2.0' + dependencies: + react: 19.2.1 + dev: false + /@rolldown/binding-android-arm64@1.0.3: resolution: {integrity: sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==} engines: {node: ^20.19.0 || >=22.12.0} diff --git a/web/public/social/cloud-v3.png b/web/public/social/cloud-v3.png new file mode 100644 index 000000000..5ab66f53f Binary files /dev/null and b/web/public/social/cloud-v3.png differ diff --git a/web/src/app/home/agents/PluginProcessorDetailContent.tsx b/web/src/app/home/agents/PluginProcessorDetailContent.tsx index 53e08f708..40a3a3b30 100644 --- a/web/src/app/home/agents/PluginProcessorDetailContent.tsx +++ b/web/src/app/home/agents/PluginProcessorDetailContent.tsx @@ -152,6 +152,23 @@ export default function PluginProcessorDetailContent({ [t, available, hasParameters, canOperate], ); + const applyInstalledProcessor = useCallback((component: RunnerDescriptor) => { + setComponents((current) => [ + ...current.filter((item) => item.id !== component.id), + component, + ]); + setComponentRef(component.id); + setParameters( + Object.fromEntries( + (component.config_schema ?? []) + .filter((field) => field.default !== undefined) + .map((field) => [field.name, field.default]), + ), + ); + setActiveTab('config'); + validate.current = null; + }, []); + const load = useCallback(async () => { setLoading(true); setFailed(false); @@ -469,6 +486,8 @@ export default function PluginProcessorDetailContent({ } titleControls={ ('runner'); const isSavingRef = useRef(false); - const hasUnsavedChangesRef = useRef(false); const loadedHostToolPolicyRef = useRef(undefined); const formSchema = z.object({ @@ -197,18 +197,20 @@ function AgentFormComponent( const savedSnapshotRef = useRef(''); const initializedStagesRef = useRef>(new Set()); - const watchedValues = form.watch(); - const hasUnsavedChanges = (() => { - if (!savedSnapshotRef.current) return false; - return JSON.stringify(watchedValues) !== savedSnapshotRef.current; - })(); - hasUnsavedChangesRef.current = hasUnsavedChanges; + const { + isDirty: hasUnsavedChanges, + dirtyRef: hasUnsavedChangesRef, + refreshDirty, + } = useSnapshotDirty(form, savedSnapshotRef); useEffect(() => { onDirtyChange?.(hasUnsavedChanges); }, [hasUnsavedChanges, onDirtyChange]); - const supportedEventPatterns = form.watch('supported_event_patterns'); + const supportedEventPatterns = useWatch({ + control: form.control, + name: 'supported_event_patterns', + }); useEffect(() => { onSupportedEventPatternsChange?.(supportedEventPatterns); }, [onSupportedEventPatternsChange, supportedEventPatterns]); @@ -279,6 +281,7 @@ function AgentFormComponent( }; form.reset(loadedValues); savedSnapshotRef.current = JSON.stringify(loadedValues); + refreshDirty(); initializedStagesRef.current.clear(); setInitialDataLoaded(true); }) @@ -290,7 +293,7 @@ function AgentFormComponent( return () => { cancelled = true; }; - }, [agentId, form, t, loadAttempt]); + }, [agentId, form, t, loadAttempt, refreshDirty]); useEffect(() => { if (!initialDataLoaded || !readPendingRunnerInstall(runnerInstallScope)) { @@ -340,7 +343,11 @@ function AgentFormComponent( void loadPluginSystemStatus(); }, [loadPluginSystemStatus]); - const currentRunner = (form.watch('runner') as Record)?.id; + const currentRunner = useWatch({ + control: form.control, + name: 'runner', + compute: (runner) => runner?.id, + }); const runnerOptions = useMemo(() => { const runnerStage = runnerConfigSchema?.stages.find( (stage) => stage.name === 'runner', @@ -358,24 +365,38 @@ function AgentFormComponent( const activeRunnerStage = runnerConfigSchema?.stages.find( (stage) => stage.name === currentRunner, ); - const runnerConfigValues = form.watch('runner_config') as Record< + const runnerConfigValues = form.getValues('runner_config') as Record< string, Record >; - const activeRunnerValues = useMemo( - () => runnerConfigValues?.[currentRunner] ?? {}, - [currentRunner, runnerConfigValues], - ); - const missingRunnerFields = useMemo( - () => - (activeRunnerStage?.config ?? []).filter( - (field) => - field.required && - isRunnerFieldVisible(field, activeRunnerValues) && - isRequiredRunnerValueMissing(activeRunnerValues[field.name]), - ), - [activeRunnerStage, activeRunnerValues], + useWatch({ + control: form.control, + compute: (values) => + (activeRunnerStage?.config ?? []) + .filter( + (field) => + field.required && + isRunnerFieldVisible( + field, + values.runner_config?.[currentRunner] ?? {}, + ) && + isRequiredRunnerValueMissing( + values.runner_config?.[currentRunner]?.[field.name], + ), + ) + .map((field) => field.name) + .join('\n'), + }); + const activeRunnerValues = runnerConfigValues?.[currentRunner] ?? {}; + const missingRunnerFields = (activeRunnerStage?.config ?? []).filter( + (field) => + field.required && + isRunnerFieldVisible(field, activeRunnerValues) && + isRequiredRunnerValueMissing(activeRunnerValues[field.name]), ); + const missingRunnerFieldNames = missingRunnerFields + .map((field) => extractI18nObject(field.label)) + .join(', '); const primarySections: Array<{ name: AgentConfigSection; label: string; @@ -448,9 +469,7 @@ function AgentFormComponent( return { label: t('agents.runnerConfigIncomplete'), description: t('agents.runnerConfigIncompleteDescription', { - fields: missingRunnerFields - .map((field) => extractI18nObject(field.label)) - .join(', '), + fields: missingRunnerFieldNames, }), tone: 'warning', }; @@ -471,7 +490,8 @@ function AgentFormComponent( pluginStatusLoading, pluginSystemStatus, runnerOptions.length, - missingRunnerFields, + missingRunnerFieldNames, + missingRunnerFields.length, selectedRunnerOption, t, ]); @@ -520,11 +540,12 @@ function AgentFormComponent( onRunnerStatusChange?.(runnerStatus); }, [onRunnerStatusChange, runnerStatus]); - function updateSnapshotIfInitial(stageKey: string) { + function updateSnapshotIfInitial(stageKey: string, wasDirty: boolean) { if (!initializedStagesRef.current.has(stageKey)) { initializedStagesRef.current.add(stageKey); - if (!hasUnsavedChanges) { + if (!wasDirty) { savedSnapshotRef.current = JSON.stringify(form.getValues()); + refreshDirty(); } } } @@ -534,9 +555,10 @@ function AgentFormComponent( stageName: string, values: object, ) { + const wasDirty = hasUnsavedChangesRef.current; if (formName === 'runner') { form.setValue('runner', values, { shouldDirty: true }); - updateSnapshotIfInitial(`runner.${stageName}`); + updateSnapshotIfInitial(`runner.${stageName}`, wasDirty); return; } @@ -550,7 +572,7 @@ function AgentFormComponent( }, { shouldDirty: true }, ); - updateSnapshotIfInitial(`runner_config.${stageName}`); + updateSnapshotIfInitial(`runner_config.${stageName}`, wasDirty); } function renderDynamicStage(stage: PipelineConfigStage) { @@ -558,8 +580,8 @@ function AgentFormComponent( if (!isRunnerSelector && stage.name !== currentRunner) return null; const initialValues = isRunnerSelector - ? (form.watch('runner') as Record) || {} - : ((form.watch('runner_config') as Record) || {})[ + ? (form.getValues('runner') as Record) || {} + : ((form.getValues('runner_config') as Record) || {})[ stage.name ] || {}; @@ -640,6 +662,7 @@ function AgentFormComponent( loadedHostToolPolicyRef.current = [...values.allowed_tools]; } savedSnapshotRef.current = submittedSnapshot; + refreshDirty(); onFinish(agent); toast.success(t('agents.saveSuccess')); return true; @@ -655,7 +678,14 @@ function AgentFormComponent( onSavingChange?.(false); } }, - [agentId, hostToolCatalogAvailable, onFinish, onSavingChange, t], + [ + agentId, + hostToolCatalogAvailable, + onFinish, + onSavingChange, + refreshDirty, + t, + ], ); function handleSubmit(values: FormValues) { @@ -682,6 +712,7 @@ function AgentFormComponent( emoji: values.emoji || '🤖', }; savedSnapshotRef.current = JSON.stringify(snapshot); + refreshDirty(); } }, async save() { @@ -693,7 +724,14 @@ function AgentFormComponent( return (await saveValues(form.getValues())) ?? false; }, }), - [form, initialDataLoaded, loadFailed, saveValues], + [ + form, + initialDataLoaded, + loadFailed, + saveValues, + hasUnsavedChangesRef, + refreshDirty, + ], ); if (loadFailed) diff --git a/web/src/app/home/agents/components/PluginProcessorSettings.tsx b/web/src/app/home/agents/components/PluginProcessorSettings.tsx index 34bc94ce8..e318259e0 100644 --- a/web/src/app/home/agents/components/PluginProcessorSettings.tsx +++ b/web/src/app/home/agents/components/PluginProcessorSettings.tsx @@ -1,128 +1,47 @@ -import { Puzzle } from 'lucide-react'; import { useTranslation } from 'react-i18next'; -import { Link } from 'react-router-dom'; import type { RunnerDescriptor } from '@/app/infra/entities/api'; -import { httpClient } from '@/app/infra/http'; -import { extractI18nObject } from '@/i18n/I18nProvider'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/ui/select'; -import { Label } from '@/components/ui/label'; -import { Button } from '@/components/ui/button'; - -function ProcessorComponentContent({ - component, - option = false, -}: { - component: RunnerDescriptor; - option?: boolean; -}) { - const label = extractI18nObject({ - en_US: component.id, - zh_Hans: component.id, - ...component.label, - }); - const pluginId = `${component.plugin_author}/${component.plugin_name}`; - return ( - - - - {label} - - {option && ( - - {pluginId} - - )} - - ); -} +import RunnerSelect from './RunnerSelect'; export default function PluginProcessorSettings({ components, value, onChange, disabled = false, + installScope, + onInstalled, }: { components: RunnerDescriptor[]; value: string; onChange: (value: string) => void; disabled?: boolean; + installScope: string; + onInstalled: (component: RunnerDescriptor) => void; }) { const { t } = useTranslation(); - const selected = components.find((item) => item.id === value); + const options = components.map((component) => ({ + name: component.id, + label: { en_US: component.id, zh_Hans: component.id, ...component.label }, + })); + if (value && !components.some((component) => component.id === value)) + options.push({ + name: value, + label: { + en_US: t('agents.eventProcessor.unavailable'), + zh_Hans: t('agents.eventProcessor.unavailable'), + }, + }); return ( -
- - +
+
); } diff --git a/web/src/app/home/agents/components/RunnerSelect.tsx b/web/src/app/home/agents/components/RunnerSelect.tsx index cb706091c..906bec6e0 100644 --- a/web/src/app/home/agents/components/RunnerSelect.tsx +++ b/web/src/app/home/agents/components/RunnerSelect.tsx @@ -1,15 +1,19 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; -import { Bot, ExternalLink, Loader2, Store } from 'lucide-react'; +import { Bot, Puzzle, ExternalLink, Loader2, Store } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { toast } from 'sonner'; -import { getCloudServiceClientSync, httpClient } from '@/app/infra/http'; +import { getCloudServiceClientSync } from '@/app/infra/http'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import type { IDynamicFormItemOption } from '@/app/infra/entities/form/dynamic'; import type { PluginV4 } from '@/app/infra/entities/plugin'; +import type { RunnerDescriptor } from '@/app/infra/entities/api'; import { RunnerMarketplaceError, getErrorMessage, installMarketplaceRunner, + installMarketplaceEventProcessor, + resumePendingEventProcessorInstall, loadRunnerCatalog, marketplacePluginId, runnerPluginPrefix, @@ -51,21 +55,25 @@ function installErrorMessage( return getErrorMessage(error) || t('wizard.aiEngine.installFailed'); } -function installedRunnerIconURL(option: IDynamicFormItemOption) { - return option.name.startsWith('plugin:') - ? (() => { - const match = option.name.match(/^plugin:([^/]+)\/([^/]+)(?:\/|$)/); - return match ? httpClient.getPluginIconURL(match[1], match[2]) : null; - })() - : null; +function pluginOptionParts(option: IDynamicFormItemOption): { + author: string; + name: string; +} | null { + if (!option.name.startsWith('plugin:')) return null; + const match = option.name.match(/^plugin:([^/]+)\/([^/]+)(?:\/|$)/); + return match ? { author: match[1], name: match[2] } : null; } function InstalledRunnerContent({ option, + eventProcessor = false, }: { option: IDynamicFormItemOption; + eventProcessor?: boolean; }) { - const iconURL = installedRunnerIconURL(option); + const Icon = eventProcessor ? Puzzle : Bot; + const parts = pluginOptionParts(option); + const iconURL = useInstalledPluginIcon(parts?.author, parts?.name); return ( @@ -76,7 +84,7 @@ function InstalledRunnerContent({ className="size-5 shrink-0 rounded object-cover" /> ) : ( - + )} {extractI18nObject(option.label)} @@ -86,11 +94,15 @@ function InstalledRunnerContent({ function InstalledRunnerOptionContent({ option, description, + eventProcessor = false, }: { option: IDynamicFormItemOption; description: string; + eventProcessor?: boolean; }) { - const iconURL = installedRunnerIconURL(option); + const Icon = eventProcessor ? Puzzle : Bot; + const parts = pluginOptionParts(option); + const iconURL = useInstalledPluginIcon(parts?.author, parts?.name); return ( @@ -101,7 +113,7 @@ function InstalledRunnerOptionContent({ className="row-span-2 size-7 shrink-0 rounded-md object-cover" /> ) : ( - + )} {extractI18nObject(option.label)} @@ -199,13 +211,19 @@ export default function RunnerSelect({ onValueChange, installScope, onInstalled, + usage = 'agent', + disabled = false, + onEventProcessorInstalled, }: { options: IDynamicFormItemOption[]; label: string; value: string; onValueChange: (value: string) => void; installScope: string; - onInstalled: (installed: InstalledRunner) => void; + onInstalled?: (installed: InstalledRunner) => void; + usage?: 'agent' | 'event'; + disabled?: boolean; + onEventProcessorInstalled?: (component: RunnerDescriptor) => void; }) { const { t } = useTranslation(); const { addTask, tasks } = usePluginInstallTasks(); @@ -227,7 +245,7 @@ export default function RunnerSelect({ setCatalogLoading(true); setCatalogError(false); try { - const catalog = await loadRunnerCatalog('agent'); + const catalog = await loadRunnerCatalog(usage); setMarketplaceRunners(catalog.marketplaceRunners); setInstalledPluginIds(catalog.installedPluginIds); setInstalledPluginDescriptions(catalog.installedPluginDescriptions); @@ -237,7 +255,37 @@ export default function RunnerSelect({ } finally { setCatalogLoading(false); } - }, []); + }, [usage]); + + useEffect(() => { + if ( + usage !== 'event' || + disabled || + !readPendingRunnerInstall(installScope) + ) + return; + let cancelled = false; + void resumePendingEventProcessorInstall(installScope) + .then((component) => { + if (!cancelled && component) { + onEventProcessorInstalled?.(component); + void loadCatalog(); + } + }) + .catch((error) => { + if (!cancelled) setInstallError(installErrorMessage(error, t)); + }); + return () => { + cancelled = true; + }; + }, [ + installScope, + usage, + disabled, + onEventProcessorInstalled, + loadCatalog, + t, + ]); useEffect(() => { void loadCatalog(); @@ -286,22 +334,28 @@ export default function RunnerSelect({ const handleInstall = useCallback( async (plugin: PluginV4) => { const pluginId = marketplacePluginId(plugin); - if (pendingInstall || installingPluginId) return; + if (disabled || pendingInstall || installingPluginId) return; setInstallingPluginId(pluginId); setInstallError(null); try { - const installed = await installMarketplaceRunner(plugin, { + const installOptions = { scope: installScope, - onTaskCreated: (taskId) => + onTaskCreated: (taskId: number) => addTask({ taskId, pluginName: marketplacePluginId(plugin), source: 'marketplace', extensionType: 'plugin', }), - }); - onInstalled(installed); + }; + if (usage === 'event') { + onEventProcessorInstalled?.( + await installMarketplaceEventProcessor(plugin, installOptions), + ); + } else { + onInstalled?.(await installMarketplaceRunner(plugin, installOptions)); + } await loadCatalog(); toast.success( t('agents.runnerInstallSuccess', { @@ -326,12 +380,16 @@ export default function RunnerSelect({ onInstalled, pendingInstall, t, + usage, + disabled, + onEventProcessorInstalled, ], ); return (