From 923ebe12af75bef27c5f010678569fd3a2a0c2d2 Mon Sep 17 00:00:00 2001 From: Hyu Date: Thu, 1 Oct 2026 18:10:42 +0800 Subject: [PATCH 01/30] feat(telemetry): attach called platform APIs to their step node on execution traces Every host step now opens a stage scope, so observations recorded inside it (including cross-task plugin/RPC calls resolved through the execution registry) carry the owning step as parent; the pipeline lane records its own pipeline/run node so acks and replies issued outside the runner still hang off a node. --- src/langbot/pkg/agent/runner/orchestrator.py | 245 ++++---- .../pkg/agent/runner/platform_tools.py | 102 ++-- src/langbot/pkg/pipeline/pipelinemgr.py | 96 ++- .../pkg/pipeline/process/handlers/chat.py | 96 ++- src/langbot/pkg/platform/botmgr.py | 157 +++-- src/langbot/pkg/telemetry/execution.py | 452 +++++++++----- src/langbot/pkg/telemetry/platform.py | 26 +- src/langbot/pkg/telemetry/trace.py | 166 +++++- src/langbot/templates/config.yaml | 4 + .../unit_tests/telemetry/test_chat_emitter.py | 110 ++++ .../telemetry/test_pipeline_lane.py | 272 +++++++++ tests/unit_tests/telemetry/test_trace.py | 561 ++++++++++++++++-- 12 files changed, 1798 insertions(+), 489 deletions(-) create mode 100644 tests/unit_tests/telemetry/test_chat_emitter.py create mode 100644 tests/unit_tests/telemetry/test_pipeline_lane.py diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index e207e743a..c7c9443eb 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -38,6 +38,7 @@ from .session_registry import AgentRunSessionRegistry, get_session_registry from .state_scope import build_state_context from ...provider.tools.loaders import skill as skill_loader from ...telemetry import trace as trace_mod +from ...telemetry.execution import bind_trace as bind_execution_trace from ...telemetry.execution import close_trace as close_execution_trace from ...telemetry.execution import record as record_execution @@ -213,13 +214,16 @@ class AgentRunOrchestrator: terminal_reason: str | None = None terminal_usage: dict[str, typing.Any] | None = None execution_outcome = 'unknown' + execution_error = '' # A run reached without a platform ingress (WebUI debug, service API) # owns its own chain; a run inside an ingress reuses that chain. trace_binding: trace_mod.TraceBinding | None = None run_token: typing.Any = None + # Workflow step node for this run; '' until the step scope is opened. + node = '' try: - trace_binding = trace_mod.bind() + trace_binding = bind_execution_trace(self.ap, run_id) run_token = trace_mod.set_run(run_id) await self.journal.create_run( event=event, @@ -261,139 +265,140 @@ class AgentRunOrchestrator: event_log_id=event_log_id, ) - async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results: - async for result_dict in results: - result_dict = dict(result_dict) - sequence = result_dict.get('sequence') - if sequence is not None: - try: - sequence_int = int(sequence) - except (TypeError, ValueError): - self.ap.logger.warning( - f'Runner {descriptor.id} returned invalid result sequence: {sequence}' - ) - sequence_int = last_sequence + 1 - result_dict['sequence'] = sequence_int - else: - if sequence_int in seen_sequences: + with trace_mod.stage_scope() as node: + async with contextlib.aclosing(self.invoker.invoke(descriptor, context)) as results: + async for result_dict in results: + result_dict = dict(result_dict) + sequence = result_dict.get('sequence') + if sequence is not None: + try: + sequence_int = int(sequence) + except (TypeError, ValueError): self.ap.logger.warning( - f'Runner {descriptor.id} returned duplicate result sequence ' - f'{sequence_int} for run {run_id}; dropping duplicate' - ) - continue - if sequence_int <= 0: - self.ap.logger.warning( - f'Runner {descriptor.id} returned non-positive result sequence ' - f'{sequence_int} for run {run_id}' + f'Runner {descriptor.id} returned invalid result sequence: {sequence}' ) sequence_int = last_sequence + 1 result_dict['sequence'] = sequence_int - elif last_sequence and sequence_int != last_sequence + 1: - self.ap.logger.warning( - f'Runner {descriptor.id} result sequence gap or out-of-order ' - f'for run {run_id}: previous={last_sequence}, current={sequence_int}' - ) + else: + if sequence_int in seen_sequences: + self.ap.logger.warning( + f'Runner {descriptor.id} returned duplicate result sequence ' + f'{sequence_int} for run {run_id}; dropping duplicate' + ) + continue + if sequence_int <= 0: + self.ap.logger.warning( + f'Runner {descriptor.id} returned non-positive result sequence ' + f'{sequence_int} for run {run_id}' + ) + sequence_int = last_sequence + 1 + result_dict['sequence'] = sequence_int + elif last_sequence and sequence_int != last_sequence + 1: + self.ap.logger.warning( + f'Runner {descriptor.id} result sequence gap or out-of-order ' + f'for run {run_id}: previous={last_sequence}, current={sequence_int}' + ) + seen_sequences.add(sequence_int) + last_sequence = max(last_sequence, sequence_int) + else: + sequence_int = last_sequence + 1 + result_dict['sequence'] = sequence_int seen_sequences.add(sequence_int) - last_sequence = max(last_sequence, sequence_int) - else: - sequence_int = last_sequence + 1 - result_dict['sequence'] = sequence_int - seen_sequences.add(sequence_int) - last_sequence = sequence_int + last_sequence = sequence_int - result_type = result_dict.get('type') - if result_type and not self.result_normalizer.validate_payload( - result_type, - result_dict.get('data', {}), - descriptor, - ): - continue - - await self.journal.append_run_result( - result_dict=result_dict, - run_id=run_id, - sequence=sequence_int, - ) - - # Trusted Host observers receive validated events before message-only normalization. - result_observer = (adapter_context or {}).get('_result_observer') - if result_observer is not None: - await result_observer(result_dict) - - if result_type == 'state.updated': - await self.journal.handle_state_updated_event( - result_dict, - event, - binding, + result_type = result_dict.get('type') + if result_type and not self.result_normalizer.validate_payload( + result_type, + result_dict.get('data', {}), descriptor, - run_id=run_id, - ) - await self.result_normalizer.normalize(result_dict, descriptor) - continue - - if result_type == 'action.requested': - if await self.interaction_manager.handle_result( - result_dict=result_dict, - event=event, - binding=binding, - descriptor=descriptor, - run_id=run_id, - adapter_context=adapter_context, ): continue - if result_type == 'run.completed': - terminal_status = 'completed' - terminal_reason = ( - result_dict.get('data', {}).get('finish_reason') - if isinstance(result_dict.get('data'), dict) - else None - ) - usage = result_dict.get('usage') - if isinstance(usage, dict): - terminal_usage = usage - elif result_type == 'run.failed': - terminal_status = 'failed' - data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {} - terminal_reason = data.get('error') or data.get('code') - usage = result_dict.get('usage') - if isinstance(usage, dict): - terminal_usage = usage - - has_completed_message = result_type == 'message.completed' or ( - result_type == 'run.completed' - and isinstance(result_dict.get('data'), dict) - and bool(result_dict['data'].get('message')) - ) - if has_completed_message and event.conversation_id and not assistant_transcript_written: - await self.journal.write_assistant_transcript( + await self.journal.append_run_result( result_dict=result_dict, - event=event, run_id=run_id, - runner_id=descriptor.id, + sequence=sequence_int, ) - assistant_transcript_written = True - result = await self.result_normalizer.normalize(result_dict, descriptor) - file_ids = result_dict.get('data', {}).get('file_ids', []) - if file_ids: - if result_type != 'message.completed' or result is None: - raise ValueError('Files must be attached to a completed message') - if binding.processor_type != 'pipeline': - raise ValueError('Agent files must be sent explicitly through the reply API') - from ...box.runner import exported_message, binding_for + # Trusted Host observers receive validated events before message-only normalization. + result_observer = (adapter_context or {}).get('_result_observer') + if result_observer is not None: + await result_observer(result_dict) - async with binding_for(execution_query).lock: - result.attachments = exported_message(execution_query, file_ids, consume=True) + if result_type == 'state.updated': + await self.journal.handle_state_updated_event( + result_dict, + event, + binding, + descriptor, + run_id=run_id, + ) + await self.result_normalizer.normalize(result_dict, descriptor) + continue - if result is not None: - yield result + if result_type == 'action.requested': + if await self.interaction_manager.handle_result( + result_dict=result_dict, + event=event, + binding=binding, + descriptor=descriptor, + run_id=run_id, + adapter_context=adapter_context, + ): + continue - run_snapshot = await self.journal.get_run(run_id) - if run_snapshot and run_snapshot.get('cancel_requested_at') is not None: - terminal_status = 'cancelled' - terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested' - break + if result_type == 'run.completed': + terminal_status = 'completed' + terminal_reason = ( + result_dict.get('data', {}).get('finish_reason') + if isinstance(result_dict.get('data'), dict) + else None + ) + usage = result_dict.get('usage') + if isinstance(usage, dict): + terminal_usage = usage + elif result_type == 'run.failed': + terminal_status = 'failed' + data = result_dict.get('data') if isinstance(result_dict.get('data'), dict) else {} + terminal_reason = data.get('error') or data.get('code') + usage = result_dict.get('usage') + if isinstance(usage, dict): + terminal_usage = usage + + has_completed_message = result_type == 'message.completed' or ( + result_type == 'run.completed' + and isinstance(result_dict.get('data'), dict) + and bool(result_dict['data'].get('message')) + ) + if has_completed_message and event.conversation_id and not assistant_transcript_written: + await self.journal.write_assistant_transcript( + result_dict=result_dict, + event=event, + run_id=run_id, + runner_id=descriptor.id, + ) + assistant_transcript_written = True + + result = await self.result_normalizer.normalize(result_dict, descriptor) + file_ids = result_dict.get('data', {}).get('file_ids', []) + if file_ids: + if result_type != 'message.completed' or result is None: + raise ValueError('Files must be attached to a completed message') + if binding.processor_type != 'pipeline': + raise ValueError('Agent files must be sent explicitly through the reply API') + from ...box.runner import exported_message, binding_for + + async with binding_for(execution_query).lock: + result.attachments = exported_message(execution_query, file_ids, consume=True) + + if result is not None: + yield result + + run_snapshot = await self.journal.get_run(run_id) + if run_snapshot and run_snapshot.get('cancel_requested_at') is not None: + terminal_status = 'cancelled' + terminal_reason = run_snapshot.get('status_reason') or 'cancel_requested' + break await self.journal.finalize_run( run_id=run_id, status=terminal_status or 'completed', @@ -403,11 +408,15 @@ class AgentRunOrchestrator: execution_outcome = {'completed': 'success', 'failed': 'failed', 'cancelled': 'cancelled'}.get( terminal_status or '', 'unknown' ) + execution_error = '' if execution_outcome == 'success' else (terminal_reason or execution_outcome) except asyncio.CancelledError: execution_outcome = 'cancelled' + execution_error = 'cancelled' raise except Exception as exc: execution_outcome = 'timeout' if self._is_deadline_exhausted(context) else 'failed' + # A chain that broke must always carry a reason, even for a bare exception. + execution_error = str(exc) or type(exc).__name__ failed_usage = terminal_usage await self.journal.finalize_run( run_id=run_id, @@ -426,6 +435,8 @@ class AgentRunOrchestrator: runner=descriptor.id, outcome=execution_outcome, synthetic=event.source == 'webui', + error=execution_error, + node=node, ) trace_mod.reset_run(run_token) if trace_binding is not None and trace_mod.unbind_root(trace_binding): diff --git a/src/langbot/pkg/agent/runner/platform_tools.py b/src/langbot/pkg/agent/runner/platform_tools.py index 5599c4953..70795f304 100644 --- a/src/langbot/pkg/agent/runner/platform_tools.py +++ b/src/langbot/pkg/agent/runner/platform_tools.py @@ -684,49 +684,71 @@ async def execute_platform_tool( normalized = _normalize_platform_params(definition, parameters) if definition.scope == 'event': normalized = _event_params(definition, context, normalized) - # This flag is frozen by the Host from the synthetic debug envelope, not tool arguments. - if delivery.get('surface') == 'webui' and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True: - result = _execute_mock_platform_tool(definition, context, normalized) - if message_chain is not None: - result['parameters']['message'] = message_chain.model_dump(mode='json') - from ...telemetry.execution import record + # Plugin/RPC actions run outside the ingress context, so bind the owning + # execution id here: nested adapter observations then join its trace. + from ...telemetry.execution import record, reset_execution_id, set_execution_id - record( - ap, - execution_context, - family='platform_api', - operation=definition.api, - mode=authorization.get('processor_type', 'none'), - synthetic=True, - outcome='success', - ) - return result - bot_id = authorization.get('bot_id') - if not bot_id: - raise ValueError('This run is not associated with a platform bot') - bot = await ap.platform_mgr.get_bot_by_uuid(execution_context, bot_id) - if bot is None: - raise ValueError(f'Bot {bot_id} is not running') - if definition.api not in set(bot.adapter.get_supported_apis() or []): - raise ValueError(f'Platform API {definition.api} is no longer supported by bot {bot_id}') - api_func = getattr(bot.adapter, definition.api, None) - if not callable(api_func): - raise ValueError(f'Platform API {definition.api} is declared but not implemented') - if definition.api == 'send_message': - normalized = { - 'target_type': _require_string(normalized, 'target_type'), - 'target_id': _require_string(normalized, 'target_id'), - 'message': message_chain - if message_chain is not None - else platform_message.MessageChain([platform_message.Plain(text=_require_string(normalized, 'text'))]), - } - from ...telemetry.platform import processing_mode - - token = processing_mode.set(authorization.get('processor_type', 'none')) + execution_id = str(session.get('run_id') or '').strip() + execution_token = set_execution_id(execution_id) try: - return await api_func(**normalized) + # This flag is frozen by the Host from the synthetic debug envelope, not tool arguments. + mock = ( + delivery.get('surface') == 'webui' + and (delivery.get('platform_capabilities') or {}).get('debug_mock') is True + ) + if mock: + outcome = 'unknown' + error_detail = '' + try: + result = _execute_mock_platform_tool(definition, context, normalized) + if message_chain is not None: + result['parameters']['message'] = message_chain.model_dump(mode='json') + outcome = 'success' + except Exception as exc: + outcome = 'failed' + error_detail = str(exc) + raise + finally: + record( + ap, + execution_context, + family='platform_api', + operation=definition.api, + mode=authorization.get('processor_type', 'none'), + synthetic=True, + outcome=outcome, + error=error_detail, + execution_id=execution_id or None, + ) + return result + bot_id = authorization.get('bot_id') + if not bot_id: + raise ValueError('This run is not associated with a platform bot') + bot = await ap.platform_mgr.get_bot_by_uuid(execution_context, bot_id) + if bot is None: + raise ValueError(f'Bot {bot_id} is not running') + if definition.api not in set(bot.adapter.get_supported_apis() or []): + raise ValueError(f'Platform API {definition.api} is no longer supported by bot {bot_id}') + api_func = getattr(bot.adapter, definition.api, None) + if not callable(api_func): + raise ValueError(f'Platform API {definition.api} is declared but not implemented') + if definition.api == 'send_message': + normalized = { + 'target_type': _require_string(normalized, 'target_type'), + 'target_id': _require_string(normalized, 'target_id'), + 'message': message_chain + if message_chain is not None + else platform_message.MessageChain([platform_message.Plain(text=_require_string(normalized, 'text'))]), + } + from ...telemetry.platform import processing_mode + + token = processing_mode.set(authorization.get('processor_type', 'none')) + try: + return await api_func(**normalized) + finally: + processing_mode.reset(token) finally: - processing_mode.reset(token) + reset_execution_id(execution_token) def _execute_mock_platform_tool( diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index a737594d4..c98b94c8a 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -3,6 +3,7 @@ from __future__ import annotations import dataclasses import typing import traceback +import asyncio import sqlalchemy @@ -257,6 +258,9 @@ class RuntimePipeline: self.ap.logger.error(result.error_notice) # Mark query as having error query.variables['_monitoring_has_error'] = True + # The lane reports failures as a value instead of raising, so record the + # reason here: without it the uploaded trace would not explain the break. + self._record_lane_failure(query, str(result.error_notice)) # Record error to monitoring system try: await self._assert_execution_active(query) @@ -368,14 +372,59 @@ class RuntimePipeline: async def process_query(self, query: pipeline_query.Query): from ..telemetry.execution import ingress + from ..telemetry.execution import record as record_execution from ..telemetry.platform import processing_mode + from ..telemetry.trace import stage_scope token = processing_mode.set('pipeline') + # The Workspace-scoped opaque query uuid is the execution identity Space + # shows for this lane; it is a stable UUID for every pooled query. + execution_id = str(getattr(query, 'query_uuid', '') or '').strip() or str(query.query_id) try: # Callers without a platform event (Webchat, HTTP API) still get one # trace for the whole Pipeline lane; nested calls reuse the trace. - with ingress(self.ap, 'pipeline_done'): - return await self._process_query(query) + with ingress(self.ap, 'pipeline_done', getattr(self, 'execution_context', None), execution_id=execution_id): + # The lane is its own workflow step: acknowledgements and replies it + # sends directly, and the runner it drives, hang under this node. + # When the lane runs under a platform route the node nests there. + with stage_scope() as node: + lane_outcome = 'success' + lane_error = '' + try: + return await self._process_query(query) + except asyncio.CancelledError: + lane_outcome = 'cancelled' + lane_error = 'cancelled' + raise + except BaseException as exc: + lane_outcome = 'failed' + lane_error = str(exc) or type(exc).__name__ + raise + finally: + try: + variables = getattr(query, 'variables', None) or {} + lane_has_error = bool(variables.get('_monitoring_has_error')) + except Exception: + lane_has_error = False + if lane_outcome == 'success' and lane_has_error: + # The lane reported the failure as a value, not an exception. + lane_outcome = 'failed' + config = getattr(query, 'pipeline_config', None) + try: + runner_id = (RunnerConfigResolver.resolve_runner_id(config) or '') if config else '' + except Exception: + runner_id = '' + record_execution( + self.ap, + getattr(self, 'execution_context', None), + family='pipeline', + operation='run', + mode='pipeline', + runner=runner_id, + outcome=lane_outcome, + error=lane_error, + node=node, + ) finally: processing_mode.reset(token) @@ -490,6 +539,9 @@ class RuntimePipeline: inst_name = query.current_stage_name if query.current_stage_name else 'unknown' self.ap.logger.error(f'Error processing query {query.query_id} stage={inst_name} : {e}') self.ap.logger.error(f'Traceback: {traceback.format_exc()}') + # The lane itself broke: land the reason on the execution trace so the + # chain is uploaded even though the error never became a StageProcessResult. + self._record_lane_failure(query, str(e) or type(e).__name__) # Record query error try: @@ -513,6 +565,46 @@ class RuntimePipeline: self.ap.logger.debug(f'Query {query.query_id} processed') await self.ap.query_pool.remove_query(query) + def _record_lane_failure(self, query: pipeline_query.Query, reason: str) -> None: + """Land one failed ``runner/execute`` stage on the owning execution trace.""" + try: + from ..telemetry.execution import record as record_execution + from ..telemetry.trace import current as current_trace + from ..telemetry.trace import stage_scope + + detail = reason or 'pipeline_lane_failed' + state = current_trace() + # The runner orchestrator already lands its own failed ``runner/execute`` + # node when the break happens inside a runner; in that case only carry the + # reason over instead of repeating the node on the trace. + already_recorded = bool( + state is not None + and any( + stage.get('family') == 'runner' and stage.get('outcome') not in ('success', 'skipped') + for stage in state.stages + ) + ) + if not already_recorded: + # The lane failure is its own workflow step, hanging under whatever + # step routed into it (or a root when nothing did). + with stage_scope() as node: + record_execution( + self.ap, + getattr(self, 'execution_context', None), + family='runner', + operation='execute', + mode='pipeline', + runner=(RunnerConfigResolver.resolve_runner_id(query.pipeline_config) or ''), + outcome='failed', + error=detail, + node=node, + ) + state = current_trace() + if state is not None: + state.mark_failure(detail) + except Exception: + pass + class PipelineManager: """流水线管理器""" diff --git a/src/langbot/pkg/pipeline/process/handlers/chat.py b/src/langbot/pkg/pipeline/process/handlers/chat.py index 2fc7b0f82..2cd681fa1 100644 --- a/src/langbot/pkg/pipeline/process/handlers/chat.py +++ b/src/langbot/pkg/pipeline/process/handlers/chat.py @@ -3,8 +3,6 @@ from __future__ import annotations import uuid import typing import traceback -import time -from datetime import datetime from .. import handler @@ -14,9 +12,7 @@ from ... import plugin_diagnostics import langbot_plugin.api.entities.events as events from ....agent.runner.config_resolver import RunnerConfigResolver from ....agent.runner import config_schema -from ....utils import constants, runner as runner_utils -from ....telemetry import features as telemetry_features -from ....telemetry.identity import workspace_identity +from ....utils import runner as runner_utils import langbot_plugin.api.entities.builtin.provider.session as provider_session import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query import langbot_plugin.api.entities.builtin.provider.message as provider_message @@ -115,9 +111,6 @@ class ChatMessageHandler(handler.MessageHandler): text_length = 0 runner = None try: - # Mark start time for telemetry - start_ts = time.time() - try_claim_steering = getattr( self.ap.agent_run_orchestrator, 'try_claim_steering_from_query', @@ -258,69 +251,48 @@ class ChatMessageHandler(handler.MessageHandler): debug_notice=traceback.format_exc(), ) finally: - # Telemetry reporting + # Telemetry: the per-execution record is built when the owning + # trace closes, so only attach the execution-scoped fields this + # lane knows here. try: - end_ts = time.time() - duration_ms = None - if 'start_ts' in locals(): - duration_ms = int((end_ts - start_ts) * 1000) + from ....telemetry.trace import current as current_trace + state = current_trace() + if state is not None: + adapter_name = query.adapter.__class__.__name__ if hasattr(query, 'adapter') else '' + runner_name = self.ap.agent_run_orchestrator.resolve_runner_id_for_telemetry(query) + state.adapter = state.adapter or adapter_name + state.runner = state.runner or (runner_name or '') + state.runner_category = state.runner_category or runner_utils.get_runner_category_from_runner( + runner_name, None, query.pipeline_config + ) + model_name = '' + try: + if getattr(query, 'use_llm_model_uuid', None): + m = await self.ap.model_mgr.get_model_by_uuid( + get_query_execution_context(query), + query.use_llm_model_uuid, + ) + if m and getattr(m, 'model_entity', None): + model_name = getattr(m.model_entity, 'name', '') or '' + except Exception: + model_name = '' + state.model_name = state.model_name or model_name + if state.pipeline_plugins is None: + state.pipeline_plugins = query.variables.get('_pipeline_bound_plugins', None) + except Exception as ex: + self.ap.logger.warning(f'Failed to attach execution telemetry fields: {ex}') + + # Trigger survey events on successful non-WebSocket responses + try: adapter_name = query.adapter.__class__.__name__ if hasattr(query, 'adapter') else None - - # Use orchestrator to resolve runner ID for telemetry - runner_name = self.ap.agent_run_orchestrator.resolve_runner_id_for_telemetry(query) - - # Model name if available - model_name = None - try: - if getattr(query, 'use_llm_model_uuid', None): - m = await self.ap.model_mgr.get_model_by_uuid( - get_query_execution_context(query), - query.use_llm_model_uuid, - ) - if m and getattr(m, 'model_entity', None): - model_name = getattr(m.model_entity, 'name', None) - except Exception: - model_name = None - - pipeline_plugins = query.variables.get('_pipeline_bound_plugins', None) - - runner_category = runner_utils.get_runner_category_from_runner( - runner_name, None, query.pipeline_config - ) - - # Feature usage collected during query processing (tool calls, - # knowledge base usage, sandbox executions, activated skills, ...) - features = telemetry_features.collect_features(query) - - payload = { - 'event_type': 'query', - 'query_id': query.query_id, - 'adapter': adapter_name, - 'runner': runner_name, - 'runner_category': runner_category, - 'duration_ms': duration_ms, - 'model_name': model_name, - 'version': constants.semantic_version, - **workspace_identity(get_query_execution_context(query)), - 'runtime_instance_id': constants.instance_id, - 'edition': constants.edition, - 'pipeline_plugins': pipeline_plugins, - 'features': features, - 'error': locals().get('error_info', None), - 'timestamp': datetime.utcnow().isoformat(), - } - - await self.ap.telemetry.start_send_task(payload) - - # Trigger survey events on successful non-WebSocket responses if not locals().get('error_info') and adapter_name and 'WebSocket' not in adapter_name: if self.ap.survey: await self.ap.survey.trigger_event('first_bot_response_success') # Counts toward the bot_response_success_100 milestone event await self.ap.survey.record_bot_response_success() except Exception as ex: - self.ap.logger.warning(f'Failed to send telemetry: {ex}') + self.ap.logger.warning(f'Failed to trigger survey event: {ex}') async def _ensure_conversation_for_history( self, diff --git a/src/langbot/pkg/platform/botmgr.py b/src/langbot/pkg/platform/botmgr.py index d4430f55d..7b942c54d 100644 --- a/src/langbot/pkg/platform/botmgr.py +++ b/src/langbot/pkg/platform/botmgr.py @@ -401,15 +401,19 @@ class RuntimeBot: from ..telemetry.execution import record with trace_mod.scope(route_ref=self._route_ref(binding, target_type, target_uuid)): - record( - getattr(self, 'ap', None), - getattr(self, 'execution_context', None), - family='event_route', - operation=event_type, - adapter=type(getattr(self, 'adapter', None)).__name__, - mode=target_type if target_type in {'pipeline', 'agent', 'event_processor'} else 'none', - outcome={'delivered': 'success', 'failed': 'failed'}.get(status, 'skipped'), - ) + # The routing decision is its own workflow step; the Pipeline or + # processor it delivers to hangs under this node. + with trace_mod.stage_scope() as node: + record( + getattr(self, 'ap', None), + getattr(self, 'execution_context', None), + family='event_route', + operation=event_type, + adapter=type(getattr(self, 'adapter', None)).__name__, + mode=target_type if target_type in {'pipeline', 'agent', 'event_processor'} else 'none', + outcome={'delivered': 'success', 'failed': 'failed'}.get(status, 'skipped'), + node=node, + ) return metadata def get_pipeline_target_for_event_type(self, event_type: str = 'message.received') -> str | None: @@ -736,12 +740,19 @@ class RuntimeBot: self, event: platform_events.EBAEvent, adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, + execution_id: str | None = None, ) -> AgentEventEnvelope: event_type = getattr(event, 'type', None) or event.__class__.__name__ event_time = getattr(event, 'timestamp', None) or time.time() - event_id = ( - getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}' - ) + if execution_id: + # Reuse the ingress execution identity so the run view, the journaled + # event and the uploaded chain all share one id. + envelope_event_id = str(execution_id) + prefix = f'platform:{self.bot_entity.uuid}:' + event_id = envelope_event_id[len(prefix) :] if envelope_event_id.startswith(prefix) else envelope_event_id + else: + event_id = self._platform_event_raw_id(event) + envelope_event_id = f'platform:{self.bot_entity.uuid}:{event_id}' target_type, target_id, target_metadata = self._infer_reply_target(event) supported_apis = self._get_adapter_supported_apis(adapter) @@ -773,7 +784,7 @@ class RuntimeBot: delivery_data['interactions'] = interaction_capabilities return AgentEventEnvelope( - event_id=f'platform:{self.bot_entity.uuid}:{event_id}', + event_id=envelope_event_id, event_type=event_type, event_time=int(event_time) if isinstance(event_time, (int, float)) else None, source='platform', @@ -863,6 +874,21 @@ class RuntimeBot: processor_id=agent.get('uuid'), ) + def _platform_event_execution_id(self, event: platform_events.EBAEvent) -> str: + """Space-visible execution identity for one inbound platform event. + + Matches the ``AgentEventEnvelope.event_id`` the same event produces, so + the stored chain and the Space run view share one identity. + """ + return f'platform:{self.bot_entity.uuid}:{self._platform_event_raw_id(event)}' + + @staticmethod + def _platform_event_raw_id(event: platform_events.EBAEvent) -> str: + event_type = getattr(event, 'type', None) or event.__class__.__name__ + return str( + getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}' + ) + async def _handle_platform_event( self, event: platform_events.EBAEvent, @@ -870,51 +896,85 @@ class RuntimeBot: ) -> None: # One inbound event owns one execution trace; every stage recorded while # it is handled (routing, runner, platform API calls) joins that trace. + from ..telemetry import trace as trace_mod from ..telemetry.execution import ingress event.bot_uuid = self.bot_entity.uuid - with ingress(getattr(self, 'ap', None), 'event_done'): - await self._handle_platform_event_body(event, adapter) + execution_id = self._platform_event_execution_id(event) + with ingress(getattr(self, 'ap', None), 'event_done', getattr(self, 'execution_context', None), execution_id): + # The inbound event is the root workflow step: everything recorded + # while handling it hangs under this node. + with trace_mod.stage_scope() as node: + await self._handle_platform_event_body(event, adapter, execution_id, node) - async def _handle_platform_event_body( - self, - event: platform_events.EBAEvent, - adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, - ) -> None: + @staticmethod + def _summarize_event_delivery(results: list[typing.Any]) -> tuple[str, str]: + outcome = 'success' + error_detail = '' + for result in results: + if isinstance(result, BaseException): + outcome = 'failed' + error_detail = error_detail or f'{type(result).__name__}: {result}' + return outcome, error_detail + + def _record_platform_event(self, event, adapter, outcome: str, error_detail: str, node: str = '') -> None: from ..telemetry.execution import record record( getattr(self, 'ap', None), getattr(self, 'execution_context', None), family='platform_event', - operation=event.type, + operation=str(getattr(event, 'type', None) or event.__class__.__name__), adapter=adapter.__class__.__name__, - outcome='success', + outcome=outcome, + error=error_detail, + node=node, ) - await self._record_adapter_event(event, adapter) - primary = ( - self._handle_interaction_submission(event, adapter) - if isinstance(event, platform_events.PlatformSpecificEvent) and event.action == 'interaction.submitted' - else self._dispatch_eba_event_to_processor(event, adapter) - ) - subscriptions = self._get_event_bindings_from_value(getattr(self.bot_entity, 'plugin_processors', [])) - seen = set() - tasks = [primary] - for subscription in subscriptions: - processor_uuid = subscription.get('processor_uuid') - if not processor_uuid or processor_uuid in seen or not subscription.get('enabled', True): - continue - seen.add(processor_uuid) - tasks.append(self._dispatch_plugin_subscription(event, adapter, processor_uuid)) - # Start all deliveries together. A slow or failed subscriber cannot block - # another subscriber or the primary route from receiving the event. - results = await asyncio.gather(*tasks, return_exceptions=True) + async def _handle_platform_event_body( + self, + event: platform_events.EBAEvent, + adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, + execution_id: str | None = None, + node: str = '', + ) -> None: + results: list[typing.Any] = [] + try: + await self._record_adapter_event(event, adapter) + + primary = ( + self._handle_interaction_submission(event, adapter, execution_id) + if isinstance(event, platform_events.PlatformSpecificEvent) and event.action == 'interaction.submitted' + else self._dispatch_eba_event_to_processor(event, adapter, execution_id=execution_id) + ) + subscriptions = self._get_event_bindings_from_value(getattr(self.bot_entity, 'plugin_processors', [])) + seen = set() + tasks = [primary] + for subscription in subscriptions: + processor_uuid = subscription.get('processor_uuid') + if not processor_uuid or processor_uuid in seen or not subscription.get('enabled', True): + continue + seen.add(processor_uuid) + tasks.append(self._dispatch_plugin_subscription(event, adapter, processor_uuid, execution_id)) + # Start all deliveries together. A slow or failed subscriber cannot block + # another subscriber or the primary route from receiving the event. + results = await asyncio.gather(*tasks, return_exceptions=True) + except BaseException as exc: + self._record_platform_event(event, adapter, 'failed', f'{type(exc).__name__}: {exc}', node) + raise + + # Recorded after handling so the inbound stage carries the real outcome + # instead of a hardcoded success. + outcome, error_detail = self._summarize_event_delivery(results) for result in results: if isinstance(result, BaseException): - await self.logger.error(f'Event delivery failed: {result}') + try: + await self.logger.error(f'Event delivery failed: {result}') + except Exception: + pass + self._record_platform_event(event, adapter, outcome, error_detail, node) - async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid): + async def _dispatch_plugin_subscription(self, event, adapter, processor_uuid, execution_id=None): event_type = event.type event_binding = { 'id': f'plugin_processor:{processor_uuid}', @@ -934,7 +994,9 @@ class RuntimeBot: if not patterns or not self._agent_supports_event_type(patterns, event_type): return agent = {**agent, 'supported_event_patterns': patterns} - return await self._dispatch_eba_event_to_processor(event, adapter, event_binding, agent) + return await self._dispatch_eba_event_to_processor( + event, adapter, event_binding, agent, execution_id=execution_id + ) except Exception as exc: return await self._record_event_route_trace( event_type=event_type, @@ -954,6 +1016,7 @@ class RuntimeBot: adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, event_binding: dict | None = None, agent: dict | None = None, + execution_id: str | None = None, ) -> dict[str, typing.Any]: event_type = getattr(event, 'type', None) or event.__class__.__name__ @@ -1101,7 +1164,7 @@ class RuntimeBot: text=f'Event {event_type} target agent has no runner: {target_uuid}', ) - envelope = self._eba_event_to_agent_envelope(event, adapter) + envelope = self._eba_event_to_agent_envelope(event, adapter, execution_id) if target_type == 'event_processor': envelope.data = event.model_dump(mode='json', exclude={'source_platform_object', 'legacy_event'}) from ..telemetry import trace as trace_mod @@ -1318,6 +1381,7 @@ class RuntimeBot: self, event: platform_events.PlatformSpecificEvent, adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, + execution_id: str | None = None, ) -> None: """Consume a platform interaction callback and resume its original processor.""" data = event.data if isinstance(event.data, dict) else {} @@ -1346,7 +1410,7 @@ class RuntimeBot: await self.ap.agent_run_orchestrator.interaction_manager.acknowledge_submission(record, adapter) if record['processor_type'] == 'agent': - await self._resume_agent_interaction(record, event, adapter, actor_id) + await self._resume_agent_interaction(record, event, adapter, actor_id, execution_id) return if record['processor_type'] != 'pipeline': raise ValueError(f'Unsupported interaction processor type: {record["processor_type"]}') @@ -1428,6 +1492,7 @@ class RuntimeBot: event: platform_events.PlatformSpecificEvent, adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter, actor_id: str | None, + execution_id: str | None = None, ) -> None: """Resume the exact independent Agent that produced an interaction.""" agent = await self.ap.agent_service.get_agent( @@ -1480,7 +1545,7 @@ class RuntimeBot: delivery_data['interactions'] = interaction_capabilities envelope = AgentEventEnvelope( - event_id=f'interaction:{record["id"]}:{uuid.uuid4()}', + event_id=execution_id or f'interaction:{record["id"]}:{uuid.uuid4()}', event_type='interaction.submitted', event_time=int(event.timestamp) if event.timestamp else int(time.time()), source='platform', diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index ff5ef28ff..f4af0defa 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -1,34 +1,33 @@ -"""Bounded, content-free execution counters for the existing telemetry sender. +"""Content-free execution records for the existing telemetry sender. This module reports observations only. Coverage catalogs and acceptance rules -belong to Space. Aggregation keys include both immutable execution identities. +belong to Space. -Two shapes share one payload type: +One complete execution is one record: the execution identity (Pipeline query, +inbound platform event or Agent run), its bounded ordered stages and its +terminal outcome. Nothing is aggregated or counted across executions — every +record carries the id of the execution it belongs to, so Space can fetch the +whole chain by primary identity. Records are buffered whole and flushed as +``{"records": [...]}`` on the configured cadence and on shutdown; an in-flight +trace is never flushed half-written. -* window counters, aggregated by ``(family, operation, mode, adapter, runner, - outcome, synthetic)`` — unchanged coverage reporting; -* per-event traces, one bounded payload per traced event, keyed by - ``query_id = trace_id`` so Space can fetch a whole chain by primary identity. - -Traces are additive: they never alter the counters, and both are bounded in -memory, batch size and send concurrency. +Memory is bounded in stages per trace, buffered traces, buffered records, the +cross-task execution registry and flush batch size. """ from __future__ import annotations import asyncio import contextlib +import contextvars +import json import time from datetime import datetime, timezone -from uuid import uuid4 from . import trace as trace_mod from .identity import workspace_identity from .trace import TraceState -MAX_KEYS = 512 -MAX_BATCH = 32 -FLUSH_SECONDS = 60 MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'}) OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'}) @@ -39,15 +38,70 @@ TRACE_MODES = frozenset({'off', 'failures', 'sampled', 'all'}) DEFAULT_TRACE_MODE = 'sampled' DEFAULT_TRACE_SAMPLE = 20 +# Record bounds: buffered records, one flush batch, flush cadence, registry. +MAX_BUFFERED_RECORDS = 512 +MAX_RECORDS_PER_FLUSH = 128 +MAX_FLUSH_BYTES = 200 * 1024 +DEFAULT_FLUSH_SECONDS = 180 +MAX_REGISTRY_KEYS = 256 + +# Set by cross-task call sites (plugin/RPC actions) that run without the +# ingress context but know which execution they belong to. +_execution_id: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_execution_id', default='') + + +def set_execution_id(execution_id: str) -> contextvars.Token: + return _execution_id.set(str(execution_id or '').strip()) + + +def reset_execution_id(token: contextvars.Token | None) -> None: + if token is None: + return + try: + _execution_id.reset(token) + except (ValueError, RuntimeError): + # A token from another context must never break execution. + pass + + +def current_execution_id() -> str: + return _execution_id.get() + + +def _bounded_error(value, limit: int = 400) -> str: + """Bound and de-fang one failure detail before it leaves the process. + + Error text is operator-facing: keep it single-line, printable and short so a + noisy failure cannot smuggle control characters or bloat the payload. + """ + if value is None: + return '' + text = ''.join(ch if ch.isprintable() else ' ' for ch in str(value)) + text = ' '.join(text.split()) + return text[:limit] + + +def _duration_ms(started_at: str, ended_at: str) -> int: + """Wall-clock duration of one execution; 0 when the interval is unusable.""" + try: + delta = datetime.fromisoformat(ended_at) - datetime.fromisoformat(started_at) + return max(int(delta.total_seconds() * 1000), 0) + except Exception: + return 0 + class ExecutionCounters: def __init__(self, manager): self.manager = manager - self.pending: dict[tuple, dict] = {} + self.records: list[dict] = [] self.task: asyncio.Task | None = None self.dropped = 0 self.traces: dict[str, TraceState] = {} self.trace_deadlines: dict[str, float] = {} + # Execution identity -> in-flight trace, for stages recorded by tasks + # that no longer share the ingress context (plugin/RPC platform calls). + self.registry: dict[str, TraceState] = {} + self.registry_deadlines: dict[str, float] = {} self.dropped_traces = 0 # ------------------------------------------------------------------ config @@ -63,6 +117,13 @@ class ExecutionCounters: sample = DEFAULT_TRACE_SAMPLE return sample if 1 <= sample <= 100000 else DEFAULT_TRACE_SAMPLE + def flush_seconds(self) -> int: + try: + seconds = int(self.manager.telemetry_config.get('telemetry_flush_seconds', DEFAULT_FLUSH_SECONDS)) + except (TypeError, ValueError): + seconds = DEFAULT_FLUSH_SECONDS + return seconds if 1 <= seconds <= 86400 else DEFAULT_FLUSH_SECONDS + # --------------------------------------------------------------- recording def record( @@ -76,67 +137,91 @@ class ExecutionCounters: runner: str = '', outcome: str = 'unknown', synthetic: bool = False, + error: str = '', + execution_id: str | None = None, + node: str = '', ): try: cfg = self.manager.telemetry_config if not cfg or cfg.get('disable_telemetry', False) or not cfg.get('url'): return - if family not in {'platform_event', 'event_route', 'runner', 'platform_api'}: + if family not in {'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}: return if mode not in MODES or outcome not in OUTCOMES: return # Only code-defined identifiers are accepted; never pass user values. - if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner)): + if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner, node)): return - identity = workspace_identity(context) state = trace_mod.current() - if state is not None: - self._record_trace_stage( - state, - identity=identity, - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=synthetic, - ) - key = ( - identity['instance_id'], - identity['workspace_uuid'], - family, - operation, - mode, - adapter, - runner, - outcome, - bool(synthetic), + if state is None: + # Cross-task work has no inherited trace; resolve the owning + # execution through the registry instead. + lookup = str(execution_id or '').strip() or current_execution_id() + if lookup: + state = self.registry.get(lookup) + if state is None: + # Every observation belongs to exactly one execution chain. + return + try: + identity = workspace_identity(context) + except Exception: + # A cross-task caller may only carry the Workspace id; reuse the + # identity already bound to the owning trace. + identity = state.identity or {} + if not identity.get('instance_id') or not identity.get('workspace_uuid'): + return + self._record_trace_stage( + state, + identity=identity, + family=family, + operation=operation, + mode=mode, + adapter=adapter, + runner=runner, + outcome=outcome, + synthetic=synthetic, + error=_bounded_error(error), + node=node, ) - row = self.pending.get(key) - if row is None: - if len(self.pending) >= MAX_KEYS: - self.dropped += 1 - return - row = {'count': 0, 'first_seen': datetime.now(timezone.utc).isoformat()} - self.pending[key] = row - row['count'] = min(row['count'] + 1, 2147483647) - row['last_seen'] = datetime.now(timezone.utc).isoformat() self._ensure_loop() except Exception: # Observability must never change execution behavior. return def _record_trace_stage( - self, state: TraceState, *, identity, family, operation, mode, adapter, runner, outcome, synthetic + self, + state: TraceState, + *, + identity, + family, + operation, + mode, + adapter, + runner, + outcome, + synthetic, + error: str = '', + node: str = '', ) -> None: if state.abandoned: return + if trace_mod.current() is state: + # Recorded by the task that owns the step: its step stack is exact. + # A step's own record sits under its enclosing step, never itself. + parent = trace_mod.current_parent() + if node and parent == node: + parent = trace_mod.enclosing_parent() + else: + # A cross-task observation (plugin/RPC) cannot see the step + # contextvar; attach it to the innermost step the owning task has + # open on the trace. + parent = state.open_nodes[-1] if state.open_nodes else '' if state.trace_id not in self.traces: # Never buffer traces this configuration would discard anyway. if self.trace_mode() == 'off' or len(self.traces) >= MAX_TRACES: self.dropped += 1 state.abandoned = True + self._unregister_state(state) return self.traces[state.trace_id] = state self.trace_deadlines[state.trace_id] = time.monotonic() + TRACE_TTL_SECONDS @@ -149,50 +234,79 @@ class ExecutionCounters: runner=runner, outcome=outcome, synthetic=bool(synthetic), + error=error, + node=node, + parent=parent, ) def _ensure_loop(self) -> None: if self.task is None or self.task.done(): self.task = asyncio.create_task(self._loop()) - # ------------------------------------------------------------------ traces + # --------------------------------------------------------------- registry - def close_trace(self, state: TraceState, reason: str = 'event_done') -> None: - """Emit one trace payload when it matches the configured sample.""" + def register_trace(self, state: TraceState, execution_id) -> None: + """Alias one more execution identity onto an in-flight trace.""" try: - if self.traces.pop(state.trace_id, None) is None: + exec_id = str(execution_id or '').strip() + if not exec_id or state.abandoned: return - self.trace_deadlines.pop(state.trace_id, None) - if not state.stages or not self._trace_emitted(state): - return - payload = self._build_trace_payload(state, reason) - if payload is not None: - self._dispatch_payload(payload) + if exec_id not in self.registry and len(self.registry) >= MAX_REGISTRY_KEYS: + oldest = next(iter(self.registry)) + self.registry.pop(oldest, None) + self.registry_deadlines.pop(oldest, None) + self.registry[exec_id] = state + self.registry_deadlines[exec_id] = time.monotonic() + TRACE_TTL_SECONDS except Exception: return - def _dispatch_payload(self, payload: dict) -> None: - """Hand one built payload to the telemetry manager from this sync context. + def _unregister_state(self, state: TraceState) -> None: + for exec_id in [key for key, value in self.registry.items() if value is state]: + self.registry.pop(exec_id, None) + self.registry_deadlines.pop(exec_id, None) - TelemetryManager.start_send_task is a coroutine, so calling it without - scheduling dropped every trace closed here. Stand-ins used by tests and - manual tools schedule synchronously, hence the coroutine check. - """ - result = self.manager.start_send_task(payload) - if not asyncio.iscoroutine(result): - return + def _sweep_registry(self, now: float) -> None: + """Drop aliases whose owning execution never closed (leaked bindings).""" + for exec_id in [key for key, deadline in self.registry_deadlines.items() if deadline <= now]: + self.registry.pop(exec_id, None) + self.registry_deadlines.pop(exec_id, None) + + # ------------------------------------------------------------------ traces + + def close_trace(self, state: TraceState, reason: str = 'event_done', error: str = '') -> None: + """Buffer one complete execution record when it matches the sample.""" try: - asyncio.get_running_loop().create_task(result) - except RuntimeError: - result.close() - logger = getattr(getattr(self.manager, 'ap', None), 'logger', None) - if logger is not None: - logger.debug('Execution trace payload dropped: no running event loop') + if state.closed: + return + state.closed = True + registered = self.traces.pop(state.trace_id, None) is not None + self.trace_deadlines.pop(state.trace_id, None) + self._unregister_state(state) + if not registered and not state.failure_reason: + return + if (not state.stages and not state.failure_reason) or not self._trace_emitted(state): + return + record = self._build_record(state, reason, error) + if record is not None: + self._enqueue_record(record) + except Exception: + return + + def _enqueue_record(self, record: dict) -> None: + if len(self.records) >= MAX_BUFFERED_RECORDS: + self.records.pop(0) + self.dropped += 1 + self.records.append(record) + self._ensure_loop() def _trace_emitted(self, state: TraceState) -> bool: mode = self.trace_mode() if mode == 'off': return False + # A chain that failed must always be uploaded, even when no stage was + # recorded before it broke: the reason is the whole point of the trace. + if state.failure_reason: + return True if mode == 'all': return True # A failed, cancelled or timed-out stage is always worth keeping. @@ -208,31 +322,69 @@ class ExecutionCounters: except ValueError: return False - def _build_trace_payload(self, state: TraceState, reason: str) -> dict | None: + def _build_record(self, state: TraceState, reason: str, error: str = '') -> dict | None: from ..utils import constants identity = state.identity if not identity.get('instance_id') or not identity.get('workspace_uuid'): return None - observations = [{**stage, 'trace_id': state.trace_id, 'count': 1} for stage in state.stages] + ended_at = datetime.now(timezone.utc).isoformat() + stages = state.stages + observations = [] + for stage in stages: + observation = { + 'seq': stage['seq'], + 'family': stage['family'], + 'operation': stage['operation'], + 'mode': stage['mode'], + 'adapter': stage['adapter'], + 'runner': stage['runner'], + 'outcome': stage['outcome'], + 'synthetic': stage['synthetic'], + 'error': stage['error'], + 'route_ref': stage['route_ref'], + 'run_id': stage['run_id'], + 'first_seen': stage['first_seen'], + 'last_seen': stage['last_seen'], + 'trace_id': state.trace_id, + } + # Workflow position is optional: records outside any step omit both + # keys entirely, so their payloads stay byte-identical. + if stage.get('node'): + observation['node'] = stage['node'] + if stage.get('parent'): + observation['parent'] = stage['parent'] + observations.append(observation) return { 'event_type': 'feature_execution', - # One payload per trace: lookup uses the indexed query_id column. + # The execution identity: one record == one chain == one Space row. 'query_id': state.trace_id, 'instance_id': identity['instance_id'], 'workspace_uuid': identity['workspace_uuid'], 'version': constants.semantic_version, 'edition': constants.edition, - 'timestamp': datetime.now(timezone.utc).isoformat(), + 'timestamp': ended_at, + 'trusted': True, + # First failure detail in this chain, so Space can show why it broke. + 'error': _bounded_error(error) + or state.failure_reason + or next((stage['error'] for stage in stages if stage['error']), ''), + # Legacy-compatible columns Space already maps; best effort. + 'duration_ms': _duration_ms(state.started_at, ended_at), + 'model_name': state.model_name or '', + 'adapter': state.adapter or next((stage['adapter'] for stage in stages if stage['adapter']), ''), + 'runner': state.runner or next((stage['runner'] for stage in stages if stage['runner']), ''), + 'runner_category': state.runner_category or '', + 'pipeline_plugins': state.pipeline_plugins, 'features': { 'schema': 1, 'observations': observations, 'trace': { 'closed_by': reason, 'started_at': state.started_at, - 'ended_at': datetime.now(timezone.utc).isoformat(), - 'route_ref': next((s['route_ref'] for s in state.stages if s['route_ref']), ''), - 'run_id': next((s['run_id'] for s in state.stages if s['run_id']), ''), + 'ended_at': ended_at, + 'route_ref': next((stage['route_ref'] for stage in stages if stage['route_ref']), ''), + 'run_id': next((stage['run_id'] for stage in stages if stage['run_id']), ''), 'outcome': state.outcome(), 'dropped_stages': state.dropped_stages, }, @@ -248,56 +400,36 @@ class ExecutionCounters: # ----------------------------------------------------------------- flushing async def _loop(self): + interval = self.flush_seconds() last_flush = time.monotonic() - while self.pending or self.traces: - # Traces expire on a short TTL; counters keep their minute cadence. - await asyncio.sleep(1 if self.traces else FLUSH_SECONDS) + while self.records or self.traces or self.registry: + # Traces expire on a short TTL; records keep their own cadence. + await asyncio.sleep(1 if (self.traces or self.registry) else interval) now = time.monotonic() self._sweep_traces(now) - if self.pending and now - last_flush >= FLUSH_SECONDS: + self._sweep_registry(now) + if self.records and now - last_flush >= interval: last_flush = now await self.flush() self.task = None async def flush(self): - from ..utils import constants - - # Remove only one bounded batch; subsequent windows drain the remainder. - # Drain one tenant per minute, at most one request, round-robin by insertion. - if not self.pending: + if not self.records: return - tenant = next(iter(self.pending))[:2] - keys = [key for key in self.pending if key[:2] == tenant][:MAX_BATCH] - groups: dict[tuple[str, str], list[dict]] = {} - for key in keys: - row = self.pending.pop(key) - instance, workspace, family, operation, mode, adapter, runner, outcome, synthetic = key - groups.setdefault((instance, workspace), []).append( - { - **row, - 'family': family, - 'operation': operation, - 'mode': mode, - 'adapter': adapter, - 'runner': runner, - 'outcome': outcome, - 'synthetic': synthetic, - } - ) - for (instance, workspace), observations in groups.items(): - payload = { - 'event_type': 'feature_execution', - 'query_id': str(uuid4()), - 'instance_id': instance, - 'workspace_uuid': workspace, - 'version': constants.semantic_version, - 'edition': constants.edition, - 'timestamp': datetime.now(timezone.utc).isoformat(), - 'features': {'schema': 1, 'observations': observations}, - } - if not await self.manager.send(payload): - await asyncio.sleep(1) - await self.manager.send(payload) + batch: list[dict] = [] + size = 0 + while self.records and len(batch) < MAX_RECORDS_PER_FLUSH: + encoded = len(json.dumps(self.records[0], ensure_ascii=False, default=str)) + if batch and size + encoded > MAX_FLUSH_BYTES: + break + batch.append(self.records.pop(0)) + size += encoded + if not batch: + return + payload = {'records': batch} + if not await self.manager.send(payload): + await asyncio.sleep(1) + await self.manager.send(payload) async def shutdown(self): if self.task is not None: @@ -305,47 +437,93 @@ class ExecutionCounters: await asyncio.gather(self.task, return_exceptions=True) self.task = None try: - await asyncio.wait_for(self.flush(), timeout=2) + deadline = time.monotonic() + 5 + while self.records and time.monotonic() < deadline: + remaining = len(self.records) + await asyncio.wait_for(self.flush(), timeout=2) + if len(self.records) >= remaining: + break except (Exception, asyncio.CancelledError): pass - self.pending.clear() - # Traces of in-flight events cannot be completed during shutdown. + self.records.clear() + # Executions still in flight cannot be completed during shutdown. self.dropped += len(self.traces) self.dropped_traces += len(self.traces) self.traces.clear() self.trace_deadlines.clear() + self.registry.clear() + self.registry_deadlines.clear() + + +def _execution_counters(ap): + counters = getattr(getattr(ap, 'telemetry', None), 'execution', None) + return counters if isinstance(counters, ExecutionCounters) else None def record(ap, context, **observation): """Best-effort bridge usable with optional telemetry and test doubles.""" try: - counters = getattr(getattr(ap, 'telemetry', None), 'execution', None) - if isinstance(counters, ExecutionCounters): + counters = _execution_counters(ap) + if counters is not None: counters.record(context, **observation) except Exception: pass -def close_trace(ap, state, reason: str = 'event_done') -> None: +def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None: """Best-effort trace close usable with optional telemetry and test doubles.""" try: - counters = getattr(getattr(ap, 'telemetry', None), 'execution', None) - if isinstance(counters, ExecutionCounters): - counters.close_trace(state, reason) + counters = _execution_counters(ap) + if counters is not None: + counters.close_trace(state, reason, error) except Exception: pass +def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding: + """Bind an execution trace and alias its identity for cross-task stages.""" + binding = trace_mod.bind(execution_id) + if execution_id: + try: + counters = _execution_counters(ap) + if counters is not None: + counters.register_trace(binding.state, execution_id) + except Exception: + pass + return binding + + @contextlib.contextmanager -def ingress(ap, reason: str = 'event_done'): - """Trace one ingress boundary; only the owner of the trace closes it. +def ingress(ap, reason: str = 'event_done', context=None, execution_id: str | None = None): + """Trace one execution boundary; only the owner of the trace closes it. Nested boundaries (an event routed into a Pipeline, a Runner invoked from a - dispatch) reuse the in-flight trace instead of starting a second one. + dispatch) reuse the in-flight trace instead of starting a second one, and + only alias their own execution id onto it. """ - binding = trace_mod.bind() + binding = trace_mod.bind(execution_id) + if binding.created and context is not None: + try: + binding.state.identity = workspace_identity(context) + except Exception: + pass + if execution_id and not binding.state.abandoned: + try: + counters = _execution_counters(ap) + if counters is not None: + counters.register_trace(binding.state, execution_id) + except Exception: + pass + failure = '' try: yield binding + except BaseException as exc: + # The boundary itself failed: keep the reason on the trace so the uploaded + # payload explains the failure instead of reporting a bare outcome. + if binding.created: + failure = f'{type(exc).__name__}: {exc}' + binding.state.mark_failure(failure) + raise finally: if trace_mod.unbind_root(binding): - close_trace(ap, binding.state, reason) + close_trace(ap, binding.state, reason, failure) diff --git a/src/langbot/pkg/telemetry/platform.py b/src/langbot/pkg/telemetry/platform.py index a7e636bcf..934f36e10 100644 --- a/src/langbot/pkg/telemetry/platform.py +++ b/src/langbot/pkg/telemetry/platform.py @@ -11,6 +11,21 @@ from .execution import record processing_mode: ContextVar[str] = ContextVar('telemetry_processing_mode', default='none') +def result_failure_detail(result) -> str: + """Pull a bounded human-readable reason out of a failed platform API result.""" + raw = getattr(result, 'raw', result) + if isinstance(raw, dict): + for key in ('error', 'message', 'msg', 'detail', 'reason'): + value = raw.get(key) + if isinstance(value, str) and value.strip(): + return value + if isinstance(value, dict): + for inner in ('message', 'msg', 'detail'): + if isinstance(value.get(inner), str) and value[inner].strip(): + return value[inner] + return '' + + def result_outcome(result): # Empty returns do not prove a remote operation succeeded. if result is None: @@ -69,18 +84,24 @@ def observe_adapter(ap, context, adapter): if action in declared: observed_operation = action outcome = 'unknown' + error_detail = '' try: result = await method(*args, **kwargs) outcome = result_outcome(result) + if outcome != 'success': + error_detail = result_failure_detail(result) return result except asyncio.CancelledError: outcome = 'cancelled' + error_detail = 'cancelled' raise - except TimeoutError: + except TimeoutError as exc: outcome = 'timeout' + error_detail = str(exc) or 'timeout' raise - except Exception: + except Exception as exc: outcome = 'failed' + error_detail = str(exc) raise finally: record( @@ -91,6 +112,7 @@ def observe_adapter(ap, context, adapter): adapter=adapter.__class__.__name__, mode=processing_mode.get(), outcome=outcome, + error=error_detail, ) return wrapped diff --git a/src/langbot/pkg/telemetry/trace.py b/src/langbot/pkg/telemetry/trace.py index a239c707e..445bb470e 100644 --- a/src/langbot/pkg/telemetry/trace.py +++ b/src/langbot/pkg/telemetry/trace.py @@ -1,9 +1,11 @@ -"""Content-free per-event execution traces. +"""Content-free per-execution traces. -One trace identity is minted per inbound platform event and survives until the -ingress handler returns. Stage records appended under it describe how that one -event was routed and processed, using only code-defined identifiers. Nothing -here is aggregated: a trace is a bounded, ordered sequence for one event. +One trace identity is one execution identity: the id a Pipeline query, inbound +platform event or Agent run is known by, so a stored chain can be joined with +the Space view of that same execution. Stage records appended under it describe +how that one execution was routed and processed, using only code-defined +identifiers. Nothing here is aggregated: a trace is a bounded, ordered sequence +for one execution. Three bounds keep memory independent of traffic volume: @@ -12,7 +14,9 @@ Three bounds keep memory independent of traffic volume: * a wall-clock TTL, enforced by the sender's sweep. The identity itself lives in a ContextVar so that asynchronous work spawned -while handling one event inherits it, mirroring ``telemetry.platform``. +while handling one execution inherits it, mirroring ``telemetry.platform``. +Cross-task stages (plugin/RPC work with no inherited context) are resolved by +the owner's execution-id registry instead. """ from __future__ import annotations @@ -28,6 +32,10 @@ MAX_STAGES = 32 _current: contextvars.ContextVar['TraceState | None'] = contextvars.ContextVar('telemetry_trace', default=None) _route: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_route', default='') _run: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_run', default='') +# Open workflow step nodes, innermost last. Records emitted inside a step carry +# the innermost node as their ``parent``; the step's own record carries the one +# below it (its enclosing step), never itself. +_node: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar('telemetry_trace_node', default=()) def _now() -> str: @@ -35,29 +43,55 @@ def _now() -> str: class TraceState: - """Bounded stage buffer for exactly one event.""" + """Bounded stage buffer for exactly one execution.""" __slots__ = ( 'trace_id', + 'execution_id', 'started_at', 'stages', 'dropped_stages', 'synthetic', 'sequence', + 'node_sequence', + 'open_nodes', 'identity', 'abandoned', + 'failure_reason', + 'closed', + 'adapter', + 'runner', + 'runner_category', + 'model_name', + 'pipeline_plugins', ) - def __init__(self) -> None: - self.trace_id = str(uuid4()) + def __init__(self, execution_id: str | None = None) -> None: + # The execution identity is the trace identity so a stored chain can be + # joined to the Space view of that same Pipeline query / event / run. + self.execution_id = str(execution_id).strip() if execution_id else '' + self.trace_id = self.execution_id or str(uuid4()) self.started_at = _now() self.stages: list[dict[str, typing.Any]] = [] self.dropped_stages = 0 self.synthetic = False self.sequence = 0 + # Node ids number the workflow steps of this one execution ("n1", "n2"...). + self.node_sequence = 0 + # Mirror of the open step stack, readable from any task: cross-task + # observations (plugin/RPC) resolve their parent through this. + self.open_nodes: tuple[str, ...] = () self.identity: dict[str, str] = {} # Set when the sender refused to buffer this trace: stop appending. self.abandoned = False + self.failure_reason = '' + self.closed = False + # Execution-scoped record fields attached by the owning lane when known. + self.adapter = '' + self.runner = '' + self.runner_category = '' + self.model_name = '' + self.pipeline_plugins: typing.Any = None def append( self, @@ -69,34 +103,62 @@ class TraceState: runner: str, outcome: str, synthetic: bool, + error: str = '', + node: str = '', + parent: str = '', ) -> None: if len(self.stages) >= MAX_STAGES: self.dropped_stages += 1 return seen = _now() - self.stages.append( - { - 'family': family, - 'operation': operation, - 'mode': mode, - 'adapter': adapter, - 'runner': runner, - 'outcome': outcome, - 'synthetic': bool(synthetic), - 'seq': self.sequence, - 'route_ref': _route.get(), - 'run_id': _run.get(), - 'first_seen': seen, - 'last_seen': seen, - } - ) + entry = { + 'family': family, + 'operation': operation, + 'mode': mode, + 'adapter': adapter, + 'runner': runner, + 'outcome': outcome, + 'synthetic': bool(synthetic), + 'seq': self.sequence, + 'route_ref': _route.get(), + 'run_id': _run.get(), + 'first_seen': seen, + 'last_seen': seen, + 'error': error, + } + # Only workflow steps carry these; legacy records stay byte-identical. + if node: + entry['node'] = node + if parent: + entry['parent'] = parent + self.stages.append(entry) self.sequence += 1 if synthetic: self.synthetic = True + def allocate_node(self) -> str: + """Mint the next step-node id for this execution ("n1", "n2", ...).""" + self.node_sequence += 1 + return f'n{self.node_sequence}' + + def mark_failure(self, reason: str) -> None: + """Remember why this chain broke so the payload can explain itself.""" + if not self.failure_reason: + self.failure_reason = reason + def outcome(self) -> str: - """Terminal outcome of the last recorded stage, for space-side display.""" - return self.stages[-1]['outcome'] if self.stages else 'unknown' + """Terminal outcome of this chain, for space-side display. + + A chain that carries a failure reason never reports ``success``: the + reason is the point of the trace. + """ + if self.stages: + last = self.stages[-1]['outcome'] + else: + last = 'unknown' + if self.failure_reason and last in ('success', 'skipped', 'unknown'): + return 'failed' + return last class TraceBinding(typing.NamedTuple): @@ -105,12 +167,16 @@ class TraceBinding(typing.NamedTuple): token: typing.Any -def bind() -> TraceBinding: - """Start a trace unless one is already in flight in this context.""" +def bind(execution_id: str | None = None) -> TraceBinding: + """Start a trace unless one is already in flight in this context. + + The execution id becomes the trace id only when this call starts the trace; + a nested boundary keeps the enclosing execution's identity. + """ existing = _current.get() if existing is not None: return TraceBinding(existing, False, None) - state = TraceState() + state = TraceState(execution_id) return TraceBinding(state, True, _current.set(state)) @@ -136,6 +202,46 @@ def current() -> TraceState | None: return _current.get() +def current_parent() -> str: + """The step node that records emitted right now attach to ('' when none).""" + stack = _node.get() + return stack[-1] if stack else '' + + +def enclosing_parent() -> str: + """The step enclosing the innermost open one; '' when it is a root step.""" + stack = _node.get() + return stack[-2] if len(stack) > 1 else '' + + +@contextlib.contextmanager +def stage_scope() -> typing.Iterator[str]: + """Open one workflow step node for the duration of the block. + + Allocates a node id unique within the execution, publishes it as the current + parent so that nested observations recorded inside the block attach to it, + and yields that id so the step's own records can be stamped with ``node=``. + Outside any trace nothing is allocated and the empty id is yielded. + """ + state = _current.get() + if state is None: + yield '' + return + node = state.allocate_node() + previous_open = state.open_nodes + state.open_nodes = previous_open + (node,) + token = _node.set(_node.get() + (node,)) + try: + yield node + finally: + state.open_nodes = previous_open + try: + _node.reset(token) + except (ValueError, RuntimeError): + # A token from another context must never break execution. + pass + + def set_run(run_id: str) -> typing.Any: return _run.set(run_id) diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 19c46ae35..e7cb87990 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -434,3 +434,7 @@ space: # (N is execution_trace_sample) and always keeps WebUI debug runs. execution_trace: sampled execution_trace_sample: 20 + # How often buffered execution records are uploaded, in seconds. + # Records of complete executions are batched as {"records": [...]}; a + # still-running execution is never uploaded half-written. + telemetry_flush_seconds: 180 diff --git a/tests/unit_tests/telemetry/test_chat_emitter.py b/tests/unit_tests/telemetry/test_chat_emitter.py new file mode 100644 index 000000000..0c66d3f83 --- /dev/null +++ b/tests/unit_tests/telemetry/test_chat_emitter.py @@ -0,0 +1,110 @@ +"""The Chat handler must not emit the legacy per-query telemetry payload. + +Per-execution telemetry is produced by the trace that owns the Pipeline lane; +the handler only annotates that trace with execution-scoped record fields. +""" + +from __future__ import annotations + +import types +from importlib import import_module +from unittest.mock import AsyncMock, Mock + +import langbot_plugin.api.entities.builtin.provider.session as provider_session + + +def get_modules(): + # Import the application package first: the handler package is wired into + # it during initialization, so importing the handler directly would re-enter + # the partially initialized pipeline package. + import_module('langbot.pkg.core.app') + return ( + import_module('langbot.pkg.pipeline.process.handlers.chat'), + import_module('langbot.pkg.telemetry.trace'), + ) + + +class FakeEvent: + """Stand-in for the plugin event models, which need a real Query.""" + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.event_name = 'person_normal_message_received' + + +class FakeEventContext: + event = types.SimpleNamespace(user_message_alter=None) + + def is_prevented_default(self) -> bool: + return False + + +async def empty_run_from_query(query): + if False: # pragma: no cover - async generator with no results + yield + + +def build_ap(): + ap = Mock() + ap.logger = Mock() + ap.plugin_connector = Mock() + ap.plugin_connector.emit_event = AsyncMock(return_value=FakeEventContext()) + ap.agent_run_orchestrator = Mock() + ap.agent_run_orchestrator.try_claim_steering_from_query = AsyncMock(return_value=False) + ap.agent_run_orchestrator.run_from_query = empty_run_from_query + ap.agent_run_orchestrator.resolve_runner_id_for_telemetry = Mock(return_value='runner-1') + ap.survey = Mock() + ap.survey.trigger_event = AsyncMock() + ap.survey.record_bot_response_success = AsyncMock() + ap.telemetry = Mock() + ap.telemetry.start_send_task = AsyncMock() + ap.telemetry.send = AsyncMock() + return ap + + +def build_query(): + query = Mock() + query.query_id = 7 + query.query_uuid = 'exec-uuid' + query.launcher_type = provider_session.LauncherTypes.PERSON + query.launcher_id = 'user-1' + query.sender_id = 'user-1' + query.message_chain = 'hello' + query.message_event = object() + query.variables = {'_pipeline_bound_plugins': ['plugin:a']} + query.pipeline_config = {'ai': {}, 'output': {'misc': {}}} + query.session = Mock(using_conversation=object()) + query.use_llm_model_uuid = None + query.adapter = Mock() + query.adapter.is_stream_output_supported = AsyncMock(return_value=False) + query.resp_messages = [] + return query + + +async def test_normal_query_sends_no_legacy_payload_and_annotates_the_trace(monkeypatch): + chat, trace_mod = get_modules() + monkeypatch.setattr(chat.events, 'PersonNormalMessageReceived', FakeEvent) + monkeypatch.setattr(chat.events, 'GroupNormalMessageReceived', FakeEvent) + + ap = build_ap() + handler = chat.ChatMessageHandler(ap) + query = build_query() + + binding = trace_mod.bind('exec-uuid') + try: + async for _ in handler.handle(query): + pass + finally: + trace_mod.unbind_root(binding) + + # The legacy `event_type='query'` emitter is gone. + ap.telemetry.start_send_task.assert_not_called() + ap.telemetry.send.assert_not_called() + # The survey trigger and its milestone bookkeeping still fire. + ap.survey.trigger_event.assert_awaited_once_with('first_bot_response_success') + ap.survey.record_bot_response_success.assert_awaited_once() + # Execution-scoped fields are attached to the owning trace instead. + assert binding.state.runner == 'runner-1' + assert binding.state.runner_category == 'unknown' + assert binding.state.pipeline_plugins == ['plugin:a'] + assert binding.state.model_name == '' diff --git a/tests/unit_tests/telemetry/test_pipeline_lane.py b/tests/unit_tests/telemetry/test_pipeline_lane.py new file mode 100644 index 000000000..f091f2bb8 --- /dev/null +++ b/tests/unit_tests/telemetry/test_pipeline_lane.py @@ -0,0 +1,272 @@ +"""The Pipeline lane must land its failure on the owning execution trace.""" + +from __future__ import annotations + +import asyncio +import types +from importlib import import_module + +import pytest + + +def get_modules(): + # Import the application package first; the pipeline package is wired into it. + import_module('langbot.pkg.core.app') + return ( + import_module('langbot.pkg.pipeline.pipelinemgr'), + import_module('langbot.pkg.telemetry.execution'), + ) + + +def get_trace(): + import_module('langbot.pkg.core.app') + return import_module('langbot.pkg.telemetry.trace') + + +class FakeManager: + def __init__(self): + self.telemetry_config = {'url': 'https://space.example.test', 'execution_trace': 'all'} + self.sent: list[dict] = [] + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +CONTEXT = types.SimpleNamespace(instance_uuid='instance-1', workspace_uuid='workspace-1') + + +def build_pipeline(pipelinemgr, ap): + pipeline = object.__new__(pipelinemgr.RuntimePipeline) + pipeline.ap = ap + pipeline.execution_context = CONTEXT + return pipeline + + +def test_lane_failure_marks_the_owned_trace_without_any_prior_stage(): + pipelinemgr, execution = get_modules() + manager = FakeManager() + counters = execution.ExecutionCounters(manager) + ap = types.SimpleNamespace(telemetry=types.SimpleNamespace(execution=counters)) + pipeline = build_pipeline(pipelinemgr, ap) + query = types.SimpleNamespace(pipeline_config={}) + + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-7'): + pipeline._record_lane_failure(query, 'sandbox exited 127') + + assert len(counters.records) == 1 + record = counters.records[0] + assert record['query_id'] == 'query-7' + assert record['error'] == 'sandbox exited 127' + assert record['features']['trace']['outcome'] == 'failed' + observations = record['features']['observations'] + assert [(row['family'], row['operation'], row['outcome']) for row in observations] == [ + ('runner', 'execute', 'failed') + ] + + +def test_lane_failure_does_not_duplicate_the_runner_node_the_orchestrator_recorded(): + """A break inside a runner is already a node; the lane only carries the reason over.""" + pipelinemgr, execution = get_modules() + manager = FakeManager() + counters = execution.ExecutionCounters(manager) + ap = types.SimpleNamespace(telemetry=types.SimpleNamespace(execution=counters)) + pipeline = build_pipeline(pipelinemgr, ap) + query = types.SimpleNamespace(pipeline_config={}) + + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-8'): + execution.record( + ap, + CONTEXT, + family='runner', + operation='execute', + mode='pipeline', + runner='plugin:langbot-team/LocalAgent/default', + outcome='failed', + error='No authorized model for local-agent', + ) + pipeline._record_lane_failure(query, 'No authorized model for local-agent') + + observations = [ + (row['family'], row['operation'], row['outcome']) + for record in counters.records + for row in record['features']['observations'] + ] + assert observations == [('runner', 'execute', 'failed')] + assert counters.records[-1]['features']['trace']['outcome'] == 'failed' + + +def build_counters(): + manager = FakeManager() + counters = import_module('langbot.pkg.telemetry.execution').ExecutionCounters(manager) + ap = types.SimpleNamespace(telemetry=types.SimpleNamespace(execution=counters)) + return counters, ap + + +def build_lane(pipelinemgr, ap): + # ``process_query`` lives on the RuntimePipeline lane, not on PipelineManager. + lane = object.__new__(pipelinemgr.RuntimePipeline) + lane.ap = ap + lane.execution_context = CONTEXT + return lane + + +def build_query(**overrides): + query = types.SimpleNamespace(query_uuid='exec-lane', query_id=1, pipeline_config={}, variables={}) + for key, value in overrides.items(): + setattr(query, key, value) + return query + + +def lane_shapes(counters): + return [ + (row['family'], row['operation'], row.get('node'), row.get('parent')) + for record in counters.records + for row in record['features']['observations'] + ] + + +async def stop_counters(counters): + """Cancel the telemetry flush task so no loop is left with pending work.""" + task = counters.task + if task is not None and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + counters.task = None + + +def test_lane_step_owns_the_apis_and_runner_it_calls(): + pipelinemgr, execution = get_modules() + trace = get_trace() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + query = build_query() + seen = {} + + async def fake_process(_query): + execution.record( + ap, CONTEXT, family='platform_api', operation='send_message', + adapter='AiocqhttpAdapter', mode='pipeline', outcome='success', + ) + with trace.stage_scope() as run_node: + seen['run'] = run_node + execution.record( + ap, CONTEXT, family='runner', operation='execute', mode='pipeline', outcome='success', node=run_node + ) + # A reply issued after the runner still belongs to the lane. + execution.record( + ap, CONTEXT, family='platform_api', operation='reply_message', + adapter='AiocqhttpAdapter', mode='pipeline', outcome='success', + ) + + lane._process_query = fake_process + + async def scenario(): + await lane.process_query(query) + await stop_counters(counters) + + asyncio.run(scenario()) + + shapes = lane_shapes(counters) + lane_node = shapes[-1][2] + assert lane_node == 'n1' + assert shapes == [ + ('platform_api', 'send_message', None, lane_node), + ('runner', 'execute', seen['run'], lane_node), + ('platform_api', 'reply_message', None, lane_node), + # The lane step is recorded last; a standalone lane is a root. + ('pipeline', 'run', lane_node, None), + ] + + +def test_lane_node_nests_under_the_platform_route_that_called_it(): + pipelinemgr, execution = get_modules() + trace = get_trace() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + query = build_query(query_uuid='exec-routed') + + async def fake_process(_query): + execution.record( + ap, CONTEXT, family='platform_api', operation='send_message', mode='pipeline', outcome='success' + ) + + lane._process_query = fake_process + + async def scenario(): + # A platform event owns the trace; the route step wraps the Pipeline lane. + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-routed'): + with trace.stage_scope() as event_node: + execution.record( + ap, CONTEXT, family='platform_event', operation='message.received', + adapter='AiocqhttpAdapter', outcome='success', node=event_node, + ) + with trace.stage_scope() as route_node: + execution.record( + ap, CONTEXT, family='event_route', operation='message.received', + adapter='AiocqhttpAdapter', outcome='success', node=route_node, + ) + await lane.process_query(query) + await stop_counters(counters) + return event_node, route_node + + event_node, route_node = asyncio.run(scenario()) + + assert (event_node, route_node) == ('n1', 'n2') + assert lane_shapes(counters) == [ + ('platform_event', 'message.received', event_node, None), + ('event_route', 'message.received', route_node, event_node), + # The API is called by the lane, so it hangs on the lane node. + ('platform_api', 'send_message', None, 'n3'), + ('pipeline', 'run', 'n3', route_node), + ] + + +def test_broken_lane_records_a_failed_step_with_its_reason(): + pipelinemgr, execution = get_modules() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + + async def fake_process(_query): + raise RuntimeError('lane exploded') + + lane._process_query = fake_process + + async def scenario(): + try: + await lane.process_query(build_query()) + finally: + await stop_counters(counters) + + with pytest.raises(RuntimeError, match='lane exploded'): + asyncio.run(scenario()) + + family, operation, node, parent = lane_shapes(counters)[-1] + row = counters.records[0]['features']['observations'][-1] + assert (family, operation, node, parent) == ('pipeline', 'run', 'n1', None) + assert row['outcome'] == 'failed' + assert row['error'] == 'lane exploded' + + +def test_lane_reported_failure_marks_the_step_failed(): + pipelinemgr, execution = get_modules() + counters, ap = build_counters() + lane = build_lane(pipelinemgr, ap) + query = build_query(variables={'_monitoring_has_error': True}) + + async def fake_process(_query): + return None + + lane._process_query = fake_process + + async def scenario(): + await lane.process_query(query) + await stop_counters(counters) + + asyncio.run(scenario()) + + row = counters.records[0]['features']['observations'][-1] + assert (row['family'], row['operation'], row['outcome']) == ('pipeline', 'run', 'failed') diff --git a/tests/unit_tests/telemetry/test_trace.py b/tests/unit_tests/telemetry/test_trace.py index f19ab2668..283cdac65 100644 --- a/tests/unit_tests/telemetry/test_trace.py +++ b/tests/unit_tests/telemetry/test_trace.py @@ -1,10 +1,13 @@ -"""Unit tests for per-event execution traces (pkg/telemetry/trace.py + execution.py).""" +"""Unit tests for per-execution execution traces (pkg/telemetry/trace.py + execution.py).""" from __future__ import annotations import asyncio +import contextvars +import json import time import types +from datetime import datetime, timedelta, timezone from importlib import import_module @@ -48,6 +51,11 @@ def trace_config(**overrides): return config +def records_sent(manager): + """Flatten the buffered-record batches a FakeManager received.""" + return [record for payload in manager.sent for record in payload.get('records', [])] + + CONTEXT = types.SimpleNamespace(instance_uuid='instance-1', workspace_uuid='workspace-1') STAGE = { @@ -85,6 +93,27 @@ class TestTraceIdentity: # A repeated unbind of the same binding must never raise. assert trace.unbind_root(outer) is True + def test_trace_id_is_the_execution_id(self): + trace, _ = get_modules() + binding = trace.bind('exec-1234') + try: + assert binding.created is True + assert binding.state.trace_id == 'exec-1234' + assert binding.state.execution_id == 'exec-1234' + finally: + trace.unbind_root(binding) + + def test_nested_boundary_keeps_the_enclosing_execution_id(self): + trace, _ = get_modules() + outer = trace.bind('event-1') + inner = trace.bind('query-9') + try: + assert inner.created is False + assert inner.state.trace_id == 'event-1' + finally: + trace.unbind_root(inner) + trace.unbind_root(outer) + def test_route_and_run_are_only_pinned_inside_the_block(self): trace, _ = get_modules() binding = trace.bind() @@ -174,23 +203,174 @@ class TestTraceRecording: assert bindings[-1].state.abandoned is True assert bindings[-1].state.stages == [] - def test_counters_keep_aggregating_while_tracing(self): + def test_observation_without_an_owning_execution_is_dropped(self): + trace, _ = get_modules() + _, counters = make_counters() + # Clustering is gone: an observation that belongs to no execution is + # discarded instead of being aggregated into a synthetic bucket. + counters.record(CONTEXT, **STAGE) + assert counters.records == [] + assert counters.traces == {} + + +class TestTraceNodeScope: + """Workflow step nodes: a step's own record and the observations it calls.""" + + @staticmethod + def _step(**overrides): + return { + 'family': 'runner', + 'operation': 'execute', + 'mode': 'agent', + 'runner': 'runner-1', + 'outcome': 'success', + **overrides, + } + + def test_nested_observation_carries_the_step_node_as_parent(self): trace, _ = get_modules() _, counters = make_counters() binding = trace.bind() try: - for _ in range(3): - counters.record(CONTEXT, **STAGE) + with trace.stage_scope() as step: + counters.record(CONTEXT, node=step, **self._step()) + counters.record(CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}) finally: trace.unbind_root(binding) - assert list(counters.pending.values())[0]['count'] == 3 + + step_row, api_row = binding.state.stages + assert step_row['node'] == step + assert 'parent' not in step_row # a root step carries no parent + assert api_row['parent'] == step + assert 'node' not in api_row # observations are not step nodes + + def test_nested_step_records_its_enclosing_step_as_parent(self): + trace, _ = get_modules() + _, counters = make_counters() + binding = trace.bind() + try: + with trace.stage_scope() as outer: + counters.record(CONTEXT, node=outer, **self._step(mode='pipeline')) + with trace.stage_scope() as inner: + counters.record(CONTEXT, node=inner, **self._step()) + counters.record(CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}) + finally: + trace.unbind_root(binding) + + outer_row, inner_row, api_row = binding.state.stages + assert (outer_row['node'], outer_row.get('parent')) == (outer, None) + # The step's own record sits under its enclosing step, never under itself. + assert (inner_row['node'], inner_row.get('parent')) == (inner, outer) + assert api_row.get('parent') == inner and 'node' not in api_row + + def test_sequential_steps_get_distinct_ids_and_roots_have_no_parent(self): + trace, _ = get_modules() + _, counters = make_counters() + binding = trace.bind() + try: + with trace.stage_scope() as first: + counters.record(CONTEXT, node=first, **self._step()) + with trace.stage_scope() as second: + counters.record(CONTEXT, node=second, **self._step()) + finally: + trace.unbind_root(binding) + + assert (first, second) == ('n1', 'n2') + assert [row['node'] for row in binding.state.stages] == [first, second] + assert all('parent' not in row for row in binding.state.stages) + + def test_cross_task_observation_attaches_to_the_open_step(self): + trace, execution = get_modules() + _, counters = make_counters() + binding = execution.bind_trace(make_ap(counters), 'exec-cross') + try: + with trace.stage_scope() as lane: + counters.record( + CONTEXT, family='pipeline', operation='run', mode='pipeline', outcome='success', node=lane + ) + # A plugin/RPC observation arrives on a task that never had the + # trace context; it is resolved through the execution registry and + # must still hang off the step the owning task has open. + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-cross') + foreign.run( + counters.record, + CONTEXT, + family='platform_api', + operation='send_message', + mode='pipeline', + outcome='success', + ) + finally: + trace.unbind_root(binding) + + rows = binding.state.stages + assert rows[0]['node'] == lane and 'parent' not in rows[0] + assert rows[1].get('parent') == lane and 'node' not in rows[1] + + def test_cross_task_observation_without_an_open_step_has_no_parent(self): + trace, execution = get_modules() + _, counters = make_counters() + binding = execution.bind_trace(make_ap(counters), 'exec-bare') + try: + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-bare') + foreign.run( + counters.record, + CONTEXT, + family='platform_api', + operation='send_message', + mode='pipeline', + outcome='success', + ) + finally: + trace.unbind_root(binding) + + row = binding.state.stages[0] + assert 'node' not in row and 'parent' not in row + + def test_scope_outside_any_trace_is_inert(self): + trace, _ = get_modules() + with trace.stage_scope() as node: + assert node == '' + assert trace.current_parent() == '' + + def test_records_outside_any_scope_serialize_without_node_keys(self): + trace, execution = get_modules() + manager, counters = make_counters(trace_config()) + binding = trace.bind('exec-legacy') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + + stage = binding.state.stages[0] + assert set(stage) == { + 'family', + 'operation', + 'mode', + 'adapter', + 'runner', + 'outcome', + 'synthetic', + 'seq', + 'route_ref', + 'run_id', + 'first_seen', + 'last_seen', + 'error', + } + observation = counters.records[0]['features']['observations'][0] + assert 'node' not in json.dumps(observation, sort_keys=True) + assert 'parent' not in json.dumps(observation, sort_keys=True) class TestTraceDelivery: - def test_close_emits_one_payload_per_trace(self): + def test_close_buffers_one_record_per_trace(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) - binding = trace.bind() + binding = trace.bind('exec-1') try: counters.record(CONTEXT, **STAGE) counters.record( @@ -201,18 +381,22 @@ class TestTraceDelivery: finally: trace.unbind_root(binding) - assert len(manager.sent) == 1 - payload = manager.sent[0] - assert payload['event_type'] == 'feature_execution' - assert payload['query_id'] == binding.state.trace_id - assert len(payload['query_id']) == 36 - assert payload['instance_id'] == 'instance-1' - assert payload['workspace_uuid'] == 'workspace-1' - features = payload['features'] + # Closing buffers; nothing is sent before the flush cadence. + assert manager.sent == [] + assert len(counters.records) == 1 + assert counters.traces == {} + + record = counters.records[0] + assert record['event_type'] == 'feature_execution' + assert record['query_id'] == 'exec-1' + assert record['instance_id'] == 'instance-1' + assert record['workspace_uuid'] == 'workspace-1' + assert record['trusted'] is True + features = record['features'] assert features['schema'] == 1 - assert [row['count'] for row in features['observations']] == [1, 1] + assert 'count' not in json.dumps(record) assert [row['seq'] for row in features['observations']] == [0, 1] - assert [row['trace_id'] for row in features['observations']] == [binding.state.trace_id] * 2 + assert [row['trace_id'] for row in features['observations']] == ['exec-1'] * 2 assert features['trace'] == { 'closed_by': 'event_done', 'started_at': binding.state.started_at, @@ -222,7 +406,6 @@ class TestTraceDelivery: 'outcome': 'success', 'dropped_stages': 0, } - assert counters.traces == {} def test_sampling_decision_is_deterministic_and_keeps_failures(self): trace, _ = get_modules() @@ -242,27 +425,28 @@ class TestTraceDelivery: def test_failures_mode_keeps_failed_and_debug_traces_only(self): trace, _ = get_modules() - manager, counters = make_counters(trace_config(execution_trace='failures', execution_trace_sample=1000000)) + _, counters = make_counters(trace_config(execution_trace='failures', execution_trace_sample=1000000)) - failed = trace.bind() + failed = trace.bind('exec-failed') counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) counters.close_trace(failed.state, 'event_done') trace.unbind_root(failed) - debug = trace.bind() + debug = trace.bind('exec-debug') counters.record(CONTEXT, **{**STAGE, 'synthetic': True}) counters.close_trace(debug.state, 'event_done') trace.unbind_root(debug) - sampled_out = trace.bind() + sampled_out = trace.bind('exec-sampled') counters.record(CONTEXT, **STAGE) counters.close_trace(sampled_out.state, 'event_done') trace.unbind_root(sampled_out) - assert [payload['features']['observations'][0]['outcome'] for payload in manager.sent] == ['failed', 'success'] - assert manager.sent[1]['features']['observations'][0]['synthetic'] is True + assert [record['query_id'] for record in counters.records] == ['exec-failed', 'exec-debug'] + assert counters.records[0]['features']['observations'][0]['outcome'] == 'failed' + assert counters.records[1]['features']['observations'][0]['synthetic'] is True - def test_off_mode_and_opt_out_emit_nothing(self): + def test_off_mode_and_opt_out_buffer_nothing(self): trace, _ = get_modules() manager, counters = make_counters(trace_config(execution_trace='off')) binding = trace.bind() @@ -272,6 +456,7 @@ class TestTraceDelivery: finally: trace.unbind_root(binding) assert manager.sent == [] + assert counters.records == [] disabled_manager, disabled = make_counters( {'url': 'https://space.example.test', 'disable_telemetry': True, 'execution_trace': 'all'} @@ -283,89 +468,359 @@ class TestTraceDelivery: finally: trace.unbind_root(binding) assert disabled_manager.sent == [] - assert disabled.pending == {} + assert disabled.records == [] assert disabled.traces == {} def test_ttl_sweep_closes_stale_trace(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) - binding = trace.bind() + binding = trace.bind('exec-ttl') try: counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) counters.trace_deadlines[binding.state.trace_id] = 0 counters._sweep_traces(time.monotonic()) finally: trace.unbind_root(binding) - assert manager.sent[0]['features']['trace']['closed_by'] == 'ttl' + assert counters.records[0]['features']['trace']['closed_by'] == 'ttl' assert counters.traces == {} assert counters.trace_deadlines == {} + assert manager.sent == [] async def test_shutdown_drops_open_traces(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) - binding = trace.bind() + binding = trace.bind('exec-open') try: counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) await counters.shutdown() finally: trace.unbind_root(binding) - # Shutdown may still flush window counters; it must not claim a trace. - assert [payload for payload in manager.sent if 'trace' in payload['features']] == [] + # Shutdown must not claim a trace that never completed. + assert records_sent(manager) == [] + assert counters.records == [] assert counters.traces == {} + assert counters.registry == {} assert counters.dropped_traces == 1 - async def test_loop_sweeps_and_sends_expired_trace(self): + async def test_loop_sweeps_and_flushes_expired_trace(self): trace, _ = get_modules() - manager, counters = make_counters(trace_config()) - binding = trace.bind() + manager, counters = make_counters(trace_config(telemetry_flush_seconds=1)) + binding = trace.bind('exec-ttl-loop') try: counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) counters.trace_deadlines[binding.state.trace_id] = 0 - await asyncio.sleep(1.2) - assert [payload['features']['trace']['closed_by'] for payload in manager.sent] == ['ttl'] + await asyncio.sleep(1.3) + assert [record['features']['trace']['closed_by'] for record in records_sent(manager)] == ['ttl'] assert counters.traces == {} finally: await counters.shutdown() trace.unbind_root(binding) + async def test_flush_sends_one_batch_of_complete_records(self): + _, execution = get_modules() + manager, counters = make_counters(trace_config()) + ap = make_ap(counters) + for index in range(2): + with execution.ingress(ap, 'event_done', CONTEXT, execution_id=f'exec-{index}'): + counters.record(CONTEXT, **STAGE) + assert len(counters.records) == 2 + + await counters.flush() + assert len(manager.sent) == 1 + payload = manager.sent[0] + assert set(payload) == {'records'} + assert [record['query_id'] for record in payload['records']] == ['exec-0', 'exec-1'] + assert 'count' not in json.dumps(payload) + + async def test_flush_interval_is_honoured(self): + _, execution = get_modules() + manager, counters = make_counters(trace_config(telemetry_flush_seconds=1)) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-timed'): + counters.record(CONTEXT, **STAGE) + assert counters.flush_seconds() == 1 + assert manager.sent == [] + await asyncio.sleep(1.3) + assert [record['query_id'] for record in records_sent(manager)] == ['exec-timed'] + await counters.shutdown() + + def test_flush_interval_config_is_bounded(self): + _, counters = make_counters(trace_config()) + assert counters.flush_seconds() == 180 + counters.manager.telemetry_config = trace_config(telemetry_flush_seconds='nope') + assert counters.flush_seconds() == 180 + counters.manager.telemetry_config = trace_config(telemetry_flush_seconds=0) + assert counters.flush_seconds() == 180 + counters.manager.telemetry_config = trace_config(telemetry_flush_seconds=300) + assert counters.flush_seconds() == 300 + + def test_buffered_record_cap_drops_oldest(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + original = execution.MAX_BUFFERED_RECORDS + execution.MAX_BUFFERED_RECORDS = 3 + try: + for index in range(5): + binding = trace.bind(f'exec-{index}') + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + trace.unbind_root(binding) + finally: + execution.MAX_BUFFERED_RECORDS = original + assert [record['query_id'] for record in counters.records] == ['exec-2', 'exec-3', 'exec-4'] + assert counters.dropped == 2 + class TestIngress: def test_ingress_closes_only_the_trace_it_started(self): _, execution = get_modules() manager, counters = make_counters(trace_config()) ap = make_ap(counters) - with execution.ingress(ap, 'event_done'): - with execution.ingress(ap, 'pipeline_done'): + with execution.ingress(ap, 'event_done', execution_id='exec-outer'): + with execution.ingress(ap, 'pipeline_done', execution_id='exec-inner'): counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - assert len(manager.sent) == 1 - assert manager.sent[0]['features']['trace']['closed_by'] == 'event_done' + assert len(counters.records) == 1 + assert counters.records[0]['query_id'] == 'exec-outer' + assert counters.records[0]['features']['trace']['closed_by'] == 'event_done' + assert manager.sent == [] def test_ingress_without_telemetry_is_a_no_op(self): _, execution = get_modules() - with execution.ingress(types.SimpleNamespace(), 'event_done'): + with execution.ingress(types.SimpleNamespace(), 'event_done', execution_id='exec-x'): pass + def test_ingress_registers_every_execution_alias(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', execution_id='event-1') as outer: + with execution.ingress(ap, 'pipeline_done', execution_id='query-1') as inner: + assert inner.state is outer.state + assert counters.registry == {} + # Closing the owning trace unregisters every alias it held. + assert counters.registry_deadlines == {} -class AsyncSendManager(FakeManager): - """Stand-in whose start_send_task is a coroutine, like TelemetryManager.""" - async def start_send_task(self, payload: dict) -> None: - self.sent.append(payload) +class TestCrossTaskAttribution: + def test_registry_joins_stages_from_a_task_without_a_trace(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'run-1') + trace.unbind_root(binding) + # The plugin/RPC task has no inherited trace context. + assert trace.current() is None + + counters.record(CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}) + counters.record( + CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}, execution_id='run-1' + ) + assert len(binding.state.stages) == 1 + assert binding.state.stages[0]['operation'] == 'send_message' + + counters.close_trace(binding.state, 'event_done') + assert counters.records[0]['query_id'] == 'run-1' + + def test_registry_aliases_a_nested_execution(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='event-1') as binding: + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-1'): + pass + assert set(counters.registry) == {'event-1', 'query-1'} + counters.record(CONTEXT, **STAGE, execution_id='query-1') + assert counters.records[0]['query_id'] == 'event-1' + assert len(binding.state.stages) == 1 + + def test_contextvar_execution_id_is_used_when_no_kwarg(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'run-2') + trace.unbind_root(binding) + token = execution.set_execution_id('run-2') + try: + counters.record(CONTEXT, **STAGE) + finally: + execution.reset_execution_id(token) + assert len(binding.state.stages) == 1 + + def test_registry_alias_is_swept_when_never_closed(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'run-3') + trace.unbind_root(binding) + counters.registry_deadlines['run-3'] = 0 + counters._sweep_registry(time.monotonic()) + assert counters.registry == {} + assert counters.registry_deadlines == {} class TestTraceDispatch: - async def test_close_trace_schedules_the_coroutine_send(self): - trace, execution = get_modules() + async def test_flush_awaits_the_manager_send(self): + _, execution = get_modules() manager = AsyncSendManager(trace_config()) counters = execution.ExecutionCounters(manager) - binding = trace.bind() + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-async'): + counters.record(CONTEXT, **STAGE) + await counters.flush() + assert [record['query_id'] for record in records_sent(manager)] == ['exec-async'] + assert manager.sent[0]['records'][0]['event_type'] == 'feature_execution' + + +class AsyncSendManager(FakeManager): + """Stand-in whose send is a coroutine, like TelemetryManager.""" + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +class TestTraceFailureDetail: + def test_failed_stage_carries_the_reason(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-fail') + try: + counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}, error='boom: sandbox exited 127') + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + record = counters.records[0] + assert record['error'] == 'boom: sandbox exited 127' + assert record['features']['observations'][0]['error'] == 'boom: sandbox exited 127' + + def test_successful_stage_reports_no_error(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-ok') try: counters.record(CONTEXT, **STAGE) counters.close_trace(binding.state, 'event_done') - # The payload is only delivered once the scheduled task runs. - assert manager.sent == [] - await asyncio.sleep(0) finally: trace.unbind_root(binding) - assert len(manager.sent) == 1 - assert manager.sent[0]['event_type'] == 'feature_execution' + record = counters.records[0] + assert record['error'] == '' + assert record['features']['observations'][0]['error'] == '' + + def test_error_detail_is_bounded_and_single_line(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-noisy') + try: + counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}, error='x' * 1000 + '\n\tboom') + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + detail = counters.records[0]['error'] + assert len(detail) <= 400 + assert '\n' not in detail and '\t' not in detail + + +class TestIngressFailureDetail: + def test_ingress_records_the_failure_reason(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + raised = False + try: + with execution.ingress(ap, 'event_done', execution_id='exec-boom'): + counters.record(CONTEXT, **STAGE) + raise RuntimeError('dispatch exploded') + except RuntimeError: + raised = True + assert raised is True + record = counters.records[0] + assert record['error'].startswith('RuntimeError: dispatch exploded') + assert record['features']['trace']['outcome'] == 'failed' + assert record['features']['trace']['closed_by'] == 'event_done' + + def test_successful_ingress_still_reports_success(self): + _, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + with execution.ingress(ap, 'event_done', execution_id='exec-fine'): + counters.record(CONTEXT, **STAGE) + record = counters.records[0] + assert record['error'] == '' + assert record['features']['trace']['outcome'] == 'success' + + +class TestFailureOnlyTrace: + def test_failure_without_stages_still_uploads(self): + _, execution = get_modules() + # sampled-out config: only failures are guaranteed to be kept + _, counters = make_counters({'url': 'https://space.example.test', 'execution_trace_sample': 1}) + ap = make_ap(counters) + try: + with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='exec-empty'): + raise RuntimeError('pipeline exploded before any stage') + except RuntimeError: + pass + assert len(counters.records) == 1 + record = counters.records[0] + assert record['error'].startswith('RuntimeError: pipeline exploded') + assert record['features']['trace']['outcome'] == 'failed' + assert record['features']['observations'] == [] + + def test_cancelled_and_timeout_chains_report_their_outcome(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + for outcome, reason in (('cancelled', 'run cancelled by operator'), ('timeout', 'deadline exhausted')): + binding = trace.bind(f'exec-{outcome}') + try: + counters.record(CONTEXT, **{**STAGE, 'outcome': outcome}, error=reason) + counters.close_trace(binding.state, 'runner_done') + finally: + trace.unbind_root(binding) + assert len(counters.records) == 1 + record = counters.records.pop(0) + assert record['query_id'] == f'exec-{outcome}' + assert record['features']['trace']['outcome'] == outcome + assert record['error'] == reason + + +class TestLegacyRecordFields: + def test_record_carries_legacy_top_level_columns(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-legacy') + binding.state.started_at = (datetime.now(timezone.utc) - timedelta(seconds=2)).isoformat() + binding.state.model_name = 'gpt-4o' + binding.state.pipeline_plugins = ['plugin:a'] + binding.state.runner_category = 'cloud' + try: + counters.record(CONTEXT, **STAGE) + counters.record( + CONTEXT, **{**STAGE, 'family': 'runner', 'operation': 'execute', 'mode': 'agent', 'runner': 'runner-1'} + ) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + record = counters.records[0] + assert record['model_name'] == 'gpt-4o' + assert record['pipeline_plugins'] == ['plugin:a'] + assert record['runner_category'] == 'cloud' + assert record['adapter'] == 'AiocqhttpAdapter' + assert record['runner'] == 'runner-1' + assert record['duration_ms'] >= 1000 + assert record['timestamp'] == record['features']['trace']['ended_at'] + + def test_legacy_string_fields_are_never_null(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-blank') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + record = counters.records[0] + assert record['model_name'] == '' + assert record['runner'] == '' + assert record['runner_category'] == '' + assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 + assert record['pipeline_plugins'] is None From 4c71de5ecab697474a80b5a83293b4b677ce4ae3 Mon Sep 17 00:00:00 2001 From: Hyu Date: Thu, 1 Oct 2026 19:01:16 +0800 Subject: [PATCH 02/30] feat(plugin): push plugin runtime ops samples to the Space control plane The shared Cloud deployment runs one Plugin Runtime for every Workspace, but nothing reported its health: the instance only exposed two aggregate numbers on its own health probe, and the per-installation failure registry, the desired state registry and the reconcile outcome lived in memory with no reader. Add RuntimeOpsReporter, which every 30 seconds reads the Runtime ``/healthz`` probe over HTTP (derived from ``plugin.runtime_ws_url``), merges it with the connector's desired-state, failure and reconcile summaries, and posts one sample to the Space control plane on the existing authenticated direction. Delivery is best-effort and bounded: a single in-flight request, a ten second timeout, dropped samples counted in ``runtime_ops`` on the instance health probe, and a failed probe recorded as a drop so the dashboard shows staleness instead of grading a stale sample as healthy. Cloud-only: a non-cloud deployment never enables the reporter, and the channel is independent of product telemetry so ``space.disable_telemetry`` cannot blind the ops dashboard. --- src/langbot/pkg/core/app.py | 6 + src/langbot/pkg/plugin/connector.py | 47 +++ src/langbot/pkg/plugin/runtime_ops.py | 389 ++++++++++++++++++ src/langbot/templates/config.yaml | 14 + .../plugin/test_runtime_ops_reporter.py | 294 +++++++++++++ 5 files changed, 750 insertions(+) create mode 100644 src/langbot/pkg/plugin/runtime_ops.py create mode 100644 tests/unit_tests/plugin/test_runtime_ops_reporter.py diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 5a4120240..7da4a0a15 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -304,6 +304,11 @@ class Application: if callable(database_snapshot): database_stats = database_snapshot() + runtime_ops_stats = {} + runtime_ops_reporter = getattr(self.plugin_connector, 'runtime_ops_reporter', None) + if runtime_ops_reporter is not None: + runtime_ops_stats = runtime_ops_reporter.get_stats() + return { 'asyncio_tasks': asyncio_tasks, 'event_loop': self.event_loop_monitor.snapshot(), @@ -314,6 +319,7 @@ class Application: 'query_pool': query_pool_stats, 'models': model_stats, 'runtimes': runtime_stats, + 'runtime_ops': runtime_ops_stats, 'telemetry_tasks': len(getattr(self.telemetry, 'send_tasks', ())), } diff --git a/src/langbot/pkg/plugin/connector.py b/src/langbot/pkg/plugin/connector.py index dc9aa9bec..6071f6cfb 100644 --- a/src/langbot/pkg/plugin/connector.py +++ b/src/langbot/pkg/plugin/connector.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import contextlib import contextvars +import datetime import hashlib import json import math @@ -21,6 +22,7 @@ from langbot_plugin.api.entities.builtin.pipeline.query import provider_session from ..core import app from . import handler +from . import runtime_ops from .errors import ( PluginRuntimeNotConnectedError, PluginInstallationFailedError, @@ -97,6 +99,12 @@ _HEARTBEAT_FAILURE_THRESHOLD = 3 _RECONNECT_MAX_DELAY_SEC = 60.0 +def _utc_now_iso() -> str: + """Return the current UTC time as a second-precision RFC 3339 string.""" + + return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + + async def _read_httpx_response_limited( response: httpx.Response, *, @@ -239,6 +247,14 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self._reconnect_task: asyncio.Task | None = None self._generation = 0 self._connected = asyncio.Event() + self.runtime_ops_reporter = runtime_ops.RuntimeOpsReporter(ap) + self._reconcile_summary: dict[str, Any] = { + 'last_started_at': None, + 'last_duration_ms': None, + 'last_ok': None, + 'failed_installations': 0, + 'missing_artifacts': 0, + } @staticmethod def _build_runtime_id() -> str: @@ -674,6 +690,21 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self._installation_failures.pop(installation_uuid, None) self._installation_failures.update(failures) + def _record_reconcile_outcome(self, started_at: str, duration_ms: float, result: dict[str, Any]) -> None: + """Expose the last reconcile outcome for the runtime ops sample.""" + + failed = result.get('failed_installations') + missing = result.get('missing_artifacts') + failed_installations = len(failed) if isinstance(failed, list) else 0 + missing_artifacts = len(missing) if isinstance(missing, list) else 0 + self._reconcile_summary = { + 'last_started_at': started_at, + 'last_duration_ms': int(max(duration_ms, 0.0)), + 'last_ok': failed_installations == 0 and missing_artifacts == 0, + 'failed_installations': failed_installations, + 'missing_artifacts': missing_artifacts, + } + async def _repair_reconcile_missing_artifacts( self, desired_states: dict[str, PluginInstallationDesiredState], @@ -754,12 +785,19 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): reconcile_timeout_seconds = max( 300.0, self._runtime_connect_timeout(self.ap.instance_config.data.get('plugin', {})) ) + reconcile_started_at = _utc_now_iso() + reconcile_started_monotonic = time.monotonic() result = await runtime_handler.reconcile_plugin_installations( tuple(self._known_desired_states.values()), timeout=reconcile_timeout_seconds, ) await self._repair_reconcile_missing_artifacts(self._known_desired_states, result) self._record_reconcile_failures(self._known_desired_states, result) + self._record_reconcile_outcome( + reconcile_started_at, + (time.monotonic() - reconcile_started_monotonic) * 1000.0, + result, + ) async def reconcile_projected_workspaces( self, @@ -775,6 +813,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): runtime_handler = self._runtime_handler() started_at = time.monotonic() + started_at_iso = _utc_now_iso() async with self._state_lock: all_states: dict[str, PluginInstallationDesiredState] = {} workspace_installations: dict[str, set[str]] = {} @@ -801,6 +840,11 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): ) await self._repair_reconcile_missing_artifacts(all_states, result) self._record_reconcile_failures(all_states, result) + self._record_reconcile_outcome( + started_at_iso, + (time.monotonic() - started_at) * 1000.0, + result, + ) for installation_uuid, previous in tuple(self._known_desired_states.items()): if installation_uuid not in all_states: runtime_handler.unregister_installation_binding(previous.binding) @@ -1068,6 +1112,8 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): if self.heartbeat_task is None or self.heartbeat_task.done(): self.heartbeat_task = asyncio.create_task(self.heartbeat_loop()) + self.runtime_ops_reporter.start() + def schedule_reconnect(self) -> None: if self._closing or not self.is_enable_plugin: return @@ -1130,6 +1176,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): await asyncio.gather(self.heartbeat_task, return_exceptions=True) self.heartbeat_task = None await self._stop_transport() + await self.runtime_ops_reporter.stop() await self._close_managed_subprocess() @staticmethod diff --git a/src/langbot/pkg/plugin/runtime_ops.py b/src/langbot/pkg/plugin/runtime_ops.py new file mode 100644 index 000000000..29f1a5de9 --- /dev/null +++ b/src/langbot/pkg/plugin/runtime_ops.py @@ -0,0 +1,389 @@ +"""Cloud Edition runtime ops sample reporter. + +Aggregates this instance's plugin Runtime health (probed from the Runtime +``/healthz`` endpoint) together with the connector's own desired-state and +failure registries, then pushes one compact sample to the Space control +plane where the internal ops dashboard stores and renders it. + +This is a Cloud-only ops channel, deliberately separate from product +telemetry so ``space.disable_telemetry`` cannot blind operations. Sending is +best-effort: a failed probe or upload never raises out of the reporter. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import datetime +import math +import os +import typing +from urllib.parse import urlsplit, urlunsplit + +import httpx + +from ..utils import constants, httpclient + +_CONTROL_PLANE_TOKEN_ENV = 'LANGBOT_SPACE_CONTROL_PLANE_TOKEN' +_RUNTIME_OPS_PATH = '/api/v1/cloud/v2/internal/runtime-ops' +_DEFAULT_INTERVAL_SECONDS = 30.0 +_DEFAULT_TIMEOUT_SECONDS = 10.0 +_DEFAULT_MAX_FAILURES = 5 +_MAX_FAILURE_MESSAGE_CHARS = 512 + + +def _utc_now_iso() -> str: + """Return the current UTC time as a second-precision RFC 3339 string.""" + + return datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds').replace('+00:00', 'Z') + + +def _coerce_bool(value: typing.Any, default: bool) -> bool: + return value if isinstance(value, bool) else default + + +def _coerce_positive_float(value: typing.Any, default: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return default + value = float(value) + if not math.isfinite(value) or value <= 0: + return default + return value + + +def _coerce_positive_int(value: typing.Any, default: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + return default + return value + + +def _sanitize_failure_message(value: typing.Any) -> str: + """Flatten and bound one failure message for the wire. + + Mirrors the connector's own sanitization so ops never receives control + characters, unbounded payloads, or newline-injected log content. + """ + + candidate = str(value or '').replace('\r', ' ').replace('\n', ' ').strip() + return candidate[:_MAX_FAILURE_MESSAGE_CHARS] + + +class RuntimeOpsReporter: + """Collect and push one runtime ops sample per interval. + + The reporter never raises out of its send loop; failures are counted and + surfaced through :meth:`get_stats`. + """ + + def __init__(self, ap: typing.Any): + self.ap = ap + instance_data = getattr(getattr(ap, 'instance_config', None), 'data', None) + if not isinstance(instance_data, dict): + instance_data = {} + plugin_config = instance_data.get('plugin', {}) + if not isinstance(plugin_config, dict): + plugin_config = {} + config = plugin_config.get('runtime_ops', {}) + if not isinstance(config, dict): + config = {} + self.config = config + + deployment_mode = getattr(getattr(ap, 'deployment', None), 'mode', 'oss') + cloud_edition = deployment_mode == 'cloud' + # Cloud-only feature: an explicit flag may turn it off, but a + # non-cloud deployment is never enabled. + self.enabled = cloud_edition and _coerce_bool(config.get('enabled'), True) + self.interval_seconds = _coerce_positive_float(config.get('interval_seconds'), _DEFAULT_INTERVAL_SECONDS) + self.timeout_seconds = _coerce_positive_float(config.get('timeout_seconds'), _DEFAULT_TIMEOUT_SECONDS) + self.max_failures = _coerce_positive_int(config.get('max_failures'), _DEFAULT_MAX_FAILURES) + + self.health_url = self._derive_health_url(plugin_config.get('runtime_ws_url')) + self.space_url = str(instance_data.get('space', {}).get('url', '') or '').rstrip('/') + self._token = str(os.environ.get(_CONTROL_PLANE_TOKEN_ENV, '') or '').strip() + + if self.enabled and not self.health_url: + self._log_info( + 'Plugin runtime ops reporter is disabled: no plugin runtime WebSocket URL is configured' + ) + self.enabled = False + if self.enabled and not self.space_url: + self._log_info('Plugin runtime ops reporter is disabled: no Space URL is configured') + self.enabled = False + + self._task: asyncio.Task | None = None + self._client: httpx.AsyncClient | None = None + self._sent_total = 0 + self._dropped_total = 0 + self._last_error: str | None = None + + def _log_info(self, message: str) -> None: + logger = getattr(self.ap, 'logger', None) + info = getattr(logger, 'info', None) + if callable(info): + info(message) + + @staticmethod + def _derive_health_url(runtime_ws_url: typing.Any) -> str | None: + """Derive the Runtime ``/healthz`` URL from the control WebSocket URL.""" + + if not isinstance(runtime_ws_url, str): + return None + candidate = runtime_ws_url.strip() + if not candidate: + return None + try: + parts = urlsplit(candidate) + port = parts.port + except ValueError: + return None + if parts.scheme not in ('ws', 'wss') or not parts.hostname: + return None + scheme = 'https' if parts.scheme == 'wss' else 'http' + netloc = parts.hostname if port is None else f'{parts.hostname}:{port}' + return urlunsplit((scheme, netloc, '/healthz', '', '')) + + def start(self) -> None: + """Start the background sample loop once the control link is up.""" + + if not self.enabled: + return + if self._task is not None and not self._task.done(): + return + self._task = asyncio.create_task(self._run()) + + async def stop(self) -> None: + """Cancel the sample loop and release the HTTP client.""" + + task = self._task + self._task = None + if task is not None and task is not asyncio.current_task(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + client = self._client + self._client = None + if client is not None and not client.is_closed: + with contextlib.suppress(Exception): + await client.aclose() + + def get_stats(self) -> dict[str, typing.Any]: + """Return in-memory counters for the instance liveness surface.""" + + return { + 'sent_total': self._sent_total, + 'dropped_total': self._dropped_total, + 'last_error': self._last_error, + } + + @contextlib.asynccontextmanager + async def _client_context(self): + if self._client is None or self._client.is_closed: + self._client = httpx.AsyncClient( + timeout=httpx.Timeout(self.timeout_seconds), + event_hooks=httpclient.httpx_response_limit_hooks(), + ) + yield self._client + + async def _run(self) -> None: + while True: + try: + await self._collect_and_send() + except asyncio.CancelledError: + raise + except Exception as exc: # never let the ops channel crash the connector + self._last_error = _sanitize_failure_message(f'sample cycle failed: {exc}') + await asyncio.sleep(self.interval_seconds) + + async def _collect_and_send(self) -> None: + health = await self._fetch_health() + if health is None: + self._dropped_total += 1 + return + await self._post(self.build_payload(health)) + + async def _fetch_health(self) -> dict[str, typing.Any] | None: + """Probe the Runtime ``/healthz`` endpoint, returning its verbatim JSON.""" + + try: + async with self._client_context() as client: + response = await asyncio.wait_for( + client.get(self.health_url), + timeout=self.timeout_seconds, + ) + if response.status_code >= 400: + self._last_error = f'health probe returned status {response.status_code}' + return None + payload = await httpclient.parse_json_response(response) + if not isinstance(payload, dict): + self._last_error = 'health probe returned a non-object payload' + return None + self._last_error = None + return payload + except asyncio.TimeoutError: + self._last_error = 'health probe timed out' + return None + except Exception as exc: + self._last_error = _sanitize_failure_message(f'health probe failed: {exc}') + return None + + async def _post(self, payload: dict[str, typing.Any]) -> None: + """Best-effort POST of one sample; never raises.""" + + url = f'{self.space_url}{_RUNTIME_OPS_PATH}' + headers = {'Authorization': f'Bearer {self._token}'} + try: + async with self._client_context() as client: + response = await asyncio.wait_for( + client.post(url, json=payload, headers=headers), + timeout=self.timeout_seconds, + ) + if response.status_code >= 400: + body = await httpclient.response_text(response, max_chars=200) + self._dropped_total += 1 + self._last_error = f'POST {url} returned status {response.status_code} - {body}' + return + self._sent_total += 1 + self._last_error = None + except asyncio.TimeoutError: + self._dropped_total += 1 + self._last_error = f'POST {url} timed out' + except Exception as exc: + self._dropped_total += 1 + self._last_error = _sanitize_failure_message(f'POST {url} failed: {exc}') + + def build_payload(self, health: dict[str, typing.Any]) -> dict[str, typing.Any]: + """Assemble one ops sample in the frozen Core -> Space contract shape.""" + + connector = getattr(self.ap, 'plugin_connector', None) + instance_data = getattr(getattr(self.ap, 'instance_config', None), 'data', None) + if not isinstance(instance_data, dict): + instance_data = {} + system_config = instance_data.get('system', {}) + if not isinstance(system_config, dict): + system_config = {} + instance_uuid = str(system_config.get('instance_id', '') or '') or str(constants.instance_id or '') + edition = str(system_config.get('edition', 'oss') or 'oss') + return { + 'event': 'runtime_ops_sample', + 'kind': 'aggregate', + 'instance_uuid': instance_uuid, + 'generated_at': _utc_now_iso(), + 'core': { + 'version': constants.semantic_version, + 'edition': edition, + 'connected': self._runtime_connected(connector), + 'runtime_profile': getattr(connector, 'runtime_profile', 'oss_dev'), + 'worker_policy': self._worker_policy(connector), + 'desired_states': self._desired_states(connector), + 'reconcile': self._reconcile_summary(connector), + }, + 'health': health, + 'failures': self._failures(connector), + } + + @staticmethod + def _runtime_connected(connector: typing.Any) -> bool: + available = getattr(connector, '_runtime_available', None) + if not callable(available): + return False + try: + return bool(available()) + except Exception: + return False + + @staticmethod + def _worker_policy(connector: typing.Any) -> dict[str, typing.Any] | None: + policy = getattr(connector, 'worker_policy', None) + if policy is None: + return None + model_dump = getattr(policy, 'model_dump', None) + if callable(model_dump): + return model_dump() + if isinstance(policy, dict): + return dict(policy) + return None + + @staticmethod + def _desired_states(connector: typing.Any) -> dict[str, typing.Any]: + states = getattr(connector, '_known_desired_states', None) + if not isinstance(states, dict): + states = {} + enabled = 0 + by_mode: dict[str, int] = {} + for state in states.values(): + if bool(getattr(state, 'enabled', False)): + enabled += 1 + mode = getattr(getattr(state, 'execution_mode', None), 'value', None) + mode = str(mode) if isinstance(mode, str) and mode else 'dedicated' + by_mode[mode] = by_mode.get(mode, 0) + 1 + return {'total': len(states), 'enabled': enabled, 'by_mode': by_mode} + + @staticmethod + def _reconcile_summary(connector: typing.Any) -> dict[str, typing.Any]: + summary = getattr(connector, '_reconcile_summary', None) + if isinstance(summary, dict): + return dict(summary) + return { + 'last_started_at': None, + 'last_duration_ms': None, + 'last_ok': None, + 'failed_installations': 0, + 'missing_artifacts': 0, + } + + def _failures(self, connector: typing.Any) -> list[dict[str, typing.Any]]: + records = getattr(connector, '_installation_failures', None) + if not isinstance(records, dict): + return [] + states = getattr(connector, '_known_desired_states', None) + if not isinstance(states, dict): + states = {} + identities = self._plugin_identities(connector) + observed_at = _utc_now_iso() + failures: list[dict[str, typing.Any]] = [] + for installation_uuid in list(records)[: self.max_failures]: + record = records.get(installation_uuid) + if not isinstance(record, dict): + continue + state = states.get(installation_uuid) + binding = getattr(state, 'binding', None) + mode = getattr(getattr(state, 'execution_mode', None), 'value', None) + mode = str(mode) if isinstance(mode, str) and mode else 'dedicated' + plugin_author, plugin_name = identities.get(str(installation_uuid), ('', '')) + failures.append( + { + 'workspace_uuid': str(getattr(binding, 'workspace_uuid', '') or ''), + 'installation_uuid': str(installation_uuid), + 'plugin_author': plugin_author, + 'plugin_name': plugin_name, + 'execution_mode': mode, + 'error_code': str(record.get('error_code', '') or ''), + 'message': _sanitize_failure_message(record.get('message')), + 'observed_at': observed_at, + } + ) + return failures + + @staticmethod + def _plugin_identities(connector: typing.Any) -> dict[str, tuple[str, str]]: + """Read the connector handler's installation identity registry. + + The Runtime connection handler binds every desired installation to + its plugin identity; the connector's failure records themselves only + carry the installation UUID, so the reporter joins them here. + """ + + handler = getattr(connector, 'handler', None) + bindings = getattr(handler, '_installation_bindings', None) + if not isinstance(bindings, dict): + return {} + identities: dict[str, tuple[str, str]] = {} + for installation_uuid, entry in bindings.items(): + try: + identity = entry[1] + except (TypeError, IndexError, KeyError): + continue + identities[str(installation_uuid)] = ( + str(getattr(identity, 'plugin_author', '') or ''), + str(getattr(identity, 'plugin_name', '') or ''), + ) + return identities diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index e7cb87990..9e5a0f7ac 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -293,6 +293,20 @@ plugin: binary_storage: # Max bytes for a single plugin binary storage value max_value_bytes: 10485760 + runtime_ops: + # Cloud-only ops telemetry: aggregates this instance's plugin Runtime + # health, desired installations and reconcile failures and pushes them + # to the Space control plane for the internal ops dashboard. This is + # separate from product telemetry (space.disable_telemetry) and has no + # effect outside the cloud edition. Defaults to on for the cloud + # edition; set false to stop pushing samples. + enabled: true + # Seconds between samples pushed to Space. + interval_seconds: 30 + # Per-request timeout for the Runtime /healthz probe and the upload. + timeout_seconds: 10 + # Maximum number of failed installations included per sample. + max_failures: 5 runner: # Host-level admin permissions for trusted control plugins. These plugins # can use existing plugin action handlers to inspect or manage Runner diff --git a/tests/unit_tests/plugin/test_runtime_ops_reporter.py b/tests/unit_tests/plugin/test_runtime_ops_reporter.py new file mode 100644 index 000000000..51e9c7861 --- /dev/null +++ b/tests/unit_tests/plugin/test_runtime_ops_reporter.py @@ -0,0 +1,294 @@ +"""Tests for the Cloud plugin runtime ops sample reporter. + +Covers the frozen Core -> Space sample contract (C2): +- Runtime /healthz URL derivation from the control WebSocket URL +- Disablement without a runtime WebSocket URL / outside the cloud edition +- Sample assembly from connector internals (desired states, failures, health) +- Best-effort upload semantics (non-2xx and transport failures never raise) +- The pushed payload never carries the runtime control token +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import Mock + +import httpx +import pytest + +from langbot.pkg.plugin.runtime_ops import RuntimeOpsReporter +from langbot_plugin.entities.io.context import PluginExecutionMode +from langbot_plugin.runtime.security import PLUGIN_RUNTIME_CONTROL_TOKEN_ENV + +_CONTROL_PLANE_TOKEN_ENV = 'LANGBOT_SPACE_CONTROL_PLANE_TOKEN' + +_DEFAULT_WS_URL = 'ws://langbot_plugin_runtime:5400/control/ws' + + +def make_policy() -> SimpleNamespace: + return SimpleNamespace(model_dump=lambda: {'max_workers': 32, 'max_cpus': 0.25}) + + +def make_connector( + *, + desired_states: dict | None = None, + failures: dict | None = None, + identities: dict | None = None, + connected: bool = True, +) -> SimpleNamespace: + return SimpleNamespace( + runtime_profile='shared', + worker_policy=make_policy(), + _known_desired_states=desired_states if desired_states is not None else {}, + _installation_failures=failures if failures is not None else {}, + _reconcile_summary={ + 'last_started_at': '2026-10-01T07:11:00Z', + 'last_duration_ms': 1234, + 'last_ok': True, + 'failed_installations': 0, + 'missing_artifacts': 0, + }, + handler=SimpleNamespace(_installation_bindings=identities if identities is not None else {}), + _runtime_available=lambda: connected, + ) + + +def make_app( + *, + runtime_ws_url: str | None = _DEFAULT_WS_URL, + runtime_ops: dict | None = None, + space_url: str = 'https://space.example', + deployment_mode: str = 'cloud', + edition: str = 'cloud', + connector: SimpleNamespace | None = None, +) -> SimpleNamespace: + plugin: dict = {} + if runtime_ws_url is not None: + plugin['runtime_ws_url'] = runtime_ws_url + if runtime_ops is not None: + plugin['runtime_ops'] = runtime_ops + return SimpleNamespace( + logger=Mock(), + instance_config=SimpleNamespace( + data={ + 'plugin': plugin, + 'space': {'url': space_url}, + 'system': {'instance_id': 'inst-1', 'edition': edition}, + } + ), + deployment=SimpleNamespace(mode=deployment_mode), + plugin_connector=connector if connector is not None else make_connector(), + ) + + +def make_desired_state(workspace_uuid: str, mode: PluginExecutionMode, *, enabled: bool = True) -> SimpleNamespace: + return SimpleNamespace( + enabled=enabled, + execution_mode=mode, + binding=SimpleNamespace(workspace_uuid=workspace_uuid), + ) + + +class TestHealthUrlDerivation: + @pytest.mark.parametrize( + ('ws_url', 'expected'), + [ + ('ws://langbot_plugin_runtime:5400/control/ws', 'http://langbot_plugin_runtime:5400/healthz'), + ('wss://runtime.example/control/ws', 'https://runtime.example/healthz'), + ('wss://runtime.example:8443/control/ws', 'https://runtime.example:8443/healthz'), + ('ws://127.0.0.1:5400', 'http://127.0.0.1:5400/healthz'), + (' ws://runtime.example:5400/control/ws ', 'http://runtime.example:5400/healthz'), + ], + ) + def test_derives_health_url(self, ws_url: str, expected: str): + reporter = RuntimeOpsReporter(make_app(runtime_ws_url=ws_url)) + + assert reporter.health_url == expected + + @pytest.mark.parametrize('ws_url', ['', None, 'http://runtime.example:5400', 'not a url']) + def test_rejects_non_websocket_url(self, ws_url): + reporter = RuntimeOpsReporter(make_app(runtime_ws_url=ws_url)) + + assert reporter.health_url is None + + +class TestEnablement: + def test_disabled_without_runtime_websocket_url(self): + reporter = RuntimeOpsReporter(make_app(runtime_ws_url=None)) + + assert reporter.enabled is False + + def test_disabled_on_non_cloud_deployment(self): + reporter = RuntimeOpsReporter(make_app(deployment_mode='oss')) + + assert reporter.enabled is False + + def test_explicit_disable_on_cloud(self): + reporter = RuntimeOpsReporter(make_app(runtime_ops={'enabled': False})) + + assert reporter.enabled is False + + def test_enabled_by_default_on_cloud(self): + reporter = RuntimeOpsReporter(make_app()) + + assert reporter.enabled is True + assert reporter.interval_seconds == 30.0 + assert reporter.timeout_seconds == 10.0 + assert reporter.max_failures == 5 + + +class TestSampleAssembly: + def test_build_payload_matches_contract(self): + desired = { + 'inst-dedicated': make_desired_state('ws-1', PluginExecutionMode.DEDICATED), + 'inst-shared-off': make_desired_state( + 'ws-2', PluginExecutionMode.SHARED_CERTIFIED, enabled=False + ), + 'inst-shared-on': make_desired_state('ws-3', PluginExecutionMode.SHARED_CERTIFIED), + } + failures = { + 'inst-dedicated': { + 'installation_uuid': 'inst-dedicated', + 'error_code': 'worker_launch_failed', + 'message': 'boom\nstack line', + }, + 'inst-shared-off': { + 'installation_uuid': 'inst-shared-off', + 'error_code': 'dependency_prepare_failed', + 'message': 'prepare failed', + }, + } + identities = { + 'inst-dedicated': ( + None, + SimpleNamespace(plugin_author='acme', plugin_name='dedicated-plugin'), + ), + } + connector = make_connector(desired_states=desired, failures=failures, identities=identities) + reporter = RuntimeOpsReporter(make_app(connector=connector)) + health = {'live': True, 'runtime': {'version': '0.7.6'}, 'shared_pool': {'workers': 12}} + + payload = reporter.build_payload(health) + + assert set(payload) == {'event', 'kind', 'instance_uuid', 'generated_at', 'core', 'health', 'failures'} + assert payload['event'] == 'runtime_ops_sample' + assert payload['kind'] == 'aggregate' + assert payload['instance_uuid'] == 'inst-1' + assert payload['generated_at'].endswith('Z') + # The health blob is forwarded verbatim. + assert payload['health'] is health + + core = payload['core'] + assert core['edition'] == 'cloud' + assert core['connected'] is True + assert core['runtime_profile'] == 'shared' + assert core['worker_policy'] == {'max_workers': 32, 'max_cpus': 0.25} + assert core['desired_states'] == { + 'total': 3, + 'enabled': 2, + 'by_mode': {'dedicated': 1, 'shared-runtime-v1': 2}, + } + assert core['reconcile'] == { + 'last_started_at': '2026-10-01T07:11:00Z', + 'last_duration_ms': 1234, + 'last_ok': True, + 'failed_installations': 0, + 'missing_artifacts': 0, + } + + assert len(payload['failures']) == 2 + first = payload['failures'][0] + assert first['workspace_uuid'] == 'ws-1' + assert first['installation_uuid'] == 'inst-dedicated' + assert first['plugin_author'] == 'acme' + assert first['plugin_name'] == 'dedicated-plugin' + assert first['execution_mode'] == 'dedicated' + assert first['error_code'] == 'worker_launch_failed' + assert first['message'] == 'boom stack line' + assert first['observed_at'].endswith('Z') + second = payload['failures'][1] + assert second['workspace_uuid'] == 'ws-2' + assert second['plugin_author'] == '' + assert second['plugin_name'] == '' + assert second['execution_mode'] == 'shared-runtime-v1' + + def test_failures_capped_by_max_failures(self): + failures = { + f'inst-{index}': { + 'installation_uuid': f'inst-{index}', + 'error_code': 'worker_launch_failed', + 'message': 'boom', + } + for index in range(8) + } + connector = make_connector(failures=failures) + reporter = RuntimeOpsReporter(make_app(runtime_ops={'max_failures': 3}, connector=connector)) + + payload = reporter.build_payload({'live': True}) + + assert len(payload['failures']) == 3 + + def test_disconnected_connector_reports_not_connected(self): + connector = make_connector(connected=False) + reporter = RuntimeOpsReporter(make_app(connector=connector)) + + payload = reporter.build_payload({'live': False}) + + assert payload['core']['connected'] is False + + +class TestUpload: + @pytest.mark.asyncio + async def test_successful_post_increments_sent(self): + reporter = RuntimeOpsReporter(make_app()) + reporter._client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={'code': 0})) + ) + + await reporter._post({'event': 'runtime_ops_sample'}) + + assert reporter.get_stats() == {'sent_total': 1, 'dropped_total': 0, 'last_error': None} + await reporter.stop() + + @pytest.mark.asyncio + async def test_non_2xx_increments_dropped_without_raising(self): + reporter = RuntimeOpsReporter(make_app()) + reporter._client = httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(503, text='unavailable')) + ) + + await reporter._post({'event': 'runtime_ops_sample'}) + + stats = reporter.get_stats() + assert stats['sent_total'] == 0 + assert stats['dropped_total'] == 1 + assert '503' in stats['last_error'] + await reporter.stop() + + @pytest.mark.asyncio + async def test_transport_exception_increments_dropped_without_raising(self): + def raise_connect_error(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError('connection refused') + + reporter = RuntimeOpsReporter(make_app()) + reporter._client = httpx.AsyncClient(transport=httpx.MockTransport(raise_connect_error)) + + await reporter._post({'event': 'runtime_ops_sample'}) + + stats = reporter.get_stats() + assert stats['dropped_total'] == 1 + assert 'connection refused' in stats['last_error'] + await reporter.stop() + + +def test_payload_never_contains_a_control_token(monkeypatch): + monkeypatch.setenv(PLUGIN_RUNTIME_CONTROL_TOKEN_ENV, 'runtime-control-secret') + monkeypatch.setenv(_CONTROL_PLANE_TOKEN_ENV, 'space-control-plane-secret') + reporter = RuntimeOpsReporter(make_app()) + + payload = reporter.build_payload({'live': True}) + serialized = json.dumps(payload) + + assert 'runtime-control-secret' not in serialized + assert 'space-control-plane-secret' not in serialized From 73797dfa04d5969b01aa12231aed92d2e296143a Mon Sep 17 00:00:00 2001 From: Hyu Date: Thu, 1 Oct 2026 19:04:43 +0800 Subject: [PATCH 03/30] style(plugin): format the runtime ops reporter with ruff --- src/langbot/pkg/plugin/runtime_ops.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/langbot/pkg/plugin/runtime_ops.py b/src/langbot/pkg/plugin/runtime_ops.py index 29f1a5de9..7a7095087 100644 --- a/src/langbot/pkg/plugin/runtime_ops.py +++ b/src/langbot/pkg/plugin/runtime_ops.py @@ -102,9 +102,7 @@ class RuntimeOpsReporter: self._token = str(os.environ.get(_CONTROL_PLANE_TOKEN_ENV, '') or '').strip() if self.enabled and not self.health_url: - self._log_info( - 'Plugin runtime ops reporter is disabled: no plugin runtime WebSocket URL is configured' - ) + self._log_info('Plugin runtime ops reporter is disabled: no plugin runtime WebSocket URL is configured') self.enabled = False if self.enabled and not self.space_url: self._log_info('Plugin runtime ops reporter is disabled: no Space URL is configured') From 41820cd1a50a2977c5ae779e72603314f7cfe98a Mon Sep 17 00:00:00 2001 From: Hyu Date: Thu, 1 Oct 2026 19:55:44 +0800 Subject: [PATCH 04/30] test(platform): follow the batched execution trace pipeline The execution trace rewrite batches records and uploads them from ExecutionCounters.flush(), replaced the per-trace direct send, dropped the window counters, made the execution id the record identity and records the inbound event stage after the routing stage. The ingress tests still asserted the old mechanism, so they failed on every Python version. Assert what the pipeline now emits instead: flush the buffered records and read them back, require the trace registry to be empty afterwards, assert the deterministic execution id, expect the route stage before the inbound stage, and give the dispatch stubs the execution_id parameter the ingress passes. --- .../platform/test_botmgr_execution_trace.py | 61 ++++++++++++------- .../platform/test_plugin_subscriptions.py | 4 +- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/tests/unit_tests/platform/test_botmgr_execution_trace.py b/tests/unit_tests/platform/test_botmgr_execution_trace.py index 12f55fde6..75605aa04 100644 --- a/tests/unit_tests/platform/test_botmgr_execution_trace.py +++ b/tests/unit_tests/platform/test_botmgr_execution_trace.py @@ -75,35 +75,51 @@ class FakeAdapter: return [] +async def flushed_records(counters, manager) -> list[dict]: + """Flush buffered telemetry and return everything the pipeline emitted. + + Records are batched and uploaded by ``ExecutionCounters.flush()``; there is + no per-trace direct send any more. + """ + + await counters.flush() + assert manager.sent, 'expected the flushed telemetry batch' + return [record for payload in manager.sent for record in payload.get('records', [])] + + @pytest.mark.asyncio async def test_route_miss_emits_one_closed_trace(): bot, manager, counters = make_bot([]) await bot._handle_platform_event(message_received_event(), FakeAdapter()) + records = await flushed_records(counters, manager) assert len(manager.sent) == 1 - payload = manager.sent[0] + trace_records = [item for item in records if item.get('event_type') == 'feature_execution'] + assert len(trace_records) == 1 + payload = trace_records[0] assert payload['event_type'] == 'feature_execution' - assert len(payload['query_id']) == 36 + # The execution identity is deterministic: one inbound event, one chain. + assert payload['query_id'] == 'platform:bot-1:message-1' assert payload['instance_id'] == 'instance-test' assert payload['workspace_uuid'] == 'workspace-test' features = payload['features'] assert features['schema'] == 1 assert [(row['family'], row['operation'], row['outcome']) for row in features['observations']] == [ - ('platform_event', 'message.received', 'success'), ('event_route', 'message.received', 'skipped'), + ('platform_event', 'message.received', 'success'), ] assert [row['seq'] for row in features['observations']] == [0, 1] assert all(row['trace_id'] == payload['query_id'] for row in features['observations']) assert all(row['adapter'] == 'FakeAdapter' for row in features['observations']) assert features['trace']['closed_by'] == 'event_done' - # Window counters are still aggregated for coverage. - assert counters.pending + # The trace is closed and deregistered: nothing dangles after the event. + assert counters.traces == {} @pytest.mark.asyncio async def test_unavailable_route_target_carries_route_identity(): - bot, manager, _ = make_bot( + bot, manager, counters = make_bot( [ { 'id': 'agent-binding', @@ -117,18 +133,18 @@ async def test_unavailable_route_target_carries_route_identity(): await bot._handle_platform_event(message_received_event(), FakeAdapter()) - stages = manager.sent[0]['features']['observations'] + stages = (await flushed_records(counters, manager))[0]['features']['observations'] assert [(stage['family'], stage['outcome'], stage['mode']) for stage in stages] == [ - ('platform_event', 'success', 'none'), ('event_route', 'failed', 'agent'), + ('platform_event', 'success', 'none'), ] - assert stages[1]['route_ref'] == 'agent:agent-1' - assert stages[0]['route_ref'] == '' + assert stages[0]['route_ref'] == 'agent:agent-1' + assert stages[1]['route_ref'] == '' @pytest.mark.asyncio async def test_discarded_route_is_visible_without_user_values(): - bot, manager, _ = make_bot( + bot, manager, counters = make_bot( [ { 'id': 'discard-binding', @@ -141,10 +157,10 @@ async def test_discarded_route_is_visible_without_user_values(): await bot._handle_platform_event(SimpleNamespace(type='platform.member.joined'), FakeAdapter()) - stages = manager.sent[0]['features']['observations'] - assert [stage['outcome'] for stage in stages] == ['success', 'skipped'] - assert stages[1]['operation'] == 'platform.member.joined' - assert stages[1]['mode'] == 'none' + stages = (await flushed_records(counters, manager))[0]['features']['observations'] + assert [stage['outcome'] for stage in stages] == ['skipped', 'success'] + assert stages[0]['operation'] == 'platform.member.joined' + assert stages[0]['mode'] == 'none' @pytest.mark.asyncio @@ -154,7 +170,7 @@ async def test_telemetry_opt_out_emits_nothing(): await bot._handle_platform_event(message_received_event(), FakeAdapter()) assert manager.sent == [] - assert counters.pending == {} + assert counters.records == [] assert counters.traces == {} @@ -162,25 +178,28 @@ async def test_telemetry_opt_out_emits_nothing(): async def test_route_trace_scope_is_restored_after_dispatch(): from langbot.pkg.telemetry import trace as trace_mod - bot, _, _ = make_bot([]) + bot, _, counters = make_bot([]) assert trace_mod.current() is None await bot._handle_platform_event(message_received_event(), FakeAdapter()) assert trace_mod.current() is None - # A later record outside the ingress must not join the closed trace. + buffered = list(counters.records) + # A later record outside the ingress must not join the closed trace, and an + # observation without an owning chain is not buffered at all. from langbot.pkg.telemetry.execution import record record(bot.ap, TEST_CONTEXT, family='platform_api', operation='send_message', outcome='success') - assert list(bot.ap.telemetry.execution.pending.values())[-1]['count'] == 1 + assert counters.records == buffered + assert counters.traces == {} @pytest.mark.asyncio async def test_adapter_call_without_ingress_still_counts(): """Mock adapter objects used by other tests must not break the ingress path.""" - bot, manager, _ = make_bot([]) + bot, manager, counters = make_bot([]) adapter = Mock() await bot._handle_platform_event(message_received_event(), adapter) - assert manager.sent + assert await flushed_records(counters, manager) diff --git a/tests/unit_tests/platform/test_plugin_subscriptions.py b/tests/unit_tests/platform/test_plugin_subscriptions.py index 44819f934..b69bc9d4f 100644 --- a/tests/unit_tests/platform/test_plugin_subscriptions.py +++ b/tests/unit_tests/platform/test_plugin_subscriptions.py @@ -33,14 +33,14 @@ async def test_slow_failed_and_duplicate_subscriptions_do_not_block_primary_rout started = [] release = asyncio.Event() - async def subscriber(event, adapter, uuid): + async def subscriber(event, adapter, uuid, execution_id=None): started.append(uuid) if uuid == 'slow': await release.wait() if uuid == 'failed': raise ValueError('plugin error') - async def primary(event, adapter): + async def primary(event, adapter, execution_id=None): started.append('primary') bot._dispatch_plugin_subscription = subscriber From 9186c8e43a999a99481120b223d72f8f7de54cdc Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Thu, 1 Oct 2026 23:11:28 +0800 Subject: [PATCH 05/30] chore(deps): upgrade plugin SDK to 0.7.7 (#2622) --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9d9b80fe8..c5ed8d8d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin @ https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl#sha256=b2f05302ca25b374ff8c094794573696b75a7d1a7731a0fe2eebbc71796e8e0d", + "langbot-plugin @ https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl#sha256=0a46c2b28ed4acc3c72f2e7f9377df6503bce4374232b4cd7189a42b98e96065", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/uv.lock b/uv.lock index 6661723ba..c90fd2387 100644 --- a/uv.lock +++ b/uv.lock @@ -2185,7 +2185,7 @@ requires-dist = [ { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" }, - { name = "langbot-plugin", url = "https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl" }, + { name = "langbot-plugin", url = "https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2255,8 +2255,8 @@ dev = [ [[package]] name = "langbot-plugin" -version = "0.7.6" -source = { url = "https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl" } +version = "0.7.7" +source = { url = "https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl" } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, @@ -2277,7 +2277,7 @@ dependencies = [ { name = "websockets" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/de/4c73851c18e01e6dd1f39e3743a7811c4016e703797075491b2fdbd2530c/langbot_plugin-0.7.6-py3-none-any.whl", hash = "sha256:b2f05302ca25b374ff8c094794573696b75a7d1a7731a0fe2eebbc71796e8e0d" }, + { url = "https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl", hash = "sha256:0a46c2b28ed4acc3c72f2e7f9377df6503bce4374232b4cd7189a42b98e96065" }, ] [package.metadata] From 38d5750785a7ac1dfc5aba412a561fe99e430896 Mon Sep 17 00:00:00 2001 From: Hyu Date: Fri, 2 Oct 2026 01:34:46 +0800 Subject: [PATCH 06/30] fix(plugin): report when an installation failure happened, not when it was sampled The connector stored only uuid/code/message, so the ops reporter stamped every entry with its own sampling time and a stale failure looked freshly broken on each report. Failures now carry failed_at and the payload publishes it. --- src/langbot/pkg/plugin/connector.py | 33 ++++++++++++++++++- src/langbot/pkg/plugin/runtime_ops.py | 5 ++- .../plugin/test_connector_reconcile.py | 13 +++++--- .../plugin/test_runtime_ops_reporter.py | 33 +++++++++++++++++++ 4 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/langbot/pkg/plugin/connector.py b/src/langbot/pkg/plugin/connector.py index 6071f6cfb..905fc459c 100644 --- a/src/langbot/pkg/plugin/connector.py +++ b/src/langbot/pkg/plugin/connector.py @@ -56,7 +56,9 @@ from langbot_plugin.api.entities.builtin.command import ( errors as command_errors, ) from langbot_plugin.runtime.plugin.mgr import PluginInstallSource +from langbot_plugin.runtime.io.handler import SHARED_WORKER_FILE_STORAGE_DIR from langbot_plugin.runtime.security import ( + PLUGIN_FILE_STORAGE_DIR_ENV, PLUGIN_RUNTIME_CONTROL_TOKEN_ENV, PLUGIN_RUNTIME_CONTROL_TOKEN_HEADER, validate_runtime_secret, @@ -231,6 +233,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): self.runtime_profile: typing.Literal['oss_dev', 'shared'] = ( 'shared' if getattr(getattr(ap, 'deployment', None), 'mode', 'oss') == 'cloud' else 'oss_dev' ) + self._align_plugin_file_transfer_root() self.runtime_identity: RuntimeIdentity | None = None self._runtime_id = self._build_runtime_id() self.worker_policy: PluginWorkerPolicy | None = None @@ -256,6 +259,24 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): 'missing_artifacts': 0, } + def _align_plugin_file_transfer_root(self) -> None: + """Use the Runtime's plugin file-transfer root when running shared. + + The SDK binds its transfer root to one directory inode with mode 0700 and + picks the default per process: a shared worker uses + ``SHARED_WORKER_FILE_STORAGE_DIR``, every other process falls back to + ``data/temp/lbp``. A host talking to a shared Runtime must use the same root, + otherwise plugin icons and assets die with "Invalid file transfer + capability" (the icon route then answers 500/404 instead of the image). + An explicit operator value always wins. + """ + + if self.runtime_profile != 'shared': + return + if os.environ.get(PLUGIN_FILE_STORAGE_DIR_ENV): + return + os.environ[PLUGIN_FILE_STORAGE_DIR_ENV] = SHARED_WORKER_FILE_STORAGE_DIR + @staticmethod def _build_runtime_id() -> str: """Return the durable identity of this instance's shared Runtime.""" @@ -634,6 +655,10 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): 'installation_uuid': installation_uuid, 'error_code': error_code, 'message': message, + # When the failure actually happened. The ops reporter must publish this + # instead of its own sampling time, otherwise every sample re-stamps the + # same failure with the newest report time. + 'failed_at': _utc_now_iso(), } self._installation_failures[installation_uuid] = failure self.ap.logger.error( @@ -675,9 +700,15 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): 'installation_uuid': installation_uuid, 'error_code': error_code, 'message': message, + # See _raise_apply_failure: never let the reporter substitute its own + # sampling time for the moment the failure was observed. + 'failed_at': _utc_now_iso(), } failures[installation_uuid] = failure - if self._installation_failures.get(installation_uuid) != failure: + previous = self._installation_failures.get(installation_uuid) or {} + if {key: value for key, value in previous.items() if key != 'failed_at'} != { + key: value for key, value in failure.items() if key != 'failed_at' + }: self.ap.logger.error( 'Plugin installation %s failed during reconcile [%s]: %s', installation_uuid, diff --git a/src/langbot/pkg/plugin/runtime_ops.py b/src/langbot/pkg/plugin/runtime_ops.py index 7a7095087..0c96820a0 100644 --- a/src/langbot/pkg/plugin/runtime_ops.py +++ b/src/langbot/pkg/plugin/runtime_ops.py @@ -356,7 +356,10 @@ class RuntimeOpsReporter: 'execution_mode': mode, 'error_code': str(record.get('error_code', '') or ''), 'message': _sanitize_failure_message(record.get('message')), - 'observed_at': observed_at, + # The moment the failure was observed by the connector. Falling back to + # the sampling time would make a stale failure look freshly broken on + # every report. + 'observed_at': str(record.get('failed_at') or observed_at), } ) return failures diff --git a/tests/unit_tests/plugin/test_connector_reconcile.py b/tests/unit_tests/plugin/test_connector_reconcile.py index 8dbb409dc..0b54f9384 100644 --- a/tests/unit_tests/plugin/test_connector_reconcile.py +++ b/tests/unit_tests/plugin/test_connector_reconcile.py @@ -735,11 +735,13 @@ async def test_apply_dependency_failure_raises_stable_observable_error(): assert error.installation_uuid == setting.installation_uuid assert error.error_code == 'dependency_prepare_failed' assert '[dependency_prepare_failed]' in str(error) - assert connector._installation_failures[setting.installation_uuid] == { + recorded = connector._installation_failures[setting.installation_uuid] + assert {key: value for key, value in recorded.items() if key != 'failed_at'} == { 'installation_uuid': setting.installation_uuid, 'error_code': 'dependency_prepare_failed', 'message': 'Plugin dependency installer exited with code 1', } + assert str(recorded['failed_at']).endswith('Z') connector.ap.logger.error.assert_called_once() @@ -767,9 +769,12 @@ async def test_shared_reconcile_records_one_failure_without_blocking_other_state setting_a.installation_uuid, setting_b.installation_uuid, } - assert connector._installation_failures == { - setting_a.installation_uuid: failure, - } + recorded_failures = connector._installation_failures + assert { + uuid: {key: value for key, value in record.items() if key != 'failed_at'} + for uuid, record in recorded_failures.items() + } == {setting_a.installation_uuid: failure} + assert str(recorded_failures[setting_a.installation_uuid]['failed_at']).endswith('Z') assert set(connector._known_desired_states) == { setting_a.installation_uuid, setting_b.installation_uuid, diff --git a/tests/unit_tests/plugin/test_runtime_ops_reporter.py b/tests/unit_tests/plugin/test_runtime_ops_reporter.py index 51e9c7861..c5fbd4b5b 100644 --- a/tests/unit_tests/plugin/test_runtime_ops_reporter.py +++ b/tests/unit_tests/plugin/test_runtime_ops_reporter.py @@ -282,6 +282,39 @@ class TestUpload: await reporter.stop() +class TestFailureTimestamps: + def test_publishes_when_the_failure_happened_not_when_it_was_sampled(self): + failures = { + 'inst-1': { + 'installation_uuid': 'inst-1', + 'error_code': 'dependency_prepare_failed', + 'message': 'prepare failed', + 'failed_at': '2026-10-01T07:11:00Z', + } + } + reporter = RuntimeOpsReporter(make_app(connector=make_connector(failures=failures))) + + payload = reporter.build_payload({'live': True}) + + entry = payload['failures'][0] + assert entry['observed_at'] == '2026-10-01T07:11:00Z' + assert entry['observed_at'] != payload['generated_at'] + + def test_legacy_records_without_a_timestamp_fall_back_to_the_sample_time(self): + failures = { + 'inst-2': { + 'installation_uuid': 'inst-2', + 'error_code': 'worker_launch_failed', + 'message': 'boom', + } + } + reporter = RuntimeOpsReporter(make_app(connector=make_connector(failures=failures))) + + payload = reporter.build_payload({'live': True}) + + assert payload['failures'][0]['observed_at'] == payload['generated_at'] + + def test_payload_never_contains_a_control_token(monkeypatch): monkeypatch.setenv(PLUGIN_RUNTIME_CONTROL_TOKEN_ENV, 'runtime-control-secret') monkeypatch.setenv(_CONTROL_PLANE_TOKEN_ENV, 'space-control-plane-secret') From 34b5fe0202299a2416e47b0c198c77793b71b84d Mon Sep 17 00:00:00 2001 From: Hyu Date: Fri, 2 Oct 2026 01:34:47 +0800 Subject: [PATCH 07/30] fix(plugin): share the runtime file-transfer root and 404 uninstalled icon requests The SDK binds one transfer root inode (mode 0700) and picks it per profile: a shared worker uses /tmp/lbp-rpc while a host without the profile env fell back to data/temp/lbp, so plugin icons and assets died with 'Invalid file transfer capability'. The host now pins the shared root, and a plugin absent from the resolved Workspace answers 404 instead of 500. --- .../pkg/api/http/controller/groups/plugins.py | 20 +++++++- .../plugin/test_plugin_file_transfer_root.py | 51 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 tests/unit_tests/plugin/test_plugin_file_transfer_root.py diff --git a/src/langbot/pkg/api/http/controller/groups/plugins.py b/src/langbot/pkg/api/http/controller/groups/plugins.py index 4bce79869..950a8954d 100644 --- a/src/langbot/pkg/api/http/controller/groups/plugins.py +++ b/src/langbot/pkg/api/http/controller/groups/plugins.py @@ -165,6 +165,16 @@ except Exception: pass +def _is_plugin_not_installed(exc: BaseException) -> bool: + """True when the connector reports a plugin absent from the resolved Workspace. + + The connector raises a plain ``ValueError`` for this case, so resource routes + answer 404 instead of surfacing a 500 the WebUI would have to swallow. + """ + + return 'is not installed in this workspace' in str(exc).lower() + + def _normalize_plugin_asset_path(filepath: str) -> str | None: filepath = filepath.replace('\\', '/') if filepath.startswith('/'): @@ -596,7 +606,15 @@ class PluginsRouterGroup(group.RouterGroup): request_context: RequestContext, ) -> quart.Response: await self._require_authenticated_plugin_runtime_context(request_context) - icon_data = await self.ap.plugin_connector.get_plugin_icon(author, plugin_name) + try: + icon_data = await self.ap.plugin_connector.get_plugin_icon(author, plugin_name) + except ValueError as exc: + # A plugin that is not installed for this Workspace is a missing + # resource, not a server fault: the WebUI asks for icons by plugin + # identity and must be able to fall back silently. + if _is_plugin_not_installed(exc): + return quart.Response('Icon not found', status=404) + raise icon_bytes = await asyncio.to_thread(base64.b64decode, icon_data['plugin_icon_base64']) return quart.Response(icon_bytes, mimetype=icon_data['mime_type']) diff --git a/tests/unit_tests/plugin/test_plugin_file_transfer_root.py b/tests/unit_tests/plugin/test_plugin_file_transfer_root.py new file mode 100644 index 000000000..fad2751ba --- /dev/null +++ b/tests/unit_tests/plugin/test_plugin_file_transfer_root.py @@ -0,0 +1,51 @@ +"""The host must share the Runtime's plugin file-transfer root in shared deployments. + +A mismatch leaves plugin icons and assets failing with +"Invalid file transfer capability" because the SDK binds one directory inode per +process (mode 0700) and picks the default from the runtime profile. +""" + +from __future__ import annotations + +import os +from importlib import import_module +from types import SimpleNamespace + +from langbot_plugin.runtime.io.handler import SHARED_WORKER_FILE_STORAGE_DIR +from langbot_plugin.runtime.security import PLUGIN_FILE_STORAGE_DIR_ENV + + +def connector_module(): + import_module('langbot.pkg.core.app') + return import_module('langbot.pkg.plugin.connector') + + +def align(profile: str) -> None: + connector = connector_module() + connector.PluginRuntimeConnector._align_plugin_file_transfer_root( + SimpleNamespace(runtime_profile=profile) + ) + + +def test_shared_profile_pins_the_runtime_transfer_root(monkeypatch): + monkeypatch.delenv(PLUGIN_FILE_STORAGE_DIR_ENV, raising=False) + + align('shared') + + assert os.environ[PLUGIN_FILE_STORAGE_DIR_ENV] == SHARED_WORKER_FILE_STORAGE_DIR + + +def test_explicit_operator_value_is_never_overridden(monkeypatch): + monkeypatch.setenv(PLUGIN_FILE_STORAGE_DIR_ENV, '/var/lib/custom-transfer') + + align('shared') + + assert os.environ[PLUGIN_FILE_STORAGE_DIR_ENV] == '/var/lib/custom-transfer' + + +def test_oss_profile_leaves_the_default_alone(monkeypatch): + monkeypatch.delenv(PLUGIN_FILE_STORAGE_DIR_ENV, raising=False) + + align('oss_dev') + + assert PLUGIN_FILE_STORAGE_DIR_ENV not in os.environ From af9011f2374ef366c7c8341eaae1b270ca93f192 Mon Sep 17 00:00:00 2001 From: Hyu Date: Fri, 2 Oct 2026 01:34:47 +0800 Subject: [PATCH 08/30] fix(web): load installed plugin icons through the authenticated route Multi-Workspace deployments fail the public plugin icon route closed (404), so plain icons broke everywhere locally and for the runner picker in production. A shared cached hook fetches the authenticated icon for installed plugins and leaves marketplace surfaces on the cloud URL. --- .../components/PluginProcessorSettings.tsx | 37 ++-- .../home/agents/components/RunnerSelect.tsx | 23 ++- .../dynamic-form/DynamicFormItemComponent.tsx | 19 +- .../pipeline-extensions/PipelineExtension.tsx | 43 +++-- .../ExtensionCardComponent.tsx | 14 +- .../PluginInstalledComponent.tsx | 4 +- .../plugin-card/PluginCardComponent.tsx | 28 ++- .../app/infra/hooks/useInstalledPluginIcon.ts | 118 ++++++++++++ web/src/app/wizard/page.tsx | 31 ++- .../hooks/useAuthenticatedPluginResource.ts | 50 ++--- .../unit/installed-plugin-icon-cache.test.mjs | 179 ++++++++++++++++++ 11 files changed, 434 insertions(+), 112 deletions(-) create mode 100644 web/src/app/infra/hooks/useInstalledPluginIcon.ts create mode 100644 web/tests/unit/installed-plugin-icon-cache.test.mjs diff --git a/web/src/app/home/agents/components/PluginProcessorSettings.tsx b/web/src/app/home/agents/components/PluginProcessorSettings.tsx index 34bc94ce8..d11fa822f 100644 --- a/web/src/app/home/agents/components/PluginProcessorSettings.tsx +++ b/web/src/app/home/agents/components/PluginProcessorSettings.tsx @@ -2,7 +2,7 @@ import { Puzzle } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { Link } from 'react-router-dom'; import type { RunnerDescriptor } from '@/app/infra/entities/api'; -import { httpClient } from '@/app/infra/http'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import { extractI18nObject } from '@/i18n/I18nProvider'; import { Select, @@ -27,6 +27,10 @@ function ProcessorComponentContent({ ...component.label, }); const pluginId = `${component.plugin_author}/${component.plugin_name}`; + const iconURL = useInstalledPluginIcon( + component.plugin_author, + component.plugin_name, + ); return ( - + {iconURL ? ( + + ) : ( + + )} {label} diff --git a/web/src/app/home/agents/components/RunnerSelect.tsx b/web/src/app/home/agents/components/RunnerSelect.tsx index cb706091c..08f6b06e7 100644 --- a/web/src/app/home/agents/components/RunnerSelect.tsx +++ b/web/src/app/home/agents/components/RunnerSelect.tsx @@ -3,7 +3,8 @@ import { Bot, ExternalLink, Loader2, Store } from 'lucide-react'; import { useTranslation } from 'react-i18next'; import { toast } from 'sonner'; -import { getCloudServiceClientSync, httpClient } from '@/app/infra/http'; +import { getCloudServiceClientSync } from '@/app/infra/http'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import type { IDynamicFormItemOption } from '@/app/infra/entities/form/dynamic'; import type { PluginV4 } from '@/app/infra/entities/plugin'; import { @@ -51,13 +52,13 @@ function installErrorMessage( return getErrorMessage(error) || t('wizard.aiEngine.installFailed'); } -function installedRunnerIconURL(option: IDynamicFormItemOption) { - return option.name.startsWith('plugin:') - ? (() => { - const match = option.name.match(/^plugin:([^/]+)\/([^/]+)(?:\/|$)/); - return match ? httpClient.getPluginIconURL(match[1], match[2]) : null; - })() - : null; +function pluginOptionParts(option: IDynamicFormItemOption): { + author: string; + name: string; +} | null { + if (!option.name.startsWith('plugin:')) return null; + const match = option.name.match(/^plugin:([^/]+)\/([^/]+)(?:\/|$)/); + return match ? { author: match[1], name: match[2] } : null; } function InstalledRunnerContent({ @@ -65,7 +66,8 @@ function InstalledRunnerContent({ }: { option: IDynamicFormItemOption; }) { - const iconURL = installedRunnerIconURL(option); + const parts = pluginOptionParts(option); + const iconURL = useInstalledPluginIcon(parts?.author, parts?.name); return ( @@ -90,7 +92,8 @@ function InstalledRunnerOptionContent({ option: IDynamicFormItemOption; description: string; }) { - const iconURL = installedRunnerIconURL(option); + const parts = pluginOptionParts(option); + const iconURL = useInstalledPluginIcon(parts?.author, parts?.name); return ( diff --git a/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx b/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx index fd7811a02..9b15ea151 100644 --- a/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx +++ b/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx @@ -22,6 +22,7 @@ import { ControllerRenderProps } from 'react-hook-form'; import { Button } from '@/components/ui/button'; import { useEffect, useState } from 'react'; import { httpClient, systemInfo, userInfo } from '@/app/infra/http'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import { LLMModel, Bot, @@ -94,19 +95,6 @@ function hasUsableOptionName(option: { name?: string | null }): boolean { return typeof option.name === 'string' && option.name.trim().length > 0; } -function getPluginComponentIconURL(value?: string): string | null { - if (!value?.startsWith('plugin:')) { - return null; - } - - const match = value.match(/^plugin:([^/]+)\/([^/]+)(?:\/|$)/); - if (!match) { - return null; - } - - return httpClient.getPluginIconURL(match[1], match[2]); -} - function SelectOptionContent({ label, value, @@ -114,7 +102,10 @@ function SelectOptionContent({ label: string; value: string; }) { - const iconURL = getPluginComponentIconURL(value); + const match = value?.startsWith('plugin:') + ? value.match(/^plugin:([^/]+)\/([^/]+)(?:\/|$)/) + : null; + const iconURL = useInstalledPluginIcon(match?.[1], match?.[2]); return (
diff --git a/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx b/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx index a26779109..c6b98c027 100644 --- a/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx +++ b/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx @@ -12,7 +12,7 @@ import { DialogFooter, } from '@/components/ui/dialog'; import { Checkbox } from '@/components/ui/checkbox'; -import { CircleHelp, Plus, X, Server, Wrench, Sparkles } from 'lucide-react'; +import { CircleHelp, Plus, X, Server, Wrench, Sparkles, Puzzle } from 'lucide-react'; import { Badge } from '@/components/ui/badge'; import { Switch } from '@/components/ui/switch'; import { Label } from '@/components/ui/label'; @@ -26,6 +26,27 @@ import { MCPServer, Skill } from '@/app/infra/entities/api'; import PluginComponentList from '@/app/home/plugins/components/plugin-installed/PluginComponentList'; import { BoxUnavailableNotice } from '@/app/home/components/BoxUnavailableNotice'; import { useBoxStatus } from '@/app/infra/hooks/useBoxStatus'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; + +function PluginThumbnail({ author, name }: { author: string; name: string }) { + const iconURL = useInstalledPluginIcon(author, name); + + if (!iconURL) { + return ( +
+ +
+ ); + } + + return ( + {name} + ); +} function InfoTooltip({ label }: { label: string }) { return ( @@ -373,13 +394,9 @@ export default function PipelineExtension({ className="flex items-center justify-between rounded-lg border p-3 hover:bg-accent" >
- {metadata.name}
{metadata.name}
@@ -665,13 +682,9 @@ export default function PipelineExtension({ onClick={() => handleTogglePlugin(pluginId)} > - {metadata.name}
{metadata.name}
diff --git a/web/src/app/home/plugins/components/plugin-installed/ExtensionCardComponent.tsx b/web/src/app/home/plugins/components/plugin-installed/ExtensionCardComponent.tsx index e3c030091..2add99f3e 100644 --- a/web/src/app/home/plugins/components/plugin-installed/ExtensionCardComponent.tsx +++ b/web/src/app/home/plugins/components/plugin-installed/ExtensionCardComponent.tsx @@ -13,7 +13,7 @@ import { Puzzle, } from 'lucide-react'; import { getCloudServiceClientSync, systemInfo } from '@/app/infra/http'; -import { useAuthenticatedPluginIcon } from '@/hooks/useAuthenticatedPluginResource'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import { Button } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; import { @@ -39,10 +39,9 @@ export default function ExtensionCardComponent({ const { t } = useTranslation(); const [dropdownOpen, setDropdownOpen] = useState(false); const [iconFailed, setIconFailed] = useState(false); - const authenticatedIcon = useAuthenticatedPluginIcon( - cardVO.author, - cardVO.name, - cardVO.type === 'plugin', + const pluginIconURL = useInstalledPluginIcon( + cardVO.type === 'plugin' ? cardVO.author : null, + cardVO.type === 'plugin' ? cardVO.name : null, ); const FallbackIcon = @@ -51,9 +50,8 @@ export default function ExtensionCardComponent({ : cardVO.type === 'skill' ? Sparkles : Puzzle; - const iconSrc = - cardVO.type === 'plugin' ? authenticatedIcon.url : cardVO.iconURL; - const showFallback = iconFailed || authenticatedIcon.error || !iconSrc; + const iconSrc = cardVO.type === 'plugin' ? pluginIconURL : cardVO.iconURL; + const showFallback = iconFailed || !iconSrc; const getTypeLabel = (type: ExtensionType) => { switch (type) { diff --git a/web/src/app/home/plugins/components/plugin-installed/PluginInstalledComponent.tsx b/web/src/app/home/plugins/components/plugin-installed/PluginInstalledComponent.tsx index 76e7ad956..be575d9b2 100644 --- a/web/src/app/home/plugins/components/plugin-installed/PluginInstalledComponent.tsx +++ b/web/src/app/home/plugins/components/plugin-installed/PluginInstalledComponent.tsx @@ -157,7 +157,9 @@ const PluginInstalledComponent = forwardRef< version: meta.version ?? '', enabled: plugin.enabled, type: marketplacePlugin?.type || 'plugin', - iconURL: httpClient.getPluginIconURL(author, name), + // Icon is resolved by ExtensionCardComponent through the shared + // installed-plugin hook (authenticated fetch in multi-Workspace + // deployments); the public /icon route 404s there. install_source: plugin.install_source, install_info: plugin.install_info, status: plugin.status, diff --git a/web/src/app/home/plugins/components/plugin-installed/plugin-card/PluginCardComponent.tsx b/web/src/app/home/plugins/components/plugin-installed/plugin-card/PluginCardComponent.tsx index e5e8bf14c..f53b73db7 100644 --- a/web/src/app/home/plugins/components/plugin-installed/plugin-card/PluginCardComponent.tsx +++ b/web/src/app/home/plugins/components/plugin-installed/plugin-card/PluginCardComponent.tsx @@ -2,9 +2,16 @@ import { PluginCardVO } from '@/app/home/plugins/components/plugin-installed/Plu import { useState } from 'react'; import { Badge } from '@/components/ui/badge'; import { useTranslation } from 'react-i18next'; -import { BugIcon, ExternalLink, Ellipsis, Trash, ArrowUp } from 'lucide-react'; +import { + BugIcon, + ExternalLink, + Ellipsis, + Trash, + ArrowUp, + Puzzle, +} from 'lucide-react'; import { getCloudServiceClientSync, systemInfo } from '@/app/infra/http'; -import { httpClient } from '@/app/infra/http/HttpClient'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import { Button } from '@/components/ui/button'; import { DropdownMenu, @@ -27,6 +34,7 @@ export default function PluginCardComponent({ }) { const { t } = useTranslation(); const [dropdownOpen, setDropdownOpen] = useState(false); + const iconURL = useInstalledPluginIcon(cardVO.author, cardVO.name); return ( <> @@ -36,11 +44,17 @@ export default function PluginCardComponent({ >
{/* Icon - fixed width */} - plugin icon + {iconURL ? ( + plugin icon + ) : ( +
+ +
+ )} {/* Content area - flexible width with min-width to prevent overflow */}
diff --git a/web/src/app/infra/hooks/useInstalledPluginIcon.ts b/web/src/app/infra/hooks/useInstalledPluginIcon.ts new file mode 100644 index 000000000..0d26a38c6 --- /dev/null +++ b/web/src/app/infra/hooks/useInstalledPluginIcon.ts @@ -0,0 +1,118 @@ +import { useEffect, useState } from 'react'; + +import { httpClient } from '@/app/infra/http'; + +/** + * Resolve the icon of an *installed* plugin in every deployment. + * + * The Core's public `/icon` route only exists in single-workspace (OSS) + * installs; in multi-Workspace deployments it 404s, so installed-plugin + * surfaces must go through the authenticated `/authenticated-icon` route and + * stream the bytes as a blob object URL. + * + * - Core backend client: blob-fetch `getAuthenticatedPluginIconURL` and cache + * the resulting object URL per `author/name` (module-level, shared across + * every mounted consumer of the same icon). + * - Cloud/Space marketplace surfaces: callers that already resolved a + * marketplace URL pass it as `marketplaceURL`; it is returned verbatim and + * no blob fetch happens. + * - Returns `null` while loading / when the plugin has no icon, so callers + * keep rendering their existing fallback. + * + * Object URLs are revoked once the last subscriber unmounts (or the last + * subscriber's fetch settles with no remaining subscriber). + */ + +type CachedIcon = { url: string }; + +// Resolved object URLs, keyed by `author/name`. +const iconCache = new Map(); +// In-flight authenticated fetches, de-duplicated per key. +const inflightIcons = new Map>(); +// Mounted subscriber count per key, so we only revoke when nobody needs it. +const iconRefCounts = new Map(); + +function loadIcon( + key: string, + author: string, + name: string, +): Promise { + const existing = inflightIcons.get(key); + if (existing) return existing; + + const request = httpClient + .getAuthenticatedPluginIconURL(author, name) + .catch(() => null) + .finally(() => { + inflightIcons.delete(key); + }); + inflightIcons.set(key, request); + return request; +} + +export function useInstalledPluginIcon( + author?: string | null, + name?: string | null, + marketplaceURL?: string | null, +): string | null { + const key = author && name ? `${author}/${name}` : null; + const [url, setUrl] = useState(() => + marketplaceURL ?? (key ? iconCache.get(key)?.url ?? null : null), + ); + + useEffect(() => { + if (marketplaceURL) { + // Cloud/Space marketplace surface: already resolved upstream. + setUrl(marketplaceURL); + return; + } + if (!key || !author || !name) { + setUrl(null); + return; + } + + let active = true; + iconRefCounts.set(key, (iconRefCounts.get(key) ?? 0) + 1); + + const cached = iconCache.get(key); + if (cached) { + setUrl(cached.url); + } else { + setUrl(null); + void loadIcon(key, author, name).then((objectURL) => { + if (!objectURL) return; + // The last subscriber may have left while the fetch was in flight. + if (iconRefCounts.get(key) === undefined) { + URL.revokeObjectURL(objectURL); + return; + } + const current = iconCache.get(key); + if (current) { + // Another subscriber resolved first; share its object URL. + if (current.url !== objectURL) URL.revokeObjectURL(objectURL); + if (active) setUrl(current.url); + return; + } + iconCache.set(key, { url: objectURL }); + if (active) setUrl(objectURL); + }); + } + + return () => { + active = false; + const remaining = (iconRefCounts.get(key) ?? 1) - 1; + if (remaining > 0) { + iconRefCounts.set(key, remaining); + return; + } + iconRefCounts.delete(key); + const entry = iconCache.get(key); + if (entry) { + iconCache.delete(key); + URL.revokeObjectURL(entry.url); + } + }; + }, [key, author, name, marketplaceURL]); + + return marketplaceURL ?? url; +} diff --git a/web/src/app/wizard/page.tsx b/web/src/app/wizard/page.tsx index f2c83e1dd..6e0fb054c 100644 --- a/web/src/app/wizard/page.tsx +++ b/web/src/app/wizard/page.tsx @@ -29,6 +29,7 @@ import { } from 'lucide-react'; import { httpClient } from '@/app/infra/http/HttpClient'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; import { systemInfo, bootstrapWorkspaceSession, @@ -1904,6 +1905,30 @@ function runnerPluginId(runner: string) { .join('/'); } +function InstalledRunnerIcon({ + author, + name, +}: { + author: string; + name: string; +}) { + const iconURL = useInstalledPluginIcon(author, name); + + if (!iconURL) { + return ( + + ); + } + + return ( + + ); +} + function StepAIEngine({ choice, onChoiceChange, @@ -2298,11 +2323,7 @@ function StepAIEngine({ > {author && name ? ( - + ) : ( )} diff --git a/web/src/hooks/useAuthenticatedPluginResource.ts b/web/src/hooks/useAuthenticatedPluginResource.ts index c271d99ff..3b534df2e 100644 --- a/web/src/hooks/useAuthenticatedPluginResource.ts +++ b/web/src/hooks/useAuthenticatedPluginResource.ts @@ -1,6 +1,7 @@ import { useEffect, useState } from 'react'; import { httpClient } from '@/app/infra/http/HttpClient'; import { useCurrentWorkspace } from '@/app/infra/http'; +import { useInstalledPluginIcon } from '@/app/infra/hooks/useInstalledPluginIcon'; type AuthenticatedResourceState = { key: string; @@ -14,50 +15,21 @@ const EMPTY_RESOURCE: AuthenticatedResourceState = { error: false, }; +/** + * Backwards-compatible wrapper around the shared installed-plugin icon hook. + * Kept for callers that want a `{ url, error }` shape; the url is '' while + * loading or when the icon is unavailable. + */ export function useAuthenticatedPluginIcon( author: string, name: string, enabled = true, ): { url: string; error: boolean } { - const [resource, setResource] = - useState(EMPTY_RESOURCE); - const currentWorkspace = useCurrentWorkspace(); - const workspaceUuid = currentWorkspace?.workspace.uuid; - const resourceKey = `${workspaceUuid ?? ''}:${author}/${name}`; - - useEffect(() => { - if (!enabled) { - setResource({ key: resourceKey, url: '', error: false }); - return; - } - let active = true; - let objectURL = ''; - setResource({ key: resourceKey, url: '', error: false }); - httpClient - .getAuthenticatedPluginIconURL(author, name) - .then((nextURL) => { - objectURL = nextURL; - if (active) { - setResource({ key: resourceKey, url: nextURL, error: false }); - } else { - URL.revokeObjectURL(nextURL); - } - }) - .catch(() => { - if (active) { - setResource({ key: resourceKey, url: '', error: true }); - } - }); - return () => { - active = false; - if (objectURL) URL.revokeObjectURL(objectURL); - }; - }, [author, enabled, name, resourceKey]); - - return { - url: resource.key === resourceKey ? resource.url : '', - error: resource.key === resourceKey && resource.error, - }; + const url = useInstalledPluginIcon( + enabled ? author : null, + enabled ? name : null, + ); + return { url: url ?? '', error: false }; } export function useAuthenticatedPluginAsset( diff --git a/web/tests/unit/installed-plugin-icon-cache.test.mjs b/web/tests/unit/installed-plugin-icon-cache.test.mjs new file mode 100644 index 000000000..81e72046e --- /dev/null +++ b/web/tests/unit/installed-plugin-icon-cache.test.mjs @@ -0,0 +1,179 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +import ts from 'typescript'; + +const currentDirectory = path.dirname(fileURLToPath(import.meta.url)); +const sourcePath = path.resolve( + currentDirectory, + '../../src/app/infra/hooks/useInstalledPluginIcon.ts', +); +const source = fs.readFileSync(sourcePath, 'utf8'); +const compiled = ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS }, +}).outputText; + +// --- minimal React + backend mocks so the hook can run under node:test --- +let stateSlots = []; +let lastCleanup; + +const reactMock = { + useState(initial) { + const value = typeof initial === 'function' ? initial() : initial; + const index = stateSlots.length; + stateSlots.push(value); + return [ + value, + (next) => { + stateSlots[index] = + typeof next === 'function' ? next(stateSlots[index]) : next; + }, + ]; + }, + useEffect(effect) { + const cleanup = effect(); + if (typeof cleanup === 'function') lastCleanup = cleanup; + }, +}; + +const httpMock = { + httpClient: { + getAuthenticatedPluginIconURL: () => { + throw new Error('fetch not configured for this test'); + }, + }, +}; + +const sourceRequire = createRequire(sourcePath); + +function loadHook() { + const loadedModule = { exports: {} }; + const requireShim = (id) => { + if (id === 'react') return reactMock; + if (id === '@/app/infra/http') return httpMock; + return sourceRequire(id); + }; + new Function('require', 'module', 'exports', compiled)( + requireShim, + loadedModule, + loadedModule.exports, + ); + return loadedModule.exports.useInstalledPluginIcon; +} + +const revoked = []; +globalThis.URL.revokeObjectURL = (url) => revoked.push(url); + +function reset() { + stateSlots = []; + lastCleanup = undefined; + revoked.length = 0; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +test('caches per author/name and revokes once the last subscriber leaves', async () => { + let fetches = 0; + let resolveFetch; + httpMock.httpClient.getAuthenticatedPluginIconURL = () => { + fetches += 1; + return new Promise((resolve) => { + resolveFetch = resolve; + }); + }; + reset(); + const useInstalledPluginIcon = loadHook(); + + const first = useInstalledPluginIcon('acme', 'demo'); + const firstCleanup = lastCleanup; + assert.equal(first, null); + assert.equal(fetches, 1); + + resolveFetch('blob:acme/demo'); + await flush(); + + // Second subscriber reuses the cached object URL synchronously. + const second = useInstalledPluginIcon('acme', 'demo'); + const secondCleanup = lastCleanup; + assert.equal(second, 'blob:acme/demo'); + assert.equal(fetches, 1); + + firstCleanup(); + assert.deepEqual(revoked, []); + secondCleanup(); + assert.deepEqual(revoked, ['blob:acme/demo']); + + // Cache was dropped with the last subscriber, so a new mount refetches. + useInstalledPluginIcon('acme', 'demo'); + assert.equal(fetches, 2); +}); + +test('de-duplicates concurrent subscribers and revokes a shared icon once', async () => { + let fetches = 0; + let resolveFetch; + httpMock.httpClient.getAuthenticatedPluginIconURL = () => { + fetches += 1; + return new Promise((resolve) => { + resolveFetch = resolve; + }); + }; + reset(); + const useInstalledPluginIcon = loadHook(); + + useInstalledPluginIcon('acme', 'demo'); + const firstCleanup = lastCleanup; + useInstalledPluginIcon('acme', 'demo'); + const secondCleanup = lastCleanup; + assert.equal(fetches, 1); + + resolveFetch('blob:shared'); + await flush(); + + firstCleanup(); + assert.deepEqual(revoked, []); + secondCleanup(); + assert.deepEqual(revoked, ['blob:shared']); +}); + +test('caches nothing and revokes nothing when the authenticated fetch fails', async () => { + let fetches = 0; + httpMock.httpClient.getAuthenticatedPluginIconURL = () => { + fetches += 1; + return Promise.reject(new Error('not found')); + }; + reset(); + const useInstalledPluginIcon = loadHook(); + + const value = useInstalledPluginIcon('acme', 'missing'); + const cleanup = lastCleanup; + assert.equal(value, null); + + await flush(); + assert.equal(useInstalledPluginIcon('acme', 'missing'), null); + assert.equal(fetches, 2); // failures are not negatively cached + + cleanup(); + assert.deepEqual(revoked, []); +}); + +test('returns a marketplace URL verbatim without fetching', () => { + let fetches = 0; + httpMock.httpClient.getAuthenticatedPluginIconURL = () => { + fetches += 1; + return Promise.resolve('blob:unused'); + }; + reset(); + const useInstalledPluginIcon = loadHook(); + + const value = useInstalledPluginIcon( + 'acme', + 'demo', + 'https://space.langbot.app/icon', + ); + assert.equal(value, 'https://space.langbot.app/icon'); + assert.equal(fetches, 0); + assert.equal(lastCleanup, undefined); +}); From 1b320aefd0481ad91ccdd8b932da99a90f4fa54d Mon Sep 17 00:00:00 2001 From: Hyu Date: Fri, 2 Oct 2026 01:36:11 +0800 Subject: [PATCH 09/30] chore(deps): upgrade plugin SDK to 0.7.8 Pins the 0.7.8 wheel published from the SDK release and refreshes uv.lock. --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c5ed8d8d6..21d711de4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin @ https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl#sha256=0a46c2b28ed4acc3c72f2e7f9377df6503bce4374232b4cd7189a42b98e96065", + "langbot-plugin @ https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl#sha256=f862fb6cce7d1020999211c1038b832571143d981f6b61b59ef87e31180bd46e", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/uv.lock b/uv.lock index c90fd2387..f45fb23ee 100644 --- a/uv.lock +++ b/uv.lock @@ -2185,7 +2185,7 @@ requires-dist = [ { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" }, - { name = "langbot-plugin", url = "https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl" }, + { name = "langbot-plugin", url = "https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2255,8 +2255,8 @@ dev = [ [[package]] name = "langbot-plugin" -version = "0.7.7" -source = { url = "https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl" } +version = "0.7.8" +source = { url = "https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl" } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, @@ -2277,7 +2277,7 @@ dependencies = [ { name = "websockets" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/d1/0bf0d7f852a238d46ce53dffd0bc28739cb1d77fcee1140f7a0477377269/langbot_plugin-0.7.7-py3-none-any.whl", hash = "sha256:0a46c2b28ed4acc3c72f2e7f9377df6503bce4374232b4cd7189a42b98e96065" }, + { url = "https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl", hash = "sha256:f862fb6cce7d1020999211c1038b832571143d981f6b61b59ef87e31180bd46e" }, ] [package.metadata] From 8549369d48bcfe0167626c2a79bdfe94d4af25cf Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 2 Oct 2026 02:18:07 +0800 Subject: [PATCH 10/30] fix(ci): format pipeline extensions and plugin icon hook (#2623) --- .../pipeline-extensions/PipelineExtension.tsx | 10 +++++++++- web/src/app/infra/hooks/useInstalledPluginIcon.ts | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx b/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx index c6b98c027..feaa11057 100644 --- a/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx +++ b/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx @@ -12,7 +12,15 @@ import { DialogFooter, } from '@/components/ui/dialog'; import { Checkbox } from '@/components/ui/checkbox'; -import { CircleHelp, Plus, X, Server, Wrench, Sparkles, Puzzle } from 'lucide-react'; +import { + CircleHelp, + Plus, + X, + Server, + Wrench, + Sparkles, + Puzzle, +} from 'lucide-react'; import { Badge } from '@/components/ui/badge'; import { Switch } from '@/components/ui/switch'; import { Label } from '@/components/ui/label'; diff --git a/web/src/app/infra/hooks/useInstalledPluginIcon.ts b/web/src/app/infra/hooks/useInstalledPluginIcon.ts index 0d26a38c6..5d8fad696 100644 --- a/web/src/app/infra/hooks/useInstalledPluginIcon.ts +++ b/web/src/app/infra/hooks/useInstalledPluginIcon.ts @@ -56,8 +56,8 @@ export function useInstalledPluginIcon( marketplaceURL?: string | null, ): string | null { const key = author && name ? `${author}/${name}` : null; - const [url, setUrl] = useState(() => - marketplaceURL ?? (key ? iconCache.get(key)?.url ?? null : null), + const [url, setUrl] = useState( + () => marketplaceURL ?? (key ? (iconCache.get(key)?.url ?? null) : null), ); useEffect(() => { From 8732fe3607220e164268d08139a6573553a7ae2d Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 2 Oct 2026 02:30:00 +0800 Subject: [PATCH 11/30] chore(release): prepare 4.11.0-beta.6 with SDK 0.7.9 (#2624) --- pyproject.toml | 4 ++-- uv.lock | 32 ++++++-------------------------- 2 files changed, 8 insertions(+), 28 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 21d711de4..d0adbb5ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "langbot" -version = "4.11.0-beta.3" +version = "4.11.0-beta.6" description = "Production-grade platform for building agentic IM bots" readme = "README.md" license-files = ["LICENSE"] @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin @ https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl#sha256=f862fb6cce7d1020999211c1038b832571143d981f6b61b59ef87e31180bd46e", + "langbot-plugin==0.7.9", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/uv.lock b/uv.lock index f45fb23ee..324e19f55 100644 --- a/uv.lock +++ b/uv.lock @@ -2059,7 +2059,7 @@ wheels = [ [[package]] name = "langbot" -version = "4.11.0b3" +version = "4.11.0b6" source = { editable = "." } dependencies = [ { name = "aiocqhttp" }, @@ -2185,7 +2185,7 @@ requires-dist = [ { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" }, - { name = "langbot-plugin", url = "https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl" }, + { name = "langbot-plugin", specifier = "==0.7.9" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2255,8 +2255,8 @@ dev = [ [[package]] name = "langbot-plugin" -version = "0.7.8" -source = { url = "https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl" } +version = "0.7.9" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, @@ -2276,29 +2276,9 @@ dependencies = [ { name = "watchdog" }, { name = "websockets" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c7/dbec32bd5bc4db152b71060b1d680f87afa591efa4ad5e38b1822fb7d7f2/langbot_plugin-0.7.9.tar.gz", hash = "sha256:d3a22eee703cbd2c6ed1e4e74ce1e2f0e80ae8c946f9422e015e59d13b9a99f2", size = 686113, upload-time = "2026-10-01T18:22:43.149Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/b3/7a5a26a005d2c79eb910aec9b37a19d386752e58a4136c977bc2d20438e8/langbot_plugin-0.7.8-py3-none-any.whl", hash = "sha256:f862fb6cce7d1020999211c1038b832571143d981f6b61b59ef87e31180bd46e" }, -] - -[package.metadata] -requires-dist = [ - { name = "aiofiles", specifier = ">=24.1.0" }, - { name = "aiohttp", specifier = ">=3.9.0" }, - { name = "dotenv", specifier = ">=0.9.9" }, - { name = "e2b", specifier = ">=2.15" }, - { name = "httpx", specifier = ">=0.28.1" }, - { name = "jinja2", specifier = ">=3.1.6" }, - { name = "packaging", specifier = ">=24.0" }, - { name = "pip", specifier = ">=25.2" }, - { name = "pydantic", specifier = ">=2.11.5" }, - { name = "pydantic-settings", specifier = ">=2.10.1" }, - { name = "pytest", specifier = ">=8.4.0" }, - { name = "pyyaml", specifier = ">=6.0.2" }, - { name = "textual", specifier = ">=3.2.0" }, - { name = "types-aiofiles", specifier = ">=24.1.0.20250516" }, - { name = "types-pyyaml", specifier = ">=6.0.12.20250516" }, - { name = "watchdog", specifier = ">=6.0.0" }, - { name = "websockets", specifier = ">=15.0.1" }, + { url = "https://files.pythonhosted.org/packages/fc/81/9fb420ea504c4c07af079ca4a9f59b1685a3aebd16e4e89306a2b9183a5b/langbot_plugin-0.7.9-py3-none-any.whl", hash = "sha256:c365b677175b20ea18c25ed364b55faa41298d3cd6dce4fe79f6dddf4ca34747", size = 441232, upload-time = "2026-10-01T18:22:41.813Z" }, ] [[package]] From c95b8581fc348fc01966c5dbc0af10e73df46e03 Mon Sep 17 00:00:00 2001 From: RockChinQ <1010553892@qq.com> Date: Fri, 2 Oct 2026 08:24:43 +0800 Subject: [PATCH 12/30] fix(plugin): upgrade SDK to 0.7.10 for legacy config compatibility (#2625) --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d0adbb5ed..d74467161 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin==0.7.9", + "langbot-plugin==0.7.10", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/uv.lock b/uv.lock index 324e19f55..d43d71643 100644 --- a/uv.lock +++ b/uv.lock @@ -2185,7 +2185,7 @@ requires-dist = [ { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" }, - { name = "langbot-plugin", specifier = "==0.7.9" }, + { name = "langbot-plugin", specifier = "==0.7.10" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2255,7 +2255,7 @@ dev = [ [[package]] name = "langbot-plugin" -version = "0.7.9" +version = "0.7.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -2276,9 +2276,9 @@ dependencies = [ { name = "watchdog" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/c7/dbec32bd5bc4db152b71060b1d680f87afa591efa4ad5e38b1822fb7d7f2/langbot_plugin-0.7.9.tar.gz", hash = "sha256:d3a22eee703cbd2c6ed1e4e74ce1e2f0e80ae8c946f9422e015e59d13b9a99f2", size = 686113, upload-time = "2026-10-01T18:22:43.149Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/0c/65013949dee8a8eb38c29b264f5246f2581d6851afd688211fa97b5a0d8d/langbot_plugin-0.7.10.tar.gz", hash = "sha256:5e6b5fea75e5c3696afc0f200a013ef0d63b1e83713935fd05552796c11fb1b2", size = 686205, upload-time = "2026-10-02T00:16:00.397Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/81/9fb420ea504c4c07af079ca4a9f59b1685a3aebd16e4e89306a2b9183a5b/langbot_plugin-0.7.9-py3-none-any.whl", hash = "sha256:c365b677175b20ea18c25ed364b55faa41298d3cd6dce4fe79f6dddf4ca34747", size = 441232, upload-time = "2026-10-01T18:22:41.813Z" }, + { url = "https://files.pythonhosted.org/packages/64/d0/9b9c9c42d3f54b8a2ae7537e4c27f1f407f57dcb9f4329aaa2a10e4e1ceb/langbot_plugin-0.7.10-py3-none-any.whl", hash = "sha256:f01957745d1db1a4db630985079662d136da91bfb353883bb59a5d99f6065fa9", size = 441294, upload-time = "2026-10-02T00:15:59.091Z" }, ] [[package]] From 2e5bbc8bddded92290dba526ab4c4931dd67f30c Mon Sep 17 00:00:00 2001 From: RockChinQ Date: Fri, 2 Oct 2026 02:16:38 +0000 Subject: [PATCH 13/30] feat(web): integrate ecosystem social preview images --- src/langbot/pkg/api/http/controller/main.py | 16 ++++++++-- .../pkg/api/http/controller/social_preview.py | 20 +++++++++++++ tests/unit_tests/test_social_preview.py | 28 ++++++++++++++++++ web/public/social/cloud-v3.png | Bin 0 -> 118683 bytes 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 src/langbot/pkg/api/http/controller/social_preview.py create mode 100644 tests/unit_tests/test_social_preview.py create mode 100644 web/public/social/cloud-v3.png diff --git a/src/langbot/pkg/api/http/controller/main.py b/src/langbot/pkg/api/http/controller/main.py index e19552c65..dfe8a685a 100644 --- a/src/langbot/pkg/api/http/controller/main.py +++ b/src/langbot/pkg/api/http/controller/main.py @@ -174,9 +174,21 @@ class HTTPController: frontend_path = paths.get_frontend_path() + async def frontend_index(): + from .social_preview import cloud_preview_html + + response = await quart.send_from_directory( + frontend_path, 'index.html', mimetype='text/html', conditional=False + ) + if quart.request.host.split(':', 1)[0].lower() == 'cloud.langbot.app': + response.set_data(cloud_preview_html(await response.get_data(as_text=True), quart.request.host)) + response.headers.pop('ETag', None) + response.headers.pop('Last-Modified', None) + return response + @self.quart_app.route('/') async def index(): - response = await quart.send_from_directory(frontend_path, 'index.html', mimetype='text/html') + response = await frontend_index() response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' response.headers['Pragma'] = 'no-cache' response.headers['Expires'] = '0' @@ -207,7 +219,7 @@ class HTTPController: return response # Fallback to index.html for SPA client-side routing - response = await quart.send_from_directory(frontend_path, 'index.html', mimetype='text/html') + response = await frontend_index() response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' response.headers['Pragma'] = 'no-cache' response.headers['Expires'] = '0' diff --git a/src/langbot/pkg/api/http/controller/social_preview.py b/src/langbot/pkg/api/http/controller/social_preview.py new file mode 100644 index 000000000..03e530ee0 --- /dev/null +++ b/src/langbot/pkg/api/http/controller/social_preview.py @@ -0,0 +1,20 @@ +"""Crawler-visible preview for the hosted Cloud frontend only.""" + +CLOUD_IMAGE = 'https://cloud.langbot.app/social/cloud-v3.png' + + +def cloud_preview_html(html: str, hostname: str) -> str: + if hostname.split(':', 1)[0].lower() != 'cloud.langbot.app': + return html + tags = ( + '' + '' + '' + f'' + '' + '' + '' + '' + f'' + ) + return html.replace('', tags + '', 1) diff --git a/tests/unit_tests/test_social_preview.py b/tests/unit_tests/test_social_preview.py new file mode 100644 index 000000000..c71ec6341 --- /dev/null +++ b/tests/unit_tests/test_social_preview.py @@ -0,0 +1,28 @@ +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('social_preview', ROOT / 'src/langbot/pkg/api/http/controller/social_preview.py') +assert spec is not None and spec.loader is not None +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class SocialPreviewTest(unittest.TestCase): + def test_cloud_has_crawler_visible_image(self): + html = (ROOT / 'web/index.html').read_text() + result = module.cloud_preview_html(html, 'cloud.langbot.app') + self.assertIn(module.CLOUD_IMAGE, result) + self.assertIn('summary_large_image', result) + self.assertEqual(result.count('property="og:image"'), 1) + self.assertTrue((ROOT / 'web/public/social/cloud-v3.png').is_file()) + + def test_self_hosted_is_unchanged(self): + html = 'LangBot' + for host in ['localhost:5300', 'example.com', 'cloud.langbot.app.evil.com']: + self.assertEqual(module.cloud_preview_html(html, host), html) + + +if __name__ == '__main__': + unittest.main() diff --git a/web/public/social/cloud-v3.png b/web/public/social/cloud-v3.png new file mode 100644 index 0000000000000000000000000000000000000000..5ab66f53f982b3ba373e7600ff4edd5a3cb21def GIT binary patch literal 118683 zcmZr%Wmr^Q*B-h{KoDu9yIT+tknZjd>FzG2yFqD?E@5Z{q=s%7O1g8TzfpPKU*A72 zuG#0TeXqXPI#EgrQW&U2r~m)}3IG6K0|3CfBf~=f(?xlR27mzoK1jS%^UOY6 z@iNlW^x}OyaRIdqk669YPTle|tee-y&>oZ|$Cl3L8c!7)mW3S6wQn&q>*u?AS)=&@ z->o(sVh`!Sz7Z7@i^LWDygC2j&@&FAe#Zm>yMQk4OyB4>QENLh$r1_Yed0>6Xvv(A zh6s-FkBM7vQ-orRAyeF%XAJL*47@!{S@n97vIJ$@R|<}Ki2l%C{9UygUDX!ENKNOK z>+mEuViZ1$w`ElS@#*+i%fM8O;%%-1b`D>8v7v83jU#`F`uGCopwAF~qUD@xz~hq& zVUX9i&Jd>pQT(PfsT7fq{cNk$a1_&|Sx@!uZ=xeb^X=UU5Vl0@g8Iql%*lfn?Joi* zFp)w5qAOs1(v&C9#Wm*~wdHPeXR(2g0`5{zroNKZyMJ)g9ecD#PvfaiyOw_IF#JKTfxfu%AWE zuNR;frN$t*F{G1jxp&exh=m{O9;H_DZmi~>?i{xg?tDE42Dy;Ce5y1QZbsPfoKoRs zStq6}dlnN6Y&j^q>U41&4UE%05Rs+VUm_0re5PY~Bd6{zD|fX+#7~KyDT6G%mWD7m z_A-I$O?2qWr<&W&Pak*9`c1pwNXbRvD4b&FyI3*rec^^nR|g)madwZrWKsL)*^xuh ztyCN`n%|U+oT4}qW~vQZG>>FAe|6H?SZ3ojS>R1yJ8qG+8p*Zhlx$}Fqjh$EC+t$N z^YuLL*p_nwWJegAK_SWL#=P8VkuVj>mik>0T^f>)kCoy1@}ldej*q9vyegq;G7=|Z zKV~JpA%IiT75t$_gIfVDt-Ac%4GRLrZ{Im(T4bcypR8=BmY)saW%z*14y;j^P0`}t zWySIEsKpHv0z>P0e;t{B0p~g?w8+B!_@sjPrTUiO#Q^+(nn^i_cQ%Vxp=j*)QaZQ@ zz||azSGgt{N)4DdY<-5AvmCU~?pYy*qn^|5t8;KkHHx{q^DzU6vdVs?Jb81kSpVekU#YB=+xaab2OZEc{Q+%N}>=7U_XK zk&H!`VI@gdwuzOmTbj{m^63z9+6L%TaDyRuh~@Jm_FOHobXv|I%#XwgLj*=7x1W~549w80G(Joa2Pq6qhqP@iTeDqqn&g4 zRa&V(RYEKo9}-OMF&fKsRhkc^#0)_B*UGE4#N(uaVnTZzMSb z=$9^qstK-0CGaH27aPew@Cv600cn<+aX~7CsvTb(O7cXg7_rmaS@6`8;qDT0!3-Nr z+!twOVhKq@3(X`7lJK52Fudgn8elx5cdsD&er7WoW^(j^z#u@j=6(%EhdyfVhq3`+ zOFt{mF?T|qos!Z=WHnuxp=1s>Pq!as?HUflm!RlYAx84z4x-KEU)^Occ;oZ9IP zRqjamJD%_!KLAigx;<3a;dD$RCwua~1R~kK!AAWPk+RhU_a=32E=UoSZD~zM@A=E; z;delaH(wpxN{jViKo&k8GzVm|c>$A+WI;T5lE?*%3Sw-%f^@<^WB5ukOQ?_~+syC) z-TX82-t$o$I$wKT8l<-C1B`lPonW8&jE zw7ak(NhOGJd2O480x8SA-F{?XlzDyq+GJrwE~RRRLl2i(nrsgA2JXBhN*sylD_53x zE!26Ns@9+aAd?-EDo^Rdh_Rnf0|cp@cwD*1+3@CDVm2)5@3Sm?WSadf_N+OS5n}jq zy$`cOE%czcK}0bvTZ_>Wx^0`A6pa}LZIZ>Tk9t``7gnuDURB9-(YG?e4_PSuWn#AK zbk0S-{mMAms`69x$WodIrixDBk?cG>9~k8-`(_;2Q`5%ZHBZ+ta1wa_oy7i)zjiW) zs(Ei@#Wf`G74apb?wq%~3U<+av%9Y%dm!PW?&NLgE*}W$Fym5-Zxb zyAu!1qnTsUV|Tba^Ehjmeevts3jLcZ1q^ zs8}KFd`hb{b= z@yLwxZO_z937LGe{V!+MFy+tADsl@Qt4A= z&5QW3jtVGUx)4&mDX=~p0ETdzvy{>&q6oXNBX@e7ZW-!8#v`iRoRq)RZhSKz;~OAJ zkzkYQ6W~)}Za-NVI9_RQtQEn4bsXs4;Bqn7AP4#R)O5{e)BqA%dM3Si5I2&n~kwKZ7YB7vTL)V zVv(p+8RPMSNC5~qa%%!{C5mqozdb28Sqf$g`?>9+G?IkS{9Ju(y07$5tzAfI_36BH z80>dTX6?zVof`9zsI|}Nwr_9>BFsHUu!7556Z(}N6-iu?L+|@2L~DR3rgHT5X^v#U zuI1Ot`~CASFF!nPC38Zxr%p*aOn+vE>DCg!D&R~ z`eUIUw&?JnrSdL6R2l8bc%D>j<+487-u9@5l)R$^ad5gQTvR_GM8DE6QL?b=70w|c z^AVxNr_+xhVMh76W_fUqR6->%wxZ3Ke~+~EK&<)@L$iM#BGDaGff9lBsaLOVwzi)N z(oABZI&@Q(h%&)Dv!b4p^HOrEK#v`Hi~@^e+;(d7!(@U6T+#U=I{p}!^1{N%owe~a z(Uw?|;M9I?p^jS>JF=+jzKL>fl5J^N`;-pCmDyfx;baC)El=&NX(l2C{=bMBz9)U)};m>hj!$+&k?856{) zuQnIsF8vQ&_EIr^)=_9i%w}6r3j|pA>Yre;wgxk{8(kM2ACLJnEwzN5V~7{N3k2*; zH}HhA4+)1T(r0fV*C3NsF3z0Yw7f&4RPoU}4UyS`WNHW9zcLp_lrnQSyFFv@nm?cd zQWn1B{DL%42+_T;3S;_HZ>?nCij{cI zonYDo7PN@=K12@b4m8ihXn&D)%Y%*?!y@6D_Rq6n(0A#k7QoyzB<42h>rA4S{C#g$ zd{!-!O32hJIq9PpfZYY;HeucHGFuKlvd7oW&??LaNT=yIfaMWzZR);SwLWRI>PIi^ z*U#@Jj38R$jwilETcATVZuz2~x;_nF&8zRd7-9sTU;UguPK#{MyO*E69b-q(tdFTk z8e_t`#kYn|xPgpR+3qLkvC>D4EuBY7cg$QpCmIZ$;PjP5!3RF>(jV8`^N*z_2GU+O z=wC~jU&*)GVQwrqb*xWQ7c7f3oQk%17#+n%rnfN4J1@$`IZqYvon2csF1}5qE||4A z)b07sO-A7Ah&eDK>X1q*TTY_Qtx;#D_BZas5^ctEpu_s?B$eBL?5}whvtBS5bv|q*8Nt`C%lUSLwl|GWIUQ~F;efa`p!vRW zZZSZKAOb6yG3|YcVLD96DUx~Ps`F>qkRf9KijUP%PTR2xNFe3w+iWlP%FUg9`1jK} zMW4~+PjsBmY%M`qNe<&BkD-Fw#pG=ct5t>p`|FXBchxBZQlWV|t`l)%hVIs?vfnU5 z;?x)$i#b@`Nkr<)Mxubg2J2fOfuXPJ(sC0Y@2+T1g5wC%P`e0U8^lGh`H=LknB&># zyGESM1X{gCTuvVPOu#X%tJ}!*d;?nsl-$bb4@EV+d+4!`(tZGgpX%%$s=gnN5CxZ{=6y z{WEE=@pLI+QkZTyQex~BA@h-Nh0a1m>O+>eWxWDS0eCl(^(gPU%Tbq?e!qua?+_4G zG}hvluoiBK?gh~+<4}VdL(inD{6}+KUhF-vL-wNU zr%3PmL(7+j_XBDAf>cAAdfUI3S2a-aS!_PGIO{A4U!*;qg>$d+*XZ^u{IM%BJ!FEf z1<{QiCAhA$za_HO)Wubg!v&q^I%eHS$J{&3JRC-^k532UYKDKpcfZ%%c4(v0a^kn7 z*Bk=St;QdDe!1f$yV^4Nv_YYnfyqZ|Vf(4F^-rTwCRXa}K(G`o$JUBjl%!ynv(qkY zZ@s1guO=`jR|-IbdR+x&Iz>~Ymt)Xjk@3Cj1{|zt`$x@iI&2wBcxenDYdj&fFa*5P zr6pUc)qFaVZ`W4quTymK_gAPf)!Pw!bclR=v%|iy=lLnWAe65%WOvQSG}f(u;a1&km_XmXizR9OyF9!eNGg4Iff81 ziG|gsU@V0^j>KBMIu$K9B6?|0OTZCBX8=*Njha+vMm!A0%)wryn*%d??g9Y2bZ9RN zb$WedF_duc^~K&w*gK*a2Bon2ESLRAG*X5n^SewwFH5KI4?*zFTx*8svN?CBm;hs-h4daseb@eB@H~YF4AVHN%`27ak1d5?Hq(dYYpu68)f}L;=IHemS;pSPg?s$zim*_`+!K*>U0Nqr+_@r&0e*34B)~F$b_M*NRgy z58zzse@jJ&v8v3}DQ!tva7l$lS23a?`5kL2Z20&?+BSsNepQ>>@8x@(G#{xIlVoI2 zt7(L!N9Evfh^flm$79o+l-0ccQPh-)ITZZyEu(HbBKM!7$XKD${$J?*DyYL9I31%Y zG_GE_+^S+M95^S}<(0jYSQuBH3bwTU!8Khe?~Yokb$7iroW{e`NIQem&q-M>0BoDT zj71m*)=g*UBHlS<)1BE%Ci_4(KCgDJpS95dQXhZ>`a)DgjU`m=KpYMU>NI9X1)yd3 zd7Pb>yH4rAhJ0G9!4dP{^3ud~S-IGqjI@7V{En|I)@WuIAfbdR6ZC48mu3z6i5z>!C&`B_Qp5a2R!dDiJ4XT4uX7 z$jMYI*+J~+sU?VGS&JPvKF_SLTk+(7Ua=3vzE3svS)UtnDy-Ip00zRQ2g4B zK^-x8$Q zZ(grTof^D%+64M@c3ivZRn{~Tnv5>dr{hSry1fKWuqIr|y|aSYLy9JK2P(A`((xN`Z6CexBd7aBFGstw$}FmV7o?oEN8$ z$)|0875#MST9c;dw(xjv$W-<1Vs;DSM0lH^B1v~!EPzILyt+wobY)w7&vH67eCHf? zi>>uoatx$}E*L=j#V=>Zk;itp(sQSBvW)9p^qd6ByvS)7yy}@ZPp88Qg!J~(h`)bO zUBGGya{Pv)%$kG{ZsjDffD`ZS(V$~{#lP&K^;#}KsO$|W=VC-uema*%-E!b?&`jQy ztE>}AM9yaVJpvoxgSzRubkPCC?av-lb*AJS_`q%8 z=Q34&Z{2W9qfky*tR@Y=a~aL=%7P*Gs{i*f=;P7_Ucah2+`2U}U-k0FS~4aJ>A0w< zUySkntDEs!2Nk1I7LlJy-L?HjZ~`0gzi)L}Kk2Y)6}3*fn(jy@>aE`goV% zVjwtdR4{#JvlUi#Eqzc+h_;??YRBVFd?ce83@-ClY#`U2;Z6kb{ z(4PFTQ-p9cmTbtqL)zH&Q6mO+cgf)AR-%38ERw9Y1^ zN@y-<;5vY5Dq}}36a+xB*YHnx*RK?agsexdmJut!=NR-|nivN?|2EClJHOPkRby$& zDINZ2AmxLzwdUQ4yL`DqOWb4nF$|6Fi`&!;?xCHb| zzZ0<3d(E`5%zJ%I2;{~3&!^C$%V`$kay?mM3u>9MX0f~P%z8H+?d8h3WNI6;-<&lz zb3g6pg=^{!%y2BfDiX7R56We~)7$rf=gSg;@XhB9>r4rV<|nUAb`jt+t#$LmQnhtv zKXsk;kdX`r)CfPu9iIl(d9fF{E20!=gk7!0-}@HuT5dYd6RBO@hBd?jqrIX1XnifwW|IIksRoZXhPV0yLl_sH3=k%W)}An+?nXN$wm1Csk#&u zA?72$&?E9|)a*=m23V;8IUZC{<%w}4LW4J8DJZdxGGe`BqZUb=3&`}GKtg;p?ZPul zWAhL#!*qU?Tk)*fM#;0dQN!6R^D+-oKW2fbQX^Q(`@a6hr}_X(c4XU|)8&&m<>+i|->)4`tI;j+&|<_Vky>IQiEj`}vceuUzTQQ`&e=K!ZB#SxlGq782Ti?B)$Nc{76}AH|2{ zSJY!5++KMm+^MU7(SD9~azQdMFQ~%cF_nYYr&k$GiS4bZ{1Ng|7Rn~ky&dx*rzIHJ z-YW;EheJ}-B6eUY2;=Jr`Oe^qU{;=x=L8I2tYNPhI?qWGPdSS3j1gdg62O@Vw&jg8 zl~D+8Z?;H?saL-0j9qODLp%?w_sk8FG1zJCIY_|CjVzgKmnb3YeC zf+gue%;>+ShP?c0Q?Yk0JZ_@%m(OM?K0|_G%M5 zJ|p4DXsVPLvutbWa4O1#GoR|=&y6hJc3RgwssG5uvCiQP2W&6i(GvRK_8B!xqF;{` zv~RTh=PO5I9?AR|$~TX!2%lNL>cQA2``mznRBBS|bCaIb5?CHxwUHmk!0E9mxd-e2lx`i%xte zNEUhBdZYjw@HU+R-6JRDJvN87*Y%mK? zZG9?46>rIcxURd+9M$M<*?_Mu5$mO^xFhn>fBV44{Z~vz;8@^4-d|36b542taxHiP+|B*_1_#TPm|~6GE-^mQ%3Rv{MfB)(;DSLzIFXOo%4P`I>SJ+66dY_wgTVozrjv1i$!5ZOo8L_q|uX;5_aoxER>joiN z`v9hg-#*oKm;DfLbO6m|>^X+}({Yue{|^!oIc4gtQKEaAleW4Pg(f9jP`2Tv~=X=S9KUCtWtT0JY&?^RH5)_bli6xeglcafn);9!_!@b^>qw6Q6a}}_(DAM1O83rPzEr*K85n$a z7vhZAD)3mqmQPF!21q`-VN>{h?a9&pr7z@h23bC*Pci}D7r*bW`;As8Qn0sip7HOm5ku5HiUwZF}txCyu}k&GRDA4j5E6U$5a>}iV zG-bM4Kv={rI>u~W%f zm3P3dNPXQJDUgGnO^7m}=agzCft6y(vheL%F8LeDdf+=swQo6AKFHL6$Eqd_<;=K# zACLM$RivB~{X#VnDtKGvw$Sp_fwAq3wqV#01Ce&dg^V#E)yIrY6%S6DS7-tn0++w| zB{Kq;^v!7CNl(Y>&jS$&QWjI*vzlj9r6&|+m&wEE`Ut3=>@snSU`jhwUlPkQ=+FZO z63-ux>E{^7^W9LN z;cC5ay|?qox*1+0DM3m9VlWuln&bgDM*fv8SjBzp`n-2ljz<`UGUKgC%ZyDpg%i9s zy}*lHe#?zZ~mD;&6bQ#b9s5wE=f!kt1Mh&w|32O!yniG0tm^ZIy3t8VWI6AU;d_gyA z4eQX&mzGJvnf@-22k-qU2&zZzJEb}&W=$Eq9a|4!#MQX(HCh^=!WQEsc%vC?~EM_}i%o-I4&s>9`cP1ljB;I$K8k^1K4 zCA?5vn;E*IjJCx;>^OTPn~RZ}l;%mJ$Nffv>5(zE+_Icj0bcfIp;(L9_lj9OSy@R= zemXDz>1L++@%O#?k@XB`#p4_vr;p=1q|}?VS>fMZK`Sv2Eeg19mU~&-EVsY8t2v=` z1HF=AP{Kk8zOpP~fZf;r<#!5(MtRz)y%TKdhfnknad_T+Gn)7*BE7Dya2xeBVxyH|X})-zB^eVQ!=c`cW6 zJqEDmbh2jcecL+4Df)3?Qu)>P6*ctlrarR{S*yB%YH5{9la;-76$k#z9lJw(h=Lv0 z>J;BkClz^&7@R^GAT5LJRPQveu|S$nja7ViVfw5OTNei^Cm0hFcluP%2|5K_L8N?V9`T3>N|dBJrX)nv@ulfsPK30aK}KDIMEPEjU`6 z=VooFc6Q6;Vu$Qz+VFm+Q}IVE2bJ=Yz)$p@dF^?03A|7?7;B88iB1y?aIH(tD@*!x zW7tVV9q2B{3u@ z*Y!5y3bKOY6qM~r5p%8X3es#lEfo8gPw3Jb+WWs)u?K+1mCTIPeNVe3H8dS;{sA-{#c4Z3CdE=i-1JwzH zyUqy#2ds_PvV9v4!iBmi>hpoHY1sVPAzjUCTw3TafBn~H8*P&-rP$p5PRiQw{neg~ z6o##u^IR}d1S!wF@quog@x_|8mtu=OQww-K&=Bc_Cqo3?W_teDwDJMn#h%Z}D)6bs z3_3IXujC?u6C8?$)IxXILdl=4I@mQCM=X-_!#|orje-QK$_=h(8P%o#0`R=4A%`-l zEQw43dh`Dsm3XLdOQzRM z|IJ4DMZi*TLa}kfp7BCy(KD;0`g-;j8>&pve*k|4&CEu=ZE`oY)Qp?>E2=``zkX4` z<7e=BMeo zjbU{oJxTuaC?}fdM=AbU7gCX*bMU3N{|Q|ef*Nenv$&OAr8bEVyHPtEe2M3OaC-jwOAp}QA|&(FUaP=Ye;$qg+)M1MRu+ty9`MNg#qHCxAhdsRv;6NW(tocj{R|CB;$LW9Jj3Yl z7slxL=c5#Vk5WCCNp6v^%KqY7|JU!d7$N%_soS=G`<6Ps{B5*mp9vkULu|5@t)-aQZ%%aPXtc{E{X{cALy|K0Xh za7bGAPn1D>)+gCtwM)K#7CPeZb(Np5+e-QO^mDZMs_@r837MadGX6WN@O(7%@6n0> z?E9~4n>0MD)&Jj7{O6-cjj_+64dx%<&rW_-kXe`_qfXD?3yl&u35IEZB4a5M$cxf< z8+?aFEbuZsi0Ku+3ET7&g#;NV7>WE(CU7GC7u;wqfnoqaX*i4p*7XP}IwG8sjKCSH z=(BtPPNbl6wW)vf#YvG6U#eA?$k-3KNBVv^92|O!lbm`M7anR#+#ieZi|{h^*!1R! zGGm@(W$D<@mmr0IrV*kyrDs3qTGFPzx=kYc1!LhDaD4zouZ6`E6+JQ}q-CY*C!|NP zu&igQB*N_vE4H>n)3IQ+&V5^Ly!g%6$vc}hlN>~VrYQWo&(i-?8&@KpU^w~NIj;ht z>qwh)-)IIs$zfN!6E!K z1OJ2b3Ijop!HV-3e#&`tFh!*&A})>~+4g8|8jr~RYJhkkrJDtFPUJUvnD=k8&`VJN znM(R4qI>eI6Vr?l8|Crf@)4MuFf&brW0fryo@GER3PAg9#si^a;8CajQ63JT)UxC; z2L`Nqy3F`e4tRIuLgUDofpY+!s`fO6k`nNth!=#fssEIkGPfdZ`siF4UnE^YicSfO zL;v3B#1A{G%2bN14N_g#U=o~ubE%zE=&G}q5ir56xq5p&(D{%zfOS0mj?Px*AopLq zkh7mu-WDo--;JKk)~w_%9=~FCSd{r09+4UyYk#XlHuw9G^^?jb012RG4p3Tyt+SPL zkNE;ta&YU8`{vbOW;()-EzKk(RoEcUIHKjhWn0~Gl*38DWE#h0J}mbs5)XM~1dKyp zLs;u)H|LOFsWHJ#yykCd$p}3ig-z(j@yy^T~TU80Gv-st){hGd~Zbk z8j=QyNh}Opw!ya-;#8DMD#!L2jlXUM1DpoU=isCnJ7)FqBuX(d01e9TO6|gB9QJk} z)8<6(x5Rq4r1~u>CDwc*OKnw#s=B}fRG{dKue`7%bv@tcfZ@2zORebN3GlhBl%>1=^K=tuX#U)hhI#dq z5(s`Hg5SE;TK$Ia++?j~g+35HIXJ8Aw%*t~B&stRQIU^Fl)R{CJ${m>GLx%HM7|K2ujZg{R1w^1IlKryHZ@9TQ<2FcC*ta)FRCc28aBANX0-1R zEc?EQ@dD#qT(mH;&}YldA*1nhlFawUp;BHB8!MBo?+xX@#GIsSw~)!MBQO*l49l=4 zS>8cAV$XZ{Ze^$Xji&}}>zr!{LFv$)$sOElKN|I%5&pr@5UQg3!sT(UO}CT#O$36< z@>F3_8&TtUy!T#io@1b^IA5o^3){Lv;?NelIi)*>bR_-t4>GhV;2Q${i%ej^f1bX$?=uxrxUdQotCAS{wd3*q{vp|hX*E= zEhO+te;k|WKN@9sP>A9u18$?Q*-T>q7;i)o_6|ZOBTRrQs;t8|dl}MuT=d+{pw5Gx zC-=us>M4m-Dvc}!rTB<&us}B~-^bTvPdSwbWN*IGtxD_ELk=I5a>VY41{*+rM~ZEV zNM9fl66iFr)|~T=>5(5ZlYJh7@zYkX(4I=l-25|>r4iNSEY5?W1u^t{isEFG{^RzD z2Ia|L8;5wMY4y6n=g_T`FJF(@4X6ZyH-liW*v3@1uzu{>!tJN1POHYNXshJTwyuqe zK;|l4UaAe-uZ42y~XNHJsAU6<08{5*QGD%zkgXjqt(b#^8F%qPOrn+t3^d!rHt?sY)X6w zbLqW`+^b&s8qV&;%g@Obo!CVVXr)};m<~eN36NR%^#kw?E$22!cKEpAC5N)Y$^7?Y z!93r97KmpuqxXsAe7QQkdgY~_+W}D1;HkCHSLWv-Fc1Y1%~Os%OHEi1J9+lQ!e~10 zlbgsD?P4fc_opsL(s*(}4$7ZS=X{uhx1HZ}X{B1WKKfU(s}kz;=Sz&@+fK@IHczqC zHDn{Du!@tYNnWC#i^}n%iLXx{N1>+wEtPmrRpXN+f1}Su{p!;_QqHg(c1#;vX>wJn zPpZEE9W7P9b^;F%5Ta$ji#c^qL!I)C#;)n)p18-HE`S1?=-HLFz`#7Pzy-- zD@)x)ps5VU^383QgpYxz8$fO(b7bwqzQf ztiT}5SnYQJT^uM)%@*LsKui#*a~KOTysvT5$O?Rr_Nkw}of=6GII}zXt(L;4M&?|Adh5M0L-9VJ}gU=s#B4Cght-3KN*$8n`+xEAj-%Kd3UuR_%lPbd58 z4g9W#WQxi!Lt_LtgG5p8_0(?~k>P)%WFy+%V%6PNB;*y)tw;HHnFdNPG;sJ`Pac3T z(O68;wb}nkgzmRGnIZ1GmUBW<^TafBjKuKJql zEq|^U@(e_>L!Kh%)Vd}-5YNB3yn~nMMEn&QI^t%lU_g-)lh$y_51wuE!jg`sg%BX` zdnaQqPCyjQ$pbUw9z25ZIq}z}BnK@EVmJ>)BxpI|=ZF9@V-bkV|LSDQM(k6UoW!j2 zmzMQK-bd7WGP5sb>0QqO{smR=g^?(|4Dla?OiV)>{hfd0qKO8{WP>g04o2R*KGS?1 z=hXhe0j+qkN4dyVhncy3W0pFtU7+mu6wx4j?IO_b;vz19d5AmKca7N$r2bU-~7U2|^R>7IuGWe6k*|1UM5m;9BfuZ|kc5@fZ~{G^13XfT8a zul0K`c^S7yp+_ntFgnRctRAe)Djru8vR$~79pl_rVebbaXV%B_8c-Cuw9YCrW+^?| zc#Trz+PLgDG2`Gla{$iJxOX&#Lou?l6XlwQJ6?!r3Ev^TJ5N-HZhaYMQ|Bg^>;K+g z0=MrJ(dIx@H-%p*FbW%ywq@?WGE&jm6SXc#h8b`E>Fz0^t#S^6{TA{Qo*j!_B{4Fc zUv?N=EKx4v_JVK-ULZ0&@4fbOE1r~YX;yWu!KWQ(!rtoHw~+&cNGkIsa()qz%4eW; zz5@cpF6(rLRdymKSpaI0HvMQJN(@p#~dp{-!WDe$u+dX9mL| ztI^2V*~Z!Qh+fTkD%lhBiOty-(-^O{ct}|GqquRUYvbg@ zNN*nS!cs&ji|3t;c~oAD<5?Y}!ClYdQ`C7(fi_6shRw(BE2F`a0SSD_TLna_GJ#wZc+iOs2c`UZ<{Lq6+K703#P=+8Bwv?eV2iTbXC+q0lyYC7Ni8g} zY3g|$={{s&MXG?-`)E7&+TrL=BR5{+@MrxZ(y@OU6!~;<(_sTX9Z;&k#(YiG?qRIw zdDAz5bvV!AvCwDTcw(_&3N1TUI@T%NtEx2Zann;USvwv{-eLNgi@bfHG~z=`yHS}- z=mqA2>-X*kuI(pj)C33{)T9gR3LAEZ1gm|P$fi>-p|hv~(z~9!xZ{QV0wKhE6)F`T zW?qn%@78Po0=u&m8b0kpv$#?imfqYQG7jV1v1^zADx37z-`SF3j2@S@br{Pq#X^R% z!NkA4J`IV(iy?|B@lN4fqv3P0S$)z7i~#TZy>`OAaw|spa?Sqw^rM)wSKP)Z85N&ap^xTjFWgHN<_1H6;&zpLIdF=VP&GHf~6+BSPrWBv# zoxE*z*53%<@YQ>un@^!iHpM9YYiMaII$}ANJr>135wp2i*b={=w8bMu{%K%&t6`0) z@xv6(uWRSAs@!1v@TKa*OzHJa62UR+aUYptu+%q0>6F#D2PTBDwIHTP*H73fhEerd zBlLcT&<*h&5fyd8wsx?ZKh>{5wXn?q=@A+v7aNMOK=fk{{hO6Y7uP!2DAm|*1KS>{ z!!@tQ+!{?8*3NVHNr2CI;A#IyuIrCpl=L2PzT7l8MKhCb4!1MrntFo1R2fs%r5Mvj ziUR;JkbLCG%UXYDVzbYBs@)u$-N)|ooYAoP;)cxl&D|DFiw;uSVqBSu5}bQ$V(_F9 zR#vw;$eg@LQ_sh8c=D<+YPB#jx3c>_wjiv;TFVLK0wG=|>I;E2-x5bwPO z6gic;MMC`SXtzUoZ1F6|&@B6Z^QW&0*lR+?inge5y+UULM={i}OnR?d@G~bVG<3Yl zr7Uu0bcJ|TPt)aBiNBkFN>pWiDR%U0jr}vScRf~a&5XzU>F#w1^NTLZM~RIcM&39& z7`(ME1k}92%g?c|2ZfABNoauJUCV6k6KqH&iSX87lu@p`chU)3swEx@y|G8Fvt42A z*`8NjE;fG_H?&#z*qh<$v&IE~C}Zk-BsusH(*VSzPtlOQU~RRyvOD3cEp&M^XBw^% zE^6tAPj4!-0`)(?^G{jF-^>gWG-Q_WT+%A-ZF1(8oTM|17!J41EpW#tgtMAO9^8sT z7IzwAH^NhkUwyVu3OMT>d&-q(l-Uc)g0q#Cj63r4hwk*| z!>j066R!fTjT&hF6u*s^>YN1nxv4o(a(6)_OS67AKd{g_ZE;;mz}CUb(%$paUgX(U zB7UmBPZIwSGLIO$bgJT@FSWJZgh_=Nv#RS7uE_8*^Q^ob$V3*`}?>I?T-@w6v5j*4B3z znnfvIf3#+V!R(Os&zMExz}#!+CHP23qK^DcbGDN=r39cBT1InWQ_;6Qh^vus&2>6C^%n+ z2rY8fqUs-v1%}20``Jps?<+~Icv^SStgm*c7|!T5UwPKkzBNr)F~p_U{ncVBWo6lunM6VqbEV$pIs%BE(V z3YF3${YHX%d8#j|s5kCo+C$~TOS@%5IM5NmNP@G?(r$CndMC+c{>6ZfB($X;hU6Uz zAf2)OXw2^<4;Xvcy3HH_n3O4N==ogjxmjh|x#HyseDyseJ|h`4d0xNz&}lr~by#0$ z77X1s$MulNubEYCxW@dS`;|Xso1ip& z0Ua3w_B@rF+zT}5NHS8hKZoBY9tU0$M?O&GadRky%DMiE+)f81h-9C?n^mU!r7jC@ zpXeGopbI}lOakMpC_Y)egQXTSBa#S6aE#q9wI2=vc%5vBO^qNh+9A5_^Eo3j##C+I z+3AqnAc|QSxb9)l1da+#NPA#?y|({UrS3VVV6A_=w=2}-G262R3PS#rJj&+MJybfL ztUgR-xGAv5Uhi@EVEsYP8kbjYwS9eQlryce?S{mxR*A$=`^xhxqnfW>o~) zEO>qao*Z`@ak&i%Kw}XwBtCSgkJulmcr6cG#@)2%lTlmH&66;zPJS6JnlQ1OmQKUIr+CSR}g>_RM@sulG-$iW3Z2ELCIiVT&CH!vnud7`_oltuN;Au~JPq@J}qO3mii zL6qFaV9G-#&1RA^q3=Z=8UvUy!!w3=o8Ebwko-hP>R-viSlSN$)?9k&KtEk0%XKau%>@sLk6=|mK!U;hXtvBKg;b{k>^a3LO3jn-*Pq;W|~_^TUnOFHVaeMTVfXE@2@?jyYs)Fr;34 zamh`h4EH7>-H)&$bwL(l7r+w@dW%|p%w9ih+;Z@w+WeVV?Pb7QHLUf0=! zV0jTH;u!GqybzuwpU-sRx}-BVS@1)S)#wj_=z}-dvy*EVcS!-Bzkkb0mGj4l1uoKP z%4K&Vha$r}`Z!$Y#WjmG72E@xN+>X$TBVC=-f$eF&-gjdXW+Z@S|=`1N=0x#!*&e~EkZ%&ce4nwd49^`7712C!t%gTjd_^I=z& zMo_O(K2TUsX)jPE#5N}hQ-!LQuqWq_5G;AmaFQo{&<%1Oea8u75Qy3#a-KTux6m?R z4WDr7`gVPugNu?ph4S&I;jD>T?)v;-r!VIqO3IrX1&DQC>i||^>xf{D{glmePCt^Jy+AK{uP zPRTr^;@o@-wBq_@DBKZiw2w#pZ_0p6EH=;Q>9A*J*~_L-Sxx2PjRvt%|AV`jHvwWX z80(,PB%rgpN#X_4u$V@&H>X=3I_tT78giO;el-ywE;lYsu&p*-%p{2|9H9d)<# zym(UU_zKr-#)2wkYGHB2h+7WsNybAxH^2h`50m5wut7KOU{ z9>HB@tksJ!sWHmlyFXs``Va-t@}Fipqv;-@o*w&Zom0ywQGdisnpCGf+l#sKq{V7 zWjT_|`CM*N-bO()N6Hc8YO4i$mXuDDa*@j!mr+QxDvF;+vjG7CY9Z>kskyeBTC822 zW?bymvc&CnU3Yo%qHiR0_Nb-{iT8~gcO|#qj|_MP`!K2O)Y%Au@h3F96wR>M7@SaG z9d+7oUMi!e^SrO7q33csL7+#y=|sVoRw{~GfI9EE##YyWdaJ!5;vb?H zEINhR!0!|AWH8pHIj|dD@%LWCkmgP(YiJ#pP2B}Y=1v_OGiCsV=E5>GMAW%xww^}a zwb#_8Xat4V*2}U57OVVYe<~Q|Q6*FUWbQY$YI(4N5?-#c zj+gwI+Ia;Xh)|wAx1eVO4#x_Olzjz>KM?_c7CJU<49_+)Gs**H{ZOXGdKc|Kbo~i$ zRi+I-JMjaBicz!G(y|xrCD%Ajpy_giuonz2zcLr}vR|n)`%Vp!v}O%Mo?C+|VgvjT`S-u_TNYY@lPmDEUxh*<=ef;|!C@NI zLr}VJPvAj=OhZ@JQ+5q@7teXZRJz)Ps;Ah0nGmO>gopW=&41-P#9!I|-ur&UvkM;0 z`j~s(7}mY@CACMvYA%_CpR4+zJFOh-eq2(=mR6Lc=vP#N>Yg~(6@-|YA4AfJkcq#w6)2NfRW!Z4547;EE#5))oX#tIhYtOB!-o2_n; z*^sZJcJ$C2cjwyL(_`mcq04=1QA!fW5-1e#s#V@s^boT`!W*66B+zyf$|`R?u0djI zdZoP*Jhv4xQ42kbNdt+DZT_KZ6a0W=rjB9tsd2C{f;J-)ViqE&wZ~oah?EN*S#Pdg z!!!7wzj!nX6*gX+LwfVm#~=hdIE#B z`%>vl+`v2bS<-D95y)jHiX#`1D0tP6%DQ40{=m9o8lHgfBU1+@!fRF2W@QZ!PtLwU zZQIjG#rNf{AuY3Xmu(BPbVt+HY!(V!9xiIqfO&SYBkR|sZ|_9rC)TQMBV|-=^tjoejePdbZpwXC&611_+Ly(VWGd@SFiHU{V0wx zsTWCD!EuU$U-?iV+*(r4gF156DyQ}8TdNZ`ZUZKyE8zzbK7PJ}mG)QrYbW8v9gIVE zBU2)Oexe}o`5xNhX_=r!N*Lz_Jy-RG)l z-59-^hNG)0`uW;)dwmK)Pm9&*PqE+UToXY}IshaU@O~pA4(kY~6*HmyDCW3|k+Eu*t^pUKnPD2s;HItD2nV<11 zh0i>@7_fMEJww+xoS)LGQt{XN94=d<^nVqEwQ-*nJ=5A&@UpD(?ct>QjJ*+X>F^UB z*G>}{%+`kHZHzoyHj~B^O?~45@k<1DJmZ?YXL`bi-soHNXh#p*R6d{$V>GDL0V~YW z-?{-g)cu8|s*HTMC-s82Pcn81(`!;NW=Bm7vf8Hf3~U^pu8yc=P|8vhA-oBVIF~`O z(-yU6YbLI2H8os2(9`geddrdOP0|rZwD8LFl(~Kj#E=w+U1sAoH3w1Ig$_EaQbVd! zh$IY4Z`zieVf3@50rjuyRFm+nOF%Irk~+l%lsMQ=v$Gd)f={tiG?SC5a`08R430+g zH(tFzWZ+yBne0|IrM(2jCiTnlTQjmQ%GpvjhGIE0DNuO^2nQ&AYt>ST`%|e~%6$kC z5*30M)K&BOu+CQy^GFU%!53B9nU9OmuBycL`nrcWwDs1z4B7q$Y2wQD1c+8URl{?Z zjb?q9Mz0{1t62wG7>wJ8%@zg~9${$aB%vi}M--;+R*sR=i z0k+CV^EvI*tets<5n<#=%57)H*hn6>JDF`nt~#K5NN>1rO_Ae!4*LU(MqUP1XOCOJ z>JNLu^1}p!@webU$`(j>(7Qt6GUt6e&Ma2OU!RJbQ5~Uqlu9rA{SoTRWobWIY_PD^ z(W!@9hajw2CmUg|yG^pKP4-6OeEb|CDhs zK`8+X+4ILl z8AkP3-)7+Sp;Tg-eLI0}3FcJL93Ip37?Cp->tDzjcS<$C{Y6zT)}OanXW+i@UG@qx zi{9f-eClX!T3$QG!_~6W4V;p6-}V?eX=fJ7r;rw2xR9*q?+a|FA#d<~lEWfgqxZz= zeuK=|G|ioQM)}4MtAFWeI&a=zW%D7vwb~+gdJPULvDI~uu&3Lk+-Kok`F%GjEf=yW z&i3W;4l!@j4makOZoEB+C=NI=TDRsoF|~=GEy=b+0BE!AIoZeVC?ifK1)Md3>pDO8uaz+}AUCIC!g{%fp=<^503X zbgs(hB(hC}x%k{FDcL|)w9JIA_!VSo zhuXl@rFuQz)xj!CbeA%uxN;@9zlM2acGsq7WUz~bNplzfP5If3MUkZo)8X!QL)*qq z@FX$G({I$a)`~W!QFrSMC(-aXsxoFs@j5OOVZ96jF7}@2UD-w3nd*3(+6xmiRARx*m#Vm)GgLa~)%4O+Xew(+?EATRlCY>~SK0OSw*Neq|Q0#9& znb^_uI2d`__`z7|s?^?c(g(7;Sq6&*n3`%3NV%E!HsxFpzwxDw^Bhza6WpbCWRs91L5kdh~9cKOv^1m2=6V=IpTTBI$8&*|xMc{7u9o zkMfI8r}CDnedr$IZ1dP2oGv@o%btpr4M0K7yt+N%)x0zdq<#xSn(A#w{04hA&`!c( zTTQ}I5|$aA^hqWw{PvCkEz1tdVARK4GCoz=lJ}>Rz;KYp(%CS?du>F);)r$M6ZRxz z*aXNT7Y^#!?~SW*nj&TJtOWc}jXygT<*h-INYSFgF&#Nfa{tXsLIsZfAW-U&TQjbO z0A+8#P(31W!l>Kp0&loBDQUWj9H`)pW)+#Rf|rfBcLB;*VJMX!b@Cdi^d1?}bZ}O` z%we02$@A7GPRsID#Q3Q4;S`ymx|tElwz3<5!(rsoFec789_w0 zaFU$H_&`1UZiGh{sBQY~HDv<{aTKmypZaaeg<>)~;nQ-yt5@oGS2lv6L>6owJSdxk zg<&075>2?==y2TwIa)cRfaub{nT`g(&H~aVroZo@C@dmaHce(zNWYJ-(fo*t9f*cq zNKb5D-*4-o0Yb;u=SdX*WW~BzjVmIJ0?)E!W^WJ7Uql%?MMCzl*dG*aFkoTRJNM}6 z?HcQDYZJb%TyYf^#x_?+P<(6f{}CTEHCm1G@eIXiPN)epm0Zjz3zREi+$!p zcq125X-`ZjN_24w%8gmGA3;`^p~^j`@YGA3c2QwJ4}qu8f7|Idd1?0)*XgMzRrH~B zC_q6+X3w(GvwO<*WAnpZ&Avj);rd*VuRzUCfMp{u;s(zfe-!eoKuG&6V6cXjJ@HJ5wlZXk=jOofC z7*LW&k}LQ7FG;#CpyXhA;&k2h9pmt#*=0Bap5dXa!$$u&G=s&UFMx^ou@jxmJwf$~ zkPgD7wX|T^{Ps(D*t$_L_mGE$RgJj>S^o~Si6%Q62lzZMP=lW94n%UaY|mznXd?SU znso5x5)3i1l_|6gJSYcPEKbJJUsNh*sG)!gSog8+3>jK5zKVNrKsF`<>}w#6;F*Pb zTc}7}LZ&>Ocke3DY;-B#*=0a(qO!tcHvGo*83e33{Iv9U&)v!6Or$q18fU<_39UP< zvPrLudVJGHJQ6Ur8b9&&qtlqhIO{*lqBMO?z&mVpT{e-yGZ8U?A@Vc>Ppmv{%ABjX5OCBQ-;IJxPov&~Llvtybd)(i4bRds@-?S`*_${_w@igt1 zFsTS=Aze5$S1iQ5Z-Glgr`f`;I=(RhSa{WG&a>=feb87sVnIj5k4xWgKRS>@SI879 zc<1qUkwwt+nZRs2bZEMh`X=YvSHAbj)3~Zt1rshxnBmph7xEJJk*X4XuJE6di=28~ zE+hTN^<@BuRQy26I`yiJSN^0`RNoV+k}R8glvIl4C3#Ev`oPP{Ps@8NH==xDRIzW4 zN065Y-_JNiFpgAmj^K^Q@bp{L7B_oXaee7dI}@Em5exL)gNHF^uNB}7PD96ARQW0= z(7HoZOmn@w^_)LWxi_yDjE$2l$zJ-jh>qb`o#S!94+gt!P31ddm!2*MR4#+}_R5U! zJE=AD55y7h?+7;Ao;`Exoyvk^2a1q2T79-kU8z2p`+bCCZEdD=H1Q(Qr|sa2oeUPXu*$w?|IS-d7#c5Zb| z{BWdAcWyRfPb{qSFtjih%X>Di{NXEusqA4uJ{(FEI0<~zu&(=H=; zrM8yUG7Kt}zh#O6-8?)@%T@7ljgwcvKB)QY=|@Nu9TsTok~<@>*E4$P-V@Z2eb*&$ zEG;i8<%bjaw=T^z5#!aSwSKZ<&f8E~M!oUT$dM8i3LR^$gobk3#QOwVXNf-xurkoR zhB5)fI$}581Ry>`(Q<>lj`uyw0fs&!)A`}YOH&*mrFfJo()0IO^PYtDtsAb60eNP` zPc6*8ENh=*UrSr0sQs8M1w=?Demwqlnh7dm~qUCBQ$aH=O69_sae*I&jd~7HI z42+Z?o@TcV=x%L?_(4pa-U5?JAxa3bYPi+axxxGx1?c+~G1r;d%8^ear)9|8Pt!d_ zRzfFZf7rN_k+y|L4ZEx@WK(x2Cd8qzy<5DA{H2zvCU=G2o%CaO>vGhe<1!+DL!(b> zZ8i#P5Ls)D&B`JGRn%GK_l-136m(S|sL6v33g7=VoU)n$2m6{(@ghl;=yWRMry^9p z=wJXiQWDM{qaBvxsh$=~SiYO>v#1u;*744a&&E{%*lAkQH@L;jv~BG;BZb3hE&2Yb zc6@<&8$WQq%!Q#>^tF)kF+<&8<7N7aZq>e;w8;tEtgCwf&h(Rl7QOhb1d{(o=v*-= zU)6ci4o!6-f>Z6J&&Af8qkB#|qRDa6Z+B?^A&UL>L)M_)tP`5V{+QX=T`u6fDVSb4O@OEIl4)*n zq}Ze0Xk`q%lhyY$SWZm>-=4dY99?PAL81o4&v%BFtFH~P0#$g>vp`4Y#A~iBtxFmp z?y3Y@T3FjunSk1jKV%Oc*dUgGq;U&?^@1aIdqm(B+ z!|pcO4z#|7L!vRC$b_)1!Ge`8<=dC1205&vvu$20)3;4$rlUb~=X>00*InRzA`Q;r z7AYHEC8si@k^lzKy07O&%+{6`44NtISekgOE3?o4f?aWwa*8=Vrd!l(XK~A!M z?BPm?Zf340Pmr%83@$jvwK?57?1HXiixP>c#sw8?M4S>fZ|RzmtAKFY%Q&aph^+Oq8Be;9KZl;~LmUO`>KzpUXJ9Mz(&q_$g)* z2(G9cg@w1Keo}qw&Ax%C!uMv{+@>fl{ZjDekYMH97bBdSsvzeONMpF2dkQ{`v!dDD zthM%{TC2>zGL*|neC!($vob#TEg)OGY{I$?XyNKI^c*frPjGAUcYk+689t&3B&d3< z>trVgabYl}kcuQ8fDYieHi9yTVKd2NS#`#GiWAs8BPkCTS8jg}36Q`)z1v|O34R0F zd*kvI+MvSb^Gd77r)%DFhBBS;ex`6g?Xtv#MsLu0_F%P=%2(2|mnDKv7*NwQJ*mCT z?}XDqT+4mwNLIbobRjUP zj?ejd>`~iB9XLX87LvgTZiFZu^D^hopUa7LL3!?2n^qTbjD-d{pcJd@U8-OzWb%X6S?R!8>@oKdd?CyZOeTMpN8lP{GsuCqKAl)x{gXJMl%YyFB{{+I`5EHWUBDG^cn(;aJ8`DeQpgAXI&s$#rj^ ztGxPEtw~nrq97}QXzuVbp!J3kQ>h{ueXl;^8wu4ri8WWejRT$QOVR5&-Ev6Ix@qeu zvLb8z-XY5JsQhNLoDt3zCCcGM!ODha?%An+*z6s%xS+Sq!NfhP-4TkD1P4%E=8wL! z3+!n5oCsGXtnj^|T*RB4n*N^+0k%K4yM;P8rNm~FP`c>`Rb{!X3 z%7l}^Gwsx-3qz^H{u=>0s=lGXp%s}Zkn^~Alj;3DzM7QSiUbs1xbUAIfxwUXJ~NsK z_>Poo7E&~(Wj~p9zW$jCWoT^Dl37552sGKr_D1n95e4-x^-jnr#D9RQLCS83WURx= zA4;F*06QOtOwSECzF|hx49Pi^DqQ1ahJgZfQJ5>Y*T5sB8Iz>*%V5dKDnAPlook)f zt{4zpT@o`CdvUqZs`wAaIFpi2DvF}LeGen_BLE>xq~MNnjCB<^uz`#tV}5m;jmTr8 zc$-)KO?RFr!@wbciM|^LfZ5_kx7_(E-mQ5mxNY%xq6&dzIx@R6%BN@LE>@b zoN&0<%E=@lgX346*1L&_QoGR0`}WDhl5yOn)}gq@?gIll0B4V1b2zq=;YWvgJ(e)O z8;T@EZD1*%LBf2+&<~I&_qA=1)%6@E2xxPK^M0fKv=lv>2thFTZhw{a4#=IH5OV}n zPkU;F+K92q7C<5_^pHY3q*v=cG(p7uuZs@dG zm?OLpF2oBEziz9eVQ&fE1m4$;kZlu!2y4iAWz;cH?MT*jyOV17u?Qk~wuq@<^AXE& z*%QhT*X66r^J4kq9)TU$q0@=6+u<p@y5^4%6aa1L{jA&36Q;`oc|jxV~9wjYcI zj92{8!I{?HgwH1=gGlG6eD!r3jZZ1dxY`H$JhSJ|)A4%NoQ*21ndGV;v9XUBbrZlw z!o~)w8r-cQl^q-o4@s?S{b^S6?t9tog_I_rr`n-(XVJ>Fsq=6<3p19p;hdLfVrh0j z-6=yhbYl1HP(D<}wVy)2)*7)<(cF9u7P!7N5*4PITV#W&wakRaL4u z(~|df#r?^<7eZ16U!I5()@Squ&4vf3fua^_4!u^RV}zB-sseM&P|OeFFv^mp-1j?n z<}D6eErCYHu~z0O%V+7@JI{+#JVOt^)jp#5Yw_1DTwTRtK{F)`{Se)9=j&~Ii6obQ zv{oj6J>6Rgy4?gRaIj%nZnfh>G_Ckc8>Qhtg?*wgoyt=bTa{SngZ}k)l7qyXX6V^M z9CAVUlXQ}s6dr4Rgz^?#G)}fsZ=qOqi^$}ODVAGid=1)$JqkjVsN^Ae!?za}p>#eh z&OPOG^7}F4c0zf?0aFS;wPc?c{VjCn-aUGByZ5UGY;)a*c{MN2pt7q;T@n zN-TRwzf_=i?V^*_iq?5K4dcAN%O#S0x`RCPqaaui^$rgI>FCn<20cUaz=rOtEk$%< zS}M+EolT4eTwZWtoAGSGeg{bOny}y{16t9tyk;uBSDrRWhQh7TM__E%d0*WGPP z0AKzQ+!tsQ40b%+l1_W+PC5|=%g8ytg%-1bC8Dm{Yak9C~n7TV=FRHBrOsw>^5R3Bpv;Dd@oq_ zyc$&isw}{zTg*UXr-pkNrQ#qagoVy3&%H%cy^!dAQ)_yRb7Fn~=czH^zj7CGeYAAv zRrX7g-RjAU7>Tud>>s6sEr%81OgwI8_67;e98GyP)9 z0g9e`iEqayRK9JD7C1q4Pc~@xCp*(VG_dG*l z<8}DNq%IQ!u5jjtv`%A5>^Dk+AJY`t!1D`+S)VQaVsyO`M3@R%H=(YC2$QGzxCK7c zF;h&|6vKLkU!k|B>A=1It_!YDb#peFA@k**yz^FDk3~vvN7)hfH$LLm4!>1IWAV_t z)^BBW>i_NzxVcd*uKm*mSKpvarVh>XAfxw*O2*@S&34u#JR>36D}QWf58x&kzw=ij z7gKD{R8y;X18_pJ_`tayuQ54fsZ;;9IjHRdO*)r%m`=bUggei5C_QPTr6|6cxaAr` zd^^9wTFn7KZFm%4rg?OtLl=8&D7du_i}{SEq8RAx2HuJ1Q#aSb`hkKrwO`4Hccnq# z#<^~ZG`Anj6gZ=PythFFXck{WgkF_Ks(C)0{|NUp!Q5BT3rArI$;Xj9`b?WdXZG=E z=*2b8#cB4Br5W@pPuMxzid2hSu47}#Er^}Sn7F>DNLM`O$J9FK!0V7!|G|E0!GAUr zQQL00h~Umh?p;hqL3` z38%tK1<3JZKXTa@`Ckfc4Y-O1$mg69l#E16PMza{UIl2*A66Ds^*j}~{8TuaF;{WB zBXR1)YDWkm(#>yKE>qiIrtk9zL}`w`XeIJ0?CSYE?-Or`#q(u``{9p#!OJ;2ftx-p zV>$G`*%js=yU32;ge%qblZP&I%69FxRkmv#Nnt!o9GZIpr3Db0{Bhrre~FK9^)$W!L=W4S z^}6Rkp7R72bUQ+nwj|6X)vV;;gC5ylA7z)-R%9Sol+p>wTjOMP?sH1#l6(u-i{)wd z9r1S^AQE)n`(u)@(sb6;Y|8LClg)&t<$Sem4mX~!?Th~T3%oL-(mcX+QVAN~Ez-fe=HsvdxfP(r}4NRyj-o|cxL zv~syo&&Gzed|ymrU`ie>$EA)%4XyT@{ue4Lan;&m(()enE1bQ}u-DA-mV(PlWxi?N zhDV(Bb~o)U?}`)j-R~Do%4ylQe;Ba3&D1|e2zlQZj_L~>rpW6P069OgoEkDrbXFPN zt+bT%`MR~tR_&Smsb!|&phoyP=FbWHeCRAtS*>ZP%OV!k@8Ix7vBvmeSLYqa*FxQN ztPEaH@1|(G;g^o4tp;*G<%Mb|{y_9Zb1fI#rJ$hu{4&JG$ixlB@<1?5_Gh|F^A*&} ziefS}xkA6Y>9BXs9de`1=~Iw;;bGw?+i$_8H+wnto;-t0tAS5x{_8MC5MivX(8I@J z`G{i=L-~mdZa~eRUVqVF2S^MQkrP={y_X?94V2CG^Se5xl^NbGNF9`O`ZGw7vvS?KuK*~a!#&t zZgKHW-`Ga8-tzNO5HVvMoo@>gZyqj_;zG-E2jSb-kw@P*isPp{bYK%bc~I?u^*PSW z5)Mvs-cX_-P*~Tq=R}Y{)Q^)ddD}xexCd3<<@!(WS9UJe`RVp}%pDb5)}8R~oa!N8VYTEFq2j$p_utt!HtwrUt3ISH8@maYtgHB``r<|New@@x}F*Mytb=I`3&T$bh z^&<`Iq+a2?(bJPu5lTbon2)d*P_3=lX4R0st!b?du}mOT`b4-dg7P_n&G>t>L)mj} zN;1x{g;RQsh@I;_xMXi{xWkKdOZa7q!Lr=R(kiY=R-KqMt$RMa=pvbH`>7-*CKlkO z^Kp~klM&>=h(f$IPwODe#99q2B!e*Tuay2mci@oyjay-gB_@Mlh~D$4v5BdHqap@0 zCs{|R7K1zKT5aGJOoRHUvEmfDlb(u=*s!M2+A2_*0JGi`%$O|hS`uv6meVZ;cVLJW zt-K>B@bzMjLYVmr6`1+O5ipDaD$bTn)%G8G2Qo?0(xAPeoKb!dROo-@(({%sV` zhIK;w;RonXhL-2aav$ zq9xwmR+d;e9J-h3b?eg6^UdfOf(zUmviEOZ7M9uX%;PjKyHrxvxlW8ZT(yG>#q`A1 zzp9X-lh1&^OmctvSQKC4L>~pVAYQ8a*ydqqY^<`@!iQCRdH#Ad)^^$bVk6?ZOc0;V z-)v8l_GM7APQ50unzkuXS1J8s8w02?fAF9I2RWgiYI`uggYlGxp zQGPQI%Nkq41(qHQEP6j9^RF!LdsbBlU;S;|PM0TTM=}+sGa)3E-Ap2W?o@yrICcx@?U0jxt83uY}Q0+cf6yifxede5tr|IBXUQM zWjP7gH`IuNe7{D$WT^F!OsBP?#;QI8moxwwECdmFHis__90CF)lEv5~VjyWnOegg@ zkBa-sqg<-1rgYjUd+9{?E=+TG&B zG7VsvdI0wjfh+UinM^{jG&@5fNB>2`4xh%eW@-nmx77*ZIP?}>~n%Zv9p zi~gnL!2RmEhXO=YzIoeTtIW|-T)r55qp4^x{IiOGVU{6Le`!GNa??Um-_Cmo^AzKQ^qZH9ih{+;|brIq2&Z-beo=4HC6*XdL5eMJ5) z=6->*<)r^>EYpv3NqU(f>SRtms94=#)UM-+g>kofnKMsa+!&p-h zh{LD+8rT|xA#`{bU+#-Dk>XJ3OndCWRlAFu%SbQ6hg6(e_N)kjr_{eF8)SoM1d-n! zY&(~b#rfz6xphhv%1Qq&GLoc~#?oAx^3s}^7w`Qx&zp0=**59kqVl&@pHmin(Z7+u z$J6KF^m!&KZKS3)E3wF;qJv?}Om$#*y$q5(Fmc*=Z_M6~rUh6E*Yp2_RKm52HN8A3 zo@UMZmcEkl(An9m&!aPh_X;1X0RqAl1t|{^(qiKbg@LZ?_KLS9qTyd072gyUc4|zm zY4a}Q)A*$M9t#iyj0eK5p;Jj}dDe;=$0R{s+Zq`M?yKGDqD%2}2qBaHcvr_sEF)r; z|CK(}>uXvMT1fl7_?^&T9}9t`A-imo%uIni6phtSyos3yR-8~#B~kjR*dlK+yA8vgOEj6@71k~Qb}fujnX z=>HfpVPGxEq!%El=M~Z45(=EKe{dPZ!NsK|Iwio44DbJ=;ozDL5=!P}7heDjs&b&X z?jM6p<$oAth<_Pm_>c|29}30(s}t?fv`N}Ew&HYgxx_8~4}bRh9~;JD`yc*nq51po zJ`O;Q(O~eeK>h(JGzkfdXZt@JnhzNFKlqv6f4%-kZQ3XYXNT3ifZdQ~{yj71?+9}E zpAY;uUb+6Up&Wni^DC{Xf-;6X1+4Ez}0uxo&nN-N#|Gqx3c6`H|3us0z04_7?H#FVW zg5hQlL`MYHZWM3Gw4%s)So_ge@v2a)H^bD$a4(|1`ZP39cYWFX-?+WuWLAoSLpPEr z;8D6U0_ZH&PJYLdRPa{0vC$@#m;p}da}hV|s`m)ck7VL9AIgKxu9mHG!vqEfI#^3C zhdsq*3fG9w9Y6f3Ivm^yNXn*0@mq}}xxyOLF8aI!@BMy*>-AvHvo3v?MWUvaxWvFc z5zyQ=>LCfA?*bTb+aHn0s?QQkv$zgo*!duaX|BK@r?h5CO+`DT&771kcVfFqQtp9jhuTeCOtBK}_f96NH~AJ)bs3RB<7wbhWGsfj80t1m>XR))9GZFV}X;E zm7C+F{CSx|$HuedgYvdZ_VTJ}*))5vsq_-Y=Mg{cy5o@S(TG($uzEjk%!oL^8!nyD z-X2Ng_Egt47S%cMkIg9qTXI$6W+P&nuVq}Lh!9#rG1{!$pG2g4c^y@!aM_ z>13^gmg~WiSQZIML)>0NtAUyU_0oo8wFK~0o)tr@8?!+wiPxi9qw>-2zIwzy47QC+l^E2%|c2oAwau<1!AK zo0`^r2(@qxYHD6aqxds^+|S&{_naa4UMn$uFKG@yili46K9A1`I7V;B~2<& zI#E4A``44mABSjkX-ZM-f4$g9i*VQv&hSDG!4I@a1m0+M2{kWD{@>~ zwA^GoV*Kn`Sil6ji*`mxMWg8b3Zdo2YQNO)-3#ea^&;)(`n~QV1WslBW(EZX z(+9i0Orf;Cn++sFHL1QzQu|nUtF`{TyE{Ga{+#zc4a&4Ss6nY@U-?4RLV|;_prOfO zU|7C{)jl%6-*j{6M-~B0xf%Ux5t8in&aop(`DOABqC{-|&ApMT84vxf^v`7Z^UX?v zp0+*2u8q~bLuy8aN`Z+sX)x+0C^-Cph?AgIXfFvKbaILVX}Z^lySF+&e;aQ?o7?1I z(Eq>%B`pp08~yxz9bW;m+^jD(&V_s_EhW`ACi<$Y{xrHjQ^@FiQ_wuiMA> zo0G73FyAh6A+MoW$OBEyQ(1-Hiz`d5ToU!J;m^qZbegtBB7W;=&AkO@*k>U++|m$! zZ*W55E3=lGfQ@6j(6*yX@^ChR_GsJpyRD5&bKO5Vp5hb@!?`eAVVwj6ih5#I@M%gA za=Sm z_cnBLrZ7Vk?wzJGyrwb>eDbdkhye=Jq9P(5=<*0X7v^FLUB<2b9uY)NwkA`T08nD* zh!;D>lyB0XEFUk7dF4D#(OXg@4D5EeO5^vo+o8-#S?YsYB+F~&KDzJ(G&-}LpBL6( zL;cRcdX92$6Pt}EN>)qqPNpAho3j|NJ1Fu?yXa#!isX41Cp_&tdv~qSKIxH%m&lsg zl6`TKEskWth%gZ(KKG8wNO0S$g$s=@OXK%Wb;^Nf)l}ZuR7wc?soKwS119mhIwpC} ztRRgDraowZYo1w*X8Y=G@ElT^9dvc1LL4FKOC48dWl2zXG~4Pu({>j~nXAf&%^o%w zh}r8qVWAB3sgM2LTgSI=C&?ho2z9W1QivzsBL2%;qRY+$*XbVHJH<16I4^Gw>C0|T zYR58MwOtzf-C~sKm}AQ@4x8-lxg@l0HnE!Z;ZvSu4k*J9*IkNKAQK zBQ<_y6=y%Um9-4`V*dNwM6WM7*B5?V+$kuiomAF9B9ixt07E5TM8Y^1klan2|yr{Ni{Jb=2;JOrzG;~h`roSu=1DG$0A`y99$9+GPB|s&2h=Jq%<&&^`TZ-e6IswLyV22*|VfH}Z zDZ;4e_W_591b5*ibisNLW89+os2)$(j5836ignY9*GDZs>N4&XX#)x2GD*G6z9Y*| zd=BKyVeE6@|DaZ}Eh;vgSZ0-%nK|uZz@HJT9JEXO2alyImHrT`*4Y;sANDgm+Wcwm zcsk=NDD!0OP<%vuLZyhdRa+x`LMSX@o+F?(TzIC|ybG~z%PqPVRN8}Qgyy<1AltQZ zT3~Hw{SB%+;8gf=0Ep44XkKS25rU60E~$}Ndrfsd8|t~=h_ptIR|y8MU<(j+X0E+4 zE)I;t@S04Mw0euvsbC*hmT<_kde8>vg;9BTP%zl#2XHNRZu#*eGqLuMNuDw@D|49y zqta;lajS{)a|%v+T|rMxbc|3?g}=}J&N*@Un0mN2{O%Ia_Ca$toO*SZNAcjtom0^A z^%pA^=fg?a2!(!MU5PIbL}i7?yJflWp$bH~{xpO0fWbY|mM0=IO!%GDut$ZmCo#64 z>89IB;I{nPM@9xi51%Zb4^eME#eE|Tc^QO?F-yH|OSkTsNHCZ^;ZN5HQRMAvfP}EqN9i6r2Ej#cKK^%7Ps(O94 z(b_Kqe{sNSIvKgA(i4F@kM_;8jr^Ny7#T4`=yv zi}lb|nrBL~HZ--MAajL&cdT@FdDBIUQYV58k_d6pb)ITJ8y8H;QpIBu&azIOW8jbY+&F=)%RNtKhyWIbGi9a~VWoF_Ev#`WPY- z6n7c|*O>|W+N9NWQbNf+#YG1?3b=ukABZ+_5jq4u|AG{j=vJ6-^|Pk{0Z2tncVTjI zy@{AyD_&DIai!k1+)xxjE{)COF7~uL013)oXHK_a=2|>-!=>fD|Di47UQL=@9uZ^p z(JPESz;m|A2y<+sl(hgg!nqyzX8+-#WzllfdFiYw8c>|Y{4fsUE_?NjbiSvVRyDUz zB_~#-hcb-ls+i`6Z) zi+~Ii;#jb=J!W>8mE~T)b2Cp_VaG$J-`!9u$69~m&Sf$6yXc(v!<_+keM#T*gy((L z2u4TAC~}FS8q11ySq(OWois|kc;5a|*-!f!B`WgiQ)Fh!*>IEdHk@nfK^{N(>&ahu ztOT?KoNcuZ1M5eZ%?EtF2_n(tJ$N(j?`O=*E&t#up0-O-3RZ!PZN`VXe!sc z9M8~$6TfXm9t;i-yA98<{ls}}SG2MeX(@xGfQ#X0B^LnQ8QJ+B%O63MW}I!Y!Q^@b zbUng;PG)e8{#W5%X@}s<-4}>0}o*zqr1&Yi1*CYjE;{0 z6k(3BUgV+cjO+7B#Iu0r=g65kbmfhfw#vQZW`WDGYyEPoma?|neDH1WV}yW)%EE7M z*020G!^!i3mjrc>kVXyAWAAg@PCOm>Dgdo_17k!3QXt~DyyZz}1-c0Z3tUao{gfZV zpZTh590skZ=6qDdoSlL-lfJXpL_GMVLWz97(Q{?p#L7!wEL$BMaz?k44(dIcK_M7t z4AyWKNN>;de2Rn1czAG?pFk0jsN49ALB|>`{RQ_fq3!a#+~-;Y@7jFWUB&63*|y~& zLGWgM>|$g!Rl4x$DWeyWns+cPSeZtC;^|_*PM(l=(8a6Nb1WPY%7%9A5(_Gg->E~B zmC7ysbyLtKRVys?r+HBGNY$2v?@zI@6A}*x=lJR7t3*om*Le| zzBuKNn+hy*eFcjljA0Qbcu}q|hs&DNOvl5;e5E$^w7PQ*n0R1rX0m^ZfY@g}AmjNe zgT!mZIpg-GnIeAe*3eR?56?f(M5g@{I_VoCKBvoMoDPwR8*gH?V3M+M4+9;Usl4g_6k1O2b~**8G01=2yYlGu9HNJyf0}QT zU)dPIJc#ZuKSW=sn#^4HUE6c@%9GR3UFq}P z2LG))QsypcGSlZqhR;>o*%^<++ojEC=Wn|M;DpsJPvBum@hj1plH|N0W*}f&5CVR> z(|){;M|FGPX`Bi1F=HCRT4MA^ai(-m<>$|R|!sqVBH0R6y5 z2xQcyeP(+bA+AY2J(l(O+cnD`@>EcoD&FXzt zWh1+hdwYGw|KjK>qoV4%@Q9QWf`Eil0}6v6Eg&V`HFOI|mvnbHbax5D(5W;GqICDr zLw9%c-S=B-e&2P^oOAZw`-!~|aQ=;1>Jpl^0GW`Doy^=tE_JL;AlCEuNIU(iucpit zy>_QPWFL8IA5i8(-wyU`=-67kqQ zZg+Lx`>|m#+@IH@apo~(9v841LA>{dTD2$FlXiXFd6GrX#PH*eG;W8;LRGY4soC*{ z$O&a2A&JnEH&GZcv|Sw9d}uiPfLzDPiLsK^3(j`MHi7iHe;BC0q-zni)sp0Ho9!TFr}z^AH_J1qxxE9#u7G&A%{ zeC;-`EdMBu!LY^nps`-h&0%Oyp+FjW6}@mPe79}?R<&4*iz&Fr{RP9W;#j4i|slpI)e|cbkkxl5nDVB2wAx z(9M^wImXN1YW;ZaecoitgE(}$E!~w3$MK=%_{-luU(5eFekcDdOd7kFj}`L4`y@>> zp?AW^@7_q@RZ>YmuGz4T;w*ow`Q0(&i`O8Ki9AH>w+-GZbF+;<=Tu@1>aoJQum5>W z1E16jM{mHGzc<6zA-1R;XLeVeaSzrv@w%Aw-cm(b*(!83H^Rv{_M8=xh)K(|T9<*q zR{?y{?<|x;Vq+6tcl|ldEy2VUtS`O@X!eutDSqbYtdn(28b!(F@gu{LFwS8Ep7nUY zww=Du9oHhA@mBru0w$8kO1-C_Qhpw%PB1g-)P_p%J?I0!}Bm1<_C z(UFKa(a#}%W_VR$DD)dS=!QC)MWOU=);I+7o;t6_z8b^JW}T(x+H4^u1Yura@i_=Q zY0NRNJ@=lf{rA4)6F!hB$6`nF1QXQl&qnw2#YoV>i+9tz$mqbd1-qUKgXOSPSoe=o zwNoDzHLar?UAYI3$48;#_KY{0R88N5p~O*=R6Zwf#pOGGhQrujT~*SUeA;9}KIg7N z-HoHDnt&RQE6V)w0E`xQGwbHt=yZw5`|C~X=pHK#8F%eF^#-qGO3}T`(@xFd@f(#@ zSWPy)SXv<=l#_eT&BB?c0&!nX!a&_T9DYi1bnzCK*49dO8(epC^t@mMNBx?;*?8e| zlL74e$xHUOT{J=EmWtiky4MX=TTN6^TA>v#LnvS+;+8VY{gteDD?Gl(g@(SAp9Ckx zI%uqjag}6e7L1lNO{F#itCmN_vBRXtwVdRv{&m?VTYl~C%ZDbL8hkszeSVXdsT@6= z-$K_7J|LKQ>9?AVvOzT#7FE&c;$b4yII>Re>a3ZJYxPx6UwuGwm77-<13OR9^~b@; z?XA^aFe(bbJ+ePfkQ17WtrLmVs@GYBUi29n&i!(=|4oywiU$VMxYu>_QTvUirtu7J zaW@S7YQ4ZD`J(lSqijyS-lVglgylBlf`_jhA2LSg>8ow9{b%e6(T!&0mPrRL&pude z_=4O##ry8BMfUlI-v)UyVfw%uFz%@9{q8We!$pU$CA%}nTLt3us*}ztk*;me=PVZ)b;T4MTj4zDXD01;c=*N%TQgm_bKSPHyKl%<7 zu+#AYZV5Kk4$C$JXm$z24B%C%3|3@!Y5l%0yB9aExMDbm~_hErLuM zl}Uw3BZJcZI?rmzlDyj3P2XrRP<4`{H#Wt3l@E=B)tcQW7rg(N$!(Ik>grI3`v#KZ zeK1&<!I)oo1|`yuZ?+RhNhNGObNwa?el80>gFyzo0l zwz#YHeKeE;TPVOaDi?l#IhF-YxRjp?0hO?ai^9UA&qdIx_s zPBIR;J;YSf;^n<-N4Zaj)7gvNJc<(?yN6oshYU6Jb#oS0{$5I0IxPFV5cXMa-?7>`gORNHu5EUnG4E51Ok|!3ur@Th{ji+3w6L?2p}Ky8Zj7Cj#EqxG zl6e~0?GIxNW*ktGf5QXDzM?t~^$a)p1o_Tf?lne8&;utJ^KM zt%aG+CEF=+7d(Mkx3GjIetDeQiL$phINOR}{MTP-z-`%~O_!)PmXhD2(6~A8TXzV% ze!;?JVy30!&WK%nvO4v>GL5&)&}f8$X%!E>1N3(E_4^BE)Nf&{Z7iw|B8yD!-_v4D z&&{LCr9TM`?||*dy(j!vG?Ncx`Gr;+DH^h9L`ANK+!53fl8Zp>wQJdk?-B zrK*j24FD0t=^rnm-3*-`=~%rVDyZKV1TGmJeJn^cCD*MxG^lc)eCuR(^{|}w?Ioc} zSuHo8%h+_xqQm&_-c=w6b-}|MMqEu*K3wIfF=erc@|p}5Phmu80w+$dD}i>Xkzi-o z&yUQBTgN1_0D2L8Oy}!6Oj;;UvEcB5ho{i@H09R!Va_iNKC16mM;F9?=JP1k)}RT& z*uTdk44|ce6GmwHr&haVA0j7t799VX8s7f>?wwmh`YK?!Z!GCGwfk9G;GMYQ^78FU z1*`AbDs@96jqud**pPk_>UuPX%4erw4_95{>(&jc?_*{EW;RiTjwoZX#a>Y^>h@xj zO92P}Bq&C(&DyQN=&R!Q#FM($tkmo_{shnY1-}{h*mP&H*#!$gpAP@ADs=RV+Vp75 zYiA7J(>Ksj(~4c^c530nPjK9d(W#P3`5DPsyalY5w~!D}&8EpJWeHpqUV0m?G!Rl& z*1y|FJ)XByKi=4j9NUsn;qtYrJa@*^DF2>JFHVQWPJ#D~FX?lrERc=;^cW2w!n$}p z?$$FA@cA==z!H3Yxa^G5P)>`!d44VEHzB_z>G}I~hx^|Sq|w~+3kuEtwr1=a^Hvp> z+9w@jNyC-Zy0&zAx(-CS{V^eCRwCrf*)Cp}WefZ}^DLykTX*6P0$ZBP@P*SwFW39U z8->$)t?DZ1J>?Q^LY>?3nFvWi*iNXkn}H`(#prK~&BoH(-O^8wS!l>6jm`EHo=9@N zc=_8I)JSH2_*jSb>aEcA)W%&ZL@6D0uS`IN+ojAzA!yul8uQ?>S0n+2Iv2d+M9zrEXix6zRU5U@6DvR z0EJCOJ}1jxpEOrot2Hv(TWTh5+Q}zyvs0Xx-HCZel&> zdb;^3f7)JP!qU^nrboZ>US|~%Ur9ru(CK)2zsuJ-H+$W7M9=k<>drq)z2W^a+}d^@ zz3ZVQc&TFmUgT3?$3Cr?YdD`FMKzt@$a$j`d>QBwF_HAGS{1}eDO;?uO|hgPKbD6~ z?(^vTKJckF^P?ep^wHd~2|!#Q>(K`lIU;ADswSM&ywu;EP)J&ESml)u-;CIUj%9us zPJ?42C(_U7NSU?CqrNt^aCD7D^LtPKxjnph(3TO|v}Sr4=6(I^&^*#-Q>txvSnf?n z3m5&`?=-(2gaz!>RWJoXDR z=$Vv~?rh5OKc(;=_LUIHFsbqA!|OiMzVpQWE7C0C{q+9L-@o5+PPL2w*q&YVnar-_ z6(-<)E;$M*yn6i&lj1Q7H-8 z@FZ!cAb2Wh)$1RU+eUx5;eNvTK4XRod}q3fOMb^>iK4?V!`@b>MMqv_+|U@OTG_|sXz z-o#}c<5$_@b=la@W0=@Ij+Xz2_5(*}In3Ef=L(G=5t?4SuV0U}tMMqz!4yBEM7iCo zY(_ThWAPZC4wD6<-q#{+pw5nr;TyZm!r4hfsZ3ZmuBCR=Q?*eMdSL>?;Qp@G2e zh_bCDUN@-l+%f;07=eF9bxG62Zoiw9JWI1|ET??)!|&17@0YBO0;50gTcX3i-5&#= zKu5u6AnYI30~{V5WAZ_CdAm2giQ{AGKstf(sH2lm-pge&z=m7UBtS-^oDSbFEpyQ8 zon9^<@-g(iF*zKP6CaDluIaBTXMOW>bV9}i#E}t4L}*@?Tikgnu~|?FXD|5Dxq~NA zdO|=**-Z%qK+}{=I(k#G7Pe5)Sks;QMfb+T2&x!v$~->SC+uCmkI36X{cpR5kN3fr z-+~8fL%^cJ>!lyn)R7eW5ig#Ao+Lu+7o0=j><|duk4lFF567*ga5>gb{Ao3uU+bVu zFWFN*7|u;6PH?{>{!Aq+h6Y*^kot+QuqtKE^b{13rr@>{y7X0N9|6mpWbhBzs1lFb?tX?dU3)?OTTvJ&weDa#popavPU?j zNT$z|mLf!&5aORO#B}T)fE&U$XhEH!U{Bc? zw$|0);lyW@Q_+$39>yeRZ=@pRTSy@Z6 zc>3Xs{Ma5xq#$E4*rC){>7P@Zhsw=M!nlBpi>?83RiXTp!z;m5N~iPy7=o2G`(qJ} z(erCXH%4?Qms{7D{s~4pXOTa=hz1LALxsr!wFH86G$;}M` zNNG1%Qqd28C=PoljjxeSm^K*u)3-Ncc5l!`Va$h8xPKu8XN%k79bA{1$i?fOPy!IIiBI(O0``wuR%+|{?%y?y_024w?WE!XEV~!y|;``3TllGJn zUSY@O-%kuau%_XX>gq6UFx}sC*H!}mIl71s)@rb#hG_6 z{aV|K*d8+;FO7zUkF37!?=NU3y$^ZmLmPT_fGpJvgT``hhR}+20BqE|6%|OcKpQPu zmp0S>-r_Ie5h|&+RY6fMisel6-BTLj#_Hlu(&yw)B^az|U27DcU+7IF(*A~zao3*n zW#wTTZ7n?^kX|XX1^)n(jnpfth#06VWaa)!*vNeTY+FWDDovFbbEITq!eY&-vw-vG zD7wj~JkYnq*`+4GvP!?a+P8tzzcic}IAM$-6D;?$m3o5T#{8X~(j6qjxx`E;=9=7f zofP_mBf8_jpJ`+ebf50^o^WLHy0wtk2fjcHAmq+x)oiWuY^RH`cAKJzt8;934Qf@S zgnXdlJPjq6APL#tI6E=d_~7SSuwawH&I!|Sp`wS3vG~l4zn9G7=g=TXLJ(JweKhI$ z`<>Lom5hGiiIn`O2aJT<7u&kAt{wc(MM1Ig&sR3)$%B$@=X0)Yfn4|UE{~G4OYQ4r zqbPbeDNsh4Mem>P-R*m@uYRpc{txQM7R8Hf2P)u8!^iEms6m!$=?=C?n5jy}*B#g6 zv{MXpg74d}18{woUrsw{6*OFAWa|FU2|DpkDukG&W7}mcD`UmZzeaw0~WC2arUL zb3q}^zjT>@za?=8PGD?O2|&!^YXTw5l0x>S$q4rch4ijV4UM_z4471KUbYKTAkraQ zFJVl@-s3mRvu7Yeij4GDJ&y?}Q$Nr*?3K%&=to&Vxm8dg;ZiH$W82R;#2 zghlPTCpfwQ2rONht`8le*_~Q`0f;4pSzC$AD-3xi{LOyd@ zruTLq_iVj)5!a=LI-JhH#Ojo2P1)M}S=W1s4T{ZADWV)-(<>aZ_H}B_7lUwI%ceW| zBGx1lp_J=^`;-v1M~D0WsL3C_Pm|RCCXC)~0VE|4P@$hv=~Qj+4}Sh4?wt4Zy%Kl~ zx4gQ<@-&cTyfTAHfHt^?wf|7C%zJK(QgQ-YSNZE_^iSP#=J*>uj`fMdfwZ5pdHqiO zy>bi|GM%|yY2@Sxwc!_C;rdA~MByi3QPcEgEWRKob9`T;(V>JjsmBh@S6RQ&tu?xi zaNO?qGEk4@i8T#0!dsyX`H4`bNi&ij^3}{^#WS1UWN2be3dkS#X)iwSWUiK^b?JX2L!xI#g8Mb*3avmQDT*8CFdEhEr20Ck=>8IduN#>zu%~Mb~zzqS= z{-uW$I;3EqSFi>t2m(Jj+h6Ffkq0NSLjq{Z>@DW2t*QS7?VnUrZ}f5lh7KAS=#PWI zTT{gxiXkm73O!Z6pr6v>kw(lEDojo{qN0kgfq0PcYHx#DN}kC@Mc}w!$cO7I9+Uzm zk(m*>?nGt63^9#J%z;3zogHLKp?kOuKD1H{>+%TWv1z`aWQP@PG0nK zZzN_=S(ud({ldVRZ@+vtN@A(`rb7|V30u0U@$*=mS-qW&Mx8GO$7}Tm2Uh7O(uJr@&o{ zH_3et{D9Cp2@tnYgMuIN(Zsn1e_R_KsA3ZS;y%QK3v1>T3ZSD}YvmaI?=O+ByK>3H&u|&q*Kt4rBk{@_ zIs!lJ?NP&yOXwq9<;8r!5H2QA&&QTet-NP4+YJVep>!gUb@xSw`%!^b^LJ~J49Ug{_s@!}YHzI^R zjktuZ>Ssr?W5mSME7a{P6?h8g`pB`L-x6C&!i#mW!Jdc~_t_sS-g0jgmAJv6K;Zg* zj4G3wM2}p?!Eu|mJ9wi{Tg*7zWU&J}Tmyo*s-%+`dg#z=iiEa@RlmGZ=`R+e*Lb+> zX|fC})u+Q0JjPc)0$YON)&zP1nd<6D)a4PU$i0ygy}E10pP7eH{M!;YEe*`W$Zl{Z zCu;cab1Zp|Jp2jGB`G1T2{b%8nZ94W+Ldd~`IQ8E7uPVUid>uHfrtH|VY0M1zxf-& zxGoP>)&BHzT_#V`_h71EJpwwy_o;yuULaU%9VO(2s~WlJtZOjwjDkc^Nej&MqQ`(* zP=u<+^yJy99qv{nz4nx@<+fgst9zFLN_vB;Z6D*_V*CjjR1iK~*sKc-l!VA}?ZfZxOzV7R z0Vrzyw>6wz_N8k?6wIt+y1sfR%}_u|c3Z^zknP!o{twz9M-kLTKzq@td^j2Fk0{P`5W}ZD(`96_CIqx-Lsw*?#eQkaRfQH zzdxC~Wo2U%p9(Q42-+v^pvU=}&5-LUkkkXaJNx;Pt(M$8_2RCV&NGTitHsPqe`2gj zt3P$NgoWF>eo}#A@%{eSkHu2r$P`iq>a(0pq(|c{36;61Nb2hX`te%iqCtxZ65-CA zzR)U|RKwc%Ha<>Ic_&_r6d0CVN>6_;|fN1F#Y@X}oVNr-o!U)17d{J3A#(Mx0v8DLDK z11XG`Tp&FV_7ozjw*!M1w}>|Qkc;5J^VXMfc4QJE7$ZLF(+=;*maokf?z?@I*4W;t zGEqeg=~msH*C*?{FHU4WR3E~iJB&oRN+mKP;z$>#cY;KOXnXUjmUDKJ1;z)JR&8)x zeWy1(tLj5(yG@g|c;8~J!vj_uYHRUCZo^|^$7BAem<@&kry!q(z{|(wezv70ZB#3y zVpn@Ok3i%lvGx3uCG6)*-)p(~eJWGn&%n{C(4E^_?#&M4x9J;<+;3kFZt5vlxHc4G zz!(Ko#@fEF@zDv)iG}v0rd$}@jK5bLV>b?scP;|vP%lpA4w9MwB|^NviQeJ7ozngX ze?bdoSWjc8TS)a>t?um4F^IF0wpys$uP6OnrR}}==G#IWCS(@<;XR0ZY zLVgo1Sg&jh5ee4vJTBXE&7JK_QicJ9>||09#jeTius}{58834sGOjYexP*1==OFYy zgr+=lH-B_CR>*@iY>Fkf9N#*G4$54Hgv=F2?_}M#(f8X$Mi!L$gj^yVR%f#?lP+(5B?%6U@2iP?xbMsL+Ra+UHIXOG z^qQJ@a_{M75bf#;Vrq3gG|twHc}T-p0AsXh@!$#)SWE~60iCF|b)SY#r_0L8 z%w#iyaW8%j7miQkBY4soRJ`Nbc(}o-9IbtWVtNR>Y zM&61)|HDv2up6RfH1{e`X+jq%FYe-rBdUp;!_90!oPwYSbLO%7*{P4Pur*8Xu_4lE z38qO5q#(${j|$IxqQ(64oW20`-6S(OLewGlFR(6*aP z`fwy$!cOwLTuOWyhi4tY$S>jc`hJs}tnc^qztqo`IH~@w!-5y?5>wK)$1jbg1-T5> zkV=I!T3K1wA%*I{Ne%SvO=0d&aX7(o8Q(`<>!1E2jx)8WtekFbL#QLQE+C@lpUhJR z-wY_BwSK$75&dKXnl0VCP@l5UF{|Mglb;)!Oc@T797x=!MKhNlch^mE=yFk*t20qz zN@}zbB>yZ9DaidbO831Et_g*&Afrpf3J$D*;9P1r>7g@z2 z>K$WJxUaW74J6;F%aDX=osyQz7YHi!6M}GW{tg)a&MR(|82`;?%C+`ee4NJi-)xix zg^~M__{*a9OOjXj#d-Gj`jtfBV&bPGH4hFTysA~R{w8f!E6RO8rAyruXdjFsh~b5r z=o2)gdkuT*O(%Dg#~OL6NK|BoV?g`aF1+=&`!L;iK@C);YfCet;4_@wI{a{Q?UPXg zz9xiMFI_W3YK$=o1){&if; zNpr6f-9?>79^>uA+PY5U2{DxcM+uwhu3p}9zOc(UYs{kJNfd{v|r0F8Y&nuNv zO}2OGKrEm58Uah44d>3F_BaV?5oi9TT+V=*90gs&F(ONkJsUW+*D^C(wFfDShvt%! zY>0R@(TLFpLWm?U57g5aaQ#7KCYOSDkpxV-m(L-1LBKfD!|N1ObT^p%f=|TTeP^vY z1j9l2{Wco86~(Z@7^k=)_MB@06xuugf;Ji-T{--qVSg6k)2EN@kB*dq07j3?hucwB zpVbucJ`O&@u#dmyhp_ZyAVHFT@OFiuO8vI|ix+?`Vncz^!Y2jwq|6cBoKBt1VzMFF30*OHWreHZJv9S( z8TD;PJKj;b1p0b@iQtOd-LUUeej>>8gwtrkmt%Am6mNV3CdtvCE&&)dbuzz3;for# z8=9x`@3~oCUfWd~P$|n6dv}FdG};c!6>hrS*Ry5$(q+x>-hRp}R7-;ohD&DTj}Y?Z zMW=#GM$RE|Fomx)2j@Pzg#CSuyb7OVMK_Y@WwDa64biP^FVh$Bk`$!&MqjS|@Qmj1 z-WZufxShVuX<$pIc7axqv(trPkEfY?i50%^K(X3Udp$7ntlvbVrmiSHL(qby4d`c5 zJ&!M0QGQ6V3{pEzX0du_G%yYSHW~Zs9siIQ^3oo%t;y{33>0Zn0%7rXXsWz_5fn{z z+<8rC*taJqy;@cQX|=Fj@6tcFu18AN44R-(qx=7t^KxZZTYX0V>~6G--%Y~TL``=E zCE1i>&HOlW?&uuEpPf0xfC2uLogxp<(ACM%g?>M&tmY%17Dg`)ORre>HmD@FMO8C= zBj49jlq*hzh7KLxUlQ4~YEi{>;yZAAWIh|i85TCzX+x~rX?W&BiDSASt}d~tJy!}~ z1=^TM7r6BRmljvv;k>@fW(p#uZh?+b)Juv>K}=j>m2zc%B+as_sDSPdPupel+kPZq zP}q}Gkp&{izffHN3+MJ9YV{3l?3*p*b++aWB}@e`tG8N0-uvDnvGB5-qA+AXV;3ie zr&2(!HZJ{iHBm*@+7fl*ux~S+n{2^*Y^3G{F;XO^uduLMU!_!N^^-?P;OZZq;EIlW zE`8zX63E58V11mBvR8Vi%C}O~9^Gxy)^-`NYxlUB@Q`HpCR6|iyzexWr4pNSmgF^I5Irpo z_&~p2X`^J6x5GYB^p!fX{_T2c<>O85`oF#`4XJu(aQfw+JD$hWx})Hvs{B~fpuURR zS#G4y`A&@p0PZJ4UDt9MiyOKI8?+m%cK|ZSeML%X@}MlLgH9wl_GO=?3vj7Ap63P3 z;f=eGKfGIpW!px%%F+cs|Fiw{UT=XnKd-2m)j+fMSIG?NPNxJv*HFzpF`wAnIgqdk z8h>_>eDSQ9(Q2!o=g_eJ=Lh2roAeMwY>GU!76DJE3OyCIVBu~jj%d$a%atFM`%P}Q zA#Vpg^AIq*VFws)b2Sc33+Mkk@*BX$XfZf~6?eG`)C|BEM?|cM-LC%b81m=({JN;i z9-RLySzue+8%4>$v%h02cdsiDR}5`8x(?Nod{!djxGGHNy{z+`_RD>*E1JGdj|7C~ zY9c3EAEw5H@Alglyh;Hb$tO%(27pmIhQ-aKA}Wl|n-ngEZ|-zYle>)L_DDEQ2L5BJ zZ)2#~#UWc&A-HC1g7pUXe|`>!>xpz~F`q0@-?vw1#rQ031m0? zkhHE;lMxh*xw_c?yU1!rmd$b4kA}Ovu7i&$`Wm#*2v82$=tGCt$c1;V1^Q97)_ z^pD4Yigoj?an>(`Bq(A3{BP6lW}zcL;nvMlqOPbv7fA-zV-NpOR5!=0hfyI`Y-?vB z|92==QNp_aCXRkUfB5ifMXR5}ij1%;Jy`GX)0|)oog|^t!;~ZFd(YdQ!d(-VQZ>M{ zN|{~1=#X9@zsA%l2}+p&>F1Dy)ya~Z*p%-2I-?euXkRi*J%ORY*dMGo8YdKoVUn*4 z-q?`6INN>Hayr#OyoZu{w7dv_V|CvtTC8N5OX78y;wUVq3BJHK znsvZG-p1Q3D}I}%I5(25$!B-=ZHS}N6+FhZI5O?Mi-kDbj#=LRSt9i zln_^2j|NlR3gcJ3EiE`>IKp>`*HqqjjW;*%{%t|u$%$t=fp2z&sT&{T%vX-d#GLi& ztBWT&&fS@rf|X@0b%x2l5D;z-)MfQoJX$F0ECfLNPVYB1=)oPUjdT^6!1>p6TMwx& z8l4eNVBfg5n$3@^U;K8R^LYGC#=$7`Ktfy}d2qd*1K0>UXE+%yw$+)EW)V)q8K)`g z$O#5V+h|%(#N&)o3lA${@5K)8P=Zkq)>khH%(wm7hkm=aZMQS_kCthYKgh_%x*epV z-ke0F-#|C5e#qQS>p<*b>gVko{7QQ;~>$AD>RW)!@3Q|_i)=|3odCM-ZGZZ;7n0;k-u!rqiH`x6yhfe!aBMm!WqQDgyNx$ zu=E0Nx}-Z7pZibUGHl5X)=MCnf}5S2tj^Aqw{Tm$o1U@Wz zM)-%0!YKV&mtLOt5VgJK;_9tw`i6^B^EiR3tWPQYrrr0RD6*DG{m*rwoZMTLk+dpp znWyYs(>x`mJiLlRCW4PeSNQ?hc_K#&PaKI6(jVc|qci32bJ3r@H`*y#1Pvv;&wNi+W=h+Y)J>rSU7EcEGydb-`i% zs-=ilO~Wz_O>8w~kmBU#WPw4sG-oJ~04UFf71?$%hteu6+XLgBDzHC%@-y@Bka3dZ z_-7yrsbT)vRQC{J_-JV)l5MmlTarALxY~3b#`~lJaKu>mtx8sXIMh_RIA`_UGs(G) zwoC`5S(}0w8G90-b#-tMj9nv^6bpU$l%>B%uEW}D_ZOvGbZGL_*W@rpj?gf1+*w>j z&QY#x`2tCN>a+GsJys_^Avqb;)tJ;PX6;6g{vmN>Q}>^ucdUe{Mz2Fa?HE|YrH{;u zlU7{L-PN(R{nu>Yc$2)l;fd%|OviI%GaF3RicRU!P)7g5r$lco3VStd{fG^>t*__wGR~GDfml@F^bW9Vp35Fhzco(t2;&9wMi*L2?XFV1bac$KjIA-6 z#N;|-6PU(Gnb4H&7czu~gCGUaeJO^nX%>tS(vH=f$}-QU+Jay2*8Iz56=THxG*#H= zxpWg96QAE?a&#QW^&I~6F_-vhIMZgRk`j*iY;}P=XWXnHsDtY2D~Tm8Ji_Gi$V}wM zwqy3@Zu=K~ZpbA|o-XIY6t7Pdt+}y9qAM0Z6cE6qZ=`0SL{C7l zo=6Io+VY5pYUlF^>-pcLWdG3HoSTZD{+TM6bFEz9JowAbX_soCoNQk|ecgJ(`ZzE@ zx*3!G_$QF^y{lu?ZgNE;8d?D{N93ABdPzpk{Upw=A?n)$<)%jD-YQ)0D_QDFrpxX? zq3^|;U?1!ksqJgSR7jqGxbHp_riEV2*67zNY~Q6~d=B3Nva1gn-1N8gH~xHs4dI20 zi|N#ypDM~eYaNtXW!HNio^M83D1IWhCW%gUy`NdV!QIPhtA|gHD9Jx98GrQ9S1gSW zLIe3@XEWTYCn7GVN_BN~el2V!fyI!gL=MPvU7h&e-?n6)M!r-uycJ&mxJkS7;>6e; z1ysHb=_vswyMEz^cHP~b*G?ZKpQ;m)6=5FSL7C=U5vP+%if?$v>K3x~+pN;z*5pSQ zqJwW*vA7lF3T(%@{LcSXo-wfyXm2FI81tD}APtvDx0EE3`Z_qrMjSR!#uyR5+DJ)y zm6Y^~WR0QbXLEa<)h-1qultxzx@I{_NyV7-a>=6fJz9j2>Uhd9VgrHGD!^erGfO{x} z-^;0e56F1{AB$;$1YrAuTYig5$iMq+H0svp$@du+P#|$D9IOHlu|Fc?DIyYmd!vXvxzD0ou=f4S|z4VKAhw+TouB}DelGVLV?O(IefHxzZJ`|Ky+V80vT>L@#IxcD_P_nwwM6hpoI^Ip^#@wBf{tLw~vbPWw zA8Ouv2e`tXKE6zwsm9`tl7j$Oq7rd;=l8hnb$#tX>*Q7+bdlf9>UR)y?Gg5ssoXkk zQ{JuFSf=$VSeeoT-C&l1~S>gnXJs)t8BYg1QW`^ z$J61ak&p&U=hStJOeB&ghMIFFrtL~{Y2FKp;t&pYy}o%9obv{-RN2+MBewgR+^ zOpfTjsp23Z=~l>4{aTiT+&uEhgzWXh-@kGzVXq*BG*!XHgvQ|{t;}&2abus$F76ky z3#mmXbI_&P5dD`3Wp$64A-=sleus+~+iF8EJH}s#S0&WNza%4w zgP*z^$;-F|i-@9CoB~1h<6dbDiTD%RO;*JpFHA5-H+DA4<=ohYVwE|$iW84IIvR-< z?|-^w`OMi2PGsf>!9=pvj?QMn)5b`Z?pGQ~hATg=zJs8l4_}y}xPICwhkX+tnrbk| zhe8bo8oqYX(3-kZ<4X{;|4!1i`nXrL&lWVwYi(0)s9!fe(b%Wc=n!g&%n^bQS3QK- z)zwB8u+BC3tyXxxPf>!PJ%_NSu`{xo^$~q#aq2^5&o#@Qk!6LCC^Nyj!G4eTZvUY( z3bsLN(ACV-A8DOWcufF8F)Mj$ja2CT;c)T%8S*p;eIiqL%g&G&N9Zr-Vx{$L`zj}v zAohYCK4F+Y`$cn(?|F8mU%GV*B3B!m5cUDkqWa^51coas?U^7S?rskcuLrsp*Bkzx z-@SQg{M<9b6==rGXUAo`a{Y}V`1FsJHsrbN^VbGpc$@CkHVv zfGT%AxLPd7_sGp{-ZyJA3=V%qOK-wFEi&aIL~hc4K$mxT3n*AdVdo3Y2Q7!J!b~3q zE?e}J+&fxG;x8pxD98s9-4Oa=#s>+23q1aR10 zF*`76kLFfe3hLAw0@)>@%xmf?mv*{|M9gDI?osiEt_U=_sc+kTA&2|zEG?WG)nm7=h0iSk)JW{p)buOtwpNz zdg~^Q0n>as-qBE}MIY%=rvx#vXm*_V6AwJ1KrkldRs zv$e9@zmkhZpKjJXrhB}*j`6#X@mqNO>Y^aHIWThmrCLs0RCf5(Md3B4RZoi0eZX#I z69}Y-EG@St7G)bMx|fv-{d=j8-2DKam`}GUuQ(0a((}o;p06lXD%cN-oWGQCv2Bg# zFmXa`T4qwxgGh>)3$`M1JP!f$!sB{QObgJ&rRhd;&cVl!CNFn44mMM1f32RPe-wAA zkE+Z|VND{afmZ}LC69(_K*qQQSy>ksHUOD+hV_mfl4K>;S0wN&=};=%G87oUt^z1L zdkj>DxtEiHXmNse(EF0_}cbx zUrZb-Lz{mXVKK3TEW8Vs8*W~`unv8(eTV~eD2Pr=h|JyX?f;N{KSt#D&@W8w^RVs# z2w~AOUO_k`xOt^`_9xDn_!56QG_ZB$Lq*55Io|=6p1*gUrPTMRen1}YKD>B8$RYz8 zn{yPOyQBGA8x(2J_{Z+~Q-E9j+Q*hrsPfseNj<>yl znFqqkPKJA8qF_b;jVzzJ9fv9Zfru*`{Tlsun7_Dd*bsRHh;eE5%wj^N&$w*t^01=# z0wl}#;!fvd)4>aCtB0w2)~81>g!a_#(+~;FJP^|c1=i6M`3@8k*%l{%F~uzdq`^?( z@^V22ir41a<+6j(k~weK*cvj3zX|oQ5zr_kc26s!9(7HAhPiKDL)tI5V3=Sq<_x`a zS84(QAyRuG+Y@==tg7iRKH=uvIi{z8J-2tvSVbhU{+@N8d>@l-U#b6@f9?JH29hYGKanlV_@xY4MCQ)8(!*kMY)R-sj1kx|_mPRgS6!cD3Ba3=)x?s7u3# ze(U;u%sV_bo>h$}_;JZ?w2?z`F+MnL~9%kfh#|gX-1+)`egkoP)b|B7zB-Lfj zfaKzG!Sj5awWZp9CVjv8;^W6U0%b1aq?Dw{H$q$UGpZlboHUrNc(W(Cr|R>N3vLTC z;h*Yjr9J~iCI#H+Yez(oPvb*L1m?vqQ8lwf$#{|0Y~65{pafH&Zz!R6mAyoz*TD_3 z%Em^1{IS-=$axTb8%Ae9Gy?$o-egew6~HSzCTA7+UH5Hm71+A~oF)|iKo~$l16vqD z)6w>6vl~F?3hwhJcyYybrfi%k#r@40TF>%3AlTm~m%Y*K4HJ`Iak{cC_;#Sv7B77s%og&!I6G!3A1bMRty#IyVmUYD zlQan|rMx#0=$|rUU;x(8(V1w=yF8+O`YxK94~R5(z@o1LYeFswbAD6)kE*v0h$?Km zy@zfEMCnFp>24UL8|jqp?v!%q?v!qn?(P`6Q&PIS&h~lU@4V;R{}037bMHH@YpvfR zYLGIFXv*R>o9o>peV#_EYJDC~59hRA0=%M72cl_q&&0$C5}xhu%R)dHe*`IM%G%CI z4LL7R@XKlXICTI0hX)=G3iHu)nQH0iS_sljINLxxutfSj##i}X!Ep%z=lKz}%H^h_ z>Vl8>cXQ|@Gj{z6=KwatP4U%{w(FW|?_DSh--J z60Db|_w(@kej5B)sI$K*EpEOF3KTvT76hC5`xE(Yk;-)5WCAAdX0IgGLNP2di8vud zPTHr`aou)2{F2DR4yi~EChPlk4!q75;AL@XIeduRr!TnB?Raf`5%)nQyydd+Y+;}> z$oO~|HyC3f>lv4NRK9FRtJtD;yvsW;_iQfa1g|q4#I!5$%YN&Lke4_Z-l zrMuzeO{=^B)e^r9efnP2YVvYX0^9QWc&Z{x4u4hkXP2i6lZi`(WAlzI&HQP&I|?Uk zh$NQg7i}We#2asNe}5;_w2+@{pcriL$$Im-+Q5p{pwVa9zZ;Kn0;O-FteuGA<>R`% z&?yI0bi^@@ccf@3lYmub#SLJ@pz=%dtp!3Vfh?(+*SextTg{LJo?IyjA78dts?QNfOE&T%F2aS#jL;~SNis3AgGZDy<&LFl6ZB~(6;3c6!tn~cW zBK3PSY3YOD3CzeFeJM-aAfidNgC4Kj_C)FaH~N@aL(I^l504qkX4!rN9i6hzZJCO&+n=v-ib=L616*;ev0($cK_Iks~JV0GPyPxDc-B3x&{8MQD zZp6>SrAsV4_kJ?aED|w2%X=qZM#zH&bfnXC80$r{m+@H1~)EL3+7x0z93Af_KtN6sAub*=VmfSET`s9vOt#S>0IOpmBA9bd<=! z>f{c{AeOE^Z2`AYk_mVM1fn1VbX&qW2|Ng=w{JP=jw&n1&-0a0mjTGNyAaHI;1WN{ z;%*LO9t?vDC1^eowBb>`s!1j32AWed3l<$T2e0c-pEdXinQDGgN`Axzx)Q5^F1If? zWI3`gd}}yDffyFq22oC=kzS*O)FZ@R`)mICuFfzIm&+>aG9fx94eYqOGA+FEks@VG z6K4>JCZA$*vHauwj*qjK^Wc^t2s1*0_8U6B5uk3(apNzchAkMpRy_;JX+PP}_v)p# zxmk$1&S&mVfR&IGp5TriSkBI*w-=BCi!d54bB0<=xHe!PL$iw5i2rD!^2&l`t!oJE zmT{pPdqoIQ^vwXJk^AikvY5PhfQsi@?(B7Bq%?1(!h)-e#56m;*f`&o=sHe5CDzvm z-&7MGa6Dx!tK2dZIDGg9|!eo2%3EZ}mAU%P0j65~F zE;e~UnQFlA#mVA49Eq#YIX?Nb@brOugB}BrMJ{Z=?`qLF2cMf7YKK^`>ARFz(W0XX zHX=ZlZ!;ZGFK*)v-@l9mdEu(bUQzK*9h)k`9>~3j|YV!OZTgrZRswIlk6rblVPSfVMrC;(G^>vG~xb3zTCPCvygH2Y^qvO4X;OhXWxPkge&e zJR8f%!0bsCeA1YsghM3D5xWZA+`7-qMh8oLo^Dd`jxyGi7LCWRuo(#qAXxsWo1OG{(&JTb%bkd7d48tZg8WKb#~edGKvO7zpv3fsj)O%8$=66k{`% zZgaM24IUTojP@eF0BAyr zFH#+w0MkjUl67zzFC@kBdC`czAN})~1@$}bf!1ngGpk`nNcy7BHb4>e_Be+UGU286 zJR^umeoY{4vR~?47pi9XW}dl9$sQFcwFk({1~6#lT3}hLzu+%b;;|JwZ=UI)%^ih{ zzfn7z-O9KKI194(eDcJCF>3_KimxlTy&Wfh7eE_C-@9v6>Rmt1F12&s$dW1+P>ci^ zg01?`=N$lCTA`P1mTa%JfWu==Q-z7!ERE^!jMh>XNsWhbM(v|-InoD!RKEYIy52pG zft-l^D zJw5q`oVzyu^T+RgqpvqY(mHJOdFQb=@Qt`bQhIQ(!t;wszjJvCECJ%DPo|Dna%fEIL@W%oty&qMdSh?<-l zZ=kvt#N@p^KH3L^Tz?%Uorz;XP_?ddS|G4(BX@8q{Z5(=%wIE%jD<6~k{ZY$cFB*# zh8>0M3_VOgR!=&barvxT2{njg$gEmXZZp*$ONiZ}T{hF4WBunx6+COlox9JKkuRZO z#dsN>B0g>NE;lh3VT$>@tUk;d0qC}}Iy~*7Pj2|$p?0%nLQJSou^(^dp8!?R>f`Ca zuyi<}wq(xH&(7_J9bpf`lpCwyb5pNe{bh36ZmBzut@&G|UlG|bb$7nwibdCGbtgO| z2)HtMFk?+D8KoZA-wI|AhErZZht`^H@7tdbto#VL4Zbm`<=|xUaI&G2buU;B+mt$Lf$cSX@pt!Ri6jqhYhG<%QcN zY?YtbOc8tgCpY&z$%N$tciJ54xvb0xDd7RVj}UYLzYJwLAgIN1PpI}RE?R$^iWtE4 zuAtyXZ)LQd{YcKtHxjR?&uR|?b%trT)2}t#t@V4Po33&DjNa$*m{lH==bZHdF0clM zr8(7AV=y|@Ly>(F-&itCS#H03Em*7&#Cqk9(}7^W2D!zd*7HJUq8;rfp=tfKmBd0;sfw6c`!wRp zbYk=2sF#V#buJi>rKLXl@LwoFo%qwvW9$)N3PfQ|%NQ)63VD5b>gTaL!}5~=9R;}d zI~@d;0qg0(Lvg0$-?Z5&ts0-bi=lUW3pZUvp$bAc!U4a9IA{8J9=(AUf+TIC^)#Hc z1jD3zB&L3Gc}BAQ4$BAQOay61K;dG(zP5f_f@heDDBjFOn$#07xFT6jP(QdpAk?e%sYs znq_oa;)plX5n}9MSeI+*@`^%EXg|mxAFXAxt?S1mR{t+Z!X3bhweK;5=ieD2CBHms zt2;!khO)pbf|Gt0WR9L0_){>m`|g)_RefGgJpTwG3(eARJdVQ;tbRv{*@;`~CUx{Q z^!$2%q@}{e_LxY_CMtpli&^U#j|y_uWkYWhs33+xMh-~v=&vcu*%9MCMkepL9UJ#P ziXs%{z{CD`kpbq;<;l%+Z2}0>(7MazxHy14 zw4hOiau#v8PjzFXh&24*mbVn|v#ga>S)38W;RDz=pyt!Z$HNER0u$1ehFjT$Q(`ij zpPm?W91LO)HiA+ROfVX?KBwWr6}il713+}4KQ>*4;C~-M!-4@U(vp*awa!Mj)#qR~ zo?2g!CC+TI!gVpH`mo*e8Zb!g3f2B{6#EF1_;#P;a2KC)oB#$HFyk0~3W$-Iq=};H zLp9Rb1H+@|E6SHSeVIK$bY-2ExF!**O+McF6_0}a&6z~8GGGku6vc6+MZh1qJ?7*6 zY6`k9le4v{wJSt&oH$Qnh+xv9F^u(F$HE|7JV9<`ZsP zT_4K5w4YWPNW?E1W~XJs^mGLP)L3%BfoBg6Np;10btSs}wZD3*TM^?w>zA`IxEY`l zq<6SSzUQdPVxurxxt#V0of3g7x)aP$y3fs20)0QwKIJVV(uQ#mhGT)^ft+!>VnK+x zp&InPGQMRXVG!{oc=WWeKN=+4?|T2z3cN;7PoCr79pGE+rF;Z@^?e_Ty5uDbd3 zLz%FQTrZ=thwXkXK%ksS57SN(qt z5wn;0TAV}O#)-?2u-oXBOTfL&&ixzWhc`RU*t{PAvO&Zbx`$nLLErP>nxxPA9RL^3 zhR>CYqro%l)UzJb=6N_!5WcxGHN;SJcjLQBJaq)`ZFvc97FSIzcQgo~5<1kh$Namd zv2~4x5?*g|XOm3otK4H)s94?kg0dJc)ZeZaxVviO{b5C6azfQOM)hE415HlsB| za>cepx{mlh5NQOK3X}^}&UJC3CGf0b9Zjhodi}o>63>%!EU^Ob5=(ubONWX|1xT7( z9YnM-t9od6aOkVJ##f>G{hxAwUA_PtVrHrjB-b?52CN}ay@x-~P#ciqZ9`0Ms;3T4 zP!7vy0Ia~ruewdw0ZNyoI<;=AdbZ}xB`!E09HDiMYAt0aK<&KEb}3}B0eS=w0i}3# zwrXtCU_g3Hm&8JGzv#HWGP)a|AK=c*wmY)UJ3@YleTSqJUK8tz*u0MEELW`SjzjT8 zAi(S4BaOW$cVO1C$z6M6f=;Xufv3oCVth|X@9Ji2Uxh3ywp`uda2_%oUW8#O<%NHc zKwhKku$J!&d=7&VDI+_I_9q#?lL)vVr&WFOS{>T-GRz|?3SAk+EQ^SzQEL2rqYrf$h24KTAWdzt?fE+dmO?GzH}H z5gO>7AF;vr8RH5?gaxBUqtvLJu>*J7C8~J|ycC#U|AvIPhaiU%(M+(DFhUTaXgc!; zw;cTUJw?>|EDpn8$OfTDxuxhY8U#ih}WrM#@)v1B&8o#NH4)6k)W&n27 zcq%XRk2>@KYGZND%@wKVQ$Ispz3$xd%1Hrv8su?jZ+!$?mW<1QCwHZ;2^8ilveRUM z@C9)07yxe8ziSb|%D#GUGzmQ3Bq!nUL{&&`Qx8AfW-}X~MSO?*w6})~>cU@0eCeft z!H#$?Dq$&^nI9QOOfRIkfPjn2*%vF}-~zO07hiBK?bO{(|3Zi`F}GHJUDdxzoDmx~ zEZIFhRscA_G>_>BrM;+)3@?D4rMDy=GF_R28Ki`=zn}-0 zY1t~XIn6J~*588+`Y7ACKHslbi$^m78!L9oLVT`ixqV8t4M6(%{2Mna6n7va9}Yg^ zH^5G5uE!kGM zY%l!2b6?5CW*3AkJqchlvfO>%nOsxy{F_c>k+Aq42E*Y5xcN@_ZQ6cf!eDB)l#HSf zSW5!eAZ;g%1irZRy@^Sn*;uQEa0l)EJwn9y)4)9}5P3+)rvX9)OtYi&q+M~>hcxh7 zxkkC@G(*3q6@f%;Vj5OtKhJFfE4~Vt;9Fb5H07iyG`#@aOSgU-x{#X23b;EbJVK+8 zWsR`dlp7lUlmL+hH?uBgtGs2ejlNUK?T2vRS1kunza@J+LsI|?vY-8*q~K+ch}8SB zXlSd$LjpZDkbPh~Jd1&WZ>wEx^Ad;#u<=GP7~=rk^a?Rekdw2eW z9CKRPE3Nk7ww4l?V?~$|hD+zv7);sBt>rD=lQ#oo5@+RkW8VtXNwBNc4gX1w{GQiK zKhLqk^X|Ef-O$Up`fi2{)l;()V6%dk-@p5j7)h+${LRT_X6o(c-2!D$0P{9zVNvSS zzl{pj_Syas&AUi2(ZI>N-P08E< zu3-difwLG~(D8`dk!FM2O0bcsj!;FT{_o@)Dq7n7E0g%M2>G+^H3Ng^$N?{Eq1*OP6FdE*R;Km8>8&mjpoCQ0$~4ZCMC z1t-@RBy{3QY*TwWlas-)_lnA?fCZJLOCHk*s${9ydlUZY=twNn6dLMlB{}+8*L4*& zDlICc-#ffGvNq9VMgeRM&rkm7KFbPo)Ih~9Cuo>nPJ@MQHp7$v4#9MWN_-?stI1@t z-eO)h)qOEYvfEMRGw81VE%JXA{K8`spiWzt@}R_W1z^k>N)(}vabvciT18@p~%h37LmDBgMyNcoG z>o|+m3&!h7IM#K7o&n_p>BUr^1AMXN_jyQ3vIJ{~#~hE?SZ(oj=$g&-7H-c|7>`UT zQym<@q{o}pXyEwTAyC&$yKjyjy&n`D@Sm+PuoJs6c%>DBcfq3UxrjMe6I;PTH?$2r# z%(u$J=6bc(l8e9_#*kSSwT6u{V*6YF{_yZ{eS!NC*o8s~-XOOh!}+H>#{N9L4>7UN ztn#+K?v}uvJuc;<_<~u7mQO$d8<2d{2sP8;a_M(bDWK*M^BNfBrC<$_A7;th-N|k= zKSIdMOc=qVD{xlHc6+tD_ByO+bP3rGr!cEO)^VAO@ol?^+b42m6?q3S+q3i8oW=O9 zksR{<2dtPZpPtolI$MFCd=-K6Fl%H}O-O4SZ_RgVqYae%r*4})g_GoEt^VX^uboV` z->1On`PNPt45>zc!1BpZ1l5y-RgsdhLNeJ3}Kh6@Ab+*!= zR<1MrFFi#eV|f3yzF%WyCA4qz&NsCKr)H103@de(xInxHQ?6?I5GfgRzxRA{I}x`- zr)I+_5x0dujx;wcEK~YP6(tx*HInnn8z*&H^MdXRw60*x%>3UA-U_k1cd zWbmG&6T46XQ68~e_~!n@w5cklbrnTS4j13X#NWu6(SHeD=c?A!_)NeaOGD_`z@y#D zX8UQ^q31D+L92Uc$!K(Nsdl=oIlI!p@aLXx>x~pV*nWRh+ELj^UvH|@(@EuD9Aha18ugg#l4Bos_Fcg z(UGiLAV?UXMU^d8ca$_Y{c07a`Lq0TrrGmBpuV=C^6TM@8NdDhG$S#HlWC#a>iXn7 zIU?P4b{i~}YiQR4Y%|Ked;+gb40xB6J`du>i!RZ&+1*@6~9$7Aq-6I;0ncTeq}0)d{Or{d5-1;@aQ)+d~x|Liiz$pyq(@*+A6hL#}j zEi6iidEIXYv1)Xa@%RGOa*K%IJ7KstJJJZUtu63Le}~|Oz-Tb-SkABQdJ=qF!`IZ0o_gq;OJa|zjo*HAP#cX?{_N{KRM^0Ud-b_^ zT{qcUHeD(3u+^!F42wsKI*v5e_j#s7T}}#TV72zJ&;-+yeXG#St4vR$BA*vCf@*@1 zm?w_R_;1TMV7X}Q0y2ee#^8Y=*5AM-@O-(K&lKI*@?0YMqewIuN;Cm5E`A zJn}vaV(?vm*eYqWsi@uzW>__4Bd`{v29o;P#2=OZMV8!q>g!v!YjelhOVOpI z`}n?hOs4RyR=b4|$w&y(&n>6_jx5ROpANf^Cx@3G(h50EulnY7o=u?FY^ErlsPAf< z#`k1FV+sW?a}#GjLLMZv)7-p#Gg#1AR0EGOrx%*%=Zey&Gn|C&r%&=nsNbXtF4+F& z)ADfnLK}=ui9)}cu~y?+W%j7H66D$h0s6c(0{bH@PG8nvdfar+UM|a1EU0Y&qWJ1W zMDcHI+gg|e2T-E3-(Mxu4s8=VE9AxeyPtnVe4y0c@z>lK;e>1s6qIM9zarb2t9 z*&g(0#FI*F78ZYbtGLWe)C^3RGP0QnS9PaQZ~vBQeoOg(hY$j+;@1k7J`=b1I0?5E zc=D)0$b=Kwtdg*Zm`3F`!^k)j_*Ufc@09bkA6_bEt@Jg8wjN6MKOo^b^fNtf_r0Ct zS8Hfl3rpuAXn#jTQTbaqu`c*@X&52+3f%GA$84R|X>g_sY)pN;amNOYfGD*AdjQIC z&h{oLX$uWRmtmf50w!V&)Z=;ev#|)a)fqe^APW!Z<2{X`HT+G2a`^5hya51^DYb1t zmkfhpT!z}ZO$)Fg%W{;Qbl{VHUogbv^} zXSw@!-8g;Ptxt0B;rZpZ_76OK(_ffJv?qf6f9IZYVR*<0d^OSD z|5TLVw4eKhyZNm?gk@?St*$-kgqitP{j#|$g>*$M(%*kxj+nlI0p}9Q^Jf@24f}Wu z$v+oQn-}ClN|=^^`Hh@}{CN(yRxm9V-5zxX6v&V22_FyArg?=781wSOG#<3eZcmoh zS6Zk}_NtR$I)NwTYx*+xE98Gx_iY@&RXpS(wdIeeRv45K0;ry?7+n~vo_6XEJA5%% z;bF(m!|B3on;r|*UvD0o=jV}TeInQ>4v4Ts!LZ88ixu)zhGVS`{a03n+qc7R)0OG2 zfoHM*1@tv~cJ=%R(1%5deHD93q`5gN&j&zV02BK*k08jrS*<3|b*pvkaqhHiMR}({ zfbv@k%T@#`IwEsOJ>?g4nl$cK^{E>W86}|#tu#2&qwLmp<*Ob3(j*#Vq>;*$hs$E+ zWS~sYu|tx_p;OR$8I9}BUHffYZ6z~F?6HFz>;#9}bd0Cb{mHGa;OFCQ<_@W)wO*{Y zkZby-+A{KQ*PZ505gh{^EdSX{Q*Dm|P>SQuQVjD{_YedsQ7A)#vgznbtHH?OtNUf{ zTm6lEe&I9V)NiJH9Xuj2N>YDRvh&FKbrDETGuBG?qoR##hws%8ao*OXEYxFb`UbXJ zfA3mGoHG131nN?6b)5yQ(F?!7;qH~oNQAw?l#H`Ih_A{(_7@f*%F=rsNGDud!5fRV z*A4Hb44c6(MuY(&2dQ%DFjwnWwHD^*nGCUsTPMy>2$r5qhSRJF@l2}MAI6tQ~=l0jd_xOk88z!Z&qx|p%HCglC$mZ2+;IFja z)LpmJj}}ihQHq=483Ay{nX;lQ()INk0pSTopY?tpuldul(?kjQd>R$8EGbFx(BfFv zRwO;GJ(OTwgb#J;b**)d3cXWa`E&|cc)aVHvroy?>TmY70nx8!9=yMQSTU85*4J!f zft1So?*0Y{#r&J2pd&7Hm!*&UcRWkOc?Sal-`__G>qB0D{ ze9AxV^L-Q(KMWA_1MvhmXdnG^=ss*$Cq3ljbHo_BbqN%*6wU6l<=!&7tru%I$DtyR zrkn!-dMytH9BSRUc~NeeBzL0H;t-xC9uO=hKJV!{s@Z&UOpPfPnPPC^qzxR&e7wyt z3miPfX2kx|Z|Uy;XEq06F$W1G8t`F86H8_Ms;JRQu_kaV6&4o$(yMXNvBSZ4`1xtW z<9!}o5PcAJf@hB5A(p@3#P*OER{sctT+nSflFr{O9gy8N zKuU;>Yn>VFBL8<=j1QYUSssun_R3Et%Dc)?Ksp1y-rf)ujluvM_l*fy-LCk}_W=AA zuKg*LzUMIipZ39VVK`Fc%TN<|1m!h**j2%j}P|R=3is z%cr@H;Z!>OK><_2PqSv-&pOd(Y1{OT0p)BWe8Zss@0?}EZ7G3I)y=4lD@lu6yQaww)bIIrS~~Psrq5srDy-NS@b|4^ zY-w6l>>RA`7CI8_G4{vAcIP<;d_cFvU^Td6^ zQ9XCkm8NgK_oal%#G6Yf1Lt4Q=ei|69AXK`R=wdN_`j2pV=WCrAxstY$2{&Pz#FG2 zV2{KK1+-$rP+fW;T9ig78%W}X|CBpXun&Vpo*VBNMjy84dy(6vf`|~62LT9_EQKtU zPW|{4@EBD2F;mIXrS-S=rv0OZ&tw0VHw-%)cB4;|B9=R%qH93w^*^nJpdcbfV9o70 zMBBd(#*6GES9?`zamcb@CGUOBdRa`X8&5aQ09tsCT7l@n^eO#?Iy@E!94f#~p(#iz zkmP$9Y^VV$rF2)o%x3rkhelYIHmZ?AOE3io)NTG}l`%I&xBl^RgUQDrfe)TwCkyaN z)YkLe8Rv8Tj4^94h%Y(;h^iP;Oa8TGR19D+10yNi*vIj) zni!y=wzxeD7P72Pr~<#7XTbk+7ZEU^m!ol8I!2F%$mAAb@V>K?&1w=bxPYMI`n``B zlpjjFV%SjoQ%>)@-%UQ5_dx{T=RGD|F%2<|0PIxuO!|_H{AI26rC&vc>q(_RDV0N= zi_KVM(!jija7yKWZ>LyM8lTtiJhI^Q`R=xT!4H1o^{9z6MVJbAE082t~(TlD=5(O>gKeGud z8SFZ4l10#G+<)I)L{t<_hTi*LsmO1@R4jsg4+6=2g+MYWjTTmmszu+cFcAryj%-{A9n)EbmO=ngTD`kv_tP8TlYZrnUk)N95iFvKPrAN!S>6{ z>PKN98JObRkG1)u9LWNI|KDd#Ux4#B7P2rj72A;305~-bmJC=_m*a{)v?GVv@l{>m z8KqUGA|iq2)g|yFjnIh)9*68+TZ*b5=B$|7x1UO)s9`!;?+!231kUbc+D2sBFZOT0 zFo@Cd4kqEmmejbU-#(w8Mms`*1OquSq?>bZm$)iLd>{?}Y9+wLYCK(FKT4a4?;goJ zWP>|PHH^&^?;(_ABzB8COL<05WF3)$70RF@1|pZlBHp3>ft)mXLo|R}^vm}R3XcaF zg)UHZ7}Ia<4@)b$iU-o3Vq)0hRrE0F3S-wIk0gs7HM0`azT(=@bncoS9L zfMf#h@#X*vBVKlqk&$-%lJ9jAmm;y=W|`;eG*-F5=7-ZKh0JV)fy;OcKBILpc^sh5 z`?R`FvRLgi{ue9oqZSDA~h%7t){$h15E2LPqQ>e{=3uhLJz zdx=22_+A!--gUbZUd1F+aD-wMf`LCHA<0{yTn~tYm0=}P8*nS?8Q4Z~G5!tv9d1Aa z=#R;o>6)(!1V?Or=ZdGWlq;mWov+kt*tzx?v1|5(2JguVddlHNivvl3_>*|x?F?f6 zGRC{uqXkY1d+1%PX}GEPvZ2q}gizwSX-KhyD?8HJMvq|H@9RNO)b5zXQ!M3+s{fXI zN$oNB+8^Ju#!S&4)rkg76iFpkNUXmF^p7e^SAje2F0k5T(!7kl)lrEdeOJBA{imxT z7}6f2Ab9)btBX%B>yNi3XTkrOafmpGqgbO3v;O}6*IWuB;w%a!1A`--D4L)9n^gMW z-trIOsensrfhfjGGks0KC+)m$#B@W;gwIp%cSZ-oREk-NtH~*zF0)aB|Ch5q?)L;F z3R(PZCxfP_mOekPzEPmJF1ht{r=h9H%=~$;~_MQwrIFs1uq?#vSBW73U3dzS^gZbleykiW(YXX;I`&SvaDIWtWfwgh2A)2vcWRu62M%1piYis zyp2dN9@2%`c}a_x*xzJABGq>IGh2FK4~MrJP81CMV=R0x0a!J~WSrm3J5-l6+iXW3 z#`u8O;PzN$(`~7kVv2hUJqY;?r9*$9z%X8Vi=_pY#lV8UpSYv9SbL9ju8#2#>Qw@;ajoIy~T1Sr^8IAuL)B} zll9Or21i|_zoMEBx6yFQrI`Y@aw;#NJ9@p)mH~W0t7*~$6mvN*KA+p{)j9+};u~EC zul({D4`FO^JzmtGpV;QEtb7&nx+Nx+a1ca)@-VH}Rj*?hay)I^3b5fV6#Ls21R^J= z5~ic=Xq6&s6BR|3@(;qc{A&Q4@@*@`>-1~f?8yE>QtIBbk%?ygM@`Op7>{;Rzs|YU zV<_}E^JX#CgQS9tiPKzoQHna^ST|+*=IIR~v)9Wof($^*<%CtVmA`8I31^zY%Y$95 zV>hn%rZj$e%>)`tHTqz`7`Jm+YDHL(;L$WWNr{y0bFl5)pH*YPvv(8}ns@0$c|6WC zZ7}O1=Y(6i)M#R+D;6J<@DAhckvgl-a0+FD@!!80qCe)yE2@HhY{z2kjXO7uzP!<@ zcMQa=ZvWX-o}a^L|B*t)j4^cd2w`O2b^(xD1B6Ruiq=s#pCeEEt~AV!xLkxL`}c48W~`wbR|0uu{B zK0mdxWf=PO6j}LzK;vqFQeRclwZj3C6WpoLOR@F9-qYxA@w_GFSxpnMu z@4+&{UR|%EJzs0uQy->V$ED|S6IQ4F`ZVaLja|~J0gtbyI%~sJrPt!{xZ0rmb*?CR zPYQfRSnqgTzs`Rb6vK**npW7w~+g+_=pJZ{{D8C z(Z^#qBSV85uNE-ymRMKkoaEI1 zaCi3rLij!K)fc-N4i*u)#i1ZTzRoh`wGG9*^eIpg2rnbi`5xEOmQ$~4%sg1op-q5+OGaS zoGj9NJ;~(4+1+z*`C+rnmDB!vXK7?Sjn&YY9b2kEnRHbDrK`07`0@RoGwZ{M^^@dR93^rwsro6=q+GqRgjhi z9>ugl@j=kvt3s36Ma`}~Y80sO<<)U|qOe94a4BQwGh(m4CAp>bWJ~}@Qw6ZIS`vJ5 zv3VZVL{qO=eR|M#>gno4Fq3o9K>^`rxNP3B^q7rnr%4tnaKAJHmfTuWavWt7qOS>J zBJfeeGr?OwXr`9gKAC<8~g4-3* zIib{ZyI*{{X2H69=j52zMGhw|*3~(}X6Dyu&{s13X?q8}^x;v87VB3fxt`k8;j(@k zCsA!aycDzV)rLLI-H#4|*_AkrddD6QHhD4%kTLqYr=OvXjS)L`^ZgQ<#GB*80X1N* zXP{?%xDHg9o8xedbn_cZiEkM8sOoMi?)qoo z6VUm}=pY92@5u70yq_|M_@6iV(i^@esn1;O=O_>okkM$bWC~ zu`Dt^s@&8tjaL(!UEIa$KRopn{=6TO`xZOZ1t)`Lc=NPV^>W&yM2VOh@@XHlCfaoa zIa?Ioz#M6w{4t9eoGN?B@ezQH69AvleuD zXuN5072wcUReF<_t+YF|x4xvQv|7fj{@=-q;eZE*93Z2=WWTmwUxnK>yub@fr%Ia{ z#ZL6?WqNj{PK%O{Txu2d5dY~1iD^`S^HFi4_<*7Bd_wCVQ-`oO>J-YteVHHy#J{g< zD%qs1GJZOVPB5{FM!q{?QJklg>#32?y!g}o9wYKxZkeu-L@1zDATJ8O7f;XqCiG8E z;*e*C6UdPY{YZbR!|ogi4W%nRfPgSLpqrti)n5rePUfRf0z)Ug4#5c7KZ?@P=%#Kr*0s5$69Y*777>^A#uaVevl0P#l$Vv7fJiX^btRaddv7m8kOSM&hA=lI~9-gY}Ouj7S9*`4BlihB6K%RH;qG&w9TrTdO|-572v!wz)svLVXJvq)96$ z#^=^w`@W3ptJ-YEdhOO7OQ4N}LGq$p%PAyF6hagFre5skD0pz`n*mhF{yp`F*pbT*GgD*diOF`5$g{lX^rUlXFgL$tKwEUSxiX@#$w(sgwIbNaEe@lk$$L9}KlX{=XGy%wsPrpJvO05m_L2>_yggu5 zm9Ca$G_ZjDZM5xowO*DgAeLffqK>OOZhrV=o_&MRbqQv`CScErj_KuCQ#1e0Ml(46 z(H~5A%w)ZV?y9EBHSjLUBtpQJ8S7r*3p}(bhTZZmM}vFYBlnR~&G(AKT2UxuqW%P$ zoK5HiY`sv?@fRep9iUGLx*)vBAXu0YhZQjE>Wh6o6&PJB&BC5M--d6o+-4c*F@Pz_ zr`atJ)2r)O+a>UG4?TMjZ|&AWLMfx;kqELq*R5a*jAPg7RryZ$JlPaLT!5WHd(a5m{yB7?}};5!))alM-1M z6yKLEY_0aJa=WzWb}YgKPy=t8#hOEM4l^JG>DsPpGgK;?az-x@L8D)Jxx8y}GaQ1p zGngPt`uU#@C!RA^<&EXF$qC;MZ=@AsNI?y9ucnZUntCBR0@VaDVQ^9jN1uv<;Hw%t z!3SMaO%)t*CBZsRC^9*PXOG=YS(3`?seq3 z_1w*mZe)L|{pKX7@z2#eVpwb?X1rhMfyfTGF72;EV)LL*r(o-`C78NvInCvvP;t-O z;UBJv&rh=af9%PGgo>Ige?Q`m`-a6zaxb)heL3NXeB2=l>;hq(c%=A~6EuGcT^bjp zfYmjsq!xNIA4|hnE-gLKiUYK^Pf0RTg@%n;$zw~gF|$b}y>`OR*@z%a|DT70iqiWY zYbV);w|w}Rcw)=00J8;@qb~AUEXP%(>b0XgKu2ue!fjWef6}3~^uTkP8GGm|{q(c7 zZo8z{8ZP|sLZ#tfUw&^q+SUvws``%4CYiaCg^cF06lB$lX(L+Q&Auf5(^gi6ibThaB;a zuw0@v;m#}mpSGnmVY%L}*BSm3W&~cda-K~RzhH%?L=8L5m;~f_#6W@K%E^?t* zdV2uB1T(c91Jvb@@ECPT1_}yD>i6I(TD+sRTuX0xYDzZa;A~*-Rpd4|CtwAG;LrA$ z!sfO_&E5V$@lOarDg&>B#x~>c2===CcW{5$z4G^KaO32(_JX8(iSGJv#B@7VYJb~PwcB%iLRwL*{m=AgMB|DE#7KIwR+4(>ota_oglFB1>w zY{qY0v))on&-QAYttgs+g;|J++8`BVK=71Ta7gNLP#4XHO7&55SH$h&#G;ancj!Zl z-wSyh*~i5wKF_w;~4hP+DAOSI0Yd&F32$OFr6t|h_|#S zbr#z9z*+|taw3>+zU~vJgI7(Ul#&(=Y9g($w2ip^OzW(B8gCm5+@>RqrfW9 zO;(x^QS7^LybxRJ)rbE7!`N8{M7>4reh?H8kTU2PK|n-WI)qVa1Vp4kVnn(-l$K6u zq@dPbbtYd?&faJIq!SF+zk7HO1jgYSaj=axtGyrien!s zc-fGYIL_yJD;6WJn3q2@+>+&i-J^#UtyVN|rrzPqot-R(#G24fQIHzuWq{KE?9{;Z z!=CMe{V>tsr-FFJ#@>f)h=jg|j7Cb;BH*ho)1_AZ5*P(_RcV!)&i0s- zB$S6+9i;p>XHeKcI9XbLZGy+BycP=+6cf9%sWATT=?JS>-9Q0^F1ZS$g@-^MX(Nprl!*hZRnELb&~umC^2fRe0`nbTSMh2z06U zHWU^P^J~T1vxWS~&weTONz8O;UeR9Od%SUjBFVNhiq}3wqB}zJyc7CGZp>C|&3Skq ziCMDB^E_6TG%-}*#AII>GlQj!~W!F)Z(M29t@s{Oq( z`7MXl#l)hr^4LQJlP`lO?hGQqFO0Yu6JphIuPVwnI=epzZy%xue^m)0f+>0N+mIIO zWz|i}Cs`G8MQZxb4a`QylH&^#?O|hE3yM$Bll}UM6K?kv=y%k%s15z?NQnhQ>+6|~ z(EizKFH?*t*5s<{s%Z3z|8jy3QA$Sk)VDx#6zBJP^u@>1-R&giZ)Hpwy#|T{!LYCY zNFUvBHAOs}ECzbRX<9K2@t=K~J=C4c6s*5{Fd(-WzDn_Dn%0>M5y)FZ!Ws0B({aRs z5QPnCp*LcTk+AUt(yL8>^%gkQ>0Fc5>!#>Pe#y=l5N~qYHP)NrFyhb?d2!xbAcX5r zkHr>G#Ytre>9RQL1u!!lY>o)=J?t_}jxnu1`6Y0X`xd8_95GqKn<8gIaScOci_lAv zKsgLY1w(z5W#kRkg3E0-pN#N`uCT3pZx|ksmfoWkY~eUaj^QmBGaur8;(NfVs&S{- zRO*>-V^wbQJ`DYt~HMV$e$@A?cDVS|w zT9c7p*HGqXEn1?R*D;hD*iA$rNh)!d{sLZgIM69%^-AFNW25L_J+zTV<*>QOvk~1Cn~WU<%c0&u2Fa&ldeRrpNJrNjlvavl+HHIN+5~ukI9Wz z9{=!+WF^UbqCo`hgrQ;pJa)HYtIEqfRtisxP7lO7jEYcQ9~d|sM&H6NK20XrgUgc* z>Kjy;v19iY2aU+u0u zla|Jw?xKokS_R4JSyc?F@br>RE9nqG(q?t<`^}9s;XYWFQXw}2VUbHPO_|@| z00YQ{7l$h!s8&iyHGi%}@(Zzk9BG!UMMT5k-j*GH>e)e~$_~d1s?d zF&fw98Sd&` zhkXKcCL7k?yz&L6pWH0Bh9~!9kDou4f^(f)76ske*k{I|2Ch(?muWw&mw&-8c4-H; z@)7mdX*r$Le3>P5d=YYNI;Cl8rOszdoUG#(58I}-4`@WfY|>i`*yDG+0&^f4NaN zmrIO^eaueI{duP?s~~KK#U}vhc*)eh%`U2UCubtH%B?lQLeBJab0i;4n&&QLcmhsWXr{*EqDlQ2cGHAeL6)n4;USBco5@b`pNY z`rcBb)?p--)*0wM0k;1u9nZ10hNUxWEC0_^Lqb%Cn5I(v5PJ$b?33t7ntcN35Gid{;+?GH6}eT4!+AOwZ=VJ|qB)J!Om#Jg*=HBK+mK0lTln`}0E+Z9Z|CZ%oR0@1~kV1u7Jb@u}59 zDdw&Vsv`L7CuY+=U&(?VtaW!-&?nxOQn%-bezTxKxE!9hNF(t9q8M1 za2j;Id(t!Q@c!~Vo=?XOi7AiFvwIQMb;o-QXg^}d2c%Trr?#HqNOLQsd;p67$ITVc zsTgjKO$_vNA&XkG-(Do+QChcsz73oITZ0(bi6_Ll3~KN&`ZccO@B$TFdUTklG~&d6 zpLtXJ!+VWPYpWR%Z#IzX+iBNtcF5i#e5P6IqM|KX9&IB&|My1O?p1j*&iAvpW(Qv$ zSz8(qzVy8onWPO4-AUHwB-}${i#(4^v}T(?ErpT*o45Z$HzdI0`9qI=UKAnqgyZ7y zA5~rgiEh!XSE#-?ha(e}Ve<1bJOyOZGOD9aMH@ma=K04N^XwtJP7@-#smr!;=HfRl z>6=tRceiWNMygI~!QAbgR`<*7TR*~iT$UCFOdIIfJBX&Q+m6t`7)Rn#)9-OVxVG$4S}VJ?0|;S-Jh%+8l9Jdmw->3g5KofnKZ z?zg712{|y5uxO9pIbb6fbD1z<3Y}M-wK`E}%E)HBdZT(}-vyhS=Js1G(_w${WI%w#zJK!WK49Sju6 zxNAJ;|AN|>DRZr``t3G1hg~F9AceluAW@oqsu~Yq@NW|$Rh=X z&9%lU1?;@SeSU`?u-KJQf7PbBEoq$K;$cy!tG+$+K}zaV&>ctGgbm&Ib_3%UagBPX zQv}n%RjNJCx~V-_thgJE45Q?>@j`DNDoEpg@j8r2WS=f;EGx@@9#t9m+qwbBiFc*t zUU`2T7OCg%W>9@i{z@sD20~rU{VEzRTZ z2lR5;_qQAd?2nx}Z_XDYC{Y4 z7E+35Tf!>`7FihA1PyQ`zM9u7%K++v^3HKsmQeCcQ!%K;S7un_4O<}@*^ zB0LR=i;h<(ZCl>K6vOMSta%Ypu?$;BA!ae`H=b4W{l zDy*zrezWPDNc4Mp38t_!wN4mA9$WgB9lQ1_hwv6SC1-upTXC}l-k6Q*+B`pk-qg=` zUHw@0>r>tKsBzlJr8oN3m@!00%-cx{qK6b-S7}-3WW8D|i%O^LB$$g(m8-bbZyW>( zSeR#D&GvRz30-Be`%S&a8I)ZiKX+#W9W1gSd;gUH*1t=u;;`hcw)VeLx->K$o3xdz zlZu;+`%0faI65r8u~zSl=Y{Vk-DYh|^Fo+HR{0g6@heH+A3mCMnFUiVBJSm|(`;w_iGCD9SYK((a{5jY;A$?yD`Wx`PGbY{}VPx@*=>(bUi`v;^?f!LFs=k-=e^to78 z2vSB7$CHiK;M2MwiVk;XHbx;vBkRd@F_KwS7SPfu4kvBs@$+#8%DT{pHKhou5pzP>?4t8z^Cb z^>dIHmsi^DF-ZFq1ZIYwPChAW!MuMYl_9(`-uK{E`8diGCJkzv5oe zeK&#nk(=_(3!*Yg;ChpIE7i%4e;9KR)2bORldE)js{{`CE%*-bIn1-Ku)uuU?6mZk zVEsvX2{yf(AKL{Hkq}>99`x>YQ~Yxc4)DSEfls=hr8xzUKFyrI`70`#|9-gK^6uaD zOZtER|DU4Q@P011a^T;hEtLKZ$!H!Bfh!7h5xswJf7qz)tu~tdpfyr4qODO~jy#g)Ti6kY^?N}?(WW!R5u9F%2jcvapv#cr_3XT&q@#TRJrwg) zeN_^C(bf)kcP96dR*ceXaIo+-dW~BZv%v}5uDF+B5of_@Y+vzN=JcJe|b&ACLaTs7B@<3bnHkCZj;czVu# z%h7eJ&jf6{;~Y?2>DNm9Q~c&ARyJ+#!!k_@1yZj5(9)U)Gg&nJorzK)GZ#v4uh}IX zpVGEf1Ld;mUyXQ8u0QM0L>zvK!a}xJk$-aJD6f4B2JE$*m$v9`?gljs1_9hguQ7H1 z3DYdq&7)Lu+KUaA$iv7rxhH6 zQs7b_v0-{eXPVSljP!%-f2jk?q)44(;+8rQA21wmCYZb{iYE)x8j#K6j+Z-%viRLp z7Mdsrr|P`Sq`bMSPA4j|D;^d3d1>H}YZBOja624z5bi}Z#b_T56aRU!6D}@=3rW^aASEW|?3m)plVS;jt58dKl>AEg?gnnM(HK^5GPmTx7x>RA3`EL;JEASQxiP z5@*W?vZq{ZQZF5?)rYIz*%Wu^VatkVF+e3YYO=0ah88ofsPd~^i6u4!oJS8M!2*AW zSUnpE0is6-y;2O!J{OsqnK$?Tg$NVf0dYhcJ;eBbcXqvco9l2n5|bqBJ1K7_3nj&q zcD=GNT13%Jb|Jr@7|=c80Uo=4^<3??2cDcM3|#gr&0~kckxOB^CJDx$R|72=lkhcM z28S_PNL^Dx1FrU7-LewlB6!YVSbh&ZkoZKF2?a`ztPbT2yHB9LNozmT z=+Sy!Px!%w?|2#eU3M!5UDTmZH2x5ey*4uCQ^?jmrxR+k?jxUYoEEYX9*cE38KT+W z7H?qLtqy#z>3-<1X18WxscpD&oecOWy!TH?E~GYh?K~Ek6i)jHRwb1_Up*c($Y{8!>d) z@QHt>SaB#C`6h&1;K$}VEyfDFj;6Zyc3p}I8Q9en;N>adNt`$H$;fz0bW2gvBMZwh z_O=fZGgTR7X)IgL5+hE`WWNJnoy9!Rbs7U#QLSAaCJ3jjdRO?NQGfzBh0Fab!UTYV z9Ep`;am~+?OPN9_hnfq=H%?hW9Y>N$R&vqk{Ky=mHf$h29s= z)#%IC8pW^49WC>r&P;K#qQ7`K4@Z~Q)IVp$unNIwIp2TpR$*A+21Oq| z=SLlo+Ss=J6=$U)NG{T8HoyAI3rc31B+~LD-|pJHxkv3ZJX$I+q&M9q4ogC|WudFL z3)j`gT>sWA+BDa;fU>)alB!Uy?%D~bqz~P-uLsWu)sd4i?4TExY++2wu_gy>o9^8e z)){Z`KW)OmQb5z%{OV-!YfRs+Pse{jZx#y6hoOrn7{D;Xd#U4*d3HJ$_lX@<|7NUS`kFTG+U_$jA^+vZh#5X;{ ziYvzXAa6XAp<~d#QOCDdC5+Z6p5m14C8!yI!uY8S-_f$aRoA<-`GKa|Pm@z-V}Y+E zvrQOO)_$>mepK|K8vJb9;zUcwq8BY}Xl|O5T?*s6oZ#mEUN{r30GCV?=g|8~lxJsb zdWv9V9h0XJE2CroYe%C=ih(-WVBNmlp7rgrdqTz`*(&}dS-aGDAB;mrv1y$CqI^y< zSnR0j)lJr~+pu1AESg}wm)S-7dP7AEj~D=@lO@JsMh($*BS9cxoH<-yr@vLYTQ7?uc*A_ zWmT|n4cXsIc4Q+~m(n;&MwWjhd=q`1qc9jj1vV2AL-Jh$?dG@*e zN=loL^rfyAa1OndpKYx_CKq3Tr$XTw6@Hv~{UC3lrQF57Sn-P&mC*;!YZeOEGW-bt zCVf|Xe|=neoa(Y$q;p!EohEw|AP%ei7p!SOP;$pn7oyluNKA zE%zNi45kH;8S}jTuhGEAxO!LVF;z$~rS)WV4LNm$4r4N((HI(>NJ=EFU0dG|x@&MX zCy8@fy9f{{VNPiomip?wB3^^OIiKhh(2+r&PuqRYWTX?*E1W3g1K`eHgXaIo88JQY z&A};>dSj@_)OuF<;3$LPcvq)zg+F%Yi;;g?=X@U}b2nbvQaL)F69wj0HPup{L&YGj z=ibYJeeMZ)Wzn?NhMR$X&ffZVp<)dw^X)E;+sWx4bak(0#mWI`=cELkL(=^U-dI=1 zwY@)-k{IZ8Tg9e@-aK;=s-Qb(vG?Utg{Q2u(DqzJPo>Az?alXl)%VE_9{U|#uRnUu zi!U!F?o+Oq8vXszIf|5B?ayd4r|BHQ_ta(QqTTZ7(O6x$XPNd9HDqU`qv7zXZEC9H z6NBA;<)culsd3NF&R{B_QyrkOQwxhsmnhT%fvmoU@o&_gvN`g>_F|AO9bJoblXK0*(C}ri2Bm87t zx2>*NlMYQ~S~ks2q5y!WG%z9w^^MHl;KX;=sB0RUFY?ns7*tFY1AZ8-EWZ+YiD1=I ztU**zn+UaeE_MnFPFn*|$g{g*Hn*eQc*~`VX(e@BKVoVPM}UylRsU zN?e&)7Rq{=Q9^Cad>$ET+iN&0fcG7bo=KieK>uLDuA|RTWc<)de9*+(!bU8_oM5{g=lvCh z(bI4;6Gb4|Gm#R$G0A?#Kdbli@^D#ClvE)ker9l$$P+Wv0 zbWxluX0EL976#8fum>aAq_vUA*8~`JGuq!JkTZw5y6V>IChRP~#x`S8=$ePA=ab(@ z)7$h7qOuUtTMo^B0|;L!luCr--_QDL=iz1W{*v2o|PG%kVQ5yf_VJ zJxB{k7~-;()KydEjQKn7m z08x2)_OPDDB~{d9U<2IL?mtxR|gGWT)>4g zRX~ftqsZ3;wD?Q{E7VDW89 zo`BN$;Kp?WqKU;+D8j0wCRb0f1Olnw_XTU0R=vKayJR`i$uD&_t6bzUX(SleGhfe9 zoSqLtEMs={Qje2`tIKW|!c|SJ@YD>`hW3wluzwMTq%YEKh^zx$2=Q2wB@5+T?|D>u zD`IQACv3w(^^cJ`-+laNz|w*~`Zkt8gf*x!fm!gG(mQt9MTaY%N+gk*3E1cCFi6LJ3;W>T$FC9+I zdKltQOi3}IhGkVK+>rgZ%q86vgejdC~;m!gxjMD9$li>#(P-v z$1SHyEs&vJMgW6pa(7Wo6n5$ML0;6Y)1t|Ug;Fg8#r?~BUxJYSS}dkZie9=KuA9Pm z#ESj$Mq)JL`e`?|WftZ!Z^%`J#??kHzTMID-)af#^Z@`11TZpVdpoY*JAtlS*CE1E zHo5U~8=`^itlDQHE6Mb6hg#i$Ep*U#l0bhr!1-4jARN1qhPHmRvBK;`C`OBqkGIY) z_EagyEaz%78uF`}@>o(n{ez5vP7p_w#lkt|hEG%NJswiYBgb`I-qY#H)H<3FU_p1# zqo@^Fy^4y?4TTW>_FpNYfrf;Lga~R}R)u!XPIKP8gG<~R1&!&t1?ZW}#q|FOi!83j zy$n}Ne2jm=hLEw3_}CO{FR?mf{1Ebqb~6>yoknKGRS4@%KlAcO&x$Prn#CN~N?y7v z3hLX9MQAU(EvNft}9nq07`UUg$y+ z_A^p1W%9(Ss%Z=Mu6G^#J{>hi8s}S0hui07Q4C$~H0DB{AP?suA^KQs0{Z(KKJ^Yb zpMOsWS?yQ@dEj1N`oCe8o`_NJH!s^*1-L0XktooCP*NJ|_ilb7NiNG?o3(df*K@9b zZh$Mq4j>YmgyNg177jneo0vcLIB(-p4Jar!1F;4Bh>(_F0?gj$Ev@opudt+tIhZ+$f7R;Y;fgo2m&Y7T+4sjG^ETuf z864~tm~TE!qDSRE&%Z(Ycbeyo2Q<_Ey(I5M80oO|esih20-8O5S^7+EO<4-2RcUMs zjmP!>3K&6ph@v=P{sc-((BGB|_%P@;uacR3MtNtHl(co!+xclHir*nuzq7*w{L*mg zFO6(G)bZ#uuO%embsd>z)GU$8Q=wqjj7*^(+R=?Im5Ue1_xGwu! z6&bSlmWUWlnIJOYJ=XYzDYGN3EWaunAw5ztW)zB_`>psHYmbjoNPuK6B?qK#SefXv zbKCDA{UnT|024a#KWG@Kwx93EVhj$xG%{km4d@!8QYpQQReeaw$HVMjtS>a~jPrDm zX*_Y%-W{f-H7o~&egcn2rWt{33r9uLr#Q~4j0Osb1jZ~^tx>;{%#6+1B@umCww824clmEE-D@c zL47JZvxy313?FKpf|~$gB8SX1?d6Fx*;#S2*+0tfHzm3Sar*idXDK)>o3m#Dunk&GE{~zX zv?4rw4{aL9sxvH2z%;8rl?0|Q%s}8o4)~2H$HJ#KXKU`yIrjd;;Gy9&5;CFB%utUC zv%Wfcp&|#Ari!o#a9+%|mzY2H&ouckPITR6d372>dkGv7^Fx)PJ(fW8aC0GwK$Odk zsn^+fKe(6skK!t#OsFoo`u)5o&=<+pC3c03XJr>O*zw|mOf(c-fb$;Zd*T1}3*LYz zF|jlW^sY22$)l~CH<8uhkLcA#uV2SW&5$IUWO06x=l-m(DWBQCk1&u1WL!}K2t)j= z``NAjglo;<=Ea!uU7kif>X8WobMla(R4?=?D4y0c+U5BnqCWO zdVcm+(eWeIjeG+Cub}DeM*m^6?$hw?M*^q3m_mu4Ck`_T}2ei3dmcm;v2lj=#1l3=yp;iPgf8v4@(X6w%?6K@e6j&w*n!Hoe7d4m|j+|0g zZ!|m1wjJQtf7|@A*xq+RXVw>AMDb`V#unsC*ZXanKyWiG;K@IjP1N|+`Cs-jAoWOfpoS%I)`F+k>+Y*#=8Pf8rj{aQt&DLZ!)G&Jj zabI&v6&C*g3w~@w<;4V)RT)fI$xI;t$L)(KHSA92_VhK1Uw;t`3~^zjU|THQ9OL6T zX5DlL)mD+S)NzmN@ti%IjDR?~ENwMdTN-6Czau!cY4ecRK0lH^tjeY~p8!C#!5sdl zIn^2Yi%)crCq)L%9dE|*N!x1d0ATs8)|%WuPB@o>qnLm(qwqt$bN|<6_34*CV`1x{ zfKUA*H_T++o*s;WKvy7GeV4yO) z*8CozL6W!PRMwscI6vVCnXF-Kd~TxFLqi>f)dbmriviF1DOJ^G{e1*PIKw1X+M62pPHx*lP5|kq;RZz^*sjGev8Jf zub$RTc7Oc%;C-p(dd_rux4e)w@@&#Q9Q{HDPsPy_(^s>;0im++69AVGJnEV{7K3JSsl+geq>q0%*V7*H|%LE@^Yxe{?9LNnzQ2|KpS~pj^i!9h2SLo z#RcD1zxWrhnQXe1ob~m_ezY|Ymq6}!)YdF^G>lk3^G|hSB)JFGvy~#fq34q+7!p8bt0NE^Gn0>>ev5MXD<8RY(h!t&u9`Ibc6>yyIGBR*axiS0%m&8sEGC;@%8~y$Y97r zHqB;W?Ag=i?K1SUjZGw71GlP1Nz#vyx`ilpew3;9i+eE;od~l846J`>^ukDhr+@Z2h+RFF`@|i>eW`GC(E&Jtbaband9(SZv0_&CLJ(i&Qk&GPdOm}C&l}U!f zLaR_Y<%sB?WOy1B;NW=d$MpSAIO#VW%PNz7&Koj|j*sS}Z=T|C^fx{6^Gn^-%N0Tz z=Jn*_TQ^|=O&&-|KrNe5Kln6=ip&6Xo|BOIsxabuHc5Z!76%c>@_5?QKUQ1W_>-L( z8{z?#hluIpbNS(Gs(d4gAS&Y>SZSrgq1z}Aek+Ar0oOfWJsaAh@}0M*_0%;F>712T z&Gdwc2Vj3UP?ZS`Fgwl;Xc>SHUKOns)=yC&X}>q$X6c$YhBq<0cXe$+PLsFl@d~i}wvsUNs#mypoB=_7Y_R?9vFZ{KEiX~K`R=Q&$ zOnz_W)ZcnsWLSS=Agnphd$x@pObMV?yPxI1i<0O0r2^WQnF@t=)XF>sHoxJ=he>6m z=R&!7cEW{^4xYZ?557Fz<+Cr*52a9lS+|Gw~fv8BW0a6JohlRTLvtI&|IO%;-KYy{c$IWNP_Dkk( za9yJ}V!A2K-q6;NaN8%U&*Kenp|_N5g~@~b0KSK;A&_Uo=oz#xN{n9=1g(v=_!8~+ zY%~1Qa)u`X7e;}5>qp)A59ZCyex5I7BI#r7MzFDjkC@~J!JLrDNYNFP)PH5&4c`E} zWTUeFhOqbQFZYqKTmBFX+;`zSxZ+q~{SKUNw(61}PZBE8m6lYw_``>~I$~Gk<70Z( zBV6%QBDmCStiLD31DR65F6`>!p<{;*&G!^J>}XePGG3)H$d!Y z5{BhfDCb41+=|TO0DW(4f{l65MZTscw}g(J49BXUh#W#BO>Za|l~-U{n4dqXq*H5z zKB}aAu|1`lES1d@g8eKAjI$TcRgAfd19*Y^uO{(`*C-0i0~7`J$5vu}5GHRnziG=5 zhdd-vf+`J=iS6J;xBJ+RjV3khSM`8FpF+a)MBsjB-IrLeJL<;{Vy|Dr_DiGQ2=u~1 zuwI=Pr>0o%!Nw2y;lZ5r3>YR!p6EB8SIGI)fj->xn5^jbAWl806mi8L$TW0X_z&CR z$}zfdoC|-luZW1W2m~=4i2Ui_2*Q|Qh5@1IZ0>j~X8w;8;VK}W?*6$8ocehF-m$nT zkO5434eZMccKz!Y;E}N%W|X3K9+dQzzLszG4&OTmSy) z2^7X~i{7{6baVb}^RZ%6Y!vB(^m-H~sv%Ec7Zfp0_KTa>a9hdW9LQ>=>yp=`l^UHn z>mu?@hYvFxN76d?u`xvg$rA-=Uf)Wmd+HZE=^eBw>TB_`H^|o~jTlmMi_}DCzKs=q z$X4G=@W9wSYAkYvBFBrxUIFH0#}rIUdZF}k+M#UHEOq!Vc}}X+5qVCqpoJ5EW-SeW z6;o!k@~yL4kK=}W5Qj%9m{2-8x{shzjW0Q3FO~JPjv#UFWZu=XasQ=}YNb&^wuuA0 z;%yQAKWK%KE^Kr-Qh0rQmD%SzDO#R^_33+W<1ZR=PVYE6l-GLyeAiM6f3z>0X3~(x z>9K}tzMieO;CJl6g~4EsGd63s#ZBXvpNJqnM5UirS^0F=Eis67JWb~;Y25aFJ+Gnc zeCy8XbYc#F7~vOxuR8Z_4dz-UaT37yP8Z2fT(;` z5QmT)!5-m5Fob=c$!1SZC69frOjGY^4;sTp|4lIZ{X-h|qphv|S1R39q|^}$Thrb! z-t(rNjWW^OZQgQY=Am%n%P*2k4dtiAMHtY(Db08<*DB;;N&BT4{Edpv>#%D6+!QBfSVg|KqTNdS7o--9x%RsdmXl}B?`NrqVi!-O=i&8s-gE5~)2h2S=NPG>orz1=_|b~t5E-*?oC z@V-ZXzIkjvy&M0Q)+f;peA>*b&Cv4>o?@pLw(JclGFk15;nC^kj&C1)xw$U)jdZ_wtvUbAh1=7^ zbi(KP#ES9XE?4u*0PmbN0I;dKZHS=jb+K~BDXyop4Z}L1C4%kY_O;+a@U=T{llj|9{SreQ` zCE*{y9Zr%!qS36=Pis*eXM3IP+N>;K0ZSluVoQ+X-V2A2VTMF?1eS|pAbf*%Jb1!#&cP3h+TuW_@3yhwyzCN z!sxo^sHD>NhY_dTQrhF-A;R0+_t;5h7dxw-S4iN0J{L*%wcjE0CfWd(<7fvv07&{} z0x5&{4nq&qB9t}WbBHxRW3L|NYKMIuUHzo@+iaR#%s2pHdgU?%npJ?kb-nIMmfNPM zEk+A%eVomOl&4cT{pTcZ7`@MCO`hG8$F)*Fgt(5> z{BhNt5HC9F`)nF5(c_+otIK)Iv)RpvcN9DbymLC&Xwk{v=m>w(f(Ck~&l}e<0OiSEio}7)Fo<#aXSY&jwEc_332<^apKd z{20$Y2yoJZ5-UAaEx<3^NzfDtQ!%uM zu%Op8HVU-1y@B*r3JfnRKE*z7b};V~YMyrMs#sZJh{E!>6fZy_lX8^TKm!%AtCi0A zlc^9`xZL!c=l9J92=PWtG)deq3;3>f9mTqOdZe?4leW*V@_7G6*FcQI56`7;-1COC zkd>R)7usbSkt zXo$2Qj9G@k<`Y^v^Am+8NnS+yK$bACN2@4p#azh>U_>_YsYDJ=~|K13VN~37!jkWj)sh!pL{*T3} z20euhTF4Hz`7AciCO`Y^Fw;f)AyBB-7%k23ii%8!c|roh^##o>XG={l;?k5Ovr9pF z3a|rz`==2A(KoNYeXGUG=m@Pc3b>gpRI0M(_(98jYB}{IV&CxK)+*|HE_ET$9*4g? z8$k@Yi1>Cf6KArT3B9&J;+s9P54p02|EffdOZo-ki`n2^T$H>uCMMcyXfigu-Y9C` zoE~iR&i;m5pKeWKDJ@4Pw0<>zm6^+2Tq}SMtDtoc_$~0Fv)ubmtjIx5>ip%V>vzP5 z2SKl=WbXdYM$x#@&ush7n^Vzg$g1_nmnacJBK@5oENSAsq+3cB9h?C;JRxt|UP8nT zZ)_2qFzCW>s?I#qPZq8#16SL!*=d+wC|s5-{Z)wT9~oP!@0?==9LI36nm+59D*a~Q zdY^W9y!a#9=Qpr2L-$l3`$;?;kIR>wkPim9nw7ZdRU>tdVp!0_5Pj44agQ5y>3053 z1+VR@*;eb=TV7LFzMhDD`ecI$)xa1*C1x?|-Q3I$0^^tpvAjFAu><11EFcu+!gLjk272zo;lY!YgX`cG4NH z(0DdXZp7FaaKTi4hZTY5?B|C=e7q=70a6egx=buO$U94W;M9}S3%ONh$g9trdRHK(pXEL z3HLn36=BNVRWcl#j;#WUIpx5g<#8NDJ98GZ{gYH$a$NMmQ2swFb1KOju?&34m72-2 z;RsCZP+O{x!Q36xlzxQC8t-@p?aOoTl2A$GEI5PW69gjtF(B=2i}lPqL16y@H6R{> z>v))l*PG^R^azu#Q|I;N#x{^nej8KM`w&5+>_s)Iwp@=z?DlRuUTm=|o-0}i1w9)H z&yy2Qk;&n%Li&BhfE~04*a?C@cJ1ESK2j`+my5FX#z~_azfPG*2+nvEbiRg(K0h3= zjK%a~Pz_2aAR!{YTy@nIvsh|@eQf?j)WTuc!}F1to5^^s?s-=9UP)hlj50}t30bYc zdj>O&>@@V5==^fhzTs1$&r2=W$Cs8T18R@|U5>#h3UZ@>rOd!bns;R3SO!!OpYX*J z0qeF`#f`f{?KrvCZI7|ZZAI!!Suh}988F|+(FVPei0TLf7LLWY3sXJ)SdivDH%XPA z?@d<)-;MhNVG_4bIavZf4LFnBy#rm;vlVt8P`u&-9lAWP+a3384X)@GdA}#uy3AI+ zdomI!#ja6>pc4atzD&Qm3W#k}^jGapo(H+ogs4bLoKL+vOl_{2GaVA$rP#fi8RAy_ ze6>sPXZ{o}Z81Tb`Bh>e=@*hsx9QN&b|7JlYMw7BIGXD|7yQcdh}E)Twpy`kkRB6& z2h+v6mXYBy(~yZ5>074tsl!`GTGOBAn;r*Uh3oMf-o*vkkXD7^uNAgwQ}lXq{aIvQ zRA=_E!ui^9(*BCmQA2qSo38WeJnd=N|73$mW_YyzZ!^BxYO~Sb#7xHKdl_H>gH2~! z8}w^69K_VsQP){i5ei4Q;}DxqgH_z@yz98x1L;0rUkNnnO`e_NHzR8ssOG&9KI1<2 zr@!}oKFKOg_({p&CUoHK+v>U*RBhG_h`Oa3t;61FMPL6;Z9-Y**HsNwC07B!gjX=s zdvSqSY;@pRKV7^&sekr%z0voI2;>(7cELSUE`i4p!&`Ov4*lzcXuOQl!BWw|9x|{L zkamP!?G=qPs%(j(Qv(*!{Z+4Ejeij0253Q*FO`5)%?J=n|!FOSa~Lz9+*P z+db7<&32QY|3k`e?cf?k)6-!ITb~_frx}%(ZQiNXMbOJ!IiGL{EJFBZLmBh(jk1lU5$DNS(c&01`j@dd&FYUuCF`NU@{-9Hr&J^(ie4 zxJ}KgE;OK(vsVi_>+|m8)HV3jhf0S6hL8BanlQ45Wn>2VFSdf}C6wqRlk7!>r69^s zNeL{^mP5UAF5)i?2P3g}0|R~v2JR7IFw%MJdX|1(JbU{zh{E6C)OMXo^=xI_0?)V5 z(B>M(m1*kYZ1iW$TnO*(z5x72IKSBL;huLpzvNaFTdQK%60(7LPTy(A!jgg;zA%NC z0~JSBFv{BO5s_Rw&K*nv0i_-uww}9KZ)L2sIq*M}+09B4iV48ypq+ z^w0WXUthd<%YBKe<{|%p6eh{E7RP3L|NkC8j1vZ~ZjKJ__@8k~+^_Ky&88~`2<$rs ztClB*n2L4W|JH@r-g*y{dB6!i4oEv)fxe3IFK8f;p)gE)dC8wu=eMXP#;EJBPEQH| z>3WHMM`+0f%ExpT)~Qpwe1>;{W4rxEB*fN}2n{%{RFJ2b0pNO*<9YTvCx1?dkHh)x z!;vS9u>1lSZr)plNZP(|YvKZtT6`ZzK^G3t%|o@LEN-$rVZpDa`cnfpWve zlqPMz=D{y^E!W|LF4HmdnG)-s(7o<&-}19pYLJ2w6$oxaki*Nj|Lz%JVP#v>d@pD- zv5+Qtr)hVrK1}4)jVtD!zeKE&l(e!8G%&#akU;g%pLRd|`zhGK0#N8W6F~nz?ukD` zz0BSLDJad?8W&sNi84}_B;OTDG_tOdqw(_b4Lcv5shr-(9G}k@X$?j+O zS#Z6G;t?hizkylaZ0 zXVjE}cC^_JRHW*%Ten!UfpgJV;hD_{wb)?>?C@c=KTc@SFXt`3;@roCM0LS;cq3^< zk0cfoWubvQv~LY{9gRmuOQqwDl0)`3%WoTZcRaaV%-Kg36FAcZ(}XvkHu$}~EZ2s` z1e0c+erGpxvUxwZqW!u>_~G}j(67k4#=HBLbz66*BdGgpn4lq&=H~8W4$!)-R1|FK zyFuWu^fkE^KEjJlOG2>-xoope0Nao7^pmBF8QsSa$60iSCicB}!CS1`Jb5VG!v(NJ zXatx1V+ve6df!i)&Nv>i3kg$a5$)Ix{6S>I+T$w!iAqJ}MjDavovF_ItA^3Lm~Gr} zw^}uQr6-St8s=bzp^t-t&XcUqzUw$2eox3Q__S#C5_bLZV4m=5VPho6NrD5`5r?h| zyFi6>re4l>unBRi`!fK4F79G2l#GM$GnS&v-481bz6H`#z`L+xeS3 z0Z5F;{#YXP1BupcVq489zv*pc@F{_i4ffS!TX-b-<`Ev{+2O!9SWde)Ogw4$Mg6kc zWy&R;@u!~x4a=`-T$#47ir>w-DP3Z8;xb}ze>#7@pP!0<@=20Tj|ke(#BsJ%UweIe z6>5ScuuxE3FD*cb(=~SO$lS~$ZwMsGIx>5GAZ+Exvwp}WZZSiQN<>vkLtQR z)Cx7H_sKS?L)0TE@$X2a(GhQ^zVoBMB?-wk(Nli{l>tV~{nX~`;}_7oEL`!=$`rO4 ze{>@YZ&FK{#KKKOt~VJEg$;ybEaKfR7(2tn9w%<{Q_Nh-#rAcxkPf6cUA)S(lhqs} zUpDxNuSv4DJ@1YLO3Kha)-A4%_r>!LD@OA*$dq|@-AZYkGP%cOJQ(6gVX}&))=ra1 zX0i%?6od$0Bi01xWsjP99a5-S482zjc>QEfKd%&~=Yq$XV324TenE75m`2N=WTkHM9C37#FIkE4#`|SPf=V@m&FM_Stw&ef>JDwFJ zSP}7TSfG^sk4X;Zv|%PI!{&B7@8?{+REj+y?&w=YhFStJTF;Kq0xvBsMGCFQnNwxtscTY-*tAaySY7KD}bb} zWi`WzOS=YOB_zhy4Tlri?q^iV7%pvt@sMEqda~%K<)2``fh^bKmD~weTaS;2}o@ zruWDo*Nbb&;7c_>pBZj4_ZD;DTb__I#q9&UW4_`XG53>IiMg_=o4Kf*Kv*kf+i*J0 z=aZ#{UiNT>sX7=et76uu6!MoFD4(@F^=TI0ZUlpOKrT;CJz<*-=ekUcUjktj{RR*7mXnXci}Noh-K1^mtSSm+Ojr( zO-++6onunK(tki zD_>Az6mv$ZG*i5g@sQAk3lhah#5Tz9um2u$uTm70ne>@XYG zuRx3-nUOBL_hs)G+d4tVP8(B#k@!*+2sy0mJ74=vW1`z9zU5m#I+U6!F$;ZOyIRgy z3_hghY4wLTEGi=rkHsoW)y35ZjNjZrYPlSK|Lo-*mC;V{sR}pew&u{TJE%`Te%1Ag zq@?TDbKn0Wse$PFv$ZM=b_<{V*2B>QOEi54RtHlJ&f2sYx$Hc@C=~Jee@g%KDUqV` zZu6WAZhAf=*D3W&t;YHH!lDM-$?yJu<2U$ry6i%Y74`WjdfXI{>g-NbD##jXSru?E za95fqLW}+UTue&4dYXWMBUm4|fX+#)!0@ddHdQ>66?BzwV#8$jD^qU}!S4%F^`Sru zDj)@o?^aEzt3-xLuAJ(0y>3J!Y-^JQt09yQH$wd3Du|<)X;gEW@eqZry?Z#_ekeQV z8-m^KrUNX-DK?bL*O5G&g{h-OMJnwV&7}e47~6AWd9uMRjeYidujp@Pjm~S?2)V{A zlhnO!zgOMiO?dp}9t4;T=#l%hdn@L>qTZuDN?8;=Mxp1O*~L^tF^;w&pZ%M^>*F62 zh~b;DNgTuxWGzI0T#gG6o|gWI<$EOyP#`^_(%ceq6f7o$YDydPv-V++e2Vf%`Li7t z6i8{9;-L5;3`Jp(k!FqyD+Lc{7$XWez4jNZtekFFz6(UD)Jd}G6Hb_I%dw~0mEr1? z3-<3NdkujZ_ftqNh?va3=Iv(zTW_0b;kwMeaExH#>De zeJ+sUQ7gY9v%p!m#3!R4HYW(fZ{BcSX}a!e5fT)Slh``r21@Gjx3_w8?{0yAu_htB z=_oQqkv8sV^sWX@J73%3_0K`i4cps=b501S8cm`x=qAY)+}r$?xg?oN7?le7+%HEj z0U|qIAmzH3xdwbHaeJk-mUT=x#?z-it88UYCwupVAgJhcYpW-Ja~~*nOyr?_@vU;* zyZ?+L5#Pk6G}7Q2e(u=9BuE3CVIsPWV_@j+$FzDf`G0qL4I}-MkDc zeM$6x@tc+jFzvpa5OP7?Slis@S0Vk9$W~+{OXktHky2^&ej7u@O6P4Xw3O?8XhF`? zmRbmHU`IY>rCQ#8gjxPI3Q*-E`q3C557+)Y{n$@^=*=6WpVphOP-ST#X%hkejxQY| z-Ps7BSv>9=0xjOi?^;d+b>_fi3KvmDMDqgjij*u2g`~o;Z)8@!)jvITUa;TG;|yOF zGWO>YGG{L|9{lvQymws+!zEh}!>8S=qfSNk|Mn;16O0su&gLEnAdUdwCnffPIHc1} zB$E&;j{0p)X;C#mUGcOIX1E#cCEcSpS*`F9AngOr)X@pOd6*&eq0Uo3^i+!z% zxy4_L1Q-B|NvM%4GEe$YGBL@;(U(RC`F9#(WY`BfKOzSv$$S=K0W}Nh9dfRLVSH&y zEPOdDyw3Blb>lIiD0P|s5$GGHfOO%Zk)v0DMH-)wyS+Le(;>AQ69#>SOG7dFG98qj zKKxslls_yMM0=WQwCuh)I4WuvGuS=F#Pq@f`kX`-6#td%gH&~`y(_IXfm=>rk(Kr6 zJa1J1cgH$Mg^^QjQgPQptpos2pW+V@nm2lrQ6wNB2)+ZNK>be?tgI21KOWMMvF?Dt zO-Xms+di6Um!&d1N0-y8q#^@R%U=ME?jYqY|AeYX2qxVo~{Xr_Km| zGVCQh;jsD6sBIexK-+L0KPqR!#1H?@lwi9MZb8=X`$jFtxYvtQF!0K737FciHj{HK zy-1*%==sqKIg}WhhW3TVd|D)7ArnnoWVKJ-8~#nkkC6nw3qi!6y(5w!A~o|HR%(Ge zPVsY4L!KKp4Dmh@F%`}Nw35hYiMjOBmAt54l|LMElg*a;$5?~0@$o|NVv~>qm*s+4 zs1Z%oW@DI-W9(BvtLX3WjiZ}5k-Dy3im|Cg+8wAKlGdl|{01L~JgOkJ`*&J9!RV zjSl^g*FOM7jlHnEI0^xCpx|WIz+*-bz7mW@T#gflT-f^i2QosIVVW6A}i4PltMe~L}7G7eN*)~$XPLc#F*9xtCUf?|r zt`2&{98K-sSCu!_{j%Ew2hm|{RX%9`t%7W-*g(i*WRBtmkhk5-K8u{*b)&==7&Qmt z%b;;woQPFz9DHG!0HLwt=H*bYux0P;$blLU%d-%C8Pux;b^82dfOC0oi9O(a=B$)U z{5%)BLWqe9>TqcjT+V=Ap+(Kw{|4|TpupC}9hdG~h>*qP-;y|Y2OvpQDRw%ncXYey z$_4gWtU>}Z><+;)XkTK0Ss&{n`eVHR)Tn#hn`E<5poA%1!Zw{$G8-1pWj-`E=|_uv zoL~0{InFx}q{B?vSEXk{hp!+#7V-J9^C#>_jln?@eP2uCaHR)C-hYTrOb*Kf{5wUY zY_&0_vFhfcF&BeAAwPl@G1}>;-KpGV`GqZYl>ij zqIvo+u?-fQ+dfRsbFk!aU;Uj;Y;Dj9e9=Pk`er4y;`a^)ew_-K1t_8h1Z#%?tf=4G zPZP;IxC4tC1qlWk&CR+e`wv@yV!pMY)lv#<8W0SMA2TP-4{&%M5{(kPE{|(tH`zcn z-CRVKvRRgElSN32XAC_y!b9RcL4og{E~6F?tJ!cqtM4@FvP2IlTmK%49wPgif%?Kc z5q#=6*5#g;Ip#?E3zC9Et#7aT)kTE3%L{Q7R%BCctv#U12gO6OidQKTgYdnEV~mC} zhp~nzFl+oKhm_(!q}oBop0CmY$&iJ%Xx=c#^!KckLou(SFtLWt&Tlt1F9#v)wPk@7 zK`b5T1Ma)lVV@3Z%@kzQvFS`JyCj&}?y|UV6763sjlY@rtvNBS{np56nX*!~Ftc_p zd7BCz8I|jw2+(8$MU%|xy9%rdvS#v60No=5z?T=?8JNs#oDKjPZubkgmf@kw*alIJ zjNtoqu#nDnQ0~fR+!KjY7{U>;uLiQGND55N zh>o2XJe*^Mi*~b`s58OVUo5D8ilj)c*r<6cY((3HM_12?%!A+?b!yIm9gqn%Sp^U_ zQUYHE-I;i~NSGGUZE3_`xe@I;%RV+jKw|~_e%R>eMo}f4pAu6EK^;`&2VDjOW#DnB zPgo7AtVdfP4_AV>S^>o^qz(Yt=XWU@Fz!Ce?bd&ajCB4Z3Ic?=m1&JHIXJxUf&o%0 zYsMX+BLI8l0MyC=XyF`w|DM8J(L+f{yUx2Q?#=52fZ4xY(o<)F@f|=-eU$3htT6R+ z{bzFpp#4$gtL4za`-c<<{XVF@E|?bgsCFYYO+KVGyy~?M$*lja`EkHK*YR1>x>oiC zzrXXVC~pwR-$EL-1H*ptokKe>*VXTZCS^gu6V+DuEu7=>eTri0ibVj&;};-nSmf#6 z8ZL!WWW(Jj(OF3aNPlLAv0+>x{NFzW2Ps>^!$Dh|`st%0trVL}z=NEn;_7-i7WuBr z6|cz#wfj%2L?HtjP8zp$$2S;ETGp8|{w%zbdNCKQF=Z}0KK`cBVj;hLA?3Yh#cO5? z2hrDz0L9PA_9u}eKCU*Ot&$}E7-hD~_Kgdz71#;5vMo220swBtX6++Gj!J+G*%(9D|jMzJqFDL~mG8N}vr&@XV9O73jh6abLbk_c%b+ z3COnOlE5!rK|*&IY+UUUKBKn|SgBxipYTk}dbC$UtU&eo<1lQa78Pt#K^ zg<~l&QDZ+eHKYm)k7-bsSsc^~dzmY9fe zey%~rwrv(U;e3%n-zWDxFFWZkm(6)^pEII&5PrAQ)ZO{CbprSMekx$4#ah4a{_PLMl)%e#lr*N20|L_Cut`O*yX44>4e%@wnk=lJ+< z*Y&45rAq}$RCwop+Fq8uE47lNzj=fP)aqVD18+N2pH8Scat{ zDx_fmk@tp?(mY&RY3v4&T?p5Ht5XZqF}>*^kft~7;FZvAuJ}=u_mhUjla~=*DL51r?`eroJRd3g`<*AF`S9 zPrhoI08z+_{hFn8_c&g9o5y9Xi@F;~nO{r@tfJuNf?(g89to<4)b`1lFQ>epIV zS$08EziYZ0vURJ^0@NhVg9F69Dcok#h1K?Tsv=(6qAjjgE(RUP5Xf0}};fe*a zCbx=KRFtmE;R8TBmb_Qxv$;m`B(CCde3lDG(4#v$=A?T@;;;VActL%6t*UB8!p`&< z(w~yiBwX;NltKc5=!g2i3}v4G7%3^drUBNgI89GVhbQ*c;uc^|>J0pfq0}Y3&iI<>jBH3>v5J(I_-!q_Q zE7G7X7`r-2Ayb~l-KN$Kft|K!pL z^7Ge&4v3xuWKC`c2@00s-ve}93>`Jm|3Dg;kPW7=4W{_1G0^y9_I-x50`cf21!pSD z(!vQM>ehe$gc=j#(@M)|$mBhXlf2T%Ie6OJtOd1OhzM1QVrSYo5IRdpNG z;S=BoJ!ZsJ>{Fnh?yb+RZ?3q6sZk{up-ns3+gehJ+~vHKrIQtUO3q&eUT@AFP;u`| zTW=0VyTlDu0pAX{s|npY{3R!Qx;e#w$1Sc#Sv34ql9#a?JfK%>bTsg z>_Ywo$IN~!P9618S$q3hiPf>+Z4(W!^VW{Uo1lDmS2v9V(~G+S2!79iT}*>nF*R#) zqgww3nYcDe3^Zu^+dfj|B3^)-LB2v)0cF#J-n6S*n`IP;lI(#qF6>+}OxKExT zaFnVfKjtNt1M8~a?#-u@)d$o-p0yqS%uVK{#DpJCq|TJB6xl4EhveA--!!!JWiQU1 zyf}qluVK#gZKd#_q&BHR12YPW80&m-9V0S$>*WGBEpPk__Wj(ZtLUN>{_QN-X~!00 zuGsVY9jND`tA)diMG1DM1n*_;UBj}QYSOmJ%*(z&9tDfJw4gb5Rl6c=C_rNo;eT<2 zuz*xBoqPp-DlC!{g9M}j{wn|#PYovm%2fWc@52jWe->&O6T|TNqdR^G9E+b-wvcf64|As( z5wAJh=aR~F8)Ff--JF1r4BNKEyU|I*j5cB6*m75l*gSTkfAv%@doVy9!l~;1(01^< zce4~>4G;=h*`UCi1LfMu-LmA1%Z19;KYOl#$clac61w6XI&HOLuGbg)Rs+2$8y!kP{#H@&?|ubI}r}(r&g>vO?3D z){K@CMasP4@crz}DQ9gBZT@y=9U*g=OeX`sR=!;hfZ6IRQpt!PeDnFd`mgvHuqgfC z8=`putY*WY|K|jnoX5xsDlbmEhQS|qr|&|m>f^-yG?hy1SY@wzwfqrhrQjivK`GjN z>CPjwEDPSbDb);P^5%8#=sz$yJD|onhY;&n(m`8^3_A^Ac)z>M%`wM*Efa7Nw*p33 zc|zF^Ef!QAWHih)l$CkEs1#C=ue(1P-7TAHRGSC}H{T^l`a{MA`cAY0ygvYS>_LG| zh&#@Za5)*85xoZIL2`hqNPKEsP1MC#BuU)zOw(hCkQ8SCn)2BZZm}|Nd8lSodVRWG z+W0=*%GYWmTh#^tjyA<58wCVJW&(A3iPi!En{ z`e(seOXJaG{G#O(bQ+j$?=aMnf`@f|dY^t&EtfiAvsmx$^23k)i(|ugKAk16c}rYg zC4yexj4l4|U7-=WKcbVow{n?(^-BSL!{oGw;>OC?XFOJSy*;xOaZw*>aCv(O=J2so zWHgRHxWPi#qg$If6m_6zCcQO_w%|Sb{kA)fLgHeFD`7LMnD1(1z5faMn?1D^5A`HA}cGI{y||5_6sTq$xsyoufQXp?X&(fs!=TZ? ze@b8*aMrz-`lx`=Jl~J#P&M!7d*janbs+dc zLSN&=zXqi60-U^{MN5bAs*yOS?{MeWJMxY12cOF@vL!rjBeB{vfG5G3ZX9ibpxxk9 z$8ugc{O|+E^W|)smdjZ`mM6-R=LLuNsOB$I^*qvXTQ%R{oqnR~fr}?4Px4>9&TK5? z!`U_f^v>x_dG>z{a{*iM=Gz>By@2J)<~4fYfbRa&({0ggo$9Yf&eK9F`bhzR15Z_W zKdSyRCai*HcY{6pRHdlUb0WU95~f($m1uwU)l8bL(4u;N-at(pimdB7rbvR1&jMEpS@unp9>WOvf#C`=U=$=Z?h{6^wM%&JE*3Ay@2nj zsr_AqM=24%cQs7{V%7kv;($X*Ff($W^W0F3mC?-;a7}N&&^Y9UEC?BST;K@qrZ+2ByCdUIb$qdl#n0k<$rQrBsqkCedCUsbYqhD%D`1)5 zSIT3Nqg$u|_h?1`y`@#A0K=Bwai;{>2M}Z3^FqGYloR(S%gcTFtEQ|Nerdc?L$Kh| zlCw7=?7MkHGX{5l`JDK&wcvEA&_>tH?3^ey3lVkKo~_zamh`Q<|JJ$nzvo~W$L>}7 zqj+@`^|fY96? z!6ftBDm|BmEk2kQ3GbT}cKGa;eB(itznhGEE|L!OYK8Lt^C-h?n){;_rDvrvpTX}8 zXSntotSzN}yEx3$IFBbM#`DpGW@WCw**>S_Kml}BT*cbWZ=May?tRc+Fk-!1a`G?y z{#{EXY4fxDoX2PWF~W$A6O-U0_~ z4iuk62nb}HuN#SjQ6!JUf)t=A6TyEkE`E`Bwm*VcaotX#I8#F4Qf{ zYp_C!8;T6TFSLUKCn(iACPX+S#ql~SHMQc`1?o?VLWnUU6{I+qD6e-3+X8N&F9?!`J7YS=VA6o=X0E2|q7 zCi^mWmUJ7`p`)fs9u+P<(5tLv=rt&8;3;61HuzHp6?#7n;2rlxB~_U<0MZ%s7(nm- zFUp#gjq`RC;{WCsJJ%PFkK4H}|D2)ze1`0|Md$aJ!_k_v_WtnRTv64O>PM;NEc9c8 z9yI{euy!wlqloPN#A?5E$4!pXe1xWMT?(`4i7;&l{}XX8K}--)i|w{w-5wyxWjUsM zy;T+p=xP4NpAxm~;qh<@+LC%hpXm|ME71WVD1;Nrc(dyKZ#owYsJ)rz1vJlN7d`xC%Es|IkpYJ^71=JMo{d3$rLE#skud&AICPvZ zG{C!rR`OX!lD?Vqo!n(5!gT*keg?6n+=4=Pu``sYJ|xwlTlev%Vh`01903Im0~mB6 z2-h(kdKy42Wd#_bJ(c);O(AQpvyQ9K%eyc|#jd(4b;h72 z<;TP<%c30Se*%8RZuLB&DlV7%ECGDKb!*3o6!@{V_gt)xbUTZuCpa1hA0Pidl_7t> zdGwUnV9KfGrdb|wwLlkq$CUU&a@^M8AN+&obNF)&yU1c{A8$)Wf`es+u|JGJ2Dl(9 zrM2$kplC{Ovw{*UK$-DdK5ALN=HiPn_m4hQTr4_Sy;UNN$3kYZZf~&K(F4PQGbZ4# z#Fyj1w~z^twx1K=)ZY&f_M48Vp@hA1cRB3T!t6*iOlfj)So{8KAN>1lO790=C(-p6 zv|BNRWlqAnFoyp;#_2x9TFL6SQ_kVErdb;PpF*!=i8-H&3IHl|JFtKAoDs(){tc%2 z%hNcmGm;AQ0K|Mo7s0}?GXwozY4nP?QQBA)BzOAbWHK>JXAI@>?(EsIZax%f284xRRI6 zux!6SlDWT)9fIs1dK0*;V3E+?JFqBH`2y_ryu+ivO@h2VX3PvcL?h+=- zpqk5ud4g41;|GFG?{8OeX*eXjOxL!s_T&=x>8i`SS}(QyQk^hx_EXyc1{@PUhG#im z3<&msgQSs~1iwS@;yWG(^(X~f-r>tCJkFNmN+z&ZLu$z|HKK$_Jq)=sG%A%#x7l_t z*`*};kmMsbHhlTOE8u1+H@MA5jv>KyOzg6v(wB~fcz7OXd*o8AztF|(l?tg~etwPP zL?Y3!R)r4OM7g)(2f#wT%_Y8C;coCKPyyiJb=p123ucL?5b!_vZrEm8ffW&Q7)R-` zjThUC`yUdFave5cd(}!L&sQ)4cNphqNcbMG-4@`%%P$uWl63;#LCu>Lr?83e{C#D6 z5(@d|sEDRPv`nF&X5xeVC};0v9iLn$Er#Jd$3j&BU#(u9?YS`zH;=rY*T4_#V>6ym z%YEK_au=LZhP5k9y}rS3%c2$uhVOF#5FGyEYli?f3;UPFVUU&k{* zw&Z8y=MvHVxonq79DD;|q_8av%aXE&8Go-^v%;0IAg6KD! z(p=$aK2bZ#yA`dV4jLQz9+jE4{n$+KC8FJe{M8FP$(xa=#+ARG80yscw`Bji6xD^n zZ-6{3q9)2}@h-qq95%ZGgGvSj9MC}HNF{n24ms7ud3wqma=C4~T-C%b4hRG8_8)<- zJ3ZUouF|Ip5}fHxtZH%0%I@d2?M zPEgsxPlg1Q0Cgyvgv%+TlkFe%uFNQU5?{}8T!gxA?szo7Fm7uRdgKaq03Wu7^%46I zI}H<;p+B1B*nXix4Gh!%^z4M4rcYb@x1xlL{uywTAJZjmI)_Vi1pW?E2KF~++K@_k zp8eI@08P8Pu)o`+>IY(CuF(h%Ju)SWxnBvpnPWDfuehnQ?bmlTTTff=VPRIsOYbv| zy%k3XF89{vp$L8NPAF2+PSNG-x)x#PvU8r$aF-hl-K{1PVYY!KppH%_FJY$;GoF1NQgj_0N?(4EsNU zRJzj|on72tJG?c~M4zB+)FWtMK`}cBR1AjZ{mTF~Tr*Mf>R17quB9nr`Zb4g_@Eelu`zGM3PO3DLjegR4$w78EJ?PHDI?|1*wgxMRRuB( zG)d1fH_%COBZqFo*f|RI29X19ASmav;E46@d85a&%iH?_|G8UY_*B1l?#*bYTHE`t z=>&M#-@yL0FnPh5UyUbr{_~a9Gw8BC;G?;l>8L%hi`k_dOCMXeTa2ty%l%oilcCmm z90^wYsT56-;$Dr4Sb|YR&QUWMQou#B=xTOJ!c{`Z_IfsA@?cUU1MF`$YLNQteqU6y zJpOwnPvyN1xQG!9E!sl^>r@be9*xGUwW~XC38Ep^B9>n(O4v$W*^$g5HWx%I#oGoi zVctm+~3DGLpLbfIsb|GA7_w+qJJkJ_cXX_`#1C-ism`(ngi)Ra>|-HFrjqRX<* zJH2li6x)zsO z=fforM!ul~ec%CaZ3oZe@~VW^CS7IlL6TFJ`Hp6cc0mkHf3LzgT$%^LfU;FHSZmZQzoW#Zz*qG?~ zhKY4=;SX;&FVxt?M8~n8U_ZP4U?ZKoQ1Ft3yZwhdnv4p>_3jE{ROqm_v_z}@+x2Fg zJ=v`NKoDV#g~M=U(lQdREG^oH3nN6;W}lr1{PJ@WjURn6?|rfT{#V|sKXRd&0b;ux zv+n+f-Fkn*^;hD(ZN=JaEJiUB3R8FnQzdIQGuR_5is~RMH}1oiG?Isn0roi(67b2p z2FdF$^~YXQN7D1m;^eW^Iw^-0@$bdR`y#kqNl_5@O9-7>N1tbySBxU`N~X{w&-Q|* zra|FU#;tV(tV#69n=$hV)(tCmFfF=z$=4FL?BR);O~}liJNfVaH|+18rD{pEh+uR7 zj*E8X%*M$vs@*L`T>4>zNTbvB?3k7_bg6BPEz~&7zWI6Paeyp2UvoKkNs)trY^%b9 zE(5HaS&K*c`F*&T8&Q-h4rI#c?ubjI#k+3iKwpFIZa1!N7%IMf%~*EA@X2)y+7@tk z-GX^j<~nxnW~Bw0b-MYo9TRC_g=Q>;|9r{^EOFQf6})5nob1YEpwOb$YS-TeEJKc@ z-G0Rm_`WL|0=Fi(u^*bh`glNoul$wBF4Rwr5??52`!pcHc_FQ6V=GtH?Xv=zQs>Ms zE5~dh6Z)l?QZD*d% zO*9zRb@E8(>b0~|Al$D?x%T}LPaG<}-^S%OhEMBE?XMt;M+FV0Zf+Ks;T|UNq~qV4 zcd<+Oi7k`d@5C)E58Ep35Q`3kDV2)gldPKZBf*Jok)1jpOP#HW&2$KCdMsjUzi7wn z*7+C-IoVX5QtKic>wFgcZ9gfXeSeeWG7zIWzb+(-=f=Szg*J=EkN?u;6x1#w{`KoEi2(_gZYAV z+2t5Xbd(S!#t;Ld8*WcjV}i^_sW32p*|bD0W+I z-5>R-oZOTM*dSLleeHQfAu+j=PH=q`#8ut`#o9OWcfmzBF4L#+J3>q1{J5gybiF-i z*BB~jI5(|cby`AZ{b5LG%l?JyaGd|uLJPH{Eha_ghcD`zw!VL`wqet+osOP)06HsY zP4*&GOc5p1`!n+@%uAXBG`$&zop7r~?S)@Y;b&KO0T)S4P4<1l>_Pv{n}M&|^xhy$ zZdlbzKHbl2exI$rQV$Wb{nD;X44Hycm4^h4p*o|gWLBDP%8f-kz&w)al{ z2nEh0VVl<$W%W%ew}Rk+jk#%xdi4mN(bqS#`aW_^>Qe4YBmLpw80lC z)8N~YKIwVCJyDH{>v6IG_%R=~UQQ?-h)_4NQ|x(?oJOB*Xinr_Kr6^a?HyZaQ5n-` z8G)SL*=rY!M^*fY^n(SboSQY=oExr26|zjh#sH7*8ol=`@SDq(n-gMwxb35Wag&OK zCO^nsBjkR&J5Fp)6F~HUp1v*2fGznT)~K9rYRASZgj7js)pKTYwr!Rc%x_pH`6g%$ z4bALbHQbILVIl6LjBvX_pxWqcJYa8pm7X{Awg@+<+GKrdtc%os@gkbuoi~*$_SsCc zCaK~1SycLjHbf8_`?n0PqRBiNedcX_1?z{o zr{J3}R9NlRmPb~CS;F;jFf_*R6m1Ol$G}YZLy<&ysb_x=bwk}v-((LD#_f~4SmqZL zX4UnhR=6>K5s%FGF!Sz?och}i9$DS5b@Et2z9aS!4j(V}?N&ZISc#ZM>K9g&b$|bX zm+#3OlG-!akGMI-*IxqzD;K%A1MKaNK0*_W2)$ifeW>gDgBP3NdL z71XjzOm;*(%C+Mro77%dq!fn@B;~?p5zb?~?aP&Cy_vw2@I-3wBO3{b4An*3@yb|I zRUiofYdaGY2=ri{-AFhkWy z39Ii(0*l0bwI^y2?&=!WRy2ji&a$siTxK?)1$i337n;bzLUvjTa{!1|vt9eJEvW^Ioaf8f z;>4@FpluV$Xh?x%?fnK8_uILdrwyT@xNR=b8>{!L@k(=!KyTqG|irPn;(z)TIpk zc3!tgUKA~zkYRMbEL@y|6e9MoQj8EKitN_~?DNN)Sy;he)k}Og5dK$2@Z-}Hc)3)fa$lFDY(x**Ew+2+zM z*z>+$t9^9UZ;F-?4-+WQD$YwK2{?+x5c3lJhlsPhYO>fxiqM8 zS#rx03#+od?BkRGKz39~iAqdeP+@*#b;rGQ*uh@~qSI^God?zv|Tc(kk*9=nFLq%B>UA zD(Dk?usuQ@%zW*@Tzq_@fcO1fy<`b?bG-Zl8q11VKZ)i$$@DNk;m0NFUIXsQyD}pU z<$krU#KvSoE}!r^$wf;nut9d&$7aJKitGL4X7URa zwc_N*H$`PUDgN;Bv<1X3x1*7HJFeq)c;XS3rPh5G5>oXiI!ht>n1y4nCh}~Xx=*R~ zS>uv^u6xJS!-6CS-4OF*!(=Q!9KS1mLCnBdljqZfKl-1BiWSW*SARVjbW*+Px@%&vHl;?&+^c+`%&0y&k{4%t1D&g^M948% zif%+cSFpRmSD*sz=ZC_9+~l^0v8qV>3iuf{eLNg1c;ctNw{~F1biOwD*|{>8NbQF| zHq&ezilidh)o%jrLXePoo*J`q?%qVX8r}C^qveims*j17)2Ax)@d=~yksJ7guli74 z>km}9nsd~!K`%vh-#aS#Z8)?Teoz%25c;?Xy^e8+4Sc-!!%#y{H@JAVI7c_z-`vaN zGNq~|Un3|AITdG4IFbLS{Q7+sWSiOtHt5*~Ejadp8ns@&u7qrNsSA6=LibCQNmcsC zmp@B>Em0|W_Pw6-Y5vvK8LmGplLm32RrsOl%>d1h&s8YWe|^t7Yq#X4!&#`)a(uA^tLA1;p4!Qpppy-g{$Mb+xIemMF3QJeaptThUlU3u>8v#Rv~`;~y?x(_ zY`Suq6-O(NWvS=#(B<&YM31JcY$UHK8PvP`>^GoAy|;Mp7;P$Z<90kR)$iN@4K2K9 zj~sFKs}$z@U<2()JYX2+-5&d4@S<4EM_sJKI6rVMPg8?xX*pKaX?@IibjI1IWyN5k zK4dZv%a1x$a4f;>cQ=OsRU6=H=ojgh`@x4-n(x{rTcJwydEienG{%E_)hmLdk>? zv}%VVb|vQcl$t_#TwB;q-ne{9cl$#+5xZ8vwOMXEY%*#Y7B*{mQvGr?OS z-bn{=yJRi|XY{R;><>eBKE8x#C5g9be{9-H>yyIsX=(Faw?|aviY;$*>p8PAA=!L( zkj?}9!;`ucr$;yqEM&HyNSvR!#jGBrb;NkNl}%!QtyK3+R{Qml;hXf>H!qVR0nnuS)|LFDcf^K<_Xmx)&IWlhY{ zJkGLQcQyGD)hxOCH6Ak~gOKM&cjGi9Vz7a(`u!Jda_K7MtKCmgq4EC~s6*JB z65IN#`)G)ZUwQ}#*t$GiVa7c6!<@o1KhK&ecF(zoNeK4)H`%_x~G3ze!QKAjz!FMyvM4VwV24b zrAd<@)El`}xIFyo)w<}XqyX#~dXxJ1PMSIBzo(*%!Y97I?Vr*WT)vS%6*nj#dX2}r zg~Wj;p7!t2H?}M%@|vua0|jm$w&93{I$_YC-P5@+N@qRmziyE1Yf+~C?`zw|sVp3` zMY&RyaYti$*BB{8ChT@7Bk&61>rM#%_A9DAmUogE_(_)QA%HM2`@Lf%z*k4Bx<5%= z7r1e&@UJO9(IM>C;wC)laD26YMyeoB2JA#=Ne@fx>_=rWH2L-w5Qaed=K*|jOU9Q* zcj10$8=Zy90tgOKMm~%pBq^2`_yQccht|#%|7-0&@u4;Im{{JW!g;j-m&%7mK>s?Z zRoXvo`SqAMSww>`GdUdw^*tHb*Y*dPHka_HY%W%!`__hZ`W zLO@^FKo1x3qtgA1-r1c3ggFInJ6XwSz;7PNKXfGVCg;a=(oIx~+*{UtIp7AR)V~`% z42mjGQ4>-BUQ>CvW(v5^F!Q5Ujs*`zW?;_$XRP7%PV(s*e$RmchSC4)Ipf3d6CQ@& zllUIO*^1u_`@Wq?xryJU#`&n$SH|1GH8e{(@9^zVxq{+ny;_=maPi2rY{QU96R zaywZ>opZ@UgLwbUrdyY}E778~WxnPAzZ7^Q2X{Yoqf(#iQCVoz{Ox$p|11m6iblQn z*^ZfV0c-Nv*Z;MAl>dc8gTB24Oz_da!O{Wl^!4DPmtMnRq6+X5`~LQFD z`x<^ie{T`~yR}70Xwtd1X+R~n0yU=qUX^Q9L@1Cdrcbi)McZKJ_$y?L@zds5--2MJwhGFavA4$E6Zvhfv;E7lH zYaiVthtN2Q)@>F#)qf`F%tU3;Vx3jtjD*5dRXegj_|kuv8s2)P67_m;{57RM^r2=* zxof;MveT*TcfZzdm%7go@6xW`PV3r^I%c3Do~jx zqI%%MhNw(v+CF`EW-^>HudSW9Taa%D~uP}<`qdei;l{X(HQtn03&WEgrROC8; zhu>JB`Fx?(_cLCHK(@P)i?V_tJ7Y=T7eTqRb%hYzPLGQc6!LQ2Qn$X6R{+x~E}B!v z-rmiR*3jI&XMN{NYdd26lVm&>CZ$amNtK%g&1FFT1@_3x zAs3`4P;psdPCaf_@$Xm1_YBk4CGcp)w5{9-5n%ogGeBX_qx>uk4#w2uRn+_ZAEcQ`e!)o-1B`8XyI&q( ziQs~V53VA{5$0IrijVVT{;U`LoHDTxjN_))Xy6r{IY)>cYuFA!y~ z${_R4JM;{NR|DhM>#Lg(O#OA??rZ3;kiK54?ex#j30|-9>SK|#8CvAo;)h>=v%c;y z@8op9Gl)N|VLb)b7MZRF#D;~g*P_W`n-!X&`<6^Cw%Xq^4OEvUBdwR;U+|LsE~1~L zUMya5Aluvr#Hko!6x+PnP=Od8Sj9%ofCkqE?d*)25!#koV<%LD^}48cfSTa(vU#@Pb` z=LYt(6P5(Ri7;#xGn+LPkqT1UmA+7q$6HQf@&#){)!OrbsD9q_vES++*zHuE9mM=zXBMFF5a z=%+!zybhjWj~QkNHsoDNJ^t8TVY*q;(a;u*b}e|$mM$2gb7`7<3=OI=rT}aZ7b#d* ztuk?gFDP0fyi^DboV7O-o+~`n-_T$=|7W%$e*-x+zZDZ)Y4f?Zr95}l`))~r;kNV0 z+=&1Re38rW+2#)L0obO}p?funGAj>3=Iy*Udr3PD=d`N0X%I2aBzi)KIRuMrLa&fZ zv&I>Vq7ME#qT@*a_}t;Fw;P6i>?u+;pxxY7cY-+3A)Nr4h`fXwGRH+n_7P!H7Mb+6 zR+YvS6v;{d07yfV+KnxNF=alV?5~M5jP)3#{$1E=V42bL4FU5iQdEN=q>b^mJI0UZ|IOMR&BSlYoSvYo2Ai$b5zWd41&Dp@1vxe z+&%cVHM-4F+}iVzr56k1e=shQ?cdS7j)8-Mk^76`*zK#e?1W5i$`CuQylii$kPRP% z$7bqrF2cir=?&%26Egs>0Ne;5@6S!-n?uJ9;OCAs0-@mBN7TmiCf;r#HFdjA67%AP ziVR&H)%wMUuXsA8td;blu;FDe1tFtv*ghuP-1YGwTk@+ z2N$>;vf8DlP)LrKcQ*bMCIhevke2ci<*X2YWFnu=6sq8SbJ=)qywB@aNiEJJkh)%s zW&1QY(yM(|p!O3L1}=Fz<{5SQWT9>oS-<9ABO1zuB?<>hJOBq)TV?;Mhzehgi*;pi z;M8q8wGRo1h}R+tn-H2v!iN}}0sH~pW`Ne>&<0gLQ7Qo9{RxM~O>JJhY-YM&$~pY` zn>W}FtS}V*k1(qc%q)HAx(^j+&R=jU;pCl;Fxq{PSW>M0W%WDl5R?TQy{-@=%piTB zi!|`?n|6r24CA#xEJgoY*Iy0Om)jc1MU|bIsvlVx{gCubR#y3x0+5&u9)r?1$k*@pw2;f0-`5_y@>;t7>h=Ljcxg^E~r@zU=S|IbKte(rOl⋙)McI>Q#9SR1J)f3cF_{@C0eV)8DKD?=yLogd2 z+eVpE=b_u)#KQerxN-P+n3FDPs&Qi&);3PskF8UGGvt8o9pQlN#>7YO1thI?(v#Is z+SV83=^dKuiU!0-X@=9nc^*b;M|Jje!2^J&8d7G(fxG%{%np6uhf9xi1h0o8Y9GDy zv;QG>u&2O*quVoJ?l$)LuUWaT{KdI>`0+EIq z?SRNilW`+8%(7HMT0R$Tg=DoQ7Xl<+%5d=uXAQ&yCZzIt_usU&1U4h~d`ob_-?UPW zF`mE~^G5D+8+5IBzG-Xfys-tL)@^McNqt8yT07Vtk3*RU#bmcg885eM1k0!H7`9(| zpXW>x!s!~iwMO7B0w1Y|U>ghx(O!5@TbYJ>gq2(p2KK3UaKA|go&MOF`~Jx!{6Gqx zNX|La6uEhq66#&Oi(sbc8dTF`_!rEE1pmuMWo8JX)-LhnXrv!&rff2r?XeKAr%l;A9^_rwa&T!9_v%8nbiy3U)tt4k%g=x5&F1d|~mkpV)8!j?e)krKRd3hVT| zD^Fx&J9#9dIF|M@%711qVy65%+E{!jgZSS$b?i9p=o1GnwByjxiJ>n5%n7Mj9E z692N+8Ji8RQ(!#&ZxZ8MW9oZrgtf( z?goeM_FUr-AkWYaqj2qGyG-wjt0nX6vg1S*qzMGLs+yTj*A?#l4Y}BHJaquQkTNv3 zGLOOWC75r~dT1cKU3lm*!%pQ3T_VcJ1ZAG0=8rH``xh-}j&+58ipA%hUQ*S|9=Y@c&`XC76VA_VTBJP!J0g)_v?740(5xBr;+ zJr_-M__B~7|Lkq~uR5=->HCtU;}<7CK?sU>SOXOl=w!k!O9QTG&qo~T;t`?o@%z%J z4;*?e$-yU_#adRM{mu{op$cl~kub&8y!#VOBUOPAY6dwlh0)Ip*c)y{FS4I zUDlSR9cQ*wS6uJqvC&BxSA;?TX1_-%e!i+HDM+XLvs{H2H;ofM`#C4o{}}t3R~6)W z_kZfD?2<~8%GlVYYOux^L)EOKYc4VyE;6qd;fMI0ujQ!}S+izbvYB#E3=+$09_y`ad0gsw4R1KtCN<|FDb|bgXpTvj_#>Ha^+b z!G|tHg00Hxepu{>qijGquY4AEa~RNmIZnQdzF-&`%Q2SW^4QL|2_&kD{S$%!C@5<27(y_cy$ zcDn_ty3xsu($Tv28H4aq!@jX#dM{@us5#DI$g|rO^0}t8q}OvrG%5eW?EspZakr59 zZJva0Wfr=DG2fC>xla%K0Sm4Vq?mbsc$qkuXbZj=p5e%H9ZCC`ZygeHm`s4{Z>k2?O1-UN7a$}Q=^9}1JtFJm z=l9P(qTs)LHDCAG*R3tnHE%YBq*K)&QBuEXBC+oS2>49>Ft!0}W-74Y>sPdl68YHctAG9=M^0Ny&IrxU22t3nlNpOy~ zqpfaFsA^8u%qYsjy`J9=WX4>rmPJfLW;sNmKWB6g^WeT-$KH>#_0U4ysv@Sln|*<8)ar2fD$vcEsir zmrT9a+HUyXxUg>zMrjg+VEiq*WDz19gSz7iRre{U*9%#N-Uf3Z`e-TKSar3GDMzVC zKczR?{Gu?qWIr?jGS+#V!D;u)yJ__scm> zbRS~yjd}c9!=^maB00yz$+GMQy{06APdSztjqiE|3Iot;s0gLtGIzrTvsmC4Pf2Mw zS$~#D#)ft=yNd*4H+0L36IcB&D3>N`C{Fme7~3cwho0T0Zz4>PgFT1=kjesfGUNC{ z?6Mqme@C9nD=R}DiXAOVMos)_CdVYs#SsEXwB9pdzCFJhkW_}a3U2mNmKYaBtBcTA zoz9%Qg65s)Yl$_}NK}Ky5PrWLLpc1o6I&HxMrgl^Z3o?*t9p>~+vX%Es{3aksgCUL zWkJL9UgOqXkX!^sS2mA@TX2?b9wNLGl0i+5I3PX&n%<)(0@JC(yO4@JVaJf43zs); z92QrhNfwfJS|Mm-sn`o3)az8Yk|?7r^RWP`K?jOW9xEC@+bj%{{u15#{nenIHpvvD zziz3=w-S#r+S3`$;gV(5V*~}(KXsf5ckaJ*$n@>4YDIBV8(it;Iz(56*|fYyBnmK7 zs;*Ax@gv6<89JvDOsqxl&gP-%_VX#eD*m`QO7ovv3W z6DQ|JBwk*KP;{4!YrSM*5kQ6XabzrLxgn6%bY8;+yEO!`8QPltJjO$YnF#51Fr>2G zetkr9{*gT1oUX`Ax}jzp7<3y1XRHisk5Rj)X;iL!?Nr=GoqhI^;e($ScE zRj$3O9L0Oe{Uk!8>1S`o>#Fxnb3N;<{lPe~8W-gq_o5Lg&2+!D3I#-6;-Lp1P5 z$MZ@(aYEK1f8mzzKA=`_Xx+!pf!a$&OBr3fsV%X# zHa3io=dC><+3(KpgEvB6*B=V_FCnAbE;4VI;9uknKDap6`)7#n<1kEu@5RylL34nv@6PuYahcAN zGH)|iYp1eSpBW;t7+FC(GXaKGM~<((#1mX9iOSe!s=%*9aSpg`#)rxxj}U-KpzN8F zX_W%Gl@CD#StHGrx|%w+y-;KH$HI$qwv(Z_u4KahiQ5o{x?&=3uEtI9aS-9h+mJUH z04^1Un)}CSMmAR6A8&yjqD}WiK>Bx5Ivy)S5VhQv3@#ia!r-j*(5u(-H|}U`Dy`M> zVDH>YD@m)Fhca`o7{P=3Z*m6F&}?tB-Jr!j8JM(KqtSQ&kd|o6J#D!A)o!2lA3x?F z{J4?c_<9Df_w(pA2w?!6kFYsam^uz%rVkczM130^GIhp}sL{F^`a03%m4`?@cWpO2 zv?xAJ;TV?E3PGbDJNl%x6BU~n!OpKB;r<&OoWqmoA^n@0s?Q~+--?VjFK$XLYf|HW zN6?vDnj)^6b4AMHxt#Aa*w26VfX{gKXBw3_Ema8;W&PllFT0-7J%D-*jn18Udp4IJ zFO%--F0@~abuzp9Ryl(8cf!uO)L91In{x~<*-ja4@lb1*>;7^`*BtJlgso#{x*nE@ z8*SDfYO#Yvg;qbm@?)U%fX_hB?Lo}y%}tzA`8mpg3JvOi(;dad()=Pj1HfO|~PP^OtobbUtgx z7}RLoXT+-hip+|jlrrtOc#lr2xwVr{MQ659*>rmTP|=(Zh`lH4we@pac$eFOpBYnq zJ#+(Za47ii`lmR*fYO9ha68d749N)?GqzP+%8=z4UD z9*<`f*ddjH+=mW5o`A;HuM`XWO|58M?J zJ6I{5UMXU4b9fs0i`Qu9&(8IPi$F1xM&t_3KS~S;IuACk?A3Z-nOKKtr^k<*`HIMF zV?R62`F6DEqHTa@Rp3=jsCH`+b6!#xd4O3c^z&vJY8VqL3Yf#Af%;P&ki8s>E%dg3 zT0UD_a{OC2%|};1i|KOTL4K$&sqbnwE~qIupk3T|I4gi!9pimlIyKZ52?($kL8l-s z>(L6fCTIuRrkZQAI((lq^fewM-A{I`Wi6+@M@5_A5eVLww~OW(FJv&YpqWgi-fO9Z z>BzI^NPl$7nA6|Z%fOZWSWPLKA#liUS+Xv@yT5(Wm$Jfa~n%ulARKWCH z)w3^?A&;!gTtboPa44L{eSV2;HSw8zU3!^g`(-sL{DfGzn_yl1LaYsI&XIzdLYtVsHVpc zoMi+FaJUIoyEMmS0%HC|LtHw^)4F8$Uutm6J%C%bo9$W|yPzu~LirCC9n$IyWYvXe z_VBScvxot{kMwM&X*jy3yesY;J#H65UOS^n@UbjOg>eYuvlJ5b?LN>mka*2@M4~s% z8rW{Pxc}5SL3x-Zvy1e=_x`tQDP<5T(`vWK)`th%H!nnkF+I1tWSB*QNV8*upo+L_ zMdk{AIOeEA3DDoCDjGJY{6AC6a$G3^;;JqtK!@J((gQLzSlX|rt zM?A(naClV;=^yt`Z5-4phaUwq>`T4ay!WUnV=StXRLNe z&6ozT0=N5?03D?S!2^*2`dJZ0+#iw-sI1Sia>tCBglGp=v3-ijLC;Yr(Z+Qb#C@H7 zX0+q-yeGkccu4PwlODt4e~(t>XPs{{yf7<}!Uk!xoC27{v?d0@Q^di)Htky+n;eYZ& zlWo6!0q;=Av(8@Pi@shcj7|$k0KvdP4PMY+JZl`zuS`0H+@>9~auM4r(81_<&1s3; z-7<%;NHUiz(5}fRdEeCx=t+DdGZ?ikb*yB=h+nhzg7kHe_b03AsIM!Vir&JPrep~gl{rLBOJyF>zyPoS@vv8^%889 zho9J@Go-qox?ayY)+0r8c%;BOKTPIZrj{7}D%lvts#XT#@v)H*ip??`tmAa*6p~zo zfj#Dv?SRdlDRvh?7~}S*+Jki4j7+Q0D3a_iN34#UHS;Sr)Zq!N&t94+ZS>_n)o3Y9neBIU5 z`YpW2llrlCq|!36m64o@r8VX8yJ`^nC#wGn7WB|dhtahbb+^F5LVYhSi{=Ng>y~j+ zV{E_~`-P`LvC#qk_>5Xz`*%LcgW1q}_OxSdN5AR0r9PRLd5C71!zz=Q@jCp19%mr2 zZFEHRdOOYv6j6}91?G6OO|X!|#8q`Ig8Fa63b2p6c{C^4f|zqgyVtIv&g@Eg_? zF#ZLXjyvEW0Fb^Cu;BAqG0=vo#Xp%5hr8aY3lpr`c?}6f`Pm@2-HM1~sT)Sx}PW)!{(hp7zdVf69E;STsNj4eqf!SP55vOpMk=wK(sB9ipRs%i zY;a>H;fV-|c?bbu2p!V=w;!QGTGnrdVBd>9Hn*7F(hShzUpe)3N+P5^r(tumq$Pl~ ze!U3~Fq3l34_!{>Nl_c*bq(c{+cky- zgyS_PqctwnE)wZQo8+8I;Lhd4;J38%HP%{zqG*Sca*k=+{c{ z&@#tE>dYBhfEYKp#Nrp21Nd?}q-5$G;&jpjtv!Zba|x6opT@U}r;!lY81fDU7{A`h zm_?XA0=2)@fy%=oVPLIBA1C`*96*h+LRG`r*v^6DoUYW%6N9JO1Evy&ulFRqJ>o3) zzcJcUQ`n66TV$^cq>sBA1$qt=-3K0!!EK^3*QZ^e~lI-UV!2d#ZyWox+ODhHB; zS)vXxO$(1Ql+EUjnzYa>clNNc@?*|l|2(4!?2?yrAbJJW7HyDChuJ(0xeC)Fniw5G ztQgT}Keqgj>i=q^nw!N%bfpmqP_NZkfqs8$=h64#UEM$nr%U_+Oi-o)X^_V=w5EdQ z9C<27_MRRwhLO;TsI6%~6JKq4wa1WYq>jhW-^hPyVz8(IR2htq&na`|O{c~r0uz?D zKabI|!Se+|21i0+xG5m0jM$)Dhi$i>}y4&xy~w1uJfH#SXm%TOhCkxEgV7)IwgxSFQ)T*@D$}0=m2G zS$`lGk*DurZE4?sF%4~B#oH94mL4Hv0(9|q{CIC05jhW2%qgfVtJ#W$aFatqHb#<$ z2Gv3K^`8ypefihVkv_=U<8iTr*Vb29hOTUk5{5r?T7^X}(NbbZDxdg=%o0w^4>`UW zT4kdhl|FkR8?7aCB@v>zJ|)5%dw!H4%MlI7!VEUharb$KT;+jPN!w|xZse<>w9&1q z<=dqidS7=#lD_=p+wjDsc~W#uT9*E|PK^8CPcMqEEo7g@tMrDyMa?}#+m0f`lMF&= zZH&vvHUo}+oxkJF4aF}>fpv3)P~h@}4IaGccChVmB2!jf-}e*c(-=Bmo^QTxO#(yD z(Zjx=@qxJBQ)owFj%Kb=w%6|-x55;yu#W++MQ5{NP`C8KdSmgiS`7a7auWj|;1$lBSeHLN5AS}uzV6Z-E{!osqZd;Chc!O+36EVS z3GAYq2d{oRg#*f--ekyKN5gJ0prf+nxhiB`S;v;v)dIn>ANA88C8`z=BX_`7uRYN@ zC_gvxr%tW4RL_vU#t@L*_3`80b;btqjmzG%3$#e9UfLIcs2$Y8axZaqVJxYep9La1 z9>`(Nw`9`E9kltCaJGG8&**}6LC}RV=U_MbS*XL6BcrF2bZfTTAJc2Jn)%sUrJmDp z`7rpX0t@GN($3Wfe2)k!`kCg{(S)~QNLCZKbvfytlT0s&-Zynuz0;%>yacM78dk$Z+{%Fjf&{|6E*jTXGmz`Bn1PL0Zd-Phaw|=3m4%S31 zhmqClv`u7bWZuVTGL0Z*msN6}Uvj?`IIy@`57-1Avo>MF;wbAK!JWvXXiV~OGzwN1 zk*bM2o|%fgIYnzV6S}%IU%%q}!juf17LI1_MH)#^cds5q5o>mtrI)`N{*_BB>c6*m z(?_IR-1s1=axDn=LB}tso1$GJ!{==BKQnW0%kRh1oqtc*n4dBP8De6bVXxDnNeV*c zI`@|hseg&AqWJ7aleKLn@r+q7LoNN5^FMh3-A=#!O}(09*(j-!9)o{fuBjb;=*_t9 zaFasn^KkO2;6-J7CmWgj$(=03Wd=v2gP6?xW7Bnb21mAQIw7_$gCi}lJxE@Yjh9>#wSJ7capFXAA)qO1-a${@E-4uX^Mox)AuExE_PvC`R)lF1m|9hA17M5R+g^t;5S*dFMSPPEU zsh29aHg=kiXadJ0yx$-=kXT$c7&gfy!1tlmErGqC=72K-AhWlu4&8TF(u%o62N-cS zd>`unY;!jFUm0@jSnym4((o%YWzgghIQ3< z^6m$d45qdeTt2yY4&qm!S4jrs)8EltYPqa$%tCuyx99+>p?LI_VEuMDvpC#+j+lR& zwiBetA+Q&6(9slVI1zN!bh1=_lU|UQmF*d=#RJ*pM@Sgr6t<~f(xMdXwOD4LQul7D zsQ2dmAJEE2U;f4PFo*Zo&uYafLoG`_ZUOSBWhNrQi+a10t&S*L)vZ-3YO=Agsd!-} z|1BOx#3E*)5k~LWznXEC$Zf9r!vzLuQV#Bg%gI;QAGY3!BYx66->D&>foiO)1uNU& zVriy4R=lp7Ag>OBA=hNT&FRlaM2S03>^RoIC$Tj`^faP7vKW&OqvR~P(=8eCp z+R-;OvENGsS17xhaa(EBMUzp31t)fvG^AlVMBrZxozU&*&iuJw`GrkxmEnN`iWw0o z7#7zLkKR2C>S`;>YO7a%ZsDwi;{Yw&-!p>bwO*~6!EUJa`xP(;Q&krcH@GaXd;@4` zs310ZtmykbeNM{y!_)eGnYh4V0d9b@jTquRb=lcP4Z=rS+|S^|sXdbSN{(L*m=K6O zu)2}~sW}Vg%zs@*+zq4K`CcTGni-X~TSjFUO)Lg+p;uP@uL*}pKXQx9Yg7h9k7pO? z_x|uY-~2`@bSk=Qk2)Z21g1>^7iJ~~FEdEtNeG?~U=d;`vl{3T4q{0qx>)dLPw5Q2 zHwKM+)_ciJYpz{Vo>wVK=ZEg=$7g%nl)JWOhWM>G78niMBX29t961Ckd}OV6&A;zO zoc^g1jCXAlC&s*1yT#%{dgx{SD&?GLotloP-&uJ1Q3zazxGv56 z`hi?UW&9xN-%kWyd{65~5u54(=fFv_nMkIwui)1sB00#c0z(8 z{^9L+?o*>oc?5%508Soc2p=oxK%jsWTIle+Tek&IVQ-}mHdAPy-`>qeLRTeM7Hl2 z7>CWNTf9y>_dZzeU{Tv@VhQ!LZ&vcXnY_&d_b*P#ZCnS7KaqQZ!FX)oP_nEdszD#f zTyKXTQEl=v5lEL6zRVCrYF=#8zZa)xb;y~D-|>E&EivxzJ#Yc95``m!@;m$=Uf;-v z)=z#N_^Lhlxv^?tB&eUA${=P3Gay0bW60q!Od4ET$t27}LoFaN~LenW<&wrB|RG9qP9_R)GMuQlBO13T*z z+#fGKTs*!SLTn*ArOMz-M6WQ(4=lFcS9Gn_zS%W)w!k=}8B$jc1MC%i?2r%Fvj->XyAJ zXXw@!2$-N*^kJYV+ys-a2iUexwG)46`Z%QAdH5p}R+#-8@{qflUH-?8bCe$y?!?hu zV2VUfr+}s@2e7ygLh%}~q54m})yQ2oe<#nr1wT8L4WfLmyr{aH0E zWvI5Eq*;{7vditd!m=KkL-<>?BAR~f?N8P9>eeVb@?O>*0*0Ooy;t2yhn{k6F<#sD z`&MkKlH31v6QchR8n1$$^S_*o6T>r9`qFAkdp{~6)JW4`-|+&4bX+>5 zcVuQ+HG2`Gi2K0w*hO|*_6^&D_cI+DEr`CB%~QHn6)Ftx*o9mG4Uu3Z0Nq-mj8RzA znDNDjH}v{>X=5(0M}-6F|8&wQK@suMs{c|p z*qEKS>C5T$>YJRZ1EwXZL?!%QcQt0r$Hrc%Y;t1Z-b=VD9*SfN+-C7#nZ;s>F`-<_ zTT11N0XD@iK6$wl?FP8LNV2tH6Y9Pw#7~3^awXFLYnEY|YUwone7kdp?$<;nxQMCF zNxze!V+oVz{K(DxU3iQFN;`!qzsyD$U>ay1Gjn=CrM&cLMU@XSL8bH<(LpcnS(w4a cl~{gHnjQ Date: Fri, 2 Oct 2026 15:46:28 +0000 Subject: [PATCH 14/30] fix(telemetry): avoid global plugin enumeration in workspace snapshots and support model adapters --- src/langbot/pkg/telemetry/heartbeat.py | 4 +-- src/langbot/pkg/telemetry/platform.py | 6 +++-- tests/unit_tests/telemetry/test_heartbeat.py | 1 + .../telemetry/test_platform_model.py | 27 +++++++++++++++++++ 4 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 tests/unit_tests/telemetry/test_platform_model.py diff --git a/src/langbot/pkg/telemetry/heartbeat.py b/src/langbot/pkg/telemetry/heartbeat.py index 45dcbf268..74718412d 100644 --- a/src/langbot/pkg/telemetry/heartbeat.py +++ b/src/langbot/pkg/telemetry/heartbeat.py @@ -189,10 +189,10 @@ async def build_heartbeat_payload( cloud_counter=lambda: len(ap.platform_mgr._bots_by_key), ) - # Plugin count (from plugin runtime) + # Scoped Cloud counts are already available; never issue tenantless RPCs. try: plugin_connector = getattr(ap, 'plugin_connector', None) - if plugin_connector is not None: + if workspace_resource is None and plugin_connector is not None: plugins = await plugin_connector.list_plugins() features['plugin_count'] = len(plugins) except Exception: diff --git a/src/langbot/pkg/telemetry/platform.py b/src/langbot/pkg/telemetry/platform.py index 934f36e10..0842d466b 100644 --- a/src/langbot/pkg/telemetry/platform.py +++ b/src/langbot/pkg/telemetry/platform.py @@ -117,5 +117,7 @@ def observe_adapter(ap, context, adapter): return wrapped - setattr(adapter, name, make_wrapper(original, name)) - setattr(adapter, '_execution_observed', True) + # Adapters are Pydantic models: method instrumentation is not a model + # field assignment. Keep wrappers instance-local, never on the class. + object.__setattr__(adapter, name, make_wrapper(original, name)) + object.__setattr__(adapter, '_execution_observed', True) diff --git a/tests/unit_tests/telemetry/test_heartbeat.py b/tests/unit_tests/telemetry/test_heartbeat.py index 86b70094b..f559f1679 100644 --- a/tests/unit_tests/telemetry/test_heartbeat.py +++ b/tests/unit_tests/telemetry/test_heartbeat.py @@ -183,6 +183,7 @@ class TestBuildHeartbeatPayload: assert by_workspace['workspace-b']['execution_generation'] == 9 assert by_workspace['workspace-b']['adapters'] == ['WorkspaceBAdapter'] assert 'workspace_resources' not in by_workspace['workspace-a'] + ap.plugin_connector.list_plugins.assert_not_awaited() ap.persistence_mgr.execute_async.assert_not_awaited() ap.workspace_service.list_active_execution_bindings.assert_awaited_once() diff --git a/tests/unit_tests/telemetry/test_platform_model.py b/tests/unit_tests/telemetry/test_platform_model.py new file mode 100644 index 000000000..8bca383ea --- /dev/null +++ b/tests/unit_tests/telemetry/test_platform_model.py @@ -0,0 +1,27 @@ +from types import SimpleNamespace + +import pytest +from pydantic import BaseModel + +from langbot.pkg.telemetry.platform import observe_adapter + + +class ModelAdapter(BaseModel): + def get_supported_apis(self): + return ['send_message'] + + async def send_message(self): + return {'message_id': 'ok'} + + +@pytest.mark.asyncio +async def test_model_adapter_instrumentation_is_instance_local(): + first, second = ModelAdapter(), ModelAdapter() + ap = SimpleNamespace(telemetry=None) + observe_adapter(ap, None, first) + wrapped = first.send_message + observe_adapter(ap, None, first) + assert first.send_message is wrapped + assert not getattr(second, '_execution_observed', False) + assert await first.send_message() == {'message_id': 'ok'} + assert await second.send_message() == {'message_id': 'ok'} From 4f1d8973477b2fd88b8f88850daa296ea137808a Mon Sep 17 00:00:00 2001 From: RockChinQ Date: Fri, 2 Oct 2026 15:52:22 +0000 Subject: [PATCH 15/30] fix(cloud): retry workspace snapshots every five minutes instead of daily --- src/langbot/pkg/telemetry/heartbeat.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/langbot/pkg/telemetry/heartbeat.py b/src/langbot/pkg/telemetry/heartbeat.py index 74718412d..2499c7e06 100644 --- a/src/langbot/pkg/telemetry/heartbeat.py +++ b/src/langbot/pkg/telemetry/heartbeat.py @@ -271,4 +271,8 @@ async def heartbeat_loop(ap: core_app.Application) -> None: ap.logger.debug(f'Telemetry heartbeat failed: {e}') except Exception: pass - await asyncio.sleep(HEARTBEAT_INTERVAL_SECONDS) + # Cloud dashboards need fresh scoped snapshots, including Workspaces + # discovered after startup. A transient startup/projection failure must + # not suppress synchronization for an entire day. + cloud_runtime = getattr(getattr(ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime' + await asyncio.sleep(300 if cloud_runtime else HEARTBEAT_INTERVAL_SECONDS) From b205c4000f04d3c8f0a892c3791488c82ab6b4d0 Mon Sep 17 00:00:00 2001 From: RockChinQ Date: Fri, 2 Oct 2026 16:13:25 +0000 Subject: [PATCH 16/30] fix(runtime): isolate bounded DNS resolution from blocking SDK work --- src/langbot/pkg/utils/bounded_executor.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/langbot/pkg/utils/bounded_executor.py b/src/langbot/pkg/utils/bounded_executor.py index 006714181..cc53beb6a 100644 --- a/src/langbot/pkg/utils/bounded_executor.py +++ b/src/langbot/pkg/utils/bounded_executor.py @@ -173,6 +173,13 @@ class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): max_workers=max_workers, thread_name_prefix=thread_name_prefix, ) + # asyncio DNS uses the default executor too. Keep a separately bounded + # resolver pool so long-lived SDK calls cannot starve control-plane DNS. + self._dns_executor = ( + BoundedThreadPoolExecutor(max_workers=2, max_pending=32, thread_name_prefix='langbot-dns') + if thread_name_prefix != 'langbot-dns' + else None + ) self.max_workers = max_workers self.max_pending = max_pending self.max_inflight_per_scope = max_inflight_per_scope @@ -194,6 +201,10 @@ class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): *args: Any, **kwargs: Any, ) -> concurrent.futures.Future: + import socket + + if self._dns_executor is not None and fn in (socket.getaddrinfo, socket.getnameinfo): + return self._dns_executor.submit(fn, *args, **kwargs) scope = current_blocking_work_scope() if not self._capacity.acquire(blocking=False): with self._stats_lock: @@ -246,6 +257,11 @@ class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor): future.add_done_callback(complete) return future + def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None: + if self._dns_executor is not None: + self._dns_executor.shutdown(wait=wait, cancel_futures=cancel_futures) + super().shutdown(wait=wait, cancel_futures=cancel_futures) + def _release_scope_locked(self, scope: str | None) -> None: if scope is None: return From dcc7e17d7751c33e8db95ea90fef2e283a8b683b Mon Sep 17 00:00:00 2001 From: RockChinQ Date: Sat, 3 Oct 2026 05:03:02 +0000 Subject: [PATCH 17/30] fix(cloud): retain all execution records for activity counts --- src/langbot/pkg/telemetry/execution.py | 6 ++++ .../telemetry/test_cloud_sampling.py | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/unit_tests/telemetry/test_cloud_sampling.py diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index f4af0defa..da0111f47 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -107,6 +107,12 @@ class ExecutionCounters: # ------------------------------------------------------------------ config def trace_mode(self) -> str: + # Cloud activity counts must not be computed from sampled successes. + # Telemetry opt-out is still enforced before recording observations. + from ..utils import constants + + if constants.edition == 'cloud': + return 'all' mode = str(self.manager.telemetry_config.get('execution_trace', DEFAULT_TRACE_MODE) or '').strip().lower() return mode if mode in TRACE_MODES else DEFAULT_TRACE_MODE diff --git a/tests/unit_tests/telemetry/test_cloud_sampling.py b/tests/unit_tests/telemetry/test_cloud_sampling.py new file mode 100644 index 000000000..b1a604489 --- /dev/null +++ b/tests/unit_tests/telemetry/test_cloud_sampling.py @@ -0,0 +1,34 @@ +from types import SimpleNamespace + +from langbot.pkg.telemetry.execution import ExecutionCounters +from langbot.pkg.utils import constants + + +def test_cloud_counts_are_not_sampled(monkeypatch): + monkeypatch.setattr(constants, 'edition', 'cloud') + counters = ExecutionCounters(SimpleNamespace(telemetry_config={'execution_trace': 'sampled'})) + assert counters.trace_mode() == 'all' + state = SimpleNamespace(failure_reason='', stages=[], synthetic=False, trace_id='00000001') + assert counters._trace_emitted(state) + + +def test_community_retains_sampling(monkeypatch): + monkeypatch.setattr(constants, 'edition', 'community') + counters = ExecutionCounters(SimpleNamespace(telemetry_config={'execution_trace': 'sampled'})) + assert counters.trace_mode() == 'sampled' + + +def test_dns_survives_saturated_executor(): + import socket + import threading + from langbot.pkg.utils.bounded_executor import BoundedThreadPoolExecutor + + gate = threading.Event() + executor = BoundedThreadPoolExecutor(max_workers=1, max_pending=0) + try: + blocked = executor.submit(gate.wait) + assert executor.submit(socket.getaddrinfo, '127.0.0.1', 443).result(timeout=2) + finally: + gate.set() + blocked.result(timeout=2) + executor.shutdown() From 4482e8d8768d7f6765b58e9d51177bc3f1d78227 Mon Sep 17 00:00:00 2001 From: Hyu Date: Wed, 7 Oct 2026 09:41:41 +0800 Subject: [PATCH 18/30] feat(telemetry)!: upload one record per execution node and one closure per chain Core no longer assembles a whole trace locally. Every workflow node is uploaded the moment it completes as its own `execution_node` record, and each execution closes with exactly one `execution_chain` record; all records of one execution carry the same `event_id`, so Space groups them back into a single chain. - every execution starts at an inbound event: event-less Pipeline lanes and WebUI/API debug runs synthesize their virtual inbound event at the boundary and are marked debug=true, so a Runner is never a trace origin - sampling is decided once per event from a stable hash of the event id (the previous `int(id[:8], 16)` raised on every `platform:`-prefixed id and so dropped every successful inbound-event chain) and is reported on each record as sample{mode,denominator}; the default mode is now `all` - the local trace buffer, the 120s TTL, the per-process trace cap and the shared open-node mirror are gone, which also removes the truncated-chain upload and the re-buffered-closed-trace leak they caused - a broken chain is always uploaded, and its reason falls back to the first failed node so the chain explains itself - flush keeps records until the sender accepts them and counts what it drops; telemetry health replaces the unused send-task queue - heartbeat reports bound Agent / plugin-processor / event-binding counts - bound Pipeline and Agent resources (plugins, MCP servers, knowledge bases, tools with their Host source, skills, models and the Runner schema selectors that picked them) are reported on the chain record - failure text is bounded and redacted instead of passing exception text through - the dead per-query feature counters and their call sites are removed Wire contract: schema 2 (`execution_node` / `execution_chain`). --- src/langbot/pkg/agent/runner/config_schema.py | 59 ++ src/langbot/pkg/agent/runner/orchestrator.py | 17 + src/langbot/pkg/api/http/service/agent.py | 53 +- src/langbot/pkg/box/service.py | 3 - src/langbot/pkg/core/app.py | 10 +- src/langbot/pkg/pipeline/pipelinemgr.py | 71 ++- src/langbot/pkg/pipeline/preproc/preproc.py | 24 + src/langbot/pkg/provider/tools/loaders/mcp.py | 3 - src/langbot/pkg/provider/tools/toolmgr.py | 7 - src/langbot/pkg/telemetry/execution.py | 588 ++++++++++-------- src/langbot/pkg/telemetry/features.py | 102 --- src/langbot/pkg/telemetry/heartbeat.py | 78 +++ src/langbot/pkg/telemetry/resources.py | 244 ++++++++ src/langbot/pkg/telemetry/telemetry.py | 103 ++- src/langbot/pkg/telemetry/trace.py | 264 ++++---- src/langbot/templates/config.yaml | 12 +- .../platform/test_botmgr_execution_trace.py | 77 ++- .../unit_tests/provider/test_mcp_resources.py | 5 - .../telemetry/test_cloud_sampling.py | 7 +- tests/unit_tests/telemetry/test_features.py | 92 --- .../telemetry/test_pipeline_lane.py | 114 ++-- tests/unit_tests/telemetry/test_telemetry.py | 51 -- tests/unit_tests/telemetry/test_trace.py | 579 ++++++++++------- tests/unit_tests/test_telemetry.py | 29 - 24 files changed, 1532 insertions(+), 1060 deletions(-) delete mode 100644 src/langbot/pkg/telemetry/features.py create mode 100644 src/langbot/pkg/telemetry/resources.py delete mode 100644 tests/unit_tests/telemetry/test_features.py delete mode 100644 tests/unit_tests/test_telemetry.py diff --git a/src/langbot/pkg/agent/runner/config_schema.py b/src/langbot/pkg/agent/runner/config_schema.py index aa7d13f7e..1fad3193f 100644 --- a/src/langbot/pkg/agent/runner/config_schema.py +++ b/src/langbot/pkg/agent/runner/config_schema.py @@ -16,6 +16,65 @@ KB_SELECTOR_TYPES = {'knowledge-base-multi-selector'} PROMPT_EDITOR_TYPES = {'prompt-editor'} NONE_SENTINELS = {'', '__none__', '__none'} +# Selector field types that pick a Host resource. The Runner config form is +# plugin-declared and open-ended (DynamicFormItemSchema allows extra types), so +# this is a registry, not a fixed list: a new selector type is one line here and +# is then reported by telemetry without touching the extraction call sites. +RESOURCE_SELECTOR_TYPES: dict[str, str] = { + 'knowledge-base-multi-selector': 'knowledge_bases', + 'tool-multi-selector': 'tools', + 'mcp-server-multi-selector': 'mcp_servers', + 'skill-multi-selector': 'skills', +} + + +def declared_resource_selectors(descriptor: RunnerDescriptor | None) -> dict[str, list[str]]: + """Map each selected resource kind to the schema field names that select it. + + Only the descriptor is read, so this reports what a Runner *can* select even + when the current configuration leaves the field empty. + """ + declared: dict[str, list[str]] = {} + if descriptor is None: + return declared + for item in descriptor.config_schema or []: + if not isinstance(item, dict): + continue + kind = RESOURCE_SELECTOR_TYPES.get(normalize_schema_item_type(item.get('type'))) + field_name = item.get('name') + if not kind or not isinstance(field_name, str) or not field_name: + continue + declared.setdefault(kind, []) + if field_name not in declared[kind]: + declared[kind].append(field_name) + return declared + + +def extract_selected_resources( + descriptor: RunnerDescriptor | None, + runner_config: dict[str, typing.Any] | None, +) -> dict[str, list[str]]: + """Extract the resource ids selected through schema-declared selector fields. + + Field names are never hard-coded: they come from the Runner's own config + schema, so a form that renames or adds a selector keeps working. + """ + selected: dict[str, list[str]] = {} + if not isinstance(runner_config, dict): + return selected + for kind, field_names in declared_resource_selectors(descriptor).items(): + values: list[str] = [] + for field_name in field_names: + value = runner_config.get(field_name) + if isinstance(value, str): + if value not in NONE_SENTINELS: + values.append(value) + elif isinstance(value, (list, tuple)): + values.extend(item for item in value if isinstance(item, str) and item not in NONE_SENTINELS) + if values: + selected[kind] = list(dict.fromkeys(values)) + return selected + def normalize_schema_item_type(item_type: typing.Any) -> typing.Any: """Normalize legacy/frontend DynamicForm aliases to protocol field types.""" diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index c7c9443eb..b04ec8394 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -37,6 +37,7 @@ from .run_journal import AgentRunJournal from .session_registry import AgentRunSessionRegistry, get_session_registry from .state_scope import build_state_context from ...provider.tools.loaders import skill as skill_loader +from ...telemetry import resources as telemetry_resources from ...telemetry import trace as trace_mod from ...telemetry.execution import bind_trace as bind_execution_trace from ...telemetry.execution import close_trace as close_execution_trace @@ -142,6 +143,22 @@ class AgentRunOrchestrator: binding=binding, descriptor=descriptor, ) + # Telemetry: report what this run was granted (never what it used). The + # materialized resource list is authoritative for tools/skills/models; + # the Runner's own config form is authoritative for its selectors. + try: + from . import config_schema as runner_config_schema + + capabilities = getattr(descriptor, 'capabilities', None) + telemetry_resources.note_agent( + runner_id=descriptor.id, + resources=resources, + selectors=runner_config_schema.extract_selected_resources(descriptor, binding.runner_config), + capabilities=capabilities.model_dump() if hasattr(capabilities, 'model_dump') else None, + tools_all=getattr(binding.resource_policy, 'allow_all_tools', None), + ) + except Exception: + pass context = await self.context_builder.build_context_from_event( event=execution_event, diff --git a/src/langbot/pkg/api/http/service/agent.py b/src/langbot/pkg/api/http/service/agent.py index 260b1f2df..5cfcf116d 100644 --- a/src/langbot/pkg/api/http/service/agent.py +++ b/src/langbot/pkg/api/http/service/agent.py @@ -348,25 +348,42 @@ class AgentService: execution_events.append(visible_result) final_text = '' - async for output in self.ap.agent_run_orchestrator.run( - event, - binding, - adapter_context={ - '_query': execution_query, - '_execution_context': execution_context, - '_result_observer': observe_result, - }, + # A debug run has no platform event: the synthetic event envelope is the + # execution's inbound boundary, so the chain is opened here with the real + # event id and marked as a debug WebUI/API execution. The Runner inside + # then joins this chain instead of becoming a trace origin of its own. + from ....telemetry.execution import ingress as execution_ingress + + principal_type = str(getattr(getattr(context, 'principal', None), 'principal_type', '') or '') + origin = 'api' if principal_type == 'api_key' else 'webui' + with execution_ingress( + self.ap, + 'event_done', + execution_context, + event_id, + debug=True, + origin=origin, + synthetic_event=event_type, ): - output_text = self._provider_output_to_text(output) - if output_text: - final_text = output_text - output_items.append( - { - 'kind': output.__class__.__name__, - 'role': str(getattr(output, 'role', '') or ''), - 'text': output_text, - } - ) + async for output in self.ap.agent_run_orchestrator.run( + event, + binding, + adapter_context={ + '_query': execution_query, + '_execution_context': execution_context, + '_result_observer': observe_result, + }, + ): + output_text = self._provider_output_to_text(output) + if output_text: + final_text = output_text + output_items.append( + { + 'kind': output.__class__.__name__, + 'role': str(getattr(output, 'role', '') or ''), + 'text': output_text, + } + ) return { 'event_id': event_id, diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index 4f51b15aa..a18ebd914 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -20,7 +20,6 @@ from langbot_plugin.box.security import BOX_SHARED_WORKSPACE_PROBE_PREFIX from .admission import SandboxAdmissionController, require_cloud_admission_policy from .connector import BoxRuntimeConnector, _get_box_config from . import secure_fs -from ..telemetry import features as telemetry_features from ..utils import httpclient from ..api.http.context import ExecutionContext from ..api.http.service.tenant import TenantContext, require_workspace_uuid @@ -636,7 +635,6 @@ class BoxService: f'query_id={query.query_id} ' f'summary={json.dumps(self._summarize_result(result), ensure_ascii=False)}' ) - telemetry_features.increment(query, 'sandbox', 'execs') return self._serialize_result(result) def resolve_box_session_id(self, query: pipeline_query.Query) -> str: @@ -2011,7 +2009,6 @@ class BoxService: # ── Observability ───────────────────────────────────────────────── def _record_error(self, exc: Exception, query: pipeline_query.Query): - telemetry_features.increment(query, 'sandbox', 'errors') self._recent_errors.append( { 'timestamp': _dt.datetime.now(_UTC).isoformat(), diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 7da4a0a15..cf3ecd13a 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -309,6 +309,14 @@ class Application: if runtime_ops_reporter is not None: runtime_ops_stats = runtime_ops_reporter.get_stats() + telemetry_stats = {} + telemetry_health = getattr(self.telemetry, 'health', None) + if callable(telemetry_health): + try: + telemetry_stats = telemetry_health() + except Exception: + telemetry_stats = {} + return { 'asyncio_tasks': asyncio_tasks, 'event_loop': self.event_loop_monitor.snapshot(), @@ -320,7 +328,7 @@ class Application: 'models': model_stats, 'runtimes': runtime_stats, 'runtime_ops': runtime_ops_stats, - 'telemetry_tasks': len(getattr(self.telemetry, 'send_tasks', ())), + 'telemetry': telemetry_stats, } async def initialize(self): diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index c98b94c8a..ad45b4c50 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -194,6 +194,19 @@ class RuntimePipeline: query.variables['_pipeline_bound_mcp_servers'] = self.bound_mcp_servers query.variables['_pipeline_mcp_resource_attachments'] = self.mcp_resource_attachments query.variables['_pipeline_mcp_resource_agent_read_enabled'] = self.mcp_resource_agent_read_enabled + # Telemetry: the bindings this lane runs with. None means "all enabled", + # which the *_all flags make explicit in the chain record. + try: + from ..telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline( + plugins=self.bound_plugins, + plugins_all=self.enable_all_plugins, + mcp_servers=self.bound_mcp_servers, + mcp_all=self.enable_all_mcp_servers, + ) + except Exception: + pass # Record query start for monitoring try: @@ -370,7 +383,38 @@ class RuntimePipeline: i += 1 + @staticmethod + def _synthetic_origin(query: pipeline_query.Query) -> str: + """Origin of a lane that starts without an inbound platform event.""" + try: + variables = getattr(query, 'variables', None) + declared = str(variables.get('_telemetry_origin') or '').strip() if isinstance(variables, dict) else '' + except Exception: + declared = '' + if declared in ('webui', 'api'): + return declared + adapter = getattr(query, 'adapter', None) + if adapter is not None and adapter.__class__.__name__ == 'WebSocketAdapter': + # WebChat and the CLI diagnostic surface both speak through the + # WebSocket proxy adapter; the CLI marks itself in its variables. + try: + variables = getattr(query, 'variables', None) + if isinstance(variables, dict) and '_cli_run_status' in variables: + return 'api' + except Exception: + pass + return 'webui' + return 'api' + + @staticmethod + def _synthetic_event_type(query: pipeline_query.Query) -> str: + """Event type of the virtual inbound event a synthetic lane stands for.""" + event = getattr(query, 'message_event', None) + event_type = str(getattr(event, 'type', None) or '').strip() + return event_type[:160] or 'message.received' + async def process_query(self, query: pipeline_query.Query): + from ..telemetry import trace as trace_mod from ..telemetry.execution import ingress from ..telemetry.execution import record as record_execution from ..telemetry.platform import processing_mode @@ -380,10 +424,21 @@ class RuntimePipeline: # The Workspace-scoped opaque query uuid is the execution identity Space # shows for this lane; it is a stable UUID for every pooled query. execution_id = str(getattr(query, 'query_uuid', '') or '').strip() or str(query.query_id) + # A lane with no inbound platform event in flight (WebChat, WebUI debug, + # HTTP pipeline run) synthesizes its virtual inbound event here, so every + # execution starts at an event boundary and the Runner is never the trace + # origin. A lane running under a platform route nests inside that ingress. + synthesizing = trace_mod.current() is None try: - # Callers without a platform event (Webchat, HTTP API) still get one - # trace for the whole Pipeline lane; nested calls reuse the trace. - with ingress(self.ap, 'pipeline_done', getattr(self, 'execution_context', None), execution_id=execution_id): + with ingress( + self.ap, + 'pipeline_done', + getattr(self, 'execution_context', None), + execution_id=execution_id, + debug=synthesizing, + origin=self._synthetic_origin(query) if synthesizing else 'platform', + synthetic_event=self._synthetic_event_type(query) if synthesizing else '', + ): # The lane is its own workflow step: acknowledgements and replies it # sends directly, and the runner it drives, hang under this node. # When the lane runs under a platform route the node nests there. @@ -576,14 +631,8 @@ class RuntimePipeline: state = current_trace() # The runner orchestrator already lands its own failed ``runner/execute`` # node when the break happens inside a runner; in that case only carry the - # reason over instead of repeating the node on the trace. - already_recorded = bool( - state is not None - and any( - stage.get('family') == 'runner' and stage.get('outcome') not in ('success', 'skipped') - for stage in state.stages - ) - ) + # reason over instead of repeating the node on the chain. + already_recorded = bool(state is not None and state.runner_failed) if not already_recorded: # The lane failure is its own workflow step, hanging under whatever # step routed into it (or a root when nothing did). diff --git a/src/langbot/pkg/pipeline/preproc/preproc.py b/src/langbot/pkg/pipeline/preproc/preproc.py index 67db3b1a4..2a0878170 100644 --- a/src/langbot/pkg/pipeline/preproc/preproc.py +++ b/src/langbot/pkg/pipeline/preproc/preproc.py @@ -108,6 +108,12 @@ class PreProcessor(stage.PipelineStage): ) catalog = ResourcePolicyProjector.filter_tools(catalog, policy) ToolManager.bind_query_tool_sources(query, catalog) + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline(tools=catalog, tools_all=policy.allow_all_tools) + except Exception: + pass return ToolManager.tools_from_catalog(catalog) def _runner_accepts_multimodal_input(self, descriptor: RunnerDescriptor | None) -> bool: @@ -393,6 +399,18 @@ class PreProcessor(stage.PipelineStage): descriptor, runner_config, ) + # Telemetry: the knowledge bases this lane may retrieve from, plus the + # schema-declared selector fields that selected them. + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline( + knowledge_bases=query.variables['_knowledge_base_uuids'], + runner_id=descriptor.id if descriptor is not None else '', + selectors=config_schema.extract_selected_resources(descriptor, runner_config), + ) + except Exception: + pass # Emit PromptPreProcessing before the runner receives the query. @@ -427,4 +445,10 @@ class PreProcessor(stage.PipelineStage): bound_skills = extensions_prefs['skills'] query.variables['_pipeline_bound_skills'] = bound_skills + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline(skills=bound_skills) + except Exception: + pass return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/provider/tools/loaders/mcp.py b/src/langbot/pkg/provider/tools/loaders/mcp.py index 48501292b..7b5a83afd 100644 --- a/src/langbot/pkg/provider/tools/loaders/mcp.py +++ b/src/langbot/pkg/provider/tools/loaders/mcp.py @@ -1464,9 +1464,6 @@ class RuntimeMCPSession: if query is None: return try: - from langbot.pkg.telemetry import features as telemetry_features - - telemetry_features.increment(query, 'mcp_resource_reads', envelope.get('source') or 'unknown') query.variables.setdefault(MCP_RESOURCE_TRACE_QUERY_KEY, []).append( { 'server_name': envelope.get('server_name'), diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index b09dff9d8..26329dad0 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -480,8 +480,6 @@ class ToolManager: query: pipeline_query.Query, source_ref: ToolSourceRef | None = None, ) -> typing.Any: - from langbot.pkg.telemetry import features as telemetry_features - source_ref = source_ref or self.get_query_tool_source(query, name) if source_ref is not None: execution_context = get_query_execution_context(query) @@ -534,7 +532,6 @@ class ToolManager: return await loader.invoke_tool(name, parameters, query, source_id=source_id) return await loader.invoke_tool(name, parameters, query) - telemetry_features.increment(query, 'tool_calls', telemetry_source) return await self._invoke_tool_with_monitoring( source=telemetry_source, name=name, @@ -547,7 +544,6 @@ class ToolManager: await self._bind_plugin_workspace(execution_context) sandbox_available = await self._workspace_sandbox_available(execution_context) if sandbox_available and await self.native_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'native') return await self._invoke_tool_with_monitoring( source='native', name=name, @@ -556,7 +552,6 @@ class ToolManager: invoke=lambda: self.native_tool_loader.invoke_tool(name, parameters, query), ) if await self.plugin_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'plugin') return await self._invoke_tool_with_monitoring( source='plugin', name=name, @@ -565,7 +560,6 @@ class ToolManager: invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query), ) if await self.mcp_tool_loader.has_tool(execution_context, name): - telemetry_features.increment(query, 'tool_calls', 'mcp') return await self._invoke_tool_with_monitoring( source='mcp', name=name, @@ -574,7 +568,6 @@ class ToolManager: invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query), ) if sandbox_available and await self.skill_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'skill') return await self._invoke_tool_with_monitoring( source='skill', name=name, diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index da0111f47..1e5a16a1c 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -1,18 +1,16 @@ -"""Content-free execution records for the existing telemetry sender. +"""Point-wise execution records for the existing telemetry sender. -This module reports observations only. Coverage catalogs and acceptance rules -belong to Space. +One execution is reported as a chain of independent records that all carry the +same ``event_id``: one ``execution_node`` per workflow node, emitted the moment +that node completes, plus exactly one ``execution_chain`` emitted when the +execution closes. The sender never assembles a trace locally, so there is no +per-trace buffer, no TTL and no per-process trace cap; memory is bounded by the +outbound record buffer, the flush batch size and the small cross-task registry. -One complete execution is one record: the execution identity (Pipeline query, -inbound platform event or Agent run), its bounded ordered stages and its -terminal outcome. Nothing is aggregated or counted across executions — every -record carries the id of the execution it belongs to, so Space can fetch the -whole chain by primary identity. Records are buffered whole and flushed as -``{"records": [...]}`` on the configured cadence and on shutdown; an in-flight -trace is never flushed half-written. - -Memory is bounded in stages per trace, buffered traces, buffered records, the -cross-task execution registry and flush batch size. +Sampling is decided exactly once per event, at ingress, from a stable hash of +the execution identity, so every record of one execution carries the same +decision. A chain that broke is always emitted even when sampling excluded it, +which is why a chain may arrive with zero or partial nodes. """ from __future__ import annotations @@ -21,29 +19,48 @@ import asyncio import contextlib import contextvars import json +import re import time +import zlib from datetime import datetime, timezone from . import trace as trace_mod from .identity import workspace_identity +from .resources import snapshot as resources_snapshot +from .resources import begin as resources_begin +from .resources import end as resources_end from .trace import TraceState MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'}) OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'}) +FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}) +ORIGINS = frozenset({'platform', 'webui', 'api'}) +CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'}) +CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'}) -# Trace bounds: stages per trace, traces buffered per process, trace lifetime. -MAX_TRACES = 64 -TRACE_TTL_SECONDS = 120 +SCHEMA = 2 TRACE_MODES = frozenset({'off', 'failures', 'sampled', 'all'}) -DEFAULT_TRACE_MODE = 'sampled' +DEFAULT_TRACE_MODE = 'all' DEFAULT_TRACE_SAMPLE = 20 -# Record bounds: buffered records, one flush batch, flush cadence, registry. +# Sampling bounds: outbound record buffer, one flush batch, flush cadence. MAX_BUFFERED_RECORDS = 512 MAX_RECORDS_PER_FLUSH = 128 MAX_FLUSH_BYTES = 200 * 1024 DEFAULT_FLUSH_SECONDS = 180 +MAX_FLUSH_ATTEMPTS = 3 +FLUSH_RETRY_BACKOFF_SECONDS = 0.2 + +# Cross-task registry: execution id -> owning execution, bounded and time-limited. MAX_REGISTRY_KEYS = 256 +REGISTRY_TTL_SECONDS = 300 + +# Failure details are operator-facing: keep the failure class and a short +# machine-ish prefix, never the free text a user or a model produced. +ERROR_LIMIT = 200 +ERROR_PREFIX_CHARS = 120 +ERROR_PREFIX_TOKENS = 12 +_ERROR_CLASS = re.compile(r'[A-Za-z_][\w.]*') # Set by cross-task call sites (plugin/RPC actions) that run without the # ingress context but know which execution they belong to. @@ -68,16 +85,36 @@ def current_execution_id() -> str: return _execution_id.get() -def _bounded_error(value, limit: int = 400) -> str: - """Bound and de-fang one failure detail before it leaves the process. +def _short_prefix(text: str) -> str: + """Keep at most a short machine-ish prefix of one failure message.""" + if len(text) > ERROR_PREFIX_CHARS: + text = text[:ERROR_PREFIX_CHARS] + parts = text.split(' ') + if len(parts) > ERROR_PREFIX_TOKENS: + text = ' '.join(parts[:ERROR_PREFIX_TOKENS]) + return text - Error text is operator-facing: keep it single-line, printable and short so a - noisy failure cannot smuggle control characters or bloat the payload. + +def _bounded_error(value, limit: int = ERROR_LIMIT) -> str: + """Bound and redact one failure detail before it leaves the process. + + The failure class and a short machine-ish prefix survive; control characters + are stripped, whitespace is collapsed and the result is capped, so a noisy + failure cannot smuggle user or model content into the payload. """ if value is None: return '' text = ''.join(ch if ch.isprintable() else ' ' for ch in str(value)) text = ' '.join(text.split()) + if not text: + return '' + head, separator, rest = text.partition(':') + head = head.strip() + if separator and len(head) <= 80 and _ERROR_CLASS.fullmatch(head): + prefix = _short_prefix(rest.strip()) + text = f'{head}: {prefix}' if prefix else head + else: + text = _short_prefix(text) return text[:limit] @@ -91,18 +128,21 @@ def _duration_ms(started_at: str, ended_at: str) -> int: class ExecutionCounters: + """Point-wise execution reporting: one record per node, one per closure.""" + def __init__(self, manager): self.manager = manager self.records: list[dict] = [] self.task: asyncio.Task | None = None + # Exposed health: records dropped or accepted by the sender. self.dropped = 0 - self.traces: dict[str, TraceState] = {} - self.trace_deadlines: dict[str, float] = {} - # Execution identity -> in-flight trace, for stages recorded by tasks + self.sent = 0 + # Execution identity -> owning execution, for nodes recorded by tasks # that no longer share the ingress context (plugin/RPC platform calls). + # It resolves identity and the sampling decision only; node payloads are + # never accumulated here. self.registry: dict[str, TraceState] = {} self.registry_deadlines: dict[str, float] = {} - self.dropped_traces = 0 # ------------------------------------------------------------------ config @@ -130,6 +170,63 @@ class ExecutionCounters: seconds = DEFAULT_FLUSH_SECONDS return seconds if 1 <= seconds <= 86400 else DEFAULT_FLUSH_SECONDS + def health(self) -> dict: + """Real telemetry health for the instance liveness snapshot.""" + return { + 'records_buffered': len(self.records), + 'records_sent': self.sent, + 'records_dropped': self.dropped, + 'executions_in_flight': len({id(state) for state in self.registry.values()}), + } + + # ----------------------------------------------------------------- sampling + + def _sampling_decision(self, event_id: str) -> tuple[str, int, bool]: + """Decide once, deterministically, whether this event is reported. + + The identity is hashed instead of being parsed, so every id shape + (``platform:...``, uuids, debug ids) samples the same way. + """ + mode = self.trace_mode() + if mode == 'sampled': + denominator = self.trace_sample() + try: + sampled_in = zlib.crc32(str(event_id).encode('utf-8')) % denominator == 0 + except Exception: + sampled_in = False + return mode, denominator, sampled_in + # 'off' reports nothing, 'failures' keeps nodes out but still reports a + # broken chain, 'all' reports everything; denominator is 1 for all three. + return mode, 1, mode == 'all' + + def configure( + self, + state: TraceState, + context=None, + *, + debug: bool = False, + origin: str = 'platform', + identity=None, + ) -> None: + """Bind the execution identity, the sampling decision and the flags.""" + if state.configured: + return + state.configured = True + state.debug = bool(debug) + state.origin = origin if origin in ORIGINS else 'platform' + mode, denominator, sampled_in = self._sampling_decision(state.event_id) + state.mode = mode + state.denominator = denominator + # Debug chains are always emitted (unless reporting is off entirely). + state.emit_nodes = mode != 'off' and (state.debug or sampled_in) + if identity is not None: + state.identity = dict(identity) + elif context is not None: + try: + state.identity = workspace_identity(context) + except Exception: + pass + # --------------------------------------------------------------- recording def record( @@ -148,102 +245,96 @@ class ExecutionCounters: node: str = '', ): try: - cfg = self.manager.telemetry_config - if not cfg or cfg.get('disable_telemetry', False) or not cfg.get('url'): + if not self._enabled(): return - if family not in {'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}: - return - if mode not in MODES or outcome not in OUTCOMES: + if family not in FAMILIES or mode not in MODES or outcome not in OUTCOMES: return # Only code-defined identifiers are accepted; never pass user values. if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner, node)): return state = trace_mod.current() if state is None: - # Cross-task work has no inherited trace; resolve the owning - # execution through the registry instead. + # Cross-task work has no inherited context; resolve the owning + # execution through the registry. It has no open step here, so + # its node becomes a root of that execution instead of being + # mirrored from another task's stack. lookup = str(execution_id or '').strip() or current_execution_id() if lookup: - state = self.registry.get(lookup) - if state is None: + state = self._resolve(lookup) + if state is None or state.closed: # Every observation belongs to exactly one execution chain. return + self.configure(state) try: identity = workspace_identity(context) except Exception: # A cross-task caller may only carry the Workspace id; reuse the - # identity already bound to the owning trace. + # identity already bound to the owning execution. identity = state.identity or {} if not identity.get('instance_id') or not identity.get('workspace_uuid'): return - self._record_trace_stage( - state, - identity=identity, - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=synthetic, - error=_bounded_error(error), - node=node, + state.identity = identity + + ended_at = datetime.now(timezone.utc).isoformat() + node_id, seq, started_at, parent = trace_mod.resolve_node(state, str(node or ''), ended_at) + root = parent == '' and not state.root_seen + if root: + state.root_seen = True + bounded_error = _bounded_error(error) + state.note_node(outcome=outcome, adapter=adapter, runner=runner, family=family, error=bounded_error) + if not state.emit_nodes: + # Sampling excluded this execution: the node is never buffered, + # only counted so the terminal chain can explain the gap. + state.dropped_nodes += 1 + return + self._enqueue_record( + { + 'event_type': 'execution_node', + 'schema': SCHEMA, + **self._common_fields(state, ended_at), + 'node_id': node_id, + 'parent_node_id': parent, + 'root': root, + 'seq': seq, + 'family': family, + 'operation': operation, + 'mode': mode, + 'adapter': adapter, + 'runner': runner, + 'outcome': outcome, + 'error': bounded_error, + 'route_ref': trace_mod.current_route(), + 'run_id': trace_mod.current_run(), + 'synthetic': bool(synthetic), + 'started_at': started_at, + 'ended_at': ended_at, + } ) - self._ensure_loop() except Exception: # Observability must never change execution behavior. return - def _record_trace_stage( - self, - state: TraceState, - *, - identity, - family, - operation, - mode, - adapter, - runner, - outcome, - synthetic, - error: str = '', - node: str = '', - ) -> None: - if state.abandoned: - return - if trace_mod.current() is state: - # Recorded by the task that owns the step: its step stack is exact. - # A step's own record sits under its enclosing step, never itself. - parent = trace_mod.current_parent() - if node and parent == node: - parent = trace_mod.enclosing_parent() - else: - # A cross-task observation (plugin/RPC) cannot see the step - # contextvar; attach it to the innermost step the owning task has - # open on the trace. - parent = state.open_nodes[-1] if state.open_nodes else '' - if state.trace_id not in self.traces: - # Never buffer traces this configuration would discard anyway. - if self.trace_mode() == 'off' or len(self.traces) >= MAX_TRACES: - self.dropped += 1 - state.abandoned = True - self._unregister_state(state) - return - self.traces[state.trace_id] = state - self.trace_deadlines[state.trace_id] = time.monotonic() + TRACE_TTL_SECONDS - state.identity = identity - state.append( - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=bool(synthetic), - error=error, - node=node, - parent=parent, - ) + def _common_fields(self, state: TraceState, timestamp: str) -> dict: + identity = state.identity + from ..utils import constants + + return { + 'event_id': state.event_id, + 'instance_id': identity['instance_id'], + 'workspace_uuid': identity['workspace_uuid'], + 'version': constants.semantic_version, + 'edition': constants.edition, + 'timestamp': timestamp, + 'debug': bool(state.debug), + 'sample': {'mode': state.mode, 'denominator': state.denominator}, + } + + def _enabled(self) -> bool: + try: + cfg = self.manager.telemetry_config + except Exception: + return False + return bool(cfg) and not cfg.get('disable_telemetry', False) and bool(cfg.get('url')) def _ensure_loop(self) -> None: if self.task is None or self.task.done(): @@ -252,20 +343,27 @@ class ExecutionCounters: # --------------------------------------------------------------- registry def register_trace(self, state: TraceState, execution_id) -> None: - """Alias one more execution identity onto an in-flight trace.""" + """Alias one more execution identity onto an in-flight execution.""" try: exec_id = str(execution_id or '').strip() - if not exec_id or state.abandoned: + if not exec_id: return + self.configure(state) + now = time.monotonic() + self._sweep_registry(now) if exec_id not in self.registry and len(self.registry) >= MAX_REGISTRY_KEYS: oldest = next(iter(self.registry)) self.registry.pop(oldest, None) self.registry_deadlines.pop(oldest, None) self.registry[exec_id] = state - self.registry_deadlines[exec_id] = time.monotonic() + TRACE_TTL_SECONDS + self.registry_deadlines[exec_id] = now + REGISTRY_TTL_SECONDS except Exception: return + def _resolve(self, execution_id: str) -> TraceState | None: + self._sweep_registry(time.monotonic()) + return self.registry.get(execution_id) + def _unregister_state(self, state: TraceState) -> None: for exec_id in [key for key, value in self.registry.items() if value is state]: self.registry.pop(exec_id, None) @@ -277,26 +375,68 @@ class ExecutionCounters: self.registry.pop(exec_id, None) self.registry_deadlines.pop(exec_id, None) - # ------------------------------------------------------------------ traces + # ------------------------------------------------------------------ chains def close_trace(self, state: TraceState, reason: str = 'event_done', error: str = '') -> None: - """Buffer one complete execution record when it matches the sample.""" + """Emit the one terminal ``execution_chain`` record of this execution.""" try: if state.closed: return state.closed = True - registered = self.traces.pop(state.trace_id, None) is not None - self.trace_deadlines.pop(state.trace_id, None) self._unregister_state(state) - if not registered and not state.failure_reason: + if not self._enabled(): + # Nothing can be uploaded; never grow the buffer for a record + # the sender would refuse to send. return - if (not state.stages and not state.failure_reason) or not self._trace_emitted(state): + self.configure(state) + if not self._chain_emitted(state): return - record = self._build_record(state, reason, error) - if record is not None: - self._enqueue_record(record) + self._enqueue_record(self._build_chain_record(state, reason, error)) except Exception: return + finally: + # The snapshot was read while building the chain record; stop + # collecting so a late sibling task cannot leak into the next chain. + resources_end() + + def _chain_emitted(self, state: TraceState) -> bool: + if state.mode == 'off': + return False + if state.emit_nodes or state.debug: + return True + # A chain that broke is always uploaded, even when sampling excluded it. + return bool(state.failure_reason) or state.outcome() in CHAIN_FAILURES + + def _build_chain_record(self, state: TraceState, reason: str, error: str = '') -> dict: + ended_at = datetime.now(timezone.utc).isoformat() + record = { + 'event_type': 'execution_chain', + 'schema': SCHEMA, + **self._common_fields(state, ended_at), + 'closed_by': reason if reason in CLOSED_BY else 'event_done', + 'outcome': state.outcome(), + # First failure detail in this chain, so Space can show why it broke. + 'error': _bounded_error(error) + or _bounded_error(state.failure_reason) + or _bounded_error(state.first_error) + or '', + 'duration_ms': _duration_ms(state.started_at, ended_at), + 'stage_count': state.stage_count, + 'dropped_nodes': state.dropped_nodes, + 'origin': state.origin, + 'model_name': state.model_name or '', + 'adapter': state.adapter or state.node_adapter, + 'runner': state.runner or state.node_runner, + 'runner_category': state.runner_category or '', + 'pipeline_plugins': state.pipeline_plugins, + } + # Bound resources of this execution, when any lane reported them. The + # key is omitted entirely when nothing was noted, so the payload stays + # byte-identical for executions with no bindings. + resources = resources_snapshot() + if resources: + record['resources'] = resources + return record def _enqueue_record(self, record: dict) -> None: if len(self.records) >= MAX_BUFFERED_RECORDS: @@ -305,137 +445,52 @@ class ExecutionCounters: self.records.append(record) self._ensure_loop() - def _trace_emitted(self, state: TraceState) -> bool: - mode = self.trace_mode() - if mode == 'off': - return False - # A chain that failed must always be uploaded, even when no stage was - # recorded before it broke: the reason is the whole point of the trace. - if state.failure_reason: - return True - if mode == 'all': - return True - # A failed, cancelled or timed-out stage is always worth keeping. - if any(stage['outcome'] not in ('success', 'skipped') for stage in state.stages): - return True - # WebUI debug runs are deliberately traced; they stay flagged synthetic. - if state.synthetic: - return True - if mode == 'failures': - return False - try: - return int(state.trace_id[:8], 16) % self.trace_sample() == 0 - except ValueError: - return False - - def _build_record(self, state: TraceState, reason: str, error: str = '') -> dict | None: - from ..utils import constants - - identity = state.identity - if not identity.get('instance_id') or not identity.get('workspace_uuid'): - return None - ended_at = datetime.now(timezone.utc).isoformat() - stages = state.stages - observations = [] - for stage in stages: - observation = { - 'seq': stage['seq'], - 'family': stage['family'], - 'operation': stage['operation'], - 'mode': stage['mode'], - 'adapter': stage['adapter'], - 'runner': stage['runner'], - 'outcome': stage['outcome'], - 'synthetic': stage['synthetic'], - 'error': stage['error'], - 'route_ref': stage['route_ref'], - 'run_id': stage['run_id'], - 'first_seen': stage['first_seen'], - 'last_seen': stage['last_seen'], - 'trace_id': state.trace_id, - } - # Workflow position is optional: records outside any step omit both - # keys entirely, so their payloads stay byte-identical. - if stage.get('node'): - observation['node'] = stage['node'] - if stage.get('parent'): - observation['parent'] = stage['parent'] - observations.append(observation) - return { - 'event_type': 'feature_execution', - # The execution identity: one record == one chain == one Space row. - 'query_id': state.trace_id, - 'instance_id': identity['instance_id'], - 'workspace_uuid': identity['workspace_uuid'], - 'version': constants.semantic_version, - 'edition': constants.edition, - 'timestamp': ended_at, - 'trusted': True, - # First failure detail in this chain, so Space can show why it broke. - 'error': _bounded_error(error) - or state.failure_reason - or next((stage['error'] for stage in stages if stage['error']), ''), - # Legacy-compatible columns Space already maps; best effort. - 'duration_ms': _duration_ms(state.started_at, ended_at), - 'model_name': state.model_name or '', - 'adapter': state.adapter or next((stage['adapter'] for stage in stages if stage['adapter']), ''), - 'runner': state.runner or next((stage['runner'] for stage in stages if stage['runner']), ''), - 'runner_category': state.runner_category or '', - 'pipeline_plugins': state.pipeline_plugins, - 'features': { - 'schema': 1, - 'observations': observations, - 'trace': { - 'closed_by': reason, - 'started_at': state.started_at, - 'ended_at': ended_at, - 'route_ref': next((stage['route_ref'] for stage in stages if stage['route_ref']), ''), - 'run_id': next((stage['run_id'] for stage in stages if stage['run_id']), ''), - 'outcome': state.outcome(), - 'dropped_stages': state.dropped_stages, - }, - }, - } - - def _sweep_traces(self, now: float) -> None: - for trace_id in [key for key, deadline in self.trace_deadlines.items() if deadline <= now]: - state = self.traces.get(trace_id) - if state is not None: - self.close_trace(state, 'ttl') - # ----------------------------------------------------------------- flushing async def _loop(self): interval = self.flush_seconds() - last_flush = time.monotonic() - while self.records or self.traces or self.registry: - # Traces expire on a short TTL; records keep their own cadence. - await asyncio.sleep(1 if (self.traces or self.registry) else interval) - now = time.monotonic() - self._sweep_traces(now) - self._sweep_registry(now) - if self.records and now - last_flush >= interval: - last_flush = now + try: + # The outbound loop exists only to flush the record buffer. + while self.records: + await asyncio.sleep(interval) await self.flush() - self.task = None + finally: + self.task = None - async def flush(self): + async def flush(self) -> bool: + """Upload the buffered prefix; records stay buffered until accepted.""" if not self.records: - return + return True batch: list[dict] = [] size = 0 - while self.records and len(batch) < MAX_RECORDS_PER_FLUSH: - encoded = len(json.dumps(self.records[0], ensure_ascii=False, default=str)) + for record in self.records: + if len(batch) >= MAX_RECORDS_PER_FLUSH: + break + encoded = len(json.dumps(record, ensure_ascii=False, default=str)) if batch and size + encoded > MAX_FLUSH_BYTES: break - batch.append(self.records.pop(0)) + batch.append(record) size += encoded if not batch: - return + return True payload = {'records': batch} - if not await self.manager.send(payload): - await asyncio.sleep(1) - await self.manager.send(payload) + for attempt in range(MAX_FLUSH_ATTEMPTS): + try: + if await self.manager.send(payload): + # Peek-then-send: nothing is removed until the sender + # accepted the exact batch it was given. + del self.records[: len(batch)] + self.sent += len(batch) + return True + except asyncio.CancelledError: + raise + except Exception: + pass + if attempt + 1 < MAX_FLUSH_ATTEMPTS: + await asyncio.sleep(FLUSH_RETRY_BACKOFF_SECONDS * (2**attempt)) + # The batch stays buffered for the next cycle; a permanently unaccepted + # record is dropped by the buffer bound above and counted there. + return False async def shutdown(self): if self.task is not None: @@ -451,12 +506,8 @@ class ExecutionCounters: break except (Exception, asyncio.CancelledError): pass + self.dropped += len(self.records) self.records.clear() - # Executions still in flight cannot be completed during shutdown. - self.dropped += len(self.traces) - self.dropped_traces += len(self.traces) - self.traces.clear() - self.trace_deadlines.clear() self.registry.clear() self.registry_deadlines.clear() @@ -477,7 +528,7 @@ def record(ap, context, **observation): def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None: - """Best-effort trace close usable with optional telemetry and test doubles.""" + """Best-effort chain close usable with optional telemetry and test doubles.""" try: counters = _execution_counters(ap) if counters is not None: @@ -487,8 +538,10 @@ def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None: def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding: - """Bind an execution trace and alias its identity for cross-task stages.""" + """Bind an execution and alias its identity for cross-task nodes.""" binding = trace_mod.bind(execution_id) + if binding.created: + resources_begin() if execution_id: try: counters = _execution_counters(ap) @@ -500,36 +553,61 @@ def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding: @contextlib.contextmanager -def ingress(ap, reason: str = 'event_done', context=None, execution_id: str | None = None): - """Trace one execution boundary; only the owner of the trace closes it. +def ingress( + ap, + reason: str = 'event_done', + context=None, + execution_id: str | None = None, + *, + debug: bool = False, + origin: str = 'platform', + synthetic_event: str = '', +): + """Open one execution boundary; only the owner of the chain closes it. Nested boundaries (an event routed into a Pipeline, a Runner invoked from a - dispatch) reuse the in-flight trace instead of starting a second one, and - only alias their own execution id onto it. + dispatch) reuse the in-flight chain instead of starting a second one, and + only alias their own execution id onto it. ``debug``/``origin`` apply when + this boundary starts the chain: a WebUI/API run without a real platform + event synthesizes its virtual inbound event here, so the chain has a real + identity and a node for the event it stands for. """ binding = trace_mod.bind(execution_id) - if binding.created and context is not None: - try: - binding.state.identity = workspace_identity(context) - except Exception: - pass - if execution_id and not binding.state.abandoned: - try: - counters = _execution_counters(ap) - if counters is not None: - counters.register_trace(binding.state, execution_id) - except Exception: - pass + state = binding.state + if binding.created: + resources_begin() + counters = _execution_counters(ap) + if counters is not None: + if binding.created: + counters.configure(state, context, debug=debug, origin=origin) + if execution_id: + counters.register_trace(state, execution_id) + scope = None + if binding.created and synthetic_event: + scope = trace_mod.stage_scope() + node = scope.__enter__() + record( + ap, + context, + family='platform_event', + operation=str(synthetic_event)[:160], + outcome='success', + synthetic=True, + node=node, + ) failure = '' try: yield binding except BaseException as exc: - # The boundary itself failed: keep the reason on the trace so the uploaded - # payload explains the failure instead of reporting a bare outcome. + # The boundary itself failed: keep the reason on the chain so the + # uploaded payload explains the failure instead of reporting a bare + # outcome, even when sampling had excluded this execution. if binding.created: failure = f'{type(exc).__name__}: {exc}' - binding.state.mark_failure(failure) + state.mark_failure(failure) raise finally: + if scope is not None: + scope.__exit__(None, None, None) if trace_mod.unbind_root(binding): - close_trace(ap, binding.state, reason, failure) + close_trace(ap, state, reason, failure) diff --git a/src/langbot/pkg/telemetry/features.py b/src/langbot/pkg/telemetry/features.py deleted file mode 100644 index ab66f9884..000000000 --- a/src/langbot/pkg/telemetry/features.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Per-query telemetry feature counters. - -Collects anonymous, content-free usage signals (tool call counts, knowledge -base usage, sandbox executions, ...) into ``query.variables`` during query -processing. The chat handler reads the accumulated dict when building the -telemetry payload and ships it as the ``features`` JSON object. - -Every helper here is defensive: telemetry must NEVER break the pipeline, so -all mutations are wrapped and failures are silently ignored. -""" - -from __future__ import annotations - -import typing - -if typing.TYPE_CHECKING: - import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query - - -FEATURES_KEY = '_telemetry_features' - - -def get_features(query: pipeline_query.Query) -> dict: - """Return the mutable features dict for this query, creating it if needed.""" - try: - return query.variables.setdefault(FEATURES_KEY, {}) - except Exception: - return {} - - -def increment(query: pipeline_query.Query, group: str, key: str | None = None, amount: int = 1) -> None: - """Increment a counter. - - ``increment(q, 'sandbox', 'execs')`` -> features['sandbox']['execs'] += 1 - ``increment(q, 'tool_call_rounds')`` -> features['tool_call_rounds'] += 1 - """ - try: - features = get_features(query) - if key is None: - features[group] = int(features.get(group, 0)) + amount - else: - nested = features.setdefault(group, {}) - if isinstance(nested, dict): - nested[key] = int(nested.get(key, 0)) + amount - except Exception: - pass - - -def set_value(query: pipeline_query.Query, group: str, value: typing.Any) -> None: - """Set a feature value (overwrites).""" - try: - get_features(query)[group] = value - except Exception: - pass - - -def collect_features(query: pipeline_query.Query) -> dict: - """Build the final ``features`` object for the telemetry payload. - - Combines the counters accumulated during processing with end-of-query - snapshots (activated skills, bound MCP servers). Returns a plain dict - that must be JSON-serializable; non-serializable values are dropped. - """ - features: dict = {} - try: - accumulated = query.variables.get(FEATURES_KEY) - if isinstance(accumulated, dict): - features.update(accumulated) - except Exception: - pass - - # Activated skills (names only, registered by the activate tool) - try: - activated = query.variables.get('_activated_skills', {}) - if isinstance(activated, dict) and activated: - features['activated_skills'] = sorted(activated.keys()) - except Exception: - pass - - # MCP servers bound to the pipeline (names only; None means "all enabled") - try: - bound_mcp = query.variables.get('_pipeline_bound_mcp_servers', None) - if bound_mcp is not None: - features['mcp_servers'] = list(bound_mcp) - except Exception: - pass - - # Drop anything that is not JSON-serializable - import json - - try: - json.dumps(features) - return features - except Exception: - safe: dict = {} - for k, v in features.items(): - try: - json.dumps({k: v}) - safe[k] = v - except Exception: - continue - return safe diff --git a/src/langbot/pkg/telemetry/heartbeat.py b/src/langbot/pkg/telemetry/heartbeat.py index 2499c7e06..7deb20400 100644 --- a/src/langbot/pkg/telemetry/heartbeat.py +++ b/src/langbot/pkg/telemetry/heartbeat.py @@ -13,6 +13,7 @@ credentials. from __future__ import annotations import asyncio +import time import typing from datetime import datetime, timezone @@ -36,10 +37,66 @@ class WorkspaceResourceSnapshot(typing.TypedDict): mcp_server_count: int extension_count: int skill_count: int + agent_count: int + event_processor_count: int + plugin_processor_count: int + event_binding_count: int adapters: list[str] execution_generation: int +# Agent rows are configuration, not runtime registries, so a Workspace count +# needs one scoped query. The heartbeat runs every five minutes on Cloud; a +# one-hour cache keeps the cost at one bounded query per Workspace per hour +# instead of one per cycle. Unavailable counts stay -1, never a fake zero. +_AGENT_COUNT_CACHE_TTL_SECONDS = 3600 +_agent_count_cache: dict[str, tuple[float, dict[str, int]]] = {} + + +async def _workspace_agent_kind_counts(ap: core_app.Application, workspace_uuid: str) -> dict[str, int]: + """Count one Workspace's Agents per ``kind``; empty when unavailable.""" + if not workspace_uuid: + return {} + now = time.monotonic() + cached = _agent_count_cache.get(workspace_uuid) + if cached is not None and cached[0] > now: + return dict(cached[1]) + counts: dict[str, int] = {} + try: + from ..entity.persistence import agent as persistence_agent + + statement = sqlalchemy.select(persistence_agent.Agent.kind, sqlalchemy.func.count()).group_by( + persistence_agent.Agent.kind + ) + tenant_uow = getattr(ap.persistence_mgr, 'tenant_uow', None) + if callable(tenant_uow): + async with tenant_uow(workspace_uuid): + result = await ap.persistence_mgr.execute_async(statement) + else: + result = await ap.persistence_mgr.execute_async(statement) + for kind, count in result.all(): + counts[str(kind or 'agent')] = int(count or 0) + except Exception: + counts = {} + _agent_count_cache[workspace_uuid] = (now + _AGENT_COUNT_CACHE_TTL_SECONDS, dict(counts)) + return counts + + +def _bot_processor_counts(bots) -> tuple[int, int]: + """Count plugin-processor subscriptions and event bindings across bots.""" + processors = 0 + bindings = 0 + for bot in bots: + entity = getattr(bot, 'bot_entity', bot) + declared_processors = getattr(entity, 'plugin_processors', None) + if isinstance(declared_processors, list): + processors += len(declared_processors) + declared_bindings = getattr(entity, 'event_bindings', None) + if isinstance(declared_bindings, list): + bindings += len(declared_bindings) + return processors, bindings + + async def _count( ap: core_app.Application, table, @@ -81,6 +138,10 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - 'mcp_server_count': 0, 'extension_count': 0, 'skill_count': 0, + 'agent_count': -1, + 'event_processor_count': -1, + 'plugin_processor_count': 0, + 'event_binding_count': 0, 'adapters': [], 'execution_generation': binding.placement_generation, } @@ -91,6 +152,9 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - for key, bot in getattr(ap.platform_mgr, '_bots_by_key', {}).items(): if len(key) >= 2 and key[1] in counts: counts[key[1]]['bot_count'] += 1 + processors, bindings = _bot_processor_counts([bot]) + counts[key[1]]['plugin_processor_count'] += processors + counts[key[1]]['event_binding_count'] += bindings adapter = getattr(bot, 'adapter', None) if adapter is not None and getattr(bot, 'enable', False): adapter_sets[key[1]].add(adapter.__class__.__name__) @@ -113,6 +177,10 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - for workspace_uuid, resource in counts.items(): resource['extension_count'] = resource['plugin_count'] + resource['mcp_server_count'] resource['adapters'] = sorted(adapter_sets[workspace_uuid]) + agent_counts = await _workspace_agent_kind_counts(ap, workspace_uuid) + if agent_counts: + resource['agent_count'] = int(agent_counts.get('agent', 0)) + resource['event_processor_count'] = int(agent_counts.get('event_processor', 0)) return list(counts.values()) @@ -163,6 +231,9 @@ async def build_heartbeat_payload( features['bot_count'] = len(platform_mgr.bots) adapters = sorted({bot.adapter.__class__.__name__ for bot in enabled_bots if getattr(bot, 'adapter', None)}) features['adapters'] = adapters + processors, bindings = _bot_processor_counts(platform_mgr.bots) + features['plugin_processor_count'] = processors + features['event_binding_count'] = bindings except Exception: pass @@ -206,6 +277,13 @@ async def build_heartbeat_payload( except Exception: pass + # Agent processors: configuration rows, so one scoped count per Workspace. + # The Cloud snapshot supplies these already and overrides below. + if workspace_resource is None: + agent_counts = await _workspace_agent_kind_counts(ap, workspace_uuid) + features['agent_count'] = int(agent_counts.get('agent', 0)) if agent_counts else -1 + features['event_processor_count'] = int(agent_counts.get('event_processor', 0)) if agent_counts else -1 + if workspace_resource is not None: features.update({key: value for key, value in workspace_resource.items() if key != 'workspace_uuid'}) diff --git a/src/langbot/pkg/telemetry/resources.py b/src/langbot/pkg/telemetry/resources.py new file mode 100644 index 000000000..5164e090e --- /dev/null +++ b/src/langbot/pkg/telemetry/resources.py @@ -0,0 +1,244 @@ +"""Bound-resource reporting for one execution. + +Answers "what was this execution given", never "what did it use": the ids and +names of the Plugins, MCP servers, knowledge bases, tools, skills and models +that were bound or granted to a Pipeline lane or an Agent run. Nothing here is +user content - only code-defined identifiers (uuids, plugin names, tool names) +and booleans. + +Two sources feed one execution-scoped snapshot: + +* the Host policy actually granted to the run (``ResourcePolicy`` / + materialized ``AgentResources``), which is authoritative for tools, skills and + platform APIs; +* the Runner's own configuration form (``DynamicFormItemSchema``), which is + where knowledge bases and models are selected. Selector field types are read + through :mod:`langbot.pkg.agent.runner.config_schema`, so a Runner that + declares a new selector type is picked up without a change here. + +The snapshot is attached to the terminal ``execution_chain`` record. It is +bounded twice - per list and in total - and it is best effort: telemetry must +never change execution behaviour, so every entry point swallows failures. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import json +import typing + +MAX_ITEMS_PER_LIST = 64 +MAX_NAME_CHARS = 160 +MAX_RESOURCES_BYTES = 8 * 1024 + +_resources: contextvars.ContextVar[dict[str, typing.Any] | None] = contextvars.ContextVar( + 'telemetry_resources', default=None +) + + +def begin() -> None: + """Start collecting for one execution (called when a chain is opened).""" + _resources.set({}) + + +def end() -> None: + """Stop collecting; the snapshot has already been read.""" + _resources.set(None) + + +def _bucket(section: str) -> dict[str, typing.Any] | None: + root = _resources.get() + if root is None: + return None + bucket = root.get(section) + if not isinstance(bucket, dict): + bucket = {} + root[section] = bucket + return bucket + + +def _names(values: typing.Any) -> list[str]: + """Bound one list of code-defined identifiers, preserving source order.""" + if not isinstance(values, (list, tuple)): + return [] + names: list[str] = [] + seen: set[str] = set() + for value in values: + name = '' + if isinstance(value, str): + name = value + elif isinstance(value, dict): + name = str(value.get('name') or value.get('kb_id') or value.get('tool_name') or '') + if not name: + continue + name = name[:MAX_NAME_CHARS] + if name in seen: + continue + seen.add(name) + names.append(name) + if len(names) >= MAX_ITEMS_PER_LIST: + break + return names + + +def _tool_refs(values: typing.Any) -> list[dict[str, str]]: + """Bound one list of tool grants, keeping their Host source.""" + if not isinstance(values, (list, tuple)): + return [] + refs: list[dict[str, str]] = [] + seen: set[str] = set() + for value in values: + if isinstance(value, str): + name, source, source_id = value, '', '' + elif isinstance(value, dict): + name = str(value.get('tool_name') or value.get('name') or '') + source = str(value.get('tool_type') or value.get('source') or '') + raw_id = value.get('source_id') + source_id = raw_id if isinstance(raw_id, str) else '' + else: + continue + name = name[:MAX_NAME_CHARS] + if not name or name in seen: + continue + seen.add(name) + ref = {'name': name} + if source: + ref['source'] = source[:64] + if source_id: + ref['source_id'] = source_id[:MAX_NAME_CHARS] + refs.append(ref) + if len(refs) >= MAX_ITEMS_PER_LIST: + break + return refs + + +def _set(bucket: dict[str, typing.Any] | None, key: str, value: typing.Any) -> None: + if bucket is None or value in (None, [], {}, ''): + return + bucket[key] = value + + +def note_pipeline( + *, + plugins: typing.Any = None, + plugins_all: typing.Any = None, + mcp_servers: typing.Any = None, + mcp_all: typing.Any = None, + knowledge_bases: typing.Any = None, + skills: typing.Any = None, + tools: typing.Any = None, + tools_all: typing.Any = None, + runner_id: str = '', + selectors: typing.Any = None, +) -> None: + """Record the resources one Pipeline lane is bound to. + + ``plugins``/``mcp_servers`` accept ``None`` to mean "all enabled", matching + the binding model; the ``*_all`` flags make that explicit in the payload. + """ + try: + bucket = _bucket('pipeline') + if bucket is None: + return + _set(bucket, 'plugins', _names(plugins)) + _set(bucket, 'mcp_servers', _names(mcp_servers)) + _set(bucket, 'knowledge_bases', _names(knowledge_bases)) + _set(bucket, 'skills', _names(skills)) + _set(bucket, 'tools', _tool_refs(tools)) + _set(bucket, 'runner_id', str(runner_id or '')[:MAX_NAME_CHARS]) + if isinstance(selectors, dict): + cleaned = { + str(kind)[:64]: _names(fields) + for kind, fields in selectors.items() + if isinstance(fields, (list, tuple)) and fields + } + _set(bucket, 'selectors', cleaned) + if plugins_all is not None: + bucket['plugins_all'] = bool(plugins_all) + if mcp_all is not None: + bucket['mcp_all'] = bool(mcp_all) + if tools_all is not None: + bucket['tools_all'] = bool(tools_all) + except Exception: + return + + +def note_agent( + *, + runner_id: str = '', + resources: typing.Any = None, + selectors: typing.Any = None, + capabilities: typing.Any = None, + tools_all: typing.Any = None, +) -> None: + """Record the resources one Agent run was granted. + + ``resources`` is the materialized ``AgentResources`` mapping the Host + actually handed to the Runner (tools carry their source), ``selectors`` + names the Runner config-form fields that selected a resource, and + ``capabilities`` records which resource categories the Runner declared. + """ + try: + bucket = _bucket('agent') + if bucket is None: + return + _set(bucket, 'runner_id', str(runner_id or '')[:MAX_NAME_CHARS]) + if isinstance(resources, dict): + _set(bucket, 'knowledge_bases', _names(resources.get('knowledge_bases'))) + _set(bucket, 'skills', _names(resources.get('skills'))) + _set(bucket, 'models', _names(resources.get('models'))) + _set(bucket, 'tools', _tool_refs(resources.get('tools'))) + _set(bucket, 'platform_apis', _names(resources.get('platform_tools'))) + if isinstance(selectors, dict): + cleaned = { + str(kind)[:64]: _names(fields) + for kind, fields in selectors.items() + if isinstance(fields, (list, tuple)) and fields + } + _set(bucket, 'selectors', cleaned) + if isinstance(capabilities, dict): + declared = {key: bool(value) for key, value in capabilities.items() if value} + _set(bucket, 'capabilities', declared) + if tools_all is not None: + bucket['tools_all'] = bool(tools_all) + except Exception: + return + + +def snapshot() -> dict[str, typing.Any] | None: + """Return the bounded snapshot of this execution; None when nothing was noted.""" + try: + root = _resources.get() + if not isinstance(root, dict) or not root: + return None + + def encoded_size(value: typing.Any) -> int: + return len(json.dumps(value, ensure_ascii=False, default=str).encode('utf-8')) + + if encoded_size(root) <= MAX_RESOURCES_BYTES: + return root + # Over budget: shed the widest detail (per-tool grants) first, then fall + # back to a marker, so an oversized binding can never block the chain + # record or bloat the upload. + reduced = json.loads(json.dumps(root, ensure_ascii=False, default=str)) + for section in ('pipeline', 'agent'): + bucket = reduced.get(section) + if isinstance(bucket, dict): + bucket.pop('tools', None) + reduced['truncated'] = True + if encoded_size(reduced) <= MAX_RESOURCES_BYTES: + return reduced + return {'truncated': True} + except Exception: + return None + + +@contextlib.contextmanager +def collected() -> typing.Iterator[None]: + """Bind a collection scope, for callers outside an execution boundary.""" + token = _resources.set({}) + try: + yield + finally: + _resources.reset(token) diff --git a/src/langbot/pkg/telemetry/telemetry.py b/src/langbot/pkg/telemetry/telemetry.py index c36b35e3c..9188a5c54 100644 --- a/src/langbot/pkg/telemetry/telemetry.py +++ b/src/langbot/pkg/telemetry/telemetry.py @@ -12,9 +12,6 @@ from ..utils import httpclient from .execution import ExecutionCounters -_MAX_INFLIGHT_TELEMETRY_TASKS = 8 - - class TelemetryManager: """TelemetryManager handles sending telemetry for a given application instance. @@ -27,36 +24,18 @@ class TelemetryManager: self.ap = ap self.telemetry_config: dict[str, typing.Any] = {} - self.send_tasks: list[asyncio.Task] = [] self._client: httpx.AsyncClient | None = None self.execution = ExecutionCounters(self) async def initialize(self): self.telemetry_config = self.ap.instance_config.data.get('space', {}) - async def start_send_task(self, payload: dict): - self.send_tasks = [task for task in self.send_tasks if not task.done()] - if len(self.send_tasks) >= _MAX_INFLIGHT_TELEMETRY_TASKS: - self.ap.logger.debug('Telemetry queue is full; dropping best-effort event') - return - task = asyncio.create_task(self.send(payload)) - self.send_tasks.append(task) - task.add_done_callback(self._send_task_done) - - def _send_task_done(self, task: asyncio.Task) -> None: - try: - self.send_tasks.remove(task) - except ValueError: - pass + def health(self) -> dict: + """Telemetry health for the instance liveness snapshot.""" + return self.execution.health() async def shutdown(self) -> None: await self.execution.shutdown() - tasks = list(self.send_tasks) - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - self.send_tasks.clear() if self._client is not None: await self._client.aclose() self._client = None @@ -95,45 +74,53 @@ class TelemetryManager: url = server.rstrip('/') + '/api/v1/telemetry' try: - # Sanitize payload so string fields are strings and not nulls + # Sanitize payload so string fields are strings and not nulls. + # A record batch is an envelope of whole execution records; it + # carries no top-level record fields, so only a single-record + # payload gets the legacy defaults. sanitized = dict(payload) - if 'query_id' in sanitized: - try: - sanitized['query_id'] = '' if sanitized['query_id'] is None else str(sanitized['query_id']) - except Exception: - sanitized['query_id'] = str(sanitized.get('query_id', '')) + if isinstance(sanitized.get('records'), list): + sanitized['records'] = [item for item in sanitized['records'] if isinstance(item, dict)] + else: + if 'query_id' in sanitized: + try: + sanitized['query_id'] = ( + '' if sanitized['query_id'] is None else str(sanitized['query_id']) + ) + except Exception: + sanitized['query_id'] = str(sanitized.get('query_id', '')) - for sfield in ( - 'adapter', - 'runner', - 'runner_category', - 'model_name', - 'version', - 'edition', - 'error', - 'timestamp', - 'event_type', - ): - if sfield not in sanitized: - continue - v = sanitized.get(sfield) - sanitized[sfield] = '' if v is None else str(v) + for sfield in ( + 'adapter', + 'runner', + 'runner_category', + 'model_name', + 'version', + 'edition', + 'error', + 'timestamp', + 'event_type', + ): + if sfield not in sanitized: + continue + v = sanitized.get(sfield) + sanitized[sfield] = '' if v is None else str(v) - # event_type defaults to 'query' for backward compatibility - if not sanitized.get('event_type'): - sanitized['event_type'] = 'query' + # event_type defaults to 'query' for backward compatibility + if not sanitized.get('event_type'): + sanitized['event_type'] = 'query' - # features must be a JSON object - if 'features' in sanitized and not isinstance(sanitized['features'], dict): - sanitized['features'] = {} + # features must be a JSON object + if 'features' in sanitized and not isinstance(sanitized['features'], dict): + sanitized['features'] = {} - if 'duration_ms' in sanitized: - try: - sanitized['duration_ms'] = ( - int(sanitized['duration_ms']) if sanitized['duration_ms'] is not None else 0 - ) - except Exception: - sanitized['duration_ms'] = 0 + if 'duration_ms' in sanitized: + try: + sanitized['duration_ms'] = ( + int(sanitized['duration_ms']) if sanitized['duration_ms'] is not None else 0 + ) + except Exception: + sanitized['duration_ms'] = 0 async with self._client_context() as client: try: diff --git a/src/langbot/pkg/telemetry/trace.py b/src/langbot/pkg/telemetry/trace.py index 445bb470e..fb4d18031 100644 --- a/src/langbot/pkg/telemetry/trace.py +++ b/src/langbot/pkg/telemetry/trace.py @@ -1,22 +1,22 @@ -"""Content-free per-execution traces. +"""Per-execution telemetry context for the v2 execution chain. -One trace identity is one execution identity: the id a Pipeline query, inbound -platform event or Agent run is known by, so a stored chain can be joined with -the Space view of that same execution. Stage records appended under it describe -how that one execution was routed and processed, using only code-defined -identifiers. Nothing here is aggregated: a trace is a bounded, ordered sequence -for one execution. +One execution identity (a Pipeline query, inbound platform event or Agent run) +is one chain. The chain is not assembled here: every workflow node is emitted +by ``telemetry.execution`` the moment it completes, carrying its own identity +and the identity of the execution it belongs to. This module only holds the +small execution-scoped context the emitter needs: -Three bounds keep memory independent of traffic volume: +* the execution state (identity, the once-per-event sampling decision, the + execution-scoped display fields); +* the routing/run identity pinned around the block that produced a node; +* the open-step stack, which lives in a ContextVar holding *one task's own + immutable tuple*. Child tasks inherit a snapshot at creation; no shared + mutable stack and no cross-task mirror exists, so sibling tasks can never + corrupt each other's parent links. -* ``MAX_STAGES`` stages per trace (further stages are counted, not kept); -* ``MAX_TRACES`` traces in flight, enforced by the sender that owns the buffer; -* a wall-clock TTL, enforced by the sender's sweep. - -The identity itself lives in a ContextVar so that asynchronous work spawned -while handling one execution inherits it, mirroring ``telemetry.platform``. -Cross-task stages (plugin/RPC work with no inherited context) are resolved by -the owner's execution-id registry instead. +Cross-task callers (plugin/RPC work with no inherited context) resolve their +execution through the owner's execution-id registry instead, which only +carries the execution state - never node payloads. """ from __future__ import annotations @@ -27,64 +27,79 @@ import typing from datetime import datetime, timezone from uuid import uuid4 -MAX_STAGES = 32 - _current: contextvars.ContextVar['TraceState | None'] = contextvars.ContextVar('telemetry_trace', default=None) _route: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_route', default='') _run: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_run', default='') -# Open workflow step nodes, innermost last. Records emitted inside a step carry -# the innermost node as their ``parent``; the step's own record carries the one -# below it (its enclosing step), never itself. -_node: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar('telemetry_trace_node', default=()) +# Open workflow step frames of the current task, innermost last. The tuple is +# never mutated in place: opening a step publishes a new tuple on this task's +# context only, and closing it restores the previous one. +_node: contextvars.ContextVar[tuple['NodeFrame', ...]] = contextvars.ContextVar('telemetry_trace_node', default=()) def _now() -> str: return datetime.now(timezone.utc).isoformat() +class NodeFrame(typing.NamedTuple): + """One workflow step open in the task that opened it.""" + + node_id: str + seq: int + started_at: str + + class TraceState: - """Bounded stage buffer for exactly one execution.""" + """The small per-execution context; node payloads never accumulate here.""" __slots__ = ( - 'trace_id', - 'execution_id', - 'started_at', - 'stages', - 'dropped_stages', - 'synthetic', - 'sequence', - 'node_sequence', - 'open_nodes', + 'event_id', 'identity', - 'abandoned', + 'configured', + 'mode', + 'denominator', + 'emit_nodes', + 'debug', + 'origin', + 'started_at', + 'node_seq', + 'root_seen', + 'stage_count', + 'dropped_nodes', + 'last_outcome', + 'first_error', 'failure_reason', + 'runner_failed', 'closed', 'adapter', 'runner', 'runner_category', 'model_name', 'pipeline_plugins', + 'node_adapter', + 'node_runner', ) def __init__(self, execution_id: str | None = None) -> None: - # The execution identity is the trace identity so a stored chain can be - # joined to the Space view of that same Pipeline query / event / run. - self.execution_id = str(execution_id).strip() if execution_id else '' - self.trace_id = self.execution_id or str(uuid4()) - self.started_at = _now() - self.stages: list[dict[str, typing.Any]] = [] - self.dropped_stages = 0 - self.synthetic = False - self.sequence = 0 - # Node ids number the workflow steps of this one execution ("n1", "n2"...). - self.node_sequence = 0 - # Mirror of the open step stack, readable from any task: cross-task - # observations (plugin/RPC) resolve their parent through this. - self.open_nodes: tuple[str, ...] = () + # The execution identity every record of this chain carries. + self.event_id = str(execution_id).strip() if execution_id else str(uuid4()) self.identity: dict[str, str] = {} - # Set when the sender refused to buffer this trace: stop appending. - self.abandoned = False + self.configured = False + # Sampling, decided exactly once per event (at ingress or first touch). + self.mode = 'all' + self.denominator = 1 + self.emit_nodes = False + self.debug = False + self.origin = 'platform' + self.started_at = _now() + self.node_seq = 0 + self.root_seen = False + self.stage_count = 0 + self.dropped_nodes = 0 + self.last_outcome = '' + self.first_error = '' self.failure_reason = '' + # Set when a runner node already reported a non-success outcome. + self.runner_failed = False self.closed = False # Execution-scoped record fields attached by the owning lane when known. self.adapter = '' @@ -92,54 +107,40 @@ class TraceState: self.runner_category = '' self.model_name = '' self.pipeline_plugins: typing.Any = None + # First adapter/runner seen on a node, used for the chain record. + self.node_adapter = '' + self.node_runner = '' - def append( + def allocate_seq(self) -> int: + """Mint the next chain-local node order.""" + seq = self.node_seq + self.node_seq += 1 + return seq + + def note_node( self, *, - family: str, - operation: str, - mode: str, - adapter: str, - runner: str, outcome: str, - synthetic: bool, + adapter: str = '', + runner: str = '', + family: str = '', error: str = '', - node: str = '', - parent: str = '', ) -> None: - if len(self.stages) >= MAX_STAGES: - self.dropped_stages += 1 - return - seen = _now() - entry = { - 'family': family, - 'operation': operation, - 'mode': mode, - 'adapter': adapter, - 'runner': runner, - 'outcome': outcome, - 'synthetic': bool(synthetic), - 'seq': self.sequence, - 'route_ref': _route.get(), - 'run_id': _run.get(), - 'first_seen': seen, - 'last_seen': seen, - 'error': error, - } - # Only workflow steps carry these; legacy records stay byte-identical. - if node: - entry['node'] = node - if parent: - entry['parent'] = parent - self.stages.append(entry) - self.sequence += 1 - if synthetic: - self.synthetic = True - - def allocate_node(self) -> str: - """Mint the next step-node id for this execution ("n1", "n2", ...).""" - self.node_sequence += 1 - return f'n{self.node_sequence}' + """Remember only what the terminal chain record needs from a node.""" + self.stage_count += 1 + self.last_outcome = outcome or 'unknown' + if adapter and not self.node_adapter: + self.node_adapter = adapter + if runner and not self.node_runner: + self.node_runner = runner + if family == 'runner' and outcome not in ('success', 'skipped'): + # The lane records its own failed runner step only when the runner + # orchestrator has not already landed one. + self.runner_failed = True + if outcome in ('failed', 'cancelled', 'timeout') and error and not self.first_error: + # The chain must explain itself even when the break was reported as + # a failed node rather than as an exception at the boundary. + self.first_error = error def mark_failure(self, reason: str) -> None: """Remember why this chain broke so the payload can explain itself.""" @@ -147,18 +148,10 @@ class TraceState: self.failure_reason = reason def outcome(self) -> str: - """Terminal outcome of this chain, for space-side display. - - A chain that carries a failure reason never reports ``success``: the - reason is the point of the trace. - """ - if self.stages: - last = self.stages[-1]['outcome'] - else: - last = 'unknown' - if self.failure_reason and last in ('success', 'skipped', 'unknown'): + """Terminal outcome of this chain; a broken chain never reports success.""" + if self.failure_reason and self.last_outcome in ('', 'success', 'skipped', 'unknown'): return 'failed' - return last + return self.last_outcome or 'unknown' class TraceBinding(typing.NamedTuple): @@ -168,9 +161,9 @@ class TraceBinding(typing.NamedTuple): def bind(execution_id: str | None = None) -> TraceBinding: - """Start a trace unless one is already in flight in this context. + """Start a chain unless one is already in flight in this context. - The execution id becomes the trace id only when this call starts the trace; + The execution id becomes the chain identity only when this call starts it; a nested boundary keeps the enclosing execution's identity. """ existing = _current.get() @@ -181,7 +174,7 @@ def bind(execution_id: str | None = None) -> TraceBinding: def unbind(binding: TraceBinding) -> bool: - """Detach this binding. Returns True when this caller owns the trace.""" + """Detach this binding. Returns True when this caller owns the chain.""" if binding.token is not None: try: _current.reset(binding.token) @@ -192,7 +185,7 @@ def unbind(binding: TraceBinding) -> bool: def unbind_root(binding: TraceBinding) -> bool: - """Detach only when this caller started the trace, else leave it in place.""" + """Detach only when this caller started the chain, else leave it in place.""" if not binding.created: return False return unbind(binding) @@ -202,39 +195,64 @@ def current() -> TraceState | None: return _current.get() +def node_stack() -> tuple[NodeFrame, ...]: + """This task's own open-step frames, innermost last.""" + return _node.get() + + def current_parent() -> str: """The step node that records emitted right now attach to ('' when none).""" stack = _node.get() - return stack[-1] if stack else '' + return stack[-1].node_id if stack else '' def enclosing_parent() -> str: """The step enclosing the innermost open one; '' when it is a root step.""" stack = _node.get() - return stack[-2] if len(stack) > 1 else '' + return stack[-2].node_id if len(stack) > 1 else '' + + +def resolve_node( + state: TraceState, + node: str, + fallback_started_at: str, +) -> tuple[str, int, str, str]: + """Resolve one record to ``(node_id, seq, started_at, parent_node_id)``. + + A record stamped with a ``node`` the calling task has open reuses that + step's identity, order and start time; anything else (a plain observation, + or a node opened by another task) is a fresh node under this task's + innermost open step. + """ + stack = _node.get() + if node: + for index in range(len(stack) - 1, -1, -1): + frame = stack[index] + if frame.node_id == node: + parent = stack[index - 1].node_id if index > 0 else '' + return frame.node_id, frame.seq, frame.started_at, parent + parent = stack[-1].node_id if stack else '' + return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent @contextlib.contextmanager def stage_scope() -> typing.Iterator[str]: """Open one workflow step node for the duration of the block. - Allocates a node id unique within the execution, publishes it as the current - parent so that nested observations recorded inside the block attach to it, - and yields that id so the step's own records can be stamped with ``node=``. - Outside any trace nothing is allocated and the empty id is yielded. + Allocates a globally unique node id and the chain-local order, publishes the + frame on this task's own stack so nested observations attach to it as their + parent, and yields the node id so the step's own record can be stamped with + ``node=``. Outside any execution nothing is allocated and '' is yielded. """ state = _current.get() if state is None: yield '' return - node = state.allocate_node() - previous_open = state.open_nodes - state.open_nodes = previous_open + (node,) - token = _node.set(_node.get() + (node,)) + frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now()) + token = _node.set(_node.get() + (frame,)) try: - yield node + yield frame.node_id finally: - state.open_nodes = previous_open try: _node.reset(token) except (ValueError, RuntimeError): @@ -258,7 +276,7 @@ def reset_run(token: typing.Any) -> None: @contextlib.contextmanager def scope(*, route_ref: str | None = None) -> typing.Iterator[None]: - """Pin routing identity for stages recorded inside this block.""" + """Pin routing identity for nodes recorded inside this block.""" if not route_ref: yield return @@ -270,3 +288,11 @@ def scope(*, route_ref: str | None = None) -> typing.Iterator[None]: _route.reset(token) except (ValueError, RuntimeError): pass + + +def current_route() -> str: + return _route.get() + + +def current_run() -> str: + return _run.get() diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 9e5a0f7ac..f8752314f 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -444,11 +444,13 @@ space: # heartbeat and execution reporting; restart after changing this setting. disable_telemetry: false # Per-event execution traces: off | failures | sampled | all. - # 'sampled' keeps every failed run plus every N-th successful one - # (N is execution_trace_sample) and always keeps WebUI debug runs. - execution_trace: sampled + # Every execution is reported by default: one record per workflow node plus + # one terminal chain record. 'failures' keeps only broken chains, 'sampled' + # keeps broken chains plus every N-th successful one (N is + # execution_trace_sample), and 'off' reports no executions at all. + execution_trace: all execution_trace_sample: 20 # How often buffered execution records are uploaded, in seconds. - # Records of complete executions are batched as {"records": [...]}; a - # still-running execution is never uploaded half-written. + # Complete execution records are batched as {"records": [...]}; records are + # removed from the buffer only once the sender accepted them. telemetry_flush_seconds: 180 diff --git a/tests/unit_tests/platform/test_botmgr_execution_trace.py b/tests/unit_tests/platform/test_botmgr_execution_trace.py index 75605aa04..8fbad1463 100644 --- a/tests/unit_tests/platform/test_botmgr_execution_trace.py +++ b/tests/unit_tests/platform/test_botmgr_execution_trace.py @@ -1,4 +1,4 @@ -"""RuntimeBot ingress tracing: one inbound event owns one execution trace.""" +"""RuntimeBot ingress tracing: one inbound event owns one v2 execution chain.""" from __future__ import annotations @@ -28,9 +28,6 @@ class FakeTelemetryManager: self.sent.append(payload) return True - def start_send_task(self, payload: dict) -> None: - self.sent.append(payload) - def make_bot(event_bindings: list[dict], config=None, agent=None): from langbot.pkg.platform.botmgr import RuntimeBot @@ -76,7 +73,7 @@ class FakeAdapter: async def flushed_records(counters, manager) -> list[dict]: - """Flush buffered telemetry and return everything the pipeline emitted. + """Flush buffered telemetry and return everything the event emitted. Records are batched and uploaded by ``ExecutionCounters.flush()``; there is no per-trace direct send any more. @@ -87,34 +84,48 @@ async def flushed_records(counters, manager) -> list[dict]: return [record for payload in manager.sent for record in payload.get('records', [])] +def execution_nodes(records: list[dict]) -> list[dict]: + return [record for record in records if record.get('event_type') == 'execution_node'] + + +def execution_chains(records: list[dict]) -> list[dict]: + return [record for record in records if record.get('event_type') == 'execution_chain'] + + @pytest.mark.asyncio -async def test_route_miss_emits_one_closed_trace(): +async def test_route_miss_emits_one_closed_chain(): bot, manager, counters = make_bot([]) await bot._handle_platform_event(message_received_event(), FakeAdapter()) records = await flushed_records(counters, manager) assert len(manager.sent) == 1 - trace_records = [item for item in records if item.get('event_type') == 'feature_execution'] - assert len(trace_records) == 1 - payload = trace_records[0] - assert payload['event_type'] == 'feature_execution' + chains = execution_chains(records) + assert len(chains) == 1 + chain = chains[0] + assert chain['schema'] == 2 # The execution identity is deterministic: one inbound event, one chain. - assert payload['query_id'] == 'platform:bot-1:message-1' - assert payload['instance_id'] == 'instance-test' - assert payload['workspace_uuid'] == 'workspace-test' - features = payload['features'] - assert features['schema'] == 1 - assert [(row['family'], row['operation'], row['outcome']) for row in features['observations']] == [ + assert chain['event_id'] == 'platform:bot-1:message-1' + assert chain['instance_id'] == 'instance-test' + assert chain['workspace_uuid'] == 'workspace-test' + assert chain['closed_by'] == 'event_done' + assert chain['stage_count'] == 2 + + rows = execution_nodes(records) + assert [(row['family'], row['operation'], row['outcome']) for row in rows] == [ ('event_route', 'message.received', 'skipped'), ('platform_event', 'message.received', 'success'), ] - assert [row['seq'] for row in features['observations']] == [0, 1] - assert all(row['trace_id'] == payload['query_id'] for row in features['observations']) - assert all(row['adapter'] == 'FakeAdapter' for row in features['observations']) - assert features['trace']['closed_by'] == 'event_done' - # The trace is closed and deregistered: nothing dangles after the event. - assert counters.traces == {} + assert {row['seq'] for row in rows} == {0, 1} + assert all(row['event_id'] == chain['event_id'] for row in rows) + assert all(row['adapter'] == 'FakeAdapter' for row in rows) + # The inbound event is the root step; the routing decision nests under it. + root = next(row for row in rows if row['root']) + route = next(row for row in rows if row['family'] == 'event_route') + assert (root['family'], root['parent_node_id']) == ('platform_event', '') + assert route['parent_node_id'] == root['node_id'] + # The chain is closed and deregistered: nothing dangles after the event. + assert counters.registry == {} @pytest.mark.asyncio @@ -133,13 +144,13 @@ async def test_unavailable_route_target_carries_route_identity(): await bot._handle_platform_event(message_received_event(), FakeAdapter()) - stages = (await flushed_records(counters, manager))[0]['features']['observations'] - assert [(stage['family'], stage['outcome'], stage['mode']) for stage in stages] == [ + nodes = execution_nodes(await flushed_records(counters, manager)) + assert [(node['family'], node['outcome'], node['mode']) for node in nodes] == [ ('event_route', 'failed', 'agent'), ('platform_event', 'success', 'none'), ] - assert stages[0]['route_ref'] == 'agent:agent-1' - assert stages[1]['route_ref'] == '' + assert nodes[0]['route_ref'] == 'agent:agent-1' + assert nodes[1]['route_ref'] == '' @pytest.mark.asyncio @@ -157,10 +168,10 @@ async def test_discarded_route_is_visible_without_user_values(): await bot._handle_platform_event(SimpleNamespace(type='platform.member.joined'), FakeAdapter()) - stages = (await flushed_records(counters, manager))[0]['features']['observations'] - assert [stage['outcome'] for stage in stages] == ['skipped', 'success'] - assert stages[0]['operation'] == 'platform.member.joined' - assert stages[0]['mode'] == 'none' + nodes = execution_nodes(await flushed_records(counters, manager)) + assert [node['outcome'] for node in nodes] == ['skipped', 'success'] + assert nodes[0]['operation'] == 'platform.member.joined' + assert nodes[0]['mode'] == 'none' @pytest.mark.asyncio @@ -171,7 +182,7 @@ async def test_telemetry_opt_out_emits_nothing(): assert manager.sent == [] assert counters.records == [] - assert counters.traces == {} + assert counters.registry == {} @pytest.mark.asyncio @@ -185,13 +196,13 @@ async def test_route_trace_scope_is_restored_after_dispatch(): assert trace_mod.current() is None buffered = list(counters.records) - # A later record outside the ingress must not join the closed trace, and an + # A later record outside the ingress must not join the closed chain, and an # observation without an owning chain is not buffered at all. from langbot.pkg.telemetry.execution import record record(bot.ap, TEST_CONTEXT, family='platform_api', operation='send_message', outcome='success') assert counters.records == buffered - assert counters.traces == {} + assert counters.registry == {} @pytest.mark.asyncio diff --git a/tests/unit_tests/provider/test_mcp_resources.py b/tests/unit_tests/provider/test_mcp_resources.py index 0ba164e9a..97b136d56 100644 --- a/tests/unit_tests/provider/test_mcp_resources.py +++ b/tests/unit_tests/provider/test_mcp_resources.py @@ -24,7 +24,6 @@ from langbot.pkg.provider.tools.loaders.mcp import ( MCPToolCallTimeoutError, RuntimeMCPSession, ) -from langbot.pkg.telemetry import features as telemetry_features from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError @@ -298,10 +297,6 @@ async def test_read_resource_envelope_truncates_caches_and_records_trace(): traces = query.variables[MCP_RESOURCE_TRACE_QUERY_KEY] assert [trace['source'] for trace in traces] == ['ui_preview', 'agent_tool'] assert traces[1]['cache_hit'] is True - assert query.variables[telemetry_features.FEATURES_KEY]['mcp_resource_reads'] == { - 'ui_preview': 1, - 'agent_tool': 1, - } @pytest.mark.asyncio diff --git a/tests/unit_tests/telemetry/test_cloud_sampling.py b/tests/unit_tests/telemetry/test_cloud_sampling.py index b1a604489..ef4de5e2e 100644 --- a/tests/unit_tests/telemetry/test_cloud_sampling.py +++ b/tests/unit_tests/telemetry/test_cloud_sampling.py @@ -1,6 +1,7 @@ from types import SimpleNamespace from langbot.pkg.telemetry.execution import ExecutionCounters +from langbot.pkg.telemetry.trace import TraceState from langbot.pkg.utils import constants @@ -8,8 +9,10 @@ def test_cloud_counts_are_not_sampled(monkeypatch): monkeypatch.setattr(constants, 'edition', 'cloud') counters = ExecutionCounters(SimpleNamespace(telemetry_config={'execution_trace': 'sampled'})) assert counters.trace_mode() == 'all' - state = SimpleNamespace(failure_reason='', stages=[], synthetic=False, trace_id='00000001') - assert counters._trace_emitted(state) + state = TraceState('00000001') + counters.configure(state) + assert state.emit_nodes is True + assert counters._chain_emitted(state) is True def test_community_retains_sampling(monkeypatch): diff --git a/tests/unit_tests/telemetry/test_features.py b/tests/unit_tests/telemetry/test_features.py deleted file mode 100644 index 6e19d0828..000000000 --- a/tests/unit_tests/telemetry/test_features.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Unit tests for telemetry feature counters (pkg/telemetry/features.py).""" - -from __future__ import annotations - -from importlib import import_module - - -def get_features_module(): - return import_module('langbot.pkg.telemetry.features') - - -class FakeQuery: - def __init__(self): - self.variables = {} - - -class TestIncrement: - def test_increment_nested_counter(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'tool_calls', 'native') - features.increment(q, 'tool_calls', 'native') - features.increment(q, 'tool_calls', 'mcp') - assert q.variables[features.FEATURES_KEY]['tool_calls'] == {'native': 2, 'mcp': 1} - - def test_increment_flat_counter(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'something') - features.increment(q, 'something', amount=2) - assert q.variables[features.FEATURES_KEY]['something'] == 3 - - def test_increment_never_raises_on_broken_query(self): - features = get_features_module() - - class Broken: - @property - def variables(self): - raise RuntimeError('boom') - - # Must not raise - features.increment(Broken(), 'tool_calls', 'native') - - def test_set_value(self): - features = get_features_module() - q = FakeQuery() - features.set_value(q, 'tool_call_rounds', 5) - assert q.variables[features.FEATURES_KEY]['tool_call_rounds'] == 5 - - -class TestCollectFeatures: - def test_collect_empty(self): - features = get_features_module() - q = FakeQuery() - assert features.collect_features(q) == {} - - def test_collect_combines_counters_and_snapshots(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'sandbox', 'execs') - features.set_value(q, 'kb', {'kb_count': 2, 'engine_plugins': ['builtin'], 'retrieved_entries': 7}) - q.variables['_activated_skills'] = {'pdf-tools': {}, 'a-skill': {}} - q.variables['_pipeline_bound_mcp_servers'] = ['srv1', 'srv2'] - - result = features.collect_features(q) - assert result['sandbox'] == {'execs': 1} - assert result['kb']['kb_count'] == 2 - assert result['activated_skills'] == ['a-skill', 'pdf-tools'] # sorted - assert result['mcp_servers'] == ['srv1', 'srv2'] - - def test_collect_omits_mcp_when_all_enabled(self): - """None means 'all enabled' and is not reported.""" - features = get_features_module() - q = FakeQuery() - q.variables['_pipeline_bound_mcp_servers'] = None - assert 'mcp_servers' not in features.collect_features(q) - - def test_collect_drops_non_json_serializable(self): - features = get_features_module() - q = FakeQuery() - features.set_value(q, 'good', 1) - features.set_value(q, 'bad', object()) - result = features.collect_features(q) - assert result == {'good': 1} - - def test_collect_is_json_serializable(self): - import json - - features = get_features_module() - q = FakeQuery() - features.increment(q, 'tool_calls', 'skill') - json.dumps(features.collect_features(q)) diff --git a/tests/unit_tests/telemetry/test_pipeline_lane.py b/tests/unit_tests/telemetry/test_pipeline_lane.py index f091f2bb8..4b3e8625e 100644 --- a/tests/unit_tests/telemetry/test_pipeline_lane.py +++ b/tests/unit_tests/telemetry/test_pipeline_lane.py @@ -1,4 +1,4 @@ -"""The Pipeline lane must land its failure on the owning execution trace.""" +"""The Pipeline lane must land its nodes on one v2 execution chain.""" from __future__ import annotations @@ -43,7 +43,7 @@ def build_pipeline(pipelinemgr, ap): return pipeline -def test_lane_failure_marks_the_owned_trace_without_any_prior_stage(): +def test_lane_failure_marks_the_owned_chain_without_any_prior_stage(): pipelinemgr, execution = get_modules() manager = FakeManager() counters = execution.ExecutionCounters(manager) @@ -54,15 +54,16 @@ def test_lane_failure_marks_the_owned_trace_without_any_prior_stage(): with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-7'): pipeline._record_lane_failure(query, 'sandbox exited 127') - assert len(counters.records) == 1 - record = counters.records[0] - assert record['query_id'] == 'query-7' - assert record['error'] == 'sandbox exited 127' - assert record['features']['trace']['outcome'] == 'failed' - observations = record['features']['observations'] - assert [(row['family'], row['operation'], row['outcome']) for row in observations] == [ - ('runner', 'execute', 'failed') - ] + rows = [record for record in counters.records if record['event_type'] == 'execution_node'] + assert len(rows) == 1 + node = rows[0] + assert node['event_id'] == 'query-7' + assert node['error'] == 'sandbox exited 127' + assert (node['family'], node['operation'], node['outcome']) == ('runner', 'execute', 'failed') + chain = [record for record in counters.records if record['event_type'] == 'execution_chain'][0] + assert chain['event_id'] == 'query-7' + assert chain['outcome'] == 'failed' + assert chain['error'] == 'sandbox exited 127' def test_lane_failure_does_not_duplicate_the_runner_node_the_orchestrator_recorded(): @@ -87,13 +88,13 @@ def test_lane_failure_does_not_duplicate_the_runner_node_the_orchestrator_record ) pipeline._record_lane_failure(query, 'No authorized model for local-agent') - observations = [ - (row['family'], row['operation'], row['outcome']) - for record in counters.records - for row in record['features']['observations'] + rows = [record for record in counters.records if record['event_type'] == 'execution_node'] + assert [(record['family'], record['operation'], record['outcome']) for record in rows] == [ + ('runner', 'execute', 'failed') ] - assert observations == [('runner', 'execute', 'failed')] - assert counters.records[-1]['features']['trace']['outcome'] == 'failed' + chain = [record for record in counters.records if record['event_type'] == 'execution_chain'][-1] + assert chain['outcome'] == 'failed' + assert chain['error'] == 'No authorized model for local-agent' def build_counters(): @@ -118,11 +119,14 @@ def build_query(**overrides): return query +def lane_nodes(counters): + return [record for record in counters.records if record['event_type'] == 'execution_node'] + + def lane_shapes(counters): return [ - (row['family'], row['operation'], row.get('node'), row.get('parent')) - for record in counters.records - for row in record['features']['observations'] + (record['family'], record['operation'], record['node_id'], record['parent_node_id']) + for record in lane_nodes(counters) ] @@ -138,7 +142,7 @@ async def stop_counters(counters): counters.task = None -def test_lane_step_owns_the_apis_and_runner_it_calls(): +async def test_lane_step_owns_the_apis_and_runner_it_calls(): pipelinemgr, execution = get_modules() trace = get_trace() counters, ap = build_counters() @@ -168,21 +172,28 @@ def test_lane_step_owns_the_apis_and_runner_it_calls(): await lane.process_query(query) await stop_counters(counters) - asyncio.run(scenario()) + await scenario() shapes = lane_shapes(counters) - lane_node = shapes[-1][2] - assert lane_node == 'n1' - assert shapes == [ - ('platform_api', 'send_message', None, lane_node), - ('runner', 'execute', seen['run'], lane_node), - ('platform_api', 'reply_message', None, lane_node), - # The lane step is recorded last; a standalone lane is a root. - ('pipeline', 'run', lane_node, None), + # The synthetic inbound event roots the chain; the lane step hangs under it. + assert [shape[0:2] for shape in shapes] == [ + ('platform_event', 'message.received'), + ('platform_api', 'send_message'), + ('runner', 'execute'), + ('platform_api', 'reply_message'), + ('pipeline', 'run'), ] + event_node = shapes[0][2] + lane_node = shapes[-1][2] + assert shapes[0][3] == '' + assert shapes[1][3] == lane_node + assert shapes[2] == ('runner', 'execute', seen['run'], lane_node) + assert shapes[3][3] == lane_node + # The lane step is recorded last and nests under the synthetic event node. + assert shapes[4] == ('pipeline', 'run', lane_node, event_node) -def test_lane_node_nests_under_the_platform_route_that_called_it(): +async def test_lane_node_nests_under_the_platform_route_that_called_it(): pipelinemgr, execution = get_modules() trace = get_trace() counters, ap = build_counters() @@ -197,7 +208,7 @@ def test_lane_node_nests_under_the_platform_route_that_called_it(): lane._process_query = fake_process async def scenario(): - # A platform event owns the trace; the route step wraps the Pipeline lane. + # A platform event owns the chain; the route step wraps the Pipeline lane. with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-routed'): with trace.stage_scope() as event_node: execution.record( @@ -213,19 +224,21 @@ def test_lane_node_nests_under_the_platform_route_that_called_it(): await stop_counters(counters) return event_node, route_node - event_node, route_node = asyncio.run(scenario()) + event_node, route_node = await scenario() - assert (event_node, route_node) == ('n1', 'n2') - assert lane_shapes(counters) == [ - ('platform_event', 'message.received', event_node, None), - ('event_route', 'message.received', route_node, event_node), - # The API is called by the lane, so it hangs on the lane node. - ('platform_api', 'send_message', None, 'n3'), - ('pipeline', 'run', 'n3', route_node), - ] + assert event_node != route_node + shapes = lane_shapes(counters) + assert shapes[0] == ('platform_event', 'message.received', event_node, '') + assert shapes[1] == ('event_route', 'message.received', route_node, event_node) + lane_node = shapes[-1][2] + # The API is called by the lane, so it hangs on the lane node. + assert shapes[2][0:2] == ('platform_api', 'send_message') + assert shapes[2][3] == lane_node and shapes[2][2] != lane_node + # The lane nests under the route step that called it. + assert shapes[3] == ('pipeline', 'run', lane_node, route_node) -def test_broken_lane_records_a_failed_step_with_its_reason(): +async def test_broken_lane_records_a_failed_step_with_its_reason(): pipelinemgr, execution = get_modules() counters, ap = build_counters() lane = build_lane(pipelinemgr, ap) @@ -242,16 +255,19 @@ def test_broken_lane_records_a_failed_step_with_its_reason(): await stop_counters(counters) with pytest.raises(RuntimeError, match='lane exploded'): - asyncio.run(scenario()) + await scenario() - family, operation, node, parent = lane_shapes(counters)[-1] - row = counters.records[0]['features']['observations'][-1] - assert (family, operation, node, parent) == ('pipeline', 'run', 'n1', None) + shapes = lane_shapes(counters) + family, operation, node, parent = shapes[-1] + row = lane_nodes(counters)[-1] + # The lane step nests under the synthetic inbound event that roots the chain. + assert (family, operation) == ('pipeline', 'run') + assert shapes[0] == ('platform_event', 'message.received', parent, '') assert row['outcome'] == 'failed' assert row['error'] == 'lane exploded' -def test_lane_reported_failure_marks_the_step_failed(): +async def test_lane_reported_failure_marks_the_step_failed(): pipelinemgr, execution = get_modules() counters, ap = build_counters() lane = build_lane(pipelinemgr, ap) @@ -266,7 +282,7 @@ def test_lane_reported_failure_marks_the_step_failed(): await lane.process_query(query) await stop_counters(counters) - asyncio.run(scenario()) + await scenario() - row = counters.records[0]['features']['observations'][-1] + row = lane_nodes(counters)[-1] assert (row['family'], row['operation'], row['outcome']) == ('pipeline', 'run', 'failed') diff --git a/tests/unit_tests/telemetry/test_telemetry.py b/tests/unit_tests/telemetry/test_telemetry.py index 004009a68..87699a18d 100644 --- a/tests/unit_tests/telemetry/test_telemetry.py +++ b/tests/unit_tests/telemetry/test_telemetry.py @@ -6,7 +6,6 @@ Tests cover: - Early return conditions (disabled, empty config, no server) - URL construction (with actual URL verification) - HTTP request success/failure scenarios -- Source code bug: send_tasks should be instance variable """ from __future__ import annotations @@ -629,53 +628,3 @@ class TestAuthenticatedWorkspaceReporter: 'Authorization': 'Bearer refreshed-workspace-owner-token' } - -class TestStartSendTask: - """Tests for start_send_task() method.""" - - @pytest.mark.asyncio - async def test_start_send_task_creates_task(self): - """Test that start_send_task creates an asyncio task.""" - telemetry = get_telemetry_module() - mock_app = Mock() - mock_app.logger = Mock() - mock_app.instance_config = Mock() - mock_app.instance_config.data = {} - - manager = telemetry.TelemetryManager(mock_app) - manager.telemetry_config = {} - - await manager.start_send_task({'query_id': 'test'}) - - # Task should be added to send_tasks list - assert len(manager.send_tasks) >= 1 - - # Clean up the task - for task in manager.send_tasks: - if not task.done(): - task.cancel() - manager.send_tasks.clear() - - @pytest.mark.asyncio - async def test_start_send_task_multiple_tasks(self): - """Test that multiple tasks are tracked.""" - telemetry = get_telemetry_module() - mock_app = Mock() - mock_app.logger = Mock() - mock_app.instance_config = Mock() - mock_app.instance_config.data = {} - - manager = telemetry.TelemetryManager(mock_app) - manager.telemetry_config = {} - - await manager.start_send_task({'query_id': 'test1'}) - await manager.start_send_task({'query_id': 'test2'}) - await manager.start_send_task({'query_id': 'test3'}) - - assert len(manager.send_tasks) >= 3 - - # Clean up - for task in manager.send_tasks: - if not task.done(): - task.cancel() - manager.send_tasks.clear() diff --git a/tests/unit_tests/telemetry/test_trace.py b/tests/unit_tests/telemetry/test_trace.py index 283cdac65..95b9f06f1 100644 --- a/tests/unit_tests/telemetry/test_trace.py +++ b/tests/unit_tests/telemetry/test_trace.py @@ -1,4 +1,10 @@ -"""Unit tests for per-execution execution traces (pkg/telemetry/trace.py + execution.py).""" +"""Unit tests for per-execution execution chains (pkg/telemetry/trace.py + execution.py). + +One execution is one chain of independent records: one ``execution_node`` per +workflow node, emitted the moment the node completes, plus exactly one +``execution_chain`` when the execution closes. Every record of a chain carries +the same ``event_id``. +""" from __future__ import annotations @@ -31,8 +37,21 @@ class FakeManager: self.sent.append(payload) return True - def start_send_task(self, payload: dict) -> None: + +class RejectingManager(FakeManager): + """Stand-in whose sender refuses every batch until told to accept them.""" + + def __init__(self, config=None): + super().__init__(config) + self.rejecting = True + self.attempts = 0 + + async def send(self, payload: dict) -> bool: + self.attempts += 1 + if self.rejecting: + return False self.sent.append(payload) + return True def make_counters(config=None): @@ -56,6 +75,14 @@ def records_sent(manager): return [record for payload in manager.sent for record in payload.get('records', [])] +def nodes(counters): + return [record for record in counters.records if record.get('event_type') == 'execution_node'] + + +def chains(counters): + return [record for record in counters.records if record.get('event_type') == 'execution_chain'] + + CONTEXT = types.SimpleNamespace(instance_uuid='instance-1', workspace_uuid='workspace-1') STAGE = { @@ -66,6 +93,35 @@ STAGE = { 'outcome': 'success', } +NODE_KEYS = { + 'event_type', + 'schema', + 'event_id', + 'instance_id', + 'workspace_uuid', + 'version', + 'edition', + 'timestamp', + 'debug', + 'sample', + 'node_id', + 'parent_node_id', + 'root', + 'seq', + 'family', + 'operation', + 'mode', + 'adapter', + 'runner', + 'outcome', + 'error', + 'route_ref', + 'run_id', + 'synthetic', + 'started_at', + 'ended_at', +} + class TestTraceIdentity: def test_nested_bind_reuses_the_in_flight_trace(self): @@ -93,13 +149,12 @@ class TestTraceIdentity: # A repeated unbind of the same binding must never raise. assert trace.unbind_root(outer) is True - def test_trace_id_is_the_execution_id(self): + def test_event_id_is_the_execution_id(self): trace, _ = get_modules() binding = trace.bind('exec-1234') try: assert binding.created is True - assert binding.state.trace_id == 'exec-1234' - assert binding.state.execution_id == 'exec-1234' + assert binding.state.event_id == 'exec-1234' finally: trace.unbind_root(binding) @@ -109,19 +164,20 @@ class TestTraceIdentity: inner = trace.bind('query-9') try: assert inner.created is False - assert inner.state.trace_id == 'event-1' + assert inner.state.event_id == 'event-1' finally: trace.unbind_root(inner) trace.unbind_root(outer) def test_route_and_run_are_only_pinned_inside_the_block(self): trace, _ = get_modules() + _, counters = make_counters(trace_config()) binding = trace.bind() token = trace.set_run('run-1') try: with trace.scope(route_ref='agent:agent-1'): - inside = trace.current() - inside.append( + counters.record( + CONTEXT, family='runner', operation='execute', mode='agent', @@ -130,8 +186,8 @@ class TestTraceIdentity: outcome='success', synthetic=False, ) - outside = trace.current() - outside.append( + counters.record( + CONTEXT, family='platform_api', operation='send_message', mode='agent', @@ -143,7 +199,7 @@ class TestTraceIdentity: finally: trace.reset_run(token) trace.unbind_root(binding) - assert [(s['route_ref'], s['run_id']) for s in binding.state.stages] == [ + assert [(row['route_ref'], row['run_id']) for row in nodes(counters)] == [ ('agent:agent-1', 'run-1'), ('', 'run-1'), ] @@ -152,65 +208,42 @@ class TestTraceIdentity: class TestTraceRecording: def test_stage_carries_sequence_identity_and_timestamps(self): trace, _ = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = trace.bind() try: counters.record(CONTEXT, **STAGE) counters.record(CONTEXT, **{**STAGE, 'family': 'runner', 'operation': 'execute', 'mode': 'pipeline'}) + counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - stages = binding.state.stages - assert [stage['seq'] for stage in stages] == [0, 1] - assert stages[0]['first_seen'] == stages[0]['last_seen'] - assert binding.state.identity == {'instance_id': 'instance-1', 'workspace_uuid': 'workspace-1'} - assert binding.state.outcome() == 'success' - - def test_stage_buffer_is_bounded_and_counts_overflow(self): - trace, _ = get_modules() - _, counters = make_counters() - binding = trace.bind() - try: - for _ in range(trace.MAX_STAGES + 5): - counters.record(CONTEXT, **STAGE) - finally: - trace.unbind_root(binding) - assert len(binding.state.stages) == trace.MAX_STAGES - assert binding.state.dropped_stages == 5 + rows = nodes(counters) + assert [row['seq'] for row in rows] == [0, 1] + # A plain observation starts and ends at the same instant. + assert rows[0]['started_at'] == rows[0]['ended_at'] + assert {(row['instance_id'], row['workspace_uuid']) for row in rows} == {('instance-1', 'workspace-1')} + chain = chains(counters)[0] + assert chain['outcome'] == 'success' + assert chain['stage_count'] == 2 def test_oversized_identifier_never_becomes_a_stage(self): trace, _ = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = trace.bind() try: counters.record(CONTEXT, **{**STAGE, 'operation': 'x' * 200}) counters.record(CONTEXT, **{**STAGE, 'family': 'not-a-family'}) finally: trace.unbind_root(binding) - assert binding.state.stages == [] - - def test_trace_limit_abandons_further_traces(self): - trace, execution = get_modules() - _, counters = make_counters() - bindings = [] - for _ in range(execution.MAX_TRACES + 1): - binding = trace.bind() - bindings.append(binding) - counters.record(CONTEXT, **STAGE) - trace.unbind_root(binding) - assert trace.current() is None - assert len(counters.traces) == execution.MAX_TRACES - assert counters.dropped == 1 - assert bindings[-1].state.abandoned is True - assert bindings[-1].state.stages == [] + assert nodes(counters) == [] def test_observation_without_an_owning_execution_is_dropped(self): - trace, _ = get_modules() + _, _ = get_modules() _, counters = make_counters() - # Clustering is gone: an observation that belongs to no execution is - # discarded instead of being aggregated into a synthetic bucket. + # An observation that belongs to no execution is discarded instead of + # being aggregated into a synthetic bucket. counters.record(CONTEXT, **STAGE) assert counters.records == [] - assert counters.traces == {} + assert counters.registry == {} class TestTraceNodeScope: @@ -238,11 +271,13 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - step_row, api_row = binding.state.stages - assert step_row['node'] == step - assert 'parent' not in step_row # a root step carries no parent - assert api_row['parent'] == step - assert 'node' not in api_row # observations are not step nodes + step_row, api_row = nodes(counters) + assert step_row['node_id'] == step + assert step_row['parent_node_id'] == '' # a root step carries no parent + assert step_row['root'] is True + assert api_row['parent_node_id'] == step + assert api_row['root'] is False + assert api_row['node_id'] != step # observations are not step nodes def test_nested_step_records_its_enclosing_step_as_parent(self): trace, _ = get_modules() @@ -257,11 +292,11 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - outer_row, inner_row, api_row = binding.state.stages - assert (outer_row['node'], outer_row.get('parent')) == (outer, None) + outer_row, inner_row, api_row = nodes(counters) + assert (outer_row['node_id'], outer_row['parent_node_id']) == (outer, '') # The step's own record sits under its enclosing step, never under itself. - assert (inner_row['node'], inner_row.get('parent')) == (inner, outer) - assert api_row.get('parent') == inner and 'node' not in api_row + assert (inner_row['node_id'], inner_row['parent_node_id']) == (inner, outer) + assert api_row['parent_node_id'] == inner and api_row['node_id'] != inner def test_sequential_steps_get_distinct_ids_and_roots_have_no_parent(self): trace, _ = get_modules() @@ -275,13 +310,16 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - assert (first, second) == ('n1', 'n2') - assert [row['node'] for row in binding.state.stages] == [first, second] - assert all('parent' not in row for row in binding.state.stages) + assert first and second and first != second + rows = nodes(counters) + assert [row['node_id'] for row in rows] == [first, second] + assert [row['seq'] for row in rows] == [0, 1] + assert rows[0]['root'] is True + assert all(row['parent_node_id'] == '' for row in rows) - def test_cross_task_observation_attaches_to_the_open_step(self): + def test_cross_task_observation_joins_the_owning_execution(self): trace, execution = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = execution.bind_trace(make_ap(counters), 'exec-cross') try: with trace.stage_scope() as lane: @@ -290,7 +328,7 @@ class TestTraceNodeScope: ) # A plugin/RPC observation arrives on a task that never had the # trace context; it is resolved through the execution registry and - # must still hang off the step the owning task has open. + # still lands on the owning chain. foreign = contextvars.Context() foreign.run(execution.set_execution_id, 'exec-cross') foreign.run( @@ -304,13 +342,16 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - rows = binding.state.stages - assert rows[0]['node'] == lane and 'parent' not in rows[0] - assert rows[1].get('parent') == lane and 'node' not in rows[1] + lane_row, foreign_row = nodes(counters) + assert lane_row['node_id'] == lane and lane_row['parent_node_id'] == '' and lane_row['root'] is True + # The cross-task node joins the same execution, but never mirrors another + # task's open-step stack: it is a root of the chain. + assert foreign_row['event_id'] == 'exec-cross' + assert foreign_row['parent_node_id'] == '' def test_cross_task_observation_without_an_open_step_has_no_parent(self): trace, execution = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = execution.bind_trace(make_ap(counters), 'exec-bare') try: foreign = contextvars.Context() @@ -326,8 +367,8 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - row = binding.state.stages[0] - assert 'node' not in row and 'parent' not in row + row = nodes(counters)[0] + assert row['parent_node_id'] == '' and row['root'] is True def test_scope_outside_any_trace_is_inert(self): trace, _ = get_modules() @@ -335,9 +376,9 @@ class TestTraceNodeScope: assert node == '' assert trace.current_parent() == '' - def test_records_outside_any_scope_serialize_without_node_keys(self): - trace, execution = get_modules() - manager, counters = make_counters(trace_config()) + def test_records_outside_any_scope_are_root_nodes(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) binding = trace.bind('exec-legacy') try: counters.record(CONTEXT, **STAGE) @@ -345,29 +386,16 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - stage = binding.state.stages[0] - assert set(stage) == { - 'family', - 'operation', - 'mode', - 'adapter', - 'runner', - 'outcome', - 'synthetic', - 'seq', - 'route_ref', - 'run_id', - 'first_seen', - 'last_seen', - 'error', - } - observation = counters.records[0]['features']['observations'][0] - assert 'node' not in json.dumps(observation, sort_keys=True) - assert 'parent' not in json.dumps(observation, sort_keys=True) + node = nodes(counters)[0] + assert set(node) == NODE_KEYS + assert node['node_id'] and node['parent_node_id'] == '' and node['root'] is True + assert node['seq'] == 0 + # Node placement lives on node records only; the chain carries none. + assert 'node_id' not in json.dumps(chains(counters)[0], sort_keys=True) class TestTraceDelivery: - def test_close_buffers_one_record_per_trace(self): + def test_close_emits_one_terminal_chain_record_per_execution(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) binding = trace.bind('exec-1') @@ -383,49 +411,55 @@ class TestTraceDelivery: # Closing buffers; nothing is sent before the flush cadence. assert manager.sent == [] - assert len(counters.records) == 1 - assert counters.traces == {} - - record = counters.records[0] - assert record['event_type'] == 'feature_execution' - assert record['query_id'] == 'exec-1' + assert len(nodes(counters)) == 2 + chain_rows = chains(counters) + assert len(chain_rows) == 1 + record = chain_rows[0] + assert record['schema'] == 2 + assert record['event_id'] == 'exec-1' assert record['instance_id'] == 'instance-1' assert record['workspace_uuid'] == 'workspace-1' - assert record['trusted'] is True - features = record['features'] - assert features['schema'] == 1 - assert 'count' not in json.dumps(record) - assert [row['seq'] for row in features['observations']] == [0, 1] - assert [row['trace_id'] for row in features['observations']] == ['exec-1'] * 2 - assert features['trace'] == { - 'closed_by': 'event_done', - 'started_at': binding.state.started_at, - 'ended_at': features['trace']['ended_at'], - 'route_ref': '', - 'run_id': '', - 'outcome': 'success', - 'dropped_stages': 0, - } + assert record['closed_by'] == 'event_done' + assert record['outcome'] == 'success' + assert record['stage_count'] == 2 + assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 + assert 'query_id' not in record and 'features' not in record def test_sampling_decision_is_deterministic_and_keeps_failures(self): trace, _ = get_modules() _, counters = make_counters(trace_config(execution_trace='sampled', execution_trace_sample=2)) - binding = trace.bind() - try: - binding.state.trace_id = '00000001-0000-4000-8000-000000000000' - counters.record(CONTEXT, **STAGE) - assert counters._trace_emitted(binding.state) is False - binding.state.trace_id = '00000000-0000-4000-8000-000000000000' - assert counters._trace_emitted(binding.state) is True - binding.state.trace_id = '00000001-0000-4000-8000-000000000001' - counters.record(CONTEXT, **{**STAGE, 'outcome': 'timeout'}) - assert counters._trace_emitted(binding.state) is True - finally: - trace.unbind_root(binding) + in_id = next(i for i in (f'sample-{n}' for n in range(64)) if counters._sampling_decision(i)[2]) + out_id = next(i for i in (f'sample-{n}' for n in range(64)) if not counters._sampling_decision(i)[2]) + # Same id, same answer: the decision is a pure function of the identity. + assert counters._sampling_decision(in_id) == counters._sampling_decision(in_id) - def test_failures_mode_keeps_failed_and_debug_traces_only(self): + excluded = trace.bind(out_id) + counters.record(CONTEXT, **STAGE) + assert excluded.state.emit_nodes is False + assert counters._chain_emitted(excluded.state) is False + counters.close_trace(excluded.state, 'event_done') + trace.unbind_root(excluded) + assert counters.records == [] + + included = trace.bind(in_id) + counters.record(CONTEXT, **STAGE) + assert included.state.emit_nodes is True + counters.close_trace(included.state, 'event_done') + trace.unbind_root(included) + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + + counters.records.clear() + broken = trace.bind(out_id) + counters.record(CONTEXT, **{**STAGE, 'outcome': 'timeout'}) + assert counters._chain_emitted(broken.state) is True + counters.close_trace(broken.state, 'event_done') + trace.unbind_root(broken) + assert [record['event_type'] for record in counters.records] == ['execution_chain'] + assert counters.records[0]['outcome'] == 'timeout' + + def test_failures_mode_keeps_failed_and_debug_chains_only(self): trace, _ = get_modules() - _, counters = make_counters(trace_config(execution_trace='failures', execution_trace_sample=1000000)) + _, counters = make_counters(trace_config(execution_trace='failures')) failed = trace.bind('exec-failed') counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) @@ -433,7 +467,8 @@ class TestTraceDelivery: trace.unbind_root(failed) debug = trace.bind('exec-debug') - counters.record(CONTEXT, **{**STAGE, 'synthetic': True}) + counters.configure(debug.state, debug=True) + counters.record(CONTEXT, **STAGE) counters.close_trace(debug.state, 'event_done') trace.unbind_root(debug) @@ -442,9 +477,12 @@ class TestTraceDelivery: counters.close_trace(sampled_out.state, 'event_done') trace.unbind_root(sampled_out) - assert [record['query_id'] for record in counters.records] == ['exec-failed', 'exec-debug'] - assert counters.records[0]['features']['observations'][0]['outcome'] == 'failed' - assert counters.records[1]['features']['observations'][0]['synthetic'] is True + assert [record['event_id'] for record in chains(counters)] == ['exec-failed', 'exec-debug'] + assert chains(counters)[0]['outcome'] == 'failed' + # Its node was sampled out, but the broken chain still explains itself. + assert chains(counters)[0]['dropped_nodes'] == 1 + assert chains(counters)[1]['debug'] is True + assert [record['event_id'] for record in nodes(counters)] == ['exec-debug'] def test_off_mode_and_opt_out_buffer_nothing(self): trace, _ = get_modules() @@ -469,24 +507,9 @@ class TestTraceDelivery: trace.unbind_root(binding) assert disabled_manager.sent == [] assert disabled.records == [] - assert disabled.traces == {} + assert disabled.registry == {} - def test_ttl_sweep_closes_stale_trace(self): - trace, _ = get_modules() - manager, counters = make_counters(trace_config()) - binding = trace.bind('exec-ttl') - try: - counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - counters.trace_deadlines[binding.state.trace_id] = 0 - counters._sweep_traces(time.monotonic()) - finally: - trace.unbind_root(binding) - assert counters.records[0]['features']['trace']['closed_by'] == 'ttl' - assert counters.traces == {} - assert counters.trace_deadlines == {} - assert manager.sent == [] - - async def test_shutdown_drops_open_traces(self): + async def test_shutdown_does_not_claim_an_unfinished_execution(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) binding = trace.bind('exec-open') @@ -495,26 +518,12 @@ class TestTraceDelivery: await counters.shutdown() finally: trace.unbind_root(binding) - # Shutdown must not claim a trace that never completed. - assert records_sent(manager) == [] + # The already-emitted node is flushed, but shutdown never fabricates a + # terminal chain for an execution that never completed. + assert [record['event_type'] for record in records_sent(manager)] == ['execution_node'] assert counters.records == [] - assert counters.traces == {} + assert chains(counters) == [] assert counters.registry == {} - assert counters.dropped_traces == 1 - - async def test_loop_sweeps_and_flushes_expired_trace(self): - trace, _ = get_modules() - manager, counters = make_counters(trace_config(telemetry_flush_seconds=1)) - binding = trace.bind('exec-ttl-loop') - try: - counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - counters.trace_deadlines[binding.state.trace_id] = 0 - await asyncio.sleep(1.3) - assert [record['features']['trace']['closed_by'] for record in records_sent(manager)] == ['ttl'] - assert counters.traces == {} - finally: - await counters.shutdown() - trace.unbind_root(binding) async def test_flush_sends_one_batch_of_complete_records(self): _, execution = get_modules() @@ -523,14 +532,20 @@ class TestTraceDelivery: for index in range(2): with execution.ingress(ap, 'event_done', CONTEXT, execution_id=f'exec-{index}'): counters.record(CONTEXT, **STAGE) - assert len(counters.records) == 2 + assert len(counters.records) == 4 await counters.flush() assert len(manager.sent) == 1 payload = manager.sent[0] assert set(payload) == {'records'} - assert [record['query_id'] for record in payload['records']] == ['exec-0', 'exec-1'] - assert 'count' not in json.dumps(payload) + assert [record['event_type'] for record in payload['records']] == [ + 'execution_node', + 'execution_chain', + 'execution_node', + 'execution_chain', + ] + assert [record['event_id'] for record in payload['records']] == ['exec-0', 'exec-0', 'exec-1', 'exec-1'] + await counters.shutdown() async def test_flush_interval_is_honoured(self): _, execution = get_modules() @@ -541,7 +556,9 @@ class TestTraceDelivery: assert counters.flush_seconds() == 1 assert manager.sent == [] await asyncio.sleep(1.3) - assert [record['query_id'] for record in records_sent(manager)] == ['exec-timed'] + sent = records_sent(manager) + assert [record['event_type'] for record in sent] == ['execution_node', 'execution_chain'] + assert {record['event_id'] for record in sent} == {'exec-timed'} await counters.shutdown() def test_flush_interval_config_is_bounded(self): @@ -563,11 +580,10 @@ class TestTraceDelivery: for index in range(5): binding = trace.bind(f'exec-{index}') counters.record(CONTEXT, **STAGE) - counters.close_trace(binding.state, 'event_done') trace.unbind_root(binding) finally: execution.MAX_BUFFERED_RECORDS = original - assert [record['query_id'] for record in counters.records] == ['exec-2', 'exec-3', 'exec-4'] + assert [record['event_id'] for record in nodes(counters)] == ['exec-2', 'exec-3', 'exec-4'] assert counters.dropped == 2 @@ -579,9 +595,9 @@ class TestIngress: with execution.ingress(ap, 'event_done', execution_id='exec-outer'): with execution.ingress(ap, 'pipeline_done', execution_id='exec-inner'): counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - assert len(counters.records) == 1 - assert counters.records[0]['query_id'] == 'exec-outer' - assert counters.records[0]['features']['trace']['closed_by'] == 'event_done' + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'exec-outer' for record in counters.records) + assert chains(counters)[0]['closed_by'] == 'event_done' assert manager.sent == [] def test_ingress_without_telemetry_is_a_no_op(self): @@ -602,7 +618,7 @@ class TestIngress: class TestCrossTaskAttribution: - def test_registry_joins_stages_from_a_task_without_a_trace(self): + def test_registry_joins_nodes_from_a_task_without_a_trace(self): trace, execution = get_modules() _, counters = make_counters(trace_config()) ap = make_ap(counters) @@ -615,23 +631,24 @@ class TestCrossTaskAttribution: counters.record( CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}, execution_id='run-1' ) - assert len(binding.state.stages) == 1 - assert binding.state.stages[0]['operation'] == 'send_message' + rows = nodes(counters) + assert len(rows) == 1 + assert rows[0]['operation'] == 'send_message' and rows[0]['event_id'] == 'run-1' counters.close_trace(binding.state, 'event_done') - assert counters.records[0]['query_id'] == 'run-1' + assert chains(counters)[0]['event_id'] == 'run-1' def test_registry_aliases_a_nested_execution(self): _, execution = get_modules() _, counters = make_counters(trace_config()) ap = make_ap(counters) - with execution.ingress(ap, 'event_done', CONTEXT, execution_id='event-1') as binding: + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='event-1'): with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-1'): pass assert set(counters.registry) == {'event-1', 'query-1'} counters.record(CONTEXT, **STAGE, execution_id='query-1') - assert counters.records[0]['query_id'] == 'event-1' - assert len(binding.state.stages) == 1 + rows = nodes(counters) + assert len(rows) == 1 and rows[0]['event_id'] == 'event-1' def test_contextvar_execution_id_is_used_when_no_kwarg(self): trace, execution = get_modules() @@ -644,7 +661,8 @@ class TestCrossTaskAttribution: counters.record(CONTEXT, **STAGE) finally: execution.reset_execution_id(token) - assert len(binding.state.stages) == 1 + rows = nodes(counters) + assert len(rows) == 1 and rows[0]['event_id'] == 'run-2' def test_registry_alias_is_swept_when_never_closed(self): trace, execution = get_modules() @@ -667,8 +685,10 @@ class TestTraceDispatch: with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-async'): counters.record(CONTEXT, **STAGE) await counters.flush() - assert [record['query_id'] for record in records_sent(manager)] == ['exec-async'] - assert manager.sent[0]['records'][0]['event_type'] == 'feature_execution' + sent = records_sent(manager) + assert [record['event_type'] for record in sent] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'exec-async' for record in sent) + await counters.shutdown() class AsyncSendManager(FakeManager): @@ -689,9 +709,9 @@ class TestTraceFailureDetail: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] - assert record['error'] == 'boom: sandbox exited 127' - assert record['features']['observations'][0]['error'] == 'boom: sandbox exited 127' + assert nodes(counters)[0]['error'] == 'boom: sandbox exited 127' + # The detail lives on the node; the chain carries the terminal outcome. + assert chains(counters)[0]['outcome'] == 'failed' def test_successful_stage_reports_no_error(self): trace, _ = get_modules() @@ -702,9 +722,8 @@ class TestTraceFailureDetail: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] - assert record['error'] == '' - assert record['features']['observations'][0]['error'] == '' + assert nodes(counters)[0]['error'] == '' + assert chains(counters)[0]['error'] == '' def test_error_detail_is_bounded_and_single_line(self): trace, _ = get_modules() @@ -715,8 +734,8 @@ class TestTraceFailureDetail: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - detail = counters.records[0]['error'] - assert len(detail) <= 400 + detail = chains(counters)[0]['error'] + assert len(detail) <= 200 assert '\n' not in detail and '\t' not in detail @@ -733,10 +752,10 @@ class TestIngressFailureDetail: except RuntimeError: raised = True assert raised is True - record = counters.records[0] - assert record['error'].startswith('RuntimeError: dispatch exploded') - assert record['features']['trace']['outcome'] == 'failed' - assert record['features']['trace']['closed_by'] == 'event_done' + chain = chains(counters)[0] + assert chain['error'].startswith('RuntimeError: dispatch exploded') + assert chain['outcome'] == 'failed' + assert chain['closed_by'] == 'event_done' def test_successful_ingress_still_reports_success(self): _, execution = get_modules() @@ -744,27 +763,29 @@ class TestIngressFailureDetail: ap = make_ap(counters) with execution.ingress(ap, 'event_done', execution_id='exec-fine'): counters.record(CONTEXT, **STAGE) - record = counters.records[0] - assert record['error'] == '' - assert record['features']['trace']['outcome'] == 'success' + chain = chains(counters)[0] + assert chain['error'] == '' + assert chain['outcome'] == 'success' class TestFailureOnlyTrace: def test_failure_without_stages_still_uploads(self): _, execution = get_modules() - # sampled-out config: only failures are guaranteed to be kept - _, counters = make_counters({'url': 'https://space.example.test', 'execution_trace_sample': 1}) + # failures mode: nodes are not reported, but a broken chain always is. + _, counters = make_counters({'url': 'https://space.example.test', 'execution_trace': 'failures'}) ap = make_ap(counters) try: with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='exec-empty'): raise RuntimeError('pipeline exploded before any stage') except RuntimeError: pass - assert len(counters.records) == 1 - record = counters.records[0] + rows = chains(counters) + assert len(rows) == 1 + record = rows[0] assert record['error'].startswith('RuntimeError: pipeline exploded') - assert record['features']['trace']['outcome'] == 'failed' - assert record['features']['observations'] == [] + assert record['outcome'] == 'failed' + assert record['stage_count'] == 0 + assert nodes(counters) == [] def test_cancelled_and_timeout_chains_report_their_outcome(self): trace, _ = get_modules() @@ -776,15 +797,14 @@ class TestFailureOnlyTrace: counters.close_trace(binding.state, 'runner_done') finally: trace.unbind_root(binding) - assert len(counters.records) == 1 - record = counters.records.pop(0) - assert record['query_id'] == f'exec-{outcome}' - assert record['features']['trace']['outcome'] == outcome - assert record['error'] == reason + record = chains(counters)[-1] + assert record['event_id'] == f'exec-{outcome}' + assert record['outcome'] == outcome + assert nodes(counters)[-1]['error'] == reason class TestLegacyRecordFields: - def test_record_carries_legacy_top_level_columns(self): + def test_chain_carries_execution_scoped_columns(self): trace, _ = get_modules() _, counters = make_counters(trace_config()) binding = trace.bind('exec-legacy') @@ -800,14 +820,13 @@ class TestLegacyRecordFields: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] + record = chains(counters)[0] assert record['model_name'] == 'gpt-4o' assert record['pipeline_plugins'] == ['plugin:a'] assert record['runner_category'] == 'cloud' assert record['adapter'] == 'AiocqhttpAdapter' assert record['runner'] == 'runner-1' assert record['duration_ms'] >= 1000 - assert record['timestamp'] == record['features']['trace']['ended_at'] def test_legacy_string_fields_are_never_null(self): trace, _ = get_modules() @@ -818,9 +837,135 @@ class TestLegacyRecordFields: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] + record = chains(counters)[0] assert record['model_name'] == '' assert record['runner'] == '' assert record['runner_category'] == '' assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 assert record['pipeline_plugins'] is None + + +class TestNoTraceTtl: + def test_nodes_more_than_two_minutes_apart_still_emit(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-long') + try: + counters.record(CONTEXT, **STAGE) + # A gap far beyond the old 120s TTL must not sweep the chain: there is + # no local trace deadline any more, so both nodes and the terminal + # chain are still emitted. + binding.state.started_at = (datetime.now(timezone.utc) - timedelta(seconds=200)).isoformat() + counters.record(CONTEXT, **{**STAGE, 'operation': 'still.alive'}) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + rows = nodes(counters) + assert [record['seq'] for record in rows] == [0, 1] + assert [record['operation'] for record in rows] == ['message.received', 'still.alive'] + chain = chains(counters)[0] + assert chain['stage_count'] == 2 + assert chain['duration_ms'] >= 120_000 + + +class TestFlushDelivery: + async def test_rejected_batch_stays_buffered_and_is_retried(self): + trace, execution = get_modules() + manager = RejectingManager(trace_config()) + counters = execution.ExecutionCounters(manager) + binding = trace.bind('exec-retry') + try: + counters.record(CONTEXT, **STAGE) + assert await counters.flush() is False + assert manager.attempts == execution.MAX_FLUSH_ATTEMPTS + # The record was never lost: it is still buffered and unsent. + assert len(counters.records) == 1 and manager.sent == [] + manager.rejecting = False + assert await counters.flush() is True + finally: + trace.unbind_root(binding) + assert counters.records == [] + assert [record['event_id'] for record in records_sent(manager)] == ['exec-retry'] + await counters.shutdown() + + async def test_health_reports_buffered_sent_and_dropped(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'exec-health') + try: + counters.record(CONTEXT, **STAGE) + health = counters.health() + assert health['records_buffered'] == 1 + assert health['records_sent'] == 0 + assert health['records_dropped'] == 0 + assert health['executions_in_flight'] == 1 + await counters.flush() + health = counters.health() + assert health['records_buffered'] == 0 + assert health['records_sent'] == 1 + finally: + trace.unbind_root(binding) + + # Overflowing the outbound buffer is surfaced as dropped records. + original = execution.MAX_BUFFERED_RECORDS + execution.MAX_BUFFERED_RECORDS = 1 + try: + for index in range(3): + inner = trace.bind(f'exec-drop-{index}') + counters.record(CONTEXT, **STAGE) + trace.unbind_root(inner) + finally: + execution.MAX_BUFFERED_RECORDS = original + health = counters.health() + assert health['records_dropped'] == 2 + assert health['records_buffered'] == 1 + await counters.shutdown() + + +class TestSiblingTaskIsolation: + async def test_sibling_stage_scopes_do_not_corrupt_parent_links(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + opened = {} + + async def work(name): + with trace.stage_scope() as node: + opened[name] = node + counters.record(CONTEXT, **{**STAGE, 'operation': name, 'node': node}) + + async def scenario(): + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-siblings'): + await asyncio.gather(work('alpha'), work('beta')) + snapshot = list(counters.records) + await counters.shutdown() + return snapshot + + records = await scenario() + rows = {record['operation']: record for record in records if record['event_type'] == 'execution_node'} + assert opened['alpha'] != opened['beta'] + # Each sibling's step is its own root: neither becomes the other's parent. + assert rows['alpha']['node_id'] == opened['alpha'] + assert rows['beta']['node_id'] == opened['beta'] + assert rows['alpha']['parent_node_id'] == '' + assert rows['beta']['parent_node_id'] == '' + assert [record['event_type'] for record in records].count('execution_chain') == 1 + + +class TestSamplingShape: + def test_platform_prefixed_event_id_is_emitted_at_default_config(self): + trace, _ = get_modules() + _, counters = make_counters() # no execution_trace override -> default mode + assert counters.trace_mode() == 'all' + # The identity is hashed, not parsed: a platform-prefixed id samples the + # same way any other id shape does. + assert counters._sampling_decision('platform:bot-1:message-1') == ('all', 1, True) + binding = trace.bind('platform:bot-1:message-1') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'platform:bot-1:message-1' for record in counters.records) diff --git a/tests/unit_tests/test_telemetry.py b/tests/unit_tests/test_telemetry.py deleted file mode 100644 index ed3994923..000000000 --- a/tests/unit_tests/test_telemetry.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -import asyncio - -import pytest - -from langbot.pkg.telemetry.telemetry import TelemetryManager - - -@pytest.mark.asyncio -async def test_send_tasks_are_scoped_to_manager_instance(monkeypatch): - async def fake_send(self, payload): - return payload - - monkeypatch.setattr(TelemetryManager, 'send', fake_send) - - first = TelemetryManager(SimpleNamespace()) - second = TelemetryManager(SimpleNamespace()) - - assert first.send_tasks is not second.send_tasks - - await first.start_send_task({'event': 'first'}) - task = first.send_tasks[0] - await task - await asyncio.sleep(0) - - assert first.send_tasks == [] - assert second.send_tasks == [] From de886edc1b60be23b9b0fac4c37296cb81dca16b Mon Sep 17 00:00:00 2001 From: Hyu Date: Wed, 7 Oct 2026 10:20:08 +0800 Subject: [PATCH 19/30] style(telemetry): apply ruff format to the telemetry manager The single-line assignment fits the configured line length; ruff format rejects the wrapped form. --- src/langbot/pkg/telemetry/telemetry.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/langbot/pkg/telemetry/telemetry.py b/src/langbot/pkg/telemetry/telemetry.py index 9188a5c54..5c9a41d22 100644 --- a/src/langbot/pkg/telemetry/telemetry.py +++ b/src/langbot/pkg/telemetry/telemetry.py @@ -84,9 +84,7 @@ class TelemetryManager: else: if 'query_id' in sanitized: try: - sanitized['query_id'] = ( - '' if sanitized['query_id'] is None else str(sanitized['query_id']) - ) + sanitized['query_id'] = '' if sanitized['query_id'] is None else str(sanitized['query_id']) except Exception: sanitized['query_id'] = str(sanitized.get('query_id', '')) From 9f76037e6427372d66dd18acd531bc3cccf02e60 Mon Sep 17 00:00:00 2001 From: Hyu Date: Wed, 7 Oct 2026 22:36:10 +0800 Subject: [PATCH 20/30] feat(monitoring): unify execution views and trace tool calls --- src/langbot/pkg/agent/runner/orchestrator.py | 15 + .../api/http/controller/groups/monitoring.py | 43 +- .../pkg/api/http/service/monitoring.py | 598 ++++++++++++++ src/langbot/pkg/pipeline/monitoring_helper.py | 97 +++ src/langbot/pkg/provider/tools/toolmgr.py | 74 ++ src/langbot/pkg/telemetry/execution.py | 8 +- src/langbot/pkg/telemetry/trace.py | 37 +- .../api/service/test_monitoring_executions.py | 350 ++++++++ .../unit_tests/provider/test_tool_manager.py | 123 +++ tests/unit_tests/telemetry/test_trace.py | 49 +- .../components/ConversationTurnList.tsx | 8 +- .../monitoring/components/ExportDropdown.tsx | 22 +- .../monitoring/components/FeedbackCard.tsx | 123 +-- .../monitoring/components/FeedbackList.tsx | 102 ++- .../components/MessageContentRenderer.tsx | 47 +- .../components/MessageDetailsCard.tsx | 30 +- .../home/monitoring/components/TabState.tsx | 50 ++ .../monitoring/components/TokenMonitoring.tsx | 285 +++---- .../executions/ExecutionDetailSheet.tsx | 361 +++++++++ .../components/executions/ExecutionTable.tsx | 169 ++++ .../components/filters/MonitoringFilters.tsx | 125 ++- .../overview-cards/ExecutionOverviewCards.tsx | 197 +++++ .../components/overview-cards/MetricCard.tsx | 138 ++-- .../overview-cards/OverviewCards.tsx | 100 --- .../overview-cards/SystemStatusCards.tsx | 45 +- .../overview-cards/TrafficChart.tsx | 22 +- .../home/monitoring/hooks/useExecutions.ts | 147 ++++ .../monitoring/hooks/useMonitoringData.ts | 49 +- web/src/app/home/monitoring/page.tsx | 752 +++++++++++------- .../app/home/monitoring/utils/callLinks.ts | 100 +++ .../app/home/monitoring/utils/dateUtils.ts | 45 ++ web/src/app/infra/entities/api/index.ts | 17 + .../entities/api/monitoring-executions.ts | 136 ++++ web/src/app/infra/http/BackendClient.ts | 65 ++ web/src/components/ui/table.tsx | 11 +- web/src/i18n/locales/en-US.ts | 77 ++ web/src/i18n/locales/zh-Hans.ts | 76 ++ web/src/i18n/locales/zh-Hant.ts | 76 ++ web/tests/e2e/fixtures/langbot-api.ts | 157 ++++ web/tests/e2e/home-smoke.spec.ts | 9 +- web/tests/e2e/monitoring-turns.spec.ts | 4 + web/tests/unit/monitoring-call-links.test.mjs | 131 +++ 42 files changed, 4181 insertions(+), 889 deletions(-) create mode 100644 tests/unit_tests/api/service/test_monitoring_executions.py create mode 100644 web/src/app/home/monitoring/components/TabState.tsx create mode 100644 web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx create mode 100644 web/src/app/home/monitoring/components/executions/ExecutionTable.tsx create mode 100644 web/src/app/home/monitoring/components/overview-cards/ExecutionOverviewCards.tsx delete mode 100644 web/src/app/home/monitoring/components/overview-cards/OverviewCards.tsx create mode 100644 web/src/app/home/monitoring/hooks/useExecutions.ts create mode 100644 web/src/app/home/monitoring/utils/callLinks.ts create mode 100644 web/src/app/infra/entities/api/monitoring-executions.ts create mode 100644 web/tests/unit/monitoring-call-links.test.mjs diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index b04ec8394..9d3db00d2 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -194,6 +194,21 @@ class AgentRunOrchestrator: state_context = build_state_context(event, binding, descriptor) run_id = context['run_id'] + + # Monitoring attributes every call to the processor that ran it and to the + # record that owns it; a Runner has to claim both itself, since no + # Pipeline did. Runs before the Runner is invoked and before the run + # context is handed to it, so its own variables stay private. + from ...pipeline import monitoring_helper + + await monitoring_helper.prepare_monitoring_identity( + self.ap, + execution_query, + processor_type=binding.processor_type, + processor_id=binding.processor_id, + runner_id=binding.runner_id, + execution_record_id=run_id, + ) context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any( tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform' for tool in resources.get('tools', []) diff --git a/src/langbot/pkg/api/http/controller/groups/monitoring.py b/src/langbot/pkg/api/http/controller/groups/monitoring.py index 01a901743..1d9656761 100644 --- a/src/langbot/pkg/api/http/controller/groups/monitoring.py +++ b/src/langbot/pkg/api/http/controller/groups/monitoring.py @@ -50,6 +50,47 @@ class MonitoringRouterGroup(group.RouterGroup): return self.success(data=metrics) + @self.route('/executions', methods=['GET'], permission=Permission.RESOURCE_VIEW) + async def get_executions(request_context: RequestContext) -> str: + """Unified execution list across agent runs and pipeline queries.""" + bot_ids = quart.request.args.getlist('botId') + pipeline_ids = quart.request.args.getlist('pipelineId') + agent_ids = quart.request.args.getlist('agentId') + statuses = quart.request.args.getlist('status') + source = quart.request.args.get('source', 'all') + mode = quart.request.args.get('mode', 'all') + start_time = parse_iso_datetime(quart.request.args.get('startTime')) + end_time = parse_iso_datetime(quart.request.args.get('endTime')) + + result = await self.ap.monitoring_service.get_executions( + request_context, + bot_ids=bot_ids if bot_ids else None, + pipeline_ids=pipeline_ids if pipeline_ids else None, + agent_ids=agent_ids if agent_ids else None, + statuses=statuses if statuses else None, + source=source, + mode=mode, + start_time=start_time, + end_time=end_time, + limit=quart.request.args.get('limit', 50), + offset=quart.request.args.get('offset', 0), + ) + + return self.success(data=result) + + @self.route( + '/executions//', + methods=['GET'], + permission=Permission.RESOURCE_VIEW, + ) + async def get_execution_detail(source: str, execution_id: str, request_context: RequestContext) -> str: + """Full trace of a single execution (agent run or pipeline query).""" + try: + detail = await self.ap.monitoring_service.get_execution_detail(request_context, source, execution_id) + except ValueError as exc: + return self.fail(404, str(exc)) + return self.success(data=detail) + @self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_token_statistics(request_context: RequestContext) -> str: """Get detailed token usage statistics (summary, per-model, timeseries).""" @@ -639,7 +680,7 @@ class MonitoringRouterGroup(group.RouterGroup): 'platform', ] else: - return self.error(message=f'Invalid export type: {export_type}', code=400) + return self.fail(400, f'Invalid export type: {export_type}') # Generate CSV content with UTF-8 BOM for Excel compatibility import io diff --git a/src/langbot/pkg/api/http/service/monitoring.py b/src/langbot/pkg/api/http/service/monitoring.py index 0bae983a6..bb7407248 100644 --- a/src/langbot/pkg/api/http/service/monitoring.py +++ b/src/langbot/pkg/api/http/service/monitoring.py @@ -10,7 +10,10 @@ from sqlalchemy.dialects import postgresql as postgresql_dialect from sqlalchemy.dialects import sqlite as sqlite_dialect from ....core import app +from ....entity.persistence import agent as persistence_agent from ....entity.persistence import monitoring as persistence_monitoring +from ....entity.persistence import agent_run as persistence_agent_run +from ....entity.persistence import agent_interaction as persistence_agent_interaction from ..authz import WorkspaceRequiredError from ..context import ExecutionContext from .tenant import TenantContext, require_workspace_uuid @@ -29,6 +32,76 @@ _HARD_MAX_MONITORING_OFFSET = 10000000 _DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4 _HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100 +# Canonical execution status groups shared by the agent ledger and pipeline query +# records so both object types can be summarised with one vocabulary. +_AGENT_STATUS_GROUP = { + 'completed': 'completed', + 'failed': 'failed', + 'timeout': 'failed', + 'cancelled': 'cancelled', + 'created': 'queued', + 'queued': 'queued', + 'claimed': 'running', + 'running': 'running', +} +_PIPELINE_STATUS_GROUP = { + 'success': 'completed', + 'error': 'failed', + 'pending': 'running', + 'discarded': 'ignored', + 'not_matched': 'ignored', +} + + +def _status_group(status: str, source: str) -> str: + table = _AGENT_STATUS_GROUP if source == 'agent' else _PIPELINE_STATUS_GROUP + return table.get(status, status) + + +def _epoch_ms(value: datetime.datetime | None) -> int | None: + """Epoch milliseconds for a DB timestamp, treating naive values as UTC.""" + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + return int(round(value.timestamp() * 1000)) + + +def _percentile_ms(values: list[float], fraction: float) -> int | None: + """Nearest-rank percentile (ms) over a non-empty, prefix-sorted list.""" + if not values: + return None + rank = max(1, min(len(values), int(round(fraction * len(values))))) + return int(round(values[rank - 1])) + + +def _collect_text(node, out: list[str], depth: int = 0) -> None: + """Best-effort extraction of human text from a serialised message chain.""" + if depth > 6: + return + if isinstance(node, dict): + for key, value in node.items(): + if key == 'text' and isinstance(value, str): + out.append(value) + elif isinstance(value, (dict, list)): + _collect_text(value, out, depth + 1) + elif isinstance(node, list): + for item in node: + _collect_text(item, out, depth + 1) + + +def _message_preview(raw: str | None, limit: int = 160) -> str: + if not raw: + return '' + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + return raw[:limit] + parts: list[str] = [] + _collect_text(parsed, parts) + text = ' '.join(part.strip() for part in parts if part.strip()) + return (text or raw)[:limit] + def _normalize_user_id(value: str | int | None) -> str | None: """Convert numeric platform IDs before binding a VARCHAR with asyncpg. @@ -885,6 +958,10 @@ class MonitoringService: active_sessions_result = await self.ap.persistence_mgr.execute_async(active_session_query) active_sessions = active_sessions_result.scalar() or 0 + # Execution-first KPI values (runs + pipeline queries, latency, token + # coverage) are served by `get_executions`, which applies the execution + # view filters. This endpoint stays the message/traffic rollup so the + # dashboard does not compute the same rollup twice per page load. return { 'total_messages': total_messages, 'llm_calls': llm_calls, @@ -894,6 +971,527 @@ class MonitoringService: 'active_sessions': active_sessions, } + # ========== Execution view ========== + + def _agent_run_conditions( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + agent_ids: list[str] | None = None, + statuses: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + debug: bool | None = None, + ) -> list: + AgentRun = persistence_agent_run.AgentRun + conditions = [AgentRun.workspace_id == workspace_uuid] + if bot_ids: + conditions.append(AgentRun.bot_id.in_(bot_ids)) + if agent_ids: + conditions.append(AgentRun.agent_id.in_(agent_ids)) + if statuses is not None: + conditions.append(AgentRun.status.in_(statuses) if statuses else sqlalchemy.false()) + if start_time: + conditions.append(AgentRun.created_at >= start_time) + if end_time: + conditions.append(AgentRun.created_at <= end_time) + if debug is not None: + debug_condition = sqlalchemy.or_( + AgentRun.binding_id.like('debug:%'), + AgentRun.conversation_id.like('debug:%'), + ) + conditions.append(debug_condition if debug else sqlalchemy.not_(debug_condition)) + return conditions + + def _pipeline_execution_conditions( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + statuses: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> list: + MonitoringMessage = persistence_monitoring.MonitoringMessage + conditions = [ + MonitoringMessage.workspace_uuid == workspace_uuid, + # Assistant replies are outputs of a query, not an execution of their own. + sqlalchemy.or_(MonitoringMessage.role.is_(None), MonitoringMessage.role != 'assistant'), + ] + if bot_ids: + conditions.append(MonitoringMessage.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(MonitoringMessage.pipeline_id.in_(pipeline_ids)) + if statuses is not None: + conditions.append(MonitoringMessage.status.in_(statuses) if statuses else sqlalchemy.false()) + if start_time: + conditions.append(MonitoringMessage.timestamp >= start_time) + if end_time: + conditions.append(MonitoringMessage.timestamp <= end_time) + return conditions + + @staticmethod + def _raw_agent_statuses(groups: list[str] | None) -> list[str] | None: + """Raw agent statuses for a group filter; empty means "no group applies".""" + if not groups: + return None + raw: list[str] = [] + for group in groups: + raw.extend(status for status, mapped in _AGENT_STATUS_GROUP.items() if mapped == group) + return raw + + @staticmethod + def _raw_pipeline_statuses(groups: list[str] | None) -> list[str] | None: + """Raw pipeline statuses for a group filter; empty means "no group applies".""" + if not groups: + return None + raw: list[str] = [] + for group in groups: + raw.extend(status for status, mapped in _PIPELINE_STATUS_GROUP.items() if mapped == group) + return raw + + @staticmethod + def _model_rows(result) -> list: + """Rows of one ``select(Model)`` executed through ``execute_async``. + + The Core keeps the connection result contract, where ``select(Model)`` + yields a *flat* row of the model's columns (``row.uuid``, ``row.status``) + rather than a one-item ORM row. ``.scalars()`` would therefore return the + first column - the id - instead of the row, so the rows are taken as-is. + """ + return list(result.all()) + + @staticmethod + def _deserialize_json(raw: str | None) -> dict: + if not raw: + return {} + try: + value = json.loads(raw) + except (TypeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + async def _agent_names(self, workspace_uuid: str, rows: list) -> dict[str, str]: + """The processor name of every Agent run in one page, in one query. + + A run stores the Agent it executed as a uuid; the list has to name the + processor, so the names are resolved for the whole page at once. + """ + agent_ids = {row.agent_id for row in rows if row.agent_id} + if not agent_ids: + return {} + Agent = persistence_agent.Agent + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(Agent.uuid, Agent.name).where( + sqlalchemy.and_( + Agent.workspace_uuid == workspace_uuid, + Agent.uuid.in_(agent_ids), + ) + ) + ) + return {row.uuid: row.name for row in self._model_rows(result)} + + def _serialize_agent_execution(self, row, agent_names: dict[str, str] | None = None) -> dict: + metadata = self._deserialize_json(row.metadata_json) + duration_ms = None + if row.started_at and row.finished_at: + duration_ms = int(round((row.finished_at - row.started_at).total_seconds() * 1000)) + created_at_ms = _epoch_ms(row.created_at) + started_at_ms = _epoch_ms(row.started_at) + finished_at_ms = _epoch_ms(row.finished_at) + is_debug = str(row.binding_id or '').startswith('debug:') or str(row.conversation_id or '').startswith('debug:') + return { + 'source': 'agent', + 'id': row.run_id, + 'event_id': row.event_id, + 'status': row.status, + 'status_group': _status_group(row.status, 'agent'), + 'title': metadata.get('event_type') or row.queue_name or row.runner_id, + 'target_kind': 'agent' if row.agent_id else 'processor', + 'target_id': row.agent_id or row.binding_id, + 'target_name': metadata.get('agent_name') + or metadata.get('target_name') + or (agent_names or {}).get(row.agent_id or ''), + 'bot_id': row.bot_id, + 'bot_name': metadata.get('bot_name'), + 'pipeline_id': None, + 'pipeline_name': None, + 'runner_id': row.runner_id, + 'conversation_id': row.conversation_id, + 'session_id': row.conversation_id, + 'created_at_ms': created_at_ms, + 'started_at_ms': started_at_ms, + 'finished_at_ms': finished_at_ms, + 'duration_ms': duration_ms, + 'usage': self._deserialize_json(row.usage_json) or None, + 'cost': self._deserialize_json(row.cost_json) or None, + 'queue_name': row.queue_name, + 'debug': is_debug, + 'has_error': row.status in {'failed', 'timeout'}, + } + + def _serialize_pipeline_execution(self, row) -> dict: + created_at_ms = _epoch_ms(row.timestamp) + return { + 'source': 'pipeline', + 'id': row.id, + 'event_id': None, + 'status': row.status, + 'status_group': _status_group(row.status, 'pipeline'), + 'title': _message_preview(row.message_content), + 'target_kind': 'pipeline', + 'target_id': row.pipeline_id, + 'target_name': row.pipeline_name, + 'bot_id': row.bot_id, + 'bot_name': row.bot_name, + 'pipeline_id': row.pipeline_id, + 'pipeline_name': row.pipeline_name, + 'runner_id': row.runner_name, + 'conversation_id': None, + 'session_id': row.session_id, + 'created_at_ms': created_at_ms, + 'started_at_ms': None, + 'finished_at_ms': None, + 'duration_ms': None, + 'usage': None, + 'cost': None, + 'queue_name': None, + 'debug': False, + 'has_error': row.status == 'error', + 'platform': row.platform, + 'user_id': row.user_id, + 'user_name': row.user_name, + } + + async def _get_token_coverage( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> dict: + LLMCall = persistence_monitoring.MonitoringLLMCall + conditions = [LLMCall.workspace_uuid == workspace_uuid] + if bot_ids: + conditions.append(LLMCall.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(LLMCall.pipeline_id.in_(pipeline_ids)) + if start_time: + conditions.append(LLMCall.timestamp >= start_time) + if end_time: + conditions.append(LLMCall.timestamp <= end_time) + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select( + sqlalchemy.func.count(LLMCall.id), + sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0), + sqlalchemy.func.sum(sqlalchemy.case((LLMCall.total_tokens > 0, 1), else_=0)), + ).where(sqlalchemy.and_(*conditions)) + ) + row = result.first() + calls, total, calls_with_usage = row if row else (0, 0, 0) + return { + 'total_tokens': int(total or 0), + 'calls': calls or 0, + 'calls_with_usage': calls_with_usage or 0, + } + + async def _get_execution_summary( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> dict: + AgentRun = persistence_agent_run.AgentRun + run_conditions = self._agent_run_conditions( + workspace_uuid, bot_ids=bot_ids, start_time=start_time, end_time=end_time + ) + run_status_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun.status, sqlalchemy.func.count(AgentRun.id)) + .where(sqlalchemy.and_(*run_conditions)) + .group_by(AgentRun.status) + ) + agent_counts = {row[0]: row[1] for row in run_status_result.all()} + agent_total = sum(agent_counts.values()) + + debug_count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where( + sqlalchemy.and_( + *run_conditions, + sqlalchemy.or_( + AgentRun.binding_id.like('debug:%'), + AgentRun.conversation_id.like('debug:%'), + ), + ) + ) + ) + debug_count = debug_count_result.scalar() or 0 + + # Percentiles are computed in Python, so the sample must stay bounded: + # take the most recent completed runs of the window and label the count + # so the UI can state how many runs the numbers rest on. + timing_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun.started_at, AgentRun.finished_at) + .where( + sqlalchemy.and_( + *run_conditions, + AgentRun.status == 'completed', + AgentRun.started_at.is_not(None), + AgentRun.finished_at.is_not(None), + ) + ) + .order_by(AgentRun.created_at.desc()) + .limit(self._detail_limit()) + ) + durations = sorted((row[1] - row[0]).total_seconds() * 1000 for row in timing_result.all() if row[0] and row[1]) + + pipeline_conditions = self._pipeline_execution_conditions( + workspace_uuid, bot_ids=bot_ids, pipeline_ids=pipeline_ids, start_time=start_time, end_time=end_time + ) + pipeline_status_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select( + persistence_monitoring.MonitoringMessage.status, + sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id), + ) + .where(sqlalchemy.and_(*pipeline_conditions)) + .group_by(persistence_monitoring.MonitoringMessage.status) + ) + pipeline_counts = {row[0]: row[1] for row in pipeline_status_result.all()} + pipeline_total = sum(pipeline_counts.values()) + + # Pending interactions are gated by the same bot/window scope as the runs + # so "waiting input" cannot drift from the rest of the card. + waiting_conditions = [ + persistence_agent_interaction.AgentInteraction.workspace_id == workspace_uuid, + persistence_agent_interaction.AgentInteraction.status == 'pending', + ] + if bot_ids: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.bot_id.in_(bot_ids)) + if start_time: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at >= start_time) + if end_time: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at <= end_time) + waiting_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(persistence_agent_interaction.AgentInteraction.id)).where( + sqlalchemy.and_(*waiting_conditions) + ) + ) + waiting = waiting_result.scalar() or 0 + + by_status: dict[str, int] = {} + for status, count in agent_counts.items(): + group = _status_group(status, 'agent') + by_status[group] = by_status.get(group, 0) + count + for status, count in pipeline_counts.items(): + group = _status_group(status, 'pipeline') + by_status[group] = by_status.get(group, 0) + count + + completed = by_status.get('completed', 0) + failed = by_status.get('failed', 0) + denominator = completed + failed + total = agent_total + pipeline_total + return { + 'total': total, + 'by_source': {'agent': agent_total, 'pipeline': pipeline_total}, + 'by_status': by_status, + 'completed': completed, + 'failed': failed, + 'running': by_status.get('running', 0), + 'queued': by_status.get('queued', 0), + 'cancelled': by_status.get('cancelled', 0), + 'ignored': by_status.get('ignored', 0), + 'waiting': waiting, + 'success_rate': round(completed / denominator * 100, 2) if denominator > 0 else None, + 'denominator': denominator, + 'p50_duration_ms': _percentile_ms(durations, 0.5), + 'p95_duration_ms': _percentile_ms(durations, 0.95), + 'duration_sample': len(durations), + 'debug': debug_count, + 'real': total - debug_count, + } + + async def get_executions( + self, + context: TenantContext, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + agent_ids: list[str] | None = None, + statuses: list[str] | None = None, + source: str = 'all', + mode: str = 'all', + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + limit: int = 50, + offset: int = 0, + ) -> dict: + """List executions from both object types (agent runs and pipeline queries).""" + workspace_uuid = require_workspace_uuid(context) + limit, offset = self.normalize_page_window(limit, offset) + fetch_limit = min(limit + offset, self._configured_query_limit('page_rows', 1000, 5000)) + if source not in {'all', 'agent', 'pipeline'}: + source = 'all' + if mode not in {'all', 'real', 'debug'}: + mode = 'all' + debug = None if mode == 'all' else (mode == 'debug') + + include_agent = source in {'all', 'agent'} and (not pipeline_ids or bool(agent_ids)) + include_pipeline = source in {'all', 'pipeline'} and (not agent_ids or bool(pipeline_ids)) + + items: list[dict] = [] + total = 0 + + if include_agent: + run_conditions = self._agent_run_conditions( + workspace_uuid, + bot_ids=bot_ids, + agent_ids=agent_ids, + statuses=self._raw_agent_statuses(statuses), + start_time=start_time, + end_time=end_time, + debug=debug, + ) + AgentRun = persistence_agent_run.AgentRun + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun) + .where(sqlalchemy.and_(*run_conditions)) + .order_by(AgentRun.created_at.desc(), AgentRun.id.desc()) + .limit(fetch_limit) + ) + rows = self._model_rows(result) + count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where(sqlalchemy.and_(*run_conditions)) + ) + total += count_result.scalar() or 0 + agent_names = await self._agent_names(workspace_uuid, rows) + items.extend(self._serialize_agent_execution(row, agent_names=agent_names) for row in rows) + + if include_pipeline and (debug is None or debug is False): + message_conditions = self._pipeline_execution_conditions( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + statuses=self._raw_pipeline_statuses(statuses), + start_time=start_time, + end_time=end_time, + ) + MonitoringMessage = persistence_monitoring.MonitoringMessage + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(MonitoringMessage) + .where(sqlalchemy.and_(*message_conditions)) + .order_by(MonitoringMessage.timestamp.desc()) + .limit(fetch_limit) + ) + rows = self._model_rows(result) + count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(MonitoringMessage.id)).where( + sqlalchemy.and_(*message_conditions) + ) + ) + total += count_result.scalar() or 0 + items.extend(self._serialize_pipeline_execution(row) for row in rows) + + items.sort(key=lambda item: item.get('created_at_ms') or 0, reverse=True) + page = items[offset : offset + limit] + return { + 'items': page, + 'total': total, + 'limit': limit, + 'offset': offset, + 'has_more': total > offset + len(page), + 'summary': { + 'executions': await self._get_execution_summary( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + ), + 'tokens': await self._get_token_coverage( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + ), + }, + } + + async def get_execution_detail( + self, + context: TenantContext, + source: str, + execution_id: str, + ) -> dict: + """Full trace of a single execution, keyed by object type.""" + workspace_uuid = require_workspace_uuid(context) + + if source == 'agent': + from ....agent.runner.run_ledger_store import RunLedgerStore + + store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine()) + run = await store.get_run(execution_id) + if run is None or run.get('workspace_id') != workspace_uuid: + raise ValueError('Execution not found') + items, next_cursor, _, has_more = await store.page_run_events(run_id=execution_id, limit=200) + return { + 'source': 'agent', + 'run': run, + 'events': items, + 'has_more': has_more, + 'next_cursor': next_cursor, + } + + if source == 'pipeline': + MonitoringMessage = persistence_monitoring.MonitoringMessage + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(MonitoringMessage).where( + sqlalchemy.and_( + MonitoringMessage.id == execution_id, + MonitoringMessage.workspace_uuid == workspace_uuid, + ) + ) + ) + rows = self._model_rows(result) + if not rows: + raise ValueError('Execution not found') + serialized = self.ap.persistence_mgr.serialize_model(MonitoringMessage, rows[0]) + + async def _related(model, order_column): + related = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(model) + .where( + sqlalchemy.and_( + model.workspace_uuid == workspace_uuid, + model.message_id == execution_id, + ) + ) + .order_by(order_column.asc()) + ) + return [self.ap.persistence_mgr.serialize_model(model, row) for row in self._model_rows(related)] + + return { + 'source': 'pipeline', + 'message': serialized, + 'llm_calls': await _related( + persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.timestamp + ), + 'tool_calls': await _related( + persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.timestamp + ), + 'errors': await _related( + persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.timestamp + ), + } + + raise ValueError('Unknown execution source') + async def get_token_statistics( self, context: TenantContext, diff --git a/src/langbot/pkg/pipeline/monitoring_helper.py b/src/langbot/pkg/pipeline/monitoring_helper.py index 15ae38f47..c4c7fc8f1 100644 --- a/src/langbot/pkg/pipeline/monitoring_helper.py +++ b/src/langbot/pkg/pipeline/monitoring_helper.py @@ -18,6 +18,103 @@ if typing.TYPE_CHECKING: from .pool import get_query_execution_context +async def resolve_processor_name( + ap: app.Application, + workspace_uuid: str, + processor_type: str, + processor_id: str | None, + runner_id: str = '', +) -> str: + """Display name of the processor kind that handled a run. + + Monitoring attributed every call to a Pipeline, so an Agent or a plugin event + processor recorded "Unknown" for the entity that actually ran. Each kind is + resolved from the table that owns it, falling back to the identity the + binding carried. + """ + import sqlalchemy + + from ..entity.persistence import agent as persistence_agent + from ..entity.persistence import pipeline as persistence_pipeline + + identifier = str(processor_id or '').strip() + if identifier: + try: + model = persistence_pipeline.LegacyPipeline if processor_type == 'pipeline' else persistence_agent.Agent + result = await ap.persistence_mgr.execute_async( + sqlalchemy.select(model.name).where( + sqlalchemy.and_( + model.uuid == identifier, + model.workspace_uuid == workspace_uuid, + ) + ) + ) + name = str(result.scalar() or '').strip() + if name: + return name + except Exception as e: + ap.logger.warning(f'Failed to resolve monitoring processor name: {e}') + # A deleted Agent or an unregistered plugin processor still names the + # component that ran, never a bare "Unknown". + return identifier or runner_id or processor_type + + +async def prepare_monitoring_identity( + ap: app.Application, + query: pipeline_query.Query, + *, + processor_type: str, + processor_id: str | None, + runner_id: str = '', + execution_record_id: str = '', +) -> None: + """Pin the monitoring identity of a runner-owned execution. + + A Pipeline pins these variables when it starts a query; a Runner started by + an Agent or a plugin event processor has to pin them itself, otherwise every + LLM/tool call it makes is attributed to "Unknown". An enclosing Pipeline + already named the run, so its label is never overwritten. + + ``_monitoring_message_id`` is the execution record the calls belong to: the + Pipeline's message, or the Agent run's id. Both are the id the executions + list shows, which is what lets a call jump back to its record. + """ + try: + variables = getattr(query, 'variables', None) + if not isinstance(variables, dict): + return + if variables.get('_monitoring_pipeline_name'): + return + + bot_name = 'WebChat' + bot_uuid = str(getattr(query, 'bot_uuid', '') or '') + workspace_uuid = str(getattr(query, 'workspace_uuid', '') or '') + if bot_uuid and workspace_uuid: + try: + bot = await ap.bot_service.get_bot( + workspace_uuid, + bot_uuid, + include_secret=False, + ) + if bot: + bot_name = bot.get('name', bot_name) + except Exception as e: + ap.logger.warning(f'Failed to resolve monitoring bot name: {e}') + + variables['_monitoring_bot_name'] = bot_name + variables['_monitoring_pipeline_name'] = await resolve_processor_name( + ap, + workspace_uuid, + processor_type, + processor_id, + runner_id, + ) + if execution_record_id: + variables['_monitoring_message_id'] = execution_record_id + except Exception as e: + ap.logger.error(f'Failed to prepare monitoring identity: {e}') + + class MonitoringHelper: """Helper class for monitoring operations""" diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index 26329dad0..97e554f42 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -2,6 +2,7 @@ from __future__ import annotations import typing import time +import asyncio import inspect from typing import TYPE_CHECKING @@ -448,6 +449,19 @@ class ToolManager: start_time = time.perf_counter() try: result = await invoke() + except asyncio.CancelledError: + duration_ms = int((time.perf_counter() - start_time) * 1000) + await self._record_tool_call( + name=name, + source=source, + parameters=parameters, + query=query, + duration_ms=duration_ms, + status='error', + error_message='cancelled', + ) + self._trace_tool_call(source=source, name=name, query=query, outcome='cancelled', error='cancelled') + raise except Exception as e: duration_ms = int((time.perf_counter() - start_time) * 1000) await self._record_tool_call( @@ -459,6 +473,14 @@ class ToolManager: status='error', error_message=str(e), ) + outcome = 'timeout' if isinstance(e, TimeoutError) else 'failed' + self._trace_tool_call( + source=source, + name=name, + query=query, + outcome=outcome, + error=str(e) or type(e).__name__, + ) raise duration_ms = int((time.perf_counter() - start_time) * 1000) @@ -471,8 +493,60 @@ class ToolManager: status='success', result=result, ) + self._trace_tool_call(source=source, name=name, query=query, outcome='success') return result + def _trace_tool_call( + self, + *, + source: str, + name: str, + query: pipeline_query.Query, + outcome: str, + error: str = '', + ) -> None: + """Report one tool invocation as its own step of the execution chain. + + Every tool call a Run makes - sandbox, skill, plugin or MCP - is the + observable action of that Run, so the chain shows it under the step that + issued it instead of collapsing the Run into a single opaque node. + + An Agent's tool call is executed for the Runtime's RPC task, which never + inherited the chain context. The Run id the caller already carries is the + alias that task's record resolves the owning execution through. + """ + # Imported here: telemetry pulls the application, and the tool manager is + # constructed while the application is still assembling. + from ...telemetry.execution import record as record_execution + from ...telemetry.platform import processing_mode + + try: + record_execution( + self.ap, + get_query_execution_context(query), + execution_id=self._trace_execution_id(query), + family='tool', + operation=name, + adapter=source, + mode=processing_mode.get(), + outcome=outcome, + error=error, + ) + except Exception as e: + self.ap.logger.warning(f'Failed to record tool call trace: {e}') + + @staticmethod + def _trace_execution_id(query: pipeline_query.Query) -> str: + """The execution chain this call belongs to, when the caller knows it. + + ``''`` lets the emitter fall back to the task's own trace context, which + is what a Pipeline-lane tool call has. + """ + session = getattr(query, '_agent_run_session', None) + if not isinstance(session, dict): + return '' + return str(session.get('run_id') or '').strip() + async def execute_func_call( self, name: str, diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index 1e5a16a1c..8a72288f2 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -33,7 +33,7 @@ from .trace import TraceState MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'}) OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'}) -FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}) +FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api', 'tool'}) ORIGINS = frozenset({'platform', 'webui', 'api'}) CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'}) CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'}) @@ -255,9 +255,9 @@ class ExecutionCounters: state = trace_mod.current() if state is None: # Cross-task work has no inherited context; resolve the owning - # execution through the registry. It has no open step here, so - # its node becomes a root of that execution instead of being - # mirrored from another task's stack. + # execution through the registry. It owns no step here, so the + # record attaches to the innermost step the chain has open - + # the step that caused this work. lookup = str(execution_id or '').strip() or current_execution_id() if lookup: state = self._resolve(lookup) diff --git a/src/langbot/pkg/telemetry/trace.py b/src/langbot/pkg/telemetry/trace.py index fb4d18031..6ae8da2d1 100644 --- a/src/langbot/pkg/telemetry/trace.py +++ b/src/langbot/pkg/telemetry/trace.py @@ -11,12 +11,17 @@ small execution-scoped context the emitter needs: * the routing/run identity pinned around the block that produced a node; * the open-step stack, which lives in a ContextVar holding *one task's own immutable tuple*. Child tasks inherit a snapshot at creation; no shared - mutable stack and no cross-task mirror exists, so sibling tasks can never - corrupt each other's parent links. + mutable stack exists, so sibling tasks can never corrupt each other's parent + links. Cross-task callers (plugin/RPC work with no inherited context) resolve their execution through the owner's execution-id registry instead, which only -carries the execution state - never node payloads. +carries the execution state - never node payloads. Such a caller owns no step +of its own, yet its work is still caused by a step of the chain: the sandbox +tool a Runner invoked, the platform API a plugin called back over RPC. The +execution state therefore also keeps the ordered set of steps any task +currently has open, and a record emitted with no open step of its own attaches +to the innermost one of those instead of becoming a root of the chain. """ from __future__ import annotations @@ -77,6 +82,7 @@ class TraceState: 'pipeline_plugins', 'node_adapter', 'node_runner', + 'open_frames', ) def __init__(self, execution_id: str | None = None) -> None: @@ -110,6 +116,10 @@ class TraceState: # First adapter/runner seen on a node, used for the chain record. self.node_adapter = '' self.node_runner = '' + # Steps currently open anywhere in this chain, in the order they were + # opened. A record emitted by a task that owns no step of its own + # attaches to the innermost entry here. + self.open_frames: dict[str, NodeFrame] = {} def allocate_seq(self) -> int: """Mint the next chain-local node order.""" @@ -117,6 +127,17 @@ class TraceState: self.node_seq += 1 return seq + def open_lane(self) -> str: + """The innermost step open in this chain, whichever task opened it.""" + frames = list(self.open_frames) + return frames[-1] if frames else '' + + def open_step(self, frame: NodeFrame) -> None: + self.open_frames[frame.node_id] = frame + + def close_step(self, frame: NodeFrame) -> None: + self.open_frames.pop(frame.node_id, None) + def note_node( self, *, @@ -221,8 +242,10 @@ def resolve_node( A record stamped with a ``node`` the calling task has open reuses that step's identity, order and start time; anything else (a plain observation, - or a node opened by another task) is a fresh node under this task's - innermost open step. + or a node opened by another task) is a fresh node under the innermost step + open in this task. A task that inherited no step at all - plugin/RPC work + that only knows the execution - attaches to the innermost step the chain + has open, which is the step that caused the work. """ stack = _node.get() if node: @@ -231,7 +254,7 @@ def resolve_node( if frame.node_id == node: parent = stack[index - 1].node_id if index > 0 else '' return frame.node_id, frame.seq, frame.started_at, parent - parent = stack[-1].node_id if stack else '' + parent = stack[-1].node_id if stack else state.open_lane() return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent @@ -250,9 +273,11 @@ def stage_scope() -> typing.Iterator[str]: return frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now()) token = _node.set(_node.get() + (frame,)) + state.open_step(frame) try: yield frame.node_id finally: + state.close_step(frame) try: _node.reset(token) except (ValueError, RuntimeError): diff --git a/tests/unit_tests/api/service/test_monitoring_executions.py b/tests/unit_tests/api/service/test_monitoring_executions.py new file mode 100644 index 000000000..cd8d767ec --- /dev/null +++ b/tests/unit_tests/api/service/test_monitoring_executions.py @@ -0,0 +1,350 @@ +"""Execution-view tests for the monitoring service. + +Covers the unified agent-run + pipeline-query execution list, its rollup +summary, and single-execution traces. +""" + +from __future__ import annotations + +import datetime +from types import SimpleNamespace + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker + +from langbot.pkg.api.http.service.monitoring import MonitoringService +from langbot.pkg.entity.persistence import agent_interaction as persistence_interaction +from langbot.pkg.entity.persistence import agent_run as persistence_agent_run +from langbot.pkg.entity.persistence import monitoring as persistence_monitoring +from langbot.pkg.entity.persistence import workspace as persistence_workspace # noqa: F401 +from langbot.pkg.entity.persistence.base import Base + +WORKSPACE = 'workspace-1' + + +class FakePersistenceManager: + """Runs statements against a real in-memory SQLite engine.""" + + def __init__(self, engine): + self._engine = engine + self._factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + def get_db_engine(self): + return self._engine + + async def execute_async(self, statement): + # Match PersistenceManager's connection-level result shape. + async with self._engine.connect() as connection: + return await connection.execute(statement) + + @staticmethod + def serialize_model(model, data): + out = {} + for column in model.__table__.columns: + value = getattr(data, column.name) + out[column.name] = value.isoformat() if isinstance(value, datetime.datetime) else value + return out + + +@pytest.fixture +async def engine(tmp_path): + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "monitoring_executions.db"}') + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield engine + await engine.dispose() + + +@pytest.fixture +def service(engine): + ap = SimpleNamespace( + persistence_mgr=FakePersistenceManager(engine), + instance_config=None, + ) + return MonitoringService(ap) + + +def _dt(hour: int, minute: int = 0, second: int = 0) -> datetime.datetime: + return datetime.datetime(2026, 1, 1, hour, minute, second) + + +async def _seed(engine) -> None: + factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with factory() as session: + session.add_all( + [ + persistence_agent_run.AgentRun( + run_id='run-done', + event_id='evt-1', + agent_id='agent-1', + binding_id='agent_agent-1_default', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='completed', + created_at=_dt(10), + started_at=_dt(10, 0, 1), + finished_at=_dt(10, 0, 3), + usage_json='{"total_tokens": 42}', + ), + persistence_agent_run.AgentRun( + run_id='run-failed', + binding_id='agent_agent-1_default', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='failed', + created_at=_dt(10, 5), + ), + persistence_agent_run.AgentRun( + run_id='run-debug', + event_id='debug:agent-1:e1', + agent_id='agent-1', + binding_id='debug:agent-1:runner-1', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='completed', + created_at=_dt(10, 6), + started_at=_dt(10, 6, 0), + finished_at=_dt(10, 6, 1), + ), + persistence_agent_run.AgentRunEvent( + run_id='run-done', + sequence=1, + type='run.completed', + data_json='{"message": {"role": "assistant", "content": "hi"}}', + ), + persistence_interaction.AgentInteraction( + interaction_id='int-1', + run_id='run-done', + binding_id='agent_agent-1_default', + runner_id='runner-1', + processor_type='agent', + processor_id='agent-1', + workspace_id=WORKSPACE, + status='pending', + request_json='{}', + callback_token_hash='hash-1', + ), + ] + ) + for message in [ + dict( + id='msg-user', + timestamp=_dt(10, 2), + session_id='session-1', + message_content='{"chain": [{"type": "Plain", "text": "hello there"}]}', + status='success', + role='user', + ), + dict( + id='msg-assistant', + timestamp=_dt(10, 3), + session_id='session-1', + message_content='{"chain": [{"type": "Plain", "text": "reply"}]}', + status='success', + role='assistant', + ), + dict( + id='msg-discarded', + timestamp=_dt(10, 4), + session_id='session-2', + message_content='{"chain": [{"type": "Plain", "text": "ignored"}]}', + status='discarded', + role='user', + ), + ]: + session.add( + persistence_monitoring.MonitoringMessage( + workspace_uuid=WORKSPACE, + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + level='info', + **message, + ) + ) + session.add_all( + [ + persistence_monitoring.MonitoringLLMCall( + id='llm-1', + workspace_uuid=WORKSPACE, + timestamp=_dt(10, 2, 30), + model_name='gpt', + input_tokens=10, + output_tokens=20, + total_tokens=30, + duration=100, + status='success', + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + session_id='session-1', + message_id='msg-user', + ), + persistence_monitoring.MonitoringLLMCall( + id='llm-2', + workspace_uuid=WORKSPACE, + timestamp=_dt(10, 4, 30), + model_name='gpt', + input_tokens=0, + output_tokens=0, + total_tokens=0, + duration=50, + status='success', + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + session_id='session-2', + message_id='msg-discarded', + ), + ] + ) + session.add( + persistence_monitoring.MonitoringSession( + workspace_uuid=WORKSPACE, + bot_id='bot-1', + session_id='session-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + message_count=2, + start_time=_dt(10, 2), + last_activity=_dt(10, 5), + is_active=True, + ) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_executions_merge_and_order_both_sources(engine, service): + await _seed(engine) + + result = await service.get_executions(WORKSPACE, limit=50) + + assert result['total'] == 5 + ids = [(item['source'], item['id']) for item in result['items']] + # Assistant replies are outputs, not executions. + assert ('pipeline', 'msg-assistant') not in ids + # Newest first across both object types. + assert ids == [ + ('agent', 'run-debug'), + ('agent', 'run-failed'), + ('pipeline', 'msg-discarded'), + ('pipeline', 'msg-user'), + ('agent', 'run-done'), + ] + + by_id = {item['id']: item for item in result['items']} + assert result['summary']['executions']['total'] == 5 + assert result['summary']['executions']['denominator'] == 4 + assert result['summary']['tokens']['calls'] == 2 + assert result['summary']['tokens']['calls_with_usage'] == 1 + assert by_id['run-done']['status_group'] == 'completed' + assert by_id['run-failed']['status_group'] == 'failed' + assert by_id['run-debug']['debug'] is True + assert by_id['run-done']['debug'] is False + assert by_id['run-done']['usage'] == {'total_tokens': 42} + assert by_id['run-done']['duration_ms'] == 2000 + assert by_id['msg-user']['status_group'] == 'completed' + assert by_id['msg-discarded']['status_group'] == 'ignored' + assert by_id['msg-user']['title'] == 'hello there' + assert by_id['msg-user']['target_id'] == 'pipeline-1' + + +@pytest.mark.asyncio +async def test_executions_source_status_and_paging(engine, service): + await _seed(engine) + + only_agents = await service.get_executions(WORKSPACE, source='agent') + assert {item['source'] for item in only_agents['items']} == {'agent'} + assert only_agents['total'] == 3 + + pipeline_only = await service.get_executions(WORKSPACE, source='pipeline') + assert {item['source'] for item in pipeline_only['items']} == {'pipeline'} + assert pipeline_only['total'] == 2 + + # A single-object target selection narrows the union to that source. + by_pipeline = await service.get_executions(WORKSPACE, pipeline_ids=['pipeline-1']) + assert {item['source'] for item in by_pipeline['items']} == {'pipeline'} + + failed = await service.get_executions(WORKSPACE, statuses=['failed']) + assert [item['id'] for item in failed['items']] == ['run-failed'] + + # `ignored` only exists for pipeline queries: it must not widen the agent + # filter back to "every status". + ignored = await service.get_executions(WORKSPACE, statuses=['ignored']) + assert [item['id'] for item in ignored['items']] == ['msg-discarded'] + + debug_only = await service.get_executions(WORKSPACE, mode='debug') + assert [item['id'] for item in debug_only['items']] == ['run-debug'] + + first_page = await service.get_executions(WORKSPACE, limit=2) + assert len(first_page['items']) == 2 + assert first_page['has_more'] is True + second_page = await service.get_executions(WORKSPACE, limit=2, offset=2) + assert first_page['items'][0]['id'] != second_page['items'][0]['id'] + + +@pytest.mark.asyncio +async def test_execution_summary_uses_explicit_denominator(engine, service): + await _seed(engine) + + summary = (await service.get_executions(WORKSPACE))['summary'] + executions = summary['executions'] + + # completed = run-done + run-debug + msg-user; failed = run-failed. + assert executions['by_source'] == {'agent': 3, 'pipeline': 2} + assert executions['completed'] == 3 + assert executions['failed'] == 1 + assert executions['ignored'] == 1 + assert executions['denominator'] == 4 + assert executions['success_rate'] == 75.0 + assert executions['debug'] == 1 + assert executions['real'] == 4 + assert executions['waiting'] == 1 + assert executions['p50_duration_ms'] == 1000 + assert executions['p95_duration_ms'] == 2000 + # Only run-done and run-debug are completed runs with a start and a finish. + assert executions['duration_sample'] == 2 + + # Token coverage is explicit: llm-2 succeeded without usage. + assert summary['tokens'] == {'total_tokens': 30, 'calls': 2, 'calls_with_usage': 1} + + +@pytest.mark.asyncio +async def test_overview_metrics_keep_message_rollup(engine, service): + await _seed(engine) + + overview = await service.get_overview_metrics(WORKSPACE) + + assert overview['total_messages'] == 3 + assert overview['llm_calls'] == 2 + assert overview['model_calls'] == 2 + assert overview['active_sessions'] == 1 + + +@pytest.mark.asyncio +async def test_execution_detail_for_agent_and_pipeline(engine, service): + await _seed(engine) + + agent_detail = await service.get_execution_detail(WORKSPACE, 'agent', 'run-done') + assert agent_detail['run']['run_id'] == 'run-done' + assert agent_detail['events'][0]['type'] == 'run.completed' + + pipeline_detail = await service.get_execution_detail(WORKSPACE, 'pipeline', 'msg-user') + assert pipeline_detail['message']['id'] == 'msg-user' + assert [call['id'] for call in pipeline_detail['llm_calls']] == ['llm-1'] + assert pipeline_detail['tool_calls'] == [] + + with pytest.raises(ValueError): + await service.get_execution_detail(WORKSPACE, 'agent', 'missing-run') + + with pytest.raises(ValueError): + await service.get_execution_detail(WORKSPACE, 'pipeline', 'missing-message') diff --git a/tests/unit_tests/provider/test_tool_manager.py b/tests/unit_tests/provider/test_tool_manager.py index d404400a4..0e81ae5e2 100644 --- a/tests/unit_tests/provider/test_tool_manager.py +++ b/tests/unit_tests/provider/test_tool_manager.py @@ -7,6 +7,8 @@ Tests cover: from __future__ import annotations +import asyncio +import contextvars from importlib import import_module from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -380,6 +382,127 @@ class TestToolManagerExecuteFuncCall: assert 'MCP resource agent reads are disabled' in record.await_args.kwargs['error_message'] +class _RecordingTelemetrySender: + """Telemetry sender stand-in that keeps every payload it was handed.""" + + def __init__(self): + self.telemetry_config = {'url': 'https://space.example.test', 'execution_trace': 'all'} + self.sent: list[dict] = [] + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +class TestToolCallTracing: + """Every tool call a Run makes is its own step of the execution chain.""" + + @pytest.fixture + def traced_app(self): + execution = import_module('langbot.pkg.telemetry.execution') + counters = execution.ExecutionCounters(_RecordingTelemetrySender()) + app = Mock() + app.logger = Mock() + app.telemetry = SimpleNamespace(execution=counters) + # Tool-call monitoring is asserted by the tests above; this path only + # needs the execution chain. + app.monitoring_service = None + + def _inert_loader(): + loader = Mock() + loader.has_tool = AsyncMock(return_value=False) + loader.invoke_tool = AsyncMock(return_value=None) + return loader + + app._native_loader = _inert_loader() + app._plugin_loader = _inert_loader() + app._mcp_loader = _inert_loader() + app._skill_loader = _inert_loader() + return app, counters + + @pytest.fixture + def sample_query(self): + query = Mock(spec=pipeline_query.Query) + query._execution_context = _CONTEXT + return query + + @staticmethod + def _manager(app): + """ToolManager wired to app-local inert loaders.""" + manager = get_toolmgr_module().ToolManager(app) + manager.native_tool_loader = app._native_loader + manager.plugin_tool_loader = app._plugin_loader + manager.mcp_tool_loader = app._mcp_loader + manager.skill_tool_loader = app._skill_loader + return manager + + @staticmethod + def _tool_nodes(counters): + return [record for record in counters.records if record['event_type'] == 'execution_node'] + + @pytest.mark.asyncio + async def test_a_tool_call_becomes_a_node_under_the_step_that_issued_it(self, traced_app, sample_query): + trace = import_module('langbot.pkg.telemetry.trace') + execution = import_module('langbot.pkg.telemetry.execution') + app, counters = traced_app + manager = self._manager(app) + + binding = execution.bind_trace(app, 'exec-tool') + # The Runtime's RPC task executes the call with no inherited trace + # context; the Run id on the query is what resolves the owning chain. + sample_query._agent_run_session = {'run_id': 'exec-tool'} + try: + with trace.stage_scope() as lane: + foreign = contextvars.Context() + task = asyncio.get_running_loop().create_task( + manager._invoke_tool_with_monitoring( + source='native', + name='exec', + parameters={'command': 'ls'}, + query=sample_query, + invoke=AsyncMock(return_value='sandbox output'), + ), + context=foreign, + ) + assert await task == 'sandbox output' + finally: + trace.unbind_root(binding) + + rows = self._tool_nodes(counters) + assert len(rows) == 1 + assert rows[0]['family'] == 'tool' + assert rows[0]['operation'] == 'exec' + assert rows[0]['adapter'] == 'native' + assert rows[0]['outcome'] == 'success' + assert rows[0]['parent_node_id'] == lane + + @pytest.mark.asyncio + async def test_a_failed_tool_call_carries_its_reason(self, traced_app, sample_query): + execution = import_module('langbot.pkg.telemetry.execution') + trace = import_module('langbot.pkg.telemetry.trace') + app, counters = traced_app + manager = self._manager(app) + + binding = execution.bind_trace(app, 'exec-tool-failed') + try: + with pytest.raises(RuntimeError): + await manager._invoke_tool_with_monitoring( + source='mcp', + name='read_resource', + parameters={}, + query=sample_query, + invoke=AsyncMock(side_effect=RuntimeError('server unreachable')), + ) + finally: + trace.unbind_root(binding) + + rows = self._tool_nodes(counters) + assert len(rows) == 1 + assert rows[0]['family'] == 'tool' + assert rows[0]['outcome'] == 'failed' + assert rows[0]['error'] == 'server unreachable' + + class TestToolManagerSourceResolution: @pytest.mark.asyncio @pytest.mark.parametrize('source', ['mcp', 'plugin']) diff --git a/tests/unit_tests/telemetry/test_trace.py b/tests/unit_tests/telemetry/test_trace.py index 95b9f06f1..381b0ee6b 100644 --- a/tests/unit_tests/telemetry/test_trace.py +++ b/tests/unit_tests/telemetry/test_trace.py @@ -344,10 +344,53 @@ class TestTraceNodeScope: lane_row, foreign_row = nodes(counters) assert lane_row['node_id'] == lane and lane_row['parent_node_id'] == '' and lane_row['root'] is True - # The cross-task node joins the same execution, but never mirrors another - # task's open-step stack: it is a root of the chain. + # The cross-task node joins the same execution and hangs off the step the + # chain had open - the step that caused the work - instead of mirroring + # any task's own stack. assert foreign_row['event_id'] == 'exec-cross' - assert foreign_row['parent_node_id'] == '' + assert foreign_row['parent_node_id'] == lane + assert foreign_row['root'] is False + + def test_cross_task_observation_attaches_to_the_innermost_open_step(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-nested') + try: + with trace.stage_scope() as outer: + counters.record(CONTEXT, **{**STAGE, 'node': outer}) + with trace.stage_scope() as inner: + counters.record(CONTEXT, **{**STAGE, 'node': inner}) + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-nested') + foreign.run(counters.record, CONTEXT, **{**STAGE, 'family': 'tool', 'operation': 'exec'}) + finally: + trace.unbind_root(binding) + + outer_row, inner_row, tool_row = nodes(counters) + assert outer_row['parent_node_id'] == '' + assert inner_row['parent_node_id'] == outer + # The lane that caused the cross-task work is the innermost one open. + assert tool_row['parent_node_id'] == inner + + def test_cross_task_observation_after_the_lane_closed_is_a_root(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-late') + try: + with trace.stage_scope() as lane: + counters.record(CONTEXT, **{**STAGE, 'node': lane}) + # A late callback arrives after the owning step returned: nothing is + # open anymore, so the node is a root of the chain rather than a + # child of a finished step. + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-late') + foreign.run(counters.record, CONTEXT, family='platform_api', operation='reply', mode='pipeline') + finally: + trace.unbind_root(binding) + + lane_row, late_row = nodes(counters) + assert late_row['parent_node_id'] == '' and late_row['root'] is False + assert lane_row['root'] is True def test_cross_task_observation_without_an_open_step_has_no_parent(self): trace, execution = get_modules() diff --git a/web/src/app/home/monitoring/components/ConversationTurnList.tsx b/web/src/app/home/monitoring/components/ConversationTurnList.tsx index afcaa1388..a1e61b03e 100644 --- a/web/src/app/home/monitoring/components/ConversationTurnList.tsx +++ b/web/src/app/home/monitoring/components/ConversationTurnList.tsx @@ -12,6 +12,7 @@ import { Wrench, } from 'lucide-react'; import { cn } from '@/lib/utils'; +import { Button } from '@/components/ui/button'; import { MessageContentRenderer } from './MessageContentRenderer'; import { ConversationTurn, @@ -507,10 +508,11 @@ export function ConversationTurnList({ key={call.id} className="border-t border-border/70 py-2 first:border-t-0 first:pt-0 last:pb-0" > - + {hasToolDetails && expandedToolCall && (
{exporting ? ( <> - - - - + {t('monitoring.export.exporting')} ) : ( diff --git a/web/src/app/home/monitoring/components/FeedbackCard.tsx b/web/src/app/home/monitoring/components/FeedbackCard.tsx index c2e93e52b..b5882b7f1 100644 --- a/web/src/app/home/monitoring/components/FeedbackCard.tsx +++ b/web/src/app/home/monitoring/components/FeedbackCard.tsx @@ -9,6 +9,7 @@ import { Heart, Smile, } from 'lucide-react'; +import { MetricCard } from './overview-cards/MetricCard'; interface FeedbackCardProps { title: string; @@ -23,6 +24,18 @@ interface FeedbackCardProps { loading?: boolean; } +// The variant only tints the icon tile; the card surface stays neutral so the +// feedback row matches the token-monitoring cards. +const variantAccents: Record< + NonNullable, + string +> = { + default: '#64748b', + success: '#10b981', + warning: '#f59e0b', + danger: '#ef4444', +}; + export function FeedbackCard({ title, value, @@ -32,89 +45,37 @@ export function FeedbackCard({ variant = 'default', loading = false, }: FeedbackCardProps) { - const variantStyles = { - default: 'bg-white dark:bg-[#2a2a2e] border-gray-200 dark:border-gray-700', - success: - 'bg-green-50 dark:bg-green-900/20 border-green-200 dark:border-green-800', - warning: - 'bg-yellow-50 dark:bg-yellow-900/20 border-yellow-200 dark:border-yellow-800', - danger: 'bg-red-50 dark:bg-red-900/20 border-red-200 dark:border-red-800', - }; - - const iconStyles = { - default: 'text-gray-500 dark:text-gray-400', - success: 'text-green-500 dark:text-green-400', - warning: 'text-yellow-500 dark:text-yellow-400', - danger: 'text-red-500 dark:text-red-400', - }; - const trendStyles = { up: 'text-green-500', - down: 'text-red-500', - neutral: 'text-gray-500', + down: 'text-destructive', + neutral: 'text-muted-foreground', }; - if (loading) { - return ( -
-
-
-
-
-
-
-
-
-
- ); - } - return ( -
-
-
-

- {title} -

-

- {value} -

- {subtitle && ( -

- {subtitle} -

- )} - {trend && ( -
- {trend.direction === 'up' && ( - - )} - {trend.direction === 'down' && ( - - )} - {trend.direction === 'neutral' && ( - - )} - - {trend.value > 0 ? '+' : ''} - {trend.value}% - -
- )} -
+ {trend && (
- {icon} + {trend.direction === 'up' && } + {trend.direction === 'down' && ( + + )} + {trend.direction === 'neutral' && } + + {trend.value > 0 ? '+' : ''} + {trend.value}% +
-
-
+ )} + ); } @@ -135,35 +96,37 @@ export function FeedbackStatsCards({ stats, loading }: FeedbackStatsProps) { { title: t('monitoring.feedback.totalFeedback'), value: stats?.totalFeedback ?? 0, - icon: , + icon: , variant: 'default' as const, }, { title: t('monitoring.feedback.totalLikes'), value: stats?.totalLikes ?? 0, - icon: , + icon: , variant: 'success' as const, }, { title: t('monitoring.feedback.totalDislikes'), value: stats?.totalDislikes ?? 0, - icon: , + icon: , variant: 'danger' as const, }, { title: t('monitoring.feedback.satisfactionRate'), value: stats ? `${stats.satisfactionRate}%` : '0%', - icon: , + icon: , + // The rate is the one feedback metric that is good or bad on its own, so + // its tile keeps encoding the verdict (as it did before the restyle). variant: (stats && stats.satisfactionRate >= 80 ? 'success' : stats && stats.satisfactionRate >= 50 ? 'warning' - : 'danger') as 'default' | 'success' | 'warning' | 'danger', + : 'danger') as 'success' | 'warning' | 'danger', }, ]; return ( -
+
{cards.map((card, index) => ( - -
- ); + return ; } if (!feedback || feedback.length === 0) { return ( -
- -

- {t('monitoring.feedback.noFeedback')} -

-

- {t('monitoring.feedback.noFeedbackDescription')} -

-
+ } + title={t('monitoring.feedback.noFeedback')} + hint={t('monitoring.feedback.noFeedbackDescription')} + /> ); } return (
{feedback.map((item) => ( -
{item.botName && ( <> - → - + → + {item.botName} )} {item.platform && ( - + {item.platform} - + )} {item.streamId && onViewMessage && ( - ))} -
+ { + if (value === 'hour' || value === 'day') setBucket(value); + }} + > + + {(['hour', 'day'] as const).map((b) => ( + + {t(`monitoring.tokens.bucket.${b}`)} + + ))} + +
@@ -372,93 +334,86 @@ export default function TokenMonitoring({
-
+ {/* Per-model breakdown */} -
+

{t('monitoring.tokens.byModel')}

-
- - - - - - - - - - - - - - {by_model.map((m) => { - const share = - summary.total_tokens > 0 - ? (m.total_tokens / summary.total_tokens) * 100 - : 0; - return ( - - - - - - - - - - ); - })} - -
- {t('monitoring.tokens.model')} - - {t('monitoring.tokens.calls')} - - {t('monitoring.tokens.inputTokens')} - - {t('monitoring.tokens.outputTokens')} - - {t('monitoring.tokens.totalTokens')} - - {t('monitoring.tokens.avgPerCall')} - - {t('monitoring.tokens.avgLatency')} -
-
- {m.model_name} -
-
-
-
-
- {m.calls} - {m.error_calls > 0 && ( - - {' '} - ({m.error_calls}✕) - - )} - - {formatNumber(m.input_tokens)} - - {formatNumber(m.output_tokens)} - - {formatNumber(m.total_tokens)} - - {formatNumber(m.avg_tokens_per_call)} - - {m.avg_duration_ms}ms -
-
-
+ + + + {t('monitoring.tokens.model')} + + {t('monitoring.tokens.calls')} + + + {t('monitoring.tokens.inputTokens')} + + + {t('monitoring.tokens.outputTokens')} + + + {t('monitoring.tokens.totalTokens')} + + + {t('monitoring.tokens.avgPerCall')} + + + {t('monitoring.tokens.avgLatency')} + + + + + {by_model.map((m) => { + const share = + summary.total_tokens > 0 + ? (m.total_tokens / summary.total_tokens) * 100 + : 0; + return ( + + +
+ {m.model_name} +
+
+
+
+ + + {m.calls} + {m.error_calls > 0 && ( + + {' '} + ({m.error_calls}✕) + + )} + + + {formatNumber(m.input_tokens)} + + + {formatNumber(m.output_tokens)} + + + {formatNumber(m.total_tokens)} + + + {formatNumber(m.avg_tokens_per_call)} + + + {m.avg_duration_ms}ms + + + ); + })} + +
+
); } diff --git a/web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx b/web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx new file mode 100644 index 000000000..f0ad3c947 --- /dev/null +++ b/web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx @@ -0,0 +1,361 @@ +import { useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Link } from 'react-router-dom'; +import { ExternalLink } from 'lucide-react'; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Skeleton } from '@/components/ui/skeleton'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { Separator } from '@/components/ui/separator'; +import { backendClient } from '@/app/infra/http'; +import type { + ExecutionCallRow, + ExecutionDetail, + ExecutionRow, +} from '@/app/infra/entities/api/monitoring-executions'; +import AgentExecutionTrace from '@/app/home/agents/components/AgentExecutionTrace'; +import { formatRunDuration } from '@/app/home/agents/components/processor-run-timing'; +import { formatDateTime } from '../../utils/dateUtils'; +import { + executionKindLabel, + executionProcessorLabel, + executionStatusLabel, + executionStatusVariant, + executionTargetLabel, +} from './ExecutionTable'; + +interface ExecutionDetailSheetProps { + row: ExecutionRow | null; + onClose: () => void; +} + +/** Walk parsed JSON, collecting every `"text"` value (OpenAI-style content). */ +function collectText(value: unknown): string { + if (typeof value === 'string') return value; + if (Array.isArray(value)) { + return value.map(collectText).filter(Boolean).join(' '); + } + if (value && typeof value === 'object') { + const record = value as Record; + if (typeof record.text === 'string') return record.text; + return Object.values(record).map(collectText).filter(Boolean).join(' '); + } + return ''; +} + +function messagePreview(raw: string | null | undefined): string { + if (!raw) return ''; + try { + return collectText(JSON.parse(raw)).trim() || raw; + } catch { + return raw; + } +} + +function callTitle(call: ExecutionCallRow, index: number): string { + const candidates = [ + call.model_name, + call.model, + call.name, + call.tool_name, + call.function, + call.error_type, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate) return candidate; + } + return typeof call.id === 'string' && call.id ? call.id : `#${index + 1}`; +} + +export default function ExecutionDetailSheet({ + row, + onClose, +}: ExecutionDetailSheetProps) { + const { t } = useTranslation(); + const [detail, setDetail] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(false); + const requestRef = useRef(0); + + useEffect(() => { + if (!row) { + setDetail(null); + setError(false); + setLoading(false); + return; + } + const requestId = requestRef.current + 1; + requestRef.current = requestId; + setLoading(true); + setError(false); + setDetail(null); + backendClient + .getExecutionDetail(row.source, row.id) + .then((result) => { + if (requestRef.current !== requestId) return; + setDetail(result); + }) + .catch(() => { + if (requestRef.current !== requestId) return; + setError(true); + }) + .finally(() => { + if (requestRef.current === requestId) setLoading(false); + }); + }, [row]); + + const isAgent = + detail != null ? detail.source === 'agent' : row?.source === 'agent'; + + const metadata: { label: string; value: string }[] = []; + if (row) { + metadata.push({ + label: t('monitoring.execution.detail.executionId'), + value: row.id, + }); + if (row.event_id) { + metadata.push({ + label: t('monitoring.execution.detail.eventId'), + value: row.event_id, + }); + } + if (row.runner_id) { + metadata.push({ + label: t('monitoring.execution.detail.runner'), + value: row.runner_id, + }); + } + if (row.queue_name) { + metadata.push({ + label: t('monitoring.execution.detail.queue'), + value: row.queue_name, + }); + } + if (row.created_at_ms != null) { + metadata.push({ + label: t('monitoring.execution.detail.created'), + value: formatDateTime(new Date(row.created_at_ms)), + }); + } + if (row.duration_ms != null) { + metadata.push({ + label: t('monitoring.execution.detail.duration'), + value: formatRunDuration(row.duration_ms), + }); + } + if (isAgent && row.usage?.total_tokens != null) { + metadata.push({ + label: t('monitoring.execution.detail.usage'), + value: row.usage.total_tokens.toLocaleString(), + }); + } + } + + const objectHref = row + ? row.target_kind === 'agent' && row.target_id + ? `/home/agents?id=${encodeURIComponent(row.target_id)}` + : row.target_kind === 'pipeline' && row.target_id + ? `/home/pipelines?id=${encodeURIComponent(row.target_id)}` + : null + : null; + + const renderCalls = (title: string, calls: ExecutionCallRow[]) => ( +
+

{title}

+ {calls.length === 0 ? ( +

+ {t('monitoring.execution.detail.noEvents')} +

+ ) : ( +
+ {calls.map((call, index) => { + const duration = + typeof call.duration === 'number' ? call.duration : null; + return ( +
+
+
{callTitle(call, index)}
+ {call.error_message && ( +
+ {call.error_message} +
+ )} +
+
+ {call.status && ( + + {call.status} + + )} + + {duration == null ? '—' : formatRunDuration(duration)} + +
+
+ ); + })} +
+ )} +
+ ); + + return ( + { + if (!open) onClose(); + }} + > + + + {t('monitoring.execution.detail.a11yDescription')} + + +
+ {row && ( + <> + + + {executionProcessorLabel(row)} + +

+ {t('monitoring.execution.triggeredBy', { + trigger: executionTargetLabel(row, t), + })} +

+
+ + {executionKindLabel(row, t)} + + + {executionStatusLabel(row, t)} + + {row.debug && ( + + {t('monitoring.execution.debug')} + + )} + {objectHref && ( + + )} +
+ + +
+ {metadata.map((item) => ( +
+
+ {item.label} +
+
+ {item.value} +
+
+ ))} +
+ + + + {isAgent ? ( +
+

+ {t('monitoring.execution.detail.timeline')} +

+ {loading && ( +
+ + +
+ )} + {!loading && error && ( +

+ {t('monitoring.execution.detail.loadError')} +

+ )} + {!loading && + !error && + detail?.source === 'agent' && + (detail.events.length > 0 ? ( + + ) : ( +

+ {t('monitoring.execution.detail.noEvents')} +

+ ))} +
+ ) : ( +
+ {loading && ( +
+ + +
+ )} + {!loading && error && ( +

+ {t('monitoring.execution.detail.loadError')} +

+ )} + {!loading && !error && detail?.source === 'pipeline' && ( + <> +
+

+ {t('monitoring.execution.detail.query')} +

+

+ {messagePreview(detail.message.message_content) || + '—'} +

+
+ {renderCalls( + t('monitoring.execution.detail.modelCalls'), + detail.llm_calls, + )} + {renderCalls( + t('monitoring.execution.detail.toolCalls'), + detail.tool_calls, + )} + {renderCalls( + t('monitoring.execution.detail.errors'), + detail.errors, + )} + + )} +
+ )} + + )} +
+ + + + ); +} diff --git a/web/src/app/home/monitoring/components/executions/ExecutionTable.tsx b/web/src/app/home/monitoring/components/executions/ExecutionTable.tsx new file mode 100644 index 000000000..901adef5d --- /dev/null +++ b/web/src/app/home/monitoring/components/executions/ExecutionTable.tsx @@ -0,0 +1,169 @@ +import React from 'react'; +import { useTranslation } from 'react-i18next'; +import type { TFunction } from 'i18next'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { Badge } from '@/components/ui/badge'; +import type { ExecutionRow } from '@/app/infra/entities/api/monitoring-executions'; +import { eventPatternLabel } from '@/app/home/components/event-patterns/event-pattern-groups'; +import { formatRunDuration } from '@/app/home/agents/components/processor-run-timing'; +import { formatDateTime } from '../../utils/dateUtils'; + +interface ExecutionTableProps { + rows: ExecutionRow[]; + selectedId?: string | null; + onSelect: (row: ExecutionRow) => void; +} + +const STATUS_GROUPS = [ + 'completed', + 'failed', + 'running', + 'queued', + 'cancelled', + 'ignored', +]; + +export function executionStatusLabel(row: ExecutionRow, t: TFunction): string { + if (STATUS_GROUPS.includes(row.status_group)) { + return t(`monitoring.execution.status.${row.status_group}`); + } + return row.status; +} + +export function executionStatusVariant( + statusGroup: string, +): 'default' | 'secondary' | 'destructive' | 'outline' { + if (statusGroup === 'failed') return 'destructive'; + if (statusGroup === 'ignored' || statusGroup === 'queued') return 'secondary'; + return 'outline'; +} + +const KIND_GROUPS = ['agent', 'pipeline', 'processor']; + +export function executionKindLabel(row: ExecutionRow, t: TFunction): string { + if (KIND_GROUPS.includes(row.target_kind)) { + return t(`monitoring.execution.kind.${row.target_kind}`); + } + return row.target_kind; +} + +export function executionTargetLabel(row: ExecutionRow, t: TFunction): string { + if (row.target_kind === 'agent') { + return eventPatternLabel(row.title, t); + } + const title = row.title ?? ''; + return title.length > 48 ? `${title.slice(0, 48)}…` : title; +} + +// The processor the row ran: the Agent or Pipeline that handled it. A run whose +// Agent was deleted still names the Runner that executed it. +export function executionProcessorLabel(row: ExecutionRow): string { + return row.target_name || row.runner_id || row.target_id || '—'; +} + +export default function ExecutionTable({ + rows, + selectedId, + onSelect, +}: ExecutionTableProps) { + const { t } = useTranslation(); + + return ( + // The page hands this table a fixed-height region: the table's own scroll + // container takes that height so the header stays sticky while rows scroll + // under it. +
+ + + + {t('monitoring.execution.columns.time')} + {t('monitoring.execution.columns.target')} + {t('monitoring.execution.columns.status')} + + {t('monitoring.execution.columns.duration')} + + + {t('monitoring.execution.columns.tokens')} + + + + + {rows.map((row) => { + const selected = selectedId === row.id; + return ( + onSelect(row)} + onKeyDown={( + event: React.KeyboardEvent, + ) => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + onSelect(row); + } + }} + > + + {row.created_at_ms == null + ? '—' + : formatDateTime(new Date(row.created_at_ms))} + + +
+
+ + {executionKindLabel(row, t)} + + + {executionProcessorLabel(row)} + +
+ + {t('monitoring.execution.triggeredBy', { + trigger: executionTargetLabel(row, t), + })} + +
+ + +
+ + {executionStatusLabel(row, t)} + + {row.debug && ( + + {t('monitoring.execution.debug')} + + )} +
+ + + {row.duration_ms == null + ? '—' + : formatRunDuration(row.duration_ms)} + + + {row.usage?.total_tokens != null + ? row.usage.total_tokens.toLocaleString() + : '—'} + + + ); + })} + +
+
+ ); +} diff --git a/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx b/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx index 6cf204e14..2470451f6 100644 --- a/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx +++ b/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx @@ -7,8 +7,13 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; +import { Label } from '@/components/ui/label'; import { backendClient } from '@/app/infra/http'; import { TimeRangeOption } from '../../types/monitoring'; +import type { + ExecutionModeFilter, + ExecutionSourceFilter, +} from '@/app/infra/entities/api/monitoring-executions'; interface MonitoringFiltersProps { selectedBots: string[]; @@ -17,8 +22,23 @@ interface MonitoringFiltersProps { onBotsChange: (bots: string[]) => void; onPipelinesChange: (pipelines: string[]) => void; onTimeRangeChange: (timeRange: TimeRangeOption) => void; + source?: ExecutionSourceFilter; + onSourceChange?: (value: ExecutionSourceFilter) => void; + mode?: ExecutionModeFilter; + onModeChange?: (value: ExecutionModeFilter) => void; + statusGroup?: string; + onStatusGroupChange?: (value: string) => void; } +const STATUS_GROUP_OPTIONS = [ + 'completed', + 'failed', + 'running', + 'queued', + 'cancelled', + 'ignored', +]; + interface Bot { uuid: string; name: string; @@ -36,6 +56,12 @@ export default function MonitoringFilters({ onBotsChange, onPipelinesChange, onTimeRangeChange, + source, + onSourceChange, + mode, + onModeChange, + statusGroup, + onStatusGroupChange, }: MonitoringFiltersProps) { const { t } = useTranslation(); const [bots, setBots] = useState([]); @@ -110,11 +136,98 @@ export default function MonitoringFilters({ return (
+ {/* Source Filter */} + {onSourceChange && ( +
+ + +
+ )} + + {/* Mode Filter */} + {onModeChange && ( +
+ + +
+ )} + + {/* Status Filter */} + {onStatusGroupChange && ( +
+ + +
+ )} + {/* Bot Filter */}
- - - onSourceChange(value as ExecutionSourceFilter) - } +
+ {/* Bot Filter */} +
+ + +
+ + {/* Processor Filter */} +
+ + -
- )} + ))} + + +
{/* Mode Filter */} {onModeChange && ( -
-
-
+ + onSelect(value === 'new' ? null : value) + } + > + + + + + + {t('assistant.newChat')} + + {conversation && + !sessions.some( + (session) => session.uuid === conversation.uuid, + ) && ( + + {t('assistant.currentChat')} + + )} + {sessions.map((session) => ( + + + {session.status === 'running' + ? '◌ ' + : session.status === 'approval' + ? '… ' + : ''} + {session.title || t('assistant.newChat')} + + + ))} + +
{ + if (event.deltaY < 0) followLatest.current = false; + }} + onScroll={(event) => { + const list = event.currentTarget; + const movingUp = list.scrollTop < scrollPosition.current - 1; + followLatest.current = + !movingUp && + list.scrollHeight - list.clientHeight - list.scrollTop < 32; + scrollPosition.current = list.scrollTop; + }} aria-live="polite" > {!conversation?.messages.length && !pendingText && ( - <> -

+

+

+ {t('assistant.subtitle')} +

+

{t('assistant.welcome')}

- {(['discover', 'build'] as const).map((key) => ( - - ))} - - )} - {conversation?.messages.map((message, index) => - message.role === 'tool' ? ( - - ) : ( -
- null, - a: ({ href, children }) => ( - - {children} - - ), - }} - > - {message.content} - +
+ {(['discover', 'build'] as const).map((key) => ( + + ))}
- ), +
)} + {messageGroups.map((group) => ( + + ))} {pendingText && ( -
+
{pendingText} {error && (

@@ -691,14 +780,28 @@ function AssistantPanel({ storageKey }: { storageKey: string }) { {busy && (

- {t('assistant.working')} + {loading + ? t('assistant.preparing') + : t( + phase === 'tool' + ? 'assistant.executing' + : 'assistant.thinking', + { round }, + )} +

+ )} + {modelError && !modelUuid && ( +

+ {t('assistant.recommendationFailed')}

)} {(error || conversation?.status === 'failed') && (

- {conversation?.error === 'model_unavailable' - ? t('assistant.modelUnavailable') - : t('assistant.error')} + {conversation?.error === 'stopped' + ? t('assistant.stopped') + : conversation?.error === 'model_unavailable' + ? t('assistant.modelUnavailable') + : t('assistant.error')}

)} {!busy && conversation?.status === 'running' && ( @@ -706,16 +809,16 @@ function AssistantPanel({ storageKey }: { storageKey: string }) { {t('assistant.running')}

)} -
{ event.preventDefault(); void submit(); }} >