Merge origin/master into feat/totp-two-factor-auth

Resolve the revision-graph conflict introduced by the release line.

Conflicts
- tests/integration/persistence/test_rag_document_identity.py: master had
  independently introduced the same dynamic-head helper (`current_head`) plus
  a `DOCUMENT_IDENTITY_REVISION` constant, and it explicitly upgrades to that
  revision. Keep master's semantics (the explicit revision matters because
  upgrading to the head now also traverses the TOTP branch) while keeping the
  module-level alembic imports on this branch.

New migration
- 0030_merge_totp_into_release joins the TOTP branch's own merge revision
  (0026_merge_totp_and_rag_identity) with the release head
  (0029_merge_rag_identity). Both reached 0025_rag_document_identity without
  including each other, which left Alembic with two heads and made
  `upgrade head` fail with "Multiple head revisions are present".

Verification
- Single Alembic head confirmed (0030_merge_totp_into_release).
- tests/integration/persistence/test_rag_document_identity.py: 34 passed.
- Full fast integration suite: 356 passed, 84 skipped.
This commit is contained in:
TyperBody
2026-09-24 02:27:37 +08:00
974 changed files with 149674 additions and 20971 deletions
+11 -1
View File
@@ -27,6 +27,7 @@ from ..authz import (
)
from ..context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
from ....cloud.support_admin import SupportAdminSessionError
from ... import management_diagnostics as diagnostics
if typing.TYPE_CHECKING:
from ....core.app import Application
@@ -224,6 +225,7 @@ class RouterGroup(abc.ABC):
try:
if request_context is not None:
diagnostics.workspace(request_context)
with bounded_executor.blocking_work_scope(request_context.workspace_uuid):
persistence_mgr = getattr(
self.ap,
@@ -277,7 +279,14 @@ class RouterGroup(abc.ABC):
)
return self.internal_error_response(request_id)
new_f = handler_error
# Observe outside authentication, using the registered Core handler
# identity rather than the URL (which can contain user identifiers).
new_f = diagnostics.observe(
diagnostics.operation_id('http', f, rule=rule, methods=options.get('methods')),
source='http',
ap=self.ap,
http=True,
)(handler_error)
# Quart/Flask requires a unique endpoint name even when the same URL
# intentionally has separate handlers for different HTTP methods.
# Include the method set so CRUD routes can declare distinct
@@ -564,6 +573,7 @@ class RouterGroup(abc.ABC):
def fail(self, code: int | str, msg: str) -> quart.Response:
"""Return an error response"""
diagnostics.outcome('failed')
return quart.jsonify(
{
'code': code,
@@ -0,0 +1,72 @@
"""Bounded, cancellable NDJSON transport for Agent debug execution."""
from __future__ import annotations
import asyncio
import contextlib
import json
import quart
from .... import management_diagnostics as diagnostics
from .....agent.runner.errors import (
RunnerError,
RunnerExecutionError,
RunnerNotAuthorizedError,
RunnerNotFoundError,
RunnerProtocolError,
)
def debug_stream_response(service, context, agent_uuid: str, payload: dict) -> quart.Response:
async def stream():
queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=32)
async def on_result(result: dict) -> None:
await queue.put({'kind': 'result', 'data': result})
async def execute() -> None:
try:
result = await service.debug_agent(context, agent_uuid, payload, on_result=on_result)
await queue.put({'kind': 'completed', 'data': result})
except Exception as exc:
# The stream still uses HTTP 200 when execution returns an
# error frame. Mark the inherited request, not its contents.
diagnostics.outcome('failed')
if isinstance(exc, RunnerExecutionError):
code, message = exc.error_code or 'runner_execution_failed', exc.message
elif isinstance(exc, RunnerNotFoundError):
code, message = 'runner_not_found', 'The configured Agent runner is unavailable'
elif isinstance(exc, RunnerNotAuthorizedError):
code, message = 'runner_not_authorized', 'The configured Agent runner is not authorized'
elif isinstance(exc, RunnerProtocolError):
code, message = 'runner_protocol_error', 'The Agent runner returned an invalid response'
elif isinstance(exc, ValueError):
code, message = 'invalid_request', str(exc)
elif isinstance(exc, RunnerError):
code, message = 'runner_error', 'The Agent runner could not complete this test'
else:
code, message = 'runner_error', 'The Agent debug execution failed'
await queue.put({'kind': 'error', 'code': code, 'msg': message})
task = asyncio.create_task(execute())
try:
while True:
try:
frame = await asyncio.wait_for(queue.get(), timeout=15)
except TimeoutError:
yield '\n'
continue
yield json.dumps(frame, ensure_ascii=False) + '\n'
if frame['kind'] in {'completed', 'error'}:
break
finally:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
response = quart.Response(stream(), content_type='application/x-ndjson; charset=utf-8')
response.timeout = None
response.headers['Cache-Control'] = 'no-store'
response.headers['X-Accel-Buffering'] = 'no'
return response
@@ -0,0 +1,179 @@
from __future__ import annotations
import quart
from .....agent.runner.errors import (
RunnerError,
RunnerExecutionError,
RunnerNotAuthorizedError,
RunnerNotFoundError,
RunnerProtocolError,
)
from ...authz import Permission, require_permission
from ...context import RequestContext
from .. import group
from .agent_debug_stream import debug_stream_response
@group.group_class('agents', '/api/v1/agents')
class AgentsRouterGroup(group.RouterGroup):
async def initialize(self) -> None:
@self.route(
'/<agent_uuid>/runs',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def processor_runs(agent_uuid: str, request_context: RequestContext):
try:
cursor = quart.request.args.get('before_id')
result = await self.ap.agent_service.get_processor_runs(
request_context,
agent_uuid,
before_id=int(cursor) if cursor else None,
)
return self.success(data=result)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'/<agent_uuid>/runs/<run_id>/events',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def processor_run_events(agent_uuid: str, run_id: str, request_context: RequestContext):
try:
cursor = quart.request.args.get('after_sequence')
result = await self.ap.agent_service.get_processor_run_events(
request_context,
agent_uuid,
run_id,
after_sequence=int(cursor) if cursor else None,
)
return self.success(data=result)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'/<agent_uuid>/debug/stream',
methods=['POST'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RUNTIME_OPERATE,
)
async def stream_debug(agent_uuid: str, request_context: RequestContext):
payload = await quart.request.get_json()
if not isinstance(payload, dict):
return self.http_status(400, -1, 'Debug payload must be an object')
return debug_stream_response(self.ap.agent_service, request_context, agent_uuid, payload)
@self.route(
'',
methods=['GET', 'POST'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def _(request_context: RequestContext) -> str:
if quart.request.method == 'GET':
sort_by = quart.request.args.get('sort_by', 'updated_at')
sort_order = quart.request.args.get('sort_order', 'DESC')
return self.success(
data={
'agents': await self.ap.agent_service.get_agents(
request_context,
sort_by,
sort_order,
)
}
)
json_data = await quart.request.json
require_permission(request_context, Permission.RESOURCE_MANAGE)
try:
created = await self.ap.agent_service.create_agent(request_context, json_data)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success(data=created)
@self.route(
'/_/metadata',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def _(request_context: RequestContext) -> str:
return self.success(data=await self.ap.agent_service.get_agent_metadata(request_context))
@self.route(
'/<agent_uuid>/debug',
methods=['POST'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RUNTIME_OPERATE,
)
async def _(agent_uuid: str, request_context: RequestContext) -> str:
json_data = await quart.request.json
try:
result = await self.ap.agent_service.debug_agent(
request_context,
agent_uuid,
json_data or {},
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
except RunnerExecutionError as exc:
return self.http_status(
422,
exc.error_code or 'runner_execution_failed',
exc.message,
)
except RunnerNotFoundError:
return self.http_status(
409,
'runner_not_found',
'The configured Agent runner is unavailable',
)
except RunnerNotAuthorizedError:
return self.http_status(
403,
'runner_not_authorized',
'The configured Agent runner is not authorized',
)
except RunnerProtocolError:
return self.http_status(
502,
'runner_protocol_error',
'The Agent runner returned an invalid response',
)
except RunnerError:
return self.http_status(
502,
'runner_error',
'The Agent runner could not complete this test',
)
return self.success(data=result)
@self.route(
'/<agent_uuid>',
methods=['GET', 'PUT', 'DELETE'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def _(agent_uuid: str, request_context: RequestContext) -> str:
if quart.request.method == 'GET':
agent = await self.ap.agent_service.get_agent(request_context, agent_uuid)
if agent is None:
return self.http_status(404, -1, 'agent not found')
return self.success(data={'agent': agent})
if quart.request.method == 'PUT':
require_permission(request_context, Permission.RESOURCE_MANAGE)
json_data = await quart.request.json
try:
await self.ap.agent_service.update_agent(request_context, agent_uuid, json_data)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success()
require_permission(request_context, Permission.RESOURCE_MANAGE)
await self.ap.agent_service.delete_agent(request_context, agent_uuid)
return self.success()
@@ -3,7 +3,7 @@
All user-facing URLs are keyed by **bot_uuid** (not pipeline_uuid) so that
internal pipeline identifiers are never exposed to end-users. Each handler
resolves the bot_uuid to the owning ``web_page_bot`` RuntimeBot and extracts
the bound pipeline_uuid for internal routing.
the message event route's Pipeline target for internal routing.
"""
import asyncio
@@ -13,6 +13,7 @@ import logging
import uuid
import hmac
import hashlib
import os
import time
import re
import httpx
@@ -20,6 +21,7 @@ import httpx
import quart
from ... import group
from ..... import management_diagnostics as diagnostics
from ......utils import httpclient, paths
from ......platform.sources.websocket_manager import WebSocketScope, is_valid_session_id, ws_connection_manager
from .websocket_chat import create_scoped_duplex_tasks, wait_for_duplex_tasks
@@ -29,6 +31,7 @@ _AUTH_TIMEOUT_SECONDS = 10.0
# Cache the widget template content
_widget_template_cache: str | None = None
_widget_template_cache_mtime_ns: int | None = None
_logo_bytes_cache: bytes | None = None
@@ -37,12 +40,14 @@ def _is_valid_uuid(s: str) -> bool:
def _get_widget_template() -> str:
"""Load and cache the widget JS template."""
global _widget_template_cache
if _widget_template_cache is None:
template_path = paths.get_resource_path('templates/embed/widget.js')
"""Load the widget template and refresh the cache when the file changes."""
global _widget_template_cache, _widget_template_cache_mtime_ns
template_path = paths.get_resource_path('templates/embed/widget.js')
template_mtime_ns = os.stat(template_path).st_mtime_ns
if _widget_template_cache is None or _widget_template_cache_mtime_ns != template_mtime_ns:
with open(template_path, 'r', encoding='utf-8') as f:
_widget_template_cache = f.read()
_widget_template_cache_mtime_ns = template_mtime_ns
return _widget_template_cache
@@ -64,16 +69,12 @@ class EmbedRouterGroup(group.RouterGroup):
"""Resolve *bot_uuid* to ``(runtime_bot, pipeline_uuid)``.
Returns ``(None, None)`` when the bot does not exist, is not a
``web_page_bot``, is disabled, or has no pipeline bound.
``web_page_bot``, is disabled, or has no Pipeline target for messages.
"""
bot = await self.ap.platform_mgr.resolve_public_bot(bot_uuid)
if (
bot is not None
and bot.bot_entity.adapter == 'web_page_bot'
and bot.bot_entity.enable
and bot.bot_entity.use_pipeline_uuid
):
return bot, bot.bot_entity.use_pipeline_uuid
pipeline_uuid = bot.get_pipeline_target_for_event_type('message.received') if bot is not None else None
if bot is not None and bot.bot_entity.adapter == 'web_page_bot' and bot.bot_entity.enable and pipeline_uuid:
return bot, pipeline_uuid
return None, None
@staticmethod
@@ -198,7 +199,9 @@ class EmbedRouterGroup(group.RouterGroup):
base_url = quart.request.host_url.rstrip('/')
webhook_prefix = self.ap.instance_config.data.get('api', {}).get('webhook_prefix', '')
if webhook_prefix:
# The wizard previews the currently connected backend, which can
# differ from the public address used by external website embeds.
if webhook_prefix and quart.request.args.get('preview') != 'wizard':
base_url = webhook_prefix.rstrip('/')
if not re.match(r'^https?://[a-zA-Z0-9._:/-]+$', base_url):
@@ -327,20 +330,24 @@ class EmbedRouterGroup(group.RouterGroup):
# -- Embed WebSocket endpoint ----------------------------------------
@self.quart_app.websocket(self.path + '/<bot_uuid>/ws/connect')
@diagnostics.observe('websocket.embed.session', source='websocket', ap=self.ap)
async def embed_websocket_connect(bot_uuid: str):
"""WebSocket connection for embed widget, keyed by bot_uuid."""
await quart.websocket.accept()
if not _is_valid_uuid(bot_uuid):
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'}))
diagnostics.outcome('rejected')
return
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
if runtime_bot is None:
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'}))
diagnostics.outcome('rejected')
return
session_type = quart.websocket.args.get('session_type', 'person')
if session_type not in ['person', 'group']:
diagnostics.outcome('rejected')
await quart.websocket.send(
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
)
@@ -349,6 +356,7 @@ class EmbedRouterGroup(group.RouterGroup):
session_id = quart.websocket.args.get('session_id', '')
if not is_valid_session_id(session_id):
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Valid session_id is required'}))
diagnostics.outcome('rejected')
return
try:
@@ -356,13 +364,16 @@ class EmbedRouterGroup(group.RouterGroup):
await self._assert_execution_active(runtime_bot)
except Exception:
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
diagnostics.outcome('rejected')
return
diagnostics.workspace(runtime_bot.execution_context)
try:
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
websocket_adapter = proxy_bot.adapter
if not websocket_adapter:
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
diagnostics.outcome('rejected')
return
connection = await ws_connection_manager.add_connection(
@@ -420,11 +431,13 @@ class EmbedRouterGroup(group.RouterGroup):
try:
await wait_for_duplex_tasks(receive_task, send_task)
except Exception as e:
diagnostics.outcome('failed')
logger.error(f'Embed WebSocket task error: {e}')
finally:
await ws_connection_manager.remove_connection(connection.connection_id)
except Exception as e:
diagnostics.outcome('failed')
logger.error(f'Embed WebSocket connection error: {e}', exc_info=True)
try:
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
@@ -439,28 +452,35 @@ class EmbedRouterGroup(group.RouterGroup):
message = await quart.websocket.receive()
await ws_connection_manager.update_activity(connection.connection_id)
try:
data = await asyncio.to_thread(json.loads, message)
message_type = data.get('type', 'message')
with diagnostics.scope(self.ap, 'websocket.embed.message', source='websocket'):
diagnostics.workspace(getattr(owner_bot, 'execution_context', None))
try:
data = await asyncio.to_thread(json.loads, message)
message_type = data.get('type', 'message')
if message_type == 'ping':
await connection.send_queue.put(
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
)
elif message_type == 'message':
try:
current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
except Exception:
await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
if message_type == 'ping':
await connection.send_queue.put(
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
)
elif message_type == 'message':
try:
current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
except Exception:
diagnostics.outcome('rejected')
await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
break
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
elif message_type == 'disconnect':
break
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
elif message_type == 'disconnect':
break
else:
diagnostics.outcome('skipped')
except json.JSONDecodeError:
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
except json.JSONDecodeError:
diagnostics.outcome('rejected')
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
except Exception as e:
diagnostics.outcome('failed')
logger.error(f'Embed receive error: {e}', exc_info=True)
finally:
connection.is_active = False
@@ -476,11 +496,13 @@ class EmbedRouterGroup(group.RouterGroup):
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
if message is None:
break
encoded = await asyncio.to_thread(json.dumps, message)
await quart.websocket.send(encoded)
with diagnostics.scope(self.ap, 'websocket.embed.send', source='websocket'):
encoded = await asyncio.to_thread(json.dumps, message)
await quart.websocket.send(encoded)
except asyncio.TimeoutError:
continue
except Exception as e:
diagnostics.outcome('failed')
logger.error(f'Embed send error: {e}', exc_info=True)
finally:
connection.is_active = False
@@ -0,0 +1,33 @@
"""User-confirmed migration only; deliberately not API-key/MCP accessible."""
import quart
from ... import group
from ....authz import Permission
from ....context import RequestContext
from ....service.pipeline_migration import PipelineMigrationService, MigrationError
@group.group_class('pipeline_migration', '/api/v1/pipelines/_/migration')
class PipelineMigrationRouterGroup(group.RouterGroup):
async def initialize(self):
self.service = PipelineMigrationService(self.ap)
@self.route(
'/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_VIEW
)
async def preview(request_context: RequestContext):
try:
return self.success(data=await self.service.preview(request_context))
except MigrationError as exc:
return self.http_status(exc.status_code, -1, exc.code)
@self.route(
'/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_MANAGE
)
async def execute(request_context: RequestContext):
try:
body = await quart.request.get_json(silent=True)
return self.success(data=await self.service.execute(request_context, body))
except MigrationError as exc:
return self.http_status(exc.status_code, -1, exc.code)
@@ -6,6 +6,10 @@ from ....authz import Permission, has_permission
from ....context import RequestContext
from ....service.secrets import redact_secrets
from ... import group
from ......pipeline.extension_preferences import (
normalize_extension_preferences,
validate_extension_preferences,
)
@group.group_class('pipelines', '/api/v1/pipelines')
@@ -41,11 +45,14 @@ class PipelinesRouterGroup(group.RouterGroup):
async def _(request_context: RequestContext) -> str:
pipeline_data = await quart.request.json
create_as_default = pipeline_data.get('is_default') is True
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(
request_context,
pipeline_data,
default=create_as_default,
)
try:
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(
request_context,
pipeline_data,
default=create_as_default,
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success(data={'uuid': pipeline_uuid})
@self.route(
@@ -122,8 +129,12 @@ class PipelinesRouterGroup(group.RouterGroup):
await self.ap.plugin_connector.require_workspace_context(request_context)
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
mcp_servers = await self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
available_skills = await self.ap.skill_service.list_skills(request_context)
extensions_prefs = pipeline.get('extensions_preferences', {})
try:
available_skills = await self.ap.skill_service.list_skills(request_context)
except Exception as exc:
self.ap.logger.warning('Unable to list skills for pipeline extensions: %s', exc)
available_skills = []
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
return self.success(
data={
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
@@ -148,16 +159,42 @@ class PipelinesRouterGroup(group.RouterGroup):
)
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
json_data = await quart.request.json
await self.ap.pipeline_service.update_pipeline_extensions(
request_context,
pipeline_uuid,
json_data.get('bound_plugins', []),
json_data.get('bound_mcp_servers', []),
json_data.get('enable_all_plugins', True),
json_data.get('enable_all_mcp_servers', True),
bound_skills=json_data.get('bound_skills', []),
enable_all_skills=json_data.get('enable_all_skills', True),
bound_mcp_resources=json_data.get('bound_mcp_resources'),
mcp_resource_agent_read_enabled=json_data.get('mcp_resource_agent_read_enabled'),
)
try:
validate_extension_preferences(
{
canonical: json_data[field]
for field, canonical in {
'bound_plugins': 'plugins',
'bound_mcp_servers': 'mcp_servers',
'bound_skills': 'skills',
'bound_mcp_resources': 'mcp_resources',
'enable_all_plugins': 'enable_all_plugins',
'enable_all_mcp_servers': 'enable_all_mcp_servers',
'enable_all_skills': 'enable_all_skills',
'mcp_resource_agent_read_enabled': 'mcp_resource_agent_read_enabled',
}.items()
if field in json_data
},
context='Pipeline extension',
field_aliases={
'plugins': 'bound_plugins',
'mcp_servers': 'bound_mcp_servers',
'skills': 'bound_skills',
'mcp_resources': 'bound_mcp_resources',
},
)
await self.ap.pipeline_service.update_pipeline_extensions(
request_context,
pipeline_uuid,
json_data.get('bound_plugins', []),
json_data.get('bound_mcp_servers', []),
json_data.get('enable_all_plugins', True),
json_data.get('enable_all_mcp_servers', True),
bound_skills=json_data.get('bound_skills', []),
enable_all_skills=json_data.get('enable_all_skills', True),
bound_mcp_resources=json_data.get('bound_mcp_resources'),
mcp_resource_agent_read_enabled=json_data.get('mcp_resource_agent_read_enabled'),
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success()
@@ -14,6 +14,7 @@ import quart
from ....authz import Permission, permissions_for_role, require_permission
from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
from ... import group
from ..... import management_diagnostics as diagnostics
from ......core.task_boundary import run_in_workspace_uow
from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
from ......utils import bounded_executor
@@ -210,6 +211,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
async def initialize(self) -> None:
@self.quart_app.websocket(self.path + '/connect')
@diagnostics.observe('websocket.dashboard.session', source='websocket', ap=self.ap)
async def websocket_connect(pipeline_uuid: str):
"""Open one authenticated dashboard debug connection."""
@@ -217,11 +219,14 @@ class WebSocketChatRouterGroup(group.RouterGroup):
try:
request_context, token = await self._authenticate_websocket()
except Exception:
diagnostics.outcome('rejected')
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
return
diagnostics.workspace(request_context)
session_type = quart.websocket.args.get('session_type', 'person')
if session_type not in ['person', 'group']:
diagnostics.outcome('rejected')
await quart.websocket.send(
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
)
@@ -230,6 +235,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
try:
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
if websocket_adapter is None:
diagnostics.outcome('rejected')
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Pipeline not found'}))
return
@@ -288,11 +294,13 @@ class WebSocketChatRouterGroup(group.RouterGroup):
try:
await wait_for_duplex_tasks(receive_task, send_task)
except Exception as exc:
diagnostics.outcome('failed')
logger.error(f'WebSocket task execution error: {exc}')
finally:
await ws_connection_manager.remove_connection(connection.connection_id)
except Exception:
diagnostics.outcome('failed')
logger.error('Dashboard WebSocket connection error', exc_info=True)
try:
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
@@ -410,28 +418,34 @@ class WebSocketChatRouterGroup(group.RouterGroup):
message = await quart.websocket.receive()
await ws_connection_manager.update_activity(connection.connection_id)
try:
data = await asyncio.to_thread(json.loads, message)
message_type = data.get('type', 'message')
if message_type == 'ping':
await connection.send_queue.put(
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
)
elif message_type == 'message':
try:
request_context = await self._revalidate_websocket_authorization(request_context, token)
except Exception:
await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
with diagnostics.scope(self.ap, 'websocket.dashboard.message', source='websocket'):
diagnostics.workspace(request_context)
try:
data = await asyncio.to_thread(json.loads, message)
message_type = data.get('type', 'message')
if message_type == 'ping':
await connection.send_queue.put(
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
)
elif message_type == 'message':
try:
request_context = await self._revalidate_websocket_authorization(request_context, token)
except Exception:
diagnostics.outcome('rejected')
await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
break
await websocket_adapter.handle_websocket_message(connection, data)
elif message_type == 'disconnect':
break
await websocket_adapter.handle_websocket_message(connection, data)
elif message_type == 'disconnect':
break
else:
logger.warning(f'Unknown WebSocket message type: {message_type}')
except json.JSONDecodeError:
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
else:
diagnostics.outcome('skipped')
logger.warning(f'Unknown WebSocket message type: {message_type}')
except json.JSONDecodeError:
diagnostics.outcome('rejected')
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
except Exception:
diagnostics.outcome('failed')
logger.error('Dashboard WebSocket receive error', exc_info=True)
finally:
connection.is_active = False
@@ -447,11 +461,13 @@ class WebSocketChatRouterGroup(group.RouterGroup):
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
if message is None:
break
encoded = await asyncio.to_thread(json.dumps, message)
await quart.websocket.send(encoded)
with diagnostics.scope(self.ap, 'websocket.dashboard.send', source='websocket'):
encoded = await asyncio.to_thread(json.dumps, message)
await quart.websocket.send(encoded)
except asyncio.TimeoutError:
continue
except Exception:
diagnostics.outcome('failed')
logger.error('Dashboard WebSocket send error', exc_info=True)
finally:
connection.is_active = False
@@ -3,11 +3,20 @@ from sqlalchemy.exc import IntegrityError
from ....authz import Permission, has_permission
from ....context import RequestContext
from ....service.bot_errors import BotApplyError, bot_error_message
from ... import group
@group.group_class('bots', '/api/v1/platform/bots')
class BotsRouterGroup(group.RouterGroup):
def _apply_error_response(self, exc: BotApplyError):
request_id = self.request_id()
logger = getattr(self.ap, 'logger', self.quart_app.logger)
logger.error(f'Bot configuration apply failed request_id={request_id} bot_uuid={exc.bot_uuid}', exc_info=True)
return quart.jsonify(
code='bot_apply_failed', msg=str(exc), data={'uuid': exc.bot_uuid}, request_id=request_id
), 400
async def initialize(self) -> None:
@self.route(
'',
@@ -34,7 +43,14 @@ class BotsRouterGroup(group.RouterGroup):
)
async def _(request_context: RequestContext) -> str:
json_data = await quart.request.json
bot_uuid = await self.ap.bot_service.create_bot(request_context, json_data)
if not isinstance(json_data, dict):
return self.http_status(400, 'invalid_bot_config', 'Bot configuration must be an object')
try:
bot_uuid = await self.ap.bot_service.create_bot(request_context, json_data)
except BotApplyError as exc:
return self._apply_error_response(exc)
except ValueError as exc:
return self.http_status(400, 'invalid_bot_config', bot_error_message(exc, json_data))
return self.success(data={'uuid': bot_uuid})
@self.route(
@@ -63,7 +79,14 @@ class BotsRouterGroup(group.RouterGroup):
async def _(bot_uuid: str, request_context: RequestContext) -> str:
if quart.request.method == 'PUT':
json_data = await quart.request.json
await self.ap.bot_service.update_bot(request_context, bot_uuid, json_data)
if not isinstance(json_data, dict):
return self.http_status(400, 'invalid_bot_config', 'Bot configuration must be an object')
try:
await self.ap.bot_service.update_bot(request_context, bot_uuid, json_data)
except BotApplyError as exc:
return self._apply_error_response(exc)
except ValueError as exc:
return self.http_status(400, 'invalid_bot_config', bot_error_message(exc, json_data))
else:
await self.ap.bot_service.delete_bot(request_context, bot_uuid)
return self.success()
@@ -83,6 +106,45 @@ class BotsRouterGroup(group.RouterGroup):
)
return self.success(data={'logs': logs, 'total_count': total_count})
@self.route(
'/<bot_uuid>/event-routes/status',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def _(bot_uuid: str, request_context: RequestContext) -> str:
return self.success(data=await self.ap.bot_service.list_event_route_statuses(request_context, bot_uuid))
async def _dry_run_event_route(bot_uuid: str, request_context: RequestContext) -> str:
json_data = await quart.request.json
if not isinstance(json_data, dict):
return self.http_status(400, -1, 'invalid request body')
result = await self.ap.bot_service.dry_run_event_route(
request_context,
bot_uuid=bot_uuid,
event_type=json_data.get('event_type'),
event_data=json_data.get('event_data', json_data.get('payload')),
context=json_data.get('context'),
event_bindings=json_data.get('event_bindings'),
)
return self.success(data=result)
self.route(
'/<bot_uuid>/event-routes/dry-run',
methods=['POST'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)(_dry_run_event_route)
# Backward-compatible alias for early local clients/tests created before
# the product route naming was settled.
self.route(
'/<bot_uuid>/event_route/dry_run',
methods=['POST'],
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)(_dry_run_event_route)
@self.route(
'/<bot_uuid>/send_message',
methods=['POST'],
@@ -426,6 +426,15 @@ class PluginsRouterGroup(group.RouterGroup):
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
ctx = taskmgr.TaskContext.new()
ctx.metadata.update(
{
'plugin_name': f'{author}/{plugin_name}',
'install_source': 'marketplace',
'operation': 'upgrade',
'progress_percent': 3,
}
)
ctx.set_current_action('checking for latest version')
wrapper = self.ap.task_mgr.create_user_task(
self._run_fenced_plugin_operation(
execution_context,
@@ -1,11 +1,15 @@
import quart
from langbot.pkg.provider.modelmgr import errors as provider_errors
from ....authz import Permission, has_permission
from ....context import RequestContext
from ... import group
from .query import resolve_include_secret
MODEL_TEST_ERROR_CODE = 'model_test_failed'
@group.group_class('models/llm', '/api/v1/provider/models/llm')
class LLMModelsRouterGroup(group.RouterGroup):
async def initialize(self) -> None:
@@ -108,7 +112,14 @@ class LLMModelsRouterGroup(group.RouterGroup):
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def _(model_uuid: str, request_context: RequestContext) -> str:
await self.ap.llm_model_service.test_llm_model(request_context, model_uuid, await quart.request.json)
try:
await self.ap.llm_model_service.test_llm_model(
request_context,
model_uuid,
await quart.request.json,
)
except provider_errors.RequesterError as exc:
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
return self.success()
@@ -214,9 +225,14 @@ class EmbeddingModelsRouterGroup(group.RouterGroup):
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def _(model_uuid: str, request_context: RequestContext) -> str:
await self.ap.embedding_models_service.test_embedding_model(
request_context, model_uuid, await quart.request.json
)
try:
await self.ap.embedding_models_service.test_embedding_model(
request_context,
model_uuid,
await quart.request.json,
)
except provider_errors.RequesterError as exc:
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
return self.success()
@@ -322,5 +338,12 @@ class RerankModelsRouterGroup(group.RouterGroup):
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def _(model_uuid: str, request_context: RequestContext) -> str:
await self.ap.rerank_models_service.test_rerank_model(request_context, model_uuid, await quart.request.json)
try:
await self.ap.rerank_models_service.test_rerank_model(
request_context,
model_uuid,
await quart.request.json,
)
except provider_errors.RequesterError as exc:
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
return self.success()
@@ -5,10 +5,40 @@ import quart
from ....authz import Permission
from ....context import RequestContext
from ... import group
from ......pipeline.extension_preferences import normalize_extension_preferences
@group.group_class('tools', '/api/v1/tools')
class ToolsRouterGroup(group.RouterGroup):
async def _get_scoped_tool_catalog(
self,
request_context: RequestContext,
) -> list[dict] | None:
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
bound_plugins: list[str] | None = None
bound_mcp_servers: list[str] | None = None
if pipeline_uuid:
pipeline = await self.ap.pipeline_service.get_pipeline(
request_context,
pipeline_uuid,
)
if pipeline is None:
return None
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
if not extensions_prefs['enable_all_plugins']:
bound_plugins = [f'{plugin["author"]}/{plugin["name"]}' for plugin in extensions_prefs['plugins']]
if not extensions_prefs['enable_all_mcp_servers']:
bound_mcp_servers = extensions_prefs['mcp_servers']
return await self.ap.tool_mgr.get_resolved_tool_catalog(
request_context,
bound_plugins,
bound_mcp_servers,
include_skill_authoring=True,
)
async def initialize(self) -> None:
@self.route(
'',
@@ -18,57 +48,35 @@ class ToolsRouterGroup(group.RouterGroup):
)
async def _(request_context: RequestContext) -> str:
"""获取所有可用工具列表"""
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
bound_plugins: list[str] | None = None
bound_mcp_servers: list[str] | None = None
if pipeline_uuid:
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
if pipeline is None:
return self.http_status(404, -1, 'pipeline not found')
extensions_prefs = pipeline.get('extensions_preferences', {}) or {}
if not extensions_prefs.get('enable_all_plugins', True):
bound_plugins = [
f'{plugin.get("author", "")}/{plugin.get("name", "")}'
for plugin in extensions_prefs.get('plugins', [])
if isinstance(plugin, dict) and plugin.get('name')
]
if not extensions_prefs.get('enable_all_mcp_servers', True):
bound_mcp_servers = [
server for server in (extensions_prefs.get('mcp_servers', []) or []) if isinstance(server, str)
]
return self.success(
data={
'tools': await self.ap.tool_mgr.get_tool_catalog(
request_context,
bound_plugins,
bound_mcp_servers,
include_skill_authoring=True,
)
}
)
catalog = await self._get_scoped_tool_catalog(request_context)
if catalog is None:
return self.http_status(404, -1, 'pipeline not found')
return self.success(data={'tools': catalog})
@self.route(
'/<tool_name>',
'/<path:tool_name>',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.RESOURCE_VIEW,
)
async def _(tool_name: str, request_context: RequestContext) -> str:
"""获取特定工具详情"""
tools = await self.ap.tool_mgr.get_all_tools(request_context, include_skill_authoring=True)
catalog = await self._get_scoped_tool_catalog(request_context)
if catalog is None:
return self.http_status(404, -1, 'pipeline not found')
for tool in tools:
if tool.name == tool_name:
for tool in catalog:
if tool.get('name') == tool_name:
return self.success(
data={
'tool': {
'name': tool.name,
'description': tool.description,
'human_desc': tool.human_desc,
'parameters': tool.parameters,
'name': tool['name'],
'description': tool.get('description') or '',
'human_desc': tool.get('human_desc') or '',
'parameters': tool.get('parameters') or {},
'source': tool.get('source'),
'source_name': tool.get('source_name'),
'source_id': tool.get('source_id'),
}
}
)
@@ -273,8 +273,8 @@ class SystemRouterGroup(group.RouterGroup):
"""Save wizard progress to metadata table.
Accepts JSON body with wizard state fields:
{ "step": int, "selected_adapter": str|null, "created_bot_uuid": str|null,
"bot_saved": bool, "selected_runner": str|null }
{ "step": int, "selected_scenario": str|null, "selected_adapter": str|null,
"created_bot_uuid": str|null, "bot_saved": bool, "selected_runner": str|null }
"""
data = await quart.request.get_json(silent=True) or {}
progress_json = json.dumps(data, ensure_ascii=False)
@@ -322,6 +322,21 @@ class SystemRouterGroup(group.RouterGroup):
return self.http_status(503, -1, str(exc))
return self.success(data=model)
@self.route(
'/model-availability',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.RESOURCE_VIEW,
)
async def _(request_context: RequestContext) -> str:
"""Expose Space's latest persisted model probes to the WebUI."""
try:
models = await self.ap.space_service.get_model_selection()
except Exception as exc:
self.ap.logger.warning(f'Failed to load LangBot Models availability: {exc}')
return self.http_status(503, -1, 'Model availability is unavailable')
return self.success(data={'models': [model.model_dump(mode='json') for model in models]})
@self.route(
'/tasks',
methods=['GET'],
+687
View File
@@ -0,0 +1,687 @@
from __future__ import annotations
import datetime
import copy
import fnmatch
import time
import uuid
import typing
from ... import management_diagnostics as diagnostics
import sqlalchemy
from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
from langbot_plugin.api.entities.builtin.runner.event import (
ActorContext,
RawEventRef,
SubjectContext,
)
from langbot_plugin.api.entities.builtin.runner.input import AgentInput
from langbot_plugin.api.entities.builtin.provider import message as provider_message
from langbot_plugin.api.entities.builtin.provider import prompt as provider_prompt
from ....core import app
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....agent.runner.config_schema import extract_prompt_config
from ....agent.runner.execution_context import build_execution_query, project_mcp_resource_config
from ....agent.runner.host_models import (
AgentBinding,
AgentEventEnvelope,
BindingScope,
DeliveryPolicy,
StatePolicy,
)
from ....agent.runner.resource_policy import ResourcePolicyProjector
from ....agent.runner.platform_tools import (
PLATFORM_TOOL_DEFINITIONS,
platform_tool_catalog,
resolve_agent_platform_tool_names,
validate_debug_mock_options,
)
from ....entity.persistence import agent as persistence_agent
from ....workspace.errors import WorkspaceNotFoundError
from ..context import ExecutionContext, RequestContext
from .tenant import TenantContext, require_workspace_uuid, scope_statement
AGENT_KIND_AGENT = 'agent'
AGENT_KIND_PIPELINE = 'pipeline'
AGENT_KIND_EVENT_PROCESSOR = 'event_processor'
PIPELINE_EVENT_PATTERNS = ['message.*']
AGENT_DEFAULT_EVENT_PATTERNS = ['*']
class AgentService:
"""Unified processor facade for the peer Agent and Pipeline types."""
ap: app.Application
def __init__(self, ap: app.Application) -> None:
self.ap = ap
async def get_agent_metadata(self, context: TenantContext) -> dict[str, typing.Any]:
"""Return metadata needed by Agent forms."""
pipeline_metadata = await self.ap.pipeline_service.get_pipeline_metadata(context)
ai_metadata = next((item for item in pipeline_metadata if item.get('name') == 'ai'), None)
host_tools: list[dict[str, typing.Any]] | None = None
get_tool_catalog = getattr(getattr(self.ap, 'tool_mgr', None), 'get_resolved_tool_catalog', None)
if get_tool_catalog is not None:
try:
host_tools = await get_tool_catalog(
context,
include_skill_authoring=True,
include_mcp_resource_tools=True,
)
except Exception as exc:
self.ap.logger.warning(f'Failed to load Agent Host tool catalog: {exc}')
event_processors = []
registry = getattr(self.ap, 'runner_registry', None)
if registry is not None:
event_processors = [
item.model_dump(mode='json')
for item in await registry.list_runners(
context,
usage='event',
use_cache=False,
)
]
return {
'event_processors': event_processors,
'runner_config': ai_metadata,
'platform_tools': platform_tool_catalog(),
'host_tools': host_tools,
'kinds': [
{
'name': AGENT_KIND_AGENT,
'supported_event_patterns': AGENT_DEFAULT_EVENT_PATTERNS,
'message_only': False,
},
{
'name': AGENT_KIND_PIPELINE,
'supported_event_patterns': PIPELINE_EVENT_PATTERNS,
'message_only': True,
},
{'name': AGENT_KIND_EVENT_PROCESSOR, 'supported_event_patterns': ['*'], 'message_only': False},
],
}
async def get_agents(
self,
context: TenantContext,
sort_by: str = 'updated_at',
sort_order: str = 'DESC',
) -> list[dict]:
agents = await self._get_agent_rows(context)
pipelines = await self.ap.pipeline_service.get_pipelines(
context,
sort_by='updated_at',
sort_order='DESC',
)
items = [self._agent_to_product_item(agent) for agent in agents]
items.extend(self._pipeline_to_product_item(pipeline) for pipeline in pipelines)
reverse = sort_order == 'DESC'
sort_key = sort_by if sort_by in {'created_at', 'updated_at'} else 'updated_at'
return sorted(items, key=lambda item: self._parse_sort_time(item.get(sort_key)), reverse=reverse)
async def get_agent(self, context: TenantContext, agent_uuid: str) -> dict | None:
agent = await self._get_agent_row(context, agent_uuid)
if agent is not None:
return self._agent_to_product_item(agent, include_config=True)
pipeline = await self.ap.pipeline_service.get_pipeline(context, agent_uuid)
if pipeline is not None:
return self._pipeline_to_product_item(pipeline, include_config=True)
return None
@diagnostics.observe('http.agent.debug_agent', source='webui_debug')
async def debug_agent(
self,
context: RequestContext,
agent_uuid: str,
payload: dict[str, typing.Any],
*,
on_result: typing.Callable[[dict[str, typing.Any]], typing.Awaitable[None]] | None = None,
) -> dict[str, typing.Any]:
"""Execute one synthetic event against a configured Agent.
The debug surface uses a trusted Workspace execution context, never
delivers outputs to a real platform, and supports both message and
non-message event envelopes.
"""
diagnostics.workspace(context)
agent = await self.get_agent(context, agent_uuid)
if agent is None or agent.get('kind') not in {AGENT_KIND_AGENT, AGENT_KIND_EVENT_PROCESSOR}:
raise ValueError('Agent not found')
event_type = str(payload.get('event_type', 'message.received')).strip()
if not event_type or len(event_type) > 128:
raise ValueError('Invalid event_type')
if not self._supports_event_type(
agent.get('supported_event_patterns'),
event_type,
):
raise ValueError('Agent does not support this event type')
text = str(payload.get('text') or '').strip()
if len(text) > 20_000:
raise ValueError('Debug input is too long')
event_data = payload.get('data', {})
if not isinstance(event_data, dict):
raise ValueError('Debug event data must be an object')
mock_options = validate_debug_mock_options(payload.get('mock', {}))
config = agent.get('config')
if not isinstance(config, dict):
raise ValueError('Agent configuration is invalid')
_, runner_id, runner_config = RunnerConfigResolver.resolve_agent_config(config)
if not runner_id:
raise ValueError('Agent has no configured runner')
conversation_id = str(payload.get('conversation_id') or f'debug:{agent_uuid}').strip()
if not conversation_id or len(conversation_id) > 256:
raise ValueError('Invalid debug conversation_id')
actor_payload = payload.get(
'actor',
{
'actor_type': 'user',
'actor_id': str(
event_data.get('member_id')
or event_data.get('requester_id')
or event_data.get('user_id')
or 'debug-user'
),
'actor_name': str(
event_data.get('member_name')
or event_data.get('requester_name')
or event_data.get('user_name')
or 'Debug User'
),
},
)
subject_payload = payload.get(
'subject',
{
'subject_type': 'message' if event_type.startswith('message.') else event_type.split('.', 1)[0],
'subject_id': 'debug-subject',
'data': event_data,
},
)
if not isinstance(actor_payload, dict) or not isinstance(subject_payload, dict):
raise ValueError('Debug actor and subject must be objects')
event_id = f'debug:{agent_uuid}:{uuid.uuid4()}'
is_group = event_type.startswith(('group.', 'bot.')) or bool(event_data.get('group_id'))
actor = ActorContext.model_validate(actor_payload)
target_id = str(event_data.get('group_id') or 'debug-group') if is_group else actor.actor_id
event = AgentEventEnvelope(
event_id=event_id,
event_type=event_type,
event_time=int(time.time()),
source='webui',
source_event_type=event_type,
workspace_id=context.workspace_uuid,
conversation_id=conversation_id,
actor=actor,
subject=SubjectContext.model_validate(subject_payload),
input=AgentInput.model_validate(
{
'text': text or event_type,
'contents': [
{'type': 'text', 'text': text or event_type},
],
'attachments': [],
}
),
delivery=DeliveryContext(
surface='webui',
reply_target={
'target_type': 'group' if is_group else 'person',
'target_id': target_id,
**({'group_id': target_id} if is_group else {}),
**(
{'message_id': str(event_data.get('message_id') or 'debug-message')}
if event_type.startswith('message.')
else {}
),
},
supports_streaming=on_result is not None,
supports_edit=False,
supports_reaction=False,
platform_capabilities={
'event_type': event_type,
'debug': True,
'debug_mock': True,
'supported_apis': sorted(
{tool.api for tool in PLATFORM_TOOL_DEFINITIONS} - set(mock_options.get('unsupported_apis', []))
),
'mock_options': mock_options,
},
),
raw_ref=RawEventRef(ref_id=event_id, storage_key=None),
data=event_data,
)
if agent.get('kind') == AGENT_KIND_EVENT_PROCESSOR:
from langbot_plugin.api.entities.builtin.platform.events import parse_eba_event
typed_event = parse_eba_event({**event_data, 'type': event_type})
from ....platform.botmgr import RuntimeBot
event.actor = RuntimeBot._infer_actor_context(typed_event)
event.subject = RuntimeBot._infer_subject_context(typed_event)
target_type, target_id, target_metadata = RuntimeBot._infer_reply_target(typed_event)
if target_id is not None:
event.delivery.reply_target = {
'target_type': target_type,
'target_id': str(target_id),
**target_metadata,
}
event.data = typed_event.model_dump(mode='json', exclude={'source_platform_object', 'legacy_event'})
binding = AgentBinding(
binding_id=(
f'event_processor:{agent_uuid}'
if agent.get('kind') == AGENT_KIND_EVENT_PROCESSOR
else f'debug:{agent_uuid}:{runner_id}'
),
scope=BindingScope(scope_type='agent', scope_id=agent_uuid),
event_types=[event_type],
runner_id=runner_id,
runner_config=runner_config,
resource_policy=ResourcePolicyProjector.from_runner_config(
runner_config,
allowed_platform_tool_names=resolve_agent_platform_tool_names(config, event_type),
allowed_host_tool_names=config.get('allowed_tools'),
override_runner_tools='allowed_tools' in config,
),
state_policy=StatePolicy(
state_scopes=['conversation', 'actor', 'subject', 'runner'],
),
delivery_policy=DeliveryPolicy(
enable_streaming=on_result is not None,
enable_reply=False,
enable_interactions=False,
),
agent_id=agent_uuid,
processor_type=agent.get('kind', 'agent'),
processor_id=agent_uuid,
)
execution_context = ExecutionContext.from_request(
context,
query_uuid=event_id,
)
# Debug bypasses pipeline preprocessing, so supply the selected Runner's
# prompt before advertising the Host prompt API. Keep explicit [] intact.
descriptor = await self.ap.runner_registry.get(execution_context, runner_id)
prompt_config = extract_prompt_config(descriptor, runner_config, [])
execution_query = build_execution_query(event, [])
execution_query.prompt = provider_prompt.Prompt(
name='default',
messages=[provider_message.Message(**message) for message in copy.deepcopy(prompt_config)],
)
for field_name in ('instance_uuid', 'workspace_uuid', 'placement_generation', 'query_uuid'):
object.__setattr__(execution_query, field_name, getattr(execution_context, field_name))
project_mcp_resource_config(execution_query, runner_config)
output_items: list[dict[str, typing.Any]] = []
execution_events: list[dict[str, typing.Any]] = []
async def observe_result(result: dict[str, typing.Any]) -> None:
if result.get('type') not in {
'message.delta',
'message.completed',
'tool.call.started',
'tool.call.completed',
'run.completed',
'run.failed',
'processor.log',
}:
return
visible_result = copy.deepcopy(
{
key: result[key]
for key in ('type', 'data', 'sequence', 'timestamp', 'run_id', 'usage')
if key in result
}
)
if on_result is not None:
await on_result(visible_result)
elif len(execution_events) < 1000:
execution_events.append(visible_result)
final_text = ''
async for output in self.ap.agent_run_orchestrator.run(
event,
binding,
adapter_context={
'_query': execution_query,
'_execution_context': execution_context,
'_result_observer': observe_result,
},
):
output_text = self._provider_output_to_text(output)
if output_text:
final_text = output_text
output_items.append(
{
'kind': output.__class__.__name__,
'role': str(getattr(output, 'role', '') or ''),
'text': output_text,
}
)
return {
'event_id': event_id,
'event_type': event_type,
'conversation_id': conversation_id,
'final_text': final_text,
'outputs': output_items,
'execution_events': execution_events,
}
@staticmethod
def _supports_event_type(patterns: typing.Any, event_type: str) -> bool:
normalized = patterns if isinstance(patterns, list) else AGENT_DEFAULT_EVENT_PATTERNS
return any(isinstance(pattern, str) and fnmatch.fnmatchcase(event_type, pattern) for pattern in normalized)
@staticmethod
def _provider_output_to_text(output: typing.Any) -> str:
all_content = getattr(output, 'all_content', None)
if all_content:
return str(all_content)
content = getattr(output, 'content', None)
if content is None:
return ''
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for item in content:
item_data = item.model_dump(mode='json') if hasattr(item, 'model_dump') else item
if isinstance(item_data, dict) and item_data.get('text') is not None:
parts.append(str(item_data['text']))
elif item_data is not None and not isinstance(item_data, dict):
parts.append(str(item_data))
return ''.join(parts)
return str(content)
async def create_agent(self, context: TenantContext, agent_data: dict) -> dict[str, str]:
workspace_uuid = require_workspace_uuid(context)
kind = agent_data.get('kind') or AGENT_KIND_AGENT
if kind == AGENT_KIND_PIPELINE:
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(
context,
{
'name': agent_data.get('name') or 'New Pipeline',
'description': agent_data.get('description') or '',
'emoji': agent_data.get('emoji') or '⚙️',
'config': {},
},
)
return {'uuid': pipeline_uuid, 'kind': AGENT_KIND_PIPELINE}
if kind not in {AGENT_KIND_AGENT, AGENT_KIND_EVENT_PROCESSOR}:
raise ValueError(f'Unsupported agent kind: {kind}')
if kind == AGENT_KIND_EVENT_PROCESSOR:
config, runner_id, patterns = await self._prepare_event_processor(context, agent_data)
else:
config = agent_data['config'] if 'config' in agent_data else await self._get_default_agent_config(context)
config, runner_id, _ = RunnerConfigResolver.resolve_agent_config(config)
await self._validate_runner_for_agent(context, runner_id)
patterns = agent_data.get('supported_event_patterns', AGENT_DEFAULT_EVENT_PATTERNS)
new_uuid = str(uuid.uuid4())
values = {
'workspace_uuid': workspace_uuid,
'uuid': new_uuid,
'name': agent_data.get('name') or 'New Agent',
'description': agent_data.get('description') or '',
'emoji': agent_data.get('emoji') or ('🧩' if kind == AGENT_KIND_EVENT_PROCESSOR else '🤖'),
'kind': kind,
'component_ref': runner_id,
'config': config,
'supported_event_patterns': patterns,
}
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_agent.Agent).values(**values))
return {'uuid': new_uuid, 'kind': kind}
async def update_agent(self, context: TenantContext, agent_uuid: str, agent_data: dict) -> None:
existing_agent = await self._get_agent_row(context, agent_uuid)
if existing_agent is None:
pipeline = await self.ap.pipeline_service.get_pipeline(context, agent_uuid)
if pipeline is None:
raise ValueError(f'Agent {agent_uuid} not found')
await self.ap.pipeline_service.update_pipeline(context, agent_uuid, agent_data)
return
update_data = {
field: agent_data[field]
for field in ('name', 'description', 'emoji', 'config', 'supported_event_patterns')
if field in agent_data
}
if existing_agent.kind == AGENT_KIND_EVENT_PROCESSOR and any(
field in agent_data for field in ('config', 'component_ref', 'parameters', 'supported_event_patterns')
):
config, runner_id, patterns = await self._prepare_event_processor(context, agent_data, existing_agent)
update_data.update(config=config, component_ref=runner_id, supported_event_patterns=patterns)
if 'config' in update_data:
config, runner_id, _ = RunnerConfigResolver.resolve_agent_config(update_data['config'])
update_data['config'] = config
else:
_, runner_id, _ = RunnerConfigResolver.resolve_agent_config(existing_agent.config)
update_data['component_ref'] = runner_id
if existing_agent.kind == AGENT_KIND_AGENT:
await self._validate_runner_for_agent(context, runner_id)
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.update(persistence_agent.Agent)
.where(persistence_agent.Agent.uuid == agent_uuid)
.values(**update_data),
persistence_agent.Agent,
context,
)
)
if getattr(result, 'rowcount', None) == 0:
raise WorkspaceNotFoundError('Agent not found')
async def delete_agent(self, context: TenantContext, agent_uuid: str) -> None:
existing_agent = await self._get_agent_row(context, agent_uuid)
if existing_agent is not None:
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.delete(persistence_agent.Agent).where(persistence_agent.Agent.uuid == agent_uuid),
persistence_agent.Agent,
context,
)
)
if getattr(result, 'rowcount', None) == 0:
raise WorkspaceNotFoundError('Agent not found')
return
pipeline = await self.ap.pipeline_service.get_pipeline(context, agent_uuid)
if pipeline is None:
raise ValueError(f'Agent {agent_uuid} not found')
await self.ap.pipeline_service.delete_pipeline(context, agent_uuid)
async def _validate_runner_for_agent(self, context, runner_id):
if runner_id:
descriptor = await self.ap.runner_registry.get(context, runner_id)
if 'agent' not in descriptor.usages:
raise ValueError('The selected Runner does not support agent usage')
async def _prepare_event_processor(self, context, data, existing=None):
"""Resolve an installed component and keep its capability declaration authoritative."""
config = copy.deepcopy(data.get('config', existing.config if existing is not None else {}))
if not isinstance(config, dict):
raise ValueError('Processor configuration must be an object')
component_ref = data.get('component_ref') or (existing.component_ref if existing is not None else None)
if component_ref is None and not config and not data.get('parameters'):
# An unconfigured instance cannot subscribe to or execute any events.
return {}, None, []
if not isinstance(component_ref, str) or not component_ref.startswith('plugin:'):
raise ValueError('Select an installed Runner component')
try:
descriptor = await self.ap.runner_registry.get(context, component_ref)
except Exception as exc:
from ....agent.runner.errors import RunnerNotFoundError
if isinstance(exc, RunnerNotFoundError):
raise ValueError('Runner component is unavailable') from exc
raise
if 'event' not in descriptor.usages or not descriptor.supported_event_patterns:
raise ValueError('The component does not declare supported events')
config['runner'] = {'id': component_ref}
parameters = data.get('parameters')
if parameters is None:
runner_config = config.get('runner_config', {})
if not isinstance(runner_config, dict):
raise ValueError('Runner configuration must be an object')
parameters = runner_config.get(component_ref)
if parameters is None:
parameters = self.ap.pipeline_service._get_default_values_from_schema(descriptor.config_schema)
if not isinstance(parameters, dict):
raise ValueError('Processor parameters must be an object')
for field in descriptor.config_schema:
if field.get('required') and parameters.get(field['name']) in (None, ''):
raise ValueError(f'Required processor parameter: {field["name"]}')
config['runner_config'] = {component_ref: parameters}
return config, component_ref, descriptor.supported_event_patterns
async def get_processor_runs(self, context, processor_id, *, before_id=None):
"""Read only this Workspace's Agent or plugin processor runs."""
from ....agent.runner.run_ledger_store import RunLedgerStore
processor = await self.get_agent(context, processor_id)
if processor is None or processor.get('kind') not in {'agent', AGENT_KIND_EVENT_PROCESSOR}:
raise ValueError('Processor not found')
store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine())
items, cursor, has_more, total = await store.list_runs(
workspace_id=require_workspace_uuid(context),
**(
{'binding_id': f'event_processor:{processor_id}'}
if processor.get('kind') == AGENT_KIND_EVENT_PROCESSOR
else {'agent_id': processor_id}
),
before_id=before_id,
)
return {'items': items, 'next_cursor': cursor, 'has_more': has_more, 'total': total}
async def get_processor_run_events(self, context, processor_id, run_id, *, after_sequence=None):
"""Authorize the parent run before exposing any trace events."""
from ....agent.runner.run_ledger_store import RunLedgerStore
processor = await self.get_agent(context, processor_id)
if processor is None or processor.get('kind') not in {'agent', AGENT_KIND_EVENT_PROCESSOR}:
raise ValueError('Processor not found')
store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine())
run = await store.get_run(run_id)
if (
run is None
or run.get('workspace_id') != require_workspace_uuid(context)
or not (
run.get('binding_id') == f'event_processor:{processor_id}'
if processor.get('kind') == AGENT_KIND_EVENT_PROCESSOR
else run.get('agent_id') == processor_id
or (
run.get('agent_id') is None
and str(run.get('binding_id', '')).startswith((f'agent_{processor_id}_', f'debug:{processor_id}:'))
)
)
):
raise ValueError('Processor run not found')
items, next_cursor, _, has_more = await store.page_run_events(
run_id=run_id,
after_sequence=after_sequence,
limit=100,
)
return {'run': run, 'items': items, 'next_cursor': next_cursor, 'has_more': has_more}
async def _get_agent_rows(self, context: TenantContext) -> list[persistence_agent.Agent]:
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_agent.Agent),
persistence_agent.Agent,
context,
)
)
return list(result.all())
async def _get_agent_row(
self,
context: TenantContext,
agent_uuid: str,
) -> persistence_agent.Agent | None:
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_agent.Agent).where(persistence_agent.Agent.uuid == agent_uuid),
persistence_agent.Agent,
context,
)
)
return result.first()
async def _get_default_agent_config(self, context: TenantContext) -> dict[str, typing.Any]:
runners = []
if getattr(self.ap, 'runner_registry', None) is not None:
try:
runners = await self.ap.runner_registry.list_runners(context, bound_plugins=None)
except Exception as e:
if getattr(self.ap, 'logger', None):
self.ap.logger.warning(f'Failed to load plugin agent runners for default agent config: {e}')
if not runners:
return {'runner': {'id': '', 'expire-time': 0}, 'runner_config': {}}
selected_runner = runners[0]
return {
'runner': {'id': selected_runner.id, 'expire-time': 0},
'runner_config': {
selected_runner.id: self.ap.pipeline_service._get_default_values_from_schema(
selected_runner.config_schema
)
},
}
def _agent_to_product_item(
self,
agent: persistence_agent.Agent,
include_config: bool = False,
) -> dict[str, typing.Any]:
item = self.ap.persistence_mgr.serialize_model(persistence_agent.Agent, agent)
item['kind'] = item.get('kind') or AGENT_KIND_AGENT
supported_event_patterns = item.get('supported_event_patterns')
item['capability'] = {
'supported_event_patterns': (
supported_event_patterns if isinstance(supported_event_patterns, list) else AGENT_DEFAULT_EVENT_PATTERNS
),
'message_only': False,
}
if not include_config:
item.pop('config', None)
return item
@staticmethod
def _pipeline_to_product_item(pipeline: dict, include_config: bool = False) -> dict[str, typing.Any]:
item = pipeline.copy()
item['kind'] = AGENT_KIND_PIPELINE
item['component_ref'] = 'pipeline'
item['supported_event_patterns'] = PIPELINE_EVENT_PATTERNS
item['capability'] = {
'supported_event_patterns': PIPELINE_EVENT_PATTERNS,
'message_only': True,
}
if not include_config:
item.pop('config', None)
return item
@staticmethod
def _parse_sort_time(value: typing.Any) -> datetime.datetime:
if isinstance(value, datetime.datetime):
return value
if isinstance(value, str):
try:
return datetime.datetime.fromisoformat(value)
except ValueError:
return datetime.datetime.min
return datetime.datetime.min
+688 -66
View File
@@ -1,26 +1,608 @@
from __future__ import annotations
import uuid
import typing
import json
import sqlalchemy
from ....core import app
from ....discover import engine
from ....entity.persistence import agent as persistence_agent
from ....entity.persistence import bot as persistence_bot
from ....entity.persistence import pipeline as persistence_pipeline
from ....workspace.errors import WorkspaceNotFoundError
from .bot_errors import BotApplyError, bot_error_message
from .tenant import TenantContext, require_workspace_uuid, scope_statement
from ....utils import httpclient
from ....platform.sources import http_bot_signing
from ....platform.adapter_names import canonical_adapter_name
from ....agent.runner.errors import RunnerNotFoundError
class BotService:
"""Bot service"""
ap: app.Application
FAILURE_ROUTE_NOT_FOUND = 'route_not_found'
FAILURE_PROCESSOR_NOT_FOUND = 'processor_not_found'
FAILURE_PROCESSOR_INCOMPATIBLE = 'processor_incompatible'
FAILURE_INVALID_EVENT = 'invalid_event'
ROUTE_TRACE_KIND = 'event_route_trace'
BOT_FIELDS = {
'uuid',
'name',
'description',
'adapter',
'adapter_config',
'enable',
'event_bindings',
'plugin_processors',
}
def __init__(self, ap: app.Application) -> None:
self.ap = ap
def _get_adapter_component(self, adapter_name: str) -> engine.Component | None:
"""Return the discovered platform adapter component for an adapter name."""
adapter_name = canonical_adapter_name(adapter_name)
for component in self.ap.discover.get_components_by_kind('MessagePlatformAdapter'):
if component.metadata.name == adapter_name:
return component
return None
def _adapter_declares_webhook_url(self, adapter_name: str) -> bool:
"""Whether the adapter manifest declares a generated webhook URL config item."""
component = self._get_adapter_component(adapter_name)
if component is None:
return False
for config_item in component.spec.get('config', []):
if config_item.get('type') == 'webhook-url':
return True
return False
@staticmethod
def _is_message_event_pattern(event_pattern: str) -> bool:
return event_pattern == 'message.*' or event_pattern.startswith('message.')
@staticmethod
def _event_pattern_covers(supported_pattern: str, binding_pattern: str) -> bool:
if supported_pattern == '*':
return True
if supported_pattern == binding_pattern:
return True
if binding_pattern == '*':
return False
if supported_pattern.endswith('.*'):
namespace = supported_pattern[:-2]
return binding_pattern == f'{namespace}.*' or binding_pattern.startswith(f'{namespace}.')
return False
@classmethod
def _agent_supports_event_pattern(cls, supported_patterns: list[str] | None, event_pattern: str) -> bool:
patterns = supported_patterns if isinstance(supported_patterns, list) else ['*']
return any(cls._event_pattern_covers(pattern, event_pattern) for pattern in patterns)
@staticmethod
def _format_diagnostic_step(step: dict[str, typing.Any]) -> str:
step_name = step.get('step') or 'diagnostic'
reason = step.get('reason') or step.get('failure_code') or 'No reason provided'
if step_name == 'evaluate_binding':
route_number = step.get('binding_index')
if not isinstance(route_number, int):
route_number = step.get('order')
route_label = f'Route {int(route_number) + 1}' if isinstance(route_number, int) else 'Route'
event_pattern = step.get('event_pattern') or '*'
if step.get('selected'):
return f'{route_label} ({event_pattern}) selected: {reason}'
if step.get('matched'):
return f'{route_label} ({event_pattern}) matched: {reason}'
return f'{route_label} ({event_pattern}) skipped: {reason}'
if step_name == 'validate_processor':
target_type = step.get('target_type') or 'processor'
target_uuid = step.get('target_uuid') or ''
suffix = f' {target_uuid}' if target_uuid else ''
return f'Validate {target_type}{suffix}: {reason}'
return str(reason)
@classmethod
def _format_diagnostic_steps(cls, diagnostic_details: list[dict[str, typing.Any]] | None) -> list[str]:
return [cls._format_diagnostic_step(step) for step in diagnostic_details or []]
@classmethod
def _event_route_status_from_log(cls, log: typing.Any) -> dict[str, typing.Any] | None:
if hasattr(log, 'to_json'):
log_data = log.to_json()
elif isinstance(log, dict):
log_data = log
else:
log_data = {
'seq_id': getattr(log, 'seq_id', None),
'timestamp': getattr(log, 'timestamp', None),
'level': getattr(getattr(log, 'level', None), 'value', getattr(log, 'level', None)),
'text': getattr(log, 'text', None),
'metadata': getattr(log, 'metadata', None),
}
metadata = log_data.get('metadata')
if not isinstance(metadata, dict) or metadata.get('kind') != cls.ROUTE_TRACE_KIND:
return None
return {
'binding_id': metadata.get('binding_id'),
'event_pattern': metadata.get('event_pattern'),
'event_type': metadata.get('event_type'),
'target_type': metadata.get('target_type'),
'target_uuid': metadata.get('target_uuid') or '',
'last_status': metadata.get('status'),
'failure_code': metadata.get('failure_code'),
'reason': metadata.get('reason') or log_data.get('text') or '',
'run_id': metadata.get('run_id'),
'timestamp': log_data.get('timestamp'),
'seq_id': log_data.get('seq_id'),
'level': log_data.get('level'),
'message': log_data.get('text') or '',
}
@staticmethod
def _target_kind(target_type: typing.Any, target_kind: str | None = None) -> str | None:
if target_kind:
return target_kind
if target_type == 'discard':
return 'discard'
if target_type in {'agent', 'pipeline', 'event_processor'}:
return str(target_type)
return None
@classmethod
def _diagnostic_result(
cls,
*,
matched: bool,
failure_code: str | None = None,
reason: str = '',
binding: dict[str, typing.Any] | None = None,
diagnostic_steps: list[dict[str, typing.Any]] | None = None,
target_name: str | None = None,
target_kind: str | None = None,
) -> dict[str, typing.Any]:
binding = binding or {}
target_type = binding.get('target_type')
target_uuid = binding.get('target_uuid') or ''
matched_binding_index = binding.get('_dry_run_index')
if not isinstance(matched_binding_index, int):
matched_binding_index = binding.get('order')
if not isinstance(matched_binding_index, int):
matched_binding_index = None
target = None
if target_type:
target = {
'target_type': target_type,
'target_uuid': target_uuid or None,
'target_name': target_name,
'kind': cls._target_kind(target_type, target_kind),
}
return {
'matched': matched,
'binding_id': binding.get('id'),
'matched_binding_id': binding.get('id'),
'matched_binding_index': matched_binding_index,
'event_pattern': binding.get('event_pattern'),
'target_type': binding.get('target_type'),
'target_uuid': target_uuid,
'target': target,
'reason': reason,
'failure_code': failure_code,
'diagnostic_steps': cls._format_diagnostic_steps(diagnostic_steps),
'diagnostic_details': diagnostic_steps or [],
}
@staticmethod
def _build_dry_run_event(event_type: str, event_data: typing.Any, context: typing.Any) -> dict[str, typing.Any]:
event: dict[str, typing.Any] = {}
if isinstance(event_data, dict):
event.update(event_data)
elif event_data is not None:
raise ValueError('event_data must be an object')
event['type'] = event_type
if context is None:
return event
if not isinstance(context, dict):
raise ValueError('context must be an object')
event['context'] = context
return event
@staticmethod
def _normalize_dry_run_bindings(bindings: typing.Any) -> list[dict[str, typing.Any]]:
if bindings is None:
return []
if not isinstance(bindings, list):
raise ValueError('event_bindings must be an array')
normalized: list[dict[str, typing.Any]] = []
for index, raw_binding in enumerate(bindings):
if not isinstance(raw_binding, dict):
continue
event_pattern = str(raw_binding.get('event_pattern') or '').strip()
target_type = str(raw_binding.get('target_type') or '').strip()
if not event_pattern or not target_type:
continue
try:
priority = int(raw_binding.get('priority') or 0)
except (TypeError, ValueError):
priority = 0
target_uuid = str(raw_binding.get('target_uuid') or '').strip()
if target_type == 'discard':
target_uuid = ''
filters = raw_binding.get('filters') if isinstance(raw_binding.get('filters'), list) else []
normalized.append(
{
'id': raw_binding.get('id'),
'event_pattern': event_pattern,
'target_type': target_type,
'target_uuid': target_uuid,
'filters': filters,
'priority': priority,
'enabled': bool(raw_binding.get('enabled', True)),
# For draft bindings, current array order is the effective order
# that would be persisted on save.
'order': index,
'_dry_run_index': index,
}
)
return normalized
@staticmethod
def _index_event_bindings(bindings: list[dict[str, typing.Any]]) -> list[dict[str, typing.Any]]:
indexed: list[dict[str, typing.Any]] = []
for index, binding in enumerate(bindings):
copied = binding.copy()
copied.setdefault('_dry_run_index', index)
indexed.append(copied)
return indexed
async def _get_pipeline_entity(
self, context: TenantContext, pipeline_uuid: str
) -> persistence_pipeline.LegacyPipeline | None:
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
),
persistence_pipeline.LegacyPipeline,
context,
)
)
return result.first()
async def _get_agent_entity(self, context: TenantContext, agent_uuid: str) -> persistence_agent.Agent | None:
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_agent.Agent).where(persistence_agent.Agent.uuid == agent_uuid),
persistence_agent.Agent,
context,
)
)
return result.first()
async def dry_run_event_route(
self,
tenant_context: TenantContext,
bot_uuid: str,
event_type: str,
event_data: dict[str, typing.Any] | None = None,
context: dict[str, typing.Any] | None = None,
event_bindings: list[dict[str, typing.Any]] | None = None,
) -> dict[str, typing.Any]:
"""Diagnose Bot event routing without dispatching to Agent, Pipeline, or platform actions."""
from ....platform.botmgr import RuntimeBot
event_type = str(event_type or '').strip()
if not event_type:
return self._diagnostic_result(
matched=False,
failure_code=self.FAILURE_INVALID_EVENT,
reason='event_type is required',
diagnostic_steps=[
{
'step': 'validate_event',
'matched': False,
'failure_code': self.FAILURE_INVALID_EVENT,
'reason': 'event_type is required',
}
],
)
bot = await self.get_bot(tenant_context, bot_uuid, include_secret=False)
if bot is None:
raise Exception('Bot not found')
try:
event = self._build_dry_run_event(event_type, event_data, context)
bindings = (
self._normalize_dry_run_bindings(event_bindings)
if event_bindings is not None
else self._index_event_bindings(
RuntimeBot._get_event_bindings_from_value(bot.get('event_bindings') or [])
)
)
except ValueError as exc:
return self._diagnostic_result(
matched=False,
failure_code=self.FAILURE_INVALID_EVENT,
reason=str(exc),
diagnostic_steps=[
{
'step': 'validate_event',
'matched': False,
'failure_code': self.FAILURE_INVALID_EVENT,
'reason': str(exc),
}
],
)
selected_binding, diagnostic_steps = RuntimeBot._evaluate_eba_event_bindings(
bindings,
event,
event_type,
)
if selected_binding is None:
return self._diagnostic_result(
matched=False,
failure_code=self.FAILURE_ROUTE_NOT_FOUND,
reason='No enabled event binding matched event_type and filters',
diagnostic_steps=diagnostic_steps,
)
target_type = selected_binding.get('target_type')
target_uuid = str(selected_binding.get('target_uuid') or '')
if target_type == 'discard':
return self._diagnostic_result(
matched=True,
binding=selected_binding,
reason='Event route matched discard target',
diagnostic_steps=diagnostic_steps,
)
if target_type == 'pipeline':
if not RuntimeBot._is_message_event_type(event_type):
return self._diagnostic_result(
matched=False,
binding=selected_binding,
failure_code=self.FAILURE_PROCESSOR_INCOMPATIBLE,
reason='Pipeline targets only support message events',
diagnostic_steps=diagnostic_steps
+ [
{
'step': 'validate_processor',
'binding_id': selected_binding.get('id'),
'target_type': target_type,
'target_uuid': target_uuid,
'matched': False,
'failure_code': self.FAILURE_PROCESSOR_INCOMPATIBLE,
'reason': 'Pipeline targets only support message events',
}
],
)
pipeline = await self._get_pipeline_entity(tenant_context, target_uuid) if target_uuid else None
if pipeline is None:
return self._diagnostic_result(
matched=False,
binding=selected_binding,
failure_code=self.FAILURE_PROCESSOR_NOT_FOUND,
reason='Pipeline target not found',
diagnostic_steps=diagnostic_steps
+ [
{
'step': 'validate_processor',
'binding_id': selected_binding.get('id'),
'target_type': target_type,
'target_uuid': target_uuid,
'matched': False,
'failure_code': self.FAILURE_PROCESSOR_NOT_FOUND,
'reason': 'Pipeline target not found',
}
],
)
return self._diagnostic_result(
matched=True,
binding=selected_binding,
target_name=getattr(pipeline, 'name', None),
reason='Event route matched pipeline target',
diagnostic_steps=diagnostic_steps,
)
if target_type in {'agent', 'event_processor'}:
agent = await self._get_agent_entity(tenant_context, target_uuid)
if agent is None or getattr(agent, 'kind', 'agent') != target_type:
return self._diagnostic_result(
matched=False,
binding=selected_binding,
failure_code=self.FAILURE_PROCESSOR_NOT_FOUND,
reason='Agent target not found',
diagnostic_steps=diagnostic_steps
+ [
{
'step': 'validate_processor',
'binding_id': selected_binding.get('id'),
'target_type': target_type,
'target_uuid': target_uuid,
'matched': False,
'failure_code': self.FAILURE_PROCESSOR_NOT_FOUND,
'reason': 'Agent target not found',
}
],
)
if not RuntimeBot._agent_supports_event_type(getattr(agent, 'supported_event_patterns', None), event_type):
return self._diagnostic_result(
matched=False,
binding=selected_binding,
failure_code=self.FAILURE_PROCESSOR_INCOMPATIBLE,
reason='Agent target does not support this event type',
diagnostic_steps=diagnostic_steps
+ [
{
'step': 'validate_processor',
'binding_id': selected_binding.get('id'),
'target_type': target_type,
'target_uuid': target_uuid,
'matched': False,
'failure_code': self.FAILURE_PROCESSOR_INCOMPATIBLE,
'reason': 'Agent target does not support this event type',
}
],
)
return self._diagnostic_result(
matched=True,
binding=selected_binding,
target_name=getattr(agent, 'name', None),
target_kind=getattr(agent, 'kind', None),
reason='Event route matched agent target',
diagnostic_steps=diagnostic_steps,
)
return self._diagnostic_result(
matched=False,
binding=selected_binding,
failure_code=self.FAILURE_PROCESSOR_INCOMPATIBLE,
reason=f'Unsupported event binding target type: {target_type}',
diagnostic_steps=diagnostic_steps
+ [
{
'step': 'validate_processor',
'binding_id': selected_binding.get('id'),
'target_type': target_type,
'target_uuid': target_uuid,
'matched': False,
'failure_code': self.FAILURE_PROCESSOR_INCOMPATIBLE,
'reason': f'Unsupported event binding target type: {target_type}',
}
],
)
async def _normalize_event_bindings(self, context: TenantContext, bindings: list[dict] | None) -> list[dict]:
"""Validate and normalize Bot event bindings."""
if not bindings:
return []
normalized: list[dict] = []
for index, raw_binding in enumerate(bindings):
if not isinstance(raw_binding, dict):
continue
event_pattern = str(raw_binding.get('event_pattern') or '').strip()
target_type = str(raw_binding.get('target_type') or '').strip()
target_uuid = str(raw_binding.get('target_uuid') or '').strip()
if not event_pattern or not target_type:
continue
if target_type == 'pipeline':
if not self._is_message_event_pattern(event_pattern):
raise ValueError('Pipeline can only be bound to message events')
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_pipeline.LegacyPipeline.uuid).where(
persistence_pipeline.LegacyPipeline.uuid == target_uuid
),
persistence_pipeline.LegacyPipeline,
context,
)
)
if result.first() is None:
raise ValueError('Pipeline not found')
elif target_type == 'agent':
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_agent.Agent).where(persistence_agent.Agent.uuid == target_uuid),
persistence_agent.Agent,
context,
)
)
agent = result.first()
if agent is None or agent.kind != target_type:
raise ValueError('Processor not found')
if not self._agent_supports_event_pattern(agent.supported_event_patterns, event_pattern):
raise ValueError('Agent does not support this event pattern')
elif target_type == 'discard':
target_uuid = ''
else:
raise ValueError(f'Unsupported event binding target type: {target_type}')
normalized.append(
{
'id': raw_binding.get('id') or str(uuid.uuid4()),
'event_pattern': event_pattern,
'target_type': target_type,
'target_uuid': target_uuid,
'filters': raw_binding.get('filters') or [],
'priority': int(raw_binding.get('priority') or 0),
'enabled': bool(raw_binding.get('enabled', True)),
'description': raw_binding.get('description') or '',
'order': index,
}
)
return normalized
async def _normalize_plugin_processors(self, context: TenantContext, subscriptions: typing.Any) -> list[dict]:
"""Validate explicit subscriptions within this Workspace; events come from the Runner."""
if not isinstance(subscriptions, list):
raise ValueError('plugin_processors must be an array')
normalized = []
seen = set()
for subscription in subscriptions:
if not isinstance(subscription, dict):
raise ValueError('Each plugin processor binding must be an object')
processor_uuid = subscription.get('processor_uuid')
if not isinstance(processor_uuid, str) or not processor_uuid.strip():
raise ValueError('Plugin processor UUID is required')
if processor_uuid in seen:
raise ValueError('A plugin processor can only be bound once per bot')
enabled = subscription.get('enabled', True)
if not isinstance(enabled, bool):
raise ValueError('Plugin processor enabled must be a boolean')
agent = await self._get_agent_entity(context, processor_uuid)
if agent is None or agent.kind != 'event_processor':
raise ValueError('Plugin processor not found')
if enabled:
try:
descriptor = await self.ap.runner_registry.get(context, agent.component_ref)
except RunnerNotFoundError as exc:
raise ValueError('Runner component is unavailable') from exc
if 'event' not in descriptor.usages or not descriptor.supported_event_patterns:
raise ValueError('Select an available Runner that declares event usage')
seen.add(processor_uuid)
normalized.append({'processor_uuid': processor_uuid, 'enabled': enabled})
return normalized
async def _prepare_bot_data(self, context: TenantContext, bot_data: dict, *, include_uuid: bool) -> dict:
"""Normalize Bot write payloads to the current event-routing model."""
update_data = bot_data.copy()
if not include_uuid:
update_data.pop('uuid', None)
update_data = {key: value for key, value in update_data.items() if key in self.BOT_FIELDS}
if 'adapter' in update_data:
update_data['adapter'] = canonical_adapter_name(update_data['adapter'])
if 'event_bindings' in update_data:
update_data['event_bindings'] = await self._normalize_event_bindings(
context, update_data.get('event_bindings')
)
if 'plugin_processors' in update_data:
update_data['plugin_processors'] = await self._normalize_plugin_processors(
context, update_data['plugin_processors']
)
return update_data
async def get_bots(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
"""获取所有机器人"""
result = await self.ap.persistence_mgr.execute_async(
@@ -33,7 +615,10 @@ class BotService:
if not include_secret:
masked_columns = ['adapter_config']
return [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
serialized = [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
for bot in serialized:
bot['adapter'] = canonical_adapter_name(bot['adapter'])
return serialized
async def get_bot(self, context: TenantContext, bot_uuid: str, include_secret: bool = False) -> dict | None:
"""获取机器人"""
@@ -54,7 +639,9 @@ class BotService:
if not include_secret:
masked_columns = ['adapter_config']
return self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
serialized = self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
serialized['adapter'] = canonical_adapter_name(serialized['adapter'])
return serialized
async def get_runtime_bot_info(
self,
@@ -73,18 +660,10 @@ class BotService:
if runtime_bot is not None:
adapter_runtime_values['bot_account_id'] = runtime_bot.adapter.bot_account_id
# Webhook URL for unified webhook adapters (independent of bot running state)
if persistence_bot['adapter'] in [
'wecom',
'wecombot',
'officialaccount',
'qqofficial',
'slack',
'wecomcs',
'LINE',
'lark',
'http_bot',
]:
# Webhook URL for adapters that declare a generated webhook config item.
# This is manifest-driven so EBA adapters do not need to be mirrored in a
# second hard-coded list.
if self._adapter_declares_webhook_url(persistence_bot['adapter']):
webhook_prefix = self.ap.instance_config.data['api'].get('webhook_prefix', 'http://127.0.0.1:5300')
extra_webhook_prefix = self.ap.instance_config.data['api'].get('extra_webhook_prefix', '')
webhook_url = f'/bots/{bot_uuid}'
@@ -114,24 +693,11 @@ class BotService:
raise ValueError(f'Maximum number of bots ({max_bots}) reached')
# TODO: 检查配置信息格式
bot_data = bot_data.copy()
bot_data = await self._prepare_bot_data(context, bot_data, include_uuid=True)
bot_data['uuid'] = str(uuid.uuid4())
bot_data['workspace_uuid'] = workspace_uuid
# bind the most recently updated pipeline if any exist
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
persistence_pipeline.LegacyPipeline,
context,
)
.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
.limit(1)
)
pipeline = result.first()
if pipeline is not None:
bot_data['use_pipeline_uuid'] = pipeline.uuid
bot_data['use_pipeline_name'] = pipeline.name
bot_data.setdefault('event_bindings', [])
bot_data.setdefault('plugin_processors', [])
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(bot_data))
@@ -139,42 +705,17 @@ class BotService:
try:
await self.ap.platform_mgr.load_bot(context, bot)
except Exception:
# The bot row was already inserted above; without this rollback a
# failing adapter constructor (e.g. a missing optional credential
# key) would leave a permanently disabled orphan bot in the DB.
await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_data['uuid'])
)
raise
except Exception as exc:
raise BotApplyError(bot_error_message(exc, bot), bot['uuid']) from exc
return bot_data['uuid']
async def update_bot(self, context: TenantContext, bot_uuid: str, bot_data: dict) -> None:
"""Update bot"""
workspace_uuid = require_workspace_uuid(context)
update_data = bot_data.copy()
update_data.pop('uuid', None)
update_data = await self._prepare_bot_data(context, bot_data, include_uuid=False)
update_data.pop('workspace_uuid', None)
# set use_pipeline_name
if 'use_pipeline_uuid' in update_data:
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
),
persistence_pipeline.LegacyPipeline,
workspace_uuid,
)
)
pipeline = result.first()
if pipeline is not None:
update_data['use_pipeline_name'] = pipeline.name
else:
raise WorkspaceNotFoundError('Pipeline not found')
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid),
@@ -184,17 +725,28 @@ class BotService:
)
if getattr(result, 'rowcount', None) == 0:
raise WorkspaceNotFoundError('Bot not found')
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
# select from db
runtime_fields = {'adapter', 'adapter_config', 'enable', 'event_bindings', 'plugin_processors'}
if not runtime_fields.intersection(update_data):
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
if runtime_bot is not None:
if 'name' in update_data:
runtime_bot.bot_entity.name = update_data['name']
if 'description' in update_data:
runtime_bot.bot_entity.description = update_data['description']
return
# Persisted configuration is distinct from applying it to the running adapter.
bot = await self.get_bot(context, bot_uuid, include_secret=True)
try:
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
if runtime_bot.enable:
await runtime_bot.run()
except Exception as exc:
raise BotApplyError(bot_error_message(exc, bot), bot['uuid']) from exc
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
if runtime_bot.enable:
await runtime_bot.run()
# update all conversation that use this bot
# Reset conversations using this bot after its configuration is applied.
for session in self.ap.sess_mgr.session_list:
if (
session.using_conversation is not None
@@ -229,6 +781,76 @@ class BotService:
return [log.to_json() for log in logs], total_count
async def list_event_route_statuses(self, context: TenantContext, bot_uuid: str) -> dict[str, typing.Any]:
"""Return recent runtime status for Bot event routes from in-memory Bot logs."""
from ....platform.botmgr import RuntimeBot
bot = await self.get_bot(context, bot_uuid, include_secret=False)
if bot is None:
raise WorkspaceNotFoundError('Bot not found')
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
latest_by_binding: dict[str, dict[str, typing.Any]] = {}
unmatched_events: list[dict[str, typing.Any]] = []
runtime_logs = getattr(getattr(runtime_bot, 'logger', None), 'logs', [])
for log in runtime_logs:
status = self._event_route_status_from_log(log)
if status is None:
continue
binding_id = status.get('binding_id')
if binding_id:
latest_by_binding[str(binding_id)] = status
else:
unmatched_events.append(status)
runtime_entity = getattr(runtime_bot, 'bot_entity', None)
raw_bindings = getattr(runtime_entity, 'event_bindings', None) if runtime_entity is not None else None
if raw_bindings is None:
raw_bindings = bot.get('event_bindings') or []
bindings = RuntimeBot._get_event_bindings_from_value(raw_bindings)
routes: list[dict[str, typing.Any]] = []
current_binding_ids: set[str] = set()
for index, binding in enumerate(bindings):
binding_id = binding.get('id')
if binding_id:
current_binding_ids.add(str(binding_id))
route_status = {
'binding_id': binding_id,
'event_pattern': binding.get('event_pattern'),
'event_type': None,
'target_type': binding.get('target_type'),
'target_uuid': binding.get('target_uuid') or '',
'last_status': None,
'failure_code': None,
'reason': None,
'run_id': None,
'timestamp': None,
'seq_id': None,
'level': None,
'message': '',
'order': binding.get('order', index),
'enabled': binding.get('enabled', True),
'current': True,
}
if binding_id and str(binding_id) in latest_by_binding:
route_status.update(latest_by_binding[str(binding_id)])
route_status['order'] = binding.get('order', index)
route_status['enabled'] = binding.get('enabled', True)
route_status['current'] = True
routes.append(route_status)
stale_routes = [
{**status, 'current': False}
for binding_id, status in latest_by_binding.items()
if binding_id not in current_binding_ids
]
return {
'routes': routes,
'unmatched_events': unmatched_events[-10:],
'stale_routes': stale_routes,
}
async def send_http_bot_test_message(
self,
context: TenantContext,
@@ -0,0 +1,43 @@
"""User-facing bot configuration errors without credentials or validation inputs."""
import json
import pydantic
from .secrets import redact_secrets
class BotApplyError(Exception):
"""Configuration was persisted, but the runtime could not apply it."""
def __init__(self, message: str, bot_uuid: str | None = None):
super().__init__(message)
self.bot_uuid = bot_uuid
def bot_error_message(error: Exception, configuration: dict) -> str:
if isinstance(error, pydantic.ValidationError):
text = '; '.join(
f'{".".join(map(str, item["loc"]))}: {item["msg"]}'
for item in error.errors(include_input=False, include_context=False, include_url=False)
)
else:
text = str(error).strip() or type(error).__name__
replacements = []
def collect(original, masked):
if isinstance(original, dict) and isinstance(masked, dict):
for key, value in original.items():
collect(value, masked.get(key))
elif isinstance(original, (list, tuple)) and isinstance(masked, (list, tuple)):
for value, replacement in zip(original, masked):
collect(value, replacement)
elif isinstance(original, str) and original and original != masked:
for representation in {original, repr(original)[1:-1], json.dumps(original, ensure_ascii=False)[1:-1]}:
replacements.append((representation, str(masked)))
collect(configuration, redact_secrets(configuration))
for original, masked in sorted(replacements, key=lambda item: len(item[0]), reverse=True):
text = text.replace(original, masked)
return text[:2000]
+64 -19
View File
@@ -71,23 +71,26 @@ class KnowledgeService:
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
retrieval_settings = kb_data.get('retrieval_settings', {})
defer_initialization = kb_data.get('defer_initialization') is True
# Validate required fields based on plugin's creation_schema and retrieval_schema
await self._validate_schema_required_fields(
context,
knowledge_engine_plugin_id,
creation_settings,
retrieval_settings,
)
if not defer_initialization:
await self._validate_schema_required_fields(
context,
knowledge_engine_plugin_id,
creation_settings,
retrieval_settings,
)
kb = await self.ap.rag_mgr.create_knowledge_base(
context,
name=kb_data.get('name', 'Untitled'),
knowledge_engine_plugin_id=knowledge_engine_plugin_id,
creation_settings=creation_settings,
retrieval_settings=retrieval_settings,
description=kb_data.get('description', ''),
)
create_kwargs = {
'name': kb_data.get('name', 'Untitled'),
'knowledge_engine_plugin_id': knowledge_engine_plugin_id,
'creation_settings': creation_settings,
'retrieval_settings': retrieval_settings,
'description': kb_data.get('description', ''),
}
if defer_initialization:
create_kwargs['initialize'] = False
kb = await self.ap.rag_mgr.create_knowledge_base(context, **create_kwargs)
return kb.uuid
async def _validate_schema_required_fields(
@@ -205,10 +208,32 @@ class KnowledgeService:
) -> None:
"""更新知识库"""
workspace_uuid = require_workspace_uuid(context)
if await self.get_knowledge_base(context, kb_uuid) is None:
current = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
if current is None:
raise WorkspaceNotFoundError('Knowledge base not found')
# Filter to only mutable fields
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
should_initialize = current.get('initialized', True) is False and kb_data.get('initialize_engine') is True
if should_initialize:
plugin_id = current.get('knowledge_engine_plugin_id')
if not plugin_id:
raise ValueError('knowledge_engine_plugin_id is required')
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
retrieval_settings = kb_data.get('retrieval_settings', {})
await self._validate_schema_required_fields(
context,
plugin_id,
creation_settings,
retrieval_settings,
)
filtered_data = {
'name': kb_data.get('name', current.get('name', 'Untitled')),
'description': kb_data.get('description', current.get('description', '')),
'creation_settings': creation_settings,
'retrieval_settings': retrieval_settings,
'initialized': True,
}
else:
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
if not filtered_data:
return
@@ -224,8 +249,28 @@ class KnowledgeService:
kb = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
if kb is None:
raise WorkspaceNotFoundError('Knowledge base not found')
if kb.get('initialized', True) is False:
return
await self.ap.rag_mgr.load_knowledge_base(context, kb)
runtime_kb = await self.ap.rag_mgr.load_knowledge_base(context, kb)
if should_initialize:
try:
await runtime_kb._on_kb_create(self._execution_context(context))
except Exception:
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(context, kb_uuid)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.KnowledgeBase)
.values(
name=current.get('name', 'Untitled'),
description=current.get('description', ''),
creation_settings=current.get('creation_settings', {}),
retrieval_settings=current.get('retrieval_settings', {}),
initialized=False,
)
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
)
raise
async def _check_doc_capability(self, context: TenantContext, kb_uuid: str, operation: str) -> None:
"""Check if the KB's Knowledge Engine supports document operations.
+109 -2
View File
@@ -142,8 +142,6 @@ class MaintenanceService:
('logs', Path('data/logs')),
('storage', Path('data/storage')),
('vector_store', Path('data/chroma')),
('plugins', Path('data/plugins')),
('mcp', Path('data/mcp')),
('temp', Path('data/temp')),
]
else:
@@ -151,6 +149,11 @@ class MaintenanceService:
roots = [('storage', scoped_storage_path)]
sections = await asyncio.to_thread(self._collect_sections, roots)
runtime_processes = await self._runtime_storage_processes(
context,
core_sections=sections,
include_instance_storage=is_oss_singleton,
)
monitoring_counts = await self._monitoring_counts(context)
binary_storage = await self._binary_storage_stats(context)
@@ -171,6 +174,12 @@ class MaintenanceService:
'log_retention_days': log_retention_days,
},
'sections': sections,
'processes': runtime_processes,
'total_size_bytes': sum(
int(process.get('size_bytes') or 0)
for process in runtime_processes
if process.get('status') == 'available'
),
'database': {
'type': database_type,
'monitoring_counts': monitoring_counts,
@@ -183,6 +192,104 @@ class MaintenanceService:
'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
}
async def _runtime_storage_processes(
self,
context: TenantContext,
*,
core_sections: list[dict[str, Any]],
include_instance_storage: bool,
) -> list[dict[str, Any]]:
core_process = {
'key': 'langbot',
'status': 'available',
'source': 'local_process',
'size_bytes': sum(int(item.get('size_bytes') or 0) for item in core_sections),
'directories': [
{
**item,
'kind': 'root',
'error_count': 0,
}
for item in core_sections
],
}
plugin_task = self._plugin_runtime_storage_process(include_instance_storage)
box_task = self._box_runtime_storage_process(context)
plugin_process, box_process = await asyncio.gather(plugin_task, box_task)
return [core_process, plugin_process, box_process]
async def _plugin_runtime_storage_process(self, include_instance_storage: bool) -> dict[str, Any]:
if not include_instance_storage:
return self._unavailable_process(
'plugin_runtime',
'not_applicable',
'Instance-wide Plugin Runtime storage is hidden in multi-Workspace deployments',
)
connector = getattr(self.ap, 'plugin_connector', None)
if connector is None or not getattr(connector, 'is_enable_plugin', True):
return self._unavailable_process('plugin_runtime', 'disabled', 'Plugin Runtime is disabled')
runtime_handler = getattr(connector, 'handler', None)
get_analysis = getattr(runtime_handler, 'get_storage_analysis', None)
if not callable(get_analysis):
return self._unavailable_process(
'plugin_runtime',
'unavailable',
'Plugin Runtime does not support storage analysis',
)
try:
result = await get_analysis()
except Exception as exc:
self.ap.logger.warning(f'Failed to collect Plugin Runtime storage analysis: {exc}')
return self._unavailable_process('plugin_runtime', 'unavailable', str(exc))
return self._available_process('plugin_runtime', result, source='runtime_rpc')
async def _box_runtime_storage_process(self, context: TenantContext) -> dict[str, Any]:
box_service = getattr(self.ap, 'box_service', None)
if box_service is None or not getattr(box_service, 'enabled', True):
return self._unavailable_process('box_runtime', 'disabled', 'Box Runtime is disabled')
get_analysis = getattr(box_service, 'get_storage_analysis', None)
if not callable(get_analysis):
return self._unavailable_process(
'box_runtime',
'unavailable',
'Box Runtime does not support storage analysis',
)
try:
result = await get_analysis(context)
except Exception as exc:
self.ap.logger.warning(f'Failed to collect Box Runtime storage analysis: {exc}')
return self._unavailable_process('box_runtime', 'unavailable', str(exc))
return self._available_process('box_runtime', result, source='runtime_rpc')
@staticmethod
def _available_process(key: str, result: Any, *, source: str) -> dict[str, Any]:
payload = result if isinstance(result, dict) else {}
directories = payload.get('directories')
if not isinstance(directories, list):
directories = []
return {
'key': key,
'status': 'available',
'source': source,
'size_bytes': int(payload.get('size_bytes') or 0),
'directories': directories,
'active_sessions': payload.get('active_sessions'),
'managed_processes': payload.get('managed_processes'),
}
@staticmethod
def _unavailable_process(key: str, status: str, error: str) -> dict[str, Any]:
return {
'key': key,
'status': status,
'source': 'runtime_rpc',
'size_bytes': None,
'directories': [],
'error': error,
}
def _collect_sections(
self,
roots: list[tuple[str, Path | None]],
+136 -46
View File
@@ -3,11 +3,17 @@ from __future__ import annotations
import uuid
import json
import sqlalchemy
import typing
from ....core import app
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....entity.persistence import pipeline as persistence_pipeline
from ....pipeline.extension_preferences import (
normalize_extension_preferences,
validate_extension_preferences,
)
from ....workspace.errors import WorkspaceNotFoundError
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
from .secrets import redact_secrets, restore_secret_placeholders
from .tenant import TenantContext, require_workspace_uuid, scope_statement
@@ -16,7 +22,6 @@ default_stage_order = [
'BanSessionCheckStage', # 封禁会话检查
'PreContentFilterStage', # 内容过滤前置阶段
'PreProcessor', # 预处理器
'ConversationMessageTruncator', # 会话消息截断器
'RequireRateLimitOccupancy', # 请求速率限制占用
'MessageProcessor', # 处理器
'ReleaseRateLimitOccupancy', # 释放速率限制占用
@@ -33,12 +38,68 @@ class PipelineService:
def __init__(self, ap: app.Application) -> None:
self.ap = ap
@staticmethod
def _get_default_values_from_schema(
config_schema: list[dict[str, typing.Any]],
) -> dict[str, typing.Any]:
return {item['name']: item['default'] for item in config_schema if item.get('name') and 'default' in item}
async def get_default_pipeline_config(self, context: TenantContext) -> dict[str, typing.Any]:
from ....utils import paths as path_utils
template_path = path_utils.get_resource_path('templates/default-pipeline-config.json')
with open(template_path, 'r', encoding='utf-8') as f:
config = json.load(f)
registry = getattr(self.ap, 'runner_registry', None)
if registry is None:
return config
try:
runners = await registry.list_runners(context, bound_plugins=None)
except Exception as exc:
self.ap.logger.warning(f'Failed to load Runner defaults for pipeline config: {exc}')
return config
if not runners:
return config
selected = runners[0]
ai_config = config.setdefault('ai', {})
runner_config = ai_config.setdefault('runner', {})
runner_config['id'] = selected.id
runner_config.setdefault('expire-time', 0)
ai_config['runner_config'] = {selected.id: self._get_default_values_from_schema(selected.config_schema)}
return config
async def get_pipeline_metadata(self, context: TenantContext) -> list[dict]:
require_workspace_uuid(context)
import copy
ai_metadata = copy.deepcopy(self.ap.pipeline_config_meta_ai)
runner_stage = next(
(stage for stage in ai_metadata.get('stages', []) if stage.get('name') == 'runner'),
None,
)
if runner_stage:
for config_item in runner_stage.get('config', []):
if config_item.get('name') != 'id':
continue
try:
runner_options, runner_stages = await self.ap.runner_registry.get_runner_metadata_for_pipeline(
context
)
config_item['options'] = runner_options
if runner_options and 'default' not in config_item:
config_item['default'] = runner_options[0]['name']
existing = {stage.get('name') for stage in ai_metadata.get('stages', [])}
ai_metadata.setdefault('stages', []).extend(
stage for stage in runner_stages if stage.get('name') not in existing
)
except Exception as exc:
self.ap.logger.warning(f'Failed to load Runner pipeline metadata: {exc}')
return [
self.ap.pipeline_config_meta_trigger,
self.ap.pipeline_config_meta_safety,
self.ap.pipeline_config_meta_ai,
ai_metadata,
self.ap.pipeline_config_meta_output,
]
@@ -101,9 +162,11 @@ class PipelineService:
return serialized if include_secret else redact_secrets(serialized)
async def create_pipeline(self, context: TenantContext, pipeline_data: dict, default: bool = False) -> str:
from ....utils import paths as path_utils
workspace_uuid = require_workspace_uuid(context)
if 'extensions_preferences' in pipeline_data:
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
if 'config' in pipeline_data:
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
# Check limitation
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
max_pipelines = limitation.get('max_pipelines', -1)
@@ -119,9 +182,8 @@ class PipelineService:
pipeline_data['stages'] = default_stage_order.copy()
pipeline_data['is_default'] = default
template_path = path_utils.get_resource_path('templates/default-pipeline-config.json')
with open(template_path, 'r', encoding='utf-8') as f:
pipeline_data['config'] = json.load(f)
pipeline_data['config'] = await self.get_default_pipeline_config(context)
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
# Ensure extensions_preferences is set with enable_all_plugins and enable_all_mcp_servers=True by default
if 'extensions_preferences' not in pipeline_data:
@@ -151,16 +213,31 @@ class PipelineService:
pipeline_data.pop(protected_field, None)
if 'config' in pipeline_data:
current_config = None
if contains_secret_placeholder(pipeline_data['config']):
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
if current_pipeline is None:
raise WorkspaceNotFoundError('Pipeline not found')
current_config = current_pipeline.get('config', {})
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
if current_pipeline is None:
raise WorkspaceNotFoundError('Pipeline not found')
current_config = current_pipeline.get('config', {})
old_ai = current_config.get('ai', {}) if isinstance(current_config, dict) else {}
old_runner = old_ai.get('runner') if isinstance(old_ai, dict) else None
legacy_binding = (
isinstance(old_runner, str)
and bool(old_runner)
or isinstance(old_runner, dict)
and bool(old_runner.get('runner'))
and not old_runner.get('id')
)
new_config = pipeline_data['config']
new_ai = new_config.get('ai', {}) if isinstance(new_config, dict) else {}
new_runner = new_ai.get('runner') if isinstance(new_ai, dict) else None
if legacy_binding and isinstance(new_runner, dict) and new_runner.get('id'):
raise ValueError('manual_migration_required')
pipeline_data['config'] = restore_secret_placeholders(
pipeline_data['config'],
current_config if current_config is not None else {},
)
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
if 'extensions_preferences' in pipeline_data:
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
@@ -178,25 +255,6 @@ class PipelineService:
if pipeline is None:
raise WorkspaceNotFoundError('Pipeline not found')
if 'name' in pipeline_data:
from ....entity.persistence import bot as persistence_bot
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_bot.Bot).where(
persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid
),
persistence_bot.Bot,
workspace_uuid,
)
)
bots = result.all()
for bot in bots:
bot_data = {'use_pipeline_name': pipeline_data['name']}
await self.ap.bot_service.update_bot(context, bot.uuid, bot_data)
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
@@ -260,18 +318,7 @@ class PipelineService:
'stages': original_pipeline.stages.copy() if original_pipeline.stages else default_stage_order.copy(),
'config': original_pipeline.config.copy() if original_pipeline.config else {},
'is_default': False,
'extensions_preferences': (
original_pipeline.extensions_preferences.copy()
if original_pipeline.extensions_preferences
else {
'enable_all_plugins': True,
'enable_all_mcp_servers': True,
'plugins': [],
'mcp_servers': [],
'mcp_resources': [],
'mcp_resource_agent_read_enabled': True,
}
),
'extensions_preferences': normalize_extension_preferences(original_pipeline.extensions_preferences),
}
# Insert the new pipeline
@@ -300,6 +347,30 @@ class PipelineService:
) -> None:
"""Update the bound plugins and MCP servers for a pipeline"""
workspace_uuid = require_workspace_uuid(context)
extension_updates: dict[str, typing.Any] = {
'enable_all_plugins': enable_all_plugins,
'enable_all_mcp_servers': enable_all_mcp_servers,
'enable_all_skills': enable_all_skills,
'plugins': bound_plugins,
}
if bound_mcp_servers is not None:
extension_updates['mcp_servers'] = bound_mcp_servers
if bound_skills is not None:
extension_updates['skills'] = bound_skills
if bound_mcp_resources is not None:
extension_updates['mcp_resources'] = bound_mcp_resources
if mcp_resource_agent_read_enabled is not None:
extension_updates['mcp_resource_agent_read_enabled'] = mcp_resource_agent_read_enabled
self._validate_extension_preferences(
extension_updates,
context='Pipeline extension',
field_aliases={
'plugins': 'bound_plugins',
'mcp_servers': 'bound_mcp_servers',
'skills': 'bound_skills',
'mcp_resources': 'bound_mcp_resources',
},
)
# Get current pipeline
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
@@ -316,7 +387,7 @@ class PipelineService:
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
# Update extensions_preferences
extensions_preferences = pipeline.extensions_preferences or {}
extensions_preferences = normalize_extension_preferences(pipeline.extensions_preferences)
extensions_preferences['enable_all_plugins'] = enable_all_plugins
extensions_preferences['enable_all_mcp_servers'] = enable_all_mcp_servers
extensions_preferences['enable_all_skills'] = enable_all_skills
@@ -344,3 +415,22 @@ class PipelineService:
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
@staticmethod
def _validate_extension_preferences(
value: typing.Any,
*,
context: str = 'Pipeline extensions_preferences',
field_aliases: typing.Mapping[str, str] | None = None,
) -> dict[str, typing.Any]:
validated = validate_extension_preferences(
value,
context=context,
field_aliases=field_aliases,
)
RunnerConfigResolver.validate_mcp_resource_attachments(
validated.get('mcp_resources'),
context=context,
field_name='mcp_resources',
)
return validated
@@ -0,0 +1,936 @@
"""Explicit, tenant-scoped migration. Preview never contacts plugin runtimes.
The task handle is process-local. Original snapshots and pending activation are
persistent; a refreshed preview is required after observation loss or restart.
"""
from __future__ import annotations
import asyncio
import copy
import hashlib
import hmac
import json
import math
import time
import secrets
import sys
import uuid
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
import httpx
import sqlalchemy as sa
from ..authz import Permission, permissions_for_role, require_permission
from ..context import ExecutionContext, PrincipalType, RequestContext
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....agent.runner import config_schema
from ....agent.runner.resource_builder import AgentResourceBuilder
from ....agent.runner.resource_policy import ResourcePolicyProjector
from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel
from ....entity.persistence.rag import KnowledgeBase
from ....entity.persistence.mcp import MCPServer
from ....core.taskmgr import TaskContext
from ....entity.persistence.pipeline import LegacyPipeline
from ....entity.persistence.agent_interaction import AgentInteraction
from ....persistence.pipeline_admission import lock_pipeline_admission
from ....entity.persistence.pipeline_migration import PipelineMigrationSnapshot
from ....entity.persistence.plugin import PluginSetting
from ....entity.persistence.user import User
from ....entity.persistence.workspace import Workspace, WorkspaceMembership, WorkspaceExecutionState
from ....plugin.errors import (
PluginRuntimeNotConnectedError,
MarketplacePluginVersionNotFoundError,
PluginInstallationFailedError,
)
from ....pipeline.legacy_config_migration import PLANNER_VERSION, plan_legacy_pipeline
class MigrationInstallContext(TaskContext):
"""Publish only structured installer progress, never upstream logs or URLs."""
ACTIONS = {
'downloading plugin package': 'downloading',
'validating plugin package': 'validating',
'preparing plugin installation': 'preparing',
'installing plugin dependencies': 'installing_deps',
'waiting for plugin initialization': 'activating',
'refreshing plugin components': 'refreshing',
'plugin installed': 'done',
}
METRICS = ('progress_percent', 'download_current', 'download_total', 'download_speed', 'deps_total')
def __init__(self, target):
super().__init__()
self.progress = {
**target,
'status': 'installing',
'stage': 'checking',
'code': None,
'started_at': time.time(),
'finished_at': None,
'steps': [],
}
self._step('checking')
def _step(self, stage):
steps = self.progress['steps']
if steps and steps[-1]['stage'] == stage:
return
now = time.time()
if steps:
steps[-1]['finished_at'] = now
steps.append({'stage': stage, 'started_at': now, 'finished_at': None})
self.progress['stage'] = stage
def set_current_action(self, action):
super().set_current_action(action)
stage = self.ACTIONS.get(action)
if stage:
self._step(stage)
self.publish()
def publish(self):
for key in self.METRICS:
value = self.metadata.get(key)
if type(value) in (int, float) and math.isfinite(value) and value >= 0:
self.progress[key] = min(value, 100) if key == 'progress_percent' else value
self.progress['updated_at'] = time.time()
def finish(self, code=None):
self.publish()
self.progress.update(status='failed' if code else 'completed', code=code, finished_at=time.time())
self.progress['steps'][-1]['finished_at'] = self.progress['finished_at']
if not code:
self.progress['progress_percent'] = 100
async def observe(self):
while True:
self.publish()
await asyncio.sleep(0.2)
class MigrationError(ValueError):
"""Only constant, credential-free codes cross the API/task boundary."""
def __init__(self, code: str, status_code: int = 409):
super().__init__(code)
self.code = code
self.status_code = status_code
def _failure_code(exc: Exception) -> str:
if isinstance(exc, MigrationError):
return exc.code
if isinstance(exc, PluginRuntimeNotConnectedError):
return 'plugin_runtime_unavailable'
return 'migration_failed'
def validate_execute_request(body) -> list[dict]:
if not isinstance(body, dict) or set(body) != {'confirmed', 'items'} or body['confirmed'] is not True:
raise MigrationError('confirmation_required', 400)
items = body['items']
if not isinstance(items, list) or not 1 <= len(items) <= 50:
raise MigrationError('invalid_selection', 400)
seen = set()
for item in items:
if not isinstance(item, dict) or set(item) != {'pipeline_uuid', 'preview_token'}:
raise MigrationError('invalid_selection', 400)
for key, maximum in [('pipeline_uuid', 255), ('preview_token', 256)]:
value = item[key]
if not isinstance(value, str) or not value.strip() or len(value) > maximum or value != value.strip():
raise MigrationError('invalid_selection', 400)
if item['pipeline_uuid'] in seen:
raise MigrationError('invalid_selection', 400)
seen.add(item['pipeline_uuid'])
return copy.deepcopy(items)
def _json(value):
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, default=str)
def _fingerprint(row):
# SQL timestamps are neither monotonic revisions nor portable CAS values.
return hashlib.sha256(
_json(
{k: v for k, v in row.items() if not k.startswith('_') and k not in ('created_at', 'updated_at')}
).encode()
).hexdigest()
class PipelineMigrationService:
def __init__(self, ap):
self.ap = ap
self.pm = ap.persistence_mgr
# Loss of this process key only invalidates previews, never snapshots.
self._token_key = secrets.token_bytes(32)
self._all_tasks: set[str] = set()
async def _binding(self, ctx, session=None):
binding = await self.ap.workspace_service.get_execution_binding(
ctx.workspace_uuid, expected_generation=ctx.placement_generation, session=session
)
if binding.instance_uuid != ctx.instance_uuid:
raise MigrationError('placement_changed')
async def _authorize(self, ctx, session=None):
require_permission(ctx, Permission.RESOURCE_MANAGE)
if ctx.principal.principal_type != PrincipalType.ACCOUNT or not ctx.account_uuid:
raise MigrationError('authorization_changed', 403)
status = (await self.pm.execute_async(sa.select(User.status).where(User.uuid == ctx.account_uuid))).scalar()
if status != 'active':
raise MigrationError('authorization_changed', 403)
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
ctx.account_uuid, ctx.workspace_uuid, session=session
)
if (
access.workspace.uuid != ctx.workspace_uuid
or access.membership.uuid != ctx.workspace.membership_uuid
or access.membership.projection_revision != ctx.workspace.membership_revision
or Permission.RESOURCE_MANAGE.value not in permissions_for_role(access.membership.role)
or access.execution.instance_uuid != ctx.instance_uuid
or access.execution.placement_generation != ctx.placement_generation
):
raise MigrationError('authorization_changed', 403)
await self._binding(ctx, session)
async def _rows(self, ctx, pipeline_uuid=None):
# Read DB-native text for JSON CAS: never compare PostgreSQL JSON using =,
# and never assume Python's JSON whitespace/key order matches stored text.
table = LegacyPipeline.__table__
stmt = sa.select(
table,
*[sa.cast(table.c[k], sa.Text).label('_raw_' + k) for k in ('config', 'stages', 'extensions_preferences')],
)
stmt = stmt.where(table.c.workspace_uuid == ctx.workspace_uuid)
if pipeline_uuid is not None:
stmt = stmt.where(table.c.uuid == pipeline_uuid)
return [dict(r._mapping) for r in (await self.pm.execute_async(stmt.order_by(table.c.uuid))).all()]
async def _plugin(self, ctx, plan):
target = plan.get('target_plugin')
if not target:
return None
t = PluginSetting.__table__
row = (
await self.pm.execute_async(
sa.select(
t.c.enabled, t.c.installation_uuid, t.c.artifact_digest, t.c.runtime_revision, t.c.install_info
).where(
t.c.workspace_uuid == ctx.workspace_uuid,
t.c.plugin_author == target['author'],
t.c.plugin_name == target['name'],
)
)
).first()
return dict(row._mapping) if row else None
def _native_interactions(self, ctx, pipeline_uuid):
"""Inspect the old in-process cache without importing/pruning it.
Native forms have no durable handoff to the plugin runner. Operators
must finish them in the original runtime before migrating; never infer
completion from the cache TTL or submit/cancel a form here.
"""
module = sys.modules.get('langbot.pkg.provider.runners.difysvapi')
if module is None:
return []
cache = getattr(module, '_PENDING_FORMS', None)
if not isinstance(cache, dict):
return ['unavailable']
observed = []
for key, forms in cache.items():
if not isinstance(key, tuple) or len(key) != 8:
if forms:
observed.append('unscoped')
continue
instance, workspace, _, _, pipeline, *_ = key
# Inspect ownership before touching any form contents. Empty scope
# cannot prove this is another tenant; block without disclosing it.
if instance and instance != ctx.instance_uuid or workspace and workspace != ctx.workspace_uuid:
continue
if pipeline and pipeline != pipeline_uuid:
continue
if forms:
observed.append(hashlib.sha256(_json(key).encode()).hexdigest())
return sorted(observed)
async def _interaction_state(self, ctx, pipeline_uuid):
"""Read only correlation/status, never request/submission/token data.
The real table has workspace_id (not workspace_uuid), and no instance
column. Execution placement is verified by the caller; processor_id is
the globally unique pipeline UUID. NULL ownership on that exact UUID
is uncertainty, not permission to query other Workspaces.
"""
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
return {'boundary': 'unavailable'}, True
t = AgentInteraction.__table__
try:
records = (
await self.pm.execute_async(
sa.select(t.c.id, t.c.status, t.c.updated_at, t.c.workspace_id)
.where(
t.c.processor_type == 'pipeline',
t.c.processor_id == pipeline_uuid,
sa.or_(
t.c.workspace_id == ctx.workspace_uuid, t.c.workspace_id.is_(None), t.c.workspace_id == ''
),
)
.order_by(t.c.id)
)
).all()
durable = [tuple(r) for r in records]
blocked = any(r.status not in {'submitted', 'cancelled', 'expired', 'delivery_failed'} for r in records)
except Exception:
# Missing schema, permissions, or a failed store must not be treated
# as an empty store. No original diagnostic crosses the API.
durable, blocked = ['unavailable'], True
native = self._native_interactions(ctx, pipeline_uuid)
return {'durable': durable, 'native': native}, blocked or bool(native)
async def _plan(self, ctx, row):
plan = plan_legacy_pipeline(row['config'], row['extensions_preferences'])
pending = None
if plan['state'] == 'already_current':
t = PipelineMigrationSnapshot.__table__
snapshots = (
await self.pm.execute_async(
sa.select(t).where(
t.c.workspace_uuid == ctx.workspace_uuid,
t.c.pipeline_uuid == row['uuid'],
t.c.state == 'activation_pending',
t.c.target_fingerprint == _fingerprint(row),
t.c.planner_version == str(PLANNER_VERSION),
)
)
).all()
if snapshots:
pending = dict(snapshots[0]._mapping)
source = pending['source_snapshot']
plan = plan_legacy_pipeline(source['config'], source['extensions_preferences'])
candidate = {**row, 'config': plan.get('config')}
if plan['state'] != 'ready' or _fingerprint(candidate) != _fingerprint(row):
raise MigrationError('planner_changed')
facts = await self._plugin(ctx, plan)
state = 'activation_pending' if pending else plan['state']
blockers = list(plan['blockers'])
if state in ('ready', 'activation_pending'):
if facts is None:
state = 'needs_plugin'
blockers.append({'code': 'plugin_missing'})
elif facts['enabled'] is not True:
state = 'blocked'
blockers.append({'code': 'plugin_disabled'})
if plan['state'] == 'ready':
observation, interaction_blocked = await self._interaction_state(ctx, row['uuid'])
plan['_interaction_state'] = observation
if interaction_blocked:
state = 'blocked'
blockers.append({'code': 'runtime.pending_interaction'})
return plan, facts, pending, state, blockers
def _token(self, ctx, row, plan, facts, pending):
claim = [
ctx.instance_uuid,
ctx.workspace_uuid,
ctx.placement_generation,
ctx.account_uuid,
ctx.workspace.membership_uuid,
ctx.workspace.membership_revision,
str(PLANNER_VERSION),
_fingerprint(row),
plan,
facts,
pending['uuid'] if pending else None,
]
# An authenticated opaque digest discloses neither credentials nor configs.
return hmac.new(self._token_key, _json(claim).encode(), hashlib.sha256).hexdigest()
async def preview(self, ctx: RequestContext):
require_permission(ctx, Permission.RESOURCE_VIEW)
async with self.pm.tenant_scope(ctx.workspace_uuid):
await self._binding(ctx)
items = []
for row in await self._rows(ctx):
plan, facts, pending, state, blockers = await self._plan(ctx, row)
item = {
k: copy.deepcopy(plan[k])
for k in ('legacy_runner', 'target_runner_id', 'target_plugin', 'changed_paths', 'warnings')
}
item.update(
pipeline_uuid=row['uuid'],
name=row['name'],
state=state,
blockers=blockers,
preview_token=self._token(ctx, row, plan, facts, pending)
if state in ('ready', 'activation_pending')
else None,
)
items.append(item)
return {
'planner_version': str(PLANNER_VERSION),
'workspace_uuid': ctx.workspace_uuid,
'items': items,
'total': len(items),
}
async def _selected(self, ctx, selection, *, data_only=False):
rows = await self._rows(ctx, selection['pipeline_uuid'])
if not rows:
raise MigrationError('pipeline_not_found', 404)
row = rows[0]
plan, facts, pending, state, blockers = await self._plan(ctx, row)
token = self._token(ctx, row, plan, facts, pending)
if not hmac.compare_digest(token, selection['preview_token']):
raise MigrationError('preview_stale')
if state not in ('ready', 'activation_pending') and not (
data_only
and plan['state'] == 'ready'
and all(b['code'] in ('plugin_missing', 'plugin_disabled') for b in blockers)
):
raise MigrationError('migration_blocked')
return row, plan, facts, pending
async def execute(self, ctx: RequestContext, body):
require_permission(ctx, Permission.RESOURCE_MANAGE)
if isinstance(body, dict) and body.get('all') is True:
if (
set(body) != {'confirmed', 'all', 'install_plugins'}
or body['confirmed'] is not True
or not isinstance(body['install_plugins'], bool)
):
raise MigrationError('confirmation_required', 400)
return await self._execute_all(ctx, install_plugins=body['install_plugins'])
items = validate_execute_request(body)
async with self.pm.tenant_scope(ctx.workspace_uuid):
await self._authorize(ctx)
# Preflight the entire selection before creating tasks or contacting runtimes.
for selection in items:
await self._selected(ctx, selection)
task_context = TaskContext.new()
task_context.metadata = {
'kind': 'pipeline_migration',
'results': [{'pipeline_uuid': i['pipeline_uuid'], 'state': 'pending', 'code': None} for i in items],
}
task = self.ap.task_mgr.create_user_task(
self._run(ctx, ExecutionContext.from_request(ctx), items, task_context),
kind='pipeline_migration',
name='pipeline_migration',
context=task_context,
instance_uuid=ctx.instance_uuid,
workspace_uuid=ctx.workspace_uuid,
placement_generation=ctx.placement_generation,
)
return {'task_id': task.id}
async def _execute_all(self, ctx, *, install_plugins):
# Capture all sources on admission. Plugin installation must not silently
# include later edits or pipelines created while the task is running.
if ctx.workspace_uuid in self._all_tasks:
raise MigrationError('migration_running')
self._all_tasks.add(ctx.workspace_uuid)
try:
async with self.pm.tenant_scope(ctx.workspace_uuid):
await self._authorize(ctx)
sources = []
for row in await self._rows(ctx):
_, _, _, state, _ = await self._plan(ctx, row)
if state not in ('already_current', 'not_legacy'):
sources.append(row)
if not sources:
raise MigrationError('nothing_to_migrate')
task_context = TaskContext.new()
task_context.metadata = {
'kind': 'pipeline_migration',
'phase': 'installing' if install_plugins else 'migrating',
'installations': [],
'results': [{'pipeline_uuid': row['uuid'], 'state': 'pending', 'code': None} for row in sources],
}
task = self.ap.task_mgr.create_user_task(
self._run_all(
ctx, ExecutionContext.from_request(ctx), sources, task_context, install_plugins=install_plugins
),
kind='pipeline_migration',
name='pipeline_migration',
context=task_context,
instance_uuid=ctx.instance_uuid,
workspace_uuid=ctx.workspace_uuid,
placement_generation=ctx.placement_generation,
)
return {'task_id': task.id, 'pipeline_uuids': [row['uuid'] for row in sources]}
except BaseException:
self._all_tasks.discard(ctx.workspace_uuid)
raise
async def _run_all(self, ctx, execution, sources, task_context, *, install_plugins):
installed = {}
try:
async with self.pm.tenant_scope(execution.workspace_uuid):
for source, result in zip(sources, task_context.metadata['results']):
if result['state'] != 'pending':
continue
try:
await self._authorize(ctx)
current = await self._rows(ctx, source['uuid'])
if not current or _fingerprint(current[0]) != _fingerprint(source):
raise MigrationError('preview_stale')
plan, facts, pending, state, blockers = await self._plan(ctx, current[0])
hard_blockers = [b for b in blockers if b['code'] not in ('plugin_missing', 'plugin_disabled')]
if hard_blockers:
raise MigrationError(hard_blockers[0]['code'])
if state in ('already_current', 'not_legacy'):
result.update(state='already_current', code=None)
continue
target = plan.get('target_plugin')
if not target:
raise MigrationError('migration_blocked')
if not install_plugins:
selection = {
'pipeline_uuid': source['uuid'],
'preview_token': self._token(ctx, current[0], plan, facts, pending),
}
await self._run(ctx, execution, [selection], task_context, results=[result], data_only=True)
continue
key = (target['author'], target['name'], target['version'])
if key not in installed:
runners = await self.ap.runner_registry.list_runners(
execution, use_cache=False, usage='agent'
)
descriptor = next((r for r in runners if r.id == plan['target_runner_id']), None)
ready = (
facts
and facts['enabled']
and descriptor
and descriptor.plugin_version == target['version']
)
if not ready:
task_context.metadata['phase'] = 'installing'
install_context = MigrationInstallContext(target)
task_context.metadata['installations'].append(install_context.progress)
observer = asyncio.create_task(install_context.observe())
install_code = 'operation_cancelled'
try:
await self._authorize(ctx)
await self.ap.plugin_connector.require_workspace_context(execution)
await self.ap.plugin_connector.install_plugin(
PluginInstallSource.MARKETPLACE,
{
'plugin_author': target['author'],
'plugin_name': target['name'],
'plugin_version': target['version'],
},
task_context=install_context,
)
install_code = None
except PluginRuntimeNotConnectedError:
install_code = 'plugin_runtime_unavailable'
except MarketplacePluginVersionNotFoundError:
install_code = 'plugin_version_unavailable'
except httpx.TimeoutException:
install_code = 'plugin_download_timeout'
except httpx.HTTPStatusError as exc:
install_code = (
'plugin_version_unavailable'
if exc.response.status_code == 404
else 'plugin_marketplace_unavailable'
)
except httpx.RequestError:
install_code = 'plugin_download_failed'
except PluginInstallationFailedError as exc:
install_code = {
'dependency_prepare_failed': 'dependency_prepare_failed',
'worker_launch_failed': 'plugin_launch_failed',
}.get(exc.error_code, 'plugin_install_failed')
except Exception:
install_code = 'plugin_install_failed'
finally:
observer.cancel()
try:
await observer
except asyncio.CancelledError:
pass
install_context.finish(install_code)
installed[key] = install_code
else:
installed[key] = None
if installed[key]:
raise MigrationError(installed[key])
await self._authorize(ctx)
current = await self._rows(ctx, source['uuid'])
if not current or _fingerprint(current[0]) != _fingerprint(source):
raise MigrationError('preview_stale')
plan, facts, pending, state, blockers = await self._plan(ctx, current[0])
if state not in ('ready', 'activation_pending'):
raise MigrationError(blockers[0]['code'] if blockers else 'migration_blocked')
selection = {
'pipeline_uuid': source['uuid'],
'preview_token': self._token(ctx, current[0], plan, facts, pending),
}
task_context.metadata['phase'] = 'migrating'
await self._run(ctx, execution, [selection], task_context, results=[result])
except Exception as exc:
result.update(
state='blocked' if isinstance(exc, MigrationError) else 'failed',
code=_failure_code(exc),
)
if any(
r['code'] in ('operation_cancelled', 'commit_outcome_unknown')
for r in task_context.metadata['results']
):
break
finally:
for result in task_context.metadata['results']:
if result['state'] == 'pending':
result.update(state='failed', code='operation_cancelled')
task_context.metadata['phase'] = 'finished'
self._all_tasks.discard(ctx.workspace_uuid)
return task_context.metadata
async def _verify_runtime(self, execution, plan):
runners = await self.ap.runner_registry.list_runners(execution, use_cache=False, usage='agent')
descriptor = next((r for r in runners if r.id == plan['target_runner_id']), None)
if descriptor is None or 'agent' not in descriptor.usages:
raise MigrationError('runner_unavailable')
if descriptor.plugin_version != plan['target_plugin']['version']:
raise MigrationError('plugin_version_incompatible')
config = plan['config']
RunnerConfigResolver.validate_pipeline_config(config)
runner_config = config['ai']['runner_config'][plan['target_runner_id']]
schema = {item['name']: item for item in descriptor.config_schema}
# Host fields have their own scope/capability checks below; plugin
# schema defaults must never implicitly authorize a foreign resource.
host_fields = {
'enable-all-tools': 'boolean',
'tools': 'array',
'knowledge-bases': 'knowledge-base-multi-selector',
'mcp-resources': 'array',
'mcp-resource-agent-read-enabled': 'boolean',
}
for name, value in runner_config.items():
field = schema.get(name)
if field is None and name in host_fields:
field = {'type': host_fields[name]}
if field is None:
raise MigrationError('runner_schema_incompatible')
if value is None and field.get('nullable') is True:
continue
kind = config_schema.normalize_schema_item_type(field.get('type'))
valid = (
(
kind
in (
'string',
'text',
'password',
'secret',
'select',
'llm-model-selector',
'embedding-model-selector',
'rerank-model-selector',
)
and isinstance(value, str)
)
or (kind == 'boolean' and type(value) is bool)
or (kind in ('integer', 'int') and type(value) is int)
or (kind in ('float', 'number') and type(value) in (int, float))
or (
kind
in (
'array',
'multi-select',
'knowledge-base-selector',
'knowledge-base-multi-selector',
'prompt-editor',
)
and isinstance(value, list)
)
or (kind in ('object', 'json') and type(value) is dict)
or (kind == 'array[string]' and type(value) is list and all(type(v) is str for v in value))
or (
kind == 'model-fallback-selector'
and (
isinstance(value, str)
or isinstance(value, dict)
and set(value) <= {'primary', 'fallbacks', 'reasoning'}
and isinstance(value.get('primary'), str)
and isinstance(value.get('fallbacks', []), list)
and all(isinstance(v, str) for v in value.get('fallbacks', []))
and isinstance(value.get('reasoning', {}), dict)
)
)
)
if not valid:
raise MigrationError('runner_schema_incompatible')
if kind == 'select' and field.get('options') and field.get('allow_custom') is not True:
if value not in [o.get('value', o.get('name')) for o in field['options']]:
raise MigrationError('runner_schema_incompatible')
if any(
f.get('required') and name not in runner_config and f.get('default') is None for name, f in schema.items()
):
raise MigrationError('runner_schema_incompatible')
await self._verify_resources(execution, descriptor, runner_config)
return _json(
[
descriptor.id,
descriptor.plugin_version,
descriptor.usages,
descriptor.config_schema,
getattr(descriptor, 'capabilities', None),
getattr(descriptor, 'permissions', None),
]
)
async def _verify_resources(self, execution, descriptor, runner_config):
"""Fail closed on references the scoped Host cannot authorize.
This runs only on explicit execution, outside the mutation transaction.
Resource use is still reauthorized by the Host on every eventual run.
"""
async def require_row(model, identifier, *, column=None):
column = model.uuid if column is None else column
found = (
await self.pm.execute_async(
sa.select(model.uuid).where(
model.workspace_uuid == execution.workspace_uuid,
column == identifier,
)
)
).first()
if found is None:
raise MigrationError('runner_resource_unavailable')
permissions = getattr(descriptor, 'permissions', None)
for model_type, model_uuid in config_schema.iter_config_model_refs(descriptor, runner_config):
allowed = set(getattr(permissions, 'models', []))
if not (allowed & ({'rerank'} if model_type == 'rerank' else {'invoke', 'stream'})):
raise MigrationError('runner_resource_unavailable')
await require_row(RerankModel if model_type == 'rerank' else LLMModel, model_uuid)
for field in config_schema.iter_schema_items(descriptor, {'embedding-model-selector'}):
value = runner_config.get(field['name'])
if value and value not in config_schema.NONE_SENTINELS:
await require_row(EmbeddingModel, value)
kb_ids = runner_config.get('knowledge-bases', [])
if not isinstance(kb_ids, list) or any(not isinstance(v, str) or not v for v in kb_ids):
raise MigrationError('runner_resource_unavailable')
if kb_ids:
if not config_schema.uses_host_knowledge_bases(descriptor) or 'retrieve' not in getattr(
permissions, 'knowledge_bases', []
):
raise MigrationError('runner_resource_unavailable')
for kb_id in kb_ids:
await require_row(KnowledgeBase, kb_id)
tool_names = runner_config.get('tools', [])
if not isinstance(tool_names, list) or any(not isinstance(v, str) or not v for v in tool_names):
raise MigrationError('runner_resource_unavailable')
tool_grant = (
runner_config.get('enable-all-tools', False)
or tool_names
or runner_config.get('mcp-resource-agent-read-enabled', False)
)
if tool_grant:
if not config_schema.uses_host_tools(descriptor) or 'call' not in getattr(permissions, 'tools', []):
raise MigrationError('runner_resource_unavailable')
policy = ResourcePolicyProjector.from_runner_config(runner_config)
tools = await AgentResourceBuilder(self.ap)._build_tools_from_binding(
execution,
permissions,
policy,
descriptor,
runner_config,
)
if not set(tool_names) <= {tool['tool_name'] for tool in tools}:
raise MigrationError('runner_resource_unavailable')
for attachment in runner_config.get('mcp-resources', []):
if not isinstance(attachment, dict):
raise MigrationError('runner_resource_unavailable')
identifier = attachment.get('server_uuid') or attachment.get('server_id')
name = attachment.get('server_name')
if identifier:
await require_row(MCPServer, identifier)
elif name:
await require_row(MCPServer, name, column=MCPServer.name)
else:
raise MigrationError('runner_resource_unavailable')
async def _lock(self, ctx, pipeline_uuid, session):
# Every pending producer shares this row and revalidates its captured
# configuration/conversation authority after acquiring admission.
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
raise MigrationError('runtime.pending_interaction')
await lock_pipeline_admission(session, ctx.workspace_uuid, pipeline_uuid)
# Hold revocable directory rows through commit (no network in this UoW).
await session.execute(sa.select(User.uuid).where(User.uuid == ctx.account_uuid).with_for_update())
for model, condition in [
(Workspace, Workspace.uuid == ctx.workspace_uuid),
(WorkspaceExecutionState, WorkspaceExecutionState.workspace_uuid == ctx.workspace_uuid),
(
WorkspaceMembership,
sa.and_(
WorkspaceMembership.workspace_uuid == ctx.workspace_uuid,
WorkspaceMembership.account_uuid == ctx.account_uuid,
),
),
(PluginSetting, PluginSetting.workspace_uuid == ctx.workspace_uuid),
]:
await session.execute(sa.select(model.__table__).where(condition).with_for_update())
async def _commit(self, ctx, selection, expected_facts, candidate, *, data_only=False):
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
await self._authorize(ctx, uow.session)
row, plan, facts, pending = await self._selected(ctx, selection, data_only=data_only)
if facts != expected_facts:
raise MigrationError('plugin_changed')
if pending:
return pending['uuid'], pending['target_fingerprint']
target = {**row, 'config': candidate['config']}
t = LegacyPipeline.__table__
cas = [t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == row['uuid']]
for key in ('config', 'stages', 'extensions_preferences'):
cas.append(sa.cast(t.c[key], sa.Text) == row['_raw_' + key])
result = await self.pm.execute_async(sa.update(t).where(*cas).values(config=candidate['config']))
if result.rowcount != 1:
raise MigrationError('preview_stale')
snapshot_uuid = str(uuid.uuid4())
target_fingerprint = _fingerprint(target)
await self.pm.execute_async(
sa.insert(PipelineMigrationSnapshot).values(
uuid=snapshot_uuid,
workspace_uuid=ctx.workspace_uuid,
pipeline_uuid=row['uuid'],
source_fingerprint=_fingerprint(row),
target_fingerprint=target_fingerprint,
source_snapshot=json.loads(_json({k: v for k, v in row.items() if not k.startswith('_')})),
planner_version=str(PLANNER_VERSION),
state='activation_pending',
)
)
if self._native_interactions(ctx, row['uuid']):
raise MigrationError('runtime.pending_interaction')
return snapshot_uuid, target_fingerprint
async def _activate(self, ctx, selection, snapshot_uuid, target_fingerprint, candidate, plan, expected_facts):
# Re-read the committed target under a short lock before synchronous
# publication. A later editor must never be replaced by our old candidate.
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
await self._authorize(ctx, uow.session)
rows = await self._rows(ctx, selection['pipeline_uuid'])
if not rows or _fingerprint(rows[0]) != target_fingerprint:
raise MigrationError('activation_source_changed')
if await self._plugin(ctx, plan) != expected_facts:
raise MigrationError('plugin_changed')
_, interaction_blocked = await self._interaction_state(ctx, selection['pipeline_uuid'])
if interaction_blocked:
raise MigrationError('runtime.pending_interaction')
# Native cache inspection above is the last await before publication
# and session invalidation; no native producer can interleave here.
self.ap.pipeline_mgr.publish_pipeline(candidate)
for session in self.ap.sess_mgr.session_list:
conversation = session.using_conversation
if (
conversation is not None
and conversation.pipeline_uuid == selection['pipeline_uuid']
and getattr(session, 'workspace_uuid', None) == ctx.workspace_uuid
):
session.using_conversation = None
t = PipelineMigrationSnapshot.__table__
await self.pm.execute_async(
sa.update(t)
.where(t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == snapshot_uuid)
.values(state='active')
)
async def _run(self, ctx, execution, items, task_context, results=None, *, data_only=False):
async with self.pm.tenant_scope(execution.workspace_uuid):
for selection, result in zip(items, results if results is not None else task_context.metadata['results']):
committed = False
commit_attempted = False
try:
await self._authorize(ctx)
row, plan, facts, pending = await self._selected(ctx, selection, data_only=data_only)
runtime_schema = None if data_only else await self._verify_runtime(execution, plan)
RunnerConfigResolver.validate_pipeline_config(plan['config'])
candidate_entity = {k: copy.deepcopy(v) for k, v in row.items() if not k.startswith('_')}
candidate_entity['config'] = copy.deepcopy(plan['config'])
runtime = await self.ap.pipeline_mgr.prepare_pipeline(execution, copy.deepcopy(candidate_entity))
if not data_only and await self._verify_runtime(execution, plan) != runtime_schema:
raise MigrationError('runner_schema_changed')
# Runtime awaits are over. Recheck authorization/source/plugin
# facts under database locks, then atomically journal and CAS.
commit_attempted = True
snapshot_uuid, target_fingerprint = await self._commit(
ctx, selection, facts, candidate_entity, **({'data_only': True} if data_only else {})
)
committed = True
await self._activate(ctx, selection, snapshot_uuid, target_fingerprint, runtime, plan, facts)
result.update(state='migrated', code='data_only' if data_only else None)
except (Exception, asyncio.CancelledError) as exc:
cancelled = isinstance(exc, asyncio.CancelledError)
reconciliation_cancel = None
code = 'operation_cancelled' if cancelled else _failure_code(exc)
if commit_attempted and not committed and not isinstance(exc, MigrationError):
# A commit can succeed while acknowledgement/connection
# cleanup fails, including cancellation. Make one read
# after the UoW has unwound, in this task's tenant scope;
# never shield/retry through another cancellation.
try:
t = PipelineMigrationSnapshot.__table__
conditions = [
t.c.workspace_uuid == ctx.workspace_uuid,
t.c.pipeline_uuid == selection['pipeline_uuid'],
t.c.source_fingerprint
== (pending['source_fingerprint'] if pending else _fingerprint(row)),
t.c.target_fingerprint == _fingerprint(candidate_entity),
t.c.planner_version == str(PLANNER_VERSION),
]
if pending:
# A retry starts from the target, not the legacy
# source. Only the original snapshot proves this
# durable migration; an unrelated journal cannot.
conditions.extend(
[
t.c.uuid == pending['uuid'],
t.c.target_fingerprint == pending['target_fingerprint'],
]
)
found = (await self.pm.execute_async(sa.select(t.c.uuid).where(*conditions))).first()
committed = found is not None
except asyncio.CancelledError as cancel_exc:
reconciliation_cancel = cancel_exc
cancelled = True
code = 'commit_outcome_unknown'
except Exception:
code = 'commit_outcome_unknown'
state = (
# Unknown is deliberately recovery-needed, not a claim
# of rollback or proof that a snapshot was committed.
'activation_pending'
if committed or code == 'commit_outcome_unknown'
else 'stale'
if code == 'preview_stale'
else 'blocked'
if isinstance(exc, MigrationError)
else 'failed'
)
result.update(state=state, code=code)
if cancelled:
for unstarted in task_context.metadata['results']:
if unstarted['state'] == 'pending':
unstarted.update(state='failed', code='operation_cancelled')
if reconciliation_cancel is not None:
# Preserve cancellation propagation during recovery,
# but record safe outcomes before leaving the task.
raise reconciliation_cancel
return task_context.metadata
# Never attach the original exception, traceback or plugin
# diagnostic to TaskContext: upstream errors can contain keys.
return task_context.metadata
+9 -1
View File
@@ -11,6 +11,7 @@ from typing import Optional
from urllib.parse import quote, unquote, urlparse
import httpx
from langbot_plugin.entities.io.errors import ActionCallTimeoutError
from ....core import app
from ....skill.utils import parse_frontmatter
@@ -119,7 +120,14 @@ class SkillService:
box_service = self._box_service()
if box_service is None:
return []
return [self._serialize_skill(skill) for skill in await box_service.list_skills(execution_context)]
try:
skills = await box_service.list_skills(execution_context)
except ActionCallTimeoutError as exc:
logger = getattr(self.ap, 'logger', None)
if logger is not None:
logger.warning(f'Box skill listing timed out: {exc}')
return []
return [self._serialize_skill(skill) for skill in skills]
async def get_skill(self, context: TenantContext, skill_name: str) -> Optional[dict]:
execution_context = await self._execution_context(context)
+14 -3
View File
@@ -245,13 +245,14 @@ class SpaceService:
models_data = data.get('data', {}).get('models', [])
return [SpaceModel.model_validate(model_dict) for model_dict in models_data]
async def get_model_selection(self, category: str) -> typing.List[SpaceModelSelection]:
async def get_model_selection(self, category: str | None = None) -> typing.List[SpaceModelSelection]:
"""Return Space models in the availability-ranked selection order."""
space_url = self._get_space_config()['url']
session = httpclient.get_session()
params = {'category': category} if category else None
async with session.get(
f'{space_url}/api/v1/models/selection',
params={'category': category},
params=params,
) as response:
if response.status != 200:
error = await httpclient.read_text_limited(response)
@@ -269,7 +270,17 @@ class SpaceService:
models = []
for selection in data:
if isinstance(selection, dict) and isinstance(selection.get('model'), dict):
models.append(selection['model'])
model = dict(selection['model'])
availability = selection.get('availability')
if not isinstance(availability, dict):
# Accept the short-lived pre-release response shape.
availability = {
key: selection[key]
for key in ('up', 'last_probed_at', 'latency_ms', 'http_code')
if key in selection
}
model['availability'] = availability
models.append(model)
else:
models.append(selection)
return [SpaceModelSelection.model_validate(model) for model in models]
@@ -0,0 +1,225 @@
"""Content-free management boundaries. Diagnostics never participate in I/O.
Only source-code identities and status categories enter these helpers. Never
pass a URL, request, frame, token, tool argument, or serialized result to them.
"""
from __future__ import annotations
import contextlib
import contextvars
import functools
import re
import quart
from quart.wrappers.response import IterableBody, ResponseBody
from ..telemetry import diagnostics as d
from ..telemetry import diagnostic_privacy as privacy
_ACTIVE_BOUNDARY = contextvars.ContextVar('management_diagnostic_boundary', default=None)
def operation_id(source, fn, *, rule='', methods=()):
"""Called at registration with a Core function, never a client tool name."""
module = fn.__module__.split('.groups.', 1)[-1]
if source != 'http':
module = ''
name = fn.__name__
if source == 'http' and name == '_':
# Many Core routes use the anonymous function name `_`. Disambiguate
# using ONLY the source-declared template and methods at registration.
# This is never quart.request.path, url_rule, endpoint or request.method.
template = re.sub(r'[^A-Za-z0-9_.:-]+', '.', rule).strip('.') or 'root'
name = '.'.join((*sorted(methods or ('GET',)), template))
return '.'.join(part for part in (source, module, name) if part)[:128]
def outcome(value, reason_code='response_error'):
try:
d.set_outcome(value, reason_code=reason_code)
except Exception:
pass
def workspace(context):
"""Annotate only this boundary's owned span with a matching trusted context."""
try:
from .http.context import ExecutionContext, RequestContext
if not isinstance(context, (ExecutionContext, RequestContext)):
return
boundary = _ACTIVE_BOUNDARY.get()
span = d.current_span()
if boundary is None or span is None or span is not boundary.span:
return
if d._manager(boundary.ap) is not span.manager:
return
instance = getattr(
getattr(boundary.ap, 'workspace_service', None),
'instance_uuid',
getattr(span.manager, 'instance_id', None),
)
# Structural recorders need not expose instance metadata. Real managers
# do, and must never receive another instance's Workspace annotation.
if instance is not None and instance != context.instance_uuid:
return
d.annotate(workspace_uuid=context.workspace_uuid)
except Exception:
pass
class Boundary:
def __init__(self, ap, operation, source):
self.ap = ap
self.span = None
try:
manager = d._manager(ap)
if manager is not None:
fields = {'source': source, 'stage': 'execute'}
if source == 'webui_debug':
fields['attributes'] = {'synthetic': True}
self.span = d.Span(manager, 'api', operation, fields)
except Exception:
pass
@contextlib.contextmanager
def activate(self):
# None is an explicit inheritance barrier, not a no-op: disabled or
# broken B work must not borrow enabled A's span through ContextVars.
token = d._CURRENT.set(self.span)
boundary_token = _ACTIVE_BOUNDARY.set(self)
try:
yield
finally:
_ACTIVE_BOUNDARY.reset(boundary_token)
d._CURRENT.reset(token)
def finish(self, error=None):
if self.span is not None:
try:
self.span.finish(error)
except Exception:
pass
class _ObservedBody(ResponseBody):
"""Carry the HTTP parent through iteration, not generator suspension."""
def __init__(self, body, boundary):
self.body = body
self.boundary = boundary
self.iterator = None
self.exhausted = False
async def __aenter__(self):
try:
with self.boundary.activate():
entered = await self.body.__aenter__()
self.iterator = entered.__aiter__()
return self
except BaseException as exc:
self.boundary.finish(exc)
raise
async def __aexit__(self, exc_type, exc_value, tb):
try:
with self.boundary.activate():
result = await self.body.__aexit__(exc_type, exc_value, tb)
except BaseException as exc:
self.boundary.finish(exc)
raise
finally:
self.boundary.finish(exc_value if exc_value is not None else (None if self.exhausted else GeneratorExit()))
return result
async def __aiter__(self):
try:
if self.iterator is None:
with self.boundary.activate():
self.iterator = self.body.__aiter__()
while True:
try:
with self.boundary.activate():
value = await anext(self.iterator)
except StopAsyncIteration:
self.exhausted = True
self.boundary.finish()
return
yield value
except BaseException as exc:
self.boundary.finish(exc)
raise
def _response_status(value):
"""Inspect response metadata only; never read or deserialize a body."""
response = value[0] if isinstance(value, tuple) else value
status = response.status_code if isinstance(response, quart.Response) else 200
if isinstance(value, tuple) and len(value) > 1 and type(value[1]) is int:
status = value[1]
if status in (401, 403):
outcome('rejected')
elif status >= 400:
outcome('failed')
elif isinstance(response, dict) and 'code' in response and response['code'] != 0:
outcome('failed')
return response
@contextlib.contextmanager
def scope(ap, operation, *, source):
"""A fixed operation around an existing non-generator statement block."""
try:
privacy.code_value('operation', operation)
except Exception:
pass
boundary = Boundary(ap, operation, source)
try:
with boundary.activate():
yield
except BaseException as exc:
boundary.finish(exc)
raise
finally:
boundary.finish()
def observe(operation, *, source, ap=None, http=False):
# The operation is supplied by Core registration code, not request data.
try:
privacy.code_value('operation', operation)
except Exception:
pass
def decorator(fn):
@functools.wraps(fn)
async def wrapped(*args, **kwargs):
try:
owner = (ap() if callable(ap) else ap) if ap is not None else getattr(args[0], 'ap', None)
except Exception:
owner = None
boundary = Boundary(owner, operation, source)
streaming = False
try:
with boundary.activate():
value = await fn(*args, **kwargs)
if http:
try:
response = _response_status(value)
if isinstance(response, quart.Response) and isinstance(response.response, IterableBody):
response.response = _ObservedBody(response.response, boundary)
streaming = True
except Exception:
pass
return value
except BaseException as exc:
boundary.finish(exc)
raise
finally:
if not streaming:
boundary.finish()
return wrapped
return decorator
+14 -7
View File
@@ -24,6 +24,7 @@ import uuid
from ..http.context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
from .context import bind_request_context, reset_request_context
from .server import LangBotMCPServer
from .. import management_diagnostics as diagnostics
if typing.TYPE_CHECKING:
from ...core import app as app_module
@@ -85,13 +86,8 @@ class MCPMount:
authenticate_api_key = self.ap.apikey_service.authenticate_api_key
is_mcp_path = self._is_mcp_path
async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def]
# Pass through non-HTTP scopes (lifespan, websocket) to Quart so its
# own startup/shutdown and websocket routes keep working.
if scope['type'] != 'http' or not is_mcp_path(scope.get('path', '')):
await quart_asgi(scope, receive, send)
return
@diagnostics.observe('mcp.request', source='mcp', ap=self.ap)
async def dispatch_mcp(scope, receive, send):
# Authenticate MCP HTTP requests with a LangBot API key.
api_key = _extract_api_key(scope.get('headers', []))
identity = None
@@ -100,6 +96,7 @@ class MCPMount:
identity = await authenticate_api_key(api_key)
if identity is None:
diagnostics.outcome('rejected')
await send(
{
'type': 'http.response.start',
@@ -126,6 +123,7 @@ class MCPMount:
entitlement = await resolver.resolve(identity.workspace_uuid)
entitlement_revision = entitlement.entitlement_revision
except Exception:
diagnostics.outcome('rejected')
await send(
{
'type': 'http.response.start',
@@ -153,6 +151,7 @@ class MCPMount:
),
entitlement_revision=entitlement_revision,
)
diagnostics.workspace(request_context)
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
if not callable(tenant_scope):
raise RuntimeError('MCP request persistence scope is unavailable')
@@ -165,4 +164,12 @@ class MCPMount:
finally:
reset_request_context(token)
async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def]
# Non-MCP traffic keeps its existing Quart boundaries. In
# particular, never derive an operation from an ASGI request path.
if scope['type'] != 'http' or not is_mcp_path(scope.get('path', '')):
await quart_asgi(scope, receive, send)
return
await dispatch_mcp(scope, receive, send)
return dispatcher
+109 -4
View File
@@ -24,6 +24,7 @@ from mcp.server.fastmcp import FastMCP
from ..http.authz import Permission, require_permission
from .context import get_request_context
from .. import management_diagnostics as diagnostics
if typing.TYPE_CHECKING:
from ...core import app as app_module
@@ -31,7 +32,7 @@ if typing.TYPE_CHECKING:
INSTRUCTIONS = """\
This MCP server manages a LangBot instance. LangBot is an LLM-native instant
messaging bot platform. Use these tools to inspect and manage bots, pipelines,
messaging bot platform. Use these tools to inspect and manage bots, agents, pipelines,
models, knowledge bases, MCP servers, and skills.
Authentication uses a LangBot API key (web-UI-created `lbk_...` key or the
@@ -52,6 +53,7 @@ def _dump(value: typing.Any) -> str:
def _authorized(permission: Permission):
context = get_request_context()
require_permission(context, permission)
diagnostics.workspace(context)
return context
@@ -76,7 +78,7 @@ class LangBotMCPServer:
# ------------------------------------------------------------------ #
def _register_tools(self) -> None:
ap = self.ap
mcp = self.mcp
mcp = self
# ----- System (read-only) -------------------------------------- #
@mcp.tool(description='Get basic LangBot system/runtime information (version, edition).')
@@ -109,14 +111,22 @@ class LangBotMCPServer:
description=(
'Create a bot. `bot_data` is a JSON object matching the LangBot '
'POST /api/v1/platform/bots body (e.g. name, adapter, config). '
'Returns the new bot UUID.'
'Use event_bindings for exclusive Agent/Pipeline routes; plugin_processors is an array '
'of {processor_uuid, enabled} subscriptions that automatically match Runner-declared events. '
'Subscriptions run independently of each other and of event_bindings. Returns the new bot UUID.'
)
)
async def create_bot(bot_data: dict) -> str:
context = _authorized(Permission.RESOURCE_MANAGE)
return _dump({'uuid': await ap.bot_service.create_bot(context, bot_data)})
@mcp.tool(description='Update a bot by UUID. `bot_data` matches the PUT bot body.')
@mcp.tool(
description=(
'Update a bot by UUID. `bot_data` matches the PUT bot body. '
'plugin_processors replaces the bot subscriptions with [{processor_uuid, enabled}]. '
'Do not put event_processor targets in event_bindings; those are exclusive Agent/Pipeline routes.'
)
)
async def update_bot(bot_uuid: str, bot_data: dict) -> str:
context = _authorized(Permission.RESOURCE_MANAGE)
await ap.bot_service.update_bot(context, bot_uuid, bot_data)
@@ -128,6 +138,10 @@ class LangBotMCPServer:
await ap.bot_service.delete_bot(context, bot_uuid)
return _dump({'ok': True})
@mcp.tool(description='List recent runtime route status for a bot event route table.')
async def list_bot_event_route_statuses(bot_uuid: str) -> str:
return _dump(await ap.bot_service.list_event_route_statuses(bot_uuid))
# ----- Pipelines ----------------------------------------------- #
@mcp.tool(description='List all pipelines.')
async def list_pipelines() -> str:
@@ -170,7 +184,89 @@ class LangBotMCPServer:
await ap.pipeline_service.delete_pipeline(context, pipeline_uuid)
return _dump({'ok': True})
# ----- Processors ---------------------------------------------- #
@mcp.tool(description='List product-level processors, including Agents, Pipelines and Event processors.')
async def list_processors() -> str:
context = _authorized(Permission.RESOURCE_VIEW)
return _dump(await ap.agent_service.get_agents(context))
@mcp.tool(description='Get an Agent, Pipeline or Event processor by UUID.')
async def get_processor(processor_uuid: str) -> str:
context = _authorized(Permission.RESOURCE_VIEW)
return _dump(await ap.agent_service.get_agent(context, processor_uuid))
@mcp.tool(
description=(
'Create an Agent, Pipeline or Event processor. Set `processor_data.kind` to '
'`agent`, `pipeline` or `event_processor`. Event processors may be created without a component; '
'then use update_processor with an installed component_ref from get_processor_metadata and optional '
'parameters. Unconfigured instances support no events. Returns UUID and kind.'
)
)
async def create_processor(processor_data: dict) -> str:
context = _authorized(Permission.RESOURCE_MANAGE)
return _dump(await ap.agent_service.create_agent(context, processor_data))
@mcp.tool(description='Update an Agent, Pipeline or Event processor by UUID.')
async def update_processor(processor_uuid: str, processor_data: dict) -> str:
context = _authorized(Permission.RESOURCE_MANAGE)
await ap.agent_service.update_agent(context, processor_uuid, processor_data)
return _dump({'ok': True})
@mcp.tool(description='Delete an Agent, Pipeline or Event processor by UUID.')
async def delete_processor(processor_uuid: str) -> str:
context = _authorized(Permission.RESOURCE_MANAGE)
await ap.agent_service.delete_agent(context, processor_uuid)
return _dump({'ok': True})
@mcp.tool(
description='Get processor kinds and installed event-capable Runner components with configuration schemas.'
)
async def get_processor_metadata() -> str:
context = _authorized(Permission.RESOURCE_VIEW)
return _dump(await ap.agent_service.get_agent_metadata(context))
@mcp.tool(
description='List one Agent or plugin processor run history; use before_id to page older runs. '
'created_at_ms, started_at_ms and finished_at_ms are Host lifecycle times in epoch milliseconds.'
)
async def list_processor_runs(processor_uuid: str, before_id: int | None = None) -> str:
context = _authorized(Permission.RESOURCE_VIEW)
return _dump(await ap.agent_service.get_processor_runs(context, processor_uuid, before_id=before_id))
@mcp.tool(
description='Read logs and action results for an Agent or plugin processor run; page using after_sequence.'
)
async def get_processor_run_events(
processor_uuid: str,
run_id: str,
after_sequence: int | None = None,
) -> str:
context = _authorized(Permission.RESOURCE_VIEW)
return _dump(
await ap.agent_service.get_processor_run_events(
context,
processor_uuid,
run_id,
after_sequence=after_sequence,
)
)
# ----- Models -------------------------------------------------- #
@mcp.tool(
description=(
'Run a synthetic event against an Agent or Event processor without platform delivery. '
'Returns final text and execution_events containing reported messages/thinking and tool calls. '
'Platform tools use mock adapters; other tools execute normally. '
'For Event processors, data contains the complete typed event fields. '
'Requires runtime.operate; payload accepts event_type, text, data, conversation_id, actor, subject and '
'mock (errors/results keyed by platform tool name; unsupported_apis lists unavailable platform APIs).'
)
)
async def debug_agent(processor_uuid: str, payload: dict) -> str:
context = _authorized(Permission.RUNTIME_OPERATE)
return _dump(await ap.agent_service.debug_agent(context, processor_uuid, payload))
@mcp.tool(description='List all configured LLM models. Secrets are redacted.')
async def list_llm_models() -> str:
context = _authorized(Permission.RESOURCE_VIEW)
@@ -230,6 +326,15 @@ class LangBotMCPServer:
context = _authorized(Permission.RESOURCE_VIEW)
return _dump(await ap.skill_service.get_skill(context, skill_name))
def tool(self, **options):
"""Register a tool boundary before its authorization and service call."""
def register(fn):
observed = diagnostics.observe(diagnostics.operation_id('mcp', fn), source='mcp', ap=self.ap)(fn)
return self.mcp.tool(**options)(observed)
return register
# ------------------------------------------------------------------ #
# ASGI app
# ------------------------------------------------------------------ #