mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-21 16:05:19 +08:00
Merge origin/master into feat/totp-two-factor-auth
Resolve the revision-graph conflict introduced by the release line. Conflicts - tests/integration/persistence/test_rag_document_identity.py: master had independently introduced the same dynamic-head helper (`current_head`) plus a `DOCUMENT_IDENTITY_REVISION` constant, and it explicitly upgrades to that revision. Keep master's semantics (the explicit revision matters because upgrading to the head now also traverses the TOTP branch) while keeping the module-level alembic imports on this branch. New migration - 0030_merge_totp_into_release joins the TOTP branch's own merge revision (0026_merge_totp_and_rag_identity) with the release head (0029_merge_rag_identity). Both reached 0025_rag_document_identity without including each other, which left Alembic with two heads and made `upgrade head` fail with "Multiple head revisions are present". Verification - Single Alembic head confirmed (0030_merge_totp_into_release). - tests/integration/persistence/test_rag_document_identity.py: 34 passed. - Full fast integration suite: 356 passed, 84 skipped.
This commit is contained in:
@@ -27,6 +27,7 @@ from ..authz import (
|
||||
)
|
||||
from ..context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from ....cloud.support_admin import SupportAdminSessionError
|
||||
from ... import management_diagnostics as diagnostics
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
@@ -224,6 +225,7 @@ class RouterGroup(abc.ABC):
|
||||
|
||||
try:
|
||||
if request_context is not None:
|
||||
diagnostics.workspace(request_context)
|
||||
with bounded_executor.blocking_work_scope(request_context.workspace_uuid):
|
||||
persistence_mgr = getattr(
|
||||
self.ap,
|
||||
@@ -277,7 +279,14 @@ class RouterGroup(abc.ABC):
|
||||
)
|
||||
return self.internal_error_response(request_id)
|
||||
|
||||
new_f = handler_error
|
||||
# Observe outside authentication, using the registered Core handler
|
||||
# identity rather than the URL (which can contain user identifiers).
|
||||
new_f = diagnostics.observe(
|
||||
diagnostics.operation_id('http', f, rule=rule, methods=options.get('methods')),
|
||||
source='http',
|
||||
ap=self.ap,
|
||||
http=True,
|
||||
)(handler_error)
|
||||
# Quart/Flask requires a unique endpoint name even when the same URL
|
||||
# intentionally has separate handlers for different HTTP methods.
|
||||
# Include the method set so CRUD routes can declare distinct
|
||||
@@ -564,6 +573,7 @@ class RouterGroup(abc.ABC):
|
||||
def fail(self, code: int | str, msg: str) -> quart.Response:
|
||||
"""Return an error response"""
|
||||
|
||||
diagnostics.outcome('failed')
|
||||
return quart.jsonify(
|
||||
{
|
||||
'code': code,
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Bounded, cancellable NDJSON transport for Agent debug execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
|
||||
import quart
|
||||
from .... import management_diagnostics as diagnostics
|
||||
|
||||
from .....agent.runner.errors import (
|
||||
RunnerError,
|
||||
RunnerExecutionError,
|
||||
RunnerNotAuthorizedError,
|
||||
RunnerNotFoundError,
|
||||
RunnerProtocolError,
|
||||
)
|
||||
|
||||
|
||||
def debug_stream_response(service, context, agent_uuid: str, payload: dict) -> quart.Response:
|
||||
async def stream():
|
||||
queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=32)
|
||||
|
||||
async def on_result(result: dict) -> None:
|
||||
await queue.put({'kind': 'result', 'data': result})
|
||||
|
||||
async def execute() -> None:
|
||||
try:
|
||||
result = await service.debug_agent(context, agent_uuid, payload, on_result=on_result)
|
||||
await queue.put({'kind': 'completed', 'data': result})
|
||||
except Exception as exc:
|
||||
# The stream still uses HTTP 200 when execution returns an
|
||||
# error frame. Mark the inherited request, not its contents.
|
||||
diagnostics.outcome('failed')
|
||||
if isinstance(exc, RunnerExecutionError):
|
||||
code, message = exc.error_code or 'runner_execution_failed', exc.message
|
||||
elif isinstance(exc, RunnerNotFoundError):
|
||||
code, message = 'runner_not_found', 'The configured Agent runner is unavailable'
|
||||
elif isinstance(exc, RunnerNotAuthorizedError):
|
||||
code, message = 'runner_not_authorized', 'The configured Agent runner is not authorized'
|
||||
elif isinstance(exc, RunnerProtocolError):
|
||||
code, message = 'runner_protocol_error', 'The Agent runner returned an invalid response'
|
||||
elif isinstance(exc, ValueError):
|
||||
code, message = 'invalid_request', str(exc)
|
||||
elif isinstance(exc, RunnerError):
|
||||
code, message = 'runner_error', 'The Agent runner could not complete this test'
|
||||
else:
|
||||
code, message = 'runner_error', 'The Agent debug execution failed'
|
||||
await queue.put({'kind': 'error', 'code': code, 'msg': message})
|
||||
|
||||
task = asyncio.create_task(execute())
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
frame = await asyncio.wait_for(queue.get(), timeout=15)
|
||||
except TimeoutError:
|
||||
yield '\n'
|
||||
continue
|
||||
yield json.dumps(frame, ensure_ascii=False) + '\n'
|
||||
if frame['kind'] in {'completed', 'error'}:
|
||||
break
|
||||
finally:
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
response = quart.Response(stream(), content_type='application/x-ndjson; charset=utf-8')
|
||||
response.timeout = None
|
||||
response.headers['Cache-Control'] = 'no-store'
|
||||
response.headers['X-Accel-Buffering'] = 'no'
|
||||
return response
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from .....agent.runner.errors import (
|
||||
RunnerError,
|
||||
RunnerExecutionError,
|
||||
RunnerNotAuthorizedError,
|
||||
RunnerNotFoundError,
|
||||
RunnerProtocolError,
|
||||
)
|
||||
from ...authz import Permission, require_permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
from .agent_debug_stream import debug_stream_response
|
||||
|
||||
|
||||
@group.group_class('agents', '/api/v1/agents')
|
||||
class AgentsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route(
|
||||
'/<agent_uuid>/runs',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def processor_runs(agent_uuid: str, request_context: RequestContext):
|
||||
try:
|
||||
cursor = quart.request.args.get('before_id')
|
||||
result = await self.ap.agent_service.get_processor_runs(
|
||||
request_context,
|
||||
agent_uuid,
|
||||
before_id=int(cursor) if cursor else None,
|
||||
)
|
||||
return self.success(data=result)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route(
|
||||
'/<agent_uuid>/runs/<run_id>/events',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def processor_run_events(agent_uuid: str, run_id: str, request_context: RequestContext):
|
||||
try:
|
||||
cursor = quart.request.args.get('after_sequence')
|
||||
result = await self.ap.agent_service.get_processor_run_events(
|
||||
request_context,
|
||||
agent_uuid,
|
||||
run_id,
|
||||
after_sequence=int(cursor) if cursor else None,
|
||||
)
|
||||
return self.success(data=result)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route(
|
||||
'/<agent_uuid>/debug/stream',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def stream_debug(agent_uuid: str, request_context: RequestContext):
|
||||
payload = await quart.request.get_json()
|
||||
if not isinstance(payload, dict):
|
||||
return self.http_status(400, -1, 'Debug payload must be an object')
|
||||
return debug_stream_response(self.ap.agent_service, request_context, agent_uuid, payload)
|
||||
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET', 'POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
sort_by = quart.request.args.get('sort_by', 'updated_at')
|
||||
sort_order = quart.request.args.get('sort_order', 'DESC')
|
||||
return self.success(
|
||||
data={
|
||||
'agents': await self.ap.agent_service.get_agents(
|
||||
request_context,
|
||||
sort_by,
|
||||
sort_order,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
json_data = await quart.request.json
|
||||
require_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
try:
|
||||
created = await self.ap.agent_service.create_agent(request_context, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data=created)
|
||||
|
||||
@self.route(
|
||||
'/_/metadata',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data=await self.ap.agent_service.get_agent_metadata(request_context))
|
||||
|
||||
@self.route(
|
||||
'/<agent_uuid>/debug',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def _(agent_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
result = await self.ap.agent_service.debug_agent(
|
||||
request_context,
|
||||
agent_uuid,
|
||||
json_data or {},
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except RunnerExecutionError as exc:
|
||||
return self.http_status(
|
||||
422,
|
||||
exc.error_code or 'runner_execution_failed',
|
||||
exc.message,
|
||||
)
|
||||
except RunnerNotFoundError:
|
||||
return self.http_status(
|
||||
409,
|
||||
'runner_not_found',
|
||||
'The configured Agent runner is unavailable',
|
||||
)
|
||||
except RunnerNotAuthorizedError:
|
||||
return self.http_status(
|
||||
403,
|
||||
'runner_not_authorized',
|
||||
'The configured Agent runner is not authorized',
|
||||
)
|
||||
except RunnerProtocolError:
|
||||
return self.http_status(
|
||||
502,
|
||||
'runner_protocol_error',
|
||||
'The Agent runner returned an invalid response',
|
||||
)
|
||||
except RunnerError:
|
||||
return self.http_status(
|
||||
502,
|
||||
'runner_error',
|
||||
'The Agent runner could not complete this test',
|
||||
)
|
||||
return self.success(data=result)
|
||||
|
||||
@self.route(
|
||||
'/<agent_uuid>',
|
||||
methods=['GET', 'PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(agent_uuid: str, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
agent = await self.ap.agent_service.get_agent(request_context, agent_uuid)
|
||||
if agent is None:
|
||||
return self.http_status(404, -1, 'agent not found')
|
||||
return self.success(data={'agent': agent})
|
||||
|
||||
if quart.request.method == 'PUT':
|
||||
require_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
await self.ap.agent_service.update_agent(request_context, agent_uuid, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
require_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
await self.ap.agent_service.delete_agent(request_context, agent_uuid)
|
||||
return self.success()
|
||||
@@ -3,7 +3,7 @@
|
||||
All user-facing URLs are keyed by **bot_uuid** (not pipeline_uuid) so that
|
||||
internal pipeline identifiers are never exposed to end-users. Each handler
|
||||
resolves the bot_uuid to the owning ``web_page_bot`` RuntimeBot and extracts
|
||||
the bound pipeline_uuid for internal routing.
|
||||
the message event route's Pipeline target for internal routing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
@@ -13,6 +13,7 @@ import logging
|
||||
import uuid
|
||||
import hmac
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
import re
|
||||
import httpx
|
||||
@@ -20,6 +21,7 @@ import httpx
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ..... import management_diagnostics as diagnostics
|
||||
from ......utils import httpclient, paths
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, is_valid_session_id, ws_connection_manager
|
||||
from .websocket_chat import create_scoped_duplex_tasks, wait_for_duplex_tasks
|
||||
@@ -29,6 +31,7 @@ _AUTH_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# Cache the widget template content
|
||||
_widget_template_cache: str | None = None
|
||||
_widget_template_cache_mtime_ns: int | None = None
|
||||
_logo_bytes_cache: bytes | None = None
|
||||
|
||||
|
||||
@@ -37,12 +40,14 @@ def _is_valid_uuid(s: str) -> bool:
|
||||
|
||||
|
||||
def _get_widget_template() -> str:
|
||||
"""Load and cache the widget JS template."""
|
||||
global _widget_template_cache
|
||||
if _widget_template_cache is None:
|
||||
template_path = paths.get_resource_path('templates/embed/widget.js')
|
||||
"""Load the widget template and refresh the cache when the file changes."""
|
||||
global _widget_template_cache, _widget_template_cache_mtime_ns
|
||||
template_path = paths.get_resource_path('templates/embed/widget.js')
|
||||
template_mtime_ns = os.stat(template_path).st_mtime_ns
|
||||
if _widget_template_cache is None or _widget_template_cache_mtime_ns != template_mtime_ns:
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
_widget_template_cache = f.read()
|
||||
_widget_template_cache_mtime_ns = template_mtime_ns
|
||||
return _widget_template_cache
|
||||
|
||||
|
||||
@@ -64,16 +69,12 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
"""Resolve *bot_uuid* to ``(runtime_bot, pipeline_uuid)``.
|
||||
|
||||
Returns ``(None, None)`` when the bot does not exist, is not a
|
||||
``web_page_bot``, is disabled, or has no pipeline bound.
|
||||
``web_page_bot``, is disabled, or has no Pipeline target for messages.
|
||||
"""
|
||||
bot = await self.ap.platform_mgr.resolve_public_bot(bot_uuid)
|
||||
if (
|
||||
bot is not None
|
||||
and bot.bot_entity.adapter == 'web_page_bot'
|
||||
and bot.bot_entity.enable
|
||||
and bot.bot_entity.use_pipeline_uuid
|
||||
):
|
||||
return bot, bot.bot_entity.use_pipeline_uuid
|
||||
pipeline_uuid = bot.get_pipeline_target_for_event_type('message.received') if bot is not None else None
|
||||
if bot is not None and bot.bot_entity.adapter == 'web_page_bot' and bot.bot_entity.enable and pipeline_uuid:
|
||||
return bot, pipeline_uuid
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
@@ -198,7 +199,9 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
|
||||
base_url = quart.request.host_url.rstrip('/')
|
||||
webhook_prefix = self.ap.instance_config.data.get('api', {}).get('webhook_prefix', '')
|
||||
if webhook_prefix:
|
||||
# The wizard previews the currently connected backend, which can
|
||||
# differ from the public address used by external website embeds.
|
||||
if webhook_prefix and quart.request.args.get('preview') != 'wizard':
|
||||
base_url = webhook_prefix.rstrip('/')
|
||||
|
||||
if not re.match(r'^https?://[a-zA-Z0-9._:/-]+$', base_url):
|
||||
@@ -327,20 +330,24 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
# -- Embed WebSocket endpoint ----------------------------------------
|
||||
|
||||
@self.quart_app.websocket(self.path + '/<bot_uuid>/ws/connect')
|
||||
@diagnostics.observe('websocket.embed.session', source='websocket', ap=self.ap)
|
||||
async def embed_websocket_connect(bot_uuid: str):
|
||||
"""WebSocket connection for embed widget, keyed by bot_uuid."""
|
||||
await quart.websocket.accept()
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'}))
|
||||
diagnostics.outcome('rejected')
|
||||
return
|
||||
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'}))
|
||||
diagnostics.outcome('rejected')
|
||||
return
|
||||
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
if session_type not in ['person', 'group']:
|
||||
diagnostics.outcome('rejected')
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
@@ -349,6 +356,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
session_id = quart.websocket.args.get('session_id', '')
|
||||
if not is_valid_session_id(session_id):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Valid session_id is required'}))
|
||||
diagnostics.outcome('rejected')
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -356,13 +364,16 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
await self._assert_execution_active(runtime_bot)
|
||||
except Exception:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
|
||||
diagnostics.outcome('rejected')
|
||||
return
|
||||
|
||||
diagnostics.workspace(runtime_bot.execution_context)
|
||||
try:
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
diagnostics.outcome('rejected')
|
||||
return
|
||||
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
@@ -420,11 +431,13 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
try:
|
||||
await wait_for_duplex_tasks(receive_task, send_task)
|
||||
except Exception as e:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error(f'Embed WebSocket task error: {e}')
|
||||
finally:
|
||||
await ws_connection_manager.remove_connection(connection.connection_id)
|
||||
|
||||
except Exception as e:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error(f'Embed WebSocket connection error: {e}', exc_info=True)
|
||||
try:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
|
||||
@@ -439,28 +452,35 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
message = await quart.websocket.receive()
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
with diagnostics.scope(self.ap, 'websocket.embed.message', source='websocket'):
|
||||
diagnostics.workspace(getattr(owner_bot, 'execution_context', None))
|
||||
try:
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
|
||||
if message_type == 'ping':
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
try:
|
||||
current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
|
||||
except Exception:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
|
||||
if message_type == 'ping':
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
try:
|
||||
current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
|
||||
except Exception:
|
||||
diagnostics.outcome('rejected')
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
else:
|
||||
diagnostics.outcome('skipped')
|
||||
|
||||
except json.JSONDecodeError:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
except json.JSONDecodeError:
|
||||
diagnostics.outcome('rejected')
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
|
||||
except Exception as e:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error(f'Embed receive error: {e}', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
@@ -476,11 +496,13 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
if message is None:
|
||||
break
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
with diagnostics.scope(self.ap, 'websocket.embed.send', source='websocket'):
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception as e:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error(f'Embed send error: {e}', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""User-confirmed migration only; deliberately not API-key/MCP accessible."""
|
||||
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ....service.pipeline_migration import PipelineMigrationService, MigrationError
|
||||
|
||||
|
||||
@group.group_class('pipeline_migration', '/api/v1/pipelines/_/migration')
|
||||
class PipelineMigrationRouterGroup(group.RouterGroup):
|
||||
async def initialize(self):
|
||||
self.service = PipelineMigrationService(self.ap)
|
||||
|
||||
@self.route(
|
||||
'/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_VIEW
|
||||
)
|
||||
async def preview(request_context: RequestContext):
|
||||
try:
|
||||
return self.success(data=await self.service.preview(request_context))
|
||||
except MigrationError as exc:
|
||||
return self.http_status(exc.status_code, -1, exc.code)
|
||||
|
||||
@self.route(
|
||||
'/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_MANAGE
|
||||
)
|
||||
async def execute(request_context: RequestContext):
|
||||
try:
|
||||
body = await quart.request.get_json(silent=True)
|
||||
return self.success(data=await self.service.execute(request_context, body))
|
||||
except MigrationError as exc:
|
||||
return self.http_status(exc.status_code, -1, exc.code)
|
||||
@@ -6,6 +6,10 @@ from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ....service.secrets import redact_secrets
|
||||
from ... import group
|
||||
from ......pipeline.extension_preferences import (
|
||||
normalize_extension_preferences,
|
||||
validate_extension_preferences,
|
||||
)
|
||||
|
||||
|
||||
@group.group_class('pipelines', '/api/v1/pipelines')
|
||||
@@ -41,11 +45,14 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
pipeline_data = await quart.request.json
|
||||
create_as_default = pipeline_data.get('is_default') is True
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(
|
||||
request_context,
|
||||
pipeline_data,
|
||||
default=create_as_default,
|
||||
)
|
||||
try:
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(
|
||||
request_context,
|
||||
pipeline_data,
|
||||
default=create_as_default,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': pipeline_uuid})
|
||||
|
||||
@self.route(
|
||||
@@ -122,8 +129,12 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
available_skills = await self.ap.skill_service.list_skills(request_context)
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
try:
|
||||
available_skills = await self.ap.skill_service.list_skills(request_context)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning('Unable to list skills for pipeline extensions: %s', exc)
|
||||
available_skills = []
|
||||
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
@@ -148,16 +159,42 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
json_data.get('bound_plugins', []),
|
||||
json_data.get('bound_mcp_servers', []),
|
||||
json_data.get('enable_all_plugins', True),
|
||||
json_data.get('enable_all_mcp_servers', True),
|
||||
bound_skills=json_data.get('bound_skills', []),
|
||||
enable_all_skills=json_data.get('enable_all_skills', True),
|
||||
bound_mcp_resources=json_data.get('bound_mcp_resources'),
|
||||
mcp_resource_agent_read_enabled=json_data.get('mcp_resource_agent_read_enabled'),
|
||||
)
|
||||
try:
|
||||
validate_extension_preferences(
|
||||
{
|
||||
canonical: json_data[field]
|
||||
for field, canonical in {
|
||||
'bound_plugins': 'plugins',
|
||||
'bound_mcp_servers': 'mcp_servers',
|
||||
'bound_skills': 'skills',
|
||||
'bound_mcp_resources': 'mcp_resources',
|
||||
'enable_all_plugins': 'enable_all_plugins',
|
||||
'enable_all_mcp_servers': 'enable_all_mcp_servers',
|
||||
'enable_all_skills': 'enable_all_skills',
|
||||
'mcp_resource_agent_read_enabled': 'mcp_resource_agent_read_enabled',
|
||||
}.items()
|
||||
if field in json_data
|
||||
},
|
||||
context='Pipeline extension',
|
||||
field_aliases={
|
||||
'plugins': 'bound_plugins',
|
||||
'mcp_servers': 'bound_mcp_servers',
|
||||
'skills': 'bound_skills',
|
||||
'mcp_resources': 'bound_mcp_resources',
|
||||
},
|
||||
)
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
json_data.get('bound_plugins', []),
|
||||
json_data.get('bound_mcp_servers', []),
|
||||
json_data.get('enable_all_plugins', True),
|
||||
json_data.get('enable_all_mcp_servers', True),
|
||||
bound_skills=json_data.get('bound_skills', []),
|
||||
enable_all_skills=json_data.get('enable_all_skills', True),
|
||||
bound_mcp_resources=json_data.get('bound_mcp_resources'),
|
||||
mcp_resource_agent_read_enabled=json_data.get('mcp_resource_agent_read_enabled'),
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
@@ -14,6 +14,7 @@ import quart
|
||||
from ....authz import Permission, permissions_for_role, require_permission
|
||||
from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from ... import group
|
||||
from ..... import management_diagnostics as diagnostics
|
||||
from ......core.task_boundary import run_in_workspace_uow
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
|
||||
from ......utils import bounded_executor
|
||||
@@ -210,6 +211,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.quart_app.websocket(self.path + '/connect')
|
||||
@diagnostics.observe('websocket.dashboard.session', source='websocket', ap=self.ap)
|
||||
async def websocket_connect(pipeline_uuid: str):
|
||||
"""Open one authenticated dashboard debug connection."""
|
||||
|
||||
@@ -217,11 +219,14 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
try:
|
||||
request_context, token = await self._authenticate_websocket()
|
||||
except Exception:
|
||||
diagnostics.outcome('rejected')
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
|
||||
return
|
||||
|
||||
diagnostics.workspace(request_context)
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
if session_type not in ['person', 'group']:
|
||||
diagnostics.outcome('rejected')
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
@@ -230,6 +235,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
try:
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
diagnostics.outcome('rejected')
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Pipeline not found'}))
|
||||
return
|
||||
|
||||
@@ -288,11 +294,13 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
try:
|
||||
await wait_for_duplex_tasks(receive_task, send_task)
|
||||
except Exception as exc:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error(f'WebSocket task execution error: {exc}')
|
||||
finally:
|
||||
await ws_connection_manager.remove_connection(connection.connection_id)
|
||||
|
||||
except Exception:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error('Dashboard WebSocket connection error', exc_info=True)
|
||||
try:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
|
||||
@@ -410,28 +418,34 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
message = await quart.websocket.receive()
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
if message_type == 'ping':
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
try:
|
||||
request_context = await self._revalidate_websocket_authorization(request_context, token)
|
||||
except Exception:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
|
||||
with diagnostics.scope(self.ap, 'websocket.dashboard.message', source='websocket'):
|
||||
diagnostics.workspace(request_context)
|
||||
try:
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
if message_type == 'ping':
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
try:
|
||||
request_context = await self._revalidate_websocket_authorization(request_context, token)
|
||||
except Exception:
|
||||
diagnostics.outcome('rejected')
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
else:
|
||||
logger.warning(f'Unknown WebSocket message type: {message_type}')
|
||||
except json.JSONDecodeError:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
else:
|
||||
diagnostics.outcome('skipped')
|
||||
logger.warning(f'Unknown WebSocket message type: {message_type}')
|
||||
except json.JSONDecodeError:
|
||||
diagnostics.outcome('rejected')
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
|
||||
except Exception:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error('Dashboard WebSocket receive error', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
@@ -447,11 +461,13 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
if message is None:
|
||||
break
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
with diagnostics.scope(self.ap, 'websocket.dashboard.send', source='websocket'):
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception:
|
||||
diagnostics.outcome('failed')
|
||||
logger.error('Dashboard WebSocket send error', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
|
||||
@@ -3,11 +3,20 @@ from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ....service.bot_errors import BotApplyError, bot_error_message
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('bots', '/api/v1/platform/bots')
|
||||
class BotsRouterGroup(group.RouterGroup):
|
||||
def _apply_error_response(self, exc: BotApplyError):
|
||||
request_id = self.request_id()
|
||||
logger = getattr(self.ap, 'logger', self.quart_app.logger)
|
||||
logger.error(f'Bot configuration apply failed request_id={request_id} bot_uuid={exc.bot_uuid}', exc_info=True)
|
||||
return quart.jsonify(
|
||||
code='bot_apply_failed', msg=str(exc), data={'uuid': exc.bot_uuid}, request_id=request_id
|
||||
), 400
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route(
|
||||
'',
|
||||
@@ -34,7 +43,14 @@ class BotsRouterGroup(group.RouterGroup):
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
bot_uuid = await self.ap.bot_service.create_bot(request_context, json_data)
|
||||
if not isinstance(json_data, dict):
|
||||
return self.http_status(400, 'invalid_bot_config', 'Bot configuration must be an object')
|
||||
try:
|
||||
bot_uuid = await self.ap.bot_service.create_bot(request_context, json_data)
|
||||
except BotApplyError as exc:
|
||||
return self._apply_error_response(exc)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, 'invalid_bot_config', bot_error_message(exc, json_data))
|
||||
return self.success(data={'uuid': bot_uuid})
|
||||
|
||||
@self.route(
|
||||
@@ -63,7 +79,14 @@ class BotsRouterGroup(group.RouterGroup):
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.bot_service.update_bot(request_context, bot_uuid, json_data)
|
||||
if not isinstance(json_data, dict):
|
||||
return self.http_status(400, 'invalid_bot_config', 'Bot configuration must be an object')
|
||||
try:
|
||||
await self.ap.bot_service.update_bot(request_context, bot_uuid, json_data)
|
||||
except BotApplyError as exc:
|
||||
return self._apply_error_response(exc)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, 'invalid_bot_config', bot_error_message(exc, json_data))
|
||||
else:
|
||||
await self.ap.bot_service.delete_bot(request_context, bot_uuid)
|
||||
return self.success()
|
||||
@@ -83,6 +106,45 @@ class BotsRouterGroup(group.RouterGroup):
|
||||
)
|
||||
return self.success(data={'logs': logs, 'total_count': total_count})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/event-routes/status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
return self.success(data=await self.ap.bot_service.list_event_route_statuses(request_context, bot_uuid))
|
||||
|
||||
async def _dry_run_event_route(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
if not isinstance(json_data, dict):
|
||||
return self.http_status(400, -1, 'invalid request body')
|
||||
|
||||
result = await self.ap.bot_service.dry_run_event_route(
|
||||
request_context,
|
||||
bot_uuid=bot_uuid,
|
||||
event_type=json_data.get('event_type'),
|
||||
event_data=json_data.get('event_data', json_data.get('payload')),
|
||||
context=json_data.get('context'),
|
||||
event_bindings=json_data.get('event_bindings'),
|
||||
)
|
||||
return self.success(data=result)
|
||||
|
||||
self.route(
|
||||
'/<bot_uuid>/event-routes/dry-run',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)(_dry_run_event_route)
|
||||
# Backward-compatible alias for early local clients/tests created before
|
||||
# the product route naming was settled.
|
||||
self.route(
|
||||
'/<bot_uuid>/event_route/dry_run',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)(_dry_run_event_route)
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/send_message',
|
||||
methods=['POST'],
|
||||
|
||||
@@ -426,6 +426,15 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata.update(
|
||||
{
|
||||
'plugin_name': f'{author}/{plugin_name}',
|
||||
'install_source': 'marketplace',
|
||||
'operation': 'upgrade',
|
||||
'progress_percent': 3,
|
||||
}
|
||||
)
|
||||
ctx.set_current_action('checking for latest version')
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
import quart
|
||||
|
||||
from langbot.pkg.provider.modelmgr import errors as provider_errors
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
from .query import resolve_include_secret
|
||||
|
||||
|
||||
MODEL_TEST_ERROR_CODE = 'model_test_failed'
|
||||
|
||||
|
||||
@group.group_class('models/llm', '/api/v1/provider/models/llm')
|
||||
class LLMModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@@ -108,7 +112,14 @@ class LLMModelsRouterGroup(group.RouterGroup):
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.llm_model_service.test_llm_model(request_context, model_uuid, await quart.request.json)
|
||||
try:
|
||||
await self.ap.llm_model_service.test_llm_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except provider_errors.RequesterError as exc:
|
||||
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
|
||||
return self.success()
|
||||
|
||||
|
||||
@@ -214,9 +225,14 @@ class EmbeddingModelsRouterGroup(group.RouterGroup):
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.embedding_models_service.test_embedding_model(
|
||||
request_context, model_uuid, await quart.request.json
|
||||
)
|
||||
try:
|
||||
await self.ap.embedding_models_service.test_embedding_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except provider_errors.RequesterError as exc:
|
||||
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
|
||||
return self.success()
|
||||
|
||||
|
||||
@@ -322,5 +338,12 @@ class RerankModelsRouterGroup(group.RouterGroup):
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.rerank_models_service.test_rerank_model(request_context, model_uuid, await quart.request.json)
|
||||
try:
|
||||
await self.ap.rerank_models_service.test_rerank_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except provider_errors.RequesterError as exc:
|
||||
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
|
||||
return self.success()
|
||||
|
||||
@@ -5,10 +5,40 @@ import quart
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
from ......pipeline.extension_preferences import normalize_extension_preferences
|
||||
|
||||
|
||||
@group.group_class('tools', '/api/v1/tools')
|
||||
class ToolsRouterGroup(group.RouterGroup):
|
||||
async def _get_scoped_tool_catalog(
|
||||
self,
|
||||
request_context: RequestContext,
|
||||
) -> list[dict] | None:
|
||||
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
|
||||
bound_plugins: list[str] | None = None
|
||||
bound_mcp_servers: list[str] | None = None
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
)
|
||||
if pipeline is None:
|
||||
return None
|
||||
|
||||
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
|
||||
if not extensions_prefs['enable_all_plugins']:
|
||||
bound_plugins = [f'{plugin["author"]}/{plugin["name"]}' for plugin in extensions_prefs['plugins']]
|
||||
if not extensions_prefs['enable_all_mcp_servers']:
|
||||
bound_mcp_servers = extensions_prefs['mcp_servers']
|
||||
|
||||
return await self.ap.tool_mgr.get_resolved_tool_catalog(
|
||||
request_context,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
)
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route(
|
||||
'',
|
||||
@@ -18,57 +48,35 @@ class ToolsRouterGroup(group.RouterGroup):
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""获取所有可用工具列表"""
|
||||
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
|
||||
bound_plugins: list[str] | None = None
|
||||
bound_mcp_servers: list[str] | None = None
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {}) or {}
|
||||
if not extensions_prefs.get('enable_all_plugins', True):
|
||||
bound_plugins = [
|
||||
f'{plugin.get("author", "")}/{plugin.get("name", "")}'
|
||||
for plugin in extensions_prefs.get('plugins', [])
|
||||
if isinstance(plugin, dict) and plugin.get('name')
|
||||
]
|
||||
if not extensions_prefs.get('enable_all_mcp_servers', True):
|
||||
bound_mcp_servers = [
|
||||
server for server in (extensions_prefs.get('mcp_servers', []) or []) if isinstance(server, str)
|
||||
]
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'tools': await self.ap.tool_mgr.get_tool_catalog(
|
||||
request_context,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
)
|
||||
}
|
||||
)
|
||||
catalog = await self._get_scoped_tool_catalog(request_context)
|
||||
if catalog is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
return self.success(data={'tools': catalog})
|
||||
|
||||
@self.route(
|
||||
'/<tool_name>',
|
||||
'/<path:tool_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(tool_name: str, request_context: RequestContext) -> str:
|
||||
"""获取特定工具详情"""
|
||||
tools = await self.ap.tool_mgr.get_all_tools(request_context, include_skill_authoring=True)
|
||||
catalog = await self._get_scoped_tool_catalog(request_context)
|
||||
if catalog is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
for tool in tools:
|
||||
if tool.name == tool_name:
|
||||
for tool in catalog:
|
||||
if tool.get('name') == tool_name:
|
||||
return self.success(
|
||||
data={
|
||||
'tool': {
|
||||
'name': tool.name,
|
||||
'description': tool.description,
|
||||
'human_desc': tool.human_desc,
|
||||
'parameters': tool.parameters,
|
||||
'name': tool['name'],
|
||||
'description': tool.get('description') or '',
|
||||
'human_desc': tool.get('human_desc') or '',
|
||||
'parameters': tool.get('parameters') or {},
|
||||
'source': tool.get('source'),
|
||||
'source_name': tool.get('source_name'),
|
||||
'source_id': tool.get('source_id'),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -273,8 +273,8 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
"""Save wizard progress to metadata table.
|
||||
|
||||
Accepts JSON body with wizard state fields:
|
||||
{ "step": int, "selected_adapter": str|null, "created_bot_uuid": str|null,
|
||||
"bot_saved": bool, "selected_runner": str|null }
|
||||
{ "step": int, "selected_scenario": str|null, "selected_adapter": str|null,
|
||||
"created_bot_uuid": str|null, "bot_saved": bool, "selected_runner": str|null }
|
||||
"""
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
progress_json = json.dumps(data, ensure_ascii=False)
|
||||
@@ -322,6 +322,21 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
return self.http_status(503, -1, str(exc))
|
||||
return self.success(data=model)
|
||||
|
||||
@self.route(
|
||||
'/model-availability',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Expose Space's latest persisted model probes to the WebUI."""
|
||||
try:
|
||||
models = await self.ap.space_service.get_model_selection()
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'Failed to load LangBot Models availability: {exc}')
|
||||
return self.http_status(503, -1, 'Model availability is unavailable')
|
||||
return self.success(data={'models': [model.model_dump(mode='json') for model in models]})
|
||||
|
||||
@self.route(
|
||||
'/tasks',
|
||||
methods=['GET'],
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import copy
|
||||
import fnmatch
|
||||
import time
|
||||
import uuid
|
||||
import typing
|
||||
from ... import management_diagnostics as diagnostics
|
||||
|
||||
import sqlalchemy
|
||||
from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
|
||||
from langbot_plugin.api.entities.builtin.runner.event import (
|
||||
ActorContext,
|
||||
RawEventRef,
|
||||
SubjectContext,
|
||||
)
|
||||
from langbot_plugin.api.entities.builtin.runner.input import AgentInput
|
||||
from langbot_plugin.api.entities.builtin.provider import message as provider_message
|
||||
from langbot_plugin.api.entities.builtin.provider import prompt as provider_prompt
|
||||
|
||||
from ....core import app
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....agent.runner.config_schema import extract_prompt_config
|
||||
from ....agent.runner.execution_context import build_execution_query, project_mcp_resource_config
|
||||
from ....agent.runner.host_models import (
|
||||
AgentBinding,
|
||||
AgentEventEnvelope,
|
||||
BindingScope,
|
||||
DeliveryPolicy,
|
||||
StatePolicy,
|
||||
)
|
||||
from ....agent.runner.resource_policy import ResourcePolicyProjector
|
||||
from ....agent.runner.platform_tools import (
|
||||
PLATFORM_TOOL_DEFINITIONS,
|
||||
platform_tool_catalog,
|
||||
resolve_agent_platform_tool_names,
|
||||
validate_debug_mock_options,
|
||||
)
|
||||
from ....entity.persistence import agent as persistence_agent
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..context import ExecutionContext, RequestContext
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
AGENT_KIND_AGENT = 'agent'
|
||||
AGENT_KIND_PIPELINE = 'pipeline'
|
||||
AGENT_KIND_EVENT_PROCESSOR = 'event_processor'
|
||||
PIPELINE_EVENT_PATTERNS = ['message.*']
|
||||
AGENT_DEFAULT_EVENT_PATTERNS = ['*']
|
||||
|
||||
|
||||
class AgentService:
|
||||
"""Unified processor facade for the peer Agent and Pipeline types."""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_agent_metadata(self, context: TenantContext) -> dict[str, typing.Any]:
|
||||
"""Return metadata needed by Agent forms."""
|
||||
pipeline_metadata = await self.ap.pipeline_service.get_pipeline_metadata(context)
|
||||
ai_metadata = next((item for item in pipeline_metadata if item.get('name') == 'ai'), None)
|
||||
host_tools: list[dict[str, typing.Any]] | None = None
|
||||
get_tool_catalog = getattr(getattr(self.ap, 'tool_mgr', None), 'get_resolved_tool_catalog', None)
|
||||
if get_tool_catalog is not None:
|
||||
try:
|
||||
host_tools = await get_tool_catalog(
|
||||
context,
|
||||
include_skill_authoring=True,
|
||||
include_mcp_resource_tools=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'Failed to load Agent Host tool catalog: {exc}')
|
||||
event_processors = []
|
||||
registry = getattr(self.ap, 'runner_registry', None)
|
||||
if registry is not None:
|
||||
event_processors = [
|
||||
item.model_dump(mode='json')
|
||||
for item in await registry.list_runners(
|
||||
context,
|
||||
usage='event',
|
||||
use_cache=False,
|
||||
)
|
||||
]
|
||||
return {
|
||||
'event_processors': event_processors,
|
||||
'runner_config': ai_metadata,
|
||||
'platform_tools': platform_tool_catalog(),
|
||||
'host_tools': host_tools,
|
||||
'kinds': [
|
||||
{
|
||||
'name': AGENT_KIND_AGENT,
|
||||
'supported_event_patterns': AGENT_DEFAULT_EVENT_PATTERNS,
|
||||
'message_only': False,
|
||||
},
|
||||
{
|
||||
'name': AGENT_KIND_PIPELINE,
|
||||
'supported_event_patterns': PIPELINE_EVENT_PATTERNS,
|
||||
'message_only': True,
|
||||
},
|
||||
{'name': AGENT_KIND_EVENT_PROCESSOR, 'supported_event_patterns': ['*'], 'message_only': False},
|
||||
],
|
||||
}
|
||||
|
||||
async def get_agents(
|
||||
self,
|
||||
context: TenantContext,
|
||||
sort_by: str = 'updated_at',
|
||||
sort_order: str = 'DESC',
|
||||
) -> list[dict]:
|
||||
agents = await self._get_agent_rows(context)
|
||||
pipelines = await self.ap.pipeline_service.get_pipelines(
|
||||
context,
|
||||
sort_by='updated_at',
|
||||
sort_order='DESC',
|
||||
)
|
||||
|
||||
items = [self._agent_to_product_item(agent) for agent in agents]
|
||||
items.extend(self._pipeline_to_product_item(pipeline) for pipeline in pipelines)
|
||||
|
||||
reverse = sort_order == 'DESC'
|
||||
sort_key = sort_by if sort_by in {'created_at', 'updated_at'} else 'updated_at'
|
||||
return sorted(items, key=lambda item: self._parse_sort_time(item.get(sort_key)), reverse=reverse)
|
||||
|
||||
async def get_agent(self, context: TenantContext, agent_uuid: str) -> dict | None:
|
||||
agent = await self._get_agent_row(context, agent_uuid)
|
||||
if agent is not None:
|
||||
return self._agent_to_product_item(agent, include_config=True)
|
||||
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(context, agent_uuid)
|
||||
if pipeline is not None:
|
||||
return self._pipeline_to_product_item(pipeline, include_config=True)
|
||||
|
||||
return None
|
||||
|
||||
@diagnostics.observe('http.agent.debug_agent', source='webui_debug')
|
||||
async def debug_agent(
|
||||
self,
|
||||
context: RequestContext,
|
||||
agent_uuid: str,
|
||||
payload: dict[str, typing.Any],
|
||||
*,
|
||||
on_result: typing.Callable[[dict[str, typing.Any]], typing.Awaitable[None]] | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Execute one synthetic event against a configured Agent.
|
||||
|
||||
The debug surface uses a trusted Workspace execution context, never
|
||||
delivers outputs to a real platform, and supports both message and
|
||||
non-message event envelopes.
|
||||
"""
|
||||
diagnostics.workspace(context)
|
||||
agent = await self.get_agent(context, agent_uuid)
|
||||
if agent is None or agent.get('kind') not in {AGENT_KIND_AGENT, AGENT_KIND_EVENT_PROCESSOR}:
|
||||
raise ValueError('Agent not found')
|
||||
|
||||
event_type = str(payload.get('event_type', 'message.received')).strip()
|
||||
if not event_type or len(event_type) > 128:
|
||||
raise ValueError('Invalid event_type')
|
||||
if not self._supports_event_type(
|
||||
agent.get('supported_event_patterns'),
|
||||
event_type,
|
||||
):
|
||||
raise ValueError('Agent does not support this event type')
|
||||
|
||||
text = str(payload.get('text') or '').strip()
|
||||
if len(text) > 20_000:
|
||||
raise ValueError('Debug input is too long')
|
||||
event_data = payload.get('data', {})
|
||||
if not isinstance(event_data, dict):
|
||||
raise ValueError('Debug event data must be an object')
|
||||
mock_options = validate_debug_mock_options(payload.get('mock', {}))
|
||||
|
||||
config = agent.get('config')
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError('Agent configuration is invalid')
|
||||
_, runner_id, runner_config = RunnerConfigResolver.resolve_agent_config(config)
|
||||
if not runner_id:
|
||||
raise ValueError('Agent has no configured runner')
|
||||
|
||||
conversation_id = str(payload.get('conversation_id') or f'debug:{agent_uuid}').strip()
|
||||
if not conversation_id or len(conversation_id) > 256:
|
||||
raise ValueError('Invalid debug conversation_id')
|
||||
|
||||
actor_payload = payload.get(
|
||||
'actor',
|
||||
{
|
||||
'actor_type': 'user',
|
||||
'actor_id': str(
|
||||
event_data.get('member_id')
|
||||
or event_data.get('requester_id')
|
||||
or event_data.get('user_id')
|
||||
or 'debug-user'
|
||||
),
|
||||
'actor_name': str(
|
||||
event_data.get('member_name')
|
||||
or event_data.get('requester_name')
|
||||
or event_data.get('user_name')
|
||||
or 'Debug User'
|
||||
),
|
||||
},
|
||||
)
|
||||
subject_payload = payload.get(
|
||||
'subject',
|
||||
{
|
||||
'subject_type': 'message' if event_type.startswith('message.') else event_type.split('.', 1)[0],
|
||||
'subject_id': 'debug-subject',
|
||||
'data': event_data,
|
||||
},
|
||||
)
|
||||
if not isinstance(actor_payload, dict) or not isinstance(subject_payload, dict):
|
||||
raise ValueError('Debug actor and subject must be objects')
|
||||
|
||||
event_id = f'debug:{agent_uuid}:{uuid.uuid4()}'
|
||||
is_group = event_type.startswith(('group.', 'bot.')) or bool(event_data.get('group_id'))
|
||||
actor = ActorContext.model_validate(actor_payload)
|
||||
target_id = str(event_data.get('group_id') or 'debug-group') if is_group else actor.actor_id
|
||||
event = AgentEventEnvelope(
|
||||
event_id=event_id,
|
||||
event_type=event_type,
|
||||
event_time=int(time.time()),
|
||||
source='webui',
|
||||
source_event_type=event_type,
|
||||
workspace_id=context.workspace_uuid,
|
||||
conversation_id=conversation_id,
|
||||
actor=actor,
|
||||
subject=SubjectContext.model_validate(subject_payload),
|
||||
input=AgentInput.model_validate(
|
||||
{
|
||||
'text': text or event_type,
|
||||
'contents': [
|
||||
{'type': 'text', 'text': text or event_type},
|
||||
],
|
||||
'attachments': [],
|
||||
}
|
||||
),
|
||||
delivery=DeliveryContext(
|
||||
surface='webui',
|
||||
reply_target={
|
||||
'target_type': 'group' if is_group else 'person',
|
||||
'target_id': target_id,
|
||||
**({'group_id': target_id} if is_group else {}),
|
||||
**(
|
||||
{'message_id': str(event_data.get('message_id') or 'debug-message')}
|
||||
if event_type.startswith('message.')
|
||||
else {}
|
||||
),
|
||||
},
|
||||
supports_streaming=on_result is not None,
|
||||
supports_edit=False,
|
||||
supports_reaction=False,
|
||||
platform_capabilities={
|
||||
'event_type': event_type,
|
||||
'debug': True,
|
||||
'debug_mock': True,
|
||||
'supported_apis': sorted(
|
||||
{tool.api for tool in PLATFORM_TOOL_DEFINITIONS} - set(mock_options.get('unsupported_apis', []))
|
||||
),
|
||||
'mock_options': mock_options,
|
||||
},
|
||||
),
|
||||
raw_ref=RawEventRef(ref_id=event_id, storage_key=None),
|
||||
data=event_data,
|
||||
)
|
||||
if agent.get('kind') == AGENT_KIND_EVENT_PROCESSOR:
|
||||
from langbot_plugin.api.entities.builtin.platform.events import parse_eba_event
|
||||
|
||||
typed_event = parse_eba_event({**event_data, 'type': event_type})
|
||||
from ....platform.botmgr import RuntimeBot
|
||||
|
||||
event.actor = RuntimeBot._infer_actor_context(typed_event)
|
||||
event.subject = RuntimeBot._infer_subject_context(typed_event)
|
||||
target_type, target_id, target_metadata = RuntimeBot._infer_reply_target(typed_event)
|
||||
if target_id is not None:
|
||||
event.delivery.reply_target = {
|
||||
'target_type': target_type,
|
||||
'target_id': str(target_id),
|
||||
**target_metadata,
|
||||
}
|
||||
event.data = typed_event.model_dump(mode='json', exclude={'source_platform_object', 'legacy_event'})
|
||||
binding = AgentBinding(
|
||||
binding_id=(
|
||||
f'event_processor:{agent_uuid}'
|
||||
if agent.get('kind') == AGENT_KIND_EVENT_PROCESSOR
|
||||
else f'debug:{agent_uuid}:{runner_id}'
|
||||
),
|
||||
scope=BindingScope(scope_type='agent', scope_id=agent_uuid),
|
||||
event_types=[event_type],
|
||||
runner_id=runner_id,
|
||||
runner_config=runner_config,
|
||||
resource_policy=ResourcePolicyProjector.from_runner_config(
|
||||
runner_config,
|
||||
allowed_platform_tool_names=resolve_agent_platform_tool_names(config, event_type),
|
||||
allowed_host_tool_names=config.get('allowed_tools'),
|
||||
override_runner_tools='allowed_tools' in config,
|
||||
),
|
||||
state_policy=StatePolicy(
|
||||
state_scopes=['conversation', 'actor', 'subject', 'runner'],
|
||||
),
|
||||
delivery_policy=DeliveryPolicy(
|
||||
enable_streaming=on_result is not None,
|
||||
enable_reply=False,
|
||||
enable_interactions=False,
|
||||
),
|
||||
agent_id=agent_uuid,
|
||||
processor_type=agent.get('kind', 'agent'),
|
||||
processor_id=agent_uuid,
|
||||
)
|
||||
execution_context = ExecutionContext.from_request(
|
||||
context,
|
||||
query_uuid=event_id,
|
||||
)
|
||||
# Debug bypasses pipeline preprocessing, so supply the selected Runner's
|
||||
# prompt before advertising the Host prompt API. Keep explicit [] intact.
|
||||
descriptor = await self.ap.runner_registry.get(execution_context, runner_id)
|
||||
prompt_config = extract_prompt_config(descriptor, runner_config, [])
|
||||
execution_query = build_execution_query(event, [])
|
||||
execution_query.prompt = provider_prompt.Prompt(
|
||||
name='default',
|
||||
messages=[provider_message.Message(**message) for message in copy.deepcopy(prompt_config)],
|
||||
)
|
||||
for field_name in ('instance_uuid', 'workspace_uuid', 'placement_generation', 'query_uuid'):
|
||||
object.__setattr__(execution_query, field_name, getattr(execution_context, field_name))
|
||||
project_mcp_resource_config(execution_query, runner_config)
|
||||
|
||||
output_items: list[dict[str, typing.Any]] = []
|
||||
execution_events: list[dict[str, typing.Any]] = []
|
||||
|
||||
async def observe_result(result: dict[str, typing.Any]) -> None:
|
||||
if result.get('type') not in {
|
||||
'message.delta',
|
||||
'message.completed',
|
||||
'tool.call.started',
|
||||
'tool.call.completed',
|
||||
'run.completed',
|
||||
'run.failed',
|
||||
'processor.log',
|
||||
}:
|
||||
return
|
||||
visible_result = copy.deepcopy(
|
||||
{
|
||||
key: result[key]
|
||||
for key in ('type', 'data', 'sequence', 'timestamp', 'run_id', 'usage')
|
||||
if key in result
|
||||
}
|
||||
)
|
||||
if on_result is not None:
|
||||
await on_result(visible_result)
|
||||
elif len(execution_events) < 1000:
|
||||
execution_events.append(visible_result)
|
||||
|
||||
final_text = ''
|
||||
async for output in self.ap.agent_run_orchestrator.run(
|
||||
event,
|
||||
binding,
|
||||
adapter_context={
|
||||
'_query': execution_query,
|
||||
'_execution_context': execution_context,
|
||||
'_result_observer': observe_result,
|
||||
},
|
||||
):
|
||||
output_text = self._provider_output_to_text(output)
|
||||
if output_text:
|
||||
final_text = output_text
|
||||
output_items.append(
|
||||
{
|
||||
'kind': output.__class__.__name__,
|
||||
'role': str(getattr(output, 'role', '') or ''),
|
||||
'text': output_text,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
'event_id': event_id,
|
||||
'event_type': event_type,
|
||||
'conversation_id': conversation_id,
|
||||
'final_text': final_text,
|
||||
'outputs': output_items,
|
||||
'execution_events': execution_events,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _supports_event_type(patterns: typing.Any, event_type: str) -> bool:
|
||||
normalized = patterns if isinstance(patterns, list) else AGENT_DEFAULT_EVENT_PATTERNS
|
||||
return any(isinstance(pattern, str) and fnmatch.fnmatchcase(event_type, pattern) for pattern in normalized)
|
||||
|
||||
@staticmethod
|
||||
def _provider_output_to_text(output: typing.Any) -> str:
|
||||
all_content = getattr(output, 'all_content', None)
|
||||
if all_content:
|
||||
return str(all_content)
|
||||
content = getattr(output, 'content', None)
|
||||
if content is None:
|
||||
return ''
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for item in content:
|
||||
item_data = item.model_dump(mode='json') if hasattr(item, 'model_dump') else item
|
||||
if isinstance(item_data, dict) and item_data.get('text') is not None:
|
||||
parts.append(str(item_data['text']))
|
||||
elif item_data is not None and not isinstance(item_data, dict):
|
||||
parts.append(str(item_data))
|
||||
return ''.join(parts)
|
||||
return str(content)
|
||||
|
||||
async def create_agent(self, context: TenantContext, agent_data: dict) -> dict[str, str]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
kind = agent_data.get('kind') or AGENT_KIND_AGENT
|
||||
if kind == AGENT_KIND_PIPELINE:
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(
|
||||
context,
|
||||
{
|
||||
'name': agent_data.get('name') or 'New Pipeline',
|
||||
'description': agent_data.get('description') or '',
|
||||
'emoji': agent_data.get('emoji') or '⚙️',
|
||||
'config': {},
|
||||
},
|
||||
)
|
||||
return {'uuid': pipeline_uuid, 'kind': AGENT_KIND_PIPELINE}
|
||||
|
||||
if kind not in {AGENT_KIND_AGENT, AGENT_KIND_EVENT_PROCESSOR}:
|
||||
raise ValueError(f'Unsupported agent kind: {kind}')
|
||||
|
||||
if kind == AGENT_KIND_EVENT_PROCESSOR:
|
||||
config, runner_id, patterns = await self._prepare_event_processor(context, agent_data)
|
||||
else:
|
||||
config = agent_data['config'] if 'config' in agent_data else await self._get_default_agent_config(context)
|
||||
config, runner_id, _ = RunnerConfigResolver.resolve_agent_config(config)
|
||||
await self._validate_runner_for_agent(context, runner_id)
|
||||
patterns = agent_data.get('supported_event_patterns', AGENT_DEFAULT_EVENT_PATTERNS)
|
||||
new_uuid = str(uuid.uuid4())
|
||||
values = {
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'uuid': new_uuid,
|
||||
'name': agent_data.get('name') or 'New Agent',
|
||||
'description': agent_data.get('description') or '',
|
||||
'emoji': agent_data.get('emoji') or ('🧩' if kind == AGENT_KIND_EVENT_PROCESSOR else '🤖'),
|
||||
'kind': kind,
|
||||
'component_ref': runner_id,
|
||||
'config': config,
|
||||
'supported_event_patterns': patterns,
|
||||
}
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_agent.Agent).values(**values))
|
||||
return {'uuid': new_uuid, 'kind': kind}
|
||||
|
||||
async def update_agent(self, context: TenantContext, agent_uuid: str, agent_data: dict) -> None:
|
||||
existing_agent = await self._get_agent_row(context, agent_uuid)
|
||||
if existing_agent is None:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(context, agent_uuid)
|
||||
if pipeline is None:
|
||||
raise ValueError(f'Agent {agent_uuid} not found')
|
||||
await self.ap.pipeline_service.update_pipeline(context, agent_uuid, agent_data)
|
||||
return
|
||||
|
||||
update_data = {
|
||||
field: agent_data[field]
|
||||
for field in ('name', 'description', 'emoji', 'config', 'supported_event_patterns')
|
||||
if field in agent_data
|
||||
}
|
||||
if existing_agent.kind == AGENT_KIND_EVENT_PROCESSOR and any(
|
||||
field in agent_data for field in ('config', 'component_ref', 'parameters', 'supported_event_patterns')
|
||||
):
|
||||
config, runner_id, patterns = await self._prepare_event_processor(context, agent_data, existing_agent)
|
||||
update_data.update(config=config, component_ref=runner_id, supported_event_patterns=patterns)
|
||||
if 'config' in update_data:
|
||||
config, runner_id, _ = RunnerConfigResolver.resolve_agent_config(update_data['config'])
|
||||
update_data['config'] = config
|
||||
else:
|
||||
_, runner_id, _ = RunnerConfigResolver.resolve_agent_config(existing_agent.config)
|
||||
update_data['component_ref'] = runner_id
|
||||
if existing_agent.kind == AGENT_KIND_AGENT:
|
||||
await self._validate_runner_for_agent(context, runner_id)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_agent.Agent)
|
||||
.where(persistence_agent.Agent.uuid == agent_uuid)
|
||||
.values(**update_data),
|
||||
persistence_agent.Agent,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Agent not found')
|
||||
|
||||
async def delete_agent(self, context: TenantContext, agent_uuid: str) -> None:
|
||||
existing_agent = await self._get_agent_row(context, agent_uuid)
|
||||
if existing_agent is not None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_agent.Agent).where(persistence_agent.Agent.uuid == agent_uuid),
|
||||
persistence_agent.Agent,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Agent not found')
|
||||
return
|
||||
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(context, agent_uuid)
|
||||
if pipeline is None:
|
||||
raise ValueError(f'Agent {agent_uuid} not found')
|
||||
await self.ap.pipeline_service.delete_pipeline(context, agent_uuid)
|
||||
|
||||
async def _validate_runner_for_agent(self, context, runner_id):
|
||||
if runner_id:
|
||||
descriptor = await self.ap.runner_registry.get(context, runner_id)
|
||||
if 'agent' not in descriptor.usages:
|
||||
raise ValueError('The selected Runner does not support agent usage')
|
||||
|
||||
async def _prepare_event_processor(self, context, data, existing=None):
|
||||
"""Resolve an installed component and keep its capability declaration authoritative."""
|
||||
config = copy.deepcopy(data.get('config', existing.config if existing is not None else {}))
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError('Processor configuration must be an object')
|
||||
component_ref = data.get('component_ref') or (existing.component_ref if existing is not None else None)
|
||||
if component_ref is None and not config and not data.get('parameters'):
|
||||
# An unconfigured instance cannot subscribe to or execute any events.
|
||||
return {}, None, []
|
||||
if not isinstance(component_ref, str) or not component_ref.startswith('plugin:'):
|
||||
raise ValueError('Select an installed Runner component')
|
||||
try:
|
||||
descriptor = await self.ap.runner_registry.get(context, component_ref)
|
||||
except Exception as exc:
|
||||
from ....agent.runner.errors import RunnerNotFoundError
|
||||
|
||||
if isinstance(exc, RunnerNotFoundError):
|
||||
raise ValueError('Runner component is unavailable') from exc
|
||||
raise
|
||||
if 'event' not in descriptor.usages or not descriptor.supported_event_patterns:
|
||||
raise ValueError('The component does not declare supported events')
|
||||
config['runner'] = {'id': component_ref}
|
||||
parameters = data.get('parameters')
|
||||
if parameters is None:
|
||||
runner_config = config.get('runner_config', {})
|
||||
if not isinstance(runner_config, dict):
|
||||
raise ValueError('Runner configuration must be an object')
|
||||
parameters = runner_config.get(component_ref)
|
||||
if parameters is None:
|
||||
parameters = self.ap.pipeline_service._get_default_values_from_schema(descriptor.config_schema)
|
||||
if not isinstance(parameters, dict):
|
||||
raise ValueError('Processor parameters must be an object')
|
||||
for field in descriptor.config_schema:
|
||||
if field.get('required') and parameters.get(field['name']) in (None, ''):
|
||||
raise ValueError(f'Required processor parameter: {field["name"]}')
|
||||
config['runner_config'] = {component_ref: parameters}
|
||||
return config, component_ref, descriptor.supported_event_patterns
|
||||
|
||||
async def get_processor_runs(self, context, processor_id, *, before_id=None):
|
||||
"""Read only this Workspace's Agent or plugin processor runs."""
|
||||
from ....agent.runner.run_ledger_store import RunLedgerStore
|
||||
|
||||
processor = await self.get_agent(context, processor_id)
|
||||
if processor is None or processor.get('kind') not in {'agent', AGENT_KIND_EVENT_PROCESSOR}:
|
||||
raise ValueError('Processor not found')
|
||||
store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine())
|
||||
items, cursor, has_more, total = await store.list_runs(
|
||||
workspace_id=require_workspace_uuid(context),
|
||||
**(
|
||||
{'binding_id': f'event_processor:{processor_id}'}
|
||||
if processor.get('kind') == AGENT_KIND_EVENT_PROCESSOR
|
||||
else {'agent_id': processor_id}
|
||||
),
|
||||
before_id=before_id,
|
||||
)
|
||||
return {'items': items, 'next_cursor': cursor, 'has_more': has_more, 'total': total}
|
||||
|
||||
async def get_processor_run_events(self, context, processor_id, run_id, *, after_sequence=None):
|
||||
"""Authorize the parent run before exposing any trace events."""
|
||||
from ....agent.runner.run_ledger_store import RunLedgerStore
|
||||
|
||||
processor = await self.get_agent(context, processor_id)
|
||||
if processor is None or processor.get('kind') not in {'agent', AGENT_KIND_EVENT_PROCESSOR}:
|
||||
raise ValueError('Processor not found')
|
||||
store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine())
|
||||
run = await store.get_run(run_id)
|
||||
if (
|
||||
run is None
|
||||
or run.get('workspace_id') != require_workspace_uuid(context)
|
||||
or not (
|
||||
run.get('binding_id') == f'event_processor:{processor_id}'
|
||||
if processor.get('kind') == AGENT_KIND_EVENT_PROCESSOR
|
||||
else run.get('agent_id') == processor_id
|
||||
or (
|
||||
run.get('agent_id') is None
|
||||
and str(run.get('binding_id', '')).startswith((f'agent_{processor_id}_', f'debug:{processor_id}:'))
|
||||
)
|
||||
)
|
||||
):
|
||||
raise ValueError('Processor run not found')
|
||||
items, next_cursor, _, has_more = await store.page_run_events(
|
||||
run_id=run_id,
|
||||
after_sequence=after_sequence,
|
||||
limit=100,
|
||||
)
|
||||
return {'run': run, 'items': items, 'next_cursor': next_cursor, 'has_more': has_more}
|
||||
|
||||
async def _get_agent_rows(self, context: TenantContext) -> list[persistence_agent.Agent]:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_agent.Agent),
|
||||
persistence_agent.Agent,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return list(result.all())
|
||||
|
||||
async def _get_agent_row(
|
||||
self,
|
||||
context: TenantContext,
|
||||
agent_uuid: str,
|
||||
) -> persistence_agent.Agent | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_agent.Agent).where(persistence_agent.Agent.uuid == agent_uuid),
|
||||
persistence_agent.Agent,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return result.first()
|
||||
|
||||
async def _get_default_agent_config(self, context: TenantContext) -> dict[str, typing.Any]:
|
||||
runners = []
|
||||
if getattr(self.ap, 'runner_registry', None) is not None:
|
||||
try:
|
||||
runners = await self.ap.runner_registry.list_runners(context, bound_plugins=None)
|
||||
except Exception as e:
|
||||
if getattr(self.ap, 'logger', None):
|
||||
self.ap.logger.warning(f'Failed to load plugin agent runners for default agent config: {e}')
|
||||
|
||||
if not runners:
|
||||
return {'runner': {'id': '', 'expire-time': 0}, 'runner_config': {}}
|
||||
|
||||
selected_runner = runners[0]
|
||||
return {
|
||||
'runner': {'id': selected_runner.id, 'expire-time': 0},
|
||||
'runner_config': {
|
||||
selected_runner.id: self.ap.pipeline_service._get_default_values_from_schema(
|
||||
selected_runner.config_schema
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
def _agent_to_product_item(
|
||||
self,
|
||||
agent: persistence_agent.Agent,
|
||||
include_config: bool = False,
|
||||
) -> dict[str, typing.Any]:
|
||||
item = self.ap.persistence_mgr.serialize_model(persistence_agent.Agent, agent)
|
||||
item['kind'] = item.get('kind') or AGENT_KIND_AGENT
|
||||
supported_event_patterns = item.get('supported_event_patterns')
|
||||
item['capability'] = {
|
||||
'supported_event_patterns': (
|
||||
supported_event_patterns if isinstance(supported_event_patterns, list) else AGENT_DEFAULT_EVENT_PATTERNS
|
||||
),
|
||||
'message_only': False,
|
||||
}
|
||||
if not include_config:
|
||||
item.pop('config', None)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_to_product_item(pipeline: dict, include_config: bool = False) -> dict[str, typing.Any]:
|
||||
item = pipeline.copy()
|
||||
item['kind'] = AGENT_KIND_PIPELINE
|
||||
item['component_ref'] = 'pipeline'
|
||||
item['supported_event_patterns'] = PIPELINE_EVENT_PATTERNS
|
||||
item['capability'] = {
|
||||
'supported_event_patterns': PIPELINE_EVENT_PATTERNS,
|
||||
'message_only': True,
|
||||
}
|
||||
if not include_config:
|
||||
item.pop('config', None)
|
||||
return item
|
||||
|
||||
@staticmethod
|
||||
def _parse_sort_time(value: typing.Any) -> datetime.datetime:
|
||||
if isinstance(value, datetime.datetime):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return datetime.datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return datetime.datetime.min
|
||||
return datetime.datetime.min
|
||||
@@ -1,26 +1,608 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import typing
|
||||
import json
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....discover import engine
|
||||
from ....entity.persistence import agent as persistence_agent
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .bot_errors import BotApplyError, bot_error_message
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
from ....utils import httpclient
|
||||
from ....platform.sources import http_bot_signing
|
||||
from ....platform.adapter_names import canonical_adapter_name
|
||||
from ....agent.runner.errors import RunnerNotFoundError
|
||||
|
||||
|
||||
class BotService:
|
||||
"""Bot service"""
|
||||
|
||||
ap: app.Application
|
||||
FAILURE_ROUTE_NOT_FOUND = 'route_not_found'
|
||||
FAILURE_PROCESSOR_NOT_FOUND = 'processor_not_found'
|
||||
FAILURE_PROCESSOR_INCOMPATIBLE = 'processor_incompatible'
|
||||
FAILURE_INVALID_EVENT = 'invalid_event'
|
||||
ROUTE_TRACE_KIND = 'event_route_trace'
|
||||
|
||||
BOT_FIELDS = {
|
||||
'uuid',
|
||||
'name',
|
||||
'description',
|
||||
'adapter',
|
||||
'adapter_config',
|
||||
'enable',
|
||||
'event_bindings',
|
||||
'plugin_processors',
|
||||
}
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
def _get_adapter_component(self, adapter_name: str) -> engine.Component | None:
|
||||
"""Return the discovered platform adapter component for an adapter name."""
|
||||
adapter_name = canonical_adapter_name(adapter_name)
|
||||
for component in self.ap.discover.get_components_by_kind('MessagePlatformAdapter'):
|
||||
if component.metadata.name == adapter_name:
|
||||
return component
|
||||
return None
|
||||
|
||||
def _adapter_declares_webhook_url(self, adapter_name: str) -> bool:
|
||||
"""Whether the adapter manifest declares a generated webhook URL config item."""
|
||||
component = self._get_adapter_component(adapter_name)
|
||||
if component is None:
|
||||
return False
|
||||
|
||||
for config_item in component.spec.get('config', []):
|
||||
if config_item.get('type') == 'webhook-url':
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_message_event_pattern(event_pattern: str) -> bool:
|
||||
return event_pattern == 'message.*' or event_pattern.startswith('message.')
|
||||
|
||||
@staticmethod
|
||||
def _event_pattern_covers(supported_pattern: str, binding_pattern: str) -> bool:
|
||||
if supported_pattern == '*':
|
||||
return True
|
||||
if supported_pattern == binding_pattern:
|
||||
return True
|
||||
if binding_pattern == '*':
|
||||
return False
|
||||
if supported_pattern.endswith('.*'):
|
||||
namespace = supported_pattern[:-2]
|
||||
return binding_pattern == f'{namespace}.*' or binding_pattern.startswith(f'{namespace}.')
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _agent_supports_event_pattern(cls, supported_patterns: list[str] | None, event_pattern: str) -> bool:
|
||||
patterns = supported_patterns if isinstance(supported_patterns, list) else ['*']
|
||||
return any(cls._event_pattern_covers(pattern, event_pattern) for pattern in patterns)
|
||||
|
||||
@staticmethod
|
||||
def _format_diagnostic_step(step: dict[str, typing.Any]) -> str:
|
||||
step_name = step.get('step') or 'diagnostic'
|
||||
reason = step.get('reason') or step.get('failure_code') or 'No reason provided'
|
||||
|
||||
if step_name == 'evaluate_binding':
|
||||
route_number = step.get('binding_index')
|
||||
if not isinstance(route_number, int):
|
||||
route_number = step.get('order')
|
||||
route_label = f'Route {int(route_number) + 1}' if isinstance(route_number, int) else 'Route'
|
||||
event_pattern = step.get('event_pattern') or '*'
|
||||
if step.get('selected'):
|
||||
return f'{route_label} ({event_pattern}) selected: {reason}'
|
||||
if step.get('matched'):
|
||||
return f'{route_label} ({event_pattern}) matched: {reason}'
|
||||
return f'{route_label} ({event_pattern}) skipped: {reason}'
|
||||
|
||||
if step_name == 'validate_processor':
|
||||
target_type = step.get('target_type') or 'processor'
|
||||
target_uuid = step.get('target_uuid') or ''
|
||||
suffix = f' {target_uuid}' if target_uuid else ''
|
||||
return f'Validate {target_type}{suffix}: {reason}'
|
||||
|
||||
return str(reason)
|
||||
|
||||
@classmethod
|
||||
def _format_diagnostic_steps(cls, diagnostic_details: list[dict[str, typing.Any]] | None) -> list[str]:
|
||||
return [cls._format_diagnostic_step(step) for step in diagnostic_details or []]
|
||||
|
||||
@classmethod
|
||||
def _event_route_status_from_log(cls, log: typing.Any) -> dict[str, typing.Any] | None:
|
||||
if hasattr(log, 'to_json'):
|
||||
log_data = log.to_json()
|
||||
elif isinstance(log, dict):
|
||||
log_data = log
|
||||
else:
|
||||
log_data = {
|
||||
'seq_id': getattr(log, 'seq_id', None),
|
||||
'timestamp': getattr(log, 'timestamp', None),
|
||||
'level': getattr(getattr(log, 'level', None), 'value', getattr(log, 'level', None)),
|
||||
'text': getattr(log, 'text', None),
|
||||
'metadata': getattr(log, 'metadata', None),
|
||||
}
|
||||
|
||||
metadata = log_data.get('metadata')
|
||||
if not isinstance(metadata, dict) or metadata.get('kind') != cls.ROUTE_TRACE_KIND:
|
||||
return None
|
||||
|
||||
return {
|
||||
'binding_id': metadata.get('binding_id'),
|
||||
'event_pattern': metadata.get('event_pattern'),
|
||||
'event_type': metadata.get('event_type'),
|
||||
'target_type': metadata.get('target_type'),
|
||||
'target_uuid': metadata.get('target_uuid') or '',
|
||||
'last_status': metadata.get('status'),
|
||||
'failure_code': metadata.get('failure_code'),
|
||||
'reason': metadata.get('reason') or log_data.get('text') or '',
|
||||
'run_id': metadata.get('run_id'),
|
||||
'timestamp': log_data.get('timestamp'),
|
||||
'seq_id': log_data.get('seq_id'),
|
||||
'level': log_data.get('level'),
|
||||
'message': log_data.get('text') or '',
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _target_kind(target_type: typing.Any, target_kind: str | None = None) -> str | None:
|
||||
if target_kind:
|
||||
return target_kind
|
||||
if target_type == 'discard':
|
||||
return 'discard'
|
||||
if target_type in {'agent', 'pipeline', 'event_processor'}:
|
||||
return str(target_type)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _diagnostic_result(
|
||||
cls,
|
||||
*,
|
||||
matched: bool,
|
||||
failure_code: str | None = None,
|
||||
reason: str = '',
|
||||
binding: dict[str, typing.Any] | None = None,
|
||||
diagnostic_steps: list[dict[str, typing.Any]] | None = None,
|
||||
target_name: str | None = None,
|
||||
target_kind: str | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
binding = binding or {}
|
||||
target_type = binding.get('target_type')
|
||||
target_uuid = binding.get('target_uuid') or ''
|
||||
matched_binding_index = binding.get('_dry_run_index')
|
||||
if not isinstance(matched_binding_index, int):
|
||||
matched_binding_index = binding.get('order')
|
||||
if not isinstance(matched_binding_index, int):
|
||||
matched_binding_index = None
|
||||
target = None
|
||||
if target_type:
|
||||
target = {
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid or None,
|
||||
'target_name': target_name,
|
||||
'kind': cls._target_kind(target_type, target_kind),
|
||||
}
|
||||
return {
|
||||
'matched': matched,
|
||||
'binding_id': binding.get('id'),
|
||||
'matched_binding_id': binding.get('id'),
|
||||
'matched_binding_index': matched_binding_index,
|
||||
'event_pattern': binding.get('event_pattern'),
|
||||
'target_type': binding.get('target_type'),
|
||||
'target_uuid': target_uuid,
|
||||
'target': target,
|
||||
'reason': reason,
|
||||
'failure_code': failure_code,
|
||||
'diagnostic_steps': cls._format_diagnostic_steps(diagnostic_steps),
|
||||
'diagnostic_details': diagnostic_steps or [],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _build_dry_run_event(event_type: str, event_data: typing.Any, context: typing.Any) -> dict[str, typing.Any]:
|
||||
event: dict[str, typing.Any] = {}
|
||||
if isinstance(event_data, dict):
|
||||
event.update(event_data)
|
||||
elif event_data is not None:
|
||||
raise ValueError('event_data must be an object')
|
||||
event['type'] = event_type
|
||||
|
||||
if context is None:
|
||||
return event
|
||||
if not isinstance(context, dict):
|
||||
raise ValueError('context must be an object')
|
||||
event['context'] = context
|
||||
return event
|
||||
|
||||
@staticmethod
|
||||
def _normalize_dry_run_bindings(bindings: typing.Any) -> list[dict[str, typing.Any]]:
|
||||
if bindings is None:
|
||||
return []
|
||||
if not isinstance(bindings, list):
|
||||
raise ValueError('event_bindings must be an array')
|
||||
|
||||
normalized: list[dict[str, typing.Any]] = []
|
||||
for index, raw_binding in enumerate(bindings):
|
||||
if not isinstance(raw_binding, dict):
|
||||
continue
|
||||
event_pattern = str(raw_binding.get('event_pattern') or '').strip()
|
||||
target_type = str(raw_binding.get('target_type') or '').strip()
|
||||
if not event_pattern or not target_type:
|
||||
continue
|
||||
|
||||
try:
|
||||
priority = int(raw_binding.get('priority') or 0)
|
||||
except (TypeError, ValueError):
|
||||
priority = 0
|
||||
|
||||
target_uuid = str(raw_binding.get('target_uuid') or '').strip()
|
||||
if target_type == 'discard':
|
||||
target_uuid = ''
|
||||
|
||||
filters = raw_binding.get('filters') if isinstance(raw_binding.get('filters'), list) else []
|
||||
normalized.append(
|
||||
{
|
||||
'id': raw_binding.get('id'),
|
||||
'event_pattern': event_pattern,
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'filters': filters,
|
||||
'priority': priority,
|
||||
'enabled': bool(raw_binding.get('enabled', True)),
|
||||
# For draft bindings, current array order is the effective order
|
||||
# that would be persisted on save.
|
||||
'order': index,
|
||||
'_dry_run_index': index,
|
||||
}
|
||||
)
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _index_event_bindings(bindings: list[dict[str, typing.Any]]) -> list[dict[str, typing.Any]]:
|
||||
indexed: list[dict[str, typing.Any]] = []
|
||||
for index, binding in enumerate(bindings):
|
||||
copied = binding.copy()
|
||||
copied.setdefault('_dry_run_index', index)
|
||||
indexed.append(copied)
|
||||
return indexed
|
||||
|
||||
async def _get_pipeline_entity(
|
||||
self, context: TenantContext, pipeline_uuid: str
|
||||
) -> persistence_pipeline.LegacyPipeline | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return result.first()
|
||||
|
||||
async def _get_agent_entity(self, context: TenantContext, agent_uuid: str) -> persistence_agent.Agent | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_agent.Agent).where(persistence_agent.Agent.uuid == agent_uuid),
|
||||
persistence_agent.Agent,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return result.first()
|
||||
|
||||
async def dry_run_event_route(
|
||||
self,
|
||||
tenant_context: TenantContext,
|
||||
bot_uuid: str,
|
||||
event_type: str,
|
||||
event_data: dict[str, typing.Any] | None = None,
|
||||
context: dict[str, typing.Any] | None = None,
|
||||
event_bindings: list[dict[str, typing.Any]] | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Diagnose Bot event routing without dispatching to Agent, Pipeline, or platform actions."""
|
||||
from ....platform.botmgr import RuntimeBot
|
||||
|
||||
event_type = str(event_type or '').strip()
|
||||
if not event_type:
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
failure_code=self.FAILURE_INVALID_EVENT,
|
||||
reason='event_type is required',
|
||||
diagnostic_steps=[
|
||||
{
|
||||
'step': 'validate_event',
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_INVALID_EVENT,
|
||||
'reason': 'event_type is required',
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
bot = await self.get_bot(tenant_context, bot_uuid, include_secret=False)
|
||||
if bot is None:
|
||||
raise Exception('Bot not found')
|
||||
|
||||
try:
|
||||
event = self._build_dry_run_event(event_type, event_data, context)
|
||||
bindings = (
|
||||
self._normalize_dry_run_bindings(event_bindings)
|
||||
if event_bindings is not None
|
||||
else self._index_event_bindings(
|
||||
RuntimeBot._get_event_bindings_from_value(bot.get('event_bindings') or [])
|
||||
)
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
failure_code=self.FAILURE_INVALID_EVENT,
|
||||
reason=str(exc),
|
||||
diagnostic_steps=[
|
||||
{
|
||||
'step': 'validate_event',
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_INVALID_EVENT,
|
||||
'reason': str(exc),
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
selected_binding, diagnostic_steps = RuntimeBot._evaluate_eba_event_bindings(
|
||||
bindings,
|
||||
event,
|
||||
event_type,
|
||||
)
|
||||
if selected_binding is None:
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
failure_code=self.FAILURE_ROUTE_NOT_FOUND,
|
||||
reason='No enabled event binding matched event_type and filters',
|
||||
diagnostic_steps=diagnostic_steps,
|
||||
)
|
||||
|
||||
target_type = selected_binding.get('target_type')
|
||||
target_uuid = str(selected_binding.get('target_uuid') or '')
|
||||
|
||||
if target_type == 'discard':
|
||||
return self._diagnostic_result(
|
||||
matched=True,
|
||||
binding=selected_binding,
|
||||
reason='Event route matched discard target',
|
||||
diagnostic_steps=diagnostic_steps,
|
||||
)
|
||||
|
||||
if target_type == 'pipeline':
|
||||
if not RuntimeBot._is_message_event_type(event_type):
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
binding=selected_binding,
|
||||
failure_code=self.FAILURE_PROCESSOR_INCOMPATIBLE,
|
||||
reason='Pipeline targets only support message events',
|
||||
diagnostic_steps=diagnostic_steps
|
||||
+ [
|
||||
{
|
||||
'step': 'validate_processor',
|
||||
'binding_id': selected_binding.get('id'),
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_PROCESSOR_INCOMPATIBLE,
|
||||
'reason': 'Pipeline targets only support message events',
|
||||
}
|
||||
],
|
||||
)
|
||||
pipeline = await self._get_pipeline_entity(tenant_context, target_uuid) if target_uuid else None
|
||||
if pipeline is None:
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
binding=selected_binding,
|
||||
failure_code=self.FAILURE_PROCESSOR_NOT_FOUND,
|
||||
reason='Pipeline target not found',
|
||||
diagnostic_steps=diagnostic_steps
|
||||
+ [
|
||||
{
|
||||
'step': 'validate_processor',
|
||||
'binding_id': selected_binding.get('id'),
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_PROCESSOR_NOT_FOUND,
|
||||
'reason': 'Pipeline target not found',
|
||||
}
|
||||
],
|
||||
)
|
||||
return self._diagnostic_result(
|
||||
matched=True,
|
||||
binding=selected_binding,
|
||||
target_name=getattr(pipeline, 'name', None),
|
||||
reason='Event route matched pipeline target',
|
||||
diagnostic_steps=diagnostic_steps,
|
||||
)
|
||||
|
||||
if target_type in {'agent', 'event_processor'}:
|
||||
agent = await self._get_agent_entity(tenant_context, target_uuid)
|
||||
if agent is None or getattr(agent, 'kind', 'agent') != target_type:
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
binding=selected_binding,
|
||||
failure_code=self.FAILURE_PROCESSOR_NOT_FOUND,
|
||||
reason='Agent target not found',
|
||||
diagnostic_steps=diagnostic_steps
|
||||
+ [
|
||||
{
|
||||
'step': 'validate_processor',
|
||||
'binding_id': selected_binding.get('id'),
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_PROCESSOR_NOT_FOUND,
|
||||
'reason': 'Agent target not found',
|
||||
}
|
||||
],
|
||||
)
|
||||
if not RuntimeBot._agent_supports_event_type(getattr(agent, 'supported_event_patterns', None), event_type):
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
binding=selected_binding,
|
||||
failure_code=self.FAILURE_PROCESSOR_INCOMPATIBLE,
|
||||
reason='Agent target does not support this event type',
|
||||
diagnostic_steps=diagnostic_steps
|
||||
+ [
|
||||
{
|
||||
'step': 'validate_processor',
|
||||
'binding_id': selected_binding.get('id'),
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_PROCESSOR_INCOMPATIBLE,
|
||||
'reason': 'Agent target does not support this event type',
|
||||
}
|
||||
],
|
||||
)
|
||||
return self._diagnostic_result(
|
||||
matched=True,
|
||||
binding=selected_binding,
|
||||
target_name=getattr(agent, 'name', None),
|
||||
target_kind=getattr(agent, 'kind', None),
|
||||
reason='Event route matched agent target',
|
||||
diagnostic_steps=diagnostic_steps,
|
||||
)
|
||||
|
||||
return self._diagnostic_result(
|
||||
matched=False,
|
||||
binding=selected_binding,
|
||||
failure_code=self.FAILURE_PROCESSOR_INCOMPATIBLE,
|
||||
reason=f'Unsupported event binding target type: {target_type}',
|
||||
diagnostic_steps=diagnostic_steps
|
||||
+ [
|
||||
{
|
||||
'step': 'validate_processor',
|
||||
'binding_id': selected_binding.get('id'),
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'matched': False,
|
||||
'failure_code': self.FAILURE_PROCESSOR_INCOMPATIBLE,
|
||||
'reason': f'Unsupported event binding target type: {target_type}',
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
async def _normalize_event_bindings(self, context: TenantContext, bindings: list[dict] | None) -> list[dict]:
|
||||
"""Validate and normalize Bot event bindings."""
|
||||
if not bindings:
|
||||
return []
|
||||
|
||||
normalized: list[dict] = []
|
||||
for index, raw_binding in enumerate(bindings):
|
||||
if not isinstance(raw_binding, dict):
|
||||
continue
|
||||
|
||||
event_pattern = str(raw_binding.get('event_pattern') or '').strip()
|
||||
target_type = str(raw_binding.get('target_type') or '').strip()
|
||||
target_uuid = str(raw_binding.get('target_uuid') or '').strip()
|
||||
if not event_pattern or not target_type:
|
||||
continue
|
||||
|
||||
if target_type == 'pipeline':
|
||||
if not self._is_message_event_pattern(event_pattern):
|
||||
raise ValueError('Pipeline can only be bound to message events')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline.uuid).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == target_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if result.first() is None:
|
||||
raise ValueError('Pipeline not found')
|
||||
elif target_type == 'agent':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_agent.Agent).where(persistence_agent.Agent.uuid == target_uuid),
|
||||
persistence_agent.Agent,
|
||||
context,
|
||||
)
|
||||
)
|
||||
agent = result.first()
|
||||
if agent is None or agent.kind != target_type:
|
||||
raise ValueError('Processor not found')
|
||||
if not self._agent_supports_event_pattern(agent.supported_event_patterns, event_pattern):
|
||||
raise ValueError('Agent does not support this event pattern')
|
||||
elif target_type == 'discard':
|
||||
target_uuid = ''
|
||||
else:
|
||||
raise ValueError(f'Unsupported event binding target type: {target_type}')
|
||||
|
||||
normalized.append(
|
||||
{
|
||||
'id': raw_binding.get('id') or str(uuid.uuid4()),
|
||||
'event_pattern': event_pattern,
|
||||
'target_type': target_type,
|
||||
'target_uuid': target_uuid,
|
||||
'filters': raw_binding.get('filters') or [],
|
||||
'priority': int(raw_binding.get('priority') or 0),
|
||||
'enabled': bool(raw_binding.get('enabled', True)),
|
||||
'description': raw_binding.get('description') or '',
|
||||
'order': index,
|
||||
}
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
||||
async def _normalize_plugin_processors(self, context: TenantContext, subscriptions: typing.Any) -> list[dict]:
|
||||
"""Validate explicit subscriptions within this Workspace; events come from the Runner."""
|
||||
if not isinstance(subscriptions, list):
|
||||
raise ValueError('plugin_processors must be an array')
|
||||
normalized = []
|
||||
seen = set()
|
||||
for subscription in subscriptions:
|
||||
if not isinstance(subscription, dict):
|
||||
raise ValueError('Each plugin processor binding must be an object')
|
||||
processor_uuid = subscription.get('processor_uuid')
|
||||
if not isinstance(processor_uuid, str) or not processor_uuid.strip():
|
||||
raise ValueError('Plugin processor UUID is required')
|
||||
if processor_uuid in seen:
|
||||
raise ValueError('A plugin processor can only be bound once per bot')
|
||||
enabled = subscription.get('enabled', True)
|
||||
if not isinstance(enabled, bool):
|
||||
raise ValueError('Plugin processor enabled must be a boolean')
|
||||
agent = await self._get_agent_entity(context, processor_uuid)
|
||||
if agent is None or agent.kind != 'event_processor':
|
||||
raise ValueError('Plugin processor not found')
|
||||
if enabled:
|
||||
try:
|
||||
descriptor = await self.ap.runner_registry.get(context, agent.component_ref)
|
||||
except RunnerNotFoundError as exc:
|
||||
raise ValueError('Runner component is unavailable') from exc
|
||||
if 'event' not in descriptor.usages or not descriptor.supported_event_patterns:
|
||||
raise ValueError('Select an available Runner that declares event usage')
|
||||
seen.add(processor_uuid)
|
||||
normalized.append({'processor_uuid': processor_uuid, 'enabled': enabled})
|
||||
return normalized
|
||||
|
||||
async def _prepare_bot_data(self, context: TenantContext, bot_data: dict, *, include_uuid: bool) -> dict:
|
||||
"""Normalize Bot write payloads to the current event-routing model."""
|
||||
update_data = bot_data.copy()
|
||||
if not include_uuid:
|
||||
update_data.pop('uuid', None)
|
||||
|
||||
update_data = {key: value for key, value in update_data.items() if key in self.BOT_FIELDS}
|
||||
if 'adapter' in update_data:
|
||||
update_data['adapter'] = canonical_adapter_name(update_data['adapter'])
|
||||
if 'event_bindings' in update_data:
|
||||
update_data['event_bindings'] = await self._normalize_event_bindings(
|
||||
context, update_data.get('event_bindings')
|
||||
)
|
||||
if 'plugin_processors' in update_data:
|
||||
update_data['plugin_processors'] = await self._normalize_plugin_processors(
|
||||
context, update_data['plugin_processors']
|
||||
)
|
||||
return update_data
|
||||
|
||||
async def get_bots(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
@@ -33,7 +615,10 @@ class BotService:
|
||||
if not include_secret:
|
||||
masked_columns = ['adapter_config']
|
||||
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
|
||||
for bot in serialized:
|
||||
bot['adapter'] = canonical_adapter_name(bot['adapter'])
|
||||
return serialized
|
||||
|
||||
async def get_bot(self, context: TenantContext, bot_uuid: str, include_secret: bool = False) -> dict | None:
|
||||
"""获取机器人"""
|
||||
@@ -54,7 +639,9 @@ class BotService:
|
||||
if not include_secret:
|
||||
masked_columns = ['adapter_config']
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
|
||||
serialized = self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
|
||||
serialized['adapter'] = canonical_adapter_name(serialized['adapter'])
|
||||
return serialized
|
||||
|
||||
async def get_runtime_bot_info(
|
||||
self,
|
||||
@@ -73,18 +660,10 @@ class BotService:
|
||||
if runtime_bot is not None:
|
||||
adapter_runtime_values['bot_account_id'] = runtime_bot.adapter.bot_account_id
|
||||
|
||||
# Webhook URL for unified webhook adapters (independent of bot running state)
|
||||
if persistence_bot['adapter'] in [
|
||||
'wecom',
|
||||
'wecombot',
|
||||
'officialaccount',
|
||||
'qqofficial',
|
||||
'slack',
|
||||
'wecomcs',
|
||||
'LINE',
|
||||
'lark',
|
||||
'http_bot',
|
||||
]:
|
||||
# Webhook URL for adapters that declare a generated webhook config item.
|
||||
# This is manifest-driven so EBA adapters do not need to be mirrored in a
|
||||
# second hard-coded list.
|
||||
if self._adapter_declares_webhook_url(persistence_bot['adapter']):
|
||||
webhook_prefix = self.ap.instance_config.data['api'].get('webhook_prefix', 'http://127.0.0.1:5300')
|
||||
extra_webhook_prefix = self.ap.instance_config.data['api'].get('extra_webhook_prefix', '')
|
||||
webhook_url = f'/bots/{bot_uuid}'
|
||||
@@ -114,24 +693,11 @@ class BotService:
|
||||
raise ValueError(f'Maximum number of bots ({max_bots}) reached')
|
||||
|
||||
# TODO: 检查配置信息格式
|
||||
bot_data = bot_data.copy()
|
||||
bot_data = await self._prepare_bot_data(context, bot_data, include_uuid=True)
|
||||
bot_data['uuid'] = str(uuid.uuid4())
|
||||
bot_data['workspace_uuid'] = workspace_uuid
|
||||
|
||||
# bind the most recently updated pipeline if any exist
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
bot_data['use_pipeline_uuid'] = pipeline.uuid
|
||||
bot_data['use_pipeline_name'] = pipeline.name
|
||||
bot_data.setdefault('event_bindings', [])
|
||||
bot_data.setdefault('plugin_processors', [])
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(bot_data))
|
||||
|
||||
@@ -139,42 +705,17 @@ class BotService:
|
||||
|
||||
try:
|
||||
await self.ap.platform_mgr.load_bot(context, bot)
|
||||
except Exception:
|
||||
# The bot row was already inserted above; without this rollback a
|
||||
# failing adapter constructor (e.g. a missing optional credential
|
||||
# key) would leave a permanently disabled orphan bot in the DB.
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_data['uuid'])
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise BotApplyError(bot_error_message(exc, bot), bot['uuid']) from exc
|
||||
|
||||
return bot_data['uuid']
|
||||
|
||||
async def update_bot(self, context: TenantContext, bot_uuid: str, bot_data: dict) -> None:
|
||||
"""Update bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
update_data = bot_data.copy()
|
||||
|
||||
update_data.pop('uuid', None)
|
||||
update_data = await self._prepare_bot_data(context, bot_data, include_uuid=False)
|
||||
update_data.pop('workspace_uuid', None)
|
||||
|
||||
# set use_pipeline_name
|
||||
if 'use_pipeline_uuid' in update_data:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
update_data['use_pipeline_name'] = pipeline.name
|
||||
else:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
@@ -184,17 +725,28 @@ class BotService:
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
# select from db
|
||||
runtime_fields = {'adapter', 'adapter_config', 'enable', 'event_bindings', 'plugin_processors'}
|
||||
if not runtime_fields.intersection(update_data):
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is not None:
|
||||
if 'name' in update_data:
|
||||
runtime_bot.bot_entity.name = update_data['name']
|
||||
if 'description' in update_data:
|
||||
runtime_bot.bot_entity.description = update_data['description']
|
||||
return
|
||||
|
||||
# Persisted configuration is distinct from applying it to the running adapter.
|
||||
bot = await self.get_bot(context, bot_uuid, include_secret=True)
|
||||
try:
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
|
||||
if runtime_bot.enable:
|
||||
await runtime_bot.run()
|
||||
except Exception as exc:
|
||||
raise BotApplyError(bot_error_message(exc, bot), bot['uuid']) from exc
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
if runtime_bot.enable:
|
||||
await runtime_bot.run()
|
||||
|
||||
# update all conversation that use this bot
|
||||
# Reset conversations using this bot after its configuration is applied.
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
@@ -229,6 +781,76 @@ class BotService:
|
||||
|
||||
return [log.to_json() for log in logs], total_count
|
||||
|
||||
async def list_event_route_statuses(self, context: TenantContext, bot_uuid: str) -> dict[str, typing.Any]:
|
||||
"""Return recent runtime status for Bot event routes from in-memory Bot logs."""
|
||||
from ....platform.botmgr import RuntimeBot
|
||||
|
||||
bot = await self.get_bot(context, bot_uuid, include_secret=False)
|
||||
if bot is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
|
||||
latest_by_binding: dict[str, dict[str, typing.Any]] = {}
|
||||
unmatched_events: list[dict[str, typing.Any]] = []
|
||||
runtime_logs = getattr(getattr(runtime_bot, 'logger', None), 'logs', [])
|
||||
for log in runtime_logs:
|
||||
status = self._event_route_status_from_log(log)
|
||||
if status is None:
|
||||
continue
|
||||
binding_id = status.get('binding_id')
|
||||
if binding_id:
|
||||
latest_by_binding[str(binding_id)] = status
|
||||
else:
|
||||
unmatched_events.append(status)
|
||||
|
||||
runtime_entity = getattr(runtime_bot, 'bot_entity', None)
|
||||
raw_bindings = getattr(runtime_entity, 'event_bindings', None) if runtime_entity is not None else None
|
||||
if raw_bindings is None:
|
||||
raw_bindings = bot.get('event_bindings') or []
|
||||
bindings = RuntimeBot._get_event_bindings_from_value(raw_bindings)
|
||||
routes: list[dict[str, typing.Any]] = []
|
||||
current_binding_ids: set[str] = set()
|
||||
for index, binding in enumerate(bindings):
|
||||
binding_id = binding.get('id')
|
||||
if binding_id:
|
||||
current_binding_ids.add(str(binding_id))
|
||||
route_status = {
|
||||
'binding_id': binding_id,
|
||||
'event_pattern': binding.get('event_pattern'),
|
||||
'event_type': None,
|
||||
'target_type': binding.get('target_type'),
|
||||
'target_uuid': binding.get('target_uuid') or '',
|
||||
'last_status': None,
|
||||
'failure_code': None,
|
||||
'reason': None,
|
||||
'run_id': None,
|
||||
'timestamp': None,
|
||||
'seq_id': None,
|
||||
'level': None,
|
||||
'message': '',
|
||||
'order': binding.get('order', index),
|
||||
'enabled': binding.get('enabled', True),
|
||||
'current': True,
|
||||
}
|
||||
if binding_id and str(binding_id) in latest_by_binding:
|
||||
route_status.update(latest_by_binding[str(binding_id)])
|
||||
route_status['order'] = binding.get('order', index)
|
||||
route_status['enabled'] = binding.get('enabled', True)
|
||||
route_status['current'] = True
|
||||
routes.append(route_status)
|
||||
|
||||
stale_routes = [
|
||||
{**status, 'current': False}
|
||||
for binding_id, status in latest_by_binding.items()
|
||||
if binding_id not in current_binding_ids
|
||||
]
|
||||
|
||||
return {
|
||||
'routes': routes,
|
||||
'unmatched_events': unmatched_events[-10:],
|
||||
'stale_routes': stale_routes,
|
||||
}
|
||||
|
||||
async def send_http_bot_test_message(
|
||||
self,
|
||||
context: TenantContext,
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""User-facing bot configuration errors without credentials or validation inputs."""
|
||||
|
||||
import json
|
||||
|
||||
import pydantic
|
||||
|
||||
from .secrets import redact_secrets
|
||||
|
||||
|
||||
class BotApplyError(Exception):
|
||||
"""Configuration was persisted, but the runtime could not apply it."""
|
||||
|
||||
def __init__(self, message: str, bot_uuid: str | None = None):
|
||||
super().__init__(message)
|
||||
self.bot_uuid = bot_uuid
|
||||
|
||||
|
||||
def bot_error_message(error: Exception, configuration: dict) -> str:
|
||||
if isinstance(error, pydantic.ValidationError):
|
||||
text = '; '.join(
|
||||
f'{".".join(map(str, item["loc"]))}: {item["msg"]}'
|
||||
for item in error.errors(include_input=False, include_context=False, include_url=False)
|
||||
)
|
||||
else:
|
||||
text = str(error).strip() or type(error).__name__
|
||||
|
||||
replacements = []
|
||||
|
||||
def collect(original, masked):
|
||||
if isinstance(original, dict) and isinstance(masked, dict):
|
||||
for key, value in original.items():
|
||||
collect(value, masked.get(key))
|
||||
elif isinstance(original, (list, tuple)) and isinstance(masked, (list, tuple)):
|
||||
for value, replacement in zip(original, masked):
|
||||
collect(value, replacement)
|
||||
elif isinstance(original, str) and original and original != masked:
|
||||
for representation in {original, repr(original)[1:-1], json.dumps(original, ensure_ascii=False)[1:-1]}:
|
||||
replacements.append((representation, str(masked)))
|
||||
|
||||
collect(configuration, redact_secrets(configuration))
|
||||
for original, masked in sorted(replacements, key=lambda item: len(item[0]), reverse=True):
|
||||
text = text.replace(original, masked)
|
||||
return text[:2000]
|
||||
@@ -71,23 +71,26 @@ class KnowledgeService:
|
||||
|
||||
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
|
||||
retrieval_settings = kb_data.get('retrieval_settings', {})
|
||||
defer_initialization = kb_data.get('defer_initialization') is True
|
||||
|
||||
# Validate required fields based on plugin's creation_schema and retrieval_schema
|
||||
await self._validate_schema_required_fields(
|
||||
context,
|
||||
knowledge_engine_plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
if not defer_initialization:
|
||||
await self._validate_schema_required_fields(
|
||||
context,
|
||||
knowledge_engine_plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
|
||||
kb = await self.ap.rag_mgr.create_knowledge_base(
|
||||
context,
|
||||
name=kb_data.get('name', 'Untitled'),
|
||||
knowledge_engine_plugin_id=knowledge_engine_plugin_id,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=retrieval_settings,
|
||||
description=kb_data.get('description', ''),
|
||||
)
|
||||
create_kwargs = {
|
||||
'name': kb_data.get('name', 'Untitled'),
|
||||
'knowledge_engine_plugin_id': knowledge_engine_plugin_id,
|
||||
'creation_settings': creation_settings,
|
||||
'retrieval_settings': retrieval_settings,
|
||||
'description': kb_data.get('description', ''),
|
||||
}
|
||||
if defer_initialization:
|
||||
create_kwargs['initialize'] = False
|
||||
kb = await self.ap.rag_mgr.create_knowledge_base(context, **create_kwargs)
|
||||
return kb.uuid
|
||||
|
||||
async def _validate_schema_required_fields(
|
||||
@@ -205,10 +208,32 @@ class KnowledgeService:
|
||||
) -> None:
|
||||
"""更新知识库"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
current = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
|
||||
if current is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
# Filter to only mutable fields
|
||||
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
|
||||
|
||||
should_initialize = current.get('initialized', True) is False and kb_data.get('initialize_engine') is True
|
||||
if should_initialize:
|
||||
plugin_id = current.get('knowledge_engine_plugin_id')
|
||||
if not plugin_id:
|
||||
raise ValueError('knowledge_engine_plugin_id is required')
|
||||
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
|
||||
retrieval_settings = kb_data.get('retrieval_settings', {})
|
||||
await self._validate_schema_required_fields(
|
||||
context,
|
||||
plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
filtered_data = {
|
||||
'name': kb_data.get('name', current.get('name', 'Untitled')),
|
||||
'description': kb_data.get('description', current.get('description', '')),
|
||||
'creation_settings': creation_settings,
|
||||
'retrieval_settings': retrieval_settings,
|
||||
'initialized': True,
|
||||
}
|
||||
else:
|
||||
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
|
||||
|
||||
if not filtered_data:
|
||||
return
|
||||
@@ -224,8 +249,28 @@ class KnowledgeService:
|
||||
kb = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
|
||||
if kb is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
if kb.get('initialized', True) is False:
|
||||
return
|
||||
|
||||
await self.ap.rag_mgr.load_knowledge_base(context, kb)
|
||||
runtime_kb = await self.ap.rag_mgr.load_knowledge_base(context, kb)
|
||||
if should_initialize:
|
||||
try:
|
||||
await runtime_kb._on_kb_create(self._execution_context(context))
|
||||
except Exception:
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(context, kb_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(
|
||||
name=current.get('name', 'Untitled'),
|
||||
description=current.get('description', ''),
|
||||
creation_settings=current.get('creation_settings', {}),
|
||||
retrieval_settings=current.get('retrieval_settings', {}),
|
||||
initialized=False,
|
||||
)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
raise
|
||||
|
||||
async def _check_doc_capability(self, context: TenantContext, kb_uuid: str, operation: str) -> None:
|
||||
"""Check if the KB's Knowledge Engine supports document operations.
|
||||
|
||||
@@ -142,8 +142,6 @@ class MaintenanceService:
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
else:
|
||||
@@ -151,6 +149,11 @@ class MaintenanceService:
|
||||
roots = [('storage', scoped_storage_path)]
|
||||
|
||||
sections = await asyncio.to_thread(self._collect_sections, roots)
|
||||
runtime_processes = await self._runtime_storage_processes(
|
||||
context,
|
||||
core_sections=sections,
|
||||
include_instance_storage=is_oss_singleton,
|
||||
)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts(context)
|
||||
binary_storage = await self._binary_storage_stats(context)
|
||||
@@ -171,6 +174,12 @@ class MaintenanceService:
|
||||
'log_retention_days': log_retention_days,
|
||||
},
|
||||
'sections': sections,
|
||||
'processes': runtime_processes,
|
||||
'total_size_bytes': sum(
|
||||
int(process.get('size_bytes') or 0)
|
||||
for process in runtime_processes
|
||||
if process.get('status') == 'available'
|
||||
),
|
||||
'database': {
|
||||
'type': database_type,
|
||||
'monitoring_counts': monitoring_counts,
|
||||
@@ -183,6 +192,104 @@ class MaintenanceService:
|
||||
'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
|
||||
}
|
||||
|
||||
async def _runtime_storage_processes(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
core_sections: list[dict[str, Any]],
|
||||
include_instance_storage: bool,
|
||||
) -> list[dict[str, Any]]:
|
||||
core_process = {
|
||||
'key': 'langbot',
|
||||
'status': 'available',
|
||||
'source': 'local_process',
|
||||
'size_bytes': sum(int(item.get('size_bytes') or 0) for item in core_sections),
|
||||
'directories': [
|
||||
{
|
||||
**item,
|
||||
'kind': 'root',
|
||||
'error_count': 0,
|
||||
}
|
||||
for item in core_sections
|
||||
],
|
||||
}
|
||||
|
||||
plugin_task = self._plugin_runtime_storage_process(include_instance_storage)
|
||||
box_task = self._box_runtime_storage_process(context)
|
||||
plugin_process, box_process = await asyncio.gather(plugin_task, box_task)
|
||||
return [core_process, plugin_process, box_process]
|
||||
|
||||
async def _plugin_runtime_storage_process(self, include_instance_storage: bool) -> dict[str, Any]:
|
||||
if not include_instance_storage:
|
||||
return self._unavailable_process(
|
||||
'plugin_runtime',
|
||||
'not_applicable',
|
||||
'Instance-wide Plugin Runtime storage is hidden in multi-Workspace deployments',
|
||||
)
|
||||
|
||||
connector = getattr(self.ap, 'plugin_connector', None)
|
||||
if connector is None or not getattr(connector, 'is_enable_plugin', True):
|
||||
return self._unavailable_process('plugin_runtime', 'disabled', 'Plugin Runtime is disabled')
|
||||
runtime_handler = getattr(connector, 'handler', None)
|
||||
get_analysis = getattr(runtime_handler, 'get_storage_analysis', None)
|
||||
if not callable(get_analysis):
|
||||
return self._unavailable_process(
|
||||
'plugin_runtime',
|
||||
'unavailable',
|
||||
'Plugin Runtime does not support storage analysis',
|
||||
)
|
||||
try:
|
||||
result = await get_analysis()
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'Failed to collect Plugin Runtime storage analysis: {exc}')
|
||||
return self._unavailable_process('plugin_runtime', 'unavailable', str(exc))
|
||||
return self._available_process('plugin_runtime', result, source='runtime_rpc')
|
||||
|
||||
async def _box_runtime_storage_process(self, context: TenantContext) -> dict[str, Any]:
|
||||
box_service = getattr(self.ap, 'box_service', None)
|
||||
if box_service is None or not getattr(box_service, 'enabled', True):
|
||||
return self._unavailable_process('box_runtime', 'disabled', 'Box Runtime is disabled')
|
||||
get_analysis = getattr(box_service, 'get_storage_analysis', None)
|
||||
if not callable(get_analysis):
|
||||
return self._unavailable_process(
|
||||
'box_runtime',
|
||||
'unavailable',
|
||||
'Box Runtime does not support storage analysis',
|
||||
)
|
||||
try:
|
||||
result = await get_analysis(context)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'Failed to collect Box Runtime storage analysis: {exc}')
|
||||
return self._unavailable_process('box_runtime', 'unavailable', str(exc))
|
||||
return self._available_process('box_runtime', result, source='runtime_rpc')
|
||||
|
||||
@staticmethod
|
||||
def _available_process(key: str, result: Any, *, source: str) -> dict[str, Any]:
|
||||
payload = result if isinstance(result, dict) else {}
|
||||
directories = payload.get('directories')
|
||||
if not isinstance(directories, list):
|
||||
directories = []
|
||||
return {
|
||||
'key': key,
|
||||
'status': 'available',
|
||||
'source': source,
|
||||
'size_bytes': int(payload.get('size_bytes') or 0),
|
||||
'directories': directories,
|
||||
'active_sessions': payload.get('active_sessions'),
|
||||
'managed_processes': payload.get('managed_processes'),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _unavailable_process(key: str, status: str, error: str) -> dict[str, Any]:
|
||||
return {
|
||||
'key': key,
|
||||
'status': status,
|
||||
'source': 'runtime_rpc',
|
||||
'size_bytes': None,
|
||||
'directories': [],
|
||||
'error': error,
|
||||
}
|
||||
|
||||
def _collect_sections(
|
||||
self,
|
||||
roots: list[tuple[str, Path | None]],
|
||||
|
||||
@@ -3,11 +3,17 @@ from __future__ import annotations
|
||||
import uuid
|
||||
import json
|
||||
import sqlalchemy
|
||||
import typing
|
||||
|
||||
from ....core import app
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....pipeline.extension_preferences import (
|
||||
normalize_extension_preferences,
|
||||
validate_extension_preferences,
|
||||
)
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .secrets import redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
@@ -16,7 +22,6 @@ default_stage_order = [
|
||||
'BanSessionCheckStage', # 封禁会话检查
|
||||
'PreContentFilterStage', # 内容过滤前置阶段
|
||||
'PreProcessor', # 预处理器
|
||||
'ConversationMessageTruncator', # 会话消息截断器
|
||||
'RequireRateLimitOccupancy', # 请求速率限制占用
|
||||
'MessageProcessor', # 处理器
|
||||
'ReleaseRateLimitOccupancy', # 释放速率限制占用
|
||||
@@ -33,12 +38,68 @@ class PipelineService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
@staticmethod
|
||||
def _get_default_values_from_schema(
|
||||
config_schema: list[dict[str, typing.Any]],
|
||||
) -> dict[str, typing.Any]:
|
||||
return {item['name']: item['default'] for item in config_schema if item.get('name') and 'default' in item}
|
||||
|
||||
async def get_default_pipeline_config(self, context: TenantContext) -> dict[str, typing.Any]:
|
||||
from ....utils import paths as path_utils
|
||||
|
||||
template_path = path_utils.get_resource_path('templates/default-pipeline-config.json')
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
config = json.load(f)
|
||||
|
||||
registry = getattr(self.ap, 'runner_registry', None)
|
||||
if registry is None:
|
||||
return config
|
||||
try:
|
||||
runners = await registry.list_runners(context, bound_plugins=None)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'Failed to load Runner defaults for pipeline config: {exc}')
|
||||
return config
|
||||
if not runners:
|
||||
return config
|
||||
|
||||
selected = runners[0]
|
||||
ai_config = config.setdefault('ai', {})
|
||||
runner_config = ai_config.setdefault('runner', {})
|
||||
runner_config['id'] = selected.id
|
||||
runner_config.setdefault('expire-time', 0)
|
||||
ai_config['runner_config'] = {selected.id: self._get_default_values_from_schema(selected.config_schema)}
|
||||
return config
|
||||
|
||||
async def get_pipeline_metadata(self, context: TenantContext) -> list[dict]:
|
||||
require_workspace_uuid(context)
|
||||
import copy
|
||||
|
||||
ai_metadata = copy.deepcopy(self.ap.pipeline_config_meta_ai)
|
||||
runner_stage = next(
|
||||
(stage for stage in ai_metadata.get('stages', []) if stage.get('name') == 'runner'),
|
||||
None,
|
||||
)
|
||||
if runner_stage:
|
||||
for config_item in runner_stage.get('config', []):
|
||||
if config_item.get('name') != 'id':
|
||||
continue
|
||||
try:
|
||||
runner_options, runner_stages = await self.ap.runner_registry.get_runner_metadata_for_pipeline(
|
||||
context
|
||||
)
|
||||
config_item['options'] = runner_options
|
||||
if runner_options and 'default' not in config_item:
|
||||
config_item['default'] = runner_options[0]['name']
|
||||
existing = {stage.get('name') for stage in ai_metadata.get('stages', [])}
|
||||
ai_metadata.setdefault('stages', []).extend(
|
||||
stage for stage in runner_stages if stage.get('name') not in existing
|
||||
)
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'Failed to load Runner pipeline metadata: {exc}')
|
||||
return [
|
||||
self.ap.pipeline_config_meta_trigger,
|
||||
self.ap.pipeline_config_meta_safety,
|
||||
self.ap.pipeline_config_meta_ai,
|
||||
ai_metadata,
|
||||
self.ap.pipeline_config_meta_output,
|
||||
]
|
||||
|
||||
@@ -101,9 +162,11 @@ class PipelineService:
|
||||
return serialized if include_secret else redact_secrets(serialized)
|
||||
|
||||
async def create_pipeline(self, context: TenantContext, pipeline_data: dict, default: bool = False) -> str:
|
||||
from ....utils import paths as path_utils
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if 'extensions_preferences' in pipeline_data:
|
||||
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
|
||||
if 'config' in pipeline_data:
|
||||
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
@@ -119,9 +182,8 @@ class PipelineService:
|
||||
pipeline_data['stages'] = default_stage_order.copy()
|
||||
pipeline_data['is_default'] = default
|
||||
|
||||
template_path = path_utils.get_resource_path('templates/default-pipeline-config.json')
|
||||
with open(template_path, 'r', encoding='utf-8') as f:
|
||||
pipeline_data['config'] = json.load(f)
|
||||
pipeline_data['config'] = await self.get_default_pipeline_config(context)
|
||||
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
|
||||
|
||||
# Ensure extensions_preferences is set with enable_all_plugins and enable_all_mcp_servers=True by default
|
||||
if 'extensions_preferences' not in pipeline_data:
|
||||
@@ -151,16 +213,31 @@ class PipelineService:
|
||||
pipeline_data.pop(protected_field, None)
|
||||
|
||||
if 'config' in pipeline_data:
|
||||
current_config = None
|
||||
if contains_secret_placeholder(pipeline_data['config']):
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
old_ai = current_config.get('ai', {}) if isinstance(current_config, dict) else {}
|
||||
old_runner = old_ai.get('runner') if isinstance(old_ai, dict) else None
|
||||
legacy_binding = (
|
||||
isinstance(old_runner, str)
|
||||
and bool(old_runner)
|
||||
or isinstance(old_runner, dict)
|
||||
and bool(old_runner.get('runner'))
|
||||
and not old_runner.get('id')
|
||||
)
|
||||
new_config = pipeline_data['config']
|
||||
new_ai = new_config.get('ai', {}) if isinstance(new_config, dict) else {}
|
||||
new_runner = new_ai.get('runner') if isinstance(new_ai, dict) else None
|
||||
if legacy_binding and isinstance(new_runner, dict) and new_runner.get('id'):
|
||||
raise ValueError('manual_migration_required')
|
||||
pipeline_data['config'] = restore_secret_placeholders(
|
||||
pipeline_data['config'],
|
||||
current_config if current_config is not None else {},
|
||||
)
|
||||
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
|
||||
if 'extensions_preferences' in pipeline_data:
|
||||
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
@@ -178,25 +255,6 @@ class PipelineService:
|
||||
if pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
if 'name' in pipeline_data:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(
|
||||
persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid
|
||||
),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
for bot in bots:
|
||||
bot_data = {'use_pipeline_name': pipeline_data['name']}
|
||||
await self.ap.bot_service.update_bot(context, bot.uuid, bot_data)
|
||||
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
@@ -260,18 +318,7 @@ class PipelineService:
|
||||
'stages': original_pipeline.stages.copy() if original_pipeline.stages else default_stage_order.copy(),
|
||||
'config': original_pipeline.config.copy() if original_pipeline.config else {},
|
||||
'is_default': False,
|
||||
'extensions_preferences': (
|
||||
original_pipeline.extensions_preferences.copy()
|
||||
if original_pipeline.extensions_preferences
|
||||
else {
|
||||
'enable_all_plugins': True,
|
||||
'enable_all_mcp_servers': True,
|
||||
'plugins': [],
|
||||
'mcp_servers': [],
|
||||
'mcp_resources': [],
|
||||
'mcp_resource_agent_read_enabled': True,
|
||||
}
|
||||
),
|
||||
'extensions_preferences': normalize_extension_preferences(original_pipeline.extensions_preferences),
|
||||
}
|
||||
|
||||
# Insert the new pipeline
|
||||
@@ -300,6 +347,30 @@ class PipelineService:
|
||||
) -> None:
|
||||
"""Update the bound plugins and MCP servers for a pipeline"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
extension_updates: dict[str, typing.Any] = {
|
||||
'enable_all_plugins': enable_all_plugins,
|
||||
'enable_all_mcp_servers': enable_all_mcp_servers,
|
||||
'enable_all_skills': enable_all_skills,
|
||||
'plugins': bound_plugins,
|
||||
}
|
||||
if bound_mcp_servers is not None:
|
||||
extension_updates['mcp_servers'] = bound_mcp_servers
|
||||
if bound_skills is not None:
|
||||
extension_updates['skills'] = bound_skills
|
||||
if bound_mcp_resources is not None:
|
||||
extension_updates['mcp_resources'] = bound_mcp_resources
|
||||
if mcp_resource_agent_read_enabled is not None:
|
||||
extension_updates['mcp_resource_agent_read_enabled'] = mcp_resource_agent_read_enabled
|
||||
self._validate_extension_preferences(
|
||||
extension_updates,
|
||||
context='Pipeline extension',
|
||||
field_aliases={
|
||||
'plugins': 'bound_plugins',
|
||||
'mcp_servers': 'bound_mcp_servers',
|
||||
'skills': 'bound_skills',
|
||||
'mcp_resources': 'bound_mcp_resources',
|
||||
},
|
||||
)
|
||||
# Get current pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
@@ -316,7 +387,7 @@ class PipelineService:
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Update extensions_preferences
|
||||
extensions_preferences = pipeline.extensions_preferences or {}
|
||||
extensions_preferences = normalize_extension_preferences(pipeline.extensions_preferences)
|
||||
extensions_preferences['enable_all_plugins'] = enable_all_plugins
|
||||
extensions_preferences['enable_all_mcp_servers'] = enable_all_mcp_servers
|
||||
extensions_preferences['enable_all_skills'] = enable_all_skills
|
||||
@@ -344,3 +415,22 @@ class PipelineService:
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
@staticmethod
|
||||
def _validate_extension_preferences(
|
||||
value: typing.Any,
|
||||
*,
|
||||
context: str = 'Pipeline extensions_preferences',
|
||||
field_aliases: typing.Mapping[str, str] | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
validated = validate_extension_preferences(
|
||||
value,
|
||||
context=context,
|
||||
field_aliases=field_aliases,
|
||||
)
|
||||
RunnerConfigResolver.validate_mcp_resource_attachments(
|
||||
validated.get('mcp_resources'),
|
||||
context=context,
|
||||
field_name='mcp_resources',
|
||||
)
|
||||
return validated
|
||||
|
||||
@@ -0,0 +1,936 @@
|
||||
"""Explicit, tenant-scoped migration. Preview never contacts plugin runtimes.
|
||||
|
||||
The task handle is process-local. Original snapshots and pending activation are
|
||||
persistent; a refreshed preview is required after observation loss or restart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
import secrets
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
import httpx
|
||||
import sqlalchemy as sa
|
||||
|
||||
from ..authz import Permission, permissions_for_role, require_permission
|
||||
from ..context import ExecutionContext, PrincipalType, RequestContext
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....agent.runner import config_schema
|
||||
from ....agent.runner.resource_builder import AgentResourceBuilder
|
||||
from ....agent.runner.resource_policy import ResourcePolicyProjector
|
||||
from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel
|
||||
from ....entity.persistence.rag import KnowledgeBase
|
||||
from ....entity.persistence.mcp import MCPServer
|
||||
from ....core.taskmgr import TaskContext
|
||||
from ....entity.persistence.pipeline import LegacyPipeline
|
||||
from ....entity.persistence.agent_interaction import AgentInteraction
|
||||
from ....persistence.pipeline_admission import lock_pipeline_admission
|
||||
from ....entity.persistence.pipeline_migration import PipelineMigrationSnapshot
|
||||
from ....entity.persistence.plugin import PluginSetting
|
||||
from ....entity.persistence.user import User
|
||||
from ....entity.persistence.workspace import Workspace, WorkspaceMembership, WorkspaceExecutionState
|
||||
from ....plugin.errors import (
|
||||
PluginRuntimeNotConnectedError,
|
||||
MarketplacePluginVersionNotFoundError,
|
||||
PluginInstallationFailedError,
|
||||
)
|
||||
from ....pipeline.legacy_config_migration import PLANNER_VERSION, plan_legacy_pipeline
|
||||
|
||||
|
||||
class MigrationInstallContext(TaskContext):
|
||||
"""Publish only structured installer progress, never upstream logs or URLs."""
|
||||
|
||||
ACTIONS = {
|
||||
'downloading plugin package': 'downloading',
|
||||
'validating plugin package': 'validating',
|
||||
'preparing plugin installation': 'preparing',
|
||||
'installing plugin dependencies': 'installing_deps',
|
||||
'waiting for plugin initialization': 'activating',
|
||||
'refreshing plugin components': 'refreshing',
|
||||
'plugin installed': 'done',
|
||||
}
|
||||
METRICS = ('progress_percent', 'download_current', 'download_total', 'download_speed', 'deps_total')
|
||||
|
||||
def __init__(self, target):
|
||||
super().__init__()
|
||||
self.progress = {
|
||||
**target,
|
||||
'status': 'installing',
|
||||
'stage': 'checking',
|
||||
'code': None,
|
||||
'started_at': time.time(),
|
||||
'finished_at': None,
|
||||
'steps': [],
|
||||
}
|
||||
self._step('checking')
|
||||
|
||||
def _step(self, stage):
|
||||
steps = self.progress['steps']
|
||||
if steps and steps[-1]['stage'] == stage:
|
||||
return
|
||||
now = time.time()
|
||||
if steps:
|
||||
steps[-1]['finished_at'] = now
|
||||
steps.append({'stage': stage, 'started_at': now, 'finished_at': None})
|
||||
self.progress['stage'] = stage
|
||||
|
||||
def set_current_action(self, action):
|
||||
super().set_current_action(action)
|
||||
stage = self.ACTIONS.get(action)
|
||||
if stage:
|
||||
self._step(stage)
|
||||
self.publish()
|
||||
|
||||
def publish(self):
|
||||
for key in self.METRICS:
|
||||
value = self.metadata.get(key)
|
||||
if type(value) in (int, float) and math.isfinite(value) and value >= 0:
|
||||
self.progress[key] = min(value, 100) if key == 'progress_percent' else value
|
||||
self.progress['updated_at'] = time.time()
|
||||
|
||||
def finish(self, code=None):
|
||||
self.publish()
|
||||
self.progress.update(status='failed' if code else 'completed', code=code, finished_at=time.time())
|
||||
self.progress['steps'][-1]['finished_at'] = self.progress['finished_at']
|
||||
if not code:
|
||||
self.progress['progress_percent'] = 100
|
||||
|
||||
async def observe(self):
|
||||
while True:
|
||||
self.publish()
|
||||
await asyncio.sleep(0.2)
|
||||
|
||||
|
||||
class MigrationError(ValueError):
|
||||
"""Only constant, credential-free codes cross the API/task boundary."""
|
||||
|
||||
def __init__(self, code: str, status_code: int = 409):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def _failure_code(exc: Exception) -> str:
|
||||
if isinstance(exc, MigrationError):
|
||||
return exc.code
|
||||
if isinstance(exc, PluginRuntimeNotConnectedError):
|
||||
return 'plugin_runtime_unavailable'
|
||||
return 'migration_failed'
|
||||
|
||||
|
||||
def validate_execute_request(body) -> list[dict]:
|
||||
if not isinstance(body, dict) or set(body) != {'confirmed', 'items'} or body['confirmed'] is not True:
|
||||
raise MigrationError('confirmation_required', 400)
|
||||
items = body['items']
|
||||
if not isinstance(items, list) or not 1 <= len(items) <= 50:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
seen = set()
|
||||
for item in items:
|
||||
if not isinstance(item, dict) or set(item) != {'pipeline_uuid', 'preview_token'}:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
for key, maximum in [('pipeline_uuid', 255), ('preview_token', 256)]:
|
||||
value = item[key]
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum or value != value.strip():
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
if item['pipeline_uuid'] in seen:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
seen.add(item['pipeline_uuid'])
|
||||
return copy.deepcopy(items)
|
||||
|
||||
|
||||
def _json(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _fingerprint(row):
|
||||
# SQL timestamps are neither monotonic revisions nor portable CAS values.
|
||||
return hashlib.sha256(
|
||||
_json(
|
||||
{k: v for k, v in row.items() if not k.startswith('_') and k not in ('created_at', 'updated_at')}
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
class PipelineMigrationService:
|
||||
def __init__(self, ap):
|
||||
self.ap = ap
|
||||
self.pm = ap.persistence_mgr
|
||||
# Loss of this process key only invalidates previews, never snapshots.
|
||||
self._token_key = secrets.token_bytes(32)
|
||||
self._all_tasks: set[str] = set()
|
||||
|
||||
async def _binding(self, ctx, session=None):
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
ctx.workspace_uuid, expected_generation=ctx.placement_generation, session=session
|
||||
)
|
||||
if binding.instance_uuid != ctx.instance_uuid:
|
||||
raise MigrationError('placement_changed')
|
||||
|
||||
async def _authorize(self, ctx, session=None):
|
||||
require_permission(ctx, Permission.RESOURCE_MANAGE)
|
||||
if ctx.principal.principal_type != PrincipalType.ACCOUNT or not ctx.account_uuid:
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
status = (await self.pm.execute_async(sa.select(User.status).where(User.uuid == ctx.account_uuid))).scalar()
|
||||
if status != 'active':
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
ctx.account_uuid, ctx.workspace_uuid, session=session
|
||||
)
|
||||
if (
|
||||
access.workspace.uuid != ctx.workspace_uuid
|
||||
or access.membership.uuid != ctx.workspace.membership_uuid
|
||||
or access.membership.projection_revision != ctx.workspace.membership_revision
|
||||
or Permission.RESOURCE_MANAGE.value not in permissions_for_role(access.membership.role)
|
||||
or access.execution.instance_uuid != ctx.instance_uuid
|
||||
or access.execution.placement_generation != ctx.placement_generation
|
||||
):
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
await self._binding(ctx, session)
|
||||
|
||||
async def _rows(self, ctx, pipeline_uuid=None):
|
||||
# Read DB-native text for JSON CAS: never compare PostgreSQL JSON using =,
|
||||
# and never assume Python's JSON whitespace/key order matches stored text.
|
||||
table = LegacyPipeline.__table__
|
||||
stmt = sa.select(
|
||||
table,
|
||||
*[sa.cast(table.c[k], sa.Text).label('_raw_' + k) for k in ('config', 'stages', 'extensions_preferences')],
|
||||
)
|
||||
stmt = stmt.where(table.c.workspace_uuid == ctx.workspace_uuid)
|
||||
if pipeline_uuid is not None:
|
||||
stmt = stmt.where(table.c.uuid == pipeline_uuid)
|
||||
return [dict(r._mapping) for r in (await self.pm.execute_async(stmt.order_by(table.c.uuid))).all()]
|
||||
|
||||
async def _plugin(self, ctx, plan):
|
||||
target = plan.get('target_plugin')
|
||||
if not target:
|
||||
return None
|
||||
t = PluginSetting.__table__
|
||||
row = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(
|
||||
t.c.enabled, t.c.installation_uuid, t.c.artifact_digest, t.c.runtime_revision, t.c.install_info
|
||||
).where(
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.plugin_author == target['author'],
|
||||
t.c.plugin_name == target['name'],
|
||||
)
|
||||
)
|
||||
).first()
|
||||
return dict(row._mapping) if row else None
|
||||
|
||||
def _native_interactions(self, ctx, pipeline_uuid):
|
||||
"""Inspect the old in-process cache without importing/pruning it.
|
||||
|
||||
Native forms have no durable handoff to the plugin runner. Operators
|
||||
must finish them in the original runtime before migrating; never infer
|
||||
completion from the cache TTL or submit/cancel a form here.
|
||||
"""
|
||||
module = sys.modules.get('langbot.pkg.provider.runners.difysvapi')
|
||||
if module is None:
|
||||
return []
|
||||
cache = getattr(module, '_PENDING_FORMS', None)
|
||||
if not isinstance(cache, dict):
|
||||
return ['unavailable']
|
||||
observed = []
|
||||
for key, forms in cache.items():
|
||||
if not isinstance(key, tuple) or len(key) != 8:
|
||||
if forms:
|
||||
observed.append('unscoped')
|
||||
continue
|
||||
instance, workspace, _, _, pipeline, *_ = key
|
||||
# Inspect ownership before touching any form contents. Empty scope
|
||||
# cannot prove this is another tenant; block without disclosing it.
|
||||
if instance and instance != ctx.instance_uuid or workspace and workspace != ctx.workspace_uuid:
|
||||
continue
|
||||
if pipeline and pipeline != pipeline_uuid:
|
||||
continue
|
||||
if forms:
|
||||
observed.append(hashlib.sha256(_json(key).encode()).hexdigest())
|
||||
return sorted(observed)
|
||||
|
||||
async def _interaction_state(self, ctx, pipeline_uuid):
|
||||
"""Read only correlation/status, never request/submission/token data.
|
||||
|
||||
The real table has workspace_id (not workspace_uuid), and no instance
|
||||
column. Execution placement is verified by the caller; processor_id is
|
||||
the globally unique pipeline UUID. NULL ownership on that exact UUID
|
||||
is uncertainty, not permission to query other Workspaces.
|
||||
"""
|
||||
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
|
||||
return {'boundary': 'unavailable'}, True
|
||||
t = AgentInteraction.__table__
|
||||
try:
|
||||
records = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(t.c.id, t.c.status, t.c.updated_at, t.c.workspace_id)
|
||||
.where(
|
||||
t.c.processor_type == 'pipeline',
|
||||
t.c.processor_id == pipeline_uuid,
|
||||
sa.or_(
|
||||
t.c.workspace_id == ctx.workspace_uuid, t.c.workspace_id.is_(None), t.c.workspace_id == ''
|
||||
),
|
||||
)
|
||||
.order_by(t.c.id)
|
||||
)
|
||||
).all()
|
||||
durable = [tuple(r) for r in records]
|
||||
blocked = any(r.status not in {'submitted', 'cancelled', 'expired', 'delivery_failed'} for r in records)
|
||||
except Exception:
|
||||
# Missing schema, permissions, or a failed store must not be treated
|
||||
# as an empty store. No original diagnostic crosses the API.
|
||||
durable, blocked = ['unavailable'], True
|
||||
native = self._native_interactions(ctx, pipeline_uuid)
|
||||
return {'durable': durable, 'native': native}, blocked or bool(native)
|
||||
|
||||
async def _plan(self, ctx, row):
|
||||
plan = plan_legacy_pipeline(row['config'], row['extensions_preferences'])
|
||||
pending = None
|
||||
if plan['state'] == 'already_current':
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
snapshots = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(t).where(
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.pipeline_uuid == row['uuid'],
|
||||
t.c.state == 'activation_pending',
|
||||
t.c.target_fingerprint == _fingerprint(row),
|
||||
t.c.planner_version == str(PLANNER_VERSION),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
if snapshots:
|
||||
pending = dict(snapshots[0]._mapping)
|
||||
source = pending['source_snapshot']
|
||||
plan = plan_legacy_pipeline(source['config'], source['extensions_preferences'])
|
||||
candidate = {**row, 'config': plan.get('config')}
|
||||
if plan['state'] != 'ready' or _fingerprint(candidate) != _fingerprint(row):
|
||||
raise MigrationError('planner_changed')
|
||||
facts = await self._plugin(ctx, plan)
|
||||
state = 'activation_pending' if pending else plan['state']
|
||||
blockers = list(plan['blockers'])
|
||||
if state in ('ready', 'activation_pending'):
|
||||
if facts is None:
|
||||
state = 'needs_plugin'
|
||||
blockers.append({'code': 'plugin_missing'})
|
||||
elif facts['enabled'] is not True:
|
||||
state = 'blocked'
|
||||
blockers.append({'code': 'plugin_disabled'})
|
||||
if plan['state'] == 'ready':
|
||||
observation, interaction_blocked = await self._interaction_state(ctx, row['uuid'])
|
||||
plan['_interaction_state'] = observation
|
||||
if interaction_blocked:
|
||||
state = 'blocked'
|
||||
blockers.append({'code': 'runtime.pending_interaction'})
|
||||
return plan, facts, pending, state, blockers
|
||||
|
||||
def _token(self, ctx, row, plan, facts, pending):
|
||||
claim = [
|
||||
ctx.instance_uuid,
|
||||
ctx.workspace_uuid,
|
||||
ctx.placement_generation,
|
||||
ctx.account_uuid,
|
||||
ctx.workspace.membership_uuid,
|
||||
ctx.workspace.membership_revision,
|
||||
str(PLANNER_VERSION),
|
||||
_fingerprint(row),
|
||||
plan,
|
||||
facts,
|
||||
pending['uuid'] if pending else None,
|
||||
]
|
||||
# An authenticated opaque digest discloses neither credentials nor configs.
|
||||
return hmac.new(self._token_key, _json(claim).encode(), hashlib.sha256).hexdigest()
|
||||
|
||||
async def preview(self, ctx: RequestContext):
|
||||
require_permission(ctx, Permission.RESOURCE_VIEW)
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._binding(ctx)
|
||||
items = []
|
||||
for row in await self._rows(ctx):
|
||||
plan, facts, pending, state, blockers = await self._plan(ctx, row)
|
||||
item = {
|
||||
k: copy.deepcopy(plan[k])
|
||||
for k in ('legacy_runner', 'target_runner_id', 'target_plugin', 'changed_paths', 'warnings')
|
||||
}
|
||||
item.update(
|
||||
pipeline_uuid=row['uuid'],
|
||||
name=row['name'],
|
||||
state=state,
|
||||
blockers=blockers,
|
||||
preview_token=self._token(ctx, row, plan, facts, pending)
|
||||
if state in ('ready', 'activation_pending')
|
||||
else None,
|
||||
)
|
||||
items.append(item)
|
||||
return {
|
||||
'planner_version': str(PLANNER_VERSION),
|
||||
'workspace_uuid': ctx.workspace_uuid,
|
||||
'items': items,
|
||||
'total': len(items),
|
||||
}
|
||||
|
||||
async def _selected(self, ctx, selection, *, data_only=False):
|
||||
rows = await self._rows(ctx, selection['pipeline_uuid'])
|
||||
if not rows:
|
||||
raise MigrationError('pipeline_not_found', 404)
|
||||
row = rows[0]
|
||||
plan, facts, pending, state, blockers = await self._plan(ctx, row)
|
||||
token = self._token(ctx, row, plan, facts, pending)
|
||||
if not hmac.compare_digest(token, selection['preview_token']):
|
||||
raise MigrationError('preview_stale')
|
||||
if state not in ('ready', 'activation_pending') and not (
|
||||
data_only
|
||||
and plan['state'] == 'ready'
|
||||
and all(b['code'] in ('plugin_missing', 'plugin_disabled') for b in blockers)
|
||||
):
|
||||
raise MigrationError('migration_blocked')
|
||||
return row, plan, facts, pending
|
||||
|
||||
async def execute(self, ctx: RequestContext, body):
|
||||
require_permission(ctx, Permission.RESOURCE_MANAGE)
|
||||
if isinstance(body, dict) and body.get('all') is True:
|
||||
if (
|
||||
set(body) != {'confirmed', 'all', 'install_plugins'}
|
||||
or body['confirmed'] is not True
|
||||
or not isinstance(body['install_plugins'], bool)
|
||||
):
|
||||
raise MigrationError('confirmation_required', 400)
|
||||
return await self._execute_all(ctx, install_plugins=body['install_plugins'])
|
||||
items = validate_execute_request(body)
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._authorize(ctx)
|
||||
# Preflight the entire selection before creating tasks or contacting runtimes.
|
||||
for selection in items:
|
||||
await self._selected(ctx, selection)
|
||||
task_context = TaskContext.new()
|
||||
task_context.metadata = {
|
||||
'kind': 'pipeline_migration',
|
||||
'results': [{'pipeline_uuid': i['pipeline_uuid'], 'state': 'pending', 'code': None} for i in items],
|
||||
}
|
||||
task = self.ap.task_mgr.create_user_task(
|
||||
self._run(ctx, ExecutionContext.from_request(ctx), items, task_context),
|
||||
kind='pipeline_migration',
|
||||
name='pipeline_migration',
|
||||
context=task_context,
|
||||
instance_uuid=ctx.instance_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
placement_generation=ctx.placement_generation,
|
||||
)
|
||||
return {'task_id': task.id}
|
||||
|
||||
async def _execute_all(self, ctx, *, install_plugins):
|
||||
# Capture all sources on admission. Plugin installation must not silently
|
||||
# include later edits or pipelines created while the task is running.
|
||||
if ctx.workspace_uuid in self._all_tasks:
|
||||
raise MigrationError('migration_running')
|
||||
self._all_tasks.add(ctx.workspace_uuid)
|
||||
try:
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._authorize(ctx)
|
||||
sources = []
|
||||
for row in await self._rows(ctx):
|
||||
_, _, _, state, _ = await self._plan(ctx, row)
|
||||
if state not in ('already_current', 'not_legacy'):
|
||||
sources.append(row)
|
||||
if not sources:
|
||||
raise MigrationError('nothing_to_migrate')
|
||||
task_context = TaskContext.new()
|
||||
task_context.metadata = {
|
||||
'kind': 'pipeline_migration',
|
||||
'phase': 'installing' if install_plugins else 'migrating',
|
||||
'installations': [],
|
||||
'results': [{'pipeline_uuid': row['uuid'], 'state': 'pending', 'code': None} for row in sources],
|
||||
}
|
||||
task = self.ap.task_mgr.create_user_task(
|
||||
self._run_all(
|
||||
ctx, ExecutionContext.from_request(ctx), sources, task_context, install_plugins=install_plugins
|
||||
),
|
||||
kind='pipeline_migration',
|
||||
name='pipeline_migration',
|
||||
context=task_context,
|
||||
instance_uuid=ctx.instance_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
placement_generation=ctx.placement_generation,
|
||||
)
|
||||
return {'task_id': task.id, 'pipeline_uuids': [row['uuid'] for row in sources]}
|
||||
except BaseException:
|
||||
self._all_tasks.discard(ctx.workspace_uuid)
|
||||
raise
|
||||
|
||||
async def _run_all(self, ctx, execution, sources, task_context, *, install_plugins):
|
||||
installed = {}
|
||||
try:
|
||||
async with self.pm.tenant_scope(execution.workspace_uuid):
|
||||
for source, result in zip(sources, task_context.metadata['results']):
|
||||
if result['state'] != 'pending':
|
||||
continue
|
||||
try:
|
||||
await self._authorize(ctx)
|
||||
current = await self._rows(ctx, source['uuid'])
|
||||
if not current or _fingerprint(current[0]) != _fingerprint(source):
|
||||
raise MigrationError('preview_stale')
|
||||
plan, facts, pending, state, blockers = await self._plan(ctx, current[0])
|
||||
hard_blockers = [b for b in blockers if b['code'] not in ('plugin_missing', 'plugin_disabled')]
|
||||
if hard_blockers:
|
||||
raise MigrationError(hard_blockers[0]['code'])
|
||||
if state in ('already_current', 'not_legacy'):
|
||||
result.update(state='already_current', code=None)
|
||||
continue
|
||||
target = plan.get('target_plugin')
|
||||
if not target:
|
||||
raise MigrationError('migration_blocked')
|
||||
if not install_plugins:
|
||||
selection = {
|
||||
'pipeline_uuid': source['uuid'],
|
||||
'preview_token': self._token(ctx, current[0], plan, facts, pending),
|
||||
}
|
||||
await self._run(ctx, execution, [selection], task_context, results=[result], data_only=True)
|
||||
continue
|
||||
key = (target['author'], target['name'], target['version'])
|
||||
if key not in installed:
|
||||
runners = await self.ap.runner_registry.list_runners(
|
||||
execution, use_cache=False, usage='agent'
|
||||
)
|
||||
descriptor = next((r for r in runners if r.id == plan['target_runner_id']), None)
|
||||
ready = (
|
||||
facts
|
||||
and facts['enabled']
|
||||
and descriptor
|
||||
and descriptor.plugin_version == target['version']
|
||||
)
|
||||
if not ready:
|
||||
task_context.metadata['phase'] = 'installing'
|
||||
install_context = MigrationInstallContext(target)
|
||||
task_context.metadata['installations'].append(install_context.progress)
|
||||
observer = asyncio.create_task(install_context.observe())
|
||||
install_code = 'operation_cancelled'
|
||||
try:
|
||||
await self._authorize(ctx)
|
||||
await self.ap.plugin_connector.require_workspace_context(execution)
|
||||
await self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE,
|
||||
{
|
||||
'plugin_author': target['author'],
|
||||
'plugin_name': target['name'],
|
||||
'plugin_version': target['version'],
|
||||
},
|
||||
task_context=install_context,
|
||||
)
|
||||
install_code = None
|
||||
except PluginRuntimeNotConnectedError:
|
||||
install_code = 'plugin_runtime_unavailable'
|
||||
except MarketplacePluginVersionNotFoundError:
|
||||
install_code = 'plugin_version_unavailable'
|
||||
except httpx.TimeoutException:
|
||||
install_code = 'plugin_download_timeout'
|
||||
except httpx.HTTPStatusError as exc:
|
||||
install_code = (
|
||||
'plugin_version_unavailable'
|
||||
if exc.response.status_code == 404
|
||||
else 'plugin_marketplace_unavailable'
|
||||
)
|
||||
except httpx.RequestError:
|
||||
install_code = 'plugin_download_failed'
|
||||
except PluginInstallationFailedError as exc:
|
||||
install_code = {
|
||||
'dependency_prepare_failed': 'dependency_prepare_failed',
|
||||
'worker_launch_failed': 'plugin_launch_failed',
|
||||
}.get(exc.error_code, 'plugin_install_failed')
|
||||
except Exception:
|
||||
install_code = 'plugin_install_failed'
|
||||
finally:
|
||||
observer.cancel()
|
||||
try:
|
||||
await observer
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
install_context.finish(install_code)
|
||||
installed[key] = install_code
|
||||
else:
|
||||
installed[key] = None
|
||||
if installed[key]:
|
||||
raise MigrationError(installed[key])
|
||||
await self._authorize(ctx)
|
||||
current = await self._rows(ctx, source['uuid'])
|
||||
if not current or _fingerprint(current[0]) != _fingerprint(source):
|
||||
raise MigrationError('preview_stale')
|
||||
plan, facts, pending, state, blockers = await self._plan(ctx, current[0])
|
||||
if state not in ('ready', 'activation_pending'):
|
||||
raise MigrationError(blockers[0]['code'] if blockers else 'migration_blocked')
|
||||
selection = {
|
||||
'pipeline_uuid': source['uuid'],
|
||||
'preview_token': self._token(ctx, current[0], plan, facts, pending),
|
||||
}
|
||||
task_context.metadata['phase'] = 'migrating'
|
||||
await self._run(ctx, execution, [selection], task_context, results=[result])
|
||||
except Exception as exc:
|
||||
result.update(
|
||||
state='blocked' if isinstance(exc, MigrationError) else 'failed',
|
||||
code=_failure_code(exc),
|
||||
)
|
||||
if any(
|
||||
r['code'] in ('operation_cancelled', 'commit_outcome_unknown')
|
||||
for r in task_context.metadata['results']
|
||||
):
|
||||
break
|
||||
finally:
|
||||
for result in task_context.metadata['results']:
|
||||
if result['state'] == 'pending':
|
||||
result.update(state='failed', code='operation_cancelled')
|
||||
task_context.metadata['phase'] = 'finished'
|
||||
self._all_tasks.discard(ctx.workspace_uuid)
|
||||
return task_context.metadata
|
||||
|
||||
async def _verify_runtime(self, execution, plan):
|
||||
runners = await self.ap.runner_registry.list_runners(execution, use_cache=False, usage='agent')
|
||||
descriptor = next((r for r in runners if r.id == plan['target_runner_id']), None)
|
||||
if descriptor is None or 'agent' not in descriptor.usages:
|
||||
raise MigrationError('runner_unavailable')
|
||||
if descriptor.plugin_version != plan['target_plugin']['version']:
|
||||
raise MigrationError('plugin_version_incompatible')
|
||||
config = plan['config']
|
||||
RunnerConfigResolver.validate_pipeline_config(config)
|
||||
runner_config = config['ai']['runner_config'][plan['target_runner_id']]
|
||||
schema = {item['name']: item for item in descriptor.config_schema}
|
||||
# Host fields have their own scope/capability checks below; plugin
|
||||
# schema defaults must never implicitly authorize a foreign resource.
|
||||
host_fields = {
|
||||
'enable-all-tools': 'boolean',
|
||||
'tools': 'array',
|
||||
'knowledge-bases': 'knowledge-base-multi-selector',
|
||||
'mcp-resources': 'array',
|
||||
'mcp-resource-agent-read-enabled': 'boolean',
|
||||
}
|
||||
for name, value in runner_config.items():
|
||||
field = schema.get(name)
|
||||
if field is None and name in host_fields:
|
||||
field = {'type': host_fields[name]}
|
||||
if field is None:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if value is None and field.get('nullable') is True:
|
||||
continue
|
||||
kind = config_schema.normalize_schema_item_type(field.get('type'))
|
||||
valid = (
|
||||
(
|
||||
kind
|
||||
in (
|
||||
'string',
|
||||
'text',
|
||||
'password',
|
||||
'secret',
|
||||
'select',
|
||||
'llm-model-selector',
|
||||
'embedding-model-selector',
|
||||
'rerank-model-selector',
|
||||
)
|
||||
and isinstance(value, str)
|
||||
)
|
||||
or (kind == 'boolean' and type(value) is bool)
|
||||
or (kind in ('integer', 'int') and type(value) is int)
|
||||
or (kind in ('float', 'number') and type(value) in (int, float))
|
||||
or (
|
||||
kind
|
||||
in (
|
||||
'array',
|
||||
'multi-select',
|
||||
'knowledge-base-selector',
|
||||
'knowledge-base-multi-selector',
|
||||
'prompt-editor',
|
||||
)
|
||||
and isinstance(value, list)
|
||||
)
|
||||
or (kind in ('object', 'json') and type(value) is dict)
|
||||
or (kind == 'array[string]' and type(value) is list and all(type(v) is str for v in value))
|
||||
or (
|
||||
kind == 'model-fallback-selector'
|
||||
and (
|
||||
isinstance(value, str)
|
||||
or isinstance(value, dict)
|
||||
and set(value) <= {'primary', 'fallbacks', 'reasoning'}
|
||||
and isinstance(value.get('primary'), str)
|
||||
and isinstance(value.get('fallbacks', []), list)
|
||||
and all(isinstance(v, str) for v in value.get('fallbacks', []))
|
||||
and isinstance(value.get('reasoning', {}), dict)
|
||||
)
|
||||
)
|
||||
)
|
||||
if not valid:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if kind == 'select' and field.get('options') and field.get('allow_custom') is not True:
|
||||
if value not in [o.get('value', o.get('name')) for o in field['options']]:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if any(
|
||||
f.get('required') and name not in runner_config and f.get('default') is None for name, f in schema.items()
|
||||
):
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
await self._verify_resources(execution, descriptor, runner_config)
|
||||
return _json(
|
||||
[
|
||||
descriptor.id,
|
||||
descriptor.plugin_version,
|
||||
descriptor.usages,
|
||||
descriptor.config_schema,
|
||||
getattr(descriptor, 'capabilities', None),
|
||||
getattr(descriptor, 'permissions', None),
|
||||
]
|
||||
)
|
||||
|
||||
async def _verify_resources(self, execution, descriptor, runner_config):
|
||||
"""Fail closed on references the scoped Host cannot authorize.
|
||||
|
||||
This runs only on explicit execution, outside the mutation transaction.
|
||||
Resource use is still reauthorized by the Host on every eventual run.
|
||||
"""
|
||||
|
||||
async def require_row(model, identifier, *, column=None):
|
||||
column = model.uuid if column is None else column
|
||||
found = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(model.uuid).where(
|
||||
model.workspace_uuid == execution.workspace_uuid,
|
||||
column == identifier,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if found is None:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
permissions = getattr(descriptor, 'permissions', None)
|
||||
for model_type, model_uuid in config_schema.iter_config_model_refs(descriptor, runner_config):
|
||||
allowed = set(getattr(permissions, 'models', []))
|
||||
if not (allowed & ({'rerank'} if model_type == 'rerank' else {'invoke', 'stream'})):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
await require_row(RerankModel if model_type == 'rerank' else LLMModel, model_uuid)
|
||||
for field in config_schema.iter_schema_items(descriptor, {'embedding-model-selector'}):
|
||||
value = runner_config.get(field['name'])
|
||||
if value and value not in config_schema.NONE_SENTINELS:
|
||||
await require_row(EmbeddingModel, value)
|
||||
|
||||
kb_ids = runner_config.get('knowledge-bases', [])
|
||||
if not isinstance(kb_ids, list) or any(not isinstance(v, str) or not v for v in kb_ids):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
if kb_ids:
|
||||
if not config_schema.uses_host_knowledge_bases(descriptor) or 'retrieve' not in getattr(
|
||||
permissions, 'knowledge_bases', []
|
||||
):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
for kb_id in kb_ids:
|
||||
await require_row(KnowledgeBase, kb_id)
|
||||
|
||||
tool_names = runner_config.get('tools', [])
|
||||
if not isinstance(tool_names, list) or any(not isinstance(v, str) or not v for v in tool_names):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
tool_grant = (
|
||||
runner_config.get('enable-all-tools', False)
|
||||
or tool_names
|
||||
or runner_config.get('mcp-resource-agent-read-enabled', False)
|
||||
)
|
||||
if tool_grant:
|
||||
if not config_schema.uses_host_tools(descriptor) or 'call' not in getattr(permissions, 'tools', []):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
policy = ResourcePolicyProjector.from_runner_config(runner_config)
|
||||
tools = await AgentResourceBuilder(self.ap)._build_tools_from_binding(
|
||||
execution,
|
||||
permissions,
|
||||
policy,
|
||||
descriptor,
|
||||
runner_config,
|
||||
)
|
||||
if not set(tool_names) <= {tool['tool_name'] for tool in tools}:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
for attachment in runner_config.get('mcp-resources', []):
|
||||
if not isinstance(attachment, dict):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
identifier = attachment.get('server_uuid') or attachment.get('server_id')
|
||||
name = attachment.get('server_name')
|
||||
if identifier:
|
||||
await require_row(MCPServer, identifier)
|
||||
elif name:
|
||||
await require_row(MCPServer, name, column=MCPServer.name)
|
||||
else:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
async def _lock(self, ctx, pipeline_uuid, session):
|
||||
# Every pending producer shares this row and revalidates its captured
|
||||
# configuration/conversation authority after acquiring admission.
|
||||
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
await lock_pipeline_admission(session, ctx.workspace_uuid, pipeline_uuid)
|
||||
# Hold revocable directory rows through commit (no network in this UoW).
|
||||
await session.execute(sa.select(User.uuid).where(User.uuid == ctx.account_uuid).with_for_update())
|
||||
for model, condition in [
|
||||
(Workspace, Workspace.uuid == ctx.workspace_uuid),
|
||||
(WorkspaceExecutionState, WorkspaceExecutionState.workspace_uuid == ctx.workspace_uuid),
|
||||
(
|
||||
WorkspaceMembership,
|
||||
sa.and_(
|
||||
WorkspaceMembership.workspace_uuid == ctx.workspace_uuid,
|
||||
WorkspaceMembership.account_uuid == ctx.account_uuid,
|
||||
),
|
||||
),
|
||||
(PluginSetting, PluginSetting.workspace_uuid == ctx.workspace_uuid),
|
||||
]:
|
||||
await session.execute(sa.select(model.__table__).where(condition).with_for_update())
|
||||
|
||||
async def _commit(self, ctx, selection, expected_facts, candidate, *, data_only=False):
|
||||
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
|
||||
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
|
||||
await self._authorize(ctx, uow.session)
|
||||
row, plan, facts, pending = await self._selected(ctx, selection, data_only=data_only)
|
||||
if facts != expected_facts:
|
||||
raise MigrationError('plugin_changed')
|
||||
if pending:
|
||||
return pending['uuid'], pending['target_fingerprint']
|
||||
target = {**row, 'config': candidate['config']}
|
||||
t = LegacyPipeline.__table__
|
||||
cas = [t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == row['uuid']]
|
||||
for key in ('config', 'stages', 'extensions_preferences'):
|
||||
cas.append(sa.cast(t.c[key], sa.Text) == row['_raw_' + key])
|
||||
result = await self.pm.execute_async(sa.update(t).where(*cas).values(config=candidate['config']))
|
||||
if result.rowcount != 1:
|
||||
raise MigrationError('preview_stale')
|
||||
snapshot_uuid = str(uuid.uuid4())
|
||||
target_fingerprint = _fingerprint(target)
|
||||
await self.pm.execute_async(
|
||||
sa.insert(PipelineMigrationSnapshot).values(
|
||||
uuid=snapshot_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
pipeline_uuid=row['uuid'],
|
||||
source_fingerprint=_fingerprint(row),
|
||||
target_fingerprint=target_fingerprint,
|
||||
source_snapshot=json.loads(_json({k: v for k, v in row.items() if not k.startswith('_')})),
|
||||
planner_version=str(PLANNER_VERSION),
|
||||
state='activation_pending',
|
||||
)
|
||||
)
|
||||
if self._native_interactions(ctx, row['uuid']):
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
return snapshot_uuid, target_fingerprint
|
||||
|
||||
async def _activate(self, ctx, selection, snapshot_uuid, target_fingerprint, candidate, plan, expected_facts):
|
||||
# Re-read the committed target under a short lock before synchronous
|
||||
# publication. A later editor must never be replaced by our old candidate.
|
||||
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
|
||||
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
|
||||
await self._authorize(ctx, uow.session)
|
||||
rows = await self._rows(ctx, selection['pipeline_uuid'])
|
||||
if not rows or _fingerprint(rows[0]) != target_fingerprint:
|
||||
raise MigrationError('activation_source_changed')
|
||||
if await self._plugin(ctx, plan) != expected_facts:
|
||||
raise MigrationError('plugin_changed')
|
||||
_, interaction_blocked = await self._interaction_state(ctx, selection['pipeline_uuid'])
|
||||
if interaction_blocked:
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
# Native cache inspection above is the last await before publication
|
||||
# and session invalidation; no native producer can interleave here.
|
||||
self.ap.pipeline_mgr.publish_pipeline(candidate)
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
conversation = session.using_conversation
|
||||
if (
|
||||
conversation is not None
|
||||
and conversation.pipeline_uuid == selection['pipeline_uuid']
|
||||
and getattr(session, 'workspace_uuid', None) == ctx.workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
await self.pm.execute_async(
|
||||
sa.update(t)
|
||||
.where(t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == snapshot_uuid)
|
||||
.values(state='active')
|
||||
)
|
||||
|
||||
async def _run(self, ctx, execution, items, task_context, results=None, *, data_only=False):
|
||||
async with self.pm.tenant_scope(execution.workspace_uuid):
|
||||
for selection, result in zip(items, results if results is not None else task_context.metadata['results']):
|
||||
committed = False
|
||||
commit_attempted = False
|
||||
try:
|
||||
await self._authorize(ctx)
|
||||
row, plan, facts, pending = await self._selected(ctx, selection, data_only=data_only)
|
||||
runtime_schema = None if data_only else await self._verify_runtime(execution, plan)
|
||||
RunnerConfigResolver.validate_pipeline_config(plan['config'])
|
||||
candidate_entity = {k: copy.deepcopy(v) for k, v in row.items() if not k.startswith('_')}
|
||||
candidate_entity['config'] = copy.deepcopy(plan['config'])
|
||||
runtime = await self.ap.pipeline_mgr.prepare_pipeline(execution, copy.deepcopy(candidate_entity))
|
||||
if not data_only and await self._verify_runtime(execution, plan) != runtime_schema:
|
||||
raise MigrationError('runner_schema_changed')
|
||||
# Runtime awaits are over. Recheck authorization/source/plugin
|
||||
# facts under database locks, then atomically journal and CAS.
|
||||
commit_attempted = True
|
||||
snapshot_uuid, target_fingerprint = await self._commit(
|
||||
ctx, selection, facts, candidate_entity, **({'data_only': True} if data_only else {})
|
||||
)
|
||||
committed = True
|
||||
await self._activate(ctx, selection, snapshot_uuid, target_fingerprint, runtime, plan, facts)
|
||||
result.update(state='migrated', code='data_only' if data_only else None)
|
||||
except (Exception, asyncio.CancelledError) as exc:
|
||||
cancelled = isinstance(exc, asyncio.CancelledError)
|
||||
reconciliation_cancel = None
|
||||
code = 'operation_cancelled' if cancelled else _failure_code(exc)
|
||||
if commit_attempted and not committed and not isinstance(exc, MigrationError):
|
||||
# A commit can succeed while acknowledgement/connection
|
||||
# cleanup fails, including cancellation. Make one read
|
||||
# after the UoW has unwound, in this task's tenant scope;
|
||||
# never shield/retry through another cancellation.
|
||||
try:
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
conditions = [
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.pipeline_uuid == selection['pipeline_uuid'],
|
||||
t.c.source_fingerprint
|
||||
== (pending['source_fingerprint'] if pending else _fingerprint(row)),
|
||||
t.c.target_fingerprint == _fingerprint(candidate_entity),
|
||||
t.c.planner_version == str(PLANNER_VERSION),
|
||||
]
|
||||
if pending:
|
||||
# A retry starts from the target, not the legacy
|
||||
# source. Only the original snapshot proves this
|
||||
# durable migration; an unrelated journal cannot.
|
||||
conditions.extend(
|
||||
[
|
||||
t.c.uuid == pending['uuid'],
|
||||
t.c.target_fingerprint == pending['target_fingerprint'],
|
||||
]
|
||||
)
|
||||
found = (await self.pm.execute_async(sa.select(t.c.uuid).where(*conditions))).first()
|
||||
committed = found is not None
|
||||
except asyncio.CancelledError as cancel_exc:
|
||||
reconciliation_cancel = cancel_exc
|
||||
cancelled = True
|
||||
code = 'commit_outcome_unknown'
|
||||
except Exception:
|
||||
code = 'commit_outcome_unknown'
|
||||
state = (
|
||||
# Unknown is deliberately recovery-needed, not a claim
|
||||
# of rollback or proof that a snapshot was committed.
|
||||
'activation_pending'
|
||||
if committed or code == 'commit_outcome_unknown'
|
||||
else 'stale'
|
||||
if code == 'preview_stale'
|
||||
else 'blocked'
|
||||
if isinstance(exc, MigrationError)
|
||||
else 'failed'
|
||||
)
|
||||
result.update(state=state, code=code)
|
||||
if cancelled:
|
||||
for unstarted in task_context.metadata['results']:
|
||||
if unstarted['state'] == 'pending':
|
||||
unstarted.update(state='failed', code='operation_cancelled')
|
||||
if reconciliation_cancel is not None:
|
||||
# Preserve cancellation propagation during recovery,
|
||||
# but record safe outcomes before leaving the task.
|
||||
raise reconciliation_cancel
|
||||
return task_context.metadata
|
||||
# Never attach the original exception, traceback or plugin
|
||||
# diagnostic to TaskContext: upstream errors can contain keys.
|
||||
return task_context.metadata
|
||||
@@ -11,6 +11,7 @@ from typing import Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
|
||||
import httpx
|
||||
from langbot_plugin.entities.io.errors import ActionCallTimeoutError
|
||||
|
||||
from ....core import app
|
||||
from ....skill.utils import parse_frontmatter
|
||||
@@ -119,7 +120,14 @@ class SkillService:
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return []
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills(execution_context)]
|
||||
try:
|
||||
skills = await box_service.list_skills(execution_context)
|
||||
except ActionCallTimeoutError as exc:
|
||||
logger = getattr(self.ap, 'logger', None)
|
||||
if logger is not None:
|
||||
logger.warning(f'Box skill listing timed out: {exc}')
|
||||
return []
|
||||
return [self._serialize_skill(skill) for skill in skills]
|
||||
|
||||
async def get_skill(self, context: TenantContext, skill_name: str) -> Optional[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
|
||||
@@ -245,13 +245,14 @@ class SpaceService:
|
||||
models_data = data.get('data', {}).get('models', [])
|
||||
return [SpaceModel.model_validate(model_dict) for model_dict in models_data]
|
||||
|
||||
async def get_model_selection(self, category: str) -> typing.List[SpaceModelSelection]:
|
||||
async def get_model_selection(self, category: str | None = None) -> typing.List[SpaceModelSelection]:
|
||||
"""Return Space models in the availability-ranked selection order."""
|
||||
space_url = self._get_space_config()['url']
|
||||
session = httpclient.get_session()
|
||||
params = {'category': category} if category else None
|
||||
async with session.get(
|
||||
f'{space_url}/api/v1/models/selection',
|
||||
params={'category': category},
|
||||
params=params,
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
error = await httpclient.read_text_limited(response)
|
||||
@@ -269,7 +270,17 @@ class SpaceService:
|
||||
models = []
|
||||
for selection in data:
|
||||
if isinstance(selection, dict) and isinstance(selection.get('model'), dict):
|
||||
models.append(selection['model'])
|
||||
model = dict(selection['model'])
|
||||
availability = selection.get('availability')
|
||||
if not isinstance(availability, dict):
|
||||
# Accept the short-lived pre-release response shape.
|
||||
availability = {
|
||||
key: selection[key]
|
||||
for key in ('up', 'last_probed_at', 'latency_ms', 'http_code')
|
||||
if key in selection
|
||||
}
|
||||
model['availability'] = availability
|
||||
models.append(model)
|
||||
else:
|
||||
models.append(selection)
|
||||
return [SpaceModelSelection.model_validate(model) for model in models]
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Content-free management boundaries. Diagnostics never participate in I/O.
|
||||
|
||||
Only source-code identities and status categories enter these helpers. Never
|
||||
pass a URL, request, frame, token, tool argument, or serialized result to them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import functools
|
||||
import re
|
||||
|
||||
import quart
|
||||
from quart.wrappers.response import IterableBody, ResponseBody
|
||||
|
||||
from ..telemetry import diagnostics as d
|
||||
from ..telemetry import diagnostic_privacy as privacy
|
||||
|
||||
_ACTIVE_BOUNDARY = contextvars.ContextVar('management_diagnostic_boundary', default=None)
|
||||
|
||||
|
||||
def operation_id(source, fn, *, rule='', methods=()):
|
||||
"""Called at registration with a Core function, never a client tool name."""
|
||||
module = fn.__module__.split('.groups.', 1)[-1]
|
||||
if source != 'http':
|
||||
module = ''
|
||||
name = fn.__name__
|
||||
if source == 'http' and name == '_':
|
||||
# Many Core routes use the anonymous function name `_`. Disambiguate
|
||||
# using ONLY the source-declared template and methods at registration.
|
||||
# This is never quart.request.path, url_rule, endpoint or request.method.
|
||||
template = re.sub(r'[^A-Za-z0-9_.:-]+', '.', rule).strip('.') or 'root'
|
||||
name = '.'.join((*sorted(methods or ('GET',)), template))
|
||||
return '.'.join(part for part in (source, module, name) if part)[:128]
|
||||
|
||||
|
||||
def outcome(value, reason_code='response_error'):
|
||||
try:
|
||||
d.set_outcome(value, reason_code=reason_code)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def workspace(context):
|
||||
"""Annotate only this boundary's owned span with a matching trusted context."""
|
||||
try:
|
||||
from .http.context import ExecutionContext, RequestContext
|
||||
|
||||
if not isinstance(context, (ExecutionContext, RequestContext)):
|
||||
return
|
||||
boundary = _ACTIVE_BOUNDARY.get()
|
||||
span = d.current_span()
|
||||
if boundary is None or span is None or span is not boundary.span:
|
||||
return
|
||||
if d._manager(boundary.ap) is not span.manager:
|
||||
return
|
||||
instance = getattr(
|
||||
getattr(boundary.ap, 'workspace_service', None),
|
||||
'instance_uuid',
|
||||
getattr(span.manager, 'instance_id', None),
|
||||
)
|
||||
# Structural recorders need not expose instance metadata. Real managers
|
||||
# do, and must never receive another instance's Workspace annotation.
|
||||
if instance is not None and instance != context.instance_uuid:
|
||||
return
|
||||
d.annotate(workspace_uuid=context.workspace_uuid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class Boundary:
|
||||
def __init__(self, ap, operation, source):
|
||||
self.ap = ap
|
||||
self.span = None
|
||||
try:
|
||||
manager = d._manager(ap)
|
||||
if manager is not None:
|
||||
fields = {'source': source, 'stage': 'execute'}
|
||||
if source == 'webui_debug':
|
||||
fields['attributes'] = {'synthetic': True}
|
||||
self.span = d.Span(manager, 'api', operation, fields)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@contextlib.contextmanager
|
||||
def activate(self):
|
||||
# None is an explicit inheritance barrier, not a no-op: disabled or
|
||||
# broken B work must not borrow enabled A's span through ContextVars.
|
||||
token = d._CURRENT.set(self.span)
|
||||
boundary_token = _ACTIVE_BOUNDARY.set(self)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_ACTIVE_BOUNDARY.reset(boundary_token)
|
||||
d._CURRENT.reset(token)
|
||||
|
||||
def finish(self, error=None):
|
||||
if self.span is not None:
|
||||
try:
|
||||
self.span.finish(error)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class _ObservedBody(ResponseBody):
|
||||
"""Carry the HTTP parent through iteration, not generator suspension."""
|
||||
|
||||
def __init__(self, body, boundary):
|
||||
self.body = body
|
||||
self.boundary = boundary
|
||||
self.iterator = None
|
||||
self.exhausted = False
|
||||
|
||||
async def __aenter__(self):
|
||||
try:
|
||||
with self.boundary.activate():
|
||||
entered = await self.body.__aenter__()
|
||||
self.iterator = entered.__aiter__()
|
||||
return self
|
||||
except BaseException as exc:
|
||||
self.boundary.finish(exc)
|
||||
raise
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, tb):
|
||||
try:
|
||||
with self.boundary.activate():
|
||||
result = await self.body.__aexit__(exc_type, exc_value, tb)
|
||||
except BaseException as exc:
|
||||
self.boundary.finish(exc)
|
||||
raise
|
||||
finally:
|
||||
self.boundary.finish(exc_value if exc_value is not None else (None if self.exhausted else GeneratorExit()))
|
||||
return result
|
||||
|
||||
async def __aiter__(self):
|
||||
try:
|
||||
if self.iterator is None:
|
||||
with self.boundary.activate():
|
||||
self.iterator = self.body.__aiter__()
|
||||
while True:
|
||||
try:
|
||||
with self.boundary.activate():
|
||||
value = await anext(self.iterator)
|
||||
except StopAsyncIteration:
|
||||
self.exhausted = True
|
||||
self.boundary.finish()
|
||||
return
|
||||
yield value
|
||||
except BaseException as exc:
|
||||
self.boundary.finish(exc)
|
||||
raise
|
||||
|
||||
|
||||
def _response_status(value):
|
||||
"""Inspect response metadata only; never read or deserialize a body."""
|
||||
response = value[0] if isinstance(value, tuple) else value
|
||||
status = response.status_code if isinstance(response, quart.Response) else 200
|
||||
if isinstance(value, tuple) and len(value) > 1 and type(value[1]) is int:
|
||||
status = value[1]
|
||||
if status in (401, 403):
|
||||
outcome('rejected')
|
||||
elif status >= 400:
|
||||
outcome('failed')
|
||||
elif isinstance(response, dict) and 'code' in response and response['code'] != 0:
|
||||
outcome('failed')
|
||||
return response
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def scope(ap, operation, *, source):
|
||||
"""A fixed operation around an existing non-generator statement block."""
|
||||
try:
|
||||
privacy.code_value('operation', operation)
|
||||
except Exception:
|
||||
pass
|
||||
boundary = Boundary(ap, operation, source)
|
||||
try:
|
||||
with boundary.activate():
|
||||
yield
|
||||
except BaseException as exc:
|
||||
boundary.finish(exc)
|
||||
raise
|
||||
finally:
|
||||
boundary.finish()
|
||||
|
||||
|
||||
def observe(operation, *, source, ap=None, http=False):
|
||||
# The operation is supplied by Core registration code, not request data.
|
||||
try:
|
||||
privacy.code_value('operation', operation)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def decorator(fn):
|
||||
@functools.wraps(fn)
|
||||
async def wrapped(*args, **kwargs):
|
||||
try:
|
||||
owner = (ap() if callable(ap) else ap) if ap is not None else getattr(args[0], 'ap', None)
|
||||
except Exception:
|
||||
owner = None
|
||||
boundary = Boundary(owner, operation, source)
|
||||
streaming = False
|
||||
try:
|
||||
with boundary.activate():
|
||||
value = await fn(*args, **kwargs)
|
||||
if http:
|
||||
try:
|
||||
response = _response_status(value)
|
||||
if isinstance(response, quart.Response) and isinstance(response.response, IterableBody):
|
||||
response.response = _ObservedBody(response.response, boundary)
|
||||
streaming = True
|
||||
except Exception:
|
||||
pass
|
||||
return value
|
||||
except BaseException as exc:
|
||||
boundary.finish(exc)
|
||||
raise
|
||||
finally:
|
||||
if not streaming:
|
||||
boundary.finish()
|
||||
|
||||
return wrapped
|
||||
|
||||
return decorator
|
||||
@@ -24,6 +24,7 @@ import uuid
|
||||
from ..http.context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from .context import bind_request_context, reset_request_context
|
||||
from .server import LangBotMCPServer
|
||||
from .. import management_diagnostics as diagnostics
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ...core import app as app_module
|
||||
@@ -85,13 +86,8 @@ class MCPMount:
|
||||
authenticate_api_key = self.ap.apikey_service.authenticate_api_key
|
||||
is_mcp_path = self._is_mcp_path
|
||||
|
||||
async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def]
|
||||
# Pass through non-HTTP scopes (lifespan, websocket) to Quart so its
|
||||
# own startup/shutdown and websocket routes keep working.
|
||||
if scope['type'] != 'http' or not is_mcp_path(scope.get('path', '')):
|
||||
await quart_asgi(scope, receive, send)
|
||||
return
|
||||
|
||||
@diagnostics.observe('mcp.request', source='mcp', ap=self.ap)
|
||||
async def dispatch_mcp(scope, receive, send):
|
||||
# Authenticate MCP HTTP requests with a LangBot API key.
|
||||
api_key = _extract_api_key(scope.get('headers', []))
|
||||
identity = None
|
||||
@@ -100,6 +96,7 @@ class MCPMount:
|
||||
identity = await authenticate_api_key(api_key)
|
||||
|
||||
if identity is None:
|
||||
diagnostics.outcome('rejected')
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
@@ -126,6 +123,7 @@ class MCPMount:
|
||||
entitlement = await resolver.resolve(identity.workspace_uuid)
|
||||
entitlement_revision = entitlement.entitlement_revision
|
||||
except Exception:
|
||||
diagnostics.outcome('rejected')
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
@@ -153,6 +151,7 @@ class MCPMount:
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
diagnostics.workspace(request_context)
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('MCP request persistence scope is unavailable')
|
||||
@@ -165,4 +164,12 @@ class MCPMount:
|
||||
finally:
|
||||
reset_request_context(token)
|
||||
|
||||
async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def]
|
||||
# Non-MCP traffic keeps its existing Quart boundaries. In
|
||||
# particular, never derive an operation from an ASGI request path.
|
||||
if scope['type'] != 'http' or not is_mcp_path(scope.get('path', '')):
|
||||
await quart_asgi(scope, receive, send)
|
||||
return
|
||||
await dispatch_mcp(scope, receive, send)
|
||||
|
||||
return dispatcher
|
||||
|
||||
@@ -24,6 +24,7 @@ from mcp.server.fastmcp import FastMCP
|
||||
|
||||
from ..http.authz import Permission, require_permission
|
||||
from .context import get_request_context
|
||||
from .. import management_diagnostics as diagnostics
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ...core import app as app_module
|
||||
@@ -31,7 +32,7 @@ if typing.TYPE_CHECKING:
|
||||
|
||||
INSTRUCTIONS = """\
|
||||
This MCP server manages a LangBot instance. LangBot is an LLM-native instant
|
||||
messaging bot platform. Use these tools to inspect and manage bots, pipelines,
|
||||
messaging bot platform. Use these tools to inspect and manage bots, agents, pipelines,
|
||||
models, knowledge bases, MCP servers, and skills.
|
||||
|
||||
Authentication uses a LangBot API key (web-UI-created `lbk_...` key or the
|
||||
@@ -52,6 +53,7 @@ def _dump(value: typing.Any) -> str:
|
||||
def _authorized(permission: Permission):
|
||||
context = get_request_context()
|
||||
require_permission(context, permission)
|
||||
diagnostics.workspace(context)
|
||||
return context
|
||||
|
||||
|
||||
@@ -76,7 +78,7 @@ class LangBotMCPServer:
|
||||
# ------------------------------------------------------------------ #
|
||||
def _register_tools(self) -> None:
|
||||
ap = self.ap
|
||||
mcp = self.mcp
|
||||
mcp = self
|
||||
|
||||
# ----- System (read-only) -------------------------------------- #
|
||||
@mcp.tool(description='Get basic LangBot system/runtime information (version, edition).')
|
||||
@@ -109,14 +111,22 @@ class LangBotMCPServer:
|
||||
description=(
|
||||
'Create a bot. `bot_data` is a JSON object matching the LangBot '
|
||||
'POST /api/v1/platform/bots body (e.g. name, adapter, config). '
|
||||
'Returns the new bot UUID.'
|
||||
'Use event_bindings for exclusive Agent/Pipeline routes; plugin_processors is an array '
|
||||
'of {processor_uuid, enabled} subscriptions that automatically match Runner-declared events. '
|
||||
'Subscriptions run independently of each other and of event_bindings. Returns the new bot UUID.'
|
||||
)
|
||||
)
|
||||
async def create_bot(bot_data: dict) -> str:
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
return _dump({'uuid': await ap.bot_service.create_bot(context, bot_data)})
|
||||
|
||||
@mcp.tool(description='Update a bot by UUID. `bot_data` matches the PUT bot body.')
|
||||
@mcp.tool(
|
||||
description=(
|
||||
'Update a bot by UUID. `bot_data` matches the PUT bot body. '
|
||||
'plugin_processors replaces the bot subscriptions with [{processor_uuid, enabled}]. '
|
||||
'Do not put event_processor targets in event_bindings; those are exclusive Agent/Pipeline routes.'
|
||||
)
|
||||
)
|
||||
async def update_bot(bot_uuid: str, bot_data: dict) -> str:
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.bot_service.update_bot(context, bot_uuid, bot_data)
|
||||
@@ -128,6 +138,10 @@ class LangBotMCPServer:
|
||||
await ap.bot_service.delete_bot(context, bot_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(description='List recent runtime route status for a bot event route table.')
|
||||
async def list_bot_event_route_statuses(bot_uuid: str) -> str:
|
||||
return _dump(await ap.bot_service.list_event_route_statuses(bot_uuid))
|
||||
|
||||
# ----- Pipelines ----------------------------------------------- #
|
||||
@mcp.tool(description='List all pipelines.')
|
||||
async def list_pipelines() -> str:
|
||||
@@ -170,7 +184,89 @@ class LangBotMCPServer:
|
||||
await ap.pipeline_service.delete_pipeline(context, pipeline_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
# ----- Processors ---------------------------------------------- #
|
||||
@mcp.tool(description='List product-level processors, including Agents, Pipelines and Event processors.')
|
||||
async def list_processors() -> str:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.agent_service.get_agents(context))
|
||||
|
||||
@mcp.tool(description='Get an Agent, Pipeline or Event processor by UUID.')
|
||||
async def get_processor(processor_uuid: str) -> str:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.agent_service.get_agent(context, processor_uuid))
|
||||
|
||||
@mcp.tool(
|
||||
description=(
|
||||
'Create an Agent, Pipeline or Event processor. Set `processor_data.kind` to '
|
||||
'`agent`, `pipeline` or `event_processor`. Event processors may be created without a component; '
|
||||
'then use update_processor with an installed component_ref from get_processor_metadata and optional '
|
||||
'parameters. Unconfigured instances support no events. Returns UUID and kind.'
|
||||
)
|
||||
)
|
||||
async def create_processor(processor_data: dict) -> str:
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
return _dump(await ap.agent_service.create_agent(context, processor_data))
|
||||
|
||||
@mcp.tool(description='Update an Agent, Pipeline or Event processor by UUID.')
|
||||
async def update_processor(processor_uuid: str, processor_data: dict) -> str:
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.agent_service.update_agent(context, processor_uuid, processor_data)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(description='Delete an Agent, Pipeline or Event processor by UUID.')
|
||||
async def delete_processor(processor_uuid: str) -> str:
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.agent_service.delete_agent(context, processor_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(
|
||||
description='Get processor kinds and installed event-capable Runner components with configuration schemas.'
|
||||
)
|
||||
async def get_processor_metadata() -> str:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.agent_service.get_agent_metadata(context))
|
||||
|
||||
@mcp.tool(
|
||||
description='List one Agent or plugin processor run history; use before_id to page older runs. '
|
||||
'created_at_ms, started_at_ms and finished_at_ms are Host lifecycle times in epoch milliseconds.'
|
||||
)
|
||||
async def list_processor_runs(processor_uuid: str, before_id: int | None = None) -> str:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.agent_service.get_processor_runs(context, processor_uuid, before_id=before_id))
|
||||
|
||||
@mcp.tool(
|
||||
description='Read logs and action results for an Agent or plugin processor run; page using after_sequence.'
|
||||
)
|
||||
async def get_processor_run_events(
|
||||
processor_uuid: str,
|
||||
run_id: str,
|
||||
after_sequence: int | None = None,
|
||||
) -> str:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(
|
||||
await ap.agent_service.get_processor_run_events(
|
||||
context,
|
||||
processor_uuid,
|
||||
run_id,
|
||||
after_sequence=after_sequence,
|
||||
)
|
||||
)
|
||||
|
||||
# ----- Models -------------------------------------------------- #
|
||||
@mcp.tool(
|
||||
description=(
|
||||
'Run a synthetic event against an Agent or Event processor without platform delivery. '
|
||||
'Returns final text and execution_events containing reported messages/thinking and tool calls. '
|
||||
'Platform tools use mock adapters; other tools execute normally. '
|
||||
'For Event processors, data contains the complete typed event fields. '
|
||||
'Requires runtime.operate; payload accepts event_type, text, data, conversation_id, actor, subject and '
|
||||
'mock (errors/results keyed by platform tool name; unsupported_apis lists unavailable platform APIs).'
|
||||
)
|
||||
)
|
||||
async def debug_agent(processor_uuid: str, payload: dict) -> str:
|
||||
context = _authorized(Permission.RUNTIME_OPERATE)
|
||||
return _dump(await ap.agent_service.debug_agent(context, processor_uuid, payload))
|
||||
|
||||
@mcp.tool(description='List all configured LLM models. Secrets are redacted.')
|
||||
async def list_llm_models() -> str:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
@@ -230,6 +326,15 @@ class LangBotMCPServer:
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.skill_service.get_skill(context, skill_name))
|
||||
|
||||
def tool(self, **options):
|
||||
"""Register a tool boundary before its authorization and service call."""
|
||||
|
||||
def register(fn):
|
||||
observed = diagnostics.observe(diagnostics.operation_id('mcp', fn), source='mcp', ap=self.ap)(fn)
|
||||
return self.mcp.tool(**options)(observed)
|
||||
|
||||
return register
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# ASGI app
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
Reference in New Issue
Block a user