mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-21 07:55:18 +08:00
feat: add verified manual pipeline migration for plugin runners
This commit is contained in:
RockChinQ
@@ -0,0 +1,33 @@
|
||||
"""User-confirmed migration only; deliberately not API-key/MCP accessible."""
|
||||
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ....service.pipeline_migration import PipelineMigrationService, MigrationError
|
||||
|
||||
|
||||
@group.group_class('pipeline_migration', '/api/v1/pipelines/_/migration')
|
||||
class PipelineMigrationRouterGroup(group.RouterGroup):
|
||||
async def initialize(self):
|
||||
self.service = PipelineMigrationService(self.ap)
|
||||
|
||||
@self.route(
|
||||
'/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_VIEW
|
||||
)
|
||||
async def preview(request_context: RequestContext):
|
||||
try:
|
||||
return self.success(data=await self.service.preview(request_context))
|
||||
except MigrationError as exc:
|
||||
return self.http_status(exc.status_code, -1, exc.code)
|
||||
|
||||
@self.route(
|
||||
'/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_MANAGE
|
||||
)
|
||||
async def execute(request_context: RequestContext):
|
||||
try:
|
||||
body = await quart.request.get_json(silent=True)
|
||||
return self.success(data=await self.service.execute(request_context, body))
|
||||
except MigrationError as exc:
|
||||
return self.http_status(exc.status_code, -1, exc.code)
|
||||
@@ -13,7 +13,7 @@ from ....pipeline.extension_preferences import (
|
||||
validate_extension_preferences,
|
||||
)
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .secrets import redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
@@ -213,12 +213,24 @@ class PipelineService:
|
||||
pipeline_data.pop(protected_field, None)
|
||||
|
||||
if 'config' in pipeline_data:
|
||||
current_config = None
|
||||
if contains_secret_placeholder(pipeline_data['config']):
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
old_ai = current_config.get('ai', {}) if isinstance(current_config, dict) else {}
|
||||
old_runner = old_ai.get('runner') if isinstance(old_ai, dict) else None
|
||||
legacy_binding = (
|
||||
isinstance(old_runner, str)
|
||||
and bool(old_runner)
|
||||
or isinstance(old_runner, dict)
|
||||
and bool(old_runner.get('runner'))
|
||||
and not old_runner.get('id')
|
||||
)
|
||||
new_config = pipeline_data['config']
|
||||
new_ai = new_config.get('ai', {}) if isinstance(new_config, dict) else {}
|
||||
new_runner = new_ai.get('runner') if isinstance(new_ai, dict) else None
|
||||
if legacy_binding and isinstance(new_runner, dict) and new_runner.get('id'):
|
||||
raise ValueError('manual_migration_required')
|
||||
pipeline_data['config'] = restore_secret_placeholders(
|
||||
pipeline_data['config'],
|
||||
current_config if current_config is not None else {},
|
||||
|
||||
@@ -0,0 +1,685 @@
|
||||
"""Explicit, tenant-scoped migration. Preview never contacts plugin runtimes.
|
||||
|
||||
The task handle is process-local. Original snapshots and pending activation are
|
||||
persistent; a refreshed preview is required after observation loss or restart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from ..authz import Permission, permissions_for_role, require_permission
|
||||
from ..context import ExecutionContext, PrincipalType, RequestContext
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....agent.runner import config_schema
|
||||
from ....agent.runner.resource_builder import AgentResourceBuilder
|
||||
from ....agent.runner.resource_policy import ResourcePolicyProjector
|
||||
from ....agent.runner.model_reasoning import extract_model_reasoning_overrides
|
||||
from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel
|
||||
from ....entity.persistence.rag import KnowledgeBase
|
||||
from ....entity.persistence.mcp import MCPServer
|
||||
from ....core.taskmgr import TaskContext
|
||||
from ....entity.persistence.pipeline import LegacyPipeline
|
||||
from ....entity.persistence.agent_interaction import AgentInteraction
|
||||
from ....persistence.pipeline_admission import lock_pipeline_admission
|
||||
from ....entity.persistence.pipeline_migration import PipelineMigrationSnapshot
|
||||
from ....entity.persistence.plugin import PluginSetting
|
||||
from ....entity.persistence.user import User
|
||||
from ....entity.persistence.workspace import Workspace, WorkspaceMembership, WorkspaceExecutionState
|
||||
from ....pipeline.legacy_config_migration import PLANNER_VERSION, plan_legacy_pipeline
|
||||
|
||||
|
||||
class MigrationError(ValueError):
|
||||
"""Only constant, credential-free codes cross the API/task boundary."""
|
||||
|
||||
def __init__(self, code: str, status_code: int = 409):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def validate_execute_request(body) -> list[dict]:
|
||||
if not isinstance(body, dict) or set(body) != {'confirmed', 'items'} or body['confirmed'] is not True:
|
||||
raise MigrationError('confirmation_required', 400)
|
||||
items = body['items']
|
||||
if not isinstance(items, list) or not 1 <= len(items) <= 50:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
seen = set()
|
||||
for item in items:
|
||||
if not isinstance(item, dict) or set(item) != {'pipeline_uuid', 'preview_token'}:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
for key, maximum in [('pipeline_uuid', 255), ('preview_token', 256)]:
|
||||
value = item[key]
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum or value != value.strip():
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
if item['pipeline_uuid'] in seen:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
seen.add(item['pipeline_uuid'])
|
||||
return copy.deepcopy(items)
|
||||
|
||||
|
||||
def _json(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _fingerprint(row):
|
||||
# SQL timestamps are neither monotonic revisions nor portable CAS values.
|
||||
return hashlib.sha256(
|
||||
_json(
|
||||
{k: v for k, v in row.items() if not k.startswith('_') and k not in ('created_at', 'updated_at')}
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
class PipelineMigrationService:
|
||||
def __init__(self, ap):
|
||||
self.ap = ap
|
||||
self.pm = ap.persistence_mgr
|
||||
# Loss of this process key only invalidates previews, never snapshots.
|
||||
self._token_key = secrets.token_bytes(32)
|
||||
|
||||
async def _binding(self, ctx, session=None):
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
ctx.workspace_uuid, expected_generation=ctx.placement_generation, session=session
|
||||
)
|
||||
if binding.instance_uuid != ctx.instance_uuid:
|
||||
raise MigrationError('placement_changed')
|
||||
|
||||
async def _authorize(self, ctx, session=None):
|
||||
require_permission(ctx, Permission.RESOURCE_MANAGE)
|
||||
if ctx.principal.principal_type != PrincipalType.ACCOUNT or not ctx.account_uuid:
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
status = (await self.pm.execute_async(sa.select(User.status).where(User.uuid == ctx.account_uuid))).scalar()
|
||||
if status != 'active':
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
ctx.account_uuid, ctx.workspace_uuid, session=session
|
||||
)
|
||||
if (
|
||||
access.workspace.uuid != ctx.workspace_uuid
|
||||
or access.membership.uuid != ctx.workspace.membership_uuid
|
||||
or access.membership.projection_revision != ctx.workspace.membership_revision
|
||||
or Permission.RESOURCE_MANAGE.value not in permissions_for_role(access.membership.role)
|
||||
or access.execution.instance_uuid != ctx.instance_uuid
|
||||
or access.execution.placement_generation != ctx.placement_generation
|
||||
):
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
await self._binding(ctx, session)
|
||||
|
||||
async def _rows(self, ctx, pipeline_uuid=None):
|
||||
# Read DB-native text for JSON CAS: never compare PostgreSQL JSON using =,
|
||||
# and never assume Python's JSON whitespace/key order matches stored text.
|
||||
table = LegacyPipeline.__table__
|
||||
stmt = sa.select(
|
||||
table,
|
||||
*[sa.cast(table.c[k], sa.Text).label('_raw_' + k) for k in ('config', 'stages', 'extensions_preferences')],
|
||||
)
|
||||
stmt = stmt.where(table.c.workspace_uuid == ctx.workspace_uuid)
|
||||
if pipeline_uuid is not None:
|
||||
stmt = stmt.where(table.c.uuid == pipeline_uuid)
|
||||
return [dict(r._mapping) for r in (await self.pm.execute_async(stmt.order_by(table.c.uuid))).all()]
|
||||
|
||||
async def _plugin(self, ctx, plan):
|
||||
target = plan.get('target_plugin')
|
||||
if not target:
|
||||
return None
|
||||
t = PluginSetting.__table__
|
||||
row = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(
|
||||
t.c.enabled, t.c.installation_uuid, t.c.artifact_digest, t.c.runtime_revision, t.c.install_info
|
||||
).where(
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.plugin_author == target['author'],
|
||||
t.c.plugin_name == target['name'],
|
||||
)
|
||||
)
|
||||
).first()
|
||||
return dict(row._mapping) if row else None
|
||||
|
||||
def _native_interactions(self, ctx, pipeline_uuid):
|
||||
"""Inspect the old in-process cache without importing/pruning it.
|
||||
|
||||
Native forms have no durable handoff to the plugin runner. Operators
|
||||
must finish them in the original runtime before migrating; never infer
|
||||
completion from the cache TTL or submit/cancel a form here.
|
||||
"""
|
||||
module = sys.modules.get('langbot.pkg.provider.runners.difysvapi')
|
||||
if module is None:
|
||||
return []
|
||||
cache = getattr(module, '_PENDING_FORMS', None)
|
||||
if not isinstance(cache, dict):
|
||||
return ['unavailable']
|
||||
observed = []
|
||||
for key, forms in cache.items():
|
||||
if not isinstance(key, tuple) or len(key) != 8:
|
||||
if forms:
|
||||
observed.append('unscoped')
|
||||
continue
|
||||
instance, workspace, _, _, pipeline, *_ = key
|
||||
# Inspect ownership before touching any form contents. Empty scope
|
||||
# cannot prove this is another tenant; block without disclosing it.
|
||||
if instance and instance != ctx.instance_uuid or workspace and workspace != ctx.workspace_uuid:
|
||||
continue
|
||||
if pipeline and pipeline != pipeline_uuid:
|
||||
continue
|
||||
if forms:
|
||||
observed.append(hashlib.sha256(_json(key).encode()).hexdigest())
|
||||
return sorted(observed)
|
||||
|
||||
async def _interaction_state(self, ctx, pipeline_uuid):
|
||||
"""Read only correlation/status, never request/submission/token data.
|
||||
|
||||
The real table has workspace_id (not workspace_uuid), and no instance
|
||||
column. Execution placement is verified by the caller; processor_id is
|
||||
the globally unique pipeline UUID. NULL ownership on that exact UUID
|
||||
is uncertainty, not permission to query other Workspaces.
|
||||
"""
|
||||
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
|
||||
return {'boundary': 'unavailable'}, True
|
||||
t = AgentInteraction.__table__
|
||||
try:
|
||||
records = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(t.c.id, t.c.status, t.c.updated_at, t.c.workspace_id)
|
||||
.where(
|
||||
t.c.processor_type == 'pipeline',
|
||||
t.c.processor_id == pipeline_uuid,
|
||||
sa.or_(
|
||||
t.c.workspace_id == ctx.workspace_uuid, t.c.workspace_id.is_(None), t.c.workspace_id == ''
|
||||
),
|
||||
)
|
||||
.order_by(t.c.id)
|
||||
)
|
||||
).all()
|
||||
durable = [tuple(r) for r in records]
|
||||
blocked = any(r.status not in {'submitted', 'cancelled', 'expired', 'delivery_failed'} for r in records)
|
||||
except Exception:
|
||||
# Missing schema, permissions, or a failed store must not be treated
|
||||
# as an empty store. No original diagnostic crosses the API.
|
||||
durable, blocked = ['unavailable'], True
|
||||
native = self._native_interactions(ctx, pipeline_uuid)
|
||||
return {'durable': durable, 'native': native}, blocked or bool(native)
|
||||
|
||||
async def _plan(self, ctx, row):
|
||||
plan = plan_legacy_pipeline(row['config'], row['extensions_preferences'])
|
||||
pending = None
|
||||
if plan['state'] == 'already_current':
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
snapshots = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(t).where(
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.pipeline_uuid == row['uuid'],
|
||||
t.c.state == 'activation_pending',
|
||||
t.c.target_fingerprint == _fingerprint(row),
|
||||
t.c.planner_version == str(PLANNER_VERSION),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
if snapshots:
|
||||
pending = dict(snapshots[0]._mapping)
|
||||
source = pending['source_snapshot']
|
||||
plan = plan_legacy_pipeline(source['config'], source['extensions_preferences'])
|
||||
candidate = {**row, 'config': plan.get('config')}
|
||||
if plan['state'] != 'ready' or _fingerprint(candidate) != _fingerprint(row):
|
||||
raise MigrationError('planner_changed')
|
||||
facts = await self._plugin(ctx, plan)
|
||||
state = 'activation_pending' if pending else plan['state']
|
||||
blockers = list(plan['blockers'])
|
||||
if state in ('ready', 'activation_pending'):
|
||||
if facts is None:
|
||||
state = 'needs_plugin'
|
||||
blockers.append({'code': 'plugin_missing'})
|
||||
elif facts['enabled'] is not True:
|
||||
state = 'blocked'
|
||||
blockers.append({'code': 'plugin_disabled'})
|
||||
if plan['state'] == 'ready':
|
||||
observation, interaction_blocked = await self._interaction_state(ctx, row['uuid'])
|
||||
plan['_interaction_state'] = observation
|
||||
if interaction_blocked:
|
||||
state = 'blocked'
|
||||
blockers.append({'code': 'runtime.pending_interaction'})
|
||||
return plan, facts, pending, state, blockers
|
||||
|
||||
def _token(self, ctx, row, plan, facts, pending):
|
||||
claim = [
|
||||
ctx.instance_uuid,
|
||||
ctx.workspace_uuid,
|
||||
ctx.placement_generation,
|
||||
ctx.account_uuid,
|
||||
ctx.workspace.membership_uuid,
|
||||
ctx.workspace.membership_revision,
|
||||
str(PLANNER_VERSION),
|
||||
_fingerprint(row),
|
||||
plan,
|
||||
facts,
|
||||
pending['uuid'] if pending else None,
|
||||
]
|
||||
# An authenticated opaque digest discloses neither credentials nor configs.
|
||||
return hmac.new(self._token_key, _json(claim).encode(), hashlib.sha256).hexdigest()
|
||||
|
||||
async def preview(self, ctx: RequestContext):
|
||||
require_permission(ctx, Permission.RESOURCE_VIEW)
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._binding(ctx)
|
||||
items = []
|
||||
for row in await self._rows(ctx):
|
||||
plan, facts, pending, state, blockers = await self._plan(ctx, row)
|
||||
item = {
|
||||
k: copy.deepcopy(plan[k])
|
||||
for k in ('legacy_runner', 'target_runner_id', 'target_plugin', 'changed_paths', 'warnings')
|
||||
}
|
||||
item.update(
|
||||
pipeline_uuid=row['uuid'],
|
||||
name=row['name'],
|
||||
state=state,
|
||||
blockers=blockers,
|
||||
preview_token=self._token(ctx, row, plan, facts, pending)
|
||||
if state in ('ready', 'activation_pending')
|
||||
else None,
|
||||
)
|
||||
items.append(item)
|
||||
return {
|
||||
'planner_version': str(PLANNER_VERSION),
|
||||
'workspace_uuid': ctx.workspace_uuid,
|
||||
'items': items,
|
||||
'total': len(items),
|
||||
}
|
||||
|
||||
async def _selected(self, ctx, selection):
|
||||
rows = await self._rows(ctx, selection['pipeline_uuid'])
|
||||
if not rows:
|
||||
raise MigrationError('pipeline_not_found', 404)
|
||||
row = rows[0]
|
||||
plan, facts, pending, state, _ = await self._plan(ctx, row)
|
||||
token = self._token(ctx, row, plan, facts, pending)
|
||||
if not hmac.compare_digest(token, selection['preview_token']):
|
||||
raise MigrationError('preview_stale')
|
||||
if state not in ('ready', 'activation_pending'):
|
||||
raise MigrationError('migration_blocked')
|
||||
return row, plan, facts, pending
|
||||
|
||||
async def execute(self, ctx: RequestContext, body):
|
||||
require_permission(ctx, Permission.RESOURCE_MANAGE)
|
||||
items = validate_execute_request(body)
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._authorize(ctx)
|
||||
# Preflight the entire selection before creating tasks or contacting runtimes.
|
||||
for selection in items:
|
||||
await self._selected(ctx, selection)
|
||||
task_context = TaskContext.new()
|
||||
task_context.metadata = {
|
||||
'kind': 'pipeline_migration',
|
||||
'results': [{'pipeline_uuid': i['pipeline_uuid'], 'state': 'pending', 'code': None} for i in items],
|
||||
}
|
||||
task = self.ap.task_mgr.create_user_task(
|
||||
self._run(ctx, ExecutionContext.from_request(ctx), items, task_context),
|
||||
kind='pipeline_migration',
|
||||
name='pipeline_migration',
|
||||
context=task_context,
|
||||
instance_uuid=ctx.instance_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
placement_generation=ctx.placement_generation,
|
||||
)
|
||||
return {'task_id': task.id}
|
||||
|
||||
async def _verify_runtime(self, execution, plan):
|
||||
runners = await self.ap.runner_registry.list_runners(execution, use_cache=False, usage='agent')
|
||||
descriptor = next((r for r in runners if r.id == plan['target_runner_id']), None)
|
||||
if descriptor is None or 'agent' not in descriptor.usages:
|
||||
raise MigrationError('runner_unavailable')
|
||||
if descriptor.plugin_version != plan['target_plugin']['version']:
|
||||
raise MigrationError('plugin_version_incompatible')
|
||||
config = plan['config']
|
||||
RunnerConfigResolver.validate_pipeline_config(config)
|
||||
runner_config = config['ai']['runner_config'][plan['target_runner_id']]
|
||||
schema = {item['name']: item for item in descriptor.config_schema}
|
||||
# Host fields have their own scope/capability checks below; plugin
|
||||
# schema defaults must never implicitly authorize a foreign resource.
|
||||
host_fields = {
|
||||
'enable-all-tools': 'boolean',
|
||||
'tools': 'array',
|
||||
'knowledge-bases': 'knowledge-base-multi-selector',
|
||||
'mcp-resources': 'array',
|
||||
'mcp-resource-agent-read-enabled': 'boolean',
|
||||
}
|
||||
for name, value in runner_config.items():
|
||||
field = schema.get(name)
|
||||
if field is None and name in host_fields:
|
||||
field = {'type': host_fields[name]}
|
||||
if field is None:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if value is None and field.get('nullable') is True:
|
||||
continue
|
||||
kind = config_schema.normalize_schema_item_type(field.get('type'))
|
||||
valid = (
|
||||
(
|
||||
kind
|
||||
in (
|
||||
'string',
|
||||
'text',
|
||||
'password',
|
||||
'secret',
|
||||
'select',
|
||||
'llm-model-selector',
|
||||
'embedding-model-selector',
|
||||
'rerank-model-selector',
|
||||
)
|
||||
and isinstance(value, str)
|
||||
)
|
||||
or (kind == 'boolean' and type(value) is bool)
|
||||
or (kind in ('integer', 'int') and type(value) is int)
|
||||
or (kind in ('float', 'number') and type(value) in (int, float))
|
||||
or (
|
||||
kind
|
||||
in (
|
||||
'array',
|
||||
'multi-select',
|
||||
'knowledge-base-selector',
|
||||
'knowledge-base-multi-selector',
|
||||
'prompt-editor',
|
||||
)
|
||||
and isinstance(value, list)
|
||||
)
|
||||
or (kind in ('object', 'json') and type(value) is dict)
|
||||
or (kind == 'array[string]' and type(value) is list and all(type(v) is str for v in value))
|
||||
or (
|
||||
kind == 'model-fallback-selector'
|
||||
and (
|
||||
isinstance(value, str)
|
||||
or isinstance(value, dict)
|
||||
and set(value) <= {'primary', 'fallbacks', 'reasoning'}
|
||||
and isinstance(value.get('primary'), str)
|
||||
and isinstance(value.get('fallbacks', []), list)
|
||||
and all(isinstance(v, str) for v in value.get('fallbacks', []))
|
||||
and isinstance(value.get('reasoning', {}), dict)
|
||||
)
|
||||
)
|
||||
)
|
||||
if not valid:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if kind == 'select' and field.get('options'):
|
||||
if value not in [o.get('value', o.get('name')) for o in field['options']]:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if any(
|
||||
f.get('required') and name not in runner_config and f.get('default') is None for name, f in schema.items()
|
||||
):
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
await self._verify_resources(execution, descriptor, runner_config)
|
||||
return _json(
|
||||
[
|
||||
descriptor.id,
|
||||
descriptor.plugin_version,
|
||||
descriptor.usages,
|
||||
descriptor.config_schema,
|
||||
getattr(descriptor, 'capabilities', None),
|
||||
getattr(descriptor, 'permissions', None),
|
||||
]
|
||||
)
|
||||
|
||||
async def _verify_resources(self, execution, descriptor, runner_config):
|
||||
"""Fail closed on references the scoped Host cannot authorize.
|
||||
|
||||
This runs only on explicit execution, outside the mutation transaction.
|
||||
Resource use is still reauthorized by the Host on every eventual run.
|
||||
"""
|
||||
|
||||
async def require_row(model, identifier, *, column=None):
|
||||
column = model.uuid if column is None else column
|
||||
found = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(model.uuid).where(
|
||||
model.workspace_uuid == execution.workspace_uuid,
|
||||
column == identifier,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if found is None:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
permissions = getattr(descriptor, 'permissions', None)
|
||||
model_resources = []
|
||||
for model_type, model_uuid in config_schema.iter_config_model_refs(descriptor, runner_config):
|
||||
allowed = set(getattr(permissions, 'models', []))
|
||||
if not (allowed & ({'rerank'} if model_type == 'rerank' else {'invoke', 'stream'})):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
await require_row(RerankModel if model_type == 'rerank' else LLMModel, model_uuid)
|
||||
model_resources.append({'model_id': model_uuid})
|
||||
for field in config_schema.iter_schema_items(descriptor, {'embedding-model-selector'}):
|
||||
value = runner_config.get(field['name'])
|
||||
if value and value not in config_schema.NONE_SENTINELS:
|
||||
await require_row(EmbeddingModel, value)
|
||||
extract_model_reasoning_overrides(descriptor, runner_config, {'models': model_resources})
|
||||
|
||||
kb_ids = runner_config.get('knowledge-bases', [])
|
||||
if not isinstance(kb_ids, list) or any(not isinstance(v, str) or not v for v in kb_ids):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
if kb_ids:
|
||||
if not config_schema.uses_host_knowledge_bases(descriptor) or 'retrieve' not in getattr(
|
||||
permissions, 'knowledge_bases', []
|
||||
):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
for kb_id in kb_ids:
|
||||
await require_row(KnowledgeBase, kb_id)
|
||||
|
||||
tool_names = runner_config.get('tools', [])
|
||||
if not isinstance(tool_names, list) or any(not isinstance(v, str) or not v for v in tool_names):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
tool_grant = (
|
||||
runner_config.get('enable-all-tools', False)
|
||||
or tool_names
|
||||
or runner_config.get('mcp-resource-agent-read-enabled', False)
|
||||
)
|
||||
if tool_grant:
|
||||
if not config_schema.uses_host_tools(descriptor) or 'call' not in getattr(permissions, 'tools', []):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
policy = ResourcePolicyProjector.from_runner_config(runner_config)
|
||||
tools = await AgentResourceBuilder(self.ap)._build_tools_from_binding(
|
||||
execution,
|
||||
permissions,
|
||||
policy,
|
||||
descriptor,
|
||||
runner_config,
|
||||
)
|
||||
if not set(tool_names) <= {tool['tool_name'] for tool in tools}:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
for attachment in runner_config.get('mcp-resources', []):
|
||||
if not isinstance(attachment, dict):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
identifier = attachment.get('server_uuid') or attachment.get('server_id')
|
||||
name = attachment.get('server_name')
|
||||
if identifier:
|
||||
await require_row(MCPServer, identifier)
|
||||
elif name:
|
||||
await require_row(MCPServer, name, column=MCPServer.name)
|
||||
else:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
async def _lock(self, ctx, pipeline_uuid, session):
|
||||
# Every pending producer shares this row and revalidates its captured
|
||||
# configuration/conversation authority after acquiring admission.
|
||||
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
await lock_pipeline_admission(session, ctx.workspace_uuid, pipeline_uuid)
|
||||
# Hold revocable directory rows through commit (no network in this UoW).
|
||||
await session.execute(sa.select(User.uuid).where(User.uuid == ctx.account_uuid).with_for_update())
|
||||
for model, condition in [
|
||||
(Workspace, Workspace.uuid == ctx.workspace_uuid),
|
||||
(WorkspaceExecutionState, WorkspaceExecutionState.workspace_uuid == ctx.workspace_uuid),
|
||||
(
|
||||
WorkspaceMembership,
|
||||
sa.and_(
|
||||
WorkspaceMembership.workspace_uuid == ctx.workspace_uuid,
|
||||
WorkspaceMembership.account_uuid == ctx.account_uuid,
|
||||
),
|
||||
),
|
||||
(PluginSetting, PluginSetting.workspace_uuid == ctx.workspace_uuid),
|
||||
]:
|
||||
await session.execute(sa.select(model.__table__).where(condition).with_for_update())
|
||||
|
||||
async def _commit(self, ctx, selection, expected_facts, candidate):
|
||||
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
|
||||
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
|
||||
await self._authorize(ctx, uow.session)
|
||||
row, plan, facts, pending = await self._selected(ctx, selection)
|
||||
if facts != expected_facts:
|
||||
raise MigrationError('plugin_changed')
|
||||
if pending:
|
||||
return pending['uuid'], pending['target_fingerprint']
|
||||
target = {**row, 'config': candidate['config']}
|
||||
t = LegacyPipeline.__table__
|
||||
cas = [t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == row['uuid']]
|
||||
for key in ('config', 'stages', 'extensions_preferences'):
|
||||
cas.append(sa.cast(t.c[key], sa.Text) == row['_raw_' + key])
|
||||
result = await self.pm.execute_async(sa.update(t).where(*cas).values(config=candidate['config']))
|
||||
if result.rowcount != 1:
|
||||
raise MigrationError('preview_stale')
|
||||
snapshot_uuid = str(uuid.uuid4())
|
||||
target_fingerprint = _fingerprint(target)
|
||||
await self.pm.execute_async(
|
||||
sa.insert(PipelineMigrationSnapshot).values(
|
||||
uuid=snapshot_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
pipeline_uuid=row['uuid'],
|
||||
source_fingerprint=_fingerprint(row),
|
||||
target_fingerprint=target_fingerprint,
|
||||
source_snapshot=json.loads(_json({k: v for k, v in row.items() if not k.startswith('_')})),
|
||||
planner_version=str(PLANNER_VERSION),
|
||||
state='activation_pending',
|
||||
)
|
||||
)
|
||||
if self._native_interactions(ctx, row['uuid']):
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
return snapshot_uuid, target_fingerprint
|
||||
|
||||
async def _activate(self, ctx, selection, snapshot_uuid, target_fingerprint, candidate, plan, expected_facts):
|
||||
# Re-read the committed target under a short lock before synchronous
|
||||
# publication. A later editor must never be replaced by our old candidate.
|
||||
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
|
||||
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
|
||||
await self._authorize(ctx, uow.session)
|
||||
rows = await self._rows(ctx, selection['pipeline_uuid'])
|
||||
if not rows or _fingerprint(rows[0]) != target_fingerprint:
|
||||
raise MigrationError('activation_source_changed')
|
||||
if await self._plugin(ctx, plan) != expected_facts:
|
||||
raise MigrationError('plugin_changed')
|
||||
_, interaction_blocked = await self._interaction_state(ctx, selection['pipeline_uuid'])
|
||||
if interaction_blocked:
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
# Native cache inspection above is the last await before publication
|
||||
# and session invalidation; no native producer can interleave here.
|
||||
self.ap.pipeline_mgr.publish_pipeline(candidate)
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
conversation = session.using_conversation
|
||||
if (
|
||||
conversation is not None
|
||||
and conversation.pipeline_uuid == selection['pipeline_uuid']
|
||||
and getattr(session, 'workspace_uuid', None) == ctx.workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
await self.pm.execute_async(
|
||||
sa.update(t)
|
||||
.where(t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == snapshot_uuid)
|
||||
.values(state='active')
|
||||
)
|
||||
|
||||
async def _run(self, ctx, execution, items, task_context):
|
||||
async with self.pm.tenant_scope(execution.workspace_uuid):
|
||||
for selection, result in zip(items, task_context.metadata['results']):
|
||||
committed = False
|
||||
commit_attempted = False
|
||||
try:
|
||||
await self._authorize(ctx)
|
||||
row, plan, facts, pending = await self._selected(ctx, selection)
|
||||
runtime_schema = await self._verify_runtime(execution, plan)
|
||||
candidate_entity = {k: copy.deepcopy(v) for k, v in row.items() if not k.startswith('_')}
|
||||
candidate_entity['config'] = copy.deepcopy(plan['config'])
|
||||
runtime = await self.ap.pipeline_mgr.prepare_pipeline(execution, copy.deepcopy(candidate_entity))
|
||||
if await self._verify_runtime(execution, plan) != runtime_schema:
|
||||
raise MigrationError('runner_schema_changed')
|
||||
# Runtime awaits are over. Recheck authorization/source/plugin
|
||||
# facts under database locks, then atomically journal and CAS.
|
||||
commit_attempted = True
|
||||
snapshot_uuid, target_fingerprint = await self._commit(ctx, selection, facts, candidate_entity)
|
||||
committed = True
|
||||
await self._activate(ctx, selection, snapshot_uuid, target_fingerprint, runtime, plan, facts)
|
||||
result.update(state='migrated', code=None)
|
||||
except (Exception, asyncio.CancelledError) as exc:
|
||||
cancelled = isinstance(exc, asyncio.CancelledError)
|
||||
reconciliation_cancel = None
|
||||
code = (
|
||||
'operation_cancelled'
|
||||
if cancelled
|
||||
else exc.code
|
||||
if isinstance(exc, MigrationError)
|
||||
else 'migration_failed'
|
||||
)
|
||||
if commit_attempted and not committed and not isinstance(exc, MigrationError):
|
||||
# A commit can succeed while acknowledgement/connection
|
||||
# cleanup fails, including cancellation. Make one read
|
||||
# after the UoW has unwound, in this task's tenant scope;
|
||||
# never shield/retry through another cancellation.
|
||||
try:
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
conditions = [
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.pipeline_uuid == selection['pipeline_uuid'],
|
||||
t.c.source_fingerprint
|
||||
== (pending['source_fingerprint'] if pending else _fingerprint(row)),
|
||||
t.c.target_fingerprint == _fingerprint(candidate_entity),
|
||||
t.c.planner_version == str(PLANNER_VERSION),
|
||||
]
|
||||
if pending:
|
||||
# A retry starts from the target, not the legacy
|
||||
# source. Only the original snapshot proves this
|
||||
# durable migration; an unrelated journal cannot.
|
||||
conditions.extend(
|
||||
[
|
||||
t.c.uuid == pending['uuid'],
|
||||
t.c.target_fingerprint == pending['target_fingerprint'],
|
||||
]
|
||||
)
|
||||
found = (await self.pm.execute_async(sa.select(t.c.uuid).where(*conditions))).first()
|
||||
committed = found is not None
|
||||
except asyncio.CancelledError as cancel_exc:
|
||||
reconciliation_cancel = cancel_exc
|
||||
cancelled = True
|
||||
code = 'commit_outcome_unknown'
|
||||
except Exception:
|
||||
code = 'commit_outcome_unknown'
|
||||
state = (
|
||||
# Unknown is deliberately recovery-needed, not a claim
|
||||
# of rollback or proof that a snapshot was committed.
|
||||
'activation_pending'
|
||||
if committed or code == 'commit_outcome_unknown'
|
||||
else 'stale'
|
||||
if code == 'preview_stale'
|
||||
else 'blocked'
|
||||
if isinstance(exc, MigrationError)
|
||||
else 'failed'
|
||||
)
|
||||
result.update(state=state, code=code)
|
||||
if cancelled:
|
||||
for unstarted in task_context.metadata['results']:
|
||||
if unstarted['state'] == 'pending':
|
||||
unstarted.update(state='failed', code='operation_cancelled')
|
||||
if reconciliation_cancel is not None:
|
||||
# Preserve cancellation propagation during recovery,
|
||||
# but record safe outcomes before leaving the task.
|
||||
raise reconciliation_cancel
|
||||
return task_context.metadata
|
||||
# Never attach the original exception, traceback or plugin
|
||||
# diagnostic to TaskContext: upstream errors can contain keys.
|
||||
return task_context.metadata
|
||||
Reference in New Issue
Block a user