mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-20 23:45:22 +08:00
feat: add verified manual pipeline migration for plugin runners
This commit is contained in:
@@ -249,6 +249,20 @@ class RunnerContextBuilder:
|
||||
'timestamp': event.event_time or int(time.time()),
|
||||
}
|
||||
|
||||
# A legacy Tbox resume must use the persisted original bot, not a
|
||||
# newly reconstructed callback envelope. Ordinary sender mode is unchanged.
|
||||
bot_id = event.bot_id
|
||||
if event.event_type == 'interaction.submitted' and binding.runner_config.get('user-id-source') == 'legacy-bot':
|
||||
identity = event.legacy_identity
|
||||
bot_id = (
|
||||
identity.bot_id
|
||||
if identity is not None
|
||||
and identity.workspace_id == event.workspace_id
|
||||
and identity.bot_id == event.bot_id
|
||||
and identity.pipeline_id == (binding.processor_id or binding.agent_id)
|
||||
else None
|
||||
)
|
||||
|
||||
# Build conversation context from event
|
||||
conversation: ConversationContext | None = None
|
||||
if event.conversation_id:
|
||||
@@ -256,13 +270,39 @@ class RunnerContextBuilder:
|
||||
'session_id': None,
|
||||
'conversation_id': event.conversation_id,
|
||||
'thread_id': event.thread_id,
|
||||
'launcher_type': None, # Will be filled from actor/subject if needed
|
||||
'launcher_type': None,
|
||||
'launcher_id': None,
|
||||
'sender_id': event.actor.actor_id if event.actor else None,
|
||||
'bot_id': event.bot_id,
|
||||
'bot_id': bot_id,
|
||||
'workspace_id': event.workspace_id,
|
||||
}
|
||||
|
||||
identity = event.legacy_identity
|
||||
if (
|
||||
binding.runner_config.get('user-id-source') == 'legacy-session'
|
||||
and identity is not None
|
||||
and identity.workspace_id == event.workspace_id
|
||||
and identity.bot_id == event.bot_id
|
||||
and identity.pipeline_id == (binding.processor_id or binding.agent_id)
|
||||
and binding.processor_type == 'pipeline'
|
||||
):
|
||||
# Coze used Query itself; Dify/n8n used Query.session, which may
|
||||
# deliberately differ (e.g. group per-member session isolation).
|
||||
if binding.runner_id == 'plugin:langbot-team/CozeAgent/default':
|
||||
launcher_type = identity.query_launcher_type
|
||||
launcher_id = identity.query_launcher_id
|
||||
elif binding.runner_id in {
|
||||
'plugin:langbot-team/DifyAgent/default',
|
||||
'plugin:langbot-team/N8nAgent/default',
|
||||
}:
|
||||
launcher_type = identity.session_launcher_type
|
||||
launcher_id = identity.session_launcher_id
|
||||
else:
|
||||
launcher_type = launcher_id = None
|
||||
if launcher_type in ('group', 'person') and launcher_id:
|
||||
conversation['launcher_type'] = launcher_type
|
||||
conversation['launcher_id'] = launcher_id
|
||||
|
||||
# Build event context (Protocol v1 event-first)
|
||||
event_context = {
|
||||
'event_id': event.event_id,
|
||||
@@ -322,7 +362,7 @@ class RunnerContextBuilder:
|
||||
'trace_id': run_id,
|
||||
'deadline_at': self._build_deadline_from_binding(binding),
|
||||
'metadata': {
|
||||
'bot_id': event.bot_id,
|
||||
'bot_id': bot_id,
|
||||
'workspace_id': event.workspace_id,
|
||||
'streaming_supported': event.delivery.supports_streaming,
|
||||
'model_context_window_tokens': model_context_window_tokens,
|
||||
|
||||
@@ -17,6 +17,18 @@ from langbot_plugin.api.entities.builtin.runner.input import AgentInput
|
||||
from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
|
||||
|
||||
|
||||
class LegacyRunnerIdentity(pydantic.BaseModel):
|
||||
"""Host-captured native Query identity; never populated from event data/params."""
|
||||
|
||||
workspace_id: str | None = None
|
||||
bot_id: str | None = None
|
||||
pipeline_id: str | None = None
|
||||
query_launcher_type: str | None = None
|
||||
query_launcher_id: str | None = None
|
||||
session_launcher_type: str | None = None
|
||||
session_launcher_id: str | None = None
|
||||
|
||||
|
||||
class AgentEventEnvelope(pydantic.BaseModel):
|
||||
"""Event envelope for LangBot Host event gateway.
|
||||
|
||||
@@ -69,6 +81,9 @@ class AgentEventEnvelope(pydantic.BaseModel):
|
||||
data: dict[str, typing.Any] = pydantic.Field(default_factory=dict)
|
||||
"""Small structured event payload. Large payloads should be referenced via raw_ref."""
|
||||
|
||||
legacy_identity: LegacyRunnerIdentity | None = None
|
||||
"""Host-only provenance, separate from untrusted event payload and adapter params."""
|
||||
|
||||
|
||||
# Binding scope types
|
||||
class BindingScope(pydantic.BaseModel):
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from langbot.pkg.telemetry import diagnostics
|
||||
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
import typing
|
||||
@@ -14,7 +15,7 @@ from langbot_plugin.api.entities.builtin.platform import message as platform_mes
|
||||
|
||||
from .descriptor import RunnerDescriptor
|
||||
from .errors import RunnerProtocolError
|
||||
from .host_models import AgentBinding, AgentEventEnvelope
|
||||
from .host_models import AgentBinding, AgentEventEnvelope, LegacyRunnerIdentity
|
||||
from .interaction_store import InteractionStore
|
||||
|
||||
|
||||
@@ -152,7 +153,12 @@ class InteractionManager:
|
||||
runner_id=descriptor.id,
|
||||
processor_type=binding.processor_type,
|
||||
processor_id=processor_id,
|
||||
request=request_data,
|
||||
request={
|
||||
**request_data,
|
||||
'_host_legacy_identity': event.legacy_identity.model_dump(mode='json')
|
||||
if event.legacy_identity is not None
|
||||
else None,
|
||||
},
|
||||
delivery_target=reply_target,
|
||||
replaces_interaction_id=(update_target or {}).get('interaction_id'),
|
||||
bot_id=event.bot_id,
|
||||
@@ -161,6 +167,7 @@ class InteractionManager:
|
||||
thread_id=event.thread_id,
|
||||
actor_id=event.actor.actor_id if event.actor else None,
|
||||
expires_at=expires_at,
|
||||
**self._pipeline_admission(binding, descriptor, adapter_context),
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -204,6 +211,50 @@ class InteractionManager:
|
||||
diagnostics.set_outcome('waiting', reason_code='waiting')
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_admission(binding, descriptor, adapter_context):
|
||||
if binding.processor_type != 'pipeline':
|
||||
return {}
|
||||
query = (adapter_context or {}).get('_query')
|
||||
expected = copy.deepcopy((adapter_context or {}).get('_pipeline_expected_config'))
|
||||
session = getattr(query, 'session', None)
|
||||
conversation = (adapter_context or {}).get('_pipeline_conversation')
|
||||
|
||||
def authority():
|
||||
from .config_resolver import RunnerConfigResolver
|
||||
|
||||
return (
|
||||
query is not None
|
||||
and session is not None
|
||||
and conversation is not None
|
||||
and isinstance(expected, dict)
|
||||
and session.using_conversation is conversation
|
||||
and getattr(query, 'pipeline_uuid', None) == binding.processor_id
|
||||
and RunnerConfigResolver.resolve_runner_id(expected) == descriptor.id == binding.runner_id
|
||||
and RunnerConfigResolver.resolve_runner_config(expected, descriptor.id) == binding.runner_config
|
||||
)
|
||||
|
||||
return {'expected_config': expected, 'authority_check': authority}
|
||||
|
||||
async def restore_legacy_identity(self, event: AgentEventEnvelope, binding: AgentBinding) -> None:
|
||||
"""Resume only from the scoped, consumed Host record, never callback fields."""
|
||||
event.legacy_identity = None
|
||||
submission = getattr(event.input, 'interaction', None)
|
||||
if submission is None:
|
||||
return
|
||||
if hasattr(submission, 'model_dump'):
|
||||
submission = submission.model_dump(mode='json')
|
||||
record = await self.store.find_resume_request(event=event, binding=binding, submission=submission)
|
||||
if record is None:
|
||||
return
|
||||
identity = record['request'].get('_host_legacy_identity')
|
||||
if not isinstance(identity, dict):
|
||||
return
|
||||
try:
|
||||
event.legacy_identity = LegacyRunnerIdentity.model_validate(identity)
|
||||
except pydantic.ValidationError:
|
||||
return
|
||||
|
||||
@diagnostics.observe('interaction', 'interaction.acknowledge', source='platform', stage='ack')
|
||||
async def acknowledge_submission(self, record: dict[str, typing.Any], adapter: typing.Any) -> None:
|
||||
"""Best-effort transition of submitted controls into a read-only state."""
|
||||
|
||||
@@ -14,6 +14,8 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from ...entity.persistence.agent_interaction import AgentInteraction
|
||||
from ...persistence.tenant_uow import TenantUnitOfWork
|
||||
from ...persistence.pipeline_admission import lock_pipeline_admission
|
||||
|
||||
|
||||
UTC = datetime.timezone.utc
|
||||
@@ -49,6 +51,12 @@ def _utc_now() -> datetime.datetime:
|
||||
return datetime.datetime.now(UTC)
|
||||
|
||||
|
||||
def _db_datetime(value):
|
||||
# Existing schema is TIMESTAMP WITHOUT TIME ZONE; persist UTC naive on
|
||||
# both backends and normalize to aware UTC only for application comparisons.
|
||||
return _as_utc(value).replace(tzinfo=None) if value is not None else None
|
||||
|
||||
|
||||
def _as_utc(value: datetime.datetime | None) -> datetime.datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
@@ -113,6 +121,8 @@ class InteractionStore:
|
||||
thread_id: str | None = None,
|
||||
actor_id: str | None = None,
|
||||
expires_at: int | float | None = None,
|
||||
expected_config: dict | None = None,
|
||||
authority_check: typing.Callable[[], bool] | None = None,
|
||||
) -> tuple[dict[str, typing.Any], str]:
|
||||
"""Persist a request and return its record plus one-time callback token."""
|
||||
if not interaction_id or not run_id or not binding_id or not runner_id or not processor_id:
|
||||
@@ -145,11 +155,32 @@ class InteractionStore:
|
||||
delivery_target_json=delivery_target_json,
|
||||
replaces_interaction_id=replaces_interaction_id,
|
||||
callback_token_hash=_token_hash(callback_token),
|
||||
expires_at=expires_at_dt,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
expires_at=_db_datetime(expires_at_dt),
|
||||
created_at=_db_datetime(now),
|
||||
updated_at=_db_datetime(now),
|
||||
)
|
||||
|
||||
if processor_type == 'pipeline':
|
||||
# Require Host-captured configuration and live conversation authority;
|
||||
# runner output must never supply either of these admission inputs.
|
||||
if not workspace_id or not isinstance(expected_config, dict) or authority_check is None:
|
||||
raise InteractionScopeError('Pipeline interaction authority unavailable')
|
||||
try:
|
||||
async with TenantUnitOfWork(self.engine, workspace_id) as uow:
|
||||
current = await lock_pipeline_admission(uow.session, workspace_id, processor_id)
|
||||
if current != expected_config or authority_check() is not True:
|
||||
raise InteractionScopeError('Pipeline interaction authority changed')
|
||||
existing = await self._get_by_run_interaction(uow.session, run_id, interaction_id)
|
||||
if existing is not None:
|
||||
raise DuplicateInteractionError('Interaction already exists')
|
||||
uow.session.add(row)
|
||||
await uow.session.flush()
|
||||
return self._to_dict(row), callback_token
|
||||
except ValueError as exc:
|
||||
raise InteractionScopeError('Pipeline interaction authority unavailable') from exc
|
||||
except IntegrityError as exc:
|
||||
raise DuplicateInteractionError('Interaction already exists') from exc
|
||||
|
||||
async with self._session_factory() as session:
|
||||
existing = await self._get_by_run_interaction(session, run_id, interaction_id)
|
||||
if existing is not None:
|
||||
@@ -182,7 +213,7 @@ class InteractionStore:
|
||||
AgentInteraction.interaction_id == interaction_id,
|
||||
AgentInteraction.status.in_(['pending', 'submitted']),
|
||||
)
|
||||
.values(delivery_result_json=payload, updated_at=_utc_now())
|
||||
.values(delivery_result_json=payload, updated_at=_db_datetime(_utc_now()))
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
@@ -228,6 +259,38 @@ class InteractionStore:
|
||||
row = result.scalar_one_or_none()
|
||||
return self._to_dict(row) if row is not None else None
|
||||
|
||||
async def find_resume_request(self, *, event, binding, submission) -> dict[str, typing.Any] | None:
|
||||
"""Resolve persisted provenance with exact tenant/processor/callback scope.
|
||||
|
||||
Ambiguous identities fail closed instead of picking the newest run.
|
||||
"""
|
||||
if not isinstance(submission, dict) or not submission.get('interaction_id'):
|
||||
return None
|
||||
target = event.delivery.reply_target or {}
|
||||
target_type, target_id = target.get('target_type'), target.get('target_id')
|
||||
conversation_id = f'{target_type}_{target_id}' if target_type and target_id else event.conversation_id
|
||||
conditions = [
|
||||
AgentInteraction.interaction_id == submission['interaction_id'],
|
||||
AgentInteraction.status == 'submitted',
|
||||
AgentInteraction.binding_id == binding.binding_id,
|
||||
AgentInteraction.runner_id == binding.runner_id,
|
||||
AgentInteraction.processor_type == binding.processor_type,
|
||||
AgentInteraction.processor_id == (binding.processor_id or binding.agent_id),
|
||||
]
|
||||
for column, value in (
|
||||
(AgentInteraction.workspace_id, event.workspace_id),
|
||||
(AgentInteraction.bot_id, event.bot_id),
|
||||
(AgentInteraction.conversation_id, conversation_id),
|
||||
(AgentInteraction.thread_id, event.thread_id),
|
||||
(AgentInteraction.actor_id, event.actor.actor_id if event.actor else None),
|
||||
):
|
||||
conditions.append(column.is_(None) if value is None else column == value)
|
||||
async with self._session_factory() as session:
|
||||
result = await session.execute(sqlalchemy.select(AgentInteraction).where(*conditions))
|
||||
records = [self._to_dict(row) for row in result.scalars()]
|
||||
matches = [record for record in records if record['submission'] == submission]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
async def get_request(self, run_id: str, interaction_id: str) -> dict[str, typing.Any] | None:
|
||||
"""Return a request by the runner-visible identity."""
|
||||
async with self._session_factory() as session:
|
||||
@@ -283,7 +346,7 @@ class InteractionStore:
|
||||
if expires_at is not None and expires_at <= now:
|
||||
row.status = 'expired'
|
||||
row.status_reason = 'interaction expired before submission'
|
||||
row.updated_at = now
|
||||
row.updated_at = _db_datetime(now)
|
||||
await session.commit()
|
||||
raise InteractionExpiredError('Interaction has expired')
|
||||
|
||||
@@ -297,9 +360,9 @@ class InteractionStore:
|
||||
)
|
||||
.values(
|
||||
status='submitted',
|
||||
submitted_at=submission_time,
|
||||
submitted_at=_db_datetime(submission_time),
|
||||
submission_json=_json_dumps(submission),
|
||||
updated_at=now,
|
||||
updated_at=_db_datetime(now),
|
||||
)
|
||||
)
|
||||
if update_result.rowcount != 1:
|
||||
@@ -323,7 +386,7 @@ class InteractionStore:
|
||||
AgentInteraction.interaction_id == interaction_id,
|
||||
AgentInteraction.status == 'pending',
|
||||
)
|
||||
.values(status='delivery_failed', status_reason=reason, updated_at=now)
|
||||
.values(status='delivery_failed', status_reason=reason, updated_at=_db_datetime(now))
|
||||
)
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
@@ -337,12 +400,12 @@ class InteractionStore:
|
||||
.where(
|
||||
AgentInteraction.status == 'pending',
|
||||
AgentInteraction.expires_at.is_not(None),
|
||||
AgentInteraction.expires_at <= cutoff,
|
||||
AgentInteraction.expires_at <= _db_datetime(cutoff),
|
||||
)
|
||||
.values(
|
||||
status='expired',
|
||||
status_reason='interaction expired',
|
||||
updated_at=cutoff,
|
||||
updated_at=_db_datetime(cutoff),
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Host-owned, run-scoped reasoning policy for schema-declared model selectors.
|
||||
|
||||
This policy is not an SDK resource or a provider kwarg. Only the Host binding
|
||||
assembler supplies it; model actions consult it after resource authorization.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import typing
|
||||
|
||||
from ...provider.modelmgr.reasoning import normalize_reasoning_config, validate_reasoning_config
|
||||
from .config_schema import NONE_SENTINELS, iter_schema_items
|
||||
from .descriptor import RunnerDescriptor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ...provider.modelmgr.requester import RuntimeLLMModel
|
||||
|
||||
|
||||
ModelReasoningOverrides = dict[str, dict[str, str]]
|
||||
|
||||
|
||||
def extract_model_reasoning_overrides(
|
||||
descriptor: RunnerDescriptor,
|
||||
runner_config: dict[str, typing.Any],
|
||||
resources: typing.Mapping[str, typing.Any],
|
||||
) -> ModelReasoningOverrides:
|
||||
"""Normalize explicit UUID-to-level mappings, intersecting selection and grants.
|
||||
|
||||
An absent entry preserves persisted model defaults. Explicit provider_default
|
||||
overrides them using the native requester's semantics. Descriptor defaults and
|
||||
undeclared config fields cannot silently introduce reasoning overrides.
|
||||
"""
|
||||
authorized = {model.get('model_id') for model in resources.get('models', [])}
|
||||
overrides: ModelReasoningOverrides = {}
|
||||
for item in iter_schema_items(descriptor, {'model-fallback-selector'}):
|
||||
field_name = item.get('name')
|
||||
if not isinstance(field_name, str):
|
||||
continue
|
||||
selection = runner_config.get(field_name)
|
||||
if not isinstance(selection, dict) or 'reasoning' not in selection:
|
||||
continue
|
||||
configured = selection['reasoning']
|
||||
if not isinstance(configured, dict):
|
||||
raise ValueError('Invalid runner model reasoning configuration')
|
||||
candidates = [selection.get('primary')]
|
||||
fallbacks = selection.get('fallbacks')
|
||||
if isinstance(fallbacks, list):
|
||||
candidates.extend(fallbacks)
|
||||
selected = {value for value in candidates if isinstance(value, str) and value not in NONE_SENTINELS}
|
||||
for model_id, level in configured.items():
|
||||
# Validate with Core, but do not echo arbitrary config/secret values.
|
||||
try:
|
||||
if not isinstance(model_id, str) or not isinstance(level, str):
|
||||
raise ValueError
|
||||
config = normalize_reasoning_config({'level': level})
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError('Invalid runner model reasoning configuration') from None
|
||||
if model_id not in selected or model_id not in authorized:
|
||||
continue
|
||||
if model_id in overrides and overrides[model_id] != config:
|
||||
raise ValueError('Conflicting runner model reasoning overrides')
|
||||
overrides[model_id] = config
|
||||
return overrides
|
||||
|
||||
|
||||
def model_with_reasoning_override(
|
||||
model: RuntimeLLMModel,
|
||||
model_id: str,
|
||||
session: typing.Mapping[str, typing.Any] | None,
|
||||
) -> RuntimeLLMModel:
|
||||
"""Clone only the runtime wrapper, after the caller has authorized the model.
|
||||
|
||||
Do not mutate shared model entities or providers. Requesters retain ownership
|
||||
of ability/capability validation and provider-specific argument translation.
|
||||
"""
|
||||
if session is None:
|
||||
return model
|
||||
overrides = session.get('authorization', {}).get('model_reasoning_overrides', {})
|
||||
if model_id not in overrides:
|
||||
return model
|
||||
config = validate_reasoning_config(overrides[model_id], model.model_entity.abilities, model.model_entity.extra_args)
|
||||
scoped_model = copy.copy(model)
|
||||
scoped_model.reasoning_config_override = copy.deepcopy(config)
|
||||
return scoped_model
|
||||
@@ -29,6 +29,7 @@ from .execution_context import (
|
||||
)
|
||||
from .host_models import AgentBinding, AgentEventEnvelope
|
||||
from .invoker import RunnerInvoker
|
||||
from .model_reasoning import extract_model_reasoning_overrides
|
||||
from .interaction_manager import InteractionManager
|
||||
from .query_bridge import QueryRunBridge
|
||||
from .registry import RunnerRegistry
|
||||
@@ -121,6 +122,12 @@ class AgentRunOrchestrator:
|
||||
project_mcp_resource_config(execution_query, binding.runner_config)
|
||||
object.__setattr__(execution_query, '_execution_context', execution_context)
|
||||
|
||||
if event.event_type == 'interaction.submitted' and binding.runner_config.get('user-id-source') in (
|
||||
'legacy-session',
|
||||
'legacy-bot',
|
||||
):
|
||||
await self.interaction_manager.restore_legacy_identity(event, binding)
|
||||
|
||||
execution_event = event
|
||||
resource_addition = await build_mcp_resource_context_addition(self.ap, execution_query)
|
||||
if resource_addition:
|
||||
@@ -133,6 +140,7 @@ class AgentRunOrchestrator:
|
||||
binding=binding,
|
||||
descriptor=descriptor,
|
||||
)
|
||||
model_reasoning_overrides = extract_model_reasoning_overrides(descriptor, binding.runner_config, resources)
|
||||
|
||||
context = await self.context_builder.build_context_from_event(
|
||||
event=execution_event,
|
||||
@@ -192,6 +200,7 @@ class AgentRunOrchestrator:
|
||||
'state_scopes': list(binding.state_policy.state_scopes),
|
||||
},
|
||||
'state_context': state_context,
|
||||
'model_reasoning_overrides': model_reasoning_overrides,
|
||||
}
|
||||
|
||||
seen_sequences: set[int] = set()
|
||||
@@ -228,6 +237,7 @@ class AgentRunOrchestrator:
|
||||
execution_query=execution_query,
|
||||
platform_context=freeze_platform_context(event),
|
||||
reply_streams=reply_streams,
|
||||
model_reasoning_overrides=model_reasoning_overrides,
|
||||
)
|
||||
|
||||
event_log_id = await self.journal.write_event_log(
|
||||
@@ -412,6 +422,10 @@ class AgentRunOrchestrator:
|
||||
plan = self.query_bridge.build_plan(query)
|
||||
adapter_context = dict(plan.adapter_context)
|
||||
adapter_context['_query'] = query
|
||||
import copy
|
||||
|
||||
adapter_context['_pipeline_expected_config'] = copy.deepcopy(query.pipeline_config)
|
||||
adapter_context['_pipeline_conversation'] = getattr(getattr(query, 'session', None), 'using_conversation', None)
|
||||
adapter_context['_execution_context'] = get_query_execution_context(query)
|
||||
|
||||
# Inbound files and subsequent runner tools must share one Host scope.
|
||||
|
||||
@@ -25,6 +25,7 @@ from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
|
||||
from .host_models import (
|
||||
AgentConfig,
|
||||
AgentEventEnvelope,
|
||||
LegacyRunnerIdentity,
|
||||
StatePolicy,
|
||||
DeliveryPolicy,
|
||||
)
|
||||
@@ -100,6 +101,24 @@ class QueryEntryAdapter:
|
||||
delivery=delivery,
|
||||
raw_ref=raw_ref,
|
||||
data=event.data,
|
||||
legacy_identity=cls._build_legacy_identity(query, execution_context.workspace_uuid),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_legacy_identity(query, workspace_id: str) -> LegacyRunnerIdentity:
|
||||
def exact(value):
|
||||
value = getattr(value, 'value', value)
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
session = getattr(query, 'session', None)
|
||||
return LegacyRunnerIdentity(
|
||||
workspace_id=workspace_id,
|
||||
bot_id=exact(getattr(query, 'bot_uuid', None)),
|
||||
pipeline_id=exact(getattr(query, 'pipeline_uuid', None)),
|
||||
query_launcher_type=exact(getattr(query, 'launcher_type', None)),
|
||||
query_launcher_id=exact(getattr(query, 'launcher_id', None)),
|
||||
session_launcher_type=exact(getattr(session, 'launcher_type', None)),
|
||||
session_launcher_id=exact(getattr(session, 'launcher_id', None)),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -13,6 +13,7 @@ import threading
|
||||
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
|
||||
|
||||
from .context_builder import AgentResources
|
||||
from .model_reasoning import ModelReasoningOverrides
|
||||
from ...provider.tools.toolmgr import ToolSourceRef
|
||||
|
||||
|
||||
@@ -52,6 +53,7 @@ class RunAuthorizationSnapshot(typing.TypedDict):
|
||||
platform_context: dict[str, typing.Any]
|
||||
authorized_ids: dict[str, set[str]]
|
||||
authorized_operations: dict[str, dict[str, set[str]]]
|
||||
model_reasoning_overrides: ModelReasoningOverrides
|
||||
|
||||
|
||||
SteeringQueueItem = dict[str, typing.Any]
|
||||
@@ -119,6 +121,7 @@ class AgentRunSessionRegistry:
|
||||
execution_query: pipeline_query.Query | None = None,
|
||||
platform_context: dict[str, typing.Any] | None = None,
|
||||
reply_streams: typing.Any = None,
|
||||
model_reasoning_overrides: ModelReasoningOverrides | None = None,
|
||||
) -> None:
|
||||
"""Register a new agent run session.
|
||||
|
||||
@@ -164,6 +167,7 @@ class AgentRunSessionRegistry:
|
||||
'platform_context': copy.deepcopy(platform_context or {}),
|
||||
'authorized_ids': self._build_authorized_ids(resources_snapshot),
|
||||
'authorized_operations': self._build_authorized_operations(resources_snapshot),
|
||||
'model_reasoning_overrides': copy.deepcopy(model_reasoning_overrides or {}),
|
||||
}
|
||||
|
||||
session: AgentRunSession = {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
"""User-confirmed migration only; deliberately not API-key/MCP accessible."""
|
||||
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ....service.pipeline_migration import PipelineMigrationService, MigrationError
|
||||
|
||||
|
||||
@group.group_class('pipeline_migration', '/api/v1/pipelines/_/migration')
|
||||
class PipelineMigrationRouterGroup(group.RouterGroup):
|
||||
async def initialize(self):
|
||||
self.service = PipelineMigrationService(self.ap)
|
||||
|
||||
@self.route(
|
||||
'/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_VIEW
|
||||
)
|
||||
async def preview(request_context: RequestContext):
|
||||
try:
|
||||
return self.success(data=await self.service.preview(request_context))
|
||||
except MigrationError as exc:
|
||||
return self.http_status(exc.status_code, -1, exc.code)
|
||||
|
||||
@self.route(
|
||||
'/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN, permission=Permission.RESOURCE_MANAGE
|
||||
)
|
||||
async def execute(request_context: RequestContext):
|
||||
try:
|
||||
body = await quart.request.get_json(silent=True)
|
||||
return self.success(data=await self.service.execute(request_context, body))
|
||||
except MigrationError as exc:
|
||||
return self.http_status(exc.status_code, -1, exc.code)
|
||||
@@ -13,7 +13,7 @@ from ....pipeline.extension_preferences import (
|
||||
validate_extension_preferences,
|
||||
)
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .secrets import redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
@@ -213,12 +213,24 @@ class PipelineService:
|
||||
pipeline_data.pop(protected_field, None)
|
||||
|
||||
if 'config' in pipeline_data:
|
||||
current_config = None
|
||||
if contains_secret_placeholder(pipeline_data['config']):
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
old_ai = current_config.get('ai', {}) if isinstance(current_config, dict) else {}
|
||||
old_runner = old_ai.get('runner') if isinstance(old_ai, dict) else None
|
||||
legacy_binding = (
|
||||
isinstance(old_runner, str)
|
||||
and bool(old_runner)
|
||||
or isinstance(old_runner, dict)
|
||||
and bool(old_runner.get('runner'))
|
||||
and not old_runner.get('id')
|
||||
)
|
||||
new_config = pipeline_data['config']
|
||||
new_ai = new_config.get('ai', {}) if isinstance(new_config, dict) else {}
|
||||
new_runner = new_ai.get('runner') if isinstance(new_ai, dict) else None
|
||||
if legacy_binding and isinstance(new_runner, dict) and new_runner.get('id'):
|
||||
raise ValueError('manual_migration_required')
|
||||
pipeline_data['config'] = restore_secret_placeholders(
|
||||
pipeline_data['config'],
|
||||
current_config if current_config is not None else {},
|
||||
|
||||
@@ -0,0 +1,685 @@
|
||||
"""Explicit, tenant-scoped migration. Preview never contacts plugin runtimes.
|
||||
|
||||
The task handle is process-local. Original snapshots and pending activation are
|
||||
persistent; a refreshed preview is required after observation loss or restart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from ..authz import Permission, permissions_for_role, require_permission
|
||||
from ..context import ExecutionContext, PrincipalType, RequestContext
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....agent.runner import config_schema
|
||||
from ....agent.runner.resource_builder import AgentResourceBuilder
|
||||
from ....agent.runner.resource_policy import ResourcePolicyProjector
|
||||
from ....agent.runner.model_reasoning import extract_model_reasoning_overrides
|
||||
from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel
|
||||
from ....entity.persistence.rag import KnowledgeBase
|
||||
from ....entity.persistence.mcp import MCPServer
|
||||
from ....core.taskmgr import TaskContext
|
||||
from ....entity.persistence.pipeline import LegacyPipeline
|
||||
from ....entity.persistence.agent_interaction import AgentInteraction
|
||||
from ....persistence.pipeline_admission import lock_pipeline_admission
|
||||
from ....entity.persistence.pipeline_migration import PipelineMigrationSnapshot
|
||||
from ....entity.persistence.plugin import PluginSetting
|
||||
from ....entity.persistence.user import User
|
||||
from ....entity.persistence.workspace import Workspace, WorkspaceMembership, WorkspaceExecutionState
|
||||
from ....pipeline.legacy_config_migration import PLANNER_VERSION, plan_legacy_pipeline
|
||||
|
||||
|
||||
class MigrationError(ValueError):
|
||||
"""Only constant, credential-free codes cross the API/task boundary."""
|
||||
|
||||
def __init__(self, code: str, status_code: int = 409):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def validate_execute_request(body) -> list[dict]:
|
||||
if not isinstance(body, dict) or set(body) != {'confirmed', 'items'} or body['confirmed'] is not True:
|
||||
raise MigrationError('confirmation_required', 400)
|
||||
items = body['items']
|
||||
if not isinstance(items, list) or not 1 <= len(items) <= 50:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
seen = set()
|
||||
for item in items:
|
||||
if not isinstance(item, dict) or set(item) != {'pipeline_uuid', 'preview_token'}:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
for key, maximum in [('pipeline_uuid', 255), ('preview_token', 256)]:
|
||||
value = item[key]
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum or value != value.strip():
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
if item['pipeline_uuid'] in seen:
|
||||
raise MigrationError('invalid_selection', 400)
|
||||
seen.add(item['pipeline_uuid'])
|
||||
return copy.deepcopy(items)
|
||||
|
||||
|
||||
def _json(value):
|
||||
return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _fingerprint(row):
|
||||
# SQL timestamps are neither monotonic revisions nor portable CAS values.
|
||||
return hashlib.sha256(
|
||||
_json(
|
||||
{k: v for k, v in row.items() if not k.startswith('_') and k not in ('created_at', 'updated_at')}
|
||||
).encode()
|
||||
).hexdigest()
|
||||
|
||||
|
||||
class PipelineMigrationService:
|
||||
def __init__(self, ap):
|
||||
self.ap = ap
|
||||
self.pm = ap.persistence_mgr
|
||||
# Loss of this process key only invalidates previews, never snapshots.
|
||||
self._token_key = secrets.token_bytes(32)
|
||||
|
||||
async def _binding(self, ctx, session=None):
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
ctx.workspace_uuid, expected_generation=ctx.placement_generation, session=session
|
||||
)
|
||||
if binding.instance_uuid != ctx.instance_uuid:
|
||||
raise MigrationError('placement_changed')
|
||||
|
||||
async def _authorize(self, ctx, session=None):
|
||||
require_permission(ctx, Permission.RESOURCE_MANAGE)
|
||||
if ctx.principal.principal_type != PrincipalType.ACCOUNT or not ctx.account_uuid:
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
status = (await self.pm.execute_async(sa.select(User.status).where(User.uuid == ctx.account_uuid))).scalar()
|
||||
if status != 'active':
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
ctx.account_uuid, ctx.workspace_uuid, session=session
|
||||
)
|
||||
if (
|
||||
access.workspace.uuid != ctx.workspace_uuid
|
||||
or access.membership.uuid != ctx.workspace.membership_uuid
|
||||
or access.membership.projection_revision != ctx.workspace.membership_revision
|
||||
or Permission.RESOURCE_MANAGE.value not in permissions_for_role(access.membership.role)
|
||||
or access.execution.instance_uuid != ctx.instance_uuid
|
||||
or access.execution.placement_generation != ctx.placement_generation
|
||||
):
|
||||
raise MigrationError('authorization_changed', 403)
|
||||
await self._binding(ctx, session)
|
||||
|
||||
async def _rows(self, ctx, pipeline_uuid=None):
|
||||
# Read DB-native text for JSON CAS: never compare PostgreSQL JSON using =,
|
||||
# and never assume Python's JSON whitespace/key order matches stored text.
|
||||
table = LegacyPipeline.__table__
|
||||
stmt = sa.select(
|
||||
table,
|
||||
*[sa.cast(table.c[k], sa.Text).label('_raw_' + k) for k in ('config', 'stages', 'extensions_preferences')],
|
||||
)
|
||||
stmt = stmt.where(table.c.workspace_uuid == ctx.workspace_uuid)
|
||||
if pipeline_uuid is not None:
|
||||
stmt = stmt.where(table.c.uuid == pipeline_uuid)
|
||||
return [dict(r._mapping) for r in (await self.pm.execute_async(stmt.order_by(table.c.uuid))).all()]
|
||||
|
||||
async def _plugin(self, ctx, plan):
|
||||
target = plan.get('target_plugin')
|
||||
if not target:
|
||||
return None
|
||||
t = PluginSetting.__table__
|
||||
row = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(
|
||||
t.c.enabled, t.c.installation_uuid, t.c.artifact_digest, t.c.runtime_revision, t.c.install_info
|
||||
).where(
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.plugin_author == target['author'],
|
||||
t.c.plugin_name == target['name'],
|
||||
)
|
||||
)
|
||||
).first()
|
||||
return dict(row._mapping) if row else None
|
||||
|
||||
def _native_interactions(self, ctx, pipeline_uuid):
|
||||
"""Inspect the old in-process cache without importing/pruning it.
|
||||
|
||||
Native forms have no durable handoff to the plugin runner. Operators
|
||||
must finish them in the original runtime before migrating; never infer
|
||||
completion from the cache TTL or submit/cancel a form here.
|
||||
"""
|
||||
module = sys.modules.get('langbot.pkg.provider.runners.difysvapi')
|
||||
if module is None:
|
||||
return []
|
||||
cache = getattr(module, '_PENDING_FORMS', None)
|
||||
if not isinstance(cache, dict):
|
||||
return ['unavailable']
|
||||
observed = []
|
||||
for key, forms in cache.items():
|
||||
if not isinstance(key, tuple) or len(key) != 8:
|
||||
if forms:
|
||||
observed.append('unscoped')
|
||||
continue
|
||||
instance, workspace, _, _, pipeline, *_ = key
|
||||
# Inspect ownership before touching any form contents. Empty scope
|
||||
# cannot prove this is another tenant; block without disclosing it.
|
||||
if instance and instance != ctx.instance_uuid or workspace and workspace != ctx.workspace_uuid:
|
||||
continue
|
||||
if pipeline and pipeline != pipeline_uuid:
|
||||
continue
|
||||
if forms:
|
||||
observed.append(hashlib.sha256(_json(key).encode()).hexdigest())
|
||||
return sorted(observed)
|
||||
|
||||
async def _interaction_state(self, ctx, pipeline_uuid):
|
||||
"""Read only correlation/status, never request/submission/token data.
|
||||
|
||||
The real table has workspace_id (not workspace_uuid), and no instance
|
||||
column. Execution placement is verified by the caller; processor_id is
|
||||
the globally unique pipeline UUID. NULL ownership on that exact UUID
|
||||
is uncertainty, not permission to query other Workspaces.
|
||||
"""
|
||||
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
|
||||
return {'boundary': 'unavailable'}, True
|
||||
t = AgentInteraction.__table__
|
||||
try:
|
||||
records = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(t.c.id, t.c.status, t.c.updated_at, t.c.workspace_id)
|
||||
.where(
|
||||
t.c.processor_type == 'pipeline',
|
||||
t.c.processor_id == pipeline_uuid,
|
||||
sa.or_(
|
||||
t.c.workspace_id == ctx.workspace_uuid, t.c.workspace_id.is_(None), t.c.workspace_id == ''
|
||||
),
|
||||
)
|
||||
.order_by(t.c.id)
|
||||
)
|
||||
).all()
|
||||
durable = [tuple(r) for r in records]
|
||||
blocked = any(r.status not in {'submitted', 'cancelled', 'expired', 'delivery_failed'} for r in records)
|
||||
except Exception:
|
||||
# Missing schema, permissions, or a failed store must not be treated
|
||||
# as an empty store. No original diagnostic crosses the API.
|
||||
durable, blocked = ['unavailable'], True
|
||||
native = self._native_interactions(ctx, pipeline_uuid)
|
||||
return {'durable': durable, 'native': native}, blocked or bool(native)
|
||||
|
||||
async def _plan(self, ctx, row):
|
||||
plan = plan_legacy_pipeline(row['config'], row['extensions_preferences'])
|
||||
pending = None
|
||||
if plan['state'] == 'already_current':
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
snapshots = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(t).where(
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.pipeline_uuid == row['uuid'],
|
||||
t.c.state == 'activation_pending',
|
||||
t.c.target_fingerprint == _fingerprint(row),
|
||||
t.c.planner_version == str(PLANNER_VERSION),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
if snapshots:
|
||||
pending = dict(snapshots[0]._mapping)
|
||||
source = pending['source_snapshot']
|
||||
plan = plan_legacy_pipeline(source['config'], source['extensions_preferences'])
|
||||
candidate = {**row, 'config': plan.get('config')}
|
||||
if plan['state'] != 'ready' or _fingerprint(candidate) != _fingerprint(row):
|
||||
raise MigrationError('planner_changed')
|
||||
facts = await self._plugin(ctx, plan)
|
||||
state = 'activation_pending' if pending else plan['state']
|
||||
blockers = list(plan['blockers'])
|
||||
if state in ('ready', 'activation_pending'):
|
||||
if facts is None:
|
||||
state = 'needs_plugin'
|
||||
blockers.append({'code': 'plugin_missing'})
|
||||
elif facts['enabled'] is not True:
|
||||
state = 'blocked'
|
||||
blockers.append({'code': 'plugin_disabled'})
|
||||
if plan['state'] == 'ready':
|
||||
observation, interaction_blocked = await self._interaction_state(ctx, row['uuid'])
|
||||
plan['_interaction_state'] = observation
|
||||
if interaction_blocked:
|
||||
state = 'blocked'
|
||||
blockers.append({'code': 'runtime.pending_interaction'})
|
||||
return plan, facts, pending, state, blockers
|
||||
|
||||
def _token(self, ctx, row, plan, facts, pending):
|
||||
claim = [
|
||||
ctx.instance_uuid,
|
||||
ctx.workspace_uuid,
|
||||
ctx.placement_generation,
|
||||
ctx.account_uuid,
|
||||
ctx.workspace.membership_uuid,
|
||||
ctx.workspace.membership_revision,
|
||||
str(PLANNER_VERSION),
|
||||
_fingerprint(row),
|
||||
plan,
|
||||
facts,
|
||||
pending['uuid'] if pending else None,
|
||||
]
|
||||
# An authenticated opaque digest discloses neither credentials nor configs.
|
||||
return hmac.new(self._token_key, _json(claim).encode(), hashlib.sha256).hexdigest()
|
||||
|
||||
async def preview(self, ctx: RequestContext):
|
||||
require_permission(ctx, Permission.RESOURCE_VIEW)
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._binding(ctx)
|
||||
items = []
|
||||
for row in await self._rows(ctx):
|
||||
plan, facts, pending, state, blockers = await self._plan(ctx, row)
|
||||
item = {
|
||||
k: copy.deepcopy(plan[k])
|
||||
for k in ('legacy_runner', 'target_runner_id', 'target_plugin', 'changed_paths', 'warnings')
|
||||
}
|
||||
item.update(
|
||||
pipeline_uuid=row['uuid'],
|
||||
name=row['name'],
|
||||
state=state,
|
||||
blockers=blockers,
|
||||
preview_token=self._token(ctx, row, plan, facts, pending)
|
||||
if state in ('ready', 'activation_pending')
|
||||
else None,
|
||||
)
|
||||
items.append(item)
|
||||
return {
|
||||
'planner_version': str(PLANNER_VERSION),
|
||||
'workspace_uuid': ctx.workspace_uuid,
|
||||
'items': items,
|
||||
'total': len(items),
|
||||
}
|
||||
|
||||
async def _selected(self, ctx, selection):
|
||||
rows = await self._rows(ctx, selection['pipeline_uuid'])
|
||||
if not rows:
|
||||
raise MigrationError('pipeline_not_found', 404)
|
||||
row = rows[0]
|
||||
plan, facts, pending, state, _ = await self._plan(ctx, row)
|
||||
token = self._token(ctx, row, plan, facts, pending)
|
||||
if not hmac.compare_digest(token, selection['preview_token']):
|
||||
raise MigrationError('preview_stale')
|
||||
if state not in ('ready', 'activation_pending'):
|
||||
raise MigrationError('migration_blocked')
|
||||
return row, plan, facts, pending
|
||||
|
||||
async def execute(self, ctx: RequestContext, body):
|
||||
require_permission(ctx, Permission.RESOURCE_MANAGE)
|
||||
items = validate_execute_request(body)
|
||||
async with self.pm.tenant_scope(ctx.workspace_uuid):
|
||||
await self._authorize(ctx)
|
||||
# Preflight the entire selection before creating tasks or contacting runtimes.
|
||||
for selection in items:
|
||||
await self._selected(ctx, selection)
|
||||
task_context = TaskContext.new()
|
||||
task_context.metadata = {
|
||||
'kind': 'pipeline_migration',
|
||||
'results': [{'pipeline_uuid': i['pipeline_uuid'], 'state': 'pending', 'code': None} for i in items],
|
||||
}
|
||||
task = self.ap.task_mgr.create_user_task(
|
||||
self._run(ctx, ExecutionContext.from_request(ctx), items, task_context),
|
||||
kind='pipeline_migration',
|
||||
name='pipeline_migration',
|
||||
context=task_context,
|
||||
instance_uuid=ctx.instance_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
placement_generation=ctx.placement_generation,
|
||||
)
|
||||
return {'task_id': task.id}
|
||||
|
||||
async def _verify_runtime(self, execution, plan):
|
||||
runners = await self.ap.runner_registry.list_runners(execution, use_cache=False, usage='agent')
|
||||
descriptor = next((r for r in runners if r.id == plan['target_runner_id']), None)
|
||||
if descriptor is None or 'agent' not in descriptor.usages:
|
||||
raise MigrationError('runner_unavailable')
|
||||
if descriptor.plugin_version != plan['target_plugin']['version']:
|
||||
raise MigrationError('plugin_version_incompatible')
|
||||
config = plan['config']
|
||||
RunnerConfigResolver.validate_pipeline_config(config)
|
||||
runner_config = config['ai']['runner_config'][plan['target_runner_id']]
|
||||
schema = {item['name']: item for item in descriptor.config_schema}
|
||||
# Host fields have their own scope/capability checks below; plugin
|
||||
# schema defaults must never implicitly authorize a foreign resource.
|
||||
host_fields = {
|
||||
'enable-all-tools': 'boolean',
|
||||
'tools': 'array',
|
||||
'knowledge-bases': 'knowledge-base-multi-selector',
|
||||
'mcp-resources': 'array',
|
||||
'mcp-resource-agent-read-enabled': 'boolean',
|
||||
}
|
||||
for name, value in runner_config.items():
|
||||
field = schema.get(name)
|
||||
if field is None and name in host_fields:
|
||||
field = {'type': host_fields[name]}
|
||||
if field is None:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if value is None and field.get('nullable') is True:
|
||||
continue
|
||||
kind = config_schema.normalize_schema_item_type(field.get('type'))
|
||||
valid = (
|
||||
(
|
||||
kind
|
||||
in (
|
||||
'string',
|
||||
'text',
|
||||
'password',
|
||||
'secret',
|
||||
'select',
|
||||
'llm-model-selector',
|
||||
'embedding-model-selector',
|
||||
'rerank-model-selector',
|
||||
)
|
||||
and isinstance(value, str)
|
||||
)
|
||||
or (kind == 'boolean' and type(value) is bool)
|
||||
or (kind in ('integer', 'int') and type(value) is int)
|
||||
or (kind in ('float', 'number') and type(value) in (int, float))
|
||||
or (
|
||||
kind
|
||||
in (
|
||||
'array',
|
||||
'multi-select',
|
||||
'knowledge-base-selector',
|
||||
'knowledge-base-multi-selector',
|
||||
'prompt-editor',
|
||||
)
|
||||
and isinstance(value, list)
|
||||
)
|
||||
or (kind in ('object', 'json') and type(value) is dict)
|
||||
or (kind == 'array[string]' and type(value) is list and all(type(v) is str for v in value))
|
||||
or (
|
||||
kind == 'model-fallback-selector'
|
||||
and (
|
||||
isinstance(value, str)
|
||||
or isinstance(value, dict)
|
||||
and set(value) <= {'primary', 'fallbacks', 'reasoning'}
|
||||
and isinstance(value.get('primary'), str)
|
||||
and isinstance(value.get('fallbacks', []), list)
|
||||
and all(isinstance(v, str) for v in value.get('fallbacks', []))
|
||||
and isinstance(value.get('reasoning', {}), dict)
|
||||
)
|
||||
)
|
||||
)
|
||||
if not valid:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if kind == 'select' and field.get('options'):
|
||||
if value not in [o.get('value', o.get('name')) for o in field['options']]:
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
if any(
|
||||
f.get('required') and name not in runner_config and f.get('default') is None for name, f in schema.items()
|
||||
):
|
||||
raise MigrationError('runner_schema_incompatible')
|
||||
await self._verify_resources(execution, descriptor, runner_config)
|
||||
return _json(
|
||||
[
|
||||
descriptor.id,
|
||||
descriptor.plugin_version,
|
||||
descriptor.usages,
|
||||
descriptor.config_schema,
|
||||
getattr(descriptor, 'capabilities', None),
|
||||
getattr(descriptor, 'permissions', None),
|
||||
]
|
||||
)
|
||||
|
||||
async def _verify_resources(self, execution, descriptor, runner_config):
|
||||
"""Fail closed on references the scoped Host cannot authorize.
|
||||
|
||||
This runs only on explicit execution, outside the mutation transaction.
|
||||
Resource use is still reauthorized by the Host on every eventual run.
|
||||
"""
|
||||
|
||||
async def require_row(model, identifier, *, column=None):
|
||||
column = model.uuid if column is None else column
|
||||
found = (
|
||||
await self.pm.execute_async(
|
||||
sa.select(model.uuid).where(
|
||||
model.workspace_uuid == execution.workspace_uuid,
|
||||
column == identifier,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if found is None:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
permissions = getattr(descriptor, 'permissions', None)
|
||||
model_resources = []
|
||||
for model_type, model_uuid in config_schema.iter_config_model_refs(descriptor, runner_config):
|
||||
allowed = set(getattr(permissions, 'models', []))
|
||||
if not (allowed & ({'rerank'} if model_type == 'rerank' else {'invoke', 'stream'})):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
await require_row(RerankModel if model_type == 'rerank' else LLMModel, model_uuid)
|
||||
model_resources.append({'model_id': model_uuid})
|
||||
for field in config_schema.iter_schema_items(descriptor, {'embedding-model-selector'}):
|
||||
value = runner_config.get(field['name'])
|
||||
if value and value not in config_schema.NONE_SENTINELS:
|
||||
await require_row(EmbeddingModel, value)
|
||||
extract_model_reasoning_overrides(descriptor, runner_config, {'models': model_resources})
|
||||
|
||||
kb_ids = runner_config.get('knowledge-bases', [])
|
||||
if not isinstance(kb_ids, list) or any(not isinstance(v, str) or not v for v in kb_ids):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
if kb_ids:
|
||||
if not config_schema.uses_host_knowledge_bases(descriptor) or 'retrieve' not in getattr(
|
||||
permissions, 'knowledge_bases', []
|
||||
):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
for kb_id in kb_ids:
|
||||
await require_row(KnowledgeBase, kb_id)
|
||||
|
||||
tool_names = runner_config.get('tools', [])
|
||||
if not isinstance(tool_names, list) or any(not isinstance(v, str) or not v for v in tool_names):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
tool_grant = (
|
||||
runner_config.get('enable-all-tools', False)
|
||||
or tool_names
|
||||
or runner_config.get('mcp-resource-agent-read-enabled', False)
|
||||
)
|
||||
if tool_grant:
|
||||
if not config_schema.uses_host_tools(descriptor) or 'call' not in getattr(permissions, 'tools', []):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
policy = ResourcePolicyProjector.from_runner_config(runner_config)
|
||||
tools = await AgentResourceBuilder(self.ap)._build_tools_from_binding(
|
||||
execution,
|
||||
permissions,
|
||||
policy,
|
||||
descriptor,
|
||||
runner_config,
|
||||
)
|
||||
if not set(tool_names) <= {tool['tool_name'] for tool in tools}:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
for attachment in runner_config.get('mcp-resources', []):
|
||||
if not isinstance(attachment, dict):
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
identifier = attachment.get('server_uuid') or attachment.get('server_id')
|
||||
name = attachment.get('server_name')
|
||||
if identifier:
|
||||
await require_row(MCPServer, identifier)
|
||||
elif name:
|
||||
await require_row(MCPServer, name, column=MCPServer.name)
|
||||
else:
|
||||
raise MigrationError('runner_resource_unavailable')
|
||||
|
||||
async def _lock(self, ctx, pipeline_uuid, session):
|
||||
# Every pending producer shares this row and revalidates its captured
|
||||
# configuration/conversation authority after acquiring admission.
|
||||
if self.pm.get_db_engine().dialect.name not in {'sqlite', 'postgresql'}:
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
await lock_pipeline_admission(session, ctx.workspace_uuid, pipeline_uuid)
|
||||
# Hold revocable directory rows through commit (no network in this UoW).
|
||||
await session.execute(sa.select(User.uuid).where(User.uuid == ctx.account_uuid).with_for_update())
|
||||
for model, condition in [
|
||||
(Workspace, Workspace.uuid == ctx.workspace_uuid),
|
||||
(WorkspaceExecutionState, WorkspaceExecutionState.workspace_uuid == ctx.workspace_uuid),
|
||||
(
|
||||
WorkspaceMembership,
|
||||
sa.and_(
|
||||
WorkspaceMembership.workspace_uuid == ctx.workspace_uuid,
|
||||
WorkspaceMembership.account_uuid == ctx.account_uuid,
|
||||
),
|
||||
),
|
||||
(PluginSetting, PluginSetting.workspace_uuid == ctx.workspace_uuid),
|
||||
]:
|
||||
await session.execute(sa.select(model.__table__).where(condition).with_for_update())
|
||||
|
||||
async def _commit(self, ctx, selection, expected_facts, candidate):
|
||||
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
|
||||
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
|
||||
await self._authorize(ctx, uow.session)
|
||||
row, plan, facts, pending = await self._selected(ctx, selection)
|
||||
if facts != expected_facts:
|
||||
raise MigrationError('plugin_changed')
|
||||
if pending:
|
||||
return pending['uuid'], pending['target_fingerprint']
|
||||
target = {**row, 'config': candidate['config']}
|
||||
t = LegacyPipeline.__table__
|
||||
cas = [t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == row['uuid']]
|
||||
for key in ('config', 'stages', 'extensions_preferences'):
|
||||
cas.append(sa.cast(t.c[key], sa.Text) == row['_raw_' + key])
|
||||
result = await self.pm.execute_async(sa.update(t).where(*cas).values(config=candidate['config']))
|
||||
if result.rowcount != 1:
|
||||
raise MigrationError('preview_stale')
|
||||
snapshot_uuid = str(uuid.uuid4())
|
||||
target_fingerprint = _fingerprint(target)
|
||||
await self.pm.execute_async(
|
||||
sa.insert(PipelineMigrationSnapshot).values(
|
||||
uuid=snapshot_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
pipeline_uuid=row['uuid'],
|
||||
source_fingerprint=_fingerprint(row),
|
||||
target_fingerprint=target_fingerprint,
|
||||
source_snapshot=json.loads(_json({k: v for k, v in row.items() if not k.startswith('_')})),
|
||||
planner_version=str(PLANNER_VERSION),
|
||||
state='activation_pending',
|
||||
)
|
||||
)
|
||||
if self._native_interactions(ctx, row['uuid']):
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
return snapshot_uuid, target_fingerprint
|
||||
|
||||
async def _activate(self, ctx, selection, snapshot_uuid, target_fingerprint, candidate, plan, expected_facts):
|
||||
# Re-read the committed target under a short lock before synchronous
|
||||
# publication. A later editor must never be replaced by our old candidate.
|
||||
async with self.pm.tenant_uow(ctx.workspace_uuid) as uow:
|
||||
await self._lock(ctx, selection['pipeline_uuid'], uow.session)
|
||||
await self._authorize(ctx, uow.session)
|
||||
rows = await self._rows(ctx, selection['pipeline_uuid'])
|
||||
if not rows or _fingerprint(rows[0]) != target_fingerprint:
|
||||
raise MigrationError('activation_source_changed')
|
||||
if await self._plugin(ctx, plan) != expected_facts:
|
||||
raise MigrationError('plugin_changed')
|
||||
_, interaction_blocked = await self._interaction_state(ctx, selection['pipeline_uuid'])
|
||||
if interaction_blocked:
|
||||
raise MigrationError('runtime.pending_interaction')
|
||||
# Native cache inspection above is the last await before publication
|
||||
# and session invalidation; no native producer can interleave here.
|
||||
self.ap.pipeline_mgr.publish_pipeline(candidate)
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
conversation = session.using_conversation
|
||||
if (
|
||||
conversation is not None
|
||||
and conversation.pipeline_uuid == selection['pipeline_uuid']
|
||||
and getattr(session, 'workspace_uuid', None) == ctx.workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
await self.pm.execute_async(
|
||||
sa.update(t)
|
||||
.where(t.c.workspace_uuid == ctx.workspace_uuid, t.c.uuid == snapshot_uuid)
|
||||
.values(state='active')
|
||||
)
|
||||
|
||||
async def _run(self, ctx, execution, items, task_context):
|
||||
async with self.pm.tenant_scope(execution.workspace_uuid):
|
||||
for selection, result in zip(items, task_context.metadata['results']):
|
||||
committed = False
|
||||
commit_attempted = False
|
||||
try:
|
||||
await self._authorize(ctx)
|
||||
row, plan, facts, pending = await self._selected(ctx, selection)
|
||||
runtime_schema = await self._verify_runtime(execution, plan)
|
||||
candidate_entity = {k: copy.deepcopy(v) for k, v in row.items() if not k.startswith('_')}
|
||||
candidate_entity['config'] = copy.deepcopy(plan['config'])
|
||||
runtime = await self.ap.pipeline_mgr.prepare_pipeline(execution, copy.deepcopy(candidate_entity))
|
||||
if await self._verify_runtime(execution, plan) != runtime_schema:
|
||||
raise MigrationError('runner_schema_changed')
|
||||
# Runtime awaits are over. Recheck authorization/source/plugin
|
||||
# facts under database locks, then atomically journal and CAS.
|
||||
commit_attempted = True
|
||||
snapshot_uuid, target_fingerprint = await self._commit(ctx, selection, facts, candidate_entity)
|
||||
committed = True
|
||||
await self._activate(ctx, selection, snapshot_uuid, target_fingerprint, runtime, plan, facts)
|
||||
result.update(state='migrated', code=None)
|
||||
except (Exception, asyncio.CancelledError) as exc:
|
||||
cancelled = isinstance(exc, asyncio.CancelledError)
|
||||
reconciliation_cancel = None
|
||||
code = (
|
||||
'operation_cancelled'
|
||||
if cancelled
|
||||
else exc.code
|
||||
if isinstance(exc, MigrationError)
|
||||
else 'migration_failed'
|
||||
)
|
||||
if commit_attempted and not committed and not isinstance(exc, MigrationError):
|
||||
# A commit can succeed while acknowledgement/connection
|
||||
# cleanup fails, including cancellation. Make one read
|
||||
# after the UoW has unwound, in this task's tenant scope;
|
||||
# never shield/retry through another cancellation.
|
||||
try:
|
||||
t = PipelineMigrationSnapshot.__table__
|
||||
conditions = [
|
||||
t.c.workspace_uuid == ctx.workspace_uuid,
|
||||
t.c.pipeline_uuid == selection['pipeline_uuid'],
|
||||
t.c.source_fingerprint
|
||||
== (pending['source_fingerprint'] if pending else _fingerprint(row)),
|
||||
t.c.target_fingerprint == _fingerprint(candidate_entity),
|
||||
t.c.planner_version == str(PLANNER_VERSION),
|
||||
]
|
||||
if pending:
|
||||
# A retry starts from the target, not the legacy
|
||||
# source. Only the original snapshot proves this
|
||||
# durable migration; an unrelated journal cannot.
|
||||
conditions.extend(
|
||||
[
|
||||
t.c.uuid == pending['uuid'],
|
||||
t.c.target_fingerprint == pending['target_fingerprint'],
|
||||
]
|
||||
)
|
||||
found = (await self.pm.execute_async(sa.select(t.c.uuid).where(*conditions))).first()
|
||||
committed = found is not None
|
||||
except asyncio.CancelledError as cancel_exc:
|
||||
reconciliation_cancel = cancel_exc
|
||||
cancelled = True
|
||||
code = 'commit_outcome_unknown'
|
||||
except Exception:
|
||||
code = 'commit_outcome_unknown'
|
||||
state = (
|
||||
# Unknown is deliberately recovery-needed, not a claim
|
||||
# of rollback or proof that a snapshot was committed.
|
||||
'activation_pending'
|
||||
if committed or code == 'commit_outcome_unknown'
|
||||
else 'stale'
|
||||
if code == 'preview_stale'
|
||||
else 'blocked'
|
||||
if isinstance(exc, MigrationError)
|
||||
else 'failed'
|
||||
)
|
||||
result.update(state=state, code=code)
|
||||
if cancelled:
|
||||
for unstarted in task_context.metadata['results']:
|
||||
if unstarted['state'] == 'pending':
|
||||
unstarted.update(state='failed', code='operation_cancelled')
|
||||
if reconciliation_cancel is not None:
|
||||
# Preserve cancellation propagation during recovery,
|
||||
# but record safe outcomes before leaving the task.
|
||||
raise reconciliation_cancel
|
||||
return task_context.metadata
|
||||
# Never attach the original exception, traceback or plugin
|
||||
# diagnostic to TaskContext: upstream errors can contain keys.
|
||||
return task_context.metadata
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Tenant-owned original configurations; never expose through public serialization."""
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PipelineMigrationSnapshot(Base):
|
||||
__tablename__ = 'pipeline_migration_snapshots'
|
||||
|
||||
uuid = sa.Column(sa.String(36), primary_key=True)
|
||||
workspace_uuid = sa.Column(sa.String(36), sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), nullable=False)
|
||||
pipeline_uuid = sa.Column(sa.String(255), nullable=False)
|
||||
source_fingerprint = sa.Column(sa.String(64), nullable=False)
|
||||
target_fingerprint = sa.Column(sa.String(64), nullable=False)
|
||||
planner_version = sa.Column(sa.String(64), nullable=False)
|
||||
source_snapshot = sa.Column(sa.JSON, nullable=False)
|
||||
state = sa.Column(sa.String(32), nullable=False)
|
||||
created_at = sa.Column(sa.DateTime, nullable=False, server_default=sa.func.now())
|
||||
|
||||
__table_args__ = (
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'pipeline_uuid'],
|
||||
['legacy_pipelines.workspace_uuid', 'legacy_pipelines.uuid'],
|
||||
name='fk_pipeline_migration_workspace_pipeline',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
'workspace_uuid', 'pipeline_uuid', 'source_fingerprint', name='uq_pipeline_migration_source'
|
||||
),
|
||||
sa.CheckConstraint("state IN ('activation_pending', 'active')", name='ck_pipeline_migration_state'),
|
||||
sa.Index('ix_pipeline_migration_workspace_pipeline', 'workspace_uuid', 'pipeline_uuid'),
|
||||
)
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Create the manual migration journal only; never convert pipeline JSON."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = '0027_pipeline_migration'
|
||||
down_revision = '0026_merge_master_beta'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
table = 'pipeline_migration_snapshots'
|
||||
if not sa.inspect(bind).has_table(table):
|
||||
op.create_table(
|
||||
table,
|
||||
sa.Column('uuid', sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
'workspace_uuid', sa.String(36), sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), nullable=False
|
||||
),
|
||||
sa.Column('pipeline_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('source_fingerprint', sa.String(64), nullable=False),
|
||||
sa.Column('target_fingerprint', sa.String(64), nullable=False),
|
||||
sa.Column('planner_version', sa.String(64), nullable=False),
|
||||
sa.Column('source_snapshot', sa.JSON(), nullable=False),
|
||||
sa.Column('state', sa.String(32), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'pipeline_uuid'],
|
||||
['legacy_pipelines.workspace_uuid', 'legacy_pipelines.uuid'],
|
||||
name='fk_pipeline_migration_workspace_pipeline',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
'workspace_uuid', 'pipeline_uuid', 'source_fingerprint', name='uq_pipeline_migration_source'
|
||||
),
|
||||
sa.CheckConstraint("state IN ('activation_pending', 'active')", name='ck_pipeline_migration_state'),
|
||||
)
|
||||
op.create_index('ix_pipeline_migration_workspace_pipeline', table, ['workspace_uuid', 'pipeline_uuid'])
|
||||
if bind.dialect.name == 'postgresql':
|
||||
op.execute(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY')
|
||||
op.execute(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY')
|
||||
op.execute(f'DROP POLICY IF EXISTS langbot_workspace_isolation ON {table}')
|
||||
expression = "workspace_uuid = NULLIF(current_setting('langbot.workspace_uuid', true), '')"
|
||||
op.execute(
|
||||
f'CREATE POLICY langbot_workspace_isolation ON {table} FOR ALL TO PUBLIC '
|
||||
f'USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('pipeline_migration_snapshots')
|
||||
@@ -71,6 +71,7 @@ _ALEMBIC_TENANT_TABLES = {
|
||||
'rerank_models',
|
||||
'legacy_pipelines',
|
||||
'pipeline_run_records',
|
||||
'pipeline_migration_snapshots',
|
||||
'plugin_settings',
|
||||
'knowledge_bases',
|
||||
'knowledge_base_files',
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Transaction-held, tenant/pipeline-scoped admission for form producers.
|
||||
|
||||
Every pending producer must hold this row until its INSERT commits. Migration
|
||||
holds the same row for both CAS and activation. No table/process/advisory lock.
|
||||
The no-op UPDATE also acquires SQLite's write reservation before any reads.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from ..entity.persistence.pipeline import LegacyPipeline
|
||||
|
||||
|
||||
async def lock_pipeline_admission(session, workspace_uuid: str, pipeline_uuid: str):
|
||||
if not workspace_uuid or not pipeline_uuid:
|
||||
raise ValueError('Pipeline admission requires exact ownership')
|
||||
t = LegacyPipeline.__table__
|
||||
result = await session.execute(
|
||||
sa.update(t)
|
||||
.where(t.c.workspace_uuid == workspace_uuid, t.c.uuid == pipeline_uuid)
|
||||
.values(updated_at=t.c.updated_at)
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
raise ValueError('Pipeline admission authority unavailable')
|
||||
return (
|
||||
await session.execute(
|
||||
sa.select(t.c.config).where(t.c.workspace_uuid == workspace_uuid, t.c.uuid == pipeline_uuid)
|
||||
)
|
||||
).scalar_one()
|
||||
@@ -58,6 +58,7 @@ TENANT_TABLE_COLUMNS: dict[str, str] = {
|
||||
'rerank_models': 'workspace_uuid',
|
||||
'legacy_pipelines': 'workspace_uuid',
|
||||
'pipeline_run_records': 'workspace_uuid',
|
||||
'pipeline_migration_snapshots': 'workspace_uuid',
|
||||
'plugin_settings': 'workspace_uuid',
|
||||
'knowledge_bases': 'workspace_uuid',
|
||||
'knowledge_base_files': 'workspace_uuid',
|
||||
@@ -467,9 +468,21 @@ def _validate_scoped_statement_call(args: tuple[typing.Any, ...], kwargs: dict[s
|
||||
raise ScopedSessionTransactionError('TenantUnitOfWork does not allow literal-execute SQL parameters')
|
||||
|
||||
if isinstance(element, sqlalchemy.sql.elements.Cast) and type(element.type) not in {Vector, HALFVEC}:
|
||||
raise ScopedSessionTransactionError(
|
||||
'TenantUnitOfWork only allows the trusted pgvector cast used by tenant vector search'
|
||||
# Manual Pipeline migration compares DB-native JSON text, because
|
||||
# PostgreSQL JSON has no equality operator. Permit only these fixed
|
||||
# mapped columns and a built-in Text target, never arbitrary casts.
|
||||
source = element.clause
|
||||
pipeline_json_cas = (
|
||||
type(element.type) is sqlalchemy.Text
|
||||
and isinstance(source, sqlalchemy.Column)
|
||||
and type(source.type) is sqlalchemy.JSON
|
||||
and getattr(getattr(source, 'table', None), 'name', None) == 'legacy_pipelines'
|
||||
and source.name in {'config', 'stages', 'extensions_preferences'}
|
||||
)
|
||||
if not pipeline_json_cas:
|
||||
raise ScopedSessionTransactionError(
|
||||
'TenantUnitOfWork only allows trusted pgvector and Pipeline JSON CAS casts'
|
||||
)
|
||||
|
||||
if isinstance(element, sqlalchemy.sql.functions.FunctionElement):
|
||||
function_name = str(getattr(element, 'name', '')).casefold()
|
||||
|
||||
@@ -0,0 +1,692 @@
|
||||
"""Pure planning for explicitly requested legacy Pipeline migrations.
|
||||
|
||||
The candidate config is internal-only: never serialize it in a preview or log
|
||||
it. ``ready`` means a config candidate, NOT permission to execute: installation,
|
||||
resource ownership and existing conversation-state checks belong to the caller.
|
||||
No IO, registry access, defaults from UI schemas, or runtime mutation occurs here.
|
||||
|
||||
Reviewed native source: 9b7ba0d64708496ace30a82866f6dbc185f089dc.
|
||||
Reviewed official plugins: local parity patches over 678b18fd65af98230806c3b97b649514c83acea9.
|
||||
Patch manifest versions below are required; version alone is not artifact proof.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
PLANNER_VERSION = '3'
|
||||
|
||||
_TARGETS = {
|
||||
'local-agent': ('LocalAgent', '0.1.6'),
|
||||
'dify-service-api': ('DifyAgent', '0.1.7'),
|
||||
'coze-api': ('CozeAgent', '0.1.7'),
|
||||
'dashscope-app-api': ('DashScopeAgent', '0.1.7'),
|
||||
'n8n-service-api': ('N8nAgent', '0.1.7'),
|
||||
'langflow-api': ('LangflowAgent', '0.1.7'),
|
||||
'deerflow-api': ('DeerFlowAgent', '0.1.7'),
|
||||
'tbox-app-api': ('TboxAgent', '0.1.5'),
|
||||
'weknora-api': ('WeKnoraAgent', '0.1.7'),
|
||||
}
|
||||
_DEERFLOW_FIELDS = {
|
||||
'api-base',
|
||||
'api-key',
|
||||
'auth-header',
|
||||
'assistant-id',
|
||||
'model-name',
|
||||
'thinking-enabled',
|
||||
'plan-mode',
|
||||
'subagent-enabled',
|
||||
'max-concurrent-subagents',
|
||||
'timeout',
|
||||
'recursion-limit',
|
||||
}
|
||||
_FIELDS = {
|
||||
'local-agent': {
|
||||
'model',
|
||||
'max-round',
|
||||
'prompt',
|
||||
'box-session-id-template',
|
||||
'rerank-model',
|
||||
'rerank-top-k',
|
||||
'enable-all-tools',
|
||||
'tools',
|
||||
'knowledge-bases',
|
||||
'knowledge-base',
|
||||
'mcp-resources',
|
||||
'mcp-resource-agent-read-enabled',
|
||||
},
|
||||
'dify-service-api': {'base-url', 'api-key', 'app-type', 'base-prompt', 'timeout'},
|
||||
'coze-api': {'api-key', 'bot-id', 'api-base', 'timeout', 'auto-save-history', 'auto_save_history'},
|
||||
'dashscope-app-api': {'app-type', 'api-key', 'app-id', 'references_quote', 'references-quote'},
|
||||
'n8n-service-api': {
|
||||
'webhook-url',
|
||||
'auth-type',
|
||||
'basic-username',
|
||||
'basic-password',
|
||||
'jwt-secret',
|
||||
'jwt-algorithm',
|
||||
'header-name',
|
||||
'header-value',
|
||||
'timeout',
|
||||
'output-key',
|
||||
'response-handling',
|
||||
},
|
||||
'langflow-api': {
|
||||
'base-url',
|
||||
'api-key',
|
||||
'flow-id',
|
||||
'input-type',
|
||||
'output-type',
|
||||
'input_type',
|
||||
'output_type',
|
||||
'tweaks',
|
||||
},
|
||||
'deerflow-api': _DEERFLOW_FIELDS,
|
||||
'tbox-app-api': {'api-key', 'app-id'},
|
||||
'weknora-api': {
|
||||
'base-url',
|
||||
'api-key',
|
||||
'app-type',
|
||||
'agent-id',
|
||||
'knowledge-base-ids',
|
||||
'web-search-enabled',
|
||||
'timeout',
|
||||
'base-prompt',
|
||||
},
|
||||
}
|
||||
_REQUIRED = {
|
||||
'local-agent': ('model', 'prompt'),
|
||||
'dify-service-api': ('base-url', 'api-key', 'app-type', 'base-prompt'),
|
||||
'coze-api': ('api-key', 'bot-id', 'api-base', 'timeout'),
|
||||
'dashscope-app-api': ('app-type', 'api-key', 'app-id'),
|
||||
'n8n-service-api': ('webhook-url',),
|
||||
'langflow-api': ('base-url', 'api-key', 'flow-id'),
|
||||
'deerflow-api': ('api-base',),
|
||||
'tbox-app-api': ('api-key', 'app-id'),
|
||||
'weknora-api': ('base-url', 'api-key', 'app-type'),
|
||||
}
|
||||
_BOOLEANS = {
|
||||
'enable-all-tools',
|
||||
'mcp-resource-agent-read-enabled',
|
||||
'auto-save-history',
|
||||
'auto_save_history',
|
||||
'thinking-enabled',
|
||||
'plan-mode',
|
||||
'subagent-enabled',
|
||||
'web-search-enabled',
|
||||
}
|
||||
_INTEGERS = {'max-round', 'rerank-top-k', 'max-concurrent-subagents', 'recursion-limit'}
|
||||
_NAME_LISTS = {'tools', 'knowledge-bases', 'knowledge-base-ids'}
|
||||
_DEERFLOW_DEFAULTS = {
|
||||
'api-key': '',
|
||||
'auth-header': '',
|
||||
'assistant-id': 'lead_agent',
|
||||
'model-name': '',
|
||||
'thinking-enabled': False,
|
||||
'plan-mode': False,
|
||||
'subagent-enabled': False,
|
||||
'max-concurrent-subagents': 3,
|
||||
'timeout': 300,
|
||||
'recursion-limit': 1000,
|
||||
}
|
||||
# Reviewed local patch manifests and implementations explicitly preserve native
|
||||
# provider identity using trusted Host conversation fields; absent context fails
|
||||
# closed in the plugin. Caller must verify the installed artifact/Host contract.
|
||||
_IDENTITY_SOURCES = {
|
||||
'dify-service-api': 'legacy-session',
|
||||
'coze-api': 'legacy-session',
|
||||
'n8n-service-api': 'legacy-session',
|
||||
'tbox-app-api': 'legacy-bot',
|
||||
}
|
||||
_LOCAL_DEFAULTS = {
|
||||
'advanced-settings': False,
|
||||
'date-grounding': True,
|
||||
'timeout': 300,
|
||||
'retrieval-top-k': 5,
|
||||
'rerank-model': '',
|
||||
'rerank-top-k': 5,
|
||||
'max-tool-iterations': 100,
|
||||
'tool-execution-mode': 'serial',
|
||||
'max-tool-result-chars': 20000,
|
||||
'context-history-fetch-limit': 50,
|
||||
'context-window-tokens': 200000,
|
||||
'context-reserve-tokens': 16384,
|
||||
'context-keep-recent-tokens': 20000,
|
||||
'context-summary-tokens': 8000,
|
||||
'enable-all-tools': True,
|
||||
'tools': [],
|
||||
'knowledge-bases': [],
|
||||
}
|
||||
_DEFAULTS = {
|
||||
'local-agent': _LOCAL_DEFAULTS,
|
||||
'dify-service-api': {'timeout': 30, 'advanced-settings': False},
|
||||
'coze-api': {'auto-save-history': True, 'advanced-settings': False},
|
||||
'dashscope-app-api': {'references_quote': '参考资料来自:', 'timeout': 120, 'advanced-settings': False},
|
||||
'n8n-service-api': {
|
||||
'auth-type': 'none',
|
||||
'basic-username': '',
|
||||
'basic-password': '',
|
||||
'jwt-secret': '',
|
||||
'jwt-algorithm': 'HS256',
|
||||
'header-name': '',
|
||||
'header-value': '',
|
||||
'timeout': 120,
|
||||
'output-key': 'response',
|
||||
'response-handling': 'reply',
|
||||
'basic-encoding': 'latin1',
|
||||
'advanced-settings': False,
|
||||
},
|
||||
'langflow-api': {'input-type': 'chat', 'output-type': 'chat', 'tweaks': {}, 'advanced-settings': False},
|
||||
'deerflow-api': _DEERFLOW_DEFAULTS,
|
||||
'tbox-app-api': {'timeout': 120},
|
||||
'weknora-api': {
|
||||
'knowledge-base-ids': [],
|
||||
'web-search-enabled': False,
|
||||
'timeout': 120,
|
||||
'base-prompt': '请回答用户的问题。',
|
||||
'advanced-settings': False,
|
||||
},
|
||||
}
|
||||
_ASSET_RUNNERS = {'dify-service-api', 'coze-api', 'dashscope-app-api', 'n8n-service-api', 'langflow-api'}
|
||||
_REMOVE_THINK_RUNNERS = {'local-agent', 'dify-service-api', 'coze-api', 'dashscope-app-api', 'tbox-app-api'}
|
||||
_REASONING_LEVELS = {'provider_default', 'disabled', 'enabled', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max'}
|
||||
|
||||
|
||||
def _strict_json(value, ancestors=None, depth=0):
|
||||
"""Reject cycles, custom objects, nonfinite numbers and non-string keys."""
|
||||
if depth > 100:
|
||||
return False
|
||||
kind = type(value)
|
||||
if value is None or kind in (str, bool, int):
|
||||
return True
|
||||
if kind is float:
|
||||
return math.isfinite(value)
|
||||
if kind not in (dict, list):
|
||||
return False
|
||||
ancestors = set() if ancestors is None else ancestors
|
||||
identity = id(value)
|
||||
if identity in ancestors:
|
||||
return False
|
||||
ancestors.add(identity)
|
||||
try:
|
||||
if kind is dict:
|
||||
return all(type(k) is str and _strict_json(v, ancestors, depth + 1) for k, v in value.items())
|
||||
return all(_strict_json(v, ancestors, depth + 1) for v in value)
|
||||
finally:
|
||||
ancestors.remove(identity)
|
||||
|
||||
|
||||
def _block(result, code, field):
|
||||
"""Only developer-owned codes and paths may reach the public projection."""
|
||||
diagnostic = {'code': code, 'field': field}
|
||||
if diagnostic not in result['blockers']:
|
||||
result['blockers'].append(diagnostic)
|
||||
result['state'] = 'blocked'
|
||||
return result
|
||||
|
||||
|
||||
def _names(value):
|
||||
return type(value) is list and all(type(item) is str and bool(item) for item in value)
|
||||
|
||||
|
||||
def _attachments(value):
|
||||
return type(value) is list and all(
|
||||
type(item) is dict and ('enabled' not in item or type(item['enabled']) is bool) for item in value
|
||||
)
|
||||
|
||||
|
||||
def _validate_preferences(result, preferences, name):
|
||||
if preferences is None:
|
||||
return
|
||||
if type(preferences) is not dict or not _strict_json(preferences):
|
||||
_block(result, 'invalid_extension_preferences', 'extensions_preferences')
|
||||
return
|
||||
for field in (
|
||||
'enable_all_plugins',
|
||||
'enable_all_mcp_servers',
|
||||
'enable_all_skills',
|
||||
'mcp_resource_agent_read_enabled',
|
||||
):
|
||||
if field in preferences and type(preferences[field]) is not bool:
|
||||
_block(result, 'invalid_extension_preferences', f'extensions_preferences.{field}')
|
||||
for field in ('mcp_servers', 'skills'):
|
||||
if field in preferences and not _names(preferences[field]):
|
||||
_block(result, 'invalid_extension_preferences', f'extensions_preferences.{field}')
|
||||
if 'mcp_resources' in preferences and not _attachments(preferences['mcp_resources']):
|
||||
_block(result, 'invalid_extension_preferences', 'extensions_preferences.mcp_resources')
|
||||
plugins = preferences.get('plugins', [])
|
||||
if type(plugins) is not list or not all(
|
||||
type(item) is dict and all(type(item.get(k)) is str and item[k] for k in ('author', 'name')) for item in plugins
|
||||
):
|
||||
_block(result, 'invalid_extension_preferences', 'extensions_preferences.plugins')
|
||||
elif preferences.get('enable_all_plugins', True) is False and not any(
|
||||
item['author'] == 'langbot-team' and item['name'] == name for item in plugins
|
||||
):
|
||||
# The immutable result contract contains no extension-preference patch.
|
||||
# Require explicit binding instead of silently flipping allow-all.
|
||||
_block(result, 'extensions.runner_excluded', 'extensions_preferences.plugins')
|
||||
|
||||
|
||||
def _warn(result, code, field):
|
||||
diagnostic = {'code': code, 'field': field}
|
||||
if diagnostic not in result['warnings']:
|
||||
result['warnings'].append(diagnostic)
|
||||
|
||||
|
||||
def _message_shape(value, shape, required=()):
|
||||
# SDK Message schemas, without importing runtime services or coercing values.
|
||||
return (
|
||||
type(value) is dict
|
||||
and not (set(value) - set(shape))
|
||||
and all(key in value for key in required)
|
||||
and all(check(value[key]) for key, check in shape.items() if key in value)
|
||||
)
|
||||
|
||||
|
||||
def _valid_prompt(prompt):
|
||||
def string(value):
|
||||
return type(value) is str
|
||||
|
||||
def optional_string(value):
|
||||
return value is None or string(value)
|
||||
|
||||
def metadata(value):
|
||||
return value is None or type(value) is dict
|
||||
|
||||
def content(value):
|
||||
if value is None or string(value):
|
||||
return True
|
||||
fields = {key: optional_string for key in ('text', 'image_base64', 'file_url', 'file_base64', 'file_name')}
|
||||
fields.update(type=string, image_url=lambda v: v is None or _message_shape(v, {'url': string}, ('url',)))
|
||||
return type(value) is list and all(_message_shape(v, fields, ('type',)) for v in value)
|
||||
|
||||
def tools(value):
|
||||
fields = {
|
||||
'id': string,
|
||||
'type': string,
|
||||
'function': lambda v: _message_shape(v, {'name': string, 'arguments': string}, ('name', 'arguments')),
|
||||
'provider_specific_fields': metadata,
|
||||
}
|
||||
return value is None or (
|
||||
type(value) is list and all(_message_shape(v, fields, ('id', 'type', 'function')) for v in value)
|
||||
)
|
||||
|
||||
fields = {key: optional_string for key in ('name', 'tool_call_id', 'resp_message_id')}
|
||||
fields.update(role=string, content=content, tool_calls=tools, provider_specific_fields=metadata)
|
||||
return type(prompt) is list and all(_message_shape(item, fields, ('role',)) for item in prompt)
|
||||
|
||||
|
||||
def _validate_local(result, section):
|
||||
prefix = 'ai.local-agent'
|
||||
model = section.get('model')
|
||||
if type(model) is dict:
|
||||
if set(model) - {'primary', 'fallbacks', 'reasoning'}:
|
||||
_block(result, 'unknown_field', f'{prefix}.model')
|
||||
if type(model.get('primary')) is not str or not model['primary'] or not _names(model.get('fallbacks', [])):
|
||||
_block(result, 'invalid_type', f'{prefix}.model')
|
||||
reasoning = model.get('reasoning', {})
|
||||
if type(reasoning) is not dict or not all(type(v) is str for v in reasoning.values()):
|
||||
_block(result, 'invalid_type', f'{prefix}.model.reasoning')
|
||||
elif any(value not in _REASONING_LEVELS for value in reasoning.values()):
|
||||
_block(result, 'local.reasoning_value', f'{prefix}.model')
|
||||
elif reasoning:
|
||||
_warn(result, 'local.model_reasoning', f'{prefix}.model.reasoning')
|
||||
elif type(model) is not str or not model:
|
||||
_block(result, 'invalid_type', f'{prefix}.model')
|
||||
if not _valid_prompt(section.get('prompt')):
|
||||
_block(result, 'local.prompt_shape', f'{prefix}.prompt')
|
||||
if section.get('box-session-id-template') not in (None, '', '{launcher_type}_{launcher_id}'):
|
||||
_block(result, 'local.box_scope', f'{prefix}.box-session-id-template')
|
||||
_warn(result, 'local.context_defaults', f'{prefix}.max-round')
|
||||
_warn(result, 'local.serial_tools_preserved', f'{prefix}.tools')
|
||||
_warn(result, 'local.retrieval_defaults', f'{prefix}.knowledge-bases')
|
||||
_warn(result, 'local.box_state_reset', f'{prefix}.box-session-id-template')
|
||||
|
||||
|
||||
def _unique_json_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
raise ValueError('duplicate_key')
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def _parse_tweaks(raw):
|
||||
if raw is None or (type(raw) is str and not raw.strip()):
|
||||
return {}
|
||||
decoded = json.loads(raw, object_pairs_hook=_unique_json_object) if type(raw) is str else raw
|
||||
if decoded is None:
|
||||
return {}
|
||||
if type(decoded) is not dict or not _strict_json(decoded):
|
||||
raise ValueError('invalid_tweaks')
|
||||
return decoded
|
||||
|
||||
|
||||
def _validate_external(result, legacy, section):
|
||||
prefix = f'ai.{legacy}'
|
||||
app_types = {
|
||||
'dify-service-api': ('chat', 'agent', 'workflow', 'chatflow'),
|
||||
'dashscope-app-api': ('agent', 'workflow'),
|
||||
'weknora-api': ('chat', 'agent'),
|
||||
}
|
||||
required_strings = {
|
||||
'dify-service-api': ('api-key', 'base-url'),
|
||||
'coze-api': ('api-key', 'bot-id'),
|
||||
'dashscope-app-api': ('api-key', 'app-id'),
|
||||
'n8n-service-api': ('webhook-url',),
|
||||
'langflow-api': ('base-url', 'api-key', 'flow-id'),
|
||||
'tbox-app-api': ('api-key', 'app-id'),
|
||||
'weknora-api': ('api-key', 'base-url'),
|
||||
}
|
||||
for field in required_strings.get(legacy, ()):
|
||||
value = section.get(field)
|
||||
if type(value) is str and (not value or (legacy == 'weknora-api' and not value.strip())):
|
||||
_block(result, 'invalid_value', f'{prefix}.{field}')
|
||||
if legacy in app_types and 'app-type' in section and section['app-type'] not in app_types[legacy]:
|
||||
_block(result, 'invalid_value', f'{prefix}.app-type')
|
||||
if legacy == 'coze-api':
|
||||
actual, alias = section.get('auto_save_history'), section.get('auto-save-history')
|
||||
if type(actual) is bool and type(alias) is bool and actual != alias:
|
||||
_block(result, 'coze.history_alias', f'{prefix}.auto_save_history')
|
||||
if 'auto_save_history' in section or 'auto-save-history' in section:
|
||||
_warn(result, 'migration.alias_repaired', f'{prefix}.auto_save_history')
|
||||
if actual is None and alias is None:
|
||||
_warn(result, 'migration.null_default', f'{prefix}.auto_save_history')
|
||||
base = section.get('api-base')
|
||||
try:
|
||||
parsed = urlsplit(base) if type(base) is str else None
|
||||
if (
|
||||
parsed is None
|
||||
or parsed.scheme not in ('http', 'https')
|
||||
or not parsed.hostname
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or any(c.isspace() or ord(c) < 32 for c in base)
|
||||
):
|
||||
raise ValueError
|
||||
_ = parsed.port
|
||||
except ValueError:
|
||||
_block(result, 'coze.custom_endpoint', f'{prefix}.api-base')
|
||||
_warn(result, 'coze.persistent_history', prefix)
|
||||
elif legacy == 'dashscope-app-api':
|
||||
if 'references-quote' in section:
|
||||
if 'references_quote' in section and section['references_quote'] != section['references-quote']:
|
||||
_block(result, 'dashscope.references_alias', f'{prefix}.references_quote')
|
||||
else:
|
||||
_warn(result, 'migration.alias_repaired', f'{prefix}.references-quote')
|
||||
elif legacy == 'langflow-api':
|
||||
for axis in ('input', 'output'):
|
||||
actual, ui = f'{axis}_type', f'{axis}-type'
|
||||
if ui in section and section[ui] != section.get(actual, 'chat'):
|
||||
_block(result, 'langflow.io_alias', f'{prefix}.{actual}')
|
||||
if actual in section:
|
||||
_warn(result, 'migration.alias_repaired', f'{prefix}.{actual}')
|
||||
try:
|
||||
_parse_tweaks(section.get('tweaks'))
|
||||
except (ValueError, RecursionError):
|
||||
_block(result, 'langflow.invalid_tweaks', f'{prefix}.tweaks')
|
||||
if section.get('tweaks') is None or (
|
||||
type(section.get('tweaks')) is str
|
||||
and (not section['tweaks'].strip() or section['tweaks'].strip() == 'null')
|
||||
):
|
||||
_warn(result, 'langflow.tweaks_default', f'{prefix}.tweaks')
|
||||
_warn(result, 'langflow.persistent_history', prefix)
|
||||
elif legacy == 'dify-service-api':
|
||||
_warn(result, 'dify.timeout_default', f'{prefix}.timeout')
|
||||
elif legacy == 'n8n-service-api':
|
||||
mode = section.get('auth-type', 'none')
|
||||
auth_fields = {
|
||||
'none': (),
|
||||
'basic': ('basic-username', 'basic-password'),
|
||||
'jwt': ('jwt-secret',),
|
||||
'header': ('header-name', 'header-value'),
|
||||
}
|
||||
if type(mode) is not str or mode not in auth_fields:
|
||||
_block(result, 'invalid_value', f'{prefix}.auth-type')
|
||||
else:
|
||||
for field in auth_fields[mode]:
|
||||
if field not in section:
|
||||
_block(result, 'missing_field', f'{prefix}.{field}')
|
||||
response = section.get('response-handling', 'reply')
|
||||
if response not in ('reply', 'ignore'):
|
||||
_block(result, 'invalid_value', f'{prefix}.response-handling')
|
||||
if mode == 'basic':
|
||||
for field in ('basic-username', 'basic-password'):
|
||||
value = section.get(field)
|
||||
if type(value) is str:
|
||||
try:
|
||||
value.encode('latin1')
|
||||
if field == 'basic-username' and ':' in value:
|
||||
raise ValueError
|
||||
except (UnicodeEncodeError, ValueError):
|
||||
_block(result, 'n8n.basic_encoding', f'{prefix}.{field}')
|
||||
if mode == 'header' and type(section.get('header-name')) is str:
|
||||
if not re.fullmatch(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+", section['header-name']):
|
||||
_block(result, 'n8n.header_name', f'{prefix}.header-name')
|
||||
_warn(result, 'n8n.session_ids_reset', prefix)
|
||||
elif legacy == 'weknora-api':
|
||||
knowledge = section.get('knowledge-base-ids')
|
||||
if type(knowledge) is list and any(type(v) is str and not v.strip() for v in knowledge):
|
||||
_block(result, 'invalid_value', f'{prefix}.knowledge-base-ids')
|
||||
if legacy in ('coze-api', 'n8n-service-api', 'weknora-api'):
|
||||
timeout = section.get('timeout', 120)
|
||||
if type(timeout) in (int, float) and timeout <= 0:
|
||||
_block(result, 'invalid_value', f'{prefix}.timeout')
|
||||
|
||||
|
||||
def _validate_section(result, legacy, section):
|
||||
prefix = f'ai.{legacy}'
|
||||
if set(section) - _FIELDS[legacy]:
|
||||
_block(result, 'unknown_field', prefix)
|
||||
for field in _REQUIRED[legacy]:
|
||||
if field not in section:
|
||||
_block(result, 'missing_field', f'{prefix}.{field}')
|
||||
# Iterate only known keys, never put an arbitrary saved key in diagnostics.
|
||||
for field in sorted(_FIELDS[legacy] & section.keys()):
|
||||
value = section[field]
|
||||
if field in ('model', 'prompt', 'tweaks') or (legacy == 'dify-service-api' and field == 'timeout'):
|
||||
continue
|
||||
if value is None and (
|
||||
(legacy == 'coze-api' and field in ('auto_save_history', 'auto-save-history'))
|
||||
or (legacy == 'weknora-api' and field in ('agent-id', 'knowledge-base-ids'))
|
||||
):
|
||||
continue
|
||||
if field in _BOOLEANS:
|
||||
valid = type(value) is bool
|
||||
elif field in _INTEGERS or (field == 'timeout' and legacy == 'deerflow-api'):
|
||||
valid = type(value) is int
|
||||
elif field == 'timeout':
|
||||
valid = type(value) in (int, float)
|
||||
elif field in _NAME_LISTS:
|
||||
valid = _names(value)
|
||||
elif field == 'mcp-resources':
|
||||
valid = _attachments(value)
|
||||
else:
|
||||
valid = type(value) is str
|
||||
if not valid:
|
||||
_block(result, 'invalid_type', f'{prefix}.{field}')
|
||||
if legacy == 'local-agent':
|
||||
_validate_local(result, section)
|
||||
else:
|
||||
_validate_external(result, legacy, section)
|
||||
if legacy == 'deerflow-api':
|
||||
base = section.get('api-base')
|
||||
if type(base) is str and not base.strip().startswith(('http://', 'https://')):
|
||||
_block(result, 'invalid_value', f'{prefix}.api-base')
|
||||
|
||||
|
||||
def _validate_output(result, config, legacy):
|
||||
output = config.get('output', {})
|
||||
if type(output) is not dict:
|
||||
_block(result, 'invalid_type', 'output')
|
||||
return
|
||||
misc = output.get('misc', {})
|
||||
if type(misc) is not dict:
|
||||
_block(result, 'invalid_type', 'output.misc')
|
||||
return
|
||||
value = misc.get('remove-think', False)
|
||||
if type(value) is not bool:
|
||||
_block(result, 'invalid_type', 'output.misc.remove-think')
|
||||
|
||||
|
||||
def _assemble(result, legacy, section, config, preferences):
|
||||
selected = {**copy.deepcopy(_DEFAULTS[legacy]), **copy.deepcopy(section)}
|
||||
prefix = f'ai.{legacy}'
|
||||
if legacy in _IDENTITY_SOURCES:
|
||||
selected['user-id-source'] = _IDENTITY_SOURCES[legacy]
|
||||
if legacy == 'local-agent':
|
||||
selected.pop('max-round', None)
|
||||
selected.pop('box-session-id-template', None)
|
||||
singular = selected.pop('knowledge-base', '')
|
||||
if not selected['knowledge-bases'] and singular and singular != '__none__':
|
||||
selected['knowledge-bases'] = [singular]
|
||||
_warn(result, 'migration.alias_repaired', f'{prefix}.knowledge-base')
|
||||
model = selected['model']
|
||||
selected['model'] = (
|
||||
{'primary': model, 'fallbacks': [], 'reasoning': {}}
|
||||
if type(model) is str
|
||||
else {'fallbacks': [], 'reasoning': {}, **model}
|
||||
)
|
||||
preferences = preferences or {}
|
||||
for field, default in (('mcp-resources', []), ('mcp-resource-agent-read-enabled', True)):
|
||||
selected[field] = copy.deepcopy(section.get(field, preferences.get(field.replace('-', '_'), default)))
|
||||
else:
|
||||
selected.update(
|
||||
{
|
||||
'enable-all-tools': False,
|
||||
'tools': [],
|
||||
'knowledge-bases': [],
|
||||
'mcp-resources': [],
|
||||
'mcp-resource-agent-read-enabled': False,
|
||||
}
|
||||
)
|
||||
if legacy in _ASSET_RUNNERS:
|
||||
selected['langbot-assets-enabled'] = False
|
||||
if legacy in _REMOVE_THINK_RUNNERS:
|
||||
selected['remove-think'] = config.get('output', {}).get('misc', {}).get('remove-think', False)
|
||||
_warn(result, 'migration.output_policy_copied', 'output.misc.remove-think')
|
||||
if legacy == 'dify-service-api':
|
||||
selected['timeout'] = 30
|
||||
elif legacy == 'coze-api':
|
||||
actual = selected.pop('auto_save_history', None)
|
||||
selected['auto-save-history'] = actual if type(actual) is bool else section.get('auto-save-history')
|
||||
if selected['auto-save-history'] is None:
|
||||
selected['auto-save-history'] = True
|
||||
elif legacy == 'dashscope-app-api':
|
||||
if 'references-quote' in selected:
|
||||
selected['references_quote'] = selected.pop('references-quote')
|
||||
elif legacy == 'langflow-api':
|
||||
for axis in ('input', 'output'):
|
||||
selected[f'{axis}-type'] = selected.pop(f'{axis}_type', selected[f'{axis}-type'])
|
||||
selected['tweaks'] = copy.deepcopy(_parse_tweaks(section.get('tweaks')))
|
||||
elif legacy == 'weknora-api':
|
||||
selected.setdefault(
|
||||
'agent-id', 'builtin-quick-answer' if selected['app-type'] == 'chat' else 'builtin-smart-reasoning'
|
||||
)
|
||||
if selected['knowledge-base-ids'] is None:
|
||||
selected['knowledge-base-ids'] = []
|
||||
_warn(result, 'weknora.session_title_changed', prefix)
|
||||
return selected
|
||||
|
||||
|
||||
def plan_legacy_pipeline(config, extensions_preferences=None) -> dict:
|
||||
"""Return a detached config candidate or safe, value-free diagnostics."""
|
||||
result = {
|
||||
'state': 'not_legacy',
|
||||
'legacy_runner': None,
|
||||
'target_runner_id': None,
|
||||
'target_plugin': None,
|
||||
'config': None,
|
||||
'changed_paths': [],
|
||||
'blockers': [],
|
||||
'warnings': [],
|
||||
}
|
||||
if type(config) is not dict:
|
||||
return _block(result, 'invalid_type', 'config')
|
||||
if not _strict_json(config):
|
||||
return _block(result, 'invalid_json_value', 'config')
|
||||
ai = config.get('ai', {})
|
||||
if type(ai) is not dict:
|
||||
return _block(result, 'invalid_type', 'ai')
|
||||
selection = ai.get('runner', {})
|
||||
if type(selection) is not dict:
|
||||
return _block(result, 'invalid_type', 'ai.runner')
|
||||
if 'id' in selection and 'runner' in selection:
|
||||
return _block(result, 'mixed_runner_selection', 'ai.runner')
|
||||
if 'id' in selection:
|
||||
current = selection['id']
|
||||
if type(current) is not str or not re.fullmatch(r'plugin:[^/\s]+/[^/\s]+/[^/\s]+', current):
|
||||
return _block(result, 'invalid_runner_id', 'ai.runner.id')
|
||||
result['state'] = 'already_current'
|
||||
return result
|
||||
if 'runner' not in selection:
|
||||
return result
|
||||
legacy = selection['runner']
|
||||
if type(legacy) is not str:
|
||||
return _block(result, 'invalid_type', 'ai.runner.runner')
|
||||
if legacy not in _TARGETS:
|
||||
return result
|
||||
name, version = _TARGETS[legacy]
|
||||
target = f'plugin:langbot-team/{name}/default'
|
||||
result.update(
|
||||
legacy_runner=legacy,
|
||||
target_runner_id=target,
|
||||
target_plugin={'author': 'langbot-team', 'name': name, 'version': version},
|
||||
)
|
||||
if 'runner_config' in ai:
|
||||
if type(ai['runner_config']) is not dict:
|
||||
return _block(result, 'invalid_type', 'ai.runner_config')
|
||||
if ai['runner_config']:
|
||||
return _block(result, 'mixed_runner_config', 'ai.runner_config')
|
||||
if 'expire-time' in selection:
|
||||
expiry = selection['expire-time']
|
||||
if type(expiry) is not int or expiry < 0:
|
||||
_block(result, 'invalid_expiry', 'ai.runner.expire-time')
|
||||
if set(ai) - set(_TARGETS) - {'runner', 'runner_config'}:
|
||||
_block(result, 'unknown_field', 'ai')
|
||||
if set(selection) - {'runner', 'expire-time'}:
|
||||
_block(result, 'unknown_field', 'ai.runner')
|
||||
if legacy not in ai:
|
||||
return _block(result, 'missing_field', f'ai.{legacy}')
|
||||
section = ai[legacy]
|
||||
if type(section) is not dict:
|
||||
return _block(result, 'invalid_type', f'ai.{legacy}')
|
||||
_validate_section(result, legacy, section)
|
||||
_validate_preferences(result, extensions_preferences, name)
|
||||
_validate_output(result, config, legacy)
|
||||
_warn(result, 'external.state_validation_required', 'ai.runner')
|
||||
_warn(result, 'migration.history_reset', 'ai.runner')
|
||||
_warn(result, 'migration.new_defaults', f'ai.{legacy}')
|
||||
_warn(result, 'migration.legacy_sections_archived', 'ai')
|
||||
if legacy in ('dify-service-api', 'dashscope-app-api', 'n8n-service-api'):
|
||||
_warn(result, 'migration.filtered_variables', f'ai.{legacy}')
|
||||
if legacy in _IDENTITY_SOURCES:
|
||||
_warn(result, 'migration.identity_preserved', f'ai.{legacy}')
|
||||
if result['blockers']:
|
||||
return result
|
||||
candidate = copy.deepcopy(config)
|
||||
candidate['ai']['runner'].pop('runner')
|
||||
candidate['ai']['runner']['id'] = target
|
||||
selected = _assemble(result, legacy, section, config, extensions_preferences)
|
||||
candidate['ai']['runner_config'] = {target: selected}
|
||||
# The durable transaction stores the full source, including inactive legacy
|
||||
# credentials. The active configuration must be canonical so the guarded
|
||||
# editor never needs to interpret mixed legacy/current containers.
|
||||
archived_paths = []
|
||||
for old_runner in _TARGETS:
|
||||
if old_runner in candidate['ai']:
|
||||
candidate['ai'].pop(old_runner)
|
||||
archived_paths.append(f'ai.{old_runner}')
|
||||
result.update(
|
||||
state='ready',
|
||||
config=candidate,
|
||||
changed_paths=['ai.runner.runner', 'ai.runner.id', 'ai.runner_config', *archived_paths],
|
||||
)
|
||||
return result
|
||||
@@ -636,6 +636,16 @@ class PipelineManager:
|
||||
return dataclasses.replace(context, pipeline_uuid=pipeline_uuid)
|
||||
|
||||
async def load_pipeline(
|
||||
self,
|
||||
context: ExecutionContext | RequestContext,
|
||||
pipeline_entity,
|
||||
*,
|
||||
_binding_validated: bool = False,
|
||||
):
|
||||
candidate = await self.prepare_pipeline(context, pipeline_entity, _binding_validated=_binding_validated)
|
||||
self.publish_pipeline(candidate)
|
||||
|
||||
async def prepare_pipeline(
|
||||
self,
|
||||
context: ExecutionContext | RequestContext,
|
||||
pipeline_entity: persistence_pipeline.LegacyPipeline
|
||||
@@ -657,8 +667,6 @@ class PipelineManager:
|
||||
execution_context.workspace_uuid,
|
||||
expected_generation=execution_context.placement_generation,
|
||||
)
|
||||
self._observe_execution_context(execution_context)
|
||||
|
||||
coerce_pipeline_config(
|
||||
pipeline_entity.config,
|
||||
getattr(self.ap, 'pipeline_config_meta_trigger', {'name': 'trigger', 'stages': []}),
|
||||
@@ -685,17 +693,25 @@ class PipelineManager:
|
||||
execution_context.workspace_uuid,
|
||||
expected_generation=execution_context.placement_generation,
|
||||
)
|
||||
self._observe_execution_context(execution_context)
|
||||
runtime_pipeline = RuntimePipeline(
|
||||
return RuntimePipeline(
|
||||
self.ap,
|
||||
pipeline_entity,
|
||||
stage_containers,
|
||||
execution_context,
|
||||
)
|
||||
|
||||
def publish_pipeline(self, runtime_pipeline: RuntimePipeline) -> None:
|
||||
"""Publish a prepared candidate without yielding.
|
||||
|
||||
Callers preparing before a database commit must recheck the committed
|
||||
source and execution binding before invoking this synchronous seam.
|
||||
"""
|
||||
execution_context = runtime_pipeline.execution_context
|
||||
self._observe_execution_context(execution_context)
|
||||
key = (
|
||||
execution_context.instance_uuid,
|
||||
execution_context.workspace_uuid,
|
||||
pipeline_entity.uuid,
|
||||
runtime_pipeline.pipeline_entity.uuid,
|
||||
)
|
||||
self._pipelines_by_key[key] = runtime_pipeline
|
||||
self._pipeline_keys_by_scope.setdefault(key[:2], set()).add(key)
|
||||
|
||||
@@ -52,6 +52,7 @@ from ..entity.persistence import model as persistence_model
|
||||
from ..core import app
|
||||
from ..utils import constants
|
||||
from ..agent.runner.session_registry import get_session_registry
|
||||
from ..agent.runner.model_reasoning import model_with_reasoning_override
|
||||
from ..agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ..agent.runner import config_schema
|
||||
from ..agent.runner.platform_tools import execute_platform_tool, get_platform_tool_detail, resolve_platform_api_call
|
||||
@@ -204,6 +205,7 @@ async def _validate_run_authorization(
|
||||
ap: app.Application,
|
||||
caller_plugin_identity: str | None = None,
|
||||
operation: str | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
) -> Union[tuple[None, handler.ActionResponse], tuple[Any, None]]:
|
||||
"""Validate run_id authorization for a resource access.
|
||||
|
||||
@@ -250,6 +252,9 @@ async def _validate_run_authorization(
|
||||
message=f'Plugin identity mismatch: caller {caller_plugin_identity} is not authorized for run_id {run_id}',
|
||||
)
|
||||
|
||||
if workspace_uuid is not None and session['authorization'].get('workspace_id') not in (None, workspace_uuid):
|
||||
return None, handler.ActionResponse.error(message='Run session belongs to another Workspace')
|
||||
|
||||
if not session_registry.is_resource_allowed(session, resource_type, resource_id, operation):
|
||||
ap.logger.warning(
|
||||
f'{resource_type.upper()}: {resource_id} operation {operation or "*"} not allowed for run_id {run_id}'
|
||||
@@ -1341,11 +1346,19 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
caller_plugin_identity = data.get('caller_plugin_identity')
|
||||
|
||||
if run_id:
|
||||
_session, error = await _validate_run_authorization(
|
||||
run_id, 'model', llm_model_uuid, self.ap, caller_plugin_identity, operation='count_tokens'
|
||||
session, error = await _validate_run_authorization(
|
||||
run_id,
|
||||
'model',
|
||||
llm_model_uuid,
|
||||
self.ap,
|
||||
caller_plugin_identity,
|
||||
operation='count_tokens',
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
)
|
||||
if error:
|
||||
return error
|
||||
else:
|
||||
session = None
|
||||
|
||||
if not await self._resource_exists(
|
||||
persistence_model.LLMModel,
|
||||
@@ -1366,6 +1379,9 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
message=f'LLM model with llm_model_uuid {llm_model_uuid} not found',
|
||||
)
|
||||
|
||||
if getattr(llm_model.model_entity, 'workspace_uuid', None) not in (None, action_context.workspace_uuid):
|
||||
return handler.ActionResponse.error(message='LLM model belongs to another Workspace')
|
||||
llm_model = model_with_reasoning_override(llm_model, llm_model_uuid, session)
|
||||
messages_obj = [provider_message.Message.model_validate(message) for message in messages]
|
||||
|
||||
async def _placeholder_func(**kwargs):
|
||||
@@ -1407,7 +1423,13 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
# Permission validation for Runner calls
|
||||
if run_id:
|
||||
session, error = await _validate_run_authorization(
|
||||
run_id, 'model', llm_model_uuid, self.ap, caller_plugin_identity, operation='invoke'
|
||||
run_id,
|
||||
'model',
|
||||
llm_model_uuid,
|
||||
self.ap,
|
||||
caller_plugin_identity,
|
||||
operation='invoke',
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
)
|
||||
if error:
|
||||
return error
|
||||
@@ -1441,6 +1463,7 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
message=f'LLM model with llm_model_uuid {llm_model_uuid} not found',
|
||||
)
|
||||
|
||||
llm_model = model_with_reasoning_override(llm_model, llm_model_uuid, session)
|
||||
messages_obj = [provider_message.Message.model_validate(message) for message in messages]
|
||||
|
||||
# The func field is excluded during model_dump() in plugin side (marked as exclude=True),
|
||||
@@ -1500,7 +1523,13 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
# Permission validation for Runner calls
|
||||
if run_id:
|
||||
session, error = await _validate_run_authorization(
|
||||
run_id, 'model', llm_model_uuid, self.ap, caller_plugin_identity, operation='stream'
|
||||
run_id,
|
||||
'model',
|
||||
llm_model_uuid,
|
||||
self.ap,
|
||||
caller_plugin_identity,
|
||||
operation='stream',
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
)
|
||||
if error:
|
||||
yield error
|
||||
@@ -1528,6 +1557,10 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
)
|
||||
return
|
||||
|
||||
if getattr(llm_model.model_entity, 'workspace_uuid', None) not in (None, action_context.workspace_uuid):
|
||||
yield handler.ActionResponse.error(message='LLM model belongs to another Workspace')
|
||||
return
|
||||
llm_model = model_with_reasoning_override(llm_model, llm_model_uuid, session)
|
||||
messages_obj = [provider_message.Message.model_validate(message) for message in messages]
|
||||
|
||||
# The func field is excluded during model_dump() in plugin side
|
||||
|
||||
@@ -235,7 +235,10 @@ class CodexRequester(requester.ProviderAPIRequester):
|
||||
'stream': True,
|
||||
'include': ['reasoning.encrypted_content'],
|
||||
}
|
||||
level = reasoning.normalize_reasoning_config(getattr(model.model_entity, 'reasoning_config', None))['level']
|
||||
reasoning_config = getattr(model, 'reasoning_config_override', None)
|
||||
if reasoning_config is None:
|
||||
reasoning_config = getattr(model.model_entity, 'reasoning_config', None)
|
||||
level = reasoning.normalize_reasoning_config(reasoning_config)['level']
|
||||
if level != 'provider_default':
|
||||
reasoning.validate_reasoning_capabilities(
|
||||
{'level': level}, self.get_reasoning_capabilities(model), model.model_entity.name
|
||||
|
||||
Reference in New Issue
Block a user