mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 04:40:57 +00:00
feat(tenancy): add Workspace multi-tenant foundation (#2353)
* Document multi-tenant workspace architecture * Add OSS and commercial workspace boundaries * docs: redesign multi-tenant workspace architecture * feat(tenancy): implement workspace isolation * docs(tenancy): record verification evidence * docs(tenancy): revise single-instance SaaS topology * docs(tenancy): refine architecture options * docs: finalize cloud v2 multi-tenant decisions * feat(tenancy): establish cloud isolation foundations * feat(tenancy): harden shared cloud runtime boundaries * docs(tenancy): record final isolation verification * fix(tenancy): close isolation and permission gaps * docs(tenancy): record final isolation verification * feat(tenancy): connect cloud workspace control plane * fix(build): install git for pinned SDK * docs(cloud): update control plane verification * chore: update multi-tenant SDK pin * fix(cloud): skip legacy model sync during startup * test(cloud): preserve minimal model manager fixtures * fix(cloud): preserve authenticated account context * fix(cloud): reuse authenticated account for user info * feat(cloud): complete Workspace settings navigation * test(web): cover Workspace dropdown menu * feat(web): place workspace controls in sidebar * refactor(web): streamline workspace controls * style(web): format workspace layout test * fix(cloud): surface runtime and workspace plan status * fix(plugin): keep runtime identity stable across restarts * fix(ui): widen and center workspace switcher * fix(ui): hide roles from workspace switcher * fix(ui): align workspace switcher with sidebar entries * feat(workspace): add in-product collaboration and direct Cloud launch * style: format collaboration changes * fix(workspace): bind collaboration APIs to tenant UoW * fix(cloud): preserve Core-owned collaboration state * test(cloud): require Space identity for invite registration * feat(cloud): complete secure invitation experience * style(web): format invitation flows * fix(cloud): recover box runtime without unscoped skill reload * feat(oss): enforce invitation account and owner billing flows * style: format OSS account service * test(oss): cover invitation logout handoff * fix(oss): resolve workspace owner in scoped session * feat(cloud): harden multi-tenant runtime resources * fix(cloud): bound runtime restart storms * fix(cloud): eliminate periodic runtime CPU spikes * fix(cloud): enforce instance capacity ceilings * fix(cloud): scope public login capability discovery * fix(cloud): bound tenant maintenance and monitoring work * fix(runtime): bound tenant resource amplification * fix(deps): pin green multi-tenant plugin SDK * fix(cloud): handle unavailable skill capability * fix(security): require authentication for image file endpoint (H-2) - Changed /api/v1/files/image from AuthType.NONE to USER_TOKEN_OR_API_KEY - Added Permission.RESOURCE_VIEW requirement - Prevents unauthenticated cross-tenant file access via leaked keys - Fixes HIGH severity finding from multi-tenant security review docs: add comprehensive database migration guide - Complete migration steps for OSS → multi-tenant - Backup, execution, verification procedures - Rollback scenarios and recovery plans - Performance tuning recommendations * test: add comprehensive cross-tenant isolation tests Added 7 critical test scenarios for multi-tenant boundaries: - Cross-tenant bot access prevention - Viewer role read-only enforcement - Removed member immediate access revocation - Model provider credential isolation - WebSocket message isolation - Invitation token workspace scoping - Multi-workspace context validation These tests address P0-2 coverage gaps for: - workspaces.py (membership & invitation flows) - user.py (authentication & authorization) - websocket_chat.py (real-time isolation) - plugins.py (resource access control) docs: finalize database migration guide * fix(security): resolve M-1, M-2, M-3 security findings M-1: WebSocket authorization TOCTOU race (FIXED) - Changed _revalidate_websocket_authorization to return RequestContext - Ensures validated context is used immediately without race window - Prevents removed members from sending messages during revalidation gap M-2: Model Manager cache workspace isolation (VERIFIED) - Confirmed _CacheKey already uses 4-tuple: (instance, workspace, generation, resource) - Cache is properly scoped per workspace, no cross-tenant leakage possible - No code change needed, documented as working correctly M-3: Invitation lock workspace scoping (FIXED) - Changed lock key from token_digest to workspace_uuid:token_digest - Prevents DoS where attacker locks token in Workspace A to block Workspace B - Locks now isolated per workspace All MEDIUM severity findings from security review now resolved. * fix(cloud): unblock tenant CI and enforce knowledge quotas * fix(tenancy): scope rerank model sync --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import types
|
||||
import typing
|
||||
|
||||
from .context import RequestContext
|
||||
|
||||
|
||||
class WorkspaceRole(enum.StrEnum):
|
||||
OWNER = 'owner'
|
||||
ADMIN = 'admin'
|
||||
DEVELOPER = 'developer'
|
||||
OPERATOR = 'operator'
|
||||
VIEWER = 'viewer'
|
||||
|
||||
|
||||
class Permission(enum.StrEnum):
|
||||
WORKSPACE_VIEW = 'workspace.view'
|
||||
WORKSPACE_UPDATE = 'workspace.update'
|
||||
WORKSPACE_DELETE = 'workspace.delete'
|
||||
OWNER_TRANSFER = 'owner.transfer'
|
||||
MEMBER_VIEW = 'member.view'
|
||||
MEMBER_INVITE = 'member.invite'
|
||||
MEMBER_UPDATE_ROLE = 'member.update_role'
|
||||
MEMBER_REMOVE = 'member.remove'
|
||||
RESOURCE_VIEW = 'resource.view'
|
||||
RESOURCE_MANAGE = 'resource.manage'
|
||||
RUNTIME_OPERATE = 'runtime.operate'
|
||||
PROVIDER_SECRET_MANAGE = 'provider_secret.manage'
|
||||
API_KEY_MANAGE = 'api_key.manage'
|
||||
AUDIT_VIEW = 'audit.view'
|
||||
DATA_EXPORT = 'data.export'
|
||||
BILLING_LINK_MANAGE = 'billing_link.manage'
|
||||
|
||||
|
||||
_VIEW_PERMISSIONS = {
|
||||
Permission.WORKSPACE_VIEW,
|
||||
Permission.MEMBER_VIEW,
|
||||
Permission.RESOURCE_VIEW,
|
||||
}
|
||||
|
||||
_ROLE_PERMISSIONS: typing.Final = types.MappingProxyType(
|
||||
{
|
||||
WorkspaceRole.OWNER: frozenset(Permission),
|
||||
WorkspaceRole.ADMIN: frozenset(
|
||||
permission
|
||||
for permission in Permission
|
||||
if permission
|
||||
not in {
|
||||
Permission.WORKSPACE_DELETE,
|
||||
Permission.OWNER_TRANSFER,
|
||||
Permission.BILLING_LINK_MANAGE,
|
||||
}
|
||||
),
|
||||
WorkspaceRole.DEVELOPER: frozenset(
|
||||
_VIEW_PERMISSIONS
|
||||
| {
|
||||
Permission.RESOURCE_MANAGE,
|
||||
Permission.RUNTIME_OPERATE,
|
||||
Permission.PROVIDER_SECRET_MANAGE,
|
||||
}
|
||||
),
|
||||
WorkspaceRole.OPERATOR: frozenset(_VIEW_PERMISSIONS | {Permission.RUNTIME_OPERATE}),
|
||||
WorkspaceRole.VIEWER: frozenset(_VIEW_PERMISSIONS),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class AuthorizationError(Exception):
|
||||
"""Base class for errors that map to an HTTP authorization response."""
|
||||
|
||||
status_code = 403
|
||||
error_code = 'forbidden'
|
||||
|
||||
|
||||
class WorkspaceRequiredError(AuthorizationError):
|
||||
status_code = 400
|
||||
error_code = 'workspace_required'
|
||||
|
||||
|
||||
class PermissionDeniedError(AuthorizationError):
|
||||
error_code = 'permission_denied'
|
||||
|
||||
def __init__(self, permission: str) -> None:
|
||||
super().__init__(f'Missing Workspace permission: {permission}')
|
||||
self.permission = permission
|
||||
|
||||
|
||||
class EditionLimitError(AuthorizationError):
|
||||
error_code = 'edition_limit'
|
||||
|
||||
|
||||
def permissions_for_role(role: str | WorkspaceRole) -> frozenset[str]:
|
||||
"""Return the canonical fixed permissions for a Workspace role."""
|
||||
|
||||
try:
|
||||
parsed_role = WorkspaceRole(role)
|
||||
except ValueError:
|
||||
return frozenset()
|
||||
return frozenset(permission.value for permission in _ROLE_PERMISSIONS[parsed_role])
|
||||
|
||||
|
||||
def has_permission(ctx: RequestContext, permission: str | Permission) -> bool:
|
||||
"""Return whether the context contains one effective permission."""
|
||||
|
||||
permission_value = permission.value if isinstance(permission, Permission) else permission
|
||||
return permission_value in ctx.workspace.permissions
|
||||
|
||||
|
||||
def require_permission(ctx: RequestContext, permission: str | Permission) -> None:
|
||||
"""Raise a stable authorization error when a permission is missing."""
|
||||
|
||||
permission_value = permission.value if isinstance(permission, Permission) else permission
|
||||
if not has_permission(ctx, permission_value):
|
||||
raise PermissionDeniedError(permission_value)
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
|
||||
|
||||
class PrincipalType(enum.StrEnum):
|
||||
"""Kinds of authenticated principals accepted by LangBot."""
|
||||
|
||||
ACCOUNT = 'account'
|
||||
API_KEY = 'api_key'
|
||||
SYSTEM = 'system'
|
||||
PUBLIC_BOT = 'public_bot'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class PrincipalContext:
|
||||
"""Authenticated identity before Workspace authorization is applied."""
|
||||
|
||||
principal_type: PrincipalType
|
||||
account_uuid: str | None = None
|
||||
api_key_uuid: str | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class WorkspaceContext:
|
||||
"""Workspace membership and effective permissions for one request."""
|
||||
|
||||
workspace_uuid: str
|
||||
membership_uuid: str | None
|
||||
role: str | None
|
||||
permissions: frozenset[str]
|
||||
membership_revision: int = 0
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class RequestContext:
|
||||
"""Trusted authorization context passed to HTTP services."""
|
||||
|
||||
instance_uuid: str
|
||||
placement_generation: int
|
||||
request_id: str
|
||||
auth_type: str
|
||||
principal: PrincipalContext
|
||||
workspace: WorkspaceContext
|
||||
entitlement_revision: int = 0
|
||||
|
||||
@property
|
||||
def workspace_uuid(self) -> str:
|
||||
"""Return the selected Workspace UUID."""
|
||||
|
||||
return self.workspace.workspace_uuid
|
||||
|
||||
@property
|
||||
def account_uuid(self) -> str | None:
|
||||
"""Return the Account UUID when the principal is an Account."""
|
||||
|
||||
return self.principal.account_uuid
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class ExecutionContext:
|
||||
"""Workspace context propagated to asynchronous and runtime work."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
bot_uuid: str | None = None
|
||||
pipeline_uuid: str | None = None
|
||||
query_uuid: str | None = None
|
||||
trigger_principal: PrincipalContext | None = None
|
||||
entitlement_revision: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_request(
|
||||
cls,
|
||||
ctx: RequestContext,
|
||||
*,
|
||||
bot_uuid: str | None = None,
|
||||
pipeline_uuid: str | None = None,
|
||||
query_uuid: str | None = None,
|
||||
) -> ExecutionContext:
|
||||
"""Create a runtime context without losing the tenant generation."""
|
||||
|
||||
return cls(
|
||||
instance_uuid=ctx.instance_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
placement_generation=ctx.placement_generation,
|
||||
bot_uuid=bot_uuid,
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
query_uuid=query_uuid,
|
||||
trigger_principal=ctx.principal,
|
||||
entitlement_revision=ctx.entitlement_revision,
|
||||
)
|
||||
@@ -5,9 +5,21 @@ import typing
|
||||
import enum
|
||||
import quart
|
||||
import traceback
|
||||
import inspect
|
||||
import uuid
|
||||
from quart.typing import RouteCallable
|
||||
|
||||
from ....core import app
|
||||
from ....utils import constants
|
||||
from ....utils import bounded_executor
|
||||
from ....workspace.collaboration import MembershipPermissionError, WorkspaceCollaborationError
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ....cloud.entitlements import EntitlementUnavailableError
|
||||
from ....core.errors import TaskCapacityError
|
||||
from ..authz import AuthorizationError, Permission, permissions_for_role, require_permission
|
||||
from ..context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
# Maximum file upload size limit (10MB)
|
||||
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
||||
@@ -33,6 +45,7 @@ class AuthType(enum.Enum):
|
||||
"""Authentication type"""
|
||||
|
||||
NONE = 'none'
|
||||
ACCOUNT_TOKEN = 'account-token'
|
||||
USER_TOKEN = 'user-token'
|
||||
API_KEY = 'api-key'
|
||||
USER_TOKEN_OR_API_KEY = 'user-token-or-api-key'
|
||||
@@ -43,11 +56,11 @@ class RouterGroup(abc.ABC):
|
||||
|
||||
path: str
|
||||
|
||||
ap: app.Application
|
||||
ap: Application
|
||||
|
||||
quart_app: quart.Quart
|
||||
|
||||
def __init__(self, ap: app.Application, quart_app: quart.Quart) -> None:
|
||||
def __init__(self, ap: Application, quart_app: quart.Quart) -> None:
|
||||
self.ap = ap
|
||||
self.quart_app = quart_app
|
||||
|
||||
@@ -59,16 +72,38 @@ class RouterGroup(abc.ABC):
|
||||
self,
|
||||
rule: str,
|
||||
auth_type: AuthType = AuthType.USER_TOKEN,
|
||||
permission: Permission | str | None = None,
|
||||
**options: typing.Any,
|
||||
) -> typing.Callable[[RouteCallable], RouteCallable]: # decorator
|
||||
"""Register a route"""
|
||||
|
||||
if auth_type == AuthType.ACCOUNT_TOKEN and permission is not None:
|
||||
raise ValueError('Account-token routes cannot declare Workspace permissions')
|
||||
|
||||
def decorator(f: RouteCallable) -> RouteCallable:
|
||||
nonlocal rule
|
||||
rule = self.path + rule
|
||||
|
||||
async def handler_error(*args, **kwargs):
|
||||
if auth_type == AuthType.USER_TOKEN:
|
||||
request_context: RequestContext | None = None
|
||||
if auth_type == AuthType.ACCOUNT_TOKEN:
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if not authorization.startswith('Bearer '):
|
||||
return self.http_status(401, -1, 'No valid user token provided')
|
||||
token = authorization.removeprefix('Bearer ')
|
||||
if not token:
|
||||
return self.http_status(401, -1, 'No valid user token provided')
|
||||
|
||||
try:
|
||||
account, user_email = await self._authenticate_account(token)
|
||||
# Account-token routes deliberately stop before Workspace
|
||||
# selection. They may bootstrap a selector, but cannot
|
||||
# receive RequestContext or enforce Workspace permissions.
|
||||
self._inject_handler_context(f, kwargs, user_email, None, account)
|
||||
except Exception as e:
|
||||
return self._auth_error_response(e)
|
||||
|
||||
elif auth_type == AuthType.USER_TOKEN:
|
||||
# get token from Authorization header
|
||||
token = quart.request.headers.get('Authorization', '').replace('Bearer ', '')
|
||||
|
||||
@@ -76,18 +111,15 @@ class RouterGroup(abc.ABC):
|
||||
return self.http_status(401, -1, 'No valid user token provided')
|
||||
|
||||
try:
|
||||
user_email = await self.ap.user_service.verify_jwt_token(token)
|
||||
|
||||
# check if this account exists
|
||||
user = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if not user:
|
||||
return self.http_status(401, -1, 'User not found')
|
||||
|
||||
# check if f accepts user_email parameter
|
||||
if 'user_email' in f.__code__.co_varnames:
|
||||
kwargs['user_email'] = user_email
|
||||
account, user_email = await self._authenticate_account(token)
|
||||
request_context = await self._resolve_account_context(account, auth_type)
|
||||
if permission is not None:
|
||||
if request_context is None:
|
||||
raise AuthorizationError('Workspace authorization is unavailable')
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, user_email, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
|
||||
elif auth_type == AuthType.API_KEY:
|
||||
# get API key from Authorization header or X-API-Key header
|
||||
@@ -101,11 +133,12 @@ class RouterGroup(abc.ABC):
|
||||
return self.http_status(401, -1, 'No valid API key provided')
|
||||
|
||||
try:
|
||||
is_valid = await self.ap.apikey_service.verify_api_key(api_key)
|
||||
if not is_valid:
|
||||
return self.http_status(401, -1, 'Invalid API key')
|
||||
request_context = await self._authenticate_api_key(api_key, auth_type)
|
||||
if permission is not None:
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, None, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
|
||||
elif auth_type == AuthType.USER_TOKEN_OR_API_KEY:
|
||||
# Try API key first (check X-API-Key header)
|
||||
@@ -114,11 +147,12 @@ class RouterGroup(abc.ABC):
|
||||
if api_key:
|
||||
# API key authentication
|
||||
try:
|
||||
is_valid = await self.ap.apikey_service.verify_api_key(api_key)
|
||||
if not is_valid:
|
||||
return self.http_status(401, -1, 'Invalid API key')
|
||||
request_context = await self._authenticate_api_key(api_key, auth_type)
|
||||
if permission is not None:
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, None, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
else:
|
||||
# Try user token authentication (Authorization header)
|
||||
token = quart.request.headers.get('Authorization', '').replace('Bearer ', '')
|
||||
@@ -129,35 +163,89 @@ class RouterGroup(abc.ABC):
|
||||
)
|
||||
|
||||
try:
|
||||
user_email = await self.ap.user_service.verify_jwt_token(token)
|
||||
|
||||
# check if this account exists
|
||||
user = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if not user:
|
||||
return self.http_status(401, -1, 'User not found')
|
||||
|
||||
# check if f accepts user_email parameter
|
||||
if 'user_email' in f.__code__.co_varnames:
|
||||
kwargs['user_email'] = user_email
|
||||
account, user_email = await self._authenticate_account(token)
|
||||
request_context = await self._resolve_account_context(account, auth_type)
|
||||
if permission is not None:
|
||||
if request_context is None:
|
||||
raise AuthorizationError('Workspace authorization is unavailable')
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, user_email, request_context)
|
||||
except (AuthorizationError, WorkspaceNotFoundError, MembershipPermissionError) as e:
|
||||
# Authentication succeeded and authorization was
|
||||
# evaluated. Do not reinterpret a denied user token
|
||||
# as an API key, which would mask the stable 403/404.
|
||||
return self._auth_error_response(e)
|
||||
except Exception:
|
||||
# If user token fails, maybe it's an API key in Authorization header
|
||||
try:
|
||||
is_valid = await self.ap.apikey_service.verify_api_key(token)
|
||||
if not is_valid:
|
||||
return self.http_status(401, -1, 'Invalid authentication credentials')
|
||||
request_context = await self._authenticate_api_key(token, auth_type)
|
||||
if permission is not None:
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, None, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
|
||||
try:
|
||||
if request_context is not None:
|
||||
with bounded_executor.blocking_work_scope(request_context.workspace_uuid):
|
||||
persistence_mgr = getattr(
|
||||
self.ap,
|
||||
'persistence_mgr',
|
||||
None,
|
||||
)
|
||||
tenant_scope_descriptor = getattr(
|
||||
type(persistence_mgr),
|
||||
'tenant_scope',
|
||||
None,
|
||||
)
|
||||
if callable(tenant_scope_descriptor):
|
||||
# Authorization discovery is complete. Carry
|
||||
# the trusted Workspace identity across the
|
||||
# handler, but do not reserve a database
|
||||
# connection while it waits on providers,
|
||||
# runtimes, uploads, or streamed clients.
|
||||
# Services that need atomic writes open a UoW.
|
||||
async with persistence_mgr.tenant_scope(request_context.workspace_uuid):
|
||||
return await f(*args, **kwargs)
|
||||
return await f(*args, **kwargs)
|
||||
return await f(*args, **kwargs)
|
||||
|
||||
except Exception as e: # 自动 500
|
||||
traceback.print_exc()
|
||||
# return self.http_status(500, -2, str(e))
|
||||
return self.http_status(500, -2, str(e))
|
||||
if isinstance(e, AuthorizationError):
|
||||
return self.http_status(e.status_code, e.error_code, str(e))
|
||||
if isinstance(e, WorkspaceNotFoundError):
|
||||
return self.http_status(404, 'resource_not_found', 'Resource not found')
|
||||
if isinstance(e, MembershipPermissionError):
|
||||
return self.http_status(403, e.code, str(e))
|
||||
if isinstance(e, WorkspaceCollaborationError):
|
||||
return self.http_status(400, e.code, str(e))
|
||||
if isinstance(e, TaskCapacityError):
|
||||
return self.http_status(429, 'task_capacity_exceeded', str(e))
|
||||
if isinstance(
|
||||
e,
|
||||
bounded_executor.BlockingWorkCapacityError,
|
||||
):
|
||||
return self.http_status(
|
||||
429,
|
||||
'blocking_work_capacity_exceeded',
|
||||
str(e),
|
||||
)
|
||||
request_id = self.request_id()
|
||||
logger = getattr(self.ap, 'logger', self.quart_app.logger)
|
||||
logger.error(
|
||||
f'Unhandled HTTP error request_id={request_id} '
|
||||
f'method={quart.request.method} path={quart.request.path}\n{traceback.format_exc()}'
|
||||
)
|
||||
return self.internal_error_response(request_id)
|
||||
|
||||
new_f = handler_error
|
||||
new_f.__name__ = (self.name + rule).replace('/', '__')
|
||||
# Quart/Flask requires a unique endpoint name even when the same URL
|
||||
# intentionally has separate handlers for different HTTP methods.
|
||||
# Include the method set so CRUD routes can declare distinct
|
||||
# permissions without colliding during application startup.
|
||||
methods = options.get('methods') or ['GET']
|
||||
method_suffix = '__'.join(sorted(str(method).upper() for method in methods))
|
||||
new_f.__name__ = (self.name + rule + '__' + method_suffix).replace('/', '__')
|
||||
new_f.__doc__ = f.__doc__
|
||||
|
||||
self.quart_app.route(rule, **options)(new_f)
|
||||
@@ -165,6 +253,192 @@ class RouterGroup(abc.ABC):
|
||||
|
||||
return decorator
|
||||
|
||||
async def _authenticate_account(self, token: str) -> tuple[typing.Any, str]:
|
||||
account: typing.Any = None
|
||||
resolver = getattr(self.ap.user_service, 'get_authenticated_account', None)
|
||||
if callable(resolver):
|
||||
resolved = resolver(token)
|
||||
if inspect.isawaitable(resolved):
|
||||
account = await resolved
|
||||
|
||||
if isinstance(account, str) or account is None:
|
||||
user_email = account or await self.ap.user_service.verify_jwt_token(token)
|
||||
account = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if account is None:
|
||||
raise ValueError('User not found')
|
||||
return account, account.user
|
||||
|
||||
async def _resolve_account_context(
|
||||
self,
|
||||
account: typing.Any,
|
||||
auth_type: AuthType,
|
||||
) -> RequestContext | None:
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
# Compatibility for isolated controller tests that do not wire the tenancy kernel.
|
||||
if collaboration_service is None or not isinstance(account_uuid, str):
|
||||
return None
|
||||
|
||||
requested_workspace_uuid = quart.request.headers.get('X-Workspace-Id')
|
||||
access = await collaboration_service.resolve_account_workspace(account_uuid, requested_workspace_uuid)
|
||||
entitlement_revision = await self._resolve_entitlement_revision(
|
||||
access.execution.instance_uuid,
|
||||
access.workspace.uuid,
|
||||
)
|
||||
request_context = RequestContext(
|
||||
instance_uuid=access.execution.instance_uuid,
|
||||
placement_generation=access.execution.placement_generation,
|
||||
request_id=self.request_id(),
|
||||
auth_type=auth_type.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.ACCOUNT,
|
||||
account_uuid=account_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=access.workspace.uuid,
|
||||
membership_uuid=access.membership.uuid,
|
||||
role=access.membership.role,
|
||||
permissions=permissions_for_role(access.membership.role),
|
||||
membership_revision=access.membership.projection_revision,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
quart.g.request_context = request_context
|
||||
quart.g.workspace_membership = access.membership
|
||||
return request_context
|
||||
|
||||
async def _authenticate_api_key(self, api_key: str, auth_type: AuthType) -> RequestContext:
|
||||
authenticator = getattr(self.ap.apikey_service, 'authenticate_api_key', None)
|
||||
if callable(authenticator):
|
||||
authenticated = authenticator(api_key)
|
||||
if inspect.isawaitable(authenticated):
|
||||
identity = await authenticated
|
||||
if identity is not None:
|
||||
entitlement_revision = await self._resolve_entitlement_revision(
|
||||
identity.instance_uuid,
|
||||
identity.workspace_uuid,
|
||||
)
|
||||
request_context = RequestContext(
|
||||
instance_uuid=identity.instance_uuid,
|
||||
placement_generation=identity.placement_generation,
|
||||
request_id=self.request_id(),
|
||||
auth_type=auth_type.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.API_KEY,
|
||||
api_key_uuid=identity.api_key_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=identity.workspace_uuid,
|
||||
membership_uuid=None,
|
||||
role=None,
|
||||
permissions=identity.permissions,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
quart.g.request_context = request_context
|
||||
return request_context
|
||||
|
||||
if not await self.ap.apikey_service.verify_api_key(api_key):
|
||||
raise ValueError('Invalid API key')
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
if workspace_service is None:
|
||||
raise ValueError('API key Workspace binding is unavailable')
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
request_context = RequestContext(
|
||||
instance_uuid=binding.instance_uuid or constants.instance_id,
|
||||
placement_generation=binding.placement_generation,
|
||||
request_id=self.request_id(),
|
||||
auth_type=auth_type.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.API_KEY,
|
||||
api_key_uuid='legacy-oss-api-key',
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
membership_uuid=None,
|
||||
role=None,
|
||||
permissions=frozenset(item.value for item in Permission),
|
||||
),
|
||||
)
|
||||
quart.g.request_context = request_context
|
||||
return request_context
|
||||
|
||||
async def _resolve_entitlement_revision(self, instance_uuid: str, workspace_uuid: str) -> int:
|
||||
deployment = getattr(self.ap, 'deployment', None)
|
||||
if deployment is None or not getattr(deployment, 'multi_workspace_enabled', False):
|
||||
return 0
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None:
|
||||
raise EntitlementUnavailableError('Workspace entitlement resolver is unavailable')
|
||||
if instance_uuid != resolver.instance_uuid:
|
||||
raise EntitlementUnavailableError('Workspace entitlement targets another LangBot instance')
|
||||
snapshot = await resolver.resolve(workspace_uuid)
|
||||
return snapshot.entitlement_revision
|
||||
|
||||
@staticmethod
|
||||
def _inject_handler_context(
|
||||
handler: RouteCallable,
|
||||
kwargs: dict[str, typing.Any],
|
||||
user_email: str | None,
|
||||
request_context: RequestContext | None,
|
||||
account: typing.Any = None,
|
||||
) -> None:
|
||||
parameters = inspect.signature(handler).parameters
|
||||
if user_email is not None and 'user_email' in parameters:
|
||||
kwargs['user_email'] = user_email
|
||||
if account is not None and 'account' in parameters:
|
||||
kwargs['account'] = account
|
||||
if request_context is not None:
|
||||
if 'request_context' in parameters:
|
||||
kwargs['request_context'] = request_context
|
||||
elif 'ctx' in parameters:
|
||||
kwargs['ctx'] = request_context
|
||||
|
||||
def _auth_error_response(self, error: Exception) -> typing.Any:
|
||||
if isinstance(error, AuthorizationError):
|
||||
return self.http_status(error.status_code, error.error_code, str(error))
|
||||
if isinstance(error, WorkspaceNotFoundError):
|
||||
return self.http_status(404, 'resource_not_found', 'Resource not found')
|
||||
if isinstance(error, MembershipPermissionError):
|
||||
return self.http_status(403, error.code, str(error))
|
||||
if isinstance(error, EntitlementUnavailableError):
|
||||
return self.http_status(403, 'entitlement_unavailable', str(error))
|
||||
request_id = self.request_id()
|
||||
logger = getattr(self.ap, 'logger', self.quart_app.logger)
|
||||
logger.warning(f'Authentication failed request_id={request_id} error_type={type(error).__name__}: {error}')
|
||||
return self.http_status(
|
||||
401,
|
||||
'invalid_authentication',
|
||||
'Invalid authentication credentials',
|
||||
)
|
||||
|
||||
def request_id(self) -> str:
|
||||
"""Return one stable request ID for authentication, logs, and errors."""
|
||||
|
||||
request_context = getattr(quart.g, 'request_context', None)
|
||||
request_id = getattr(request_context, 'request_id', None) or getattr(quart.g, 'request_id', None)
|
||||
if not request_id:
|
||||
candidate = str(quart.request.headers.get('X-Request-Id') or '').strip()
|
||||
if not candidate or len(candidate) > 128 or any(ord(char) < 32 for char in candidate):
|
||||
candidate = str(uuid.uuid4())
|
||||
request_id = candidate
|
||||
quart.g.request_id = request_id
|
||||
return str(request_id)
|
||||
|
||||
def internal_error_response(self, request_id: str | None = None) -> typing.Tuple[quart.Response, int]:
|
||||
"""Return a stable 500 response without exposing the underlying exception."""
|
||||
|
||||
resolved_request_id = request_id or self.request_id()
|
||||
response = quart.jsonify(
|
||||
{
|
||||
'code': 'internal_error',
|
||||
'msg': 'Internal server error',
|
||||
'request_id': resolved_request_id,
|
||||
}
|
||||
)
|
||||
response.headers['X-Request-Id'] = resolved_request_id
|
||||
return response, 500
|
||||
|
||||
def success(self, data: typing.Any = None) -> quart.Response:
|
||||
"""Return a 200 response"""
|
||||
return quart.jsonify(
|
||||
@@ -175,7 +449,7 @@ class RouterGroup(abc.ABC):
|
||||
}
|
||||
)
|
||||
|
||||
def fail(self, code: int, msg: str) -> quart.Response:
|
||||
def fail(self, code: int | str, msg: str) -> quart.Response:
|
||||
"""Return an error response"""
|
||||
|
||||
return quart.jsonify(
|
||||
@@ -185,6 +459,6 @@ class RouterGroup(abc.ABC):
|
||||
}
|
||||
)
|
||||
|
||||
def http_status(self, status: int, code: int, msg: str) -> typing.Tuple[quart.Response, int]:
|
||||
def http_status(self, status: int, code: int | str, msg: str) -> typing.Tuple[quart.Response, int]:
|
||||
"""返回一个指定状态码的响应"""
|
||||
return (self.fail(code, msg), status)
|
||||
|
||||
@@ -1,43 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('apikeys', '/api/v1/apikeys')
|
||||
class ApiKeysRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'])
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
keys = await self.ap.apikey_service.get_api_keys()
|
||||
return self.success(data={'keys': keys})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name', '')
|
||||
description = json_data.get('description', '')
|
||||
@self.route('', methods=['GET'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
keys = await self.ap.apikey_service.get_api_keys(request_context)
|
||||
return self.success(data={'keys': keys})
|
||||
|
||||
if not name:
|
||||
return self.http_status(400, -1, 'Name is required')
|
||||
@self.route('', methods=['POST'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
expires_at = json_data.get('expires_at')
|
||||
parsed_expiry = None
|
||||
if expires_at:
|
||||
try:
|
||||
parsed_expiry = datetime.datetime.fromisoformat(str(expires_at).replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
return self.http_status(400, 'invalid_expiry', 'Invalid API key expiry')
|
||||
try:
|
||||
key = await self.ap.apikey_service.create_api_key(
|
||||
request_context,
|
||||
json_data.get('name', ''),
|
||||
json_data.get('description', ''),
|
||||
scopes=json_data.get('scopes'),
|
||||
expires_at=parsed_expiry,
|
||||
)
|
||||
except ValueError as error:
|
||||
return self.http_status(400, 'invalid_api_key', str(error))
|
||||
return self.success(data={'key': key})
|
||||
|
||||
key = await self.ap.apikey_service.create_api_key(name, description)
|
||||
return self.success(data={'key': key})
|
||||
@self.route('/<int:key_id>', methods=['GET'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(key_id: int, request_context: RequestContext) -> str:
|
||||
key = await self.ap.apikey_service.get_api_key(request_context, key_id)
|
||||
if key is None:
|
||||
return self.http_status(404, 'resource_not_found', 'API key not found')
|
||||
return self.success(data={'key': key})
|
||||
|
||||
@self.route('/<int:key_id>', methods=['GET', 'PUT', 'DELETE'])
|
||||
async def _(key_id: int) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
key = await self.ap.apikey_service.get_api_key(key_id)
|
||||
if key is None:
|
||||
return self.http_status(404, -1, 'API key not found')
|
||||
return self.success(data={'key': key})
|
||||
@self.route('/<int:key_id>', methods=['PUT'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(key_id: int, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
await self.ap.apikey_service.update_api_key(
|
||||
request_context,
|
||||
key_id,
|
||||
json_data.get('name'),
|
||||
json_data.get('description'),
|
||||
)
|
||||
except ValueError as error:
|
||||
return self.http_status(400, 'invalid_api_key', str(error))
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name')
|
||||
description = json_data.get('description')
|
||||
|
||||
await self.ap.apikey_service.update_api_key(key_id, name, description)
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.apikey_service.delete_api_key(key_id)
|
||||
return self.success()
|
||||
@self.route('/<int:key_id>', methods=['DELETE'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(key_id: int, request_context: RequestContext) -> str:
|
||||
await self.ap.apikey_service.delete_api_key(request_context, key_id)
|
||||
return self.success()
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from langbot.pkg.utils import constants
|
||||
from langbot_plugin.box.errors import BoxAdmissionError
|
||||
|
||||
from langbot.pkg.cloud.entitlements import EntitlementUnavailableError
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
from .box_visibility import should_hide_box_runtime_status
|
||||
|
||||
@@ -9,18 +13,56 @@ from .box_visibility import should_hide_box_runtime_status
|
||||
@group.group_class('box', '/api/v1/box')
|
||||
class BoxRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
status = await self.ap.box_service.get_status()
|
||||
@self.route(
|
||||
'/status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
status = await self.ap.box_service.get_status(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
status['hidden'] = should_hide_box_runtime_status(constants.edition, status.get('enabled'))
|
||||
return self.success(data=status)
|
||||
|
||||
@self.route('/sessions', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
sessions = await self.ap.box_service.get_sessions()
|
||||
@self.route(
|
||||
'/runtime-status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
del request_context
|
||||
status = await self.ap.box_service.get_backend_status()
|
||||
status['hidden'] = should_hide_box_runtime_status(constants.edition, status.get('enabled'))
|
||||
return self.success(data=status)
|
||||
|
||||
@self.route(
|
||||
'/sessions',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
sessions = await self.ap.box_service.get_sessions(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
return self.success(data=sessions)
|
||||
|
||||
@self.route('/errors', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
errors = self.ap.box_service.get_recent_errors()
|
||||
@self.route(
|
||||
'/errors',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
if getattr(self.ap.box_service, 'managed_admission_required', False):
|
||||
await self.ap.box_service.require_workspace_sandbox(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
errors = self.ap.box_service.get_recent_errors(request_context)
|
||||
return self.success(data=errors)
|
||||
|
||||
@@ -3,6 +3,9 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from ...service.secrets import redact_secrets
|
||||
from .. import group
|
||||
|
||||
|
||||
@@ -11,12 +14,29 @@ class ExtensionsRouterGroup(group.RouterGroup):
|
||||
"""Unified API for installed extensions (plugins, MCP servers, skills)."""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> quart.Response:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> quart.Response:
|
||||
if self.ap.plugin_connector.is_enable_plugin:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
async def read_in_task_scope(operation):
|
||||
tenant_scope = getattr(getattr(self.ap, 'persistence_mgr', None), 'tenant_scope', None)
|
||||
if callable(tenant_scope):
|
||||
async with tenant_scope(request_context.workspace_uuid):
|
||||
return await operation()
|
||||
return await operation()
|
||||
|
||||
plugins, mcp_servers, skills = await asyncio.gather(
|
||||
self.ap.plugin_connector.list_plugins(),
|
||||
self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True),
|
||||
self.ap.skill_service.list_skills(),
|
||||
read_in_task_scope(self.ap.plugin_connector.list_plugins),
|
||||
read_in_task_scope(
|
||||
lambda: self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
),
|
||||
read_in_task_scope(lambda: self.ap.skill_service.list_skills(request_context)),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
@@ -39,7 +59,7 @@ class ExtensionsRouterGroup(group.RouterGroup):
|
||||
extensions: list[dict] = []
|
||||
if isinstance(plugins, list):
|
||||
for plugin in plugins:
|
||||
extensions.append({'type': 'plugin', 'plugin': plugin})
|
||||
extensions.append({'type': 'plugin', 'plugin': redact_secrets(plugin)})
|
||||
if isinstance(mcp_servers, list):
|
||||
for server in mcp_servers:
|
||||
extensions.append({'type': 'mcp', 'server': server})
|
||||
|
||||
@@ -7,29 +7,53 @@ import asyncio
|
||||
|
||||
import quart.datastructures
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
def _storage_owner(context: RequestContext) -> str:
|
||||
if context.principal.account_uuid:
|
||||
return f'account:{context.principal.account_uuid}'
|
||||
if context.principal.api_key_uuid:
|
||||
return f'api-key:{context.principal.api_key_uuid}'
|
||||
return f'principal:{context.principal.principal_type.value}'
|
||||
|
||||
|
||||
@group.group_class('files', '/api/v1/files')
|
||||
class FilesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/image/<path:image_key>', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _(image_key: str) -> quart.Response:
|
||||
if '..' in image_key or '\\' in image_key:
|
||||
@self.route(
|
||||
'/image/<path:image_key>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(image_key: str, request_context: RequestContext) -> quart.Response:
|
||||
image_bytes = await self.ap.storage_mgr.resolve_public_object(
|
||||
image_key,
|
||||
expected_owner_type='upload_image',
|
||||
)
|
||||
if image_bytes is None:
|
||||
image_bytes = await self.ap.storage_mgr.resolve_public_object(
|
||||
image_key,
|
||||
expected_owner_type='bot_log',
|
||||
)
|
||||
if image_bytes is None:
|
||||
return quart.Response(status=404)
|
||||
|
||||
if not await self.ap.storage_mgr.storage_provider.exists(image_key):
|
||||
return quart.Response(status=404)
|
||||
|
||||
image_bytes = await self.ap.storage_mgr.storage_provider.load(image_key)
|
||||
mime_type = mimetypes.guess_type(image_key)[0]
|
||||
if mime_type is None:
|
||||
mime_type = 'image/jpeg'
|
||||
|
||||
return quart.Response(image_bytes, mimetype=mime_type)
|
||||
|
||||
@self.route('/images', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def upload_image() -> quart.Response:
|
||||
@self.route(
|
||||
'/images',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def upload_image(request_context: RequestContext) -> quart.Response:
|
||||
request = quart.request
|
||||
|
||||
# Check file size limit before reading the file
|
||||
@@ -66,18 +90,29 @@ class FilesRouterGroup(group.RouterGroup):
|
||||
if '/' in file_name or '\\' in file_name:
|
||||
return self.fail(400, 'File name contains invalid characters')
|
||||
|
||||
file_key = file_name + '_' + str(uuid.uuid4())[:8] + '.' + extension
|
||||
logical_key = f'{uuid.uuid4()}.{extension}'
|
||||
|
||||
# save file to storage
|
||||
await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
|
||||
file_key = await self.ap.storage_mgr.save_scoped(
|
||||
request_context,
|
||||
owner_type='upload_image',
|
||||
owner=_storage_owner(request_context),
|
||||
key=logical_key,
|
||||
value=file_bytes,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'file_key': file_key,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/documents', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def upload_document() -> quart.Response:
|
||||
@self.route(
|
||||
'/documents',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def upload_document(request_context: RequestContext) -> quart.Response:
|
||||
request = quart.request
|
||||
|
||||
# Check file size limit before reading the file
|
||||
@@ -110,12 +145,18 @@ class FilesRouterGroup(group.RouterGroup):
|
||||
if '/' in file_name or '\\' in file_name:
|
||||
return self.fail(400, 'File name contains invalid characters')
|
||||
|
||||
file_key = file_name + '_' + str(uuid.uuid4())[:8]
|
||||
logical_key = str(uuid.uuid4())
|
||||
if extension:
|
||||
file_key += '.' + extension
|
||||
logical_key += '.' + extension
|
||||
|
||||
# save file to storage
|
||||
await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
|
||||
file_key = await self.ap.storage_mgr.save_scoped(
|
||||
request_context,
|
||||
owner_type='upload_document',
|
||||
owner=_storage_owner(request_context),
|
||||
key=logical_key,
|
||||
value=file_bytes,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'file_id': file_key,
|
||||
|
||||
@@ -1,100 +1,146 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('knowledge_base', '/api/v1/knowledge/bases')
|
||||
class KnowledgeBaseRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['POST', 'GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def handle_knowledge_bases() -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
knowledge_bases = await self.ap.knowledge_service.get_knowledge_bases()
|
||||
return self.success(data={'bases': knowledge_bases})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def handle_knowledge_bases(request_context: RequestContext) -> quart.Response:
|
||||
knowledge_bases = await self.ap.knowledge_service.get_knowledge_bases(
|
||||
request_context,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
return self.success(data={'bases': knowledge_bases})
|
||||
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
knowledge_base_uuid = await self.ap.knowledge_service.create_knowledge_base(json_data)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
return self.success(data={'uuid': knowledge_base_uuid})
|
||||
|
||||
return self.http_status(405, -1, 'Method not allowed')
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def create_knowledge_base(request_context: RequestContext) -> quart.Response:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
knowledge_base_uuid = await self.ap.knowledge_service.create_knowledge_base(
|
||||
request_context,
|
||||
json_data,
|
||||
)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
return self.success(data={'uuid': knowledge_base_uuid})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>',
|
||||
methods=['GET', 'DELETE', 'PUT'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def handle_specific_knowledge_base(knowledge_base_uuid: str) -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
knowledge_base = await self.ap.knowledge_service.get_knowledge_base(knowledge_base_uuid)
|
||||
async def get_specific_knowledge_base(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
knowledge_base = await self.ap.knowledge_service.get_knowledge_base(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
if knowledge_base is None:
|
||||
return self.http_status(404, 'resource_not_found', 'knowledge base not found')
|
||||
return self.success(data={'base': knowledge_base})
|
||||
|
||||
if knowledge_base is None:
|
||||
return self.http_status(404, -1, 'knowledge base not found')
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'base': knowledge_base,
|
||||
}
|
||||
)
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>',
|
||||
methods=['DELETE', 'PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def mutate_specific_knowledge_base(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.knowledge_service.update_knowledge_base(knowledge_base_uuid, json_data)
|
||||
await self.ap.knowledge_service.update_knowledge_base(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
json_data,
|
||||
)
|
||||
return self.success(data={'uuid': knowledge_base_uuid})
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.knowledge_service.delete_knowledge_base(knowledge_base_uuid)
|
||||
return self.success({})
|
||||
await self.ap.knowledge_service.delete_knowledge_base(request_context, knowledge_base_uuid)
|
||||
return self.success({})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/files',
|
||||
methods=['GET', 'POST'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_knowledge_base_files(knowledge_base_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
files = await self.ap.knowledge_service.get_files_by_knowledge_base(knowledge_base_uuid)
|
||||
return self.success(
|
||||
data={
|
||||
'files': files,
|
||||
}
|
||||
)
|
||||
async def get_knowledge_base_files(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
files = await self.ap.knowledge_service.get_files_by_knowledge_base(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
return self.success(data={'files': files})
|
||||
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
file_id = json_data.get('file_id')
|
||||
if not file_id:
|
||||
return self.http_status(400, -1, 'File ID is required')
|
||||
|
||||
parser_plugin_id = json_data.get('parser_plugin_id')
|
||||
|
||||
# 调用服务层方法将文件与知识库关联
|
||||
task_id = await self.ap.knowledge_service.store_file(
|
||||
knowledge_base_uuid, file_id, parser_plugin_id=parser_plugin_id
|
||||
)
|
||||
return self.success(
|
||||
{
|
||||
'task_id': task_id,
|
||||
}
|
||||
)
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/files',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def add_knowledge_base_file(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
json_data = await quart.request.json
|
||||
file_id = json_data.get('file_id')
|
||||
if not file_id:
|
||||
return self.http_status(400, -1, 'File ID is required')
|
||||
parser_plugin_id = json_data.get('parser_plugin_id')
|
||||
task_id = await self.ap.knowledge_service.store_file(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
file_id,
|
||||
parser_plugin_id=parser_plugin_id,
|
||||
)
|
||||
return self.success({'task_id': task_id})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/files/<file_id>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def delete_specific_file_in_kb(file_id: str, knowledge_base_uuid: str) -> str:
|
||||
await self.ap.knowledge_service.delete_file(knowledge_base_uuid, file_id)
|
||||
async def delete_specific_file_in_kb(
|
||||
file_id: str,
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
await self.ap.knowledge_service.delete_file(request_context, knowledge_base_uuid, file_id)
|
||||
return self.success({})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/retrieve',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def retrieve_knowledge_base(knowledge_base_uuid: str) -> str:
|
||||
async def retrieve_knowledge_base(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
json_data = await quart.request.json
|
||||
query = json_data.get('query')
|
||||
|
||||
@@ -104,6 +150,9 @@ class KnowledgeBaseRouterGroup(group.RouterGroup):
|
||||
# Extract retrieval_settings to allow dynamic control over Knowledge Engine behavior (e.g. top_k, filters)
|
||||
retrieval_settings = json_data.get('retrieval_settings', {})
|
||||
results = await self.ap.knowledge_service.retrieve_knowledge_base(
|
||||
knowledge_base_uuid, query, retrieval_settings
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
query,
|
||||
retrieval_settings,
|
||||
)
|
||||
return self.success(data={'results': results})
|
||||
|
||||
@@ -1,25 +1,39 @@
|
||||
import quart
|
||||
from urllib.parse import unquote
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('knowledge_engines', '/api/v1/knowledge/engines')
|
||||
class KnowledgeEnginesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_knowledge_engines() -> quart.Response:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_knowledge_engines(request_context: RequestContext) -> quart.Response:
|
||||
"""List all available Knowledge Engines from plugins.
|
||||
|
||||
Returns a list of Knowledge Engines with their capabilities and configuration schemas.
|
||||
This is used by the frontend to render the knowledge base creation wizard.
|
||||
"""
|
||||
engines = await self.ap.knowledge_service.list_knowledge_engines()
|
||||
engines = await self.ap.knowledge_service.list_knowledge_engines(request_context)
|
||||
return self.success(data={'engines': engines})
|
||||
|
||||
@self.route(
|
||||
'/<path:plugin_id>/creation-schema', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<path:plugin_id>/creation-schema',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_engine_creation_schema(plugin_id: str) -> quart.Response:
|
||||
async def get_engine_creation_schema(
|
||||
plugin_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
"""Get creation settings schema for a specific Knowledge Engine.
|
||||
|
||||
plugin_id is in 'author/name' format, captured via <path:> converter.
|
||||
@@ -27,13 +41,19 @@ class KnowledgeEnginesRouterGroup(group.RouterGroup):
|
||||
plugin_id = unquote(plugin_id)
|
||||
if '/' not in plugin_id:
|
||||
return self.http_status(400, -1, 'Invalid plugin_id format. Expected author/name.')
|
||||
schema = await self.ap.knowledge_service.get_engine_creation_schema(plugin_id)
|
||||
schema = await self.ap.knowledge_service.get_engine_creation_schema(request_context, plugin_id)
|
||||
return self.success(data={'schema': schema})
|
||||
|
||||
@self.route(
|
||||
'/<path:plugin_id>/retrieval-schema', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<path:plugin_id>/retrieval-schema',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_engine_retrieval_schema(plugin_id: str) -> quart.Response:
|
||||
async def get_engine_retrieval_schema(
|
||||
plugin_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
"""Get retrieval settings schema for a specific Knowledge Engine.
|
||||
|
||||
plugin_id is in 'author/name' format, captured via <path:> converter.
|
||||
@@ -41,5 +61,5 @@ class KnowledgeEnginesRouterGroup(group.RouterGroup):
|
||||
plugin_id = unquote(plugin_id)
|
||||
if '/' not in plugin_id:
|
||||
return self.http_status(400, -1, 'Invalid plugin_id format. Expected author/name.')
|
||||
schema = await self.ap.knowledge_service.get_engine_retrieval_schema(plugin_id)
|
||||
schema = await self.ap.knowledge_service.get_engine_retrieval_schema(request_context, plugin_id)
|
||||
return self.success(data={'schema': schema})
|
||||
|
||||
@@ -6,8 +6,12 @@ import quart
|
||||
import sqlalchemy
|
||||
|
||||
from ... import group
|
||||
from ....authz import Permission
|
||||
from ....context import ExecutionContext, RequestContext
|
||||
from ......core import taskmgr
|
||||
from ......entity.persistence import metadata as persistence_metadata
|
||||
from ......workspace.errors import WorkspaceError, WorkspaceNotFoundError
|
||||
from ......utils import httpclient
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
LANGRAG_PLUGIN_AUTHOR = 'langbot-team'
|
||||
@@ -31,24 +35,100 @@ EXTERNAL_PLUGIN_CREATION_FIELDS: dict[str, set[str] | None] = {
|
||||
'langbot-team/FastGPTConnector': None, # all fields -> creation_settings
|
||||
}
|
||||
|
||||
_INFORMATION_SCHEMA_TABLES = sqlalchemy.table(
|
||||
'tables',
|
||||
sqlalchemy.column('table_schema'),
|
||||
sqlalchemy.column('table_name'),
|
||||
schema='information_schema',
|
||||
)
|
||||
_SQLITE_MASTER = sqlalchemy.table(
|
||||
'sqlite_master',
|
||||
sqlalchemy.column('type'),
|
||||
sqlalchemy.column('name'),
|
||||
)
|
||||
_LEGACY_KNOWLEDGE_BASE_BACKUP = sqlalchemy.table(
|
||||
'knowledge_bases_backup',
|
||||
sqlalchemy.column('uuid'),
|
||||
sqlalchemy.column('name'),
|
||||
sqlalchemy.column('description'),
|
||||
sqlalchemy.column('emoji'),
|
||||
sqlalchemy.column('embedding_model_uuid'),
|
||||
sqlalchemy.column('top_k'),
|
||||
sqlalchemy.column('created_at'),
|
||||
sqlalchemy.column('updated_at'),
|
||||
)
|
||||
_LEGACY_EXTERNAL_KNOWLEDGE_BASE = sqlalchemy.table(
|
||||
'external_knowledge_bases',
|
||||
sqlalchemy.column('uuid'),
|
||||
sqlalchemy.column('name'),
|
||||
sqlalchemy.column('description'),
|
||||
sqlalchemy.column('emoji'),
|
||||
sqlalchemy.column('plugin_author'),
|
||||
sqlalchemy.column('plugin_name'),
|
||||
sqlalchemy.column('retriever_config'),
|
||||
sqlalchemy.column('created_at'),
|
||||
)
|
||||
_CURRENT_KNOWLEDGE_BASE = sqlalchemy.table(
|
||||
'knowledge_bases',
|
||||
sqlalchemy.column('uuid'),
|
||||
sqlalchemy.column('workspace_uuid'),
|
||||
sqlalchemy.column('name'),
|
||||
sqlalchemy.column('description'),
|
||||
sqlalchemy.column('emoji'),
|
||||
sqlalchemy.column('created_at'),
|
||||
sqlalchemy.column('updated_at'),
|
||||
sqlalchemy.column('knowledge_engine_plugin_id'),
|
||||
sqlalchemy.column('collection_id'),
|
||||
sqlalchemy.column('creation_settings'),
|
||||
sqlalchemy.column('retrieval_settings'),
|
||||
)
|
||||
|
||||
|
||||
@group.group_class('knowledge/migration', '/api/v1/knowledge/migration')
|
||||
class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
async def _get_migration_flag(self) -> bool:
|
||||
async def _require_local_migration_context(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
) -> ExecutionContext:
|
||||
"""Fence legacy-table migration to the OSS singleton Workspace.
|
||||
|
||||
The backup tables predate Workspace scoping and are deliberately
|
||||
instance-global. A cloud projection must therefore never be allowed
|
||||
to inspect or restore them, even when it has a valid execution lease.
|
||||
"""
|
||||
try:
|
||||
binding = await self.ap.workspace_service.get_local_execution_binding(
|
||||
execution_context.workspace_uuid,
|
||||
expected_generation=execution_context.placement_generation,
|
||||
)
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except WorkspaceError as exc:
|
||||
raise WorkspaceNotFoundError('RAG migration is unavailable') from exc
|
||||
|
||||
if binding.instance_uuid != execution_context.instance_uuid:
|
||||
raise WorkspaceNotFoundError('RAG migration is unavailable')
|
||||
return ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
)
|
||||
|
||||
async def _get_migration_flag(self, execution_context: ExecutionContext) -> bool:
|
||||
"""Check if rag_plugin_migration_needed flag is set."""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_metadata.Metadata).where(
|
||||
persistence_metadata.Metadata.key == 'rag_plugin_migration_needed'
|
||||
)
|
||||
sqlalchemy.select(persistence_metadata.WorkspaceMetadata.value)
|
||||
.where(persistence_metadata.WorkspaceMetadata.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_metadata.WorkspaceMetadata.key == 'rag_plugin_migration_needed')
|
||||
)
|
||||
row = result.first()
|
||||
return row is not None and row.value == 'true'
|
||||
return result.scalar_one_or_none() == 'true'
|
||||
|
||||
async def _set_migration_flag(self, value: str):
|
||||
async def _set_migration_flag(self, execution_context: ExecutionContext, value: str):
|
||||
"""Set rag_plugin_migration_needed flag."""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_metadata.Metadata)
|
||||
.where(persistence_metadata.Metadata.key == 'rag_plugin_migration_needed')
|
||||
sqlalchemy.update(persistence_metadata.WorkspaceMetadata)
|
||||
.where(persistence_metadata.WorkspaceMetadata.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_metadata.WorkspaceMetadata.key == 'rag_plugin_migration_needed')
|
||||
.values(value=value)
|
||||
)
|
||||
|
||||
@@ -56,35 +136,47 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
"""Check if a table exists."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
|
||||
).bindparams(table_name=table_name)
|
||||
sqlalchemy.select(_INFORMATION_SCHEMA_TABLES.c.table_name)
|
||||
.where(_INFORMATION_SCHEMA_TABLES.c.table_schema == 'public')
|
||||
.where(_INFORMATION_SCHEMA_TABLES.c.table_name == table_name)
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar()
|
||||
return result.first() is not None
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
|
||||
table_name=table_name
|
||||
)
|
||||
sqlalchemy.select(_SQLITE_MASTER.c.name)
|
||||
.where(_SQLITE_MASTER.c.type == 'table')
|
||||
.where(_SQLITE_MASTER.c.name == table_name)
|
||||
.limit(1)
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _install_plugin_from_marketplace(
|
||||
self, plugin_id: str, task_context: taskmgr.TaskContext, space_url: str
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
plugin_id: str,
|
||||
task_context: taskmgr.TaskContext,
|
||||
space_url: str,
|
||||
) -> None:
|
||||
"""Install a single plugin from the marketplace."""
|
||||
p_author, p_name = plugin_id.split('/', 1)
|
||||
self.ap.logger.info(f'RAG migration: installing plugin {plugin_id} from marketplace...')
|
||||
task_context.trace(f'Installing plugin {plugin_id} from marketplace...')
|
||||
|
||||
async with httpx.AsyncClient(trust_env=True, timeout=15) as client:
|
||||
async with httpx.AsyncClient(
|
||||
trust_env=True,
|
||||
timeout=15,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(),
|
||||
) as client:
|
||||
resp = await client.get(f'{space_url}/api/v1/marketplace/plugins/{p_author}/{p_name}')
|
||||
resp.raise_for_status()
|
||||
p_data = resp.json().get('data', {}).get('plugin', {})
|
||||
response_data = await httpclient.parse_json_response(resp)
|
||||
p_data = response_data.get('data', {}).get('plugin', {})
|
||||
p_version = p_data.get('latest_version')
|
||||
if not p_version:
|
||||
raise Exception(f'Could not determine latest version for {plugin_id}')
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
await self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE,
|
||||
{
|
||||
@@ -96,8 +188,15 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
)
|
||||
self.ap.logger.info(f'RAG migration: plugin {plugin_id} install request sent.')
|
||||
|
||||
async def _execute_rag_migration(self, task_context: taskmgr.TaskContext, install_plugin: bool = True):
|
||||
async def _execute_rag_migration(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
task_context: taskmgr.TaskContext,
|
||||
install_plugin: bool = True,
|
||||
):
|
||||
"""Execute RAG migration: install required plugins and restore backup data."""
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
warnings = []
|
||||
|
||||
# Collect all plugins we need: LangRAG (always) + connector plugins (from external KBs)
|
||||
@@ -108,7 +207,10 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
has_external = await self._table_exists('external_knowledge_bases')
|
||||
if has_external:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT DISTINCT plugin_author, plugin_name FROM external_knowledge_bases;')
|
||||
sqlalchemy.select(
|
||||
_LEGACY_EXTERNAL_KNOWLEDGE_BASE.c.plugin_author,
|
||||
_LEGACY_EXTERNAL_KNOWLEDGE_BASE.c.plugin_name,
|
||||
).distinct()
|
||||
)
|
||||
for row in result.fetchall():
|
||||
plugin_author = row[0] or ''
|
||||
@@ -127,7 +229,14 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
|
||||
for plugin_id in needed_plugins:
|
||||
try:
|
||||
await self._install_plugin_from_marketplace(plugin_id, task_context, space_url)
|
||||
await self._install_plugin_from_marketplace(
|
||||
execution_context,
|
||||
plugin_id,
|
||||
task_context,
|
||||
space_url,
|
||||
)
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'RAG migration: plugin {plugin_id} install returned: {e}')
|
||||
task_context.trace(f'Plugin install note ({plugin_id}): {e}')
|
||||
@@ -141,8 +250,11 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
engine_id_set: set[str] = set()
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
if all(pid in engine_id_set for pid in needed_plugins):
|
||||
@@ -158,17 +270,18 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
await asyncio.sleep(2)
|
||||
else:
|
||||
try:
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except Exception:
|
||||
engine_id_set = set()
|
||||
|
||||
# Step 3: Restore internal knowledge bases from backup
|
||||
task_context.trace('Restoring internal knowledge bases...', action='restore-internal')
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM knowledge_bases_backup;')
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(_LEGACY_KNOWLEDGE_BASE_BACKUP))
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
@@ -183,30 +296,30 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
created_at = row_dict.get('created_at')
|
||||
updated_at = row_dict.get('updated_at')
|
||||
|
||||
# DB migration 20 created these columns as TEXT, while a fresh
|
||||
# schema uses SQLAlchemy JSON. Keep the statement structured,
|
||||
# but retain untyped bound values so both physical schemas and
|
||||
# SQLite's string-valued legacy DATETIME rows remain valid.
|
||||
creation_settings = json.dumps({'embedding_model_uuid': embedding_model_uuid})
|
||||
retrieval_settings = json.dumps({'top_k': top_k})
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
sqlalchemy.insert(_CURRENT_KNOWLEDGE_BASE).values(
|
||||
uuid=kb_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
plugin_id=LANGRAG_PLUGIN_ID,
|
||||
knowledge_engine_plugin_id=LANGRAG_PLUGIN_ID,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=retrieval_settings,
|
||||
)
|
||||
)
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
try:
|
||||
config = {'embedding_model_uuid': embedding_model_uuid}
|
||||
await self.ap.plugin_connector.rag_on_kb_create(LANGRAG_PLUGIN_ID, kb_uuid, config)
|
||||
@@ -221,9 +334,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
# Step 4: Restore external knowledge bases
|
||||
task_context.trace('Restoring external knowledge bases...', action='restore-external')
|
||||
if has_external:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(_LEGACY_EXTERNAL_KNOWLEDGE_BASE))
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
@@ -266,20 +377,15 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
retrieval_settings_dict = {k: v for k, v in retriever_config.items() if k not in creation_fields}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
sqlalchemy.insert(_CURRENT_KNOWLEDGE_BASE).values(
|
||||
uuid=kb_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=created_at,
|
||||
plugin_id=external_plugin_id,
|
||||
knowledge_engine_plugin_id=external_plugin_id,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=json.dumps(creation_settings_dict),
|
||||
retrieval_settings=json.dumps(retrieval_settings_dict),
|
||||
@@ -294,6 +400,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
else:
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
try:
|
||||
await self.ap.plugin_connector.rag_on_kb_create(
|
||||
external_plugin_id, kb_uuid, creation_settings_dict
|
||||
@@ -307,16 +414,23 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
await self.ap.rag_mgr.load_knowledge_bases_from_db()
|
||||
|
||||
# Step 5: Clear migration flag
|
||||
await self._set_migration_flag('false')
|
||||
await self._set_migration_flag(execution_context, 'false')
|
||||
task_context.trace('RAG migration completed.', action='done')
|
||||
|
||||
if warnings:
|
||||
task_context.trace(f'Completed with {len(warnings)} warning(s).')
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
@self.route(
|
||||
'/status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
needed = await self._get_migration_flag(execution_context)
|
||||
|
||||
internal_kb_count = 0
|
||||
external_kb_count = 0
|
||||
@@ -324,13 +438,13 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
if needed:
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases_backup;')
|
||||
sqlalchemy.select(sqlalchemy.func.count()).select_from(_LEGACY_KNOWLEDGE_BASE_BACKUP)
|
||||
)
|
||||
internal_kb_count = result.scalar() or 0
|
||||
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
sqlalchemy.select(sqlalchemy.func.count()).select_from(_LEGACY_EXTERNAL_KNOWLEDGE_BASE)
|
||||
)
|
||||
external_kb_count = result.scalar() or 0
|
||||
|
||||
@@ -342,9 +456,16 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
@self.route(
|
||||
'/execute',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
needed = await self._get_migration_flag(execution_context)
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
@@ -353,20 +474,34 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._execute_rag_migration(task_context=ctx, install_plugin=install_plugin),
|
||||
self._execute_rag_migration(
|
||||
execution_context,
|
||||
task_context=ctx,
|
||||
install_plugin=install_plugin,
|
||||
),
|
||||
kind='rag-migration',
|
||||
name='rag-migration-execute',
|
||||
label='Migrating knowledge bases to plugin architecture',
|
||||
context=ctx,
|
||||
instance_uuid=execution_context.instance_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
placement_generation=execution_context.placement_generation,
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/dismiss', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
@self.route(
|
||||
'/dismiss',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
needed = await self._get_migration_flag(execution_context)
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
await self._set_migration_flag('false')
|
||||
await self._set_migration_flag(execution_context, 'false')
|
||||
return self.success()
|
||||
|
||||
@@ -1,16 +1,24 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('parsers', '/api/v1/knowledge/parsers')
|
||||
class ParsersRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_parsers() -> quart.Response:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_parsers(request_context: RequestContext) -> quart.Response:
|
||||
"""List all available parsers from plugins.
|
||||
|
||||
Optional query parameter `mime_type` to filter parsers by supported MIME type.
|
||||
"""
|
||||
mime_type = quart.request.args.get('mime_type')
|
||||
parsers = await self.ap.knowledge_service.list_parsers(mime_type)
|
||||
parsers = await self.ap.knowledge_service.list_parsers(request_context, mime_type)
|
||||
return self.success(data={'parsers': parsers})
|
||||
|
||||
@@ -3,14 +3,23 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('logs', '/api/v1/logs')
|
||||
class LogsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route('', methods=['GET'], permission=Permission.AUDIT_VIEW)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
# The process log is instance-global. It is safe to expose only in
|
||||
# the OSS singleton Workspace; SaaS must use Workspace-scoped
|
||||
# observability records instead of leaking another tenant's lines.
|
||||
await self.ap.workspace_service.get_local_execution_binding(
|
||||
request_context.workspace_uuid,
|
||||
expected_generation=request_context.placement_generation,
|
||||
)
|
||||
start_page_number = int(quart.request.args.get('start_page_number', 0))
|
||||
start_offset = int(quart.request.args.get('start_offset', 0))
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import datetime
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@@ -24,8 +26,8 @@ def parse_iso_datetime(datetime_str: str | None) -> datetime.datetime | None:
|
||||
@group.group_class('monitoring', '/api/v1/monitoring')
|
||||
class MonitoringRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/overview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_overview() -> str:
|
||||
@self.route('/overview', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_overview(request_context: RequestContext) -> str:
|
||||
"""Get overview metrics"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -38,6 +40,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
metrics = await self.ap.monitoring_service.get_overview_metrics(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -46,8 +49,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=metrics)
|
||||
|
||||
@self.route('/token-statistics', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_token_statistics() -> str:
|
||||
@self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_token_statistics(request_context: RequestContext) -> str:
|
||||
"""Get detailed token usage statistics (summary, per-model, timeseries)."""
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
@@ -61,6 +64,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
stats = await self.ap.monitoring_service.get_token_statistics(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -70,8 +74,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=stats)
|
||||
|
||||
@self.route('/messages', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_messages() -> str:
|
||||
@self.route('/messages', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_messages(request_context: RequestContext) -> str:
|
||||
"""Get message logs"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -87,6 +91,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
messages, total = await self.ap.monitoring_service.get_messages(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
session_ids=session_ids if session_ids else None,
|
||||
@@ -105,8 +110,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/llm-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_llm_calls() -> str:
|
||||
@self.route('/llm-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_llm_calls(request_context: RequestContext) -> str:
|
||||
"""Get LLM call records"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -121,6 +126,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
llm_calls, total = await self.ap.monitoring_service.get_llm_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -138,8 +144,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/tool-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_tool_calls() -> str:
|
||||
@self.route('/tool-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_tool_calls(request_context: RequestContext) -> str:
|
||||
"""Get tool call records"""
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
@@ -153,6 +159,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
tool_calls, total = await self.ap.monitoring_service.get_tool_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
session_ids=session_ids if session_ids else None,
|
||||
@@ -171,8 +178,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/embedding-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_embedding_calls() -> str:
|
||||
@self.route('/embedding-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_embedding_calls(request_context: RequestContext) -> str:
|
||||
"""Get embedding call records"""
|
||||
# Parse query parameters
|
||||
start_time_str = quart.request.args.get('startTime')
|
||||
@@ -186,6 +193,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
embedding_calls, total = await self.ap.monitoring_service.get_embedding_calls(
|
||||
request_context,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
knowledge_base_id=knowledge_base_id if knowledge_base_id else None,
|
||||
@@ -202,8 +210,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/sessions', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_sessions() -> str:
|
||||
@self.route('/sessions', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_sessions(request_context: RequestContext) -> str:
|
||||
"""Get session information"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -224,6 +232,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
is_active = is_active_str.lower() == 'true'
|
||||
|
||||
sessions, total = await self.ap.monitoring_service.get_sessions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -242,8 +251,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/errors', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_errors() -> str:
|
||||
@self.route('/errors', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_errors(request_context: RequestContext) -> str:
|
||||
"""Get error logs"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -258,6 +267,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
errors, total = await self.ap.monitoring_service.get_errors(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -275,8 +285,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/data', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_all_data() -> str:
|
||||
@self.route('/data', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_all_data(request_context: RequestContext) -> str:
|
||||
"""Get all monitoring data in a single request"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -291,6 +301,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get overview metrics
|
||||
overview = await self.ap.monitoring_service.get_overview_metrics(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -299,6 +310,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get messages
|
||||
messages, messages_total = await self.ap.monitoring_service.get_messages(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -309,6 +321,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get LLM calls
|
||||
llm_calls, llm_calls_total = await self.ap.monitoring_service.get_llm_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -319,6 +332,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get tool calls
|
||||
tool_calls, tool_calls_total = await self.ap.monitoring_service.get_tool_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -329,6 +343,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get sessions
|
||||
sessions, sessions_total = await self.ap.monitoring_service.get_sessions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -340,6 +355,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get errors
|
||||
errors, errors_total = await self.ap.monitoring_service.get_errors(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -350,6 +366,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get embedding calls
|
||||
embedding_calls, embedding_calls_total = await self.ap.monitoring_service.get_embedding_calls(
|
||||
request_context,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
@@ -376,27 +393,27 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/sessions/<session_id>/analysis', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_session_analysis(session_id: str) -> str:
|
||||
@self.route('/sessions/<session_id>/analysis', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_session_analysis(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Get detailed analysis for a specific session"""
|
||||
analysis = await self.ap.monitoring_service.get_session_analysis(session_id)
|
||||
analysis = await self.ap.monitoring_service.get_session_analysis(request_context, session_id)
|
||||
|
||||
# Always return success with the analysis data
|
||||
# The frontend will handle the 'found: false' case
|
||||
return self.success(data=analysis)
|
||||
|
||||
@self.route('/messages/<message_id>/details', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_message_details(message_id: str) -> str:
|
||||
@self.route('/messages/<message_id>/details', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_message_details(message_id: str, request_context: RequestContext) -> str:
|
||||
"""Get detailed information for a specific message"""
|
||||
details = await self.ap.monitoring_service.get_message_details(message_id)
|
||||
details = await self.ap.monitoring_service.get_message_details(request_context, message_id)
|
||||
|
||||
if not details.get('found'):
|
||||
return self.error(message=f'Message {message_id} not found', code=404)
|
||||
|
||||
return self.success(data=details)
|
||||
|
||||
@self.route('/export', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def export_data() -> tuple[str, int]:
|
||||
@self.route('/export', methods=['GET'], permission=Permission.DATA_EXPORT)
|
||||
async def export_data(request_context: RequestContext) -> tuple[str, int]:
|
||||
"""Export monitoring data as CSV"""
|
||||
# Parse query parameters
|
||||
export_type = quart.request.args.get('type', 'messages')
|
||||
@@ -413,6 +430,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
# Get data based on export type
|
||||
if export_type == 'messages':
|
||||
data = await self.ap.monitoring_service.export_messages(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -437,6 +455,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'llm-calls':
|
||||
data = await self.ap.monitoring_service.export_llm_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -463,6 +482,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'embedding-calls':
|
||||
data = await self.ap.monitoring_service.export_embedding_calls(
|
||||
request_context,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
@@ -485,6 +505,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'errors':
|
||||
data = await self.ap.monitoring_service.export_errors(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -506,6 +527,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'sessions':
|
||||
data = await self.ap.monitoring_service.export_sessions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -527,6 +549,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'feedback':
|
||||
data = await self.ap.monitoring_service.export_feedback(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -581,8 +604,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return response, 200
|
||||
|
||||
@self.route('/feedback/stats', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_feedback_stats() -> str:
|
||||
@self.route('/feedback/stats', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_feedback_stats(request_context: RequestContext) -> str:
|
||||
"""Get feedback statistics"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -595,6 +618,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
stats = await self.ap.monitoring_service.get_feedback_stats(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -603,8 +627,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=stats)
|
||||
|
||||
@self.route('/feedback', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_feedback() -> str:
|
||||
@self.route('/feedback', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_feedback(request_context: RequestContext) -> str:
|
||||
"""Get feedback list"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -623,6 +647,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
feedback_type = int(feedback_type_str) if feedback_type_str else None
|
||||
|
||||
feedback_list, total = await self.ap.monitoring_service.get_feedback_list(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
feedback_type=feedback_type,
|
||||
|
||||
@@ -20,10 +20,12 @@ import httpx
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ......utils import paths
|
||||
from ......platform.sources.websocket_manager import is_valid_session_id, ws_connection_manager
|
||||
from ......utils import httpclient, paths
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, is_valid_session_id, ws_connection_manager
|
||||
from .websocket_chat import create_scoped_duplex_tasks, wait_for_duplex_tasks
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_AUTH_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# Cache the widget template content
|
||||
_widget_template_cache: str | None = None
|
||||
@@ -58,37 +60,31 @@ def _get_logo_bytes() -> bytes:
|
||||
class EmbedRouterGroup(group.RouterGroup):
|
||||
# -- helpers -------------------------------------------------------------
|
||||
|
||||
def _resolve_bot(self, bot_uuid: str):
|
||||
async def _resolve_bot(self, bot_uuid: str):
|
||||
"""Resolve *bot_uuid* to ``(runtime_bot, pipeline_uuid)``.
|
||||
|
||||
Returns ``(None, None)`` when the bot does not exist, is not a
|
||||
``web_page_bot``, is disabled, or has no pipeline bound.
|
||||
"""
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if (
|
||||
bot.bot_entity.uuid == bot_uuid
|
||||
and bot.bot_entity.adapter == 'web_page_bot'
|
||||
and bot.bot_entity.enable
|
||||
and bot.bot_entity.use_pipeline_uuid
|
||||
):
|
||||
return bot, bot.bot_entity.use_pipeline_uuid
|
||||
bot = await self.ap.platform_mgr.resolve_public_bot(bot_uuid)
|
||||
if (
|
||||
bot is not None
|
||||
and bot.bot_entity.adapter == 'web_page_bot'
|
||||
and bot.bot_entity.enable
|
||||
and bot.bot_entity.use_pipeline_uuid
|
||||
):
|
||||
return bot, bot.bot_entity.use_pipeline_uuid
|
||||
return None, None
|
||||
|
||||
def _get_bot_config(self, bot_uuid: str) -> dict:
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if bot.bot_entity.uuid == bot_uuid and bot.bot_entity.adapter == 'web_page_bot':
|
||||
return bot.bot_entity.adapter_config
|
||||
return {}
|
||||
@staticmethod
|
||||
def _get_bot_config(runtime_bot) -> dict:
|
||||
return runtime_bot.bot_entity.adapter_config
|
||||
|
||||
async def _verify_session_token(self, request, bot_uuid: str) -> bool:
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
def _verify_session_token_value(self, token: str, runtime_bot) -> bool:
|
||||
config = self._get_bot_config(runtime_bot)
|
||||
secret = config.get('turnstile_secret_key', '')
|
||||
if not secret:
|
||||
return True
|
||||
auth_header = request.headers.get('Authorization', '')
|
||||
if not auth_header.startswith('Bearer '):
|
||||
return False
|
||||
token = auth_header[7:]
|
||||
try:
|
||||
ts_str, mac = token.split('.', 1)
|
||||
ts = float(ts_str)
|
||||
@@ -99,6 +95,50 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def _verify_session_token(self, request, runtime_bot) -> bool:
|
||||
auth_header = request.headers.get('Authorization', '')
|
||||
token = auth_header[7:] if auth_header.startswith('Bearer ') else ''
|
||||
return self._verify_session_token_value(token, runtime_bot)
|
||||
|
||||
async def _authenticate_websocket(self, runtime_bot) -> None:
|
||||
"""Require the embed session token as the first WebSocket frame."""
|
||||
|
||||
raw_message = await asyncio.wait_for(quart.websocket.receive(), timeout=_AUTH_TIMEOUT_SECONDS)
|
||||
payload = await asyncio.to_thread(json.loads, raw_message)
|
||||
if not isinstance(payload, dict) or payload.get('type') != 'authenticate':
|
||||
raise ValueError('Authentication is required')
|
||||
token = str(payload.get('token') or '')
|
||||
if not self._verify_session_token_value(token, runtime_bot):
|
||||
raise ValueError('Authentication is required')
|
||||
|
||||
async def _assert_execution_active(self, runtime_bot) -> None:
|
||||
context = runtime_bot.execution_context
|
||||
await self.ap.workspace_service.get_execution_binding(
|
||||
context.workspace_uuid,
|
||||
expected_generation=context.placement_generation,
|
||||
)
|
||||
|
||||
async def _resolve_connected_bot(self, owner_bot, pipeline_uuid: str):
|
||||
"""Re-resolve mutable bot state before every public message."""
|
||||
current_bot, current_pipeline_uuid = await self._resolve_bot(owner_bot.bot_entity.uuid)
|
||||
if current_bot is None or current_pipeline_uuid != pipeline_uuid:
|
||||
raise RuntimeError('Bot is unavailable')
|
||||
|
||||
owner_context = owner_bot.execution_context
|
||||
current_context = current_bot.execution_context
|
||||
if (
|
||||
current_context.instance_uuid,
|
||||
current_context.workspace_uuid,
|
||||
current_context.placement_generation,
|
||||
) != (
|
||||
owner_context.instance_uuid,
|
||||
owner_context.workspace_uuid,
|
||||
owner_context.placement_generation,
|
||||
):
|
||||
raise RuntimeError('Bot is unavailable')
|
||||
await self._assert_execution_active(current_bot)
|
||||
return current_bot
|
||||
|
||||
# -- routes --------------------------------------------------------------
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@@ -106,7 +146,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def verify_turnstile(bot_uuid: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
try:
|
||||
@@ -115,18 +155,18 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not token:
|
||||
return self.http_status(400, -1, 'Token is required')
|
||||
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
config = self._get_bot_config(runtime_bot)
|
||||
secret = config.get('turnstile_secret_key', '')
|
||||
if not secret:
|
||||
ts = time.time()
|
||||
return self.success(data={'token': f'{ts}.dummy'})
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(event_hooks=httpclient.httpx_response_limit_hooks()) as client:
|
||||
resp = await client.post(
|
||||
'https://challenges.cloudflare.com/turnstile/v0/siteverify',
|
||||
data={'secret': secret, 'response': token},
|
||||
)
|
||||
result = resp.json()
|
||||
result = await httpclient.parse_json_response(resp)
|
||||
|
||||
if not result.get('success'):
|
||||
return self.http_status(403, -1, 'Turnstile verification failed')
|
||||
@@ -146,7 +186,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
"""Serve the embed widget JavaScript with injected configuration."""
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return quart.Response(
|
||||
'// Bot not found or not available', status=404, content_type='application/javascript'
|
||||
@@ -164,7 +204,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not re.match(r'^https?://[a-zA-Z0-9._:/-]+$', base_url):
|
||||
base_url = quart.request.host_url.rstrip('/')
|
||||
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
config = self._get_bot_config(runtime_bot)
|
||||
site_key = config.get('turnstile_site_key', '')
|
||||
locale = config.get('language', 'en_US') or 'en_US'
|
||||
bubble_icon = config.get('bubble_icon', 'logo') or 'logo'
|
||||
@@ -194,10 +234,10 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def get_embed_messages(bot_uuid: str, session_type: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
if not await self._verify_session_token(quart.request, runtime_bot):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
@@ -207,7 +247,8 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not is_valid_session_id(session_id):
|
||||
return self.http_status(400, -1, 'Valid session_id is required')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
@@ -222,10 +263,10 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def reset_embed_session(bot_uuid: str, session_type: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
if not await self._verify_session_token(quart.request, runtime_bot):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
@@ -235,7 +276,8 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not is_valid_session_id(session_id):
|
||||
return self.http_status(400, -1, 'Valid session_id is required')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
@@ -250,10 +292,10 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def submit_feedback(bot_uuid: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
if not await self._verify_session_token(quart.request, runtime_bot):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
data = await quart.request.get_json()
|
||||
@@ -266,6 +308,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
feedback_id = f'embed_{uuid.uuid4().hex[:12]}'
|
||||
|
||||
await self.ap.monitoring_service.record_feedback(
|
||||
runtime_bot.execution_context,
|
||||
feedback_id=feedback_id,
|
||||
feedback_type=feedback_type,
|
||||
bot_id=runtime_bot.bot_entity.uuid,
|
||||
@@ -286,11 +329,12 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
@self.quart_app.websocket(self.path + '/<bot_uuid>/ws/connect')
|
||||
async def embed_websocket_connect(bot_uuid: str):
|
||||
"""WebSocket connection for embed widget, keyed by bot_uuid."""
|
||||
await quart.websocket.accept()
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'}))
|
||||
return
|
||||
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'}))
|
||||
return
|
||||
@@ -307,18 +351,42 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Valid session_id is required'}))
|
||||
return
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
try:
|
||||
await self._authenticate_websocket(runtime_bot)
|
||||
await self._assert_execution_active(runtime_bot)
|
||||
except Exception:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
|
||||
return
|
||||
|
||||
try:
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
return
|
||||
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
websocket=quart.websocket._get_current_object(),
|
||||
scope=WebSocketScope.from_context(runtime_bot.execution_context),
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
session_type=session_type,
|
||||
session_id=session_id,
|
||||
metadata={'user_agent': quart.websocket.headers.get('User-Agent', '')},
|
||||
send_queue_size=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('send_queue_size', 100)
|
||||
),
|
||||
max_connections=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections', 1024)
|
||||
),
|
||||
max_connections_per_workspace=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections_per_workspace', 32)
|
||||
),
|
||||
)
|
||||
|
||||
await quart.websocket.send(
|
||||
@@ -338,11 +406,19 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
f'(bot={bot_uuid}, pipeline={pipeline_uuid}, session_type={session_type})'
|
||||
)
|
||||
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter, runtime_bot))
|
||||
send_task = asyncio.create_task(self._handle_send(connection))
|
||||
receive_task, send_task = create_scoped_duplex_tasks(
|
||||
self._handle_receive(
|
||||
connection,
|
||||
websocket_adapter,
|
||||
runtime_bot,
|
||||
pipeline_uuid,
|
||||
),
|
||||
self._handle_send(connection),
|
||||
runtime_bot.execution_context.workspace_uuid,
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.gather(receive_task, send_task)
|
||||
await wait_for_duplex_tasks(receive_task, send_task)
|
||||
except Exception as e:
|
||||
logger.error(f'Embed WebSocket task error: {e}')
|
||||
finally:
|
||||
@@ -357,14 +433,14 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
|
||||
# -- WebSocket receive/send helpers --------------------------------------
|
||||
|
||||
async def _handle_receive(self, connection, websocket_adapter, owner_bot):
|
||||
async def _handle_receive(self, connection, websocket_adapter, owner_bot, pipeline_uuid: str):
|
||||
try:
|
||||
while connection.is_active:
|
||||
message = await quart.websocket.receive()
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
|
||||
if message_type == 'ping':
|
||||
@@ -372,7 +448,12 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=owner_bot)
|
||||
try:
|
||||
current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
|
||||
except Exception:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
|
||||
@@ -383,13 +464,20 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
logger.error(f'Embed receive error: {e}', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
try:
|
||||
connection.send_queue.put_nowait(None)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
async def _handle_send(self, connection):
|
||||
try:
|
||||
while connection.is_active:
|
||||
while connection.is_active or not connection.send_queue.empty():
|
||||
try:
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
await quart.websocket.send(json.dumps(message))
|
||||
if message is None:
|
||||
break
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception as e:
|
||||
|
||||
@@ -2,120 +2,156 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ....service.secrets import redact_secrets
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('pipelines', '/api/v1/pipelines')
|
||||
class PipelinesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
sort_by = quart.request.args.get('sort_by', 'created_at')
|
||||
sort_order = quart.request.args.get('sort_order', 'DESC')
|
||||
return self.success(
|
||||
data={'pipelines': await self.ap.pipeline_service.get_pipelines(sort_by, sort_order)}
|
||||
)
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(json_data)
|
||||
|
||||
return self.success(data={'uuid': pipeline_uuid})
|
||||
|
||||
@self.route('/_/metadata', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
return self.success(data={'configs': await self.ap.pipeline_service.get_pipeline_metadata()})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
sort_by = quart.request.args.get('sort_by', 'created_at')
|
||||
sort_order = quart.request.args.get('sort_order', 'DESC')
|
||||
include_secret = has_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
return self.success(
|
||||
data={
|
||||
'pipelines': await self.ap.pipeline_service.get_pipelines(
|
||||
request_context,
|
||||
sort_by,
|
||||
sort_order,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@self.route(
|
||||
'/<pipeline_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(request_context, await quart.request.json)
|
||||
return self.success(data={'uuid': pipeline_uuid})
|
||||
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
@self.route(
|
||||
'/_/metadata',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data={'configs': await self.ap.pipeline_service.get_pipeline_metadata(request_context)})
|
||||
|
||||
return self.success(data={'pipeline': pipeline})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@self.route(
|
||||
'/<pipeline_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
return self.success(data={'pipeline': pipeline})
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline_uuid, json_data)
|
||||
@self.route(
|
||||
'/<pipeline_uuid>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
try:
|
||||
await self.ap.pipeline_service.update_pipeline(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
else:
|
||||
await self.ap.pipeline_service.delete_pipeline(request_context, pipeline_uuid)
|
||||
return self.success()
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.pipeline_service.delete_pipeline(pipeline_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<pipeline_uuid>/copy', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(pipeline_uuid: str) -> str:
|
||||
@self.route(
|
||||
'/<pipeline_uuid>/copy',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
new_uuid = await self.ap.pipeline_service.copy_pipeline(pipeline_uuid)
|
||||
new_uuid = await self.ap.pipeline_service.copy_pipeline(request_context, pipeline_uuid)
|
||||
return self.success(data={'uuid': new_uuid})
|
||||
except ValueError as e:
|
||||
return self.http_status(404, -1, str(e))
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@self.route(
|
||||
'/<pipeline_uuid>/extensions', methods=['GET', 'PUT'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<pipeline_uuid>/extensions',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(pipeline_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
# Get current extensions and available plugins
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
# Only include plugins with pipeline-related components (Command, EventListener, Tool)
|
||||
# Plugins that only have KnowledgeEngine components are not suitable for pipeline extensions
|
||||
pipeline_component_kinds = ['Command', 'EventListener', 'Tool']
|
||||
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True)
|
||||
pipeline_component_kinds = ['Command', 'EventListener', 'Tool']
|
||||
if self.ap.plugin_connector.is_enable_plugin:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
available_skills = await self.ap.skill_service.list_skills(request_context)
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
|
||||
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
|
||||
'bound_plugins': extensions_prefs.get('plugins', []),
|
||||
'available_plugins': redact_secrets(plugins),
|
||||
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
|
||||
'available_mcp_servers': mcp_servers,
|
||||
'bound_mcp_resources': extensions_prefs.get('mcp_resources', []),
|
||||
'mcp_resource_agent_read_enabled': extensions_prefs.get('mcp_resource_agent_read_enabled', True),
|
||||
'bound_skills': extensions_prefs.get('skills', []),
|
||||
'available_skills': available_skills,
|
||||
}
|
||||
)
|
||||
|
||||
# Get available skills
|
||||
available_skills = await self.ap.skill_service.list_skills()
|
||||
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
|
||||
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
|
||||
'bound_plugins': extensions_prefs.get('plugins', []),
|
||||
'available_plugins': plugins,
|
||||
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
|
||||
'available_mcp_servers': mcp_servers,
|
||||
'bound_mcp_resources': extensions_prefs.get('mcp_resources', []),
|
||||
'mcp_resource_agent_read_enabled': extensions_prefs.get(
|
||||
'mcp_resource_agent_read_enabled', True
|
||||
),
|
||||
'bound_skills': extensions_prefs.get('skills', []),
|
||||
'available_skills': available_skills,
|
||||
}
|
||||
)
|
||||
elif quart.request.method == 'PUT':
|
||||
# Update bound plugins and MCP servers for this pipeline
|
||||
json_data = await quart.request.json
|
||||
enable_all_plugins = json_data.get('enable_all_plugins', True)
|
||||
enable_all_mcp_servers = json_data.get('enable_all_mcp_servers', True)
|
||||
enable_all_skills = json_data.get('enable_all_skills', True)
|
||||
bound_plugins = json_data.get('bound_plugins', [])
|
||||
bound_mcp_servers = json_data.get('bound_mcp_servers', [])
|
||||
bound_skills = json_data.get('bound_skills', [])
|
||||
bound_mcp_resources = json_data.get('bound_mcp_resources')
|
||||
mcp_resource_agent_read_enabled = json_data.get('mcp_resource_agent_read_enabled')
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
pipeline_uuid,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
enable_all_plugins,
|
||||
enable_all_mcp_servers,
|
||||
bound_skills=bound_skills,
|
||||
enable_all_skills=enable_all_skills,
|
||||
bound_mcp_resources=bound_mcp_resources,
|
||||
mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled,
|
||||
)
|
||||
|
||||
return self.success()
|
||||
@self.route(
|
||||
'/<pipeline_uuid>/extensions',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
json_data.get('bound_plugins', []),
|
||||
json_data.get('bound_mcp_servers', []),
|
||||
json_data.get('enable_all_plugins', True),
|
||||
json_data.get('enable_all_mcp_servers', True),
|
||||
bound_skills=json_data.get('bound_skills', []),
|
||||
enable_all_skills=json_data.get('enable_all_skills', True),
|
||||
bound_mcp_resources=json_data.get('bound_mcp_resources'),
|
||||
mcp_resource_agent_read_enabled=json_data.get('mcp_resource_agent_read_enabled'),
|
||||
)
|
||||
return self.success()
|
||||
|
||||
@@ -1,64 +1,234 @@
|
||||
"""WebSocket聊天路由 - 支持双向实时通信"""
|
||||
"""Authenticated dashboard WebSocket chat routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, permissions_for_role, require_permission
|
||||
from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from ... import group
|
||||
from ......platform.sources.websocket_manager import ws_connection_manager
|
||||
from ......core.task_boundary import run_in_workspace_uow
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
|
||||
from ......utils import bounded_executor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_AUTH_TIMEOUT_SECONDS = 10.0
|
||||
_DUPLEX_DRAIN_TIMEOUT_SECONDS = 0.25
|
||||
|
||||
|
||||
def create_scoped_duplex_tasks(
|
||||
receive_coro: typing.Coroutine[typing.Any, typing.Any, None],
|
||||
send_coro: typing.Coroutine[typing.Any, typing.Any, None],
|
||||
workspace_uuid: str,
|
||||
) -> tuple[asyncio.Task[None], asyncio.Task[None]]:
|
||||
"""Create both socket directions under one trusted Workspace budget."""
|
||||
|
||||
return (
|
||||
asyncio.create_task(
|
||||
bounded_executor.run_in_blocking_work_scope(
|
||||
receive_coro,
|
||||
workspace_uuid,
|
||||
)
|
||||
),
|
||||
asyncio.create_task(
|
||||
bounded_executor.run_in_blocking_work_scope(
|
||||
send_coro,
|
||||
workspace_uuid,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def wait_for_duplex_tasks(
|
||||
receive_task: asyncio.Task,
|
||||
send_task: asyncio.Task,
|
||||
) -> None:
|
||||
"""Stop the peer direction as soon as either socket task terminates."""
|
||||
|
||||
try:
|
||||
done, _ = await asyncio.wait(
|
||||
{receive_task, send_task},
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
# A receive task may enqueue a terminal authorization/error frame and
|
||||
# then finish. Give the sender a short deterministic drain window
|
||||
# instead of cancelling it before that frame reaches the client.
|
||||
if receive_task in done and not send_task.done():
|
||||
await asyncio.wait(
|
||||
{send_task},
|
||||
timeout=_DUPLEX_DRAIN_TIMEOUT_SECONDS,
|
||||
)
|
||||
finally:
|
||||
for task in (receive_task, send_task):
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(
|
||||
receive_task,
|
||||
send_task,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
|
||||
@group.group_class('websocket_chat', '/api/v1/pipelines/<pipeline_uuid>/ws')
|
||||
class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
async def _authenticate_websocket(self) -> tuple[RequestContext, str]:
|
||||
"""Authenticate the first dashboard WebSocket message.
|
||||
|
||||
Browsers cannot attach the normal Authorization/X-Workspace-Id headers
|
||||
to a WebSocket handshake. The client therefore sends one auth frame
|
||||
immediately after opening the socket; no connection is registered and
|
||||
no runtime object is resolved before this method succeeds.
|
||||
"""
|
||||
|
||||
raw_message = await asyncio.wait_for(quart.websocket.receive(), timeout=_AUTH_TIMEOUT_SECONDS)
|
||||
payload = await asyncio.to_thread(json.loads, raw_message)
|
||||
if not isinstance(payload, dict) or payload.get('type') != 'authenticate':
|
||||
raise ValueError('Authentication is required')
|
||||
|
||||
token = str(payload.get('token') or '').strip()
|
||||
workspace_uuid = str(payload.get('workspace_uuid') or '').strip()
|
||||
if not token or not workspace_uuid:
|
||||
raise ValueError('Authentication is required')
|
||||
|
||||
account, _ = await self._authenticate_account(token)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
if not isinstance(account_uuid, str) or collaboration_service is None:
|
||||
raise ValueError('Workspace authentication is unavailable')
|
||||
|
||||
access = await collaboration_service.resolve_account_workspace(account_uuid, workspace_uuid)
|
||||
request_context = RequestContext(
|
||||
instance_uuid=access.execution.instance_uuid,
|
||||
placement_generation=access.execution.placement_generation,
|
||||
request_id=quart.websocket.headers.get('X-Request-Id') or str(uuid.uuid4()),
|
||||
auth_type=group.AuthType.USER_TOKEN.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.ACCOUNT,
|
||||
account_uuid=account_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=access.workspace.uuid,
|
||||
membership_uuid=access.membership.uuid,
|
||||
role=access.membership.role,
|
||||
permissions=permissions_for_role(access.membership.role),
|
||||
membership_revision=access.membership.projection_revision,
|
||||
),
|
||||
)
|
||||
require_permission(request_context, Permission.RUNTIME_OPERATE)
|
||||
return request_context, token
|
||||
|
||||
async def _revalidate_websocket_authorization(
|
||||
self,
|
||||
request_context: RequestContext,
|
||||
token: str,
|
||||
) -> RequestContext:
|
||||
"""Recheck revocable account, membership, permission, and placement state."""
|
||||
|
||||
account, _ = await self._authenticate_account(token)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
if account_uuid != request_context.account_uuid:
|
||||
raise ValueError('WebSocket account changed')
|
||||
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
if collaboration_service is None or not isinstance(account_uuid, str):
|
||||
raise ValueError('Workspace authentication is unavailable')
|
||||
access = await collaboration_service.resolve_account_workspace(
|
||||
account_uuid,
|
||||
request_context.workspace_uuid,
|
||||
)
|
||||
if (
|
||||
access.workspace.uuid != request_context.workspace_uuid
|
||||
or access.membership.uuid != request_context.workspace.membership_uuid
|
||||
or access.membership.projection_revision != request_context.workspace.membership_revision
|
||||
or access.execution.instance_uuid != request_context.instance_uuid
|
||||
or access.execution.placement_generation != request_context.placement_generation
|
||||
):
|
||||
raise ValueError('WebSocket authorization changed')
|
||||
|
||||
current_context = RequestContext(
|
||||
instance_uuid=access.execution.instance_uuid,
|
||||
placement_generation=access.execution.placement_generation,
|
||||
request_id=request_context.request_id,
|
||||
auth_type=request_context.auth_type,
|
||||
principal=request_context.principal,
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=access.workspace.uuid,
|
||||
membership_uuid=access.membership.uuid,
|
||||
role=access.membership.role,
|
||||
permissions=permissions_for_role(access.membership.role),
|
||||
membership_revision=access.membership.projection_revision,
|
||||
),
|
||||
entitlement_revision=request_context.entitlement_revision,
|
||||
)
|
||||
require_permission(current_context, Permission.RUNTIME_OPERATE)
|
||||
return current_context
|
||||
|
||||
async def _get_scoped_adapter(self, request_context: RequestContext, pipeline_uuid: str):
|
||||
pipeline = await run_in_workspace_uow(
|
||||
self.ap,
|
||||
request_context.workspace_uuid,
|
||||
lambda: self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid),
|
||||
)
|
||||
if pipeline is None:
|
||||
return None
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(request_context)
|
||||
return proxy_bot.adapter
|
||||
|
||||
async def initialize(self) -> None:
|
||||
# 直接使用 quart_app 注册 WebSocket 路由
|
||||
@self.quart_app.websocket(self.path + '/connect')
|
||||
async def websocket_connect(pipeline_uuid: str):
|
||||
"""
|
||||
建立WebSocket连接
|
||||
"""Open one authenticated dashboard debug connection."""
|
||||
|
||||
URL参数:
|
||||
- pipeline_uuid: 流水线UUID
|
||||
- session_type: 会话类型 (person/group)
|
||||
"""
|
||||
await quart.websocket.accept()
|
||||
try:
|
||||
# 获取参数 - 在WebSocket上下文中使用 quart.websocket.args
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
request_context, token = await self._authenticate_websocket()
|
||||
except Exception:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
|
||||
return
|
||||
|
||||
if session_type not in ['person', 'group']:
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
if session_type not in ['person', 'group']:
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Pipeline not found'}))
|
||||
return
|
||||
|
||||
# 获取WebSocket适配器
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
return
|
||||
|
||||
# Dashboard pipeline-debug sessions must always run under the
|
||||
# built-in websocket_proxy_bot identity. We deliberately do NOT
|
||||
# resolve a web_page_bot owner here — even if one is bound to
|
||||
# the same pipeline, debug requests must not be attributed to
|
||||
# it. The embed widget path (`/api/v1/embed/<bot>/ws/connect`)
|
||||
# is the one that carries the page-bot identity.
|
||||
|
||||
# 注册连接
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
websocket=quart.websocket._get_current_object(),
|
||||
scope=WebSocketScope.from_context(request_context),
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
session_type=session_type,
|
||||
metadata={'user_agent': quart.websocket.headers.get('User-Agent', '')},
|
||||
send_queue_size=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('send_queue_size', 100)
|
||||
),
|
||||
max_connections=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections', 1024)
|
||||
),
|
||||
max_connections_per_workspace=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections_per_workspace', 32)
|
||||
),
|
||||
)
|
||||
|
||||
# 发送连接成功消息
|
||||
await quart.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
@@ -72,182 +242,188 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f'WebSocket connection established: {connection.connection_id} '
|
||||
f'(pipeline={pipeline_uuid}, session_type={session_type})'
|
||||
f'Dashboard WebSocket connected: {connection.connection_id} '
|
||||
f'(workspace={connection.workspace_uuid}, pipeline={pipeline_uuid}, '
|
||||
f'session_type={session_type})'
|
||||
)
|
||||
|
||||
# 创建接收和发送任务
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter))
|
||||
send_task = asyncio.create_task(self._handle_send(connection))
|
||||
|
||||
# 等待任务完成
|
||||
receive_task, send_task = create_scoped_duplex_tasks(
|
||||
self._handle_receive(
|
||||
connection,
|
||||
websocket_adapter,
|
||||
request_context,
|
||||
token,
|
||||
),
|
||||
self._handle_send(connection),
|
||||
request_context.workspace_uuid,
|
||||
)
|
||||
try:
|
||||
await asyncio.gather(receive_task, send_task)
|
||||
except Exception as e:
|
||||
logger.error(f'WebSocket task execution error: {e}')
|
||||
await wait_for_duplex_tasks(receive_task, send_task)
|
||||
except Exception as exc:
|
||||
logger.error(f'WebSocket task execution error: {exc}')
|
||||
finally:
|
||||
# 清理连接
|
||||
await ws_connection_manager.remove_connection(connection.connection_id)
|
||||
logger.debug(f'WebSocket connection cleaned: {connection.connection_id}')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'WebSocket connection error: {e}', exc_info=True)
|
||||
except Exception:
|
||||
logger.error('Dashboard WebSocket connection error', exc_info=True)
|
||||
try:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': str(e)}))
|
||||
except:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@self.route('/messages/<session_type>', methods=['GET'])
|
||||
async def get_messages(pipeline_uuid: str, session_type: str) -> str:
|
||||
"""获取消息历史"""
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
@self.route(
|
||||
'/messages/<session_type>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def get_messages(
|
||||
pipeline_uuid: str,
|
||||
session_type: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
messages = websocket_adapter.get_websocket_messages(pipeline_uuid, session_type)
|
||||
return self.success(data={'messages': messages})
|
||||
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
@self.route(
|
||||
'/reset/<session_type>',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def reset_session(
|
||||
pipeline_uuid: str,
|
||||
session_type: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
messages = websocket_adapter.get_websocket_messages(pipeline_uuid, session_type)
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
websocket_adapter.reset_session(pipeline_uuid, session_type)
|
||||
return self.success(data={'message': 'Session reset successfully'})
|
||||
|
||||
return self.success(data={'messages': messages})
|
||||
@self.route(
|
||||
'/connections',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def get_connections(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
if await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid) is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
@self.route('/reset/<session_type>', methods=['POST'])
|
||||
async def reset_session(pipeline_uuid: str, session_type: str) -> str:
|
||||
"""重置会话"""
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
websocket_adapter.reset_session(pipeline_uuid, session_type)
|
||||
|
||||
return self.success(data={'message': 'Session reset successfully'})
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
@self.route('/connections', methods=['GET'])
|
||||
async def get_connections(pipeline_uuid: str) -> str:
|
||||
"""获取当前连接统计"""
|
||||
try:
|
||||
stats = ws_connection_manager.get_stats()
|
||||
connections = await ws_connection_manager.get_connections_by_pipeline(pipeline_uuid)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'stats': stats,
|
||||
'connections': [
|
||||
{
|
||||
'connection_id': conn.connection_id,
|
||||
'session_type': conn.session_type,
|
||||
'created_at': conn.created_at.isoformat(),
|
||||
'last_active': conn.last_active.isoformat(),
|
||||
'is_active': conn.is_active,
|
||||
}
|
||||
for conn in connections
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
@self.route('/broadcast', methods=['POST'])
|
||||
async def broadcast_message(pipeline_uuid: str) -> str:
|
||||
"""向所有连接广播消息(后端主动推送)"""
|
||||
try:
|
||||
data = await quart.request.get_json()
|
||||
message = data.get('message')
|
||||
|
||||
if not message:
|
||||
return self.http_status(400, -1, 'message is required')
|
||||
|
||||
# 广播消息
|
||||
broadcast_data = {
|
||||
'type': 'broadcast',
|
||||
'message': message,
|
||||
'timestamp': datetime.datetime.now().isoformat(),
|
||||
scope = WebSocketScope.from_context(request_context)
|
||||
stats = ws_connection_manager.get_stats(scope=scope)
|
||||
connections = await ws_connection_manager.get_connections_by_pipeline(
|
||||
pipeline_uuid,
|
||||
scope=scope,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'stats': stats,
|
||||
'connections': [
|
||||
{
|
||||
'connection_id': connection.connection_id,
|
||||
'session_type': connection.session_type,
|
||||
'created_at': connection.created_at.isoformat(),
|
||||
'last_active': connection.last_active.isoformat(),
|
||||
'is_active': connection.is_active,
|
||||
}
|
||||
for connection in connections
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
await ws_connection_manager.broadcast_to_pipeline(pipeline_uuid, broadcast_data)
|
||||
@self.route(
|
||||
'/broadcast',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def broadcast_message(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
if await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid) is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
|
||||
return self.success(data={'message': 'Broadcast sent successfully'})
|
||||
data = await quart.request.get_json()
|
||||
message = data.get('message')
|
||||
if not message:
|
||||
return self.http_status(400, -1, 'message is required')
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
broadcast_data = {
|
||||
'type': 'broadcast',
|
||||
'message': message,
|
||||
'timestamp': datetime.datetime.now().isoformat(),
|
||||
}
|
||||
await ws_connection_manager.broadcast_to_pipeline(
|
||||
pipeline_uuid,
|
||||
broadcast_data,
|
||||
scope=WebSocketScope.from_context(request_context),
|
||||
)
|
||||
return self.success(data={'message': 'Broadcast sent successfully'})
|
||||
|
||||
async def _handle_receive(self, connection, websocket_adapter):
|
||||
"""处理接收消息的任务"""
|
||||
async def _handle_receive(
|
||||
self,
|
||||
connection,
|
||||
websocket_adapter,
|
||||
request_context: RequestContext,
|
||||
token: str,
|
||||
):
|
||||
try:
|
||||
while connection.is_active:
|
||||
# 接收消息
|
||||
message = await quart.websocket.receive()
|
||||
|
||||
# 更新活跃时间
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
|
||||
if message_type == 'ping':
|
||||
# 心跳响应
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
|
||||
elif message_type == 'message':
|
||||
# 处理用户消息
|
||||
logger.debug(f'收到消息: {data} from {connection.connection_id}')
|
||||
|
||||
# 处理消息(不等待响应,响应会通过broadcast异步发送)
|
||||
# owner_bot is intentionally NOT passed: the dashboard
|
||||
# debug WebSocket must always run under the proxy bot,
|
||||
# never under a coincidentally-bound web_page_bot.
|
||||
try:
|
||||
await self._revalidate_websocket_authorization(request_context, token)
|
||||
except Exception:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data)
|
||||
|
||||
elif message_type == 'disconnect':
|
||||
# 客户端主动断开
|
||||
logger.debug(f'Client disconnected: {connection.connection_id}')
|
||||
break
|
||||
|
||||
else:
|
||||
logger.warning(f'Unknown message type: {message_type}')
|
||||
|
||||
logger.warning(f'Unknown WebSocket message type: {message_type}')
|
||||
except json.JSONDecodeError:
|
||||
logger.error(f'Invalid JSON message: {message}')
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Receive message error: {e}', exc_info=True)
|
||||
except Exception:
|
||||
logger.error('Dashboard WebSocket receive error', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
try:
|
||||
connection.send_queue.put_nowait(None)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
async def _handle_send(self, connection):
|
||||
"""处理发送消息的任务"""
|
||||
try:
|
||||
while connection.is_active:
|
||||
# 从队列获取消息
|
||||
while connection.is_active or not connection.send_queue.empty():
|
||||
try:
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
|
||||
# 发送消息
|
||||
await quart.websocket.send(json.dumps(message))
|
||||
|
||||
if message is None:
|
||||
break
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
except asyncio.TimeoutError:
|
||||
# 超时继续循环
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Send message error: {e}', exc_info=True)
|
||||
except Exception:
|
||||
logger.error('Dashboard WebSocket send error', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
|
||||
@@ -1,8 +1,133 @@
|
||||
import quart
|
||||
import mimetypes
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import mimetypes
|
||||
|
||||
import quart
|
||||
|
||||
from langbot.pkg.api.http.authz import Permission
|
||||
from langbot.pkg.api.http.context import RequestContext
|
||||
from langbot.pkg.core.errors import TaskCapacityError
|
||||
from langbot.pkg.utils import httpclient, importutil
|
||||
|
||||
from ... import group
|
||||
from langbot.pkg.utils import importutil
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class _AdapterSessionScope:
|
||||
"""Immutable tenant and principal binding for a credential exchange."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
principal_type: str
|
||||
account_uuid: str | None
|
||||
api_key_uuid: str | None
|
||||
|
||||
@classmethod
|
||||
def from_request_context(cls, request_context: RequestContext) -> '_AdapterSessionScope':
|
||||
principal = request_context.principal
|
||||
return cls(
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
principal_type=principal.principal_type.value,
|
||||
account_uuid=principal.account_uuid,
|
||||
api_key_uuid=principal.api_key_uuid,
|
||||
)
|
||||
|
||||
def matches(self, request_context: RequestContext) -> bool:
|
||||
"""Return whether a request is from the exact initiating tenant principal."""
|
||||
|
||||
return self == self.from_request_context(request_context)
|
||||
|
||||
|
||||
def _bind_session_scope(session: dict, request_context: RequestContext) -> None:
|
||||
session['scope'] = _AdapterSessionScope.from_request_context(request_context)
|
||||
|
||||
|
||||
def _get_owned_session(
|
||||
sessions: dict[str, dict],
|
||||
session_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> dict | None:
|
||||
"""Resolve a session without revealing sessions owned by another scope."""
|
||||
|
||||
session = sessions.get(session_id)
|
||||
scope = session.get('scope') if session is not None else None
|
||||
if not isinstance(scope, _AdapterSessionScope) or not scope.matches(request_context):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def _pop_owned_session(
|
||||
sessions: dict[str, dict],
|
||||
session_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> dict | None:
|
||||
"""Remove an owned session without allowing cross-scope cancellation."""
|
||||
|
||||
session = _get_owned_session(sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return None
|
||||
return sessions.pop(session_id, None)
|
||||
|
||||
|
||||
_MAX_ADAPTER_SESSIONS = 100
|
||||
_MAX_ADAPTER_SESSIONS_PER_WORKSPACE = 10
|
||||
|
||||
|
||||
def _start_adapter_session_task(
|
||||
ap,
|
||||
coro,
|
||||
*,
|
||||
adapter: str,
|
||||
session_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> asyncio.Task | None:
|
||||
"""Attach one credential exchange to tenant admission and app shutdown."""
|
||||
|
||||
try:
|
||||
wrapper = ap.task_mgr.create_user_task(
|
||||
coro,
|
||||
kind='platform-adapter-credential-exchange',
|
||||
name=f'{adapter}-credential-{session_id}',
|
||||
label=f'{adapter} credential exchange',
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
)
|
||||
except TaskCapacityError:
|
||||
coro.close()
|
||||
return None
|
||||
return wrapper.task
|
||||
|
||||
|
||||
def _make_room_for_session(
|
||||
sessions: dict[str, dict],
|
||||
request_context: RequestContext,
|
||||
) -> None:
|
||||
"""Bound credential-exchange sessions globally and per workspace."""
|
||||
|
||||
workspace_uuid = request_context.workspace_uuid
|
||||
owned = [
|
||||
(session_id, session)
|
||||
for session_id, session in sessions.items()
|
||||
if getattr(session.get('scope'), 'workspace_uuid', None) == workspace_uuid
|
||||
]
|
||||
evict_workspace_session = len(owned) >= _MAX_ADAPTER_SESSIONS_PER_WORKSPACE
|
||||
evict_global_session = len(sessions) >= _MAX_ADAPTER_SESSIONS
|
||||
if not evict_workspace_session and not evict_global_session:
|
||||
return
|
||||
|
||||
candidates = owned if evict_workspace_session else list(sessions.items())
|
||||
session_id, _ = min(
|
||||
candidates,
|
||||
key=lambda item: float(item[1].get('created_at', 0.0)),
|
||||
)
|
||||
session = sessions.pop(session_id, None)
|
||||
task = session.get('task') if session is not None else None
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
|
||||
|
||||
def _decrypt_qqofficial_secret(encrypted_b64: str, key: bytes) -> str:
|
||||
@@ -84,8 +209,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/lark/create-app', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/lark/create-app', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start Feishu one-click app registration. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -106,6 +231,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_create_app_sessions, request_context)
|
||||
_create_app_sessions[session_id] = session
|
||||
|
||||
def on_qr_code(info):
|
||||
@@ -137,7 +264,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_registration())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_registration(),
|
||||
adapter='lark',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_create_app_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -160,10 +296,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/lark/create-app/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/lark/create-app/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll registration status."""
|
||||
session = _create_app_sessions.get(session_id)
|
||||
_cleanup_expired_sessions()
|
||||
session = _get_owned_session(_create_app_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -179,10 +320,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/lark/create-app/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/lark/create-app/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a registration session."""
|
||||
session = _create_app_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_create_app_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -206,8 +353,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/weixin/login', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/weixin/login', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start WeChat QR code login. Returns session_id + QR code data URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -229,6 +376,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_weixin_login_sessions, request_context)
|
||||
_weixin_login_sessions[session_id] = session
|
||||
|
||||
client = OpenClawWeixinClient(
|
||||
@@ -267,7 +416,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
task = asyncio.create_task(run_login())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_login(),
|
||||
adapter='weixin',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_weixin_login_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -290,10 +448,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/weixin/login/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/weixin/login/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll WeChat login status."""
|
||||
session = _weixin_login_sessions.get(session_id)
|
||||
_cleanup_expired_weixin_sessions()
|
||||
session = _get_owned_session(_weixin_login_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -317,10 +480,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/weixin/login/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/weixin/login/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a WeChat login session."""
|
||||
session = _weixin_login_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_weixin_login_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -344,8 +513,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/dingtalk/create-app', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/dingtalk/create-app', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start DingTalk one-click app creation via Device Flow. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -368,6 +537,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'device_code': None,
|
||||
'interval': 5,
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_dingtalk_sessions, request_context)
|
||||
_dingtalk_sessions[session_id] = session
|
||||
|
||||
async def run_device_flow():
|
||||
@@ -380,7 +551,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
json={'source': 'langbot'},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from DingTalk service'
|
||||
@@ -397,7 +568,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
json={'nonce': nonce},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from DingTalk service'
|
||||
@@ -428,7 +599,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
json={'device_code': device_code},
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json()
|
||||
poll_data = await httpclient.read_json_limited(poll_resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -464,7 +635,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_device_flow())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_device_flow(),
|
||||
adapter='dingtalk',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_dingtalk_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -491,11 +671,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/dingtalk/create-app/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/dingtalk/create-app/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll DingTalk Device Flow status."""
|
||||
_cleanup_expired_dingtalk_sessions()
|
||||
session = _dingtalk_sessions.get(session_id)
|
||||
session = _get_owned_session(_dingtalk_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -511,10 +695,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/dingtalk/create-app/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/dingtalk/create-app/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a DingTalk Device Flow session."""
|
||||
session = _dingtalk_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_dingtalk_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -538,8 +728,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/wecombot/create-bot', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/wecombot/create-bot', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start WeComBot one-click creation via QR code. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -563,6 +753,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'scode': None,
|
||||
'task': None,
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_wecombot_sessions, request_context)
|
||||
_wecombot_sessions[session_id] = session
|
||||
|
||||
async def run_qr_flow():
|
||||
@@ -574,7 +766,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
f'{WECOM_QC_GENERATE_URL}?source=langbot&plat=0',
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from WeCom service'
|
||||
@@ -601,7 +793,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
f'{WECOM_QC_QUERY_URL}?scode={scode}',
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json()
|
||||
poll_data = await httpclient.read_json_limited(poll_resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -628,7 +820,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_qr_flow())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_qr_flow(),
|
||||
adapter='wecombot',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_wecombot_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -655,11 +856,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/wecombot/create-bot/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/wecombot/create-bot/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll WeComBot creation status."""
|
||||
_cleanup_expired_wecombot_sessions()
|
||||
session = _wecombot_sessions.get(session_id)
|
||||
session = _get_owned_session(_wecombot_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -675,10 +880,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/wecombot/create-bot/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/wecombot/create-bot/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a WeComBot creation session."""
|
||||
session = _wecombot_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_wecombot_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -702,8 +913,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/qqofficial/bind', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/qqofficial/bind', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start QQ Official QR binding. Returns session_id + QR URL.
|
||||
|
||||
Flow: generate a local AES-256 key, register it with
|
||||
@@ -739,6 +950,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'bind_key_bytes': bind_key_bytes,
|
||||
'interval': 2,
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_qqofficial_sessions, request_context)
|
||||
_qqofficial_sessions[session_id] = session
|
||||
|
||||
async def run_qr_binding():
|
||||
@@ -752,7 +965,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
headers={'Accept': 'application/json'},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json(content_type=None)
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from QQ bind service'
|
||||
@@ -790,7 +1003,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
headers={'Accept': 'application/json'},
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json(content_type=None)
|
||||
poll_data = await httpclient.read_json_limited(poll_resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -843,7 +1056,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_qr_binding())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_qr_binding(),
|
||||
adapter='qqofficial',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_qqofficial_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait up to 10s for the QR URL to be ready before responding.
|
||||
@@ -870,11 +1092,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/qqofficial/bind/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/qqofficial/bind/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll QQ Official QR binding status."""
|
||||
_cleanup_expired_qqofficial_sessions()
|
||||
session = _qqofficial_sessions.get(session_id)
|
||||
session = _get_owned_session(_qqofficial_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -892,10 +1118,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/qqofficial/bind/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/qqofficial/bind/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a QQ Official QR binding session."""
|
||||
session = _qqofficial_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_qqofficial_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
|
||||
@@ -1,45 +1,95 @@
|
||||
import quart
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('bots', '/api/v1/platform/bots')
|
||||
class BotsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
return self.success(data={'bots': await self.ap.bot_service.get_bots()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
bot_uuid = await self.ap.bot_service.create_bot(json_data)
|
||||
return self.success(data={'uuid': bot_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
include_secret = has_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
return self.success(
|
||||
data={
|
||||
'bots': await self.ap.bot_service.get_bots(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<bot_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
bot = await self.ap.bot_service.get_runtime_bot_info(bot_uuid)
|
||||
if bot is None:
|
||||
return self.http_status(404, -1, 'bot not found')
|
||||
return self.success(data={'bot': bot})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.bot_service.update_bot(bot_uuid, json_data)
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.bot_service.delete_bot(bot_uuid)
|
||||
return self.success()
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
bot_uuid = await self.ap.bot_service.create_bot(request_context, json_data)
|
||||
return self.success(data={'uuid': bot_uuid})
|
||||
|
||||
@self.route('/<bot_uuid>/logs', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
@self.route(
|
||||
'/<bot_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
include_secret = has_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
bot = await self.ap.bot_service.get_runtime_bot_info(
|
||||
request_context,
|
||||
bot_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
if bot is None:
|
||||
return self.http_status(404, -1, 'bot not found')
|
||||
return self.success(data={'bot': bot})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.bot_service.update_bot(request_context, bot_uuid, json_data)
|
||||
else:
|
||||
await self.ap.bot_service.delete_bot(request_context, bot_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/logs',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
from_index = json_data.get('from_index', -1)
|
||||
max_count = json_data.get('max_count', 10)
|
||||
logs, total_count = await self.ap.bot_service.list_event_logs(bot_uuid, from_index, max_count)
|
||||
logs, total_count = await self.ap.bot_service.list_event_logs(
|
||||
request_context, bot_uuid, from_index, max_count
|
||||
)
|
||||
return self.success(data={'logs': logs, 'total_count': total_count})
|
||||
|
||||
@self.route('/<bot_uuid>/send_message', methods=['POST'], auth_type=group.AuthType.API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
@self.route(
|
||||
'/<bot_uuid>/send_message',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
target_type = json_data.get('target_type')
|
||||
target_id = json_data.get('target_id')
|
||||
@@ -54,37 +104,51 @@ class BotsRouterGroup(group.RouterGroup):
|
||||
if target_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'target_type must be either "person" or "group"')
|
||||
|
||||
try:
|
||||
await self.ap.bot_service.send_message(bot_uuid, target_type, target_id, message_chain_data)
|
||||
return self.success(data={'sent': True})
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return self.http_status(500, -1, f'Failed to send message: {str(e)}')
|
||||
|
||||
# ============ Bot Admins ============
|
||||
|
||||
@self.route('/<bot_uuid>/admins', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
admins = await self.ap.bot_service.get_bot_admins(bot_uuid)
|
||||
return self.success(data={'admins': admins})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
launcher_type = json_data.get('launcher_type', '').strip()
|
||||
launcher_id = str(json_data.get('launcher_id', '')).strip()
|
||||
if not launcher_type or not launcher_id:
|
||||
return self.http_status(400, -1, 'launcher_type and launcher_id are required')
|
||||
try:
|
||||
admin_id = await self.ap.bot_service.add_bot_admin(bot_uuid, launcher_type, launcher_id)
|
||||
return self.success(data={'id': admin_id})
|
||||
except Exception as e:
|
||||
return self.http_status(409, -1, str(e))
|
||||
await self.ap.bot_service.send_message(
|
||||
request_context,
|
||||
bot_uuid,
|
||||
target_type,
|
||||
target_id,
|
||||
message_chain_data,
|
||||
)
|
||||
return self.success(data={'sent': True})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/admins/<int:admin_id>', methods=['DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<bot_uuid>/admins',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, admin_id: int) -> str:
|
||||
await self.ap.bot_service.delete_bot_admin(bot_uuid, admin_id)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
admins = await self.ap.bot_service.get_bot_admins(request_context, bot_uuid)
|
||||
return self.success(data={'admins': admins})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/admins',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
launcher_type = json_data.get('launcher_type', '').strip()
|
||||
launcher_id = str(json_data.get('launcher_id', '')).strip()
|
||||
if not launcher_type or not launcher_id:
|
||||
return self.http_status(400, -1, 'launcher_type and launcher_id are required')
|
||||
try:
|
||||
admin_id = await self.ap.bot_service.add_bot_admin(
|
||||
request_context, bot_uuid, launcher_type, launcher_id
|
||||
)
|
||||
return self.success(data={'id': admin_id})
|
||||
except IntegrityError as e:
|
||||
return self.http_status(409, -1, str(e))
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/admins/<int:admin_id>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(bot_uuid: str, admin_id: int, request_context: RequestContext) -> str:
|
||||
await self.ap.bot_service.delete_bot_admin(request_context, bot_uuid, admin_id)
|
||||
return self.success()
|
||||
|
||||
@@ -1,23 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import collections.abc
|
||||
import copy
|
||||
import quart
|
||||
import re
|
||||
import httpx
|
||||
import uuid
|
||||
import os
|
||||
import zipfile
|
||||
import yaml
|
||||
from urllib.parse import urlparse
|
||||
import posixpath
|
||||
import sqlalchemy
|
||||
|
||||
from .....core import taskmgr
|
||||
from .....core.task_boundary import run_in_workspace_uow
|
||||
from .....entity.persistence import plugin as persistence_plugin
|
||||
from ...authz import Permission
|
||||
from ...context import ExecutionContext, RequestContext
|
||||
from .. import group
|
||||
from .....workspace.errors import WorkspaceNotFoundError
|
||||
from .....plugin.github import validate_github_plugin_install_info
|
||||
from .....plugin.archive import inspect_plugin_archive_metadata
|
||||
from .....utils import httpclient
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
|
||||
_SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
_SENSITIVE_CONFIG_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'apikey',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_CONFIG_TOKENS = frozenset(
|
||||
{
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def _is_sensitive_config_key(key: object) -> bool:
|
||||
normalized = _normalize_config_key(key)
|
||||
if normalized in _SENSITIVE_CONFIG_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_CONFIG_TOKENS:
|
||||
return True
|
||||
return 'key' in tokens and bool(tokens & _SENSITIVE_KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def _mask_secret_structure(value):
|
||||
"""Mask every non-empty leaf while preserving container structure."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: _mask_secret_structure(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_mask_secret_structure(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_mask_secret_structure(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return _SECRET_MASK
|
||||
|
||||
|
||||
def redact_plugin_secrets(value):
|
||||
"""Return a recursively redacted copy of plugin-facing data."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (_mask_secret_structure(item) if _is_sensitive_config_key(key) else redact_plugin_secrets(item))
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_plugin_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_plugin_secrets(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def restore_plugin_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from the current config before a management write."""
|
||||
|
||||
if sensitive and value == _SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked plugin secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: restore_plugin_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or _is_sensitive_config_key(key),
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_plugin_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_plugin_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
# Resolve the built-in page SDK JS from the langbot_plugin package
|
||||
_PAGE_SDK_PATH = None
|
||||
try:
|
||||
@@ -148,18 +283,78 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'subdir': subdir,
|
||||
}
|
||||
|
||||
async def _check_extensions_limit(self) -> str | None:
|
||||
async def _check_extensions_limit(self, request_context: RequestContext) -> str | None:
|
||||
"""Check if extensions limit is reached. Returns error response if limit exceeded, None otherwise."""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_extensions = limitation.get('max_extensions', -1)
|
||||
if max_extensions >= 0:
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers()
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(request_context)
|
||||
total_extensions = len(plugins) + len(mcp_servers)
|
||||
if total_extensions >= max_extensions:
|
||||
return self.http_status(400, -1, f'Maximum number of extensions ({max_extensions}) reached')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _task_scope(request_context: RequestContext) -> dict[str, str | int]:
|
||||
return {
|
||||
'instance_uuid': request_context.instance_uuid,
|
||||
'workspace_uuid': request_context.workspace_uuid,
|
||||
'placement_generation': request_context.placement_generation,
|
||||
}
|
||||
|
||||
async def _run_fenced_plugin_operation(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
operation: collections.abc.Callable[[], collections.abc.Awaitable],
|
||||
):
|
||||
"""Revalidate a captured task context immediately before Runtime I/O."""
|
||||
|
||||
await run_in_workspace_uow(
|
||||
self.ap,
|
||||
execution_context.workspace_uuid,
|
||||
lambda: self.ap.plugin_connector.require_workspace_context(execution_context),
|
||||
)
|
||||
return await operation()
|
||||
|
||||
async def _require_public_plugin_runtime_context(self) -> ExecutionContext:
|
||||
"""Resolve public assets only for the OSS singleton Workspace.
|
||||
|
||||
Public image and iframe requests cannot carry the WebUI bearer token.
|
||||
They therefore remain available for the one-Workspace Core deployment,
|
||||
but fail closed instead of guessing a Workspace when multi-Workspace
|
||||
policy is active.
|
||||
"""
|
||||
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
policy = getattr(workspace_service, 'policy', None)
|
||||
if workspace_service is None or policy is None or getattr(policy, 'multi_workspace_enabled', False):
|
||||
raise WorkspaceNotFoundError('Plugin resource not found')
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
execution_context = ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
)
|
||||
return await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
|
||||
async def _get_stored_plugin_config(
|
||||
self,
|
||||
request_context: RequestContext,
|
||||
author: str,
|
||||
plugin_name: str,
|
||||
plugin: dict,
|
||||
):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting.config)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == request_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
)
|
||||
persisted_config = result.scalar_one_or_none()
|
||||
return persisted_config if persisted_config is not None else plugin['plugin_config']
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/_sdk/page-sdk.js', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> quart.Response:
|
||||
@@ -170,15 +365,27 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return quart.Response(content, mimetype='application/javascript')
|
||||
return quart.Response('// SDK not found', status=404, mimetype='application/javascript')
|
||||
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
|
||||
return self.success(data={'plugins': plugins})
|
||||
return self.success(data={'plugins': redact_plugin_secrets(plugins)})
|
||||
|
||||
@self.route('/debug-info', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/debug-info',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get plugin debug information including debug URL and key"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
debug_info = await self.ap.plugin_connector.get_debug_info()
|
||||
|
||||
# Get debug URL from config
|
||||
@@ -196,77 +403,121 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/<author>/<plugin_name>/upgrade',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.upgrade_plugin(author, plugin_name, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.upgrade_plugin(author, plugin_name, task_context=ctx),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name=f'plugin-upgrade-{plugin_name}',
|
||||
label=f'Upgrading plugin {plugin_name}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>',
|
||||
methods=['GET', 'DELETE'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
return self.success(data={'plugin': plugin})
|
||||
elif quart.request.method == 'DELETE':
|
||||
delete_data = quart.request.args.get('delete_data', 'false').lower() == 'true'
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.delete_plugin(
|
||||
author, plugin_name, delete_data=delete_data, task_context=ctx
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name=f'plugin-remove-{plugin_name}',
|
||||
label=f'Removing plugin {plugin_name}',
|
||||
context=ctx,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
return self.success(data={'plugin': redact_plugin_secrets(plugin)})
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
delete_data = quart.request.args.get('delete_data', 'false').lower() == 'true'
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.delete_plugin(
|
||||
author,
|
||||
plugin_name,
|
||||
delete_data=delete_data,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name=f'plugin-remove-{plugin_name}',
|
||||
label=f'Removing plugin {plugin_name}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/config',
|
||||
methods=['GET', 'PUT'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
|
||||
if quart.request.method == 'GET':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting.config)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
config = await self._get_stored_plugin_config(
|
||||
request_context,
|
||||
author,
|
||||
plugin_name,
|
||||
plugin,
|
||||
)
|
||||
return self.success(data={'config': redact_plugin_secrets(config)})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/config',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
current_config = await self._get_stored_plugin_config(
|
||||
request_context,
|
||||
author,
|
||||
plugin_name,
|
||||
plugin,
|
||||
)
|
||||
try:
|
||||
config = restore_plugin_secret_placeholders(
|
||||
await quart.request.json,
|
||||
current_config,
|
||||
)
|
||||
persisted_config = result.scalar_one_or_none()
|
||||
|
||||
config = persisted_config if persisted_config is not None else plugin['plugin_config']
|
||||
return self.success(data={'config': config})
|
||||
elif quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
|
||||
await self.ap.plugin_connector.set_plugin_config(author, plugin_name, data)
|
||||
|
||||
return self.success(data={})
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
await self.ap.plugin_connector.set_plugin_config(author, plugin_name, config)
|
||||
return self.success(data={})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/readme',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
language = quart.request.args.get('language', 'en')
|
||||
readme = await self.ap.plugin_connector.get_plugin_readme(author, plugin_name, language=language)
|
||||
return self.success(data={'readme': readme})
|
||||
@@ -275,8 +526,10 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/<author>/<plugin_name>/logs',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
try:
|
||||
limit = int(quart.request.args.get('limit', 200))
|
||||
except (TypeError, ValueError):
|
||||
@@ -291,11 +544,12 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
auth_type=group.AuthType.NONE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
await self._require_public_plugin_runtime_context()
|
||||
icon_data = await self.ap.plugin_connector.get_plugin_icon(author, plugin_name)
|
||||
icon_base64 = icon_data['plugin_icon_base64']
|
||||
mime_type = icon_data['mime_type']
|
||||
|
||||
icon_data = base64.b64decode(icon_base64)
|
||||
icon_data = await asyncio.to_thread(base64.b64decode, icon_base64)
|
||||
|
||||
return quart.Response(icon_data, mimetype=mime_type)
|
||||
|
||||
@@ -305,6 +559,7 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
auth_type=group.AuthType.NONE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, filepath: str) -> quart.Response:
|
||||
await self._require_public_plugin_runtime_context()
|
||||
asset_path = _normalize_plugin_asset_path(filepath)
|
||||
if asset_path is None:
|
||||
return quart.Response('Asset not found', status=404)
|
||||
@@ -312,7 +567,10 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
asset_data = await self.ap.plugin_connector.get_plugin_assets(author, plugin_name, asset_path)
|
||||
if not asset_data.get('asset_base64'):
|
||||
return quart.Response('Asset not found', status=404)
|
||||
asset_bytes = base64.b64decode(asset_data['asset_base64'])
|
||||
asset_bytes = await asyncio.to_thread(
|
||||
base64.b64decode,
|
||||
asset_data['asset_base64'],
|
||||
)
|
||||
mime_type = asset_data['mime_type']
|
||||
resp = quart.Response(asset_bytes, mimetype=mime_type)
|
||||
# CSP for HTML pages served to sandboxed iframes (opaque origin).
|
||||
@@ -334,9 +592,11 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/<author>/<plugin_name>/page-api',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
"""Forward a page API request to the plugin."""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
data = await quart.request.json
|
||||
if not isinstance(data, dict):
|
||||
return self.http_status(400, -1, 'invalid request body')
|
||||
@@ -357,9 +617,15 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, result['error'])
|
||||
return self.success(data=result.get('data'))
|
||||
|
||||
@self.route('/github/releases', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/github/releases',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get releases from a GitHub repository URL"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
data = await quart.request.json
|
||||
repo_url = data.get('repo_url', '')
|
||||
|
||||
@@ -400,10 +666,11 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
trust_env=True,
|
||||
follow_redirects=True,
|
||||
timeout=10,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(),
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
releases = response.json()
|
||||
releases = await httpclient.parse_json_response(response)
|
||||
|
||||
# Format releases data for frontend
|
||||
formatted_releases = []
|
||||
@@ -427,16 +694,18 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'source_subdir': requested_subdir,
|
||||
}
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
return self.http_status(500, -1, f'Failed to fetch releases: {str(e)}')
|
||||
except httpx.RequestError:
|
||||
raise
|
||||
|
||||
@self.route(
|
||||
'/github/release-assets',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _() -> str:
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get assets from a specific GitHub release"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
data = await quart.request.json
|
||||
owner = data.get('owner', '')
|
||||
repo = data.get('repo', '')
|
||||
@@ -452,12 +721,13 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
trust_env=True,
|
||||
follow_redirects=True,
|
||||
timeout=10,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(),
|
||||
) as client:
|
||||
response = await client.get(
|
||||
url,
|
||||
)
|
||||
response.raise_for_status()
|
||||
release = response.json()
|
||||
release = await httpclient.parse_json_response(response)
|
||||
|
||||
# Format assets data for frontend
|
||||
formatted_assets = []
|
||||
@@ -484,42 +754,61 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
# )
|
||||
|
||||
return self.success(data={'assets': formatted_assets})
|
||||
except httpx.RequestError as e:
|
||||
return self.http_status(500, -1, f'Failed to fetch release assets: {str(e)}')
|
||||
except httpx.RequestError:
|
||||
raise
|
||||
|
||||
@self.route('/install/github', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/install/github',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Install plugin from GitHub release asset"""
|
||||
limit_error = await self._check_extensions_limit()
|
||||
limit_error = await self._check_extensions_limit(request_context)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
data = await quart.request.json
|
||||
asset_url = data.get('asset_url', '')
|
||||
owner = data.get('owner', '')
|
||||
repo = data.get('repo', '')
|
||||
release_tag = data.get('release_tag', '')
|
||||
data = await quart.request.json or {}
|
||||
try:
|
||||
install_info = validate_github_plugin_install_info(
|
||||
{
|
||||
'asset_url': data.get('asset_url'),
|
||||
'asset_id': data.get('asset_id'),
|
||||
'release_id': data.get('release_id'),
|
||||
'owner': data.get('owner'),
|
||||
'repo': data.get('repo'),
|
||||
'release_tag': data.get('release_tag'),
|
||||
'github_url': f'https://github.com/{data.get("owner", "")}/{data.get("repo", "")}',
|
||||
}
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
if not asset_url:
|
||||
return self.http_status(400, -1, 'Missing asset_url parameter')
|
||||
owner = install_info['owner']
|
||||
repo = install_info['repo']
|
||||
release_tag = install_info['release_tag']
|
||||
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{owner}/{repo}'
|
||||
ctx.metadata['install_source'] = 'github'
|
||||
install_info = {
|
||||
'asset_url': asset_url,
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'release_tag': release_tag,
|
||||
'github_url': f'https://github.com/{owner}/{repo}',
|
||||
}
|
||||
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.GITHUB, install_info, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.GITHUB,
|
||||
install_info,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-github',
|
||||
label=f'Installing plugin from GitHub {owner}/{repo}@{release_tag}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
@@ -528,9 +817,10 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/install/marketplace',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _() -> str:
|
||||
limit_error = await self._check_extensions_limit()
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
limit_error = await self._check_extensions_limit(request_context)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
@@ -538,23 +828,37 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
|
||||
plugin_author = data.get('plugin_author', '')
|
||||
plugin_name = data.get('plugin_name', '')
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{plugin_author}/{plugin_name}'
|
||||
ctx.metadata['install_source'] = 'marketplace'
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.MARKETPLACE, data, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE,
|
||||
data,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-marketplace',
|
||||
label=f'Installing plugin from marketplace {plugin_author}/{plugin_name}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/install/local', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
limit_error = await self._check_extensions_limit()
|
||||
@self.route(
|
||||
'/install/local',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
limit_error = await self._check_extensions_limit(request_context)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
@@ -563,6 +867,7 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
file_bytes = file.read()
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
data = {
|
||||
'plugin_file': file_bytes,
|
||||
@@ -572,74 +877,72 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
ctx.metadata['plugin_name'] = file.filename or 'local plugin'
|
||||
ctx.metadata['install_source'] = 'local'
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.LOCAL, data, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.LOCAL,
|
||||
data,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-local',
|
||||
label=f'Installing plugin from local {file.filename}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/install/local/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/install/local/preview',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
file_bytes = file.read()
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(file_bytes)) as zf:
|
||||
names = [name for name in zf.namelist() if not name.endswith('/')]
|
||||
manifest_name = next(
|
||||
(
|
||||
name
|
||||
for name in names
|
||||
if name.replace('\\', '/').strip('/').lower() in ('manifest.yaml', 'manifest.yml')
|
||||
),
|
||||
None,
|
||||
)
|
||||
if manifest_name is None:
|
||||
return self.http_status(400, -1, 'manifest.yaml is required')
|
||||
manifest, requirements, names = await asyncio.to_thread(
|
||||
inspect_plugin_archive_metadata,
|
||||
file_bytes,
|
||||
)
|
||||
spec = manifest.get('spec') or {}
|
||||
components = spec.get('components') or {}
|
||||
component_counts = self._count_plugin_components(components, names)
|
||||
component_types = list(component_counts.keys())
|
||||
|
||||
manifest = yaml.safe_load(zf.read(manifest_name).decode('utf-8')) or {}
|
||||
requirements: list[str] = []
|
||||
requirements_name = next(
|
||||
(name for name in names if name.replace('\\', '/').strip('/').lower() == 'requirements.txt'),
|
||||
None,
|
||||
)
|
||||
if requirements_name is not None:
|
||||
requirements = [
|
||||
line.strip()
|
||||
for line in zf.read(requirements_name).decode('utf-8', errors='ignore').splitlines()
|
||||
if line.strip() and not line.strip().startswith('#')
|
||||
]
|
||||
return self.success(
|
||||
data={
|
||||
'filename': file.filename or 'local plugin',
|
||||
'size': len(file_bytes),
|
||||
'manifest': manifest,
|
||||
'metadata': manifest.get('metadata') or {},
|
||||
'component_types': component_types,
|
||||
'component_counts': component_counts,
|
||||
'requirements': requirements,
|
||||
'file_count': len(names),
|
||||
}
|
||||
)
|
||||
except (zipfile.BadZipFile, ValueError) as exc:
|
||||
return self.http_status(400, -1, str(exc) or 'invalid .lbpkg file')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
spec = manifest.get('spec') or {}
|
||||
components = spec.get('components') or {}
|
||||
component_counts = self._count_plugin_components(components, names)
|
||||
component_types = list(component_counts.keys())
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'filename': file.filename or 'local plugin',
|
||||
'size': len(file_bytes),
|
||||
'manifest': manifest,
|
||||
'metadata': manifest.get('metadata') or {},
|
||||
'component_types': component_types,
|
||||
'component_counts': component_counts,
|
||||
'requirements': requirements,
|
||||
'file_count': len(names),
|
||||
}
|
||||
)
|
||||
except zipfile.BadZipFile:
|
||||
return self.http_status(400, -1, 'invalid .lbpkg file')
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview plugin package: {exc}')
|
||||
|
||||
@self.route('/config-files', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/config-files',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Upload a file for plugin configuration"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
@@ -650,25 +953,37 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
if len(file_bytes) > MAX_FILE_SIZE:
|
||||
return self.http_status(400, -1, 'file size exceeds 10MB limit')
|
||||
|
||||
# Generate unique file key with original extension
|
||||
original_filename = file.filename
|
||||
original_filename = file.filename or 'config.bin'
|
||||
_, ext = os.path.splitext(original_filename)
|
||||
file_key = f'plugin_config_{uuid.uuid4().hex}{ext}'
|
||||
|
||||
# Save file using storage manager
|
||||
await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
|
||||
logical_key = f'plugin_config_{uuid.uuid4().hex}{ext}'
|
||||
file_key = await self.ap.storage_mgr.save_scoped(
|
||||
request_context,
|
||||
owner_type='plugin_config',
|
||||
owner=request_context.workspace_uuid,
|
||||
key=logical_key,
|
||||
value=file_bytes,
|
||||
)
|
||||
|
||||
return self.success(data={'file_key': file_key})
|
||||
|
||||
@self.route('/config-files/<file_key>', methods=['DELETE'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(file_key: str) -> str:
|
||||
@self.route(
|
||||
'/config-files/<path:file_key>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(file_key: str, request_context: RequestContext) -> str:
|
||||
"""Delete a plugin configuration file"""
|
||||
# Only allow deletion of files with plugin_config_ prefix for security
|
||||
if not file_key.startswith('plugin_config_'):
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
if not self.ap.storage_mgr.is_scoped_object_key(file_key, expected_owner_type='plugin_config'):
|
||||
return self.http_status(400, -1, 'invalid file key')
|
||||
|
||||
try:
|
||||
await self.ap.storage_mgr.storage_provider.delete(file_key)
|
||||
await self.ap.storage_mgr.delete_scoped_object_key(
|
||||
request_context,
|
||||
file_key,
|
||||
expected_owner_type='plugin_config',
|
||||
)
|
||||
return self.success(data={'deleted': True})
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'failed to delete file: {str(e)}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@@ -1,147 +1,292 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('models/llm', '/api/v1/provider/models/llm')
|
||||
class LLMModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={'models': await self.ap.llm_model_service.get_llm_models_by_provider(provider_uuid)}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.llm_model_service.get_llm_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.llm_model_service.create_llm_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
|
||||
if provider_uuid:
|
||||
models = await self.ap.llm_model_service.get_llm_models_by_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
else:
|
||||
models = await self.ap.llm_model_service.get_llm_models(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
return self.success(data={'models': models})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.llm_model_service.get_llm_model(model_uuid)
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_uuid = await self.ap.llm_model_service.create_llm_model(
|
||||
request_context,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
model = await self.ap.llm_model_service.get_llm_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
return self.success(data={'model': model})
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.llm_model_service.update_llm_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
await self.ap.llm_model_service.update_llm_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.llm_model_service.delete_llm_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.llm_model_service.test_llm_model(model_uuid, json_data)
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.llm_model_service.delete_llm_model(request_context, model_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.llm_model_service.test_llm_model(request_context, model_uuid, await quart.request.json)
|
||||
return self.success()
|
||||
|
||||
|
||||
@group.group_class('models/embedding', '/api/v1/provider/models/embedding')
|
||||
class EmbeddingModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={
|
||||
'models': await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
provider_uuid
|
||||
)
|
||||
}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.embedding_models_service.get_embedding_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.embedding_models_service.create_embedding_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
|
||||
if provider_uuid:
|
||||
models = await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
else:
|
||||
models = await self.ap.embedding_models_service.get_embedding_models(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
return self.success(data={'models': models})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.embedding_models_service.get_embedding_model(model_uuid)
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_uuid = await self.ap.embedding_models_service.create_embedding_model(
|
||||
request_context,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
model = await self.ap.embedding_models_service.get_embedding_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
return self.success(data={'model': model})
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.embedding_models_service.update_embedding_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
await self.ap.embedding_models_service.update_embedding_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.embedding_models_service.delete_embedding_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.embedding_models_service.test_embedding_model(model_uuid, json_data)
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.embedding_models_service.delete_embedding_model(request_context, model_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.embedding_models_service.test_embedding_model(
|
||||
request_context, model_uuid, await quart.request.json
|
||||
)
|
||||
return self.success()
|
||||
|
||||
|
||||
@group.group_class('models/rerank', '/api/v1/provider/models/rerank')
|
||||
class RerankModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={
|
||||
'models': await self.ap.rerank_models_service.get_rerank_models_by_provider(provider_uuid)
|
||||
}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.rerank_models_service.get_rerank_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.rerank_models_service.create_rerank_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
|
||||
if provider_uuid:
|
||||
models = await self.ap.rerank_models_service.get_rerank_models_by_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
else:
|
||||
models = await self.ap.rerank_models_service.get_rerank_models(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
return self.success(data={'models': models})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.rerank_models_service.get_rerank_model(model_uuid)
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_uuid = await self.ap.rerank_models_service.create_rerank_model(
|
||||
request_context,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.rerank_models_service.update_rerank_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.rerank_models_service.delete_rerank_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.rerank_models_service.test_rerank_model(model_uuid, json_data)
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
model = await self.ap.rerank_models_service.get_rerank_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
return self.success(data={'model': model})
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.rerank_models_service.update_rerank_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.rerank_models_service.delete_rerank_model(request_context, model_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.rerank_models_service.test_rerank_model(request_context, model_uuid, await quart.request.json)
|
||||
return self.success()
|
||||
|
||||
@@ -1,56 +1,102 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('models/providers', '/api/v1/provider/providers')
|
||||
class ModelProvidersRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
providers = await self.ap.provider_service.get_providers()
|
||||
# Add model counts
|
||||
for provider in providers:
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(provider['uuid'])
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'providers': providers})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
provider_uuid = await self.ap.provider_service.create_provider(json_data)
|
||||
return self.success(data={'uuid': provider_uuid})
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(provider_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider = await self.ap.provider_service.get_provider(provider_uuid)
|
||||
if provider is None:
|
||||
return self.http_status(404, -1, 'provider not found')
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(provider_uuid)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
providers = await self.ap.provider_service.get_providers(
|
||||
request_context,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
for provider in providers:
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider['uuid'])
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'provider': provider})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.provider_service.update_provider(provider_uuid, json_data)
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
try:
|
||||
await self.ap.provider_service.delete_provider(provider_uuid)
|
||||
return self.success()
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
return self.success(data={'providers': providers})
|
||||
|
||||
@self.route('/<provider_uuid>/scan-models', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(provider_uuid: str) -> str:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
provider_uuid = await self.ap.provider_service.create_provider(request_context, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': provider_uuid})
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
provider = await self.ap.provider_service.get_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if provider is None:
|
||||
return self.http_status(404, -1, 'provider not found')
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider_uuid)
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'provider': provider})
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
await self.ap.provider_service.update_provider(request_context, provider_uuid, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.provider_service.delete_provider(request_context, provider_uuid)
|
||||
return self.success()
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>/scan-models',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_type = quart.request.args.get('type')
|
||||
result = await self.ap.provider_service.scan_provider_models(provider_uuid, model_type)
|
||||
result = await self.ap.provider_service.scan_provider_models(request_context, provider_uuid, model_type)
|
||||
return self.success(data=result)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@@ -1,103 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import quart
|
||||
import traceback
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ......provider.tools.loaders.mcp_policy import MCPStdioDisabledError
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('mcp', '/api/v1/mcp')
|
||||
class MCPRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/servers', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""获取MCP服务器列表"""
|
||||
if quart.request.method == 'GET':
|
||||
servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True)
|
||||
|
||||
return self.success(data={'servers': servers})
|
||||
|
||||
elif quart.request.method == 'POST':
|
||||
data = await quart.request.json
|
||||
|
||||
try:
|
||||
uuid = await self.ap.mcp_service.create_mcp_server(data)
|
||||
return self.success(data={'uuid': uuid})
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return self.http_status(500, -1, f'Failed to create MCP server: {str(e)}')
|
||||
@self.route(
|
||||
'/servers',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
servers = await self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
return self.success(data={'servers': servers})
|
||||
|
||||
@self.route(
|
||||
'/servers/<path:server_name>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN
|
||||
'/servers',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(server_name: str) -> str:
|
||||
"""获取、更新或删除MCP服务器配置"""
|
||||
server_name = unquote(server_name)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
data = await quart.request.json
|
||||
try:
|
||||
server_uuid = await self.ap.mcp_service.create_mcp_server(request_context, data)
|
||||
except MCPStdioDisabledError as exc:
|
||||
return self.http_status(403, exc.code, str(exc))
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': server_uuid})
|
||||
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(server_name)
|
||||
@self.route(
|
||||
'/servers/<path:server_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
server_name = unquote(server_name)
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(request_context, server_name)
|
||||
if server_data is None:
|
||||
return self.http_status(404, -1, 'Server not found')
|
||||
return self.success(data={'server': server_data})
|
||||
|
||||
if quart.request.method == 'GET':
|
||||
return self.success(data={'server': server_data})
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
@self.route(
|
||||
'/servers/<path:server_name>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
server_name = unquote(server_name)
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(request_context, server_name)
|
||||
if server_data is None:
|
||||
return self.http_status(404, -1, 'Server not found')
|
||||
if quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
try:
|
||||
await self.ap.mcp_service.update_mcp_server(server_data['uuid'], data)
|
||||
return self.success()
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to update MCP server: {str(e)}')
|
||||
await self.ap.mcp_service.update_mcp_server(request_context, server_data['uuid'], data)
|
||||
except MCPStdioDisabledError as exc:
|
||||
return self.http_status(403, exc.code, str(exc))
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
else:
|
||||
await self.ap.mcp_service.delete_mcp_server(request_context, server_data['uuid'])
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
try:
|
||||
await self.ap.mcp_service.delete_mcp_server(server_data['uuid'])
|
||||
return self.success()
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to delete MCP server: {str(e)}')
|
||||
|
||||
@self.route('/servers/<path:server_name>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""测试MCP服务器连接"""
|
||||
server_name = unquote(server_name)
|
||||
server_data = await quart.request.json
|
||||
task_id = await self.ap.mcp_service.test_mcp_server(server_name=server_name, server_data=server_data)
|
||||
try:
|
||||
task_id = await self.ap.mcp_service.test_mcp_server(
|
||||
request_context,
|
||||
server_name=server_name,
|
||||
server_data=server_data,
|
||||
)
|
||||
except MCPStdioDisabledError as exc:
|
||||
return self.http_status(403, exc.code, str(exc))
|
||||
return self.success(data={'task_id': task_id})
|
||||
|
||||
@self.route('/servers/<path:server_name>/resources', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/resources',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Get resources from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
try:
|
||||
resources = await self.ap.mcp_service.get_mcp_server_resources(server_name)
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(server_name)
|
||||
runtime_info = await self.ap.mcp_service.get_runtime_info(server_name)
|
||||
return self.success(
|
||||
data={
|
||||
'resources': resources,
|
||||
'resource_templates': templates,
|
||||
'resource_capabilities': (runtime_info or {}).get('resource_capabilities', {}),
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to get resources: {str(e)}')
|
||||
resources = await self.ap.mcp_service.get_mcp_server_resources(request_context, server_name)
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(request_context, server_name)
|
||||
runtime_info = await self.ap.mcp_service.get_runtime_info(request_context, server_name)
|
||||
return self.success(
|
||||
data={
|
||||
'resources': resources,
|
||||
'resource_templates': templates,
|
||||
'resource_capabilities': (runtime_info or {}).get('resource_capabilities', {}),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/resource-templates', methods=['GET'], auth_type=group.AuthType.USER_TOKEN
|
||||
'/servers/<path:server_name>/resource-templates',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str) -> str:
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Get resource templates from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
try:
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(server_name)
|
||||
return self.success(data={'resource_templates': templates})
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to get resource templates: {str(e)}')
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(request_context, server_name)
|
||||
return self.success(data={'resource_templates': templates})
|
||||
|
||||
@self.route('/servers/<path:server_name>/logs', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/logs',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Get logs from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
try:
|
||||
@@ -106,24 +141,32 @@ class MCPRouterGroup(group.RouterGroup):
|
||||
limit = 200
|
||||
limit = min(limit, 500)
|
||||
level = quart.request.args.get('level') or None
|
||||
logs = await self.ap.mcp_service.get_mcp_server_logs(server_name, limit=limit, level=level)
|
||||
logs = await self.ap.mcp_service.get_mcp_server_logs(
|
||||
request_context,
|
||||
server_name,
|
||||
limit=limit,
|
||||
level=level,
|
||||
)
|
||||
return self.success(data={'logs': logs})
|
||||
|
||||
@self.route('/servers/<path:server_name>/resources/read', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/resources/read',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Read a resource from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
data = await quart.request.json
|
||||
uri = data.get('uri')
|
||||
if not uri:
|
||||
return self.http_status(400, -1, 'URI is required')
|
||||
try:
|
||||
envelope = await self.ap.mcp_service.read_mcp_server_resource_envelope(
|
||||
server_name,
|
||||
uri,
|
||||
max_bytes=data.get('max_bytes'),
|
||||
include_blob=bool(data.get('include_blob', False)),
|
||||
)
|
||||
return self.success(data=envelope)
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to read resource: {str(e)}')
|
||||
envelope = await self.ap.mcp_service.read_mcp_server_resource_envelope(
|
||||
request_context,
|
||||
server_name,
|
||||
uri,
|
||||
max_bytes=data.get('max_bytes'),
|
||||
include_blob=bool(data.get('include_blob', False)),
|
||||
)
|
||||
return self.success(data=envelope)
|
||||
|
||||
@@ -2,21 +2,28 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('tools', '/api/v1/tools')
|
||||
class ToolsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""获取所有可用工具列表"""
|
||||
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
|
||||
bound_plugins: list[str] | None = None
|
||||
bound_mcp_servers: list[str] | None = None
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
@@ -35,6 +42,7 @@ class ToolsRouterGroup(group.RouterGroup):
|
||||
return self.success(
|
||||
data={
|
||||
'tools': await self.ap.tool_mgr.get_tool_catalog(
|
||||
request_context,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
@@ -42,10 +50,15 @@ class ToolsRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(tool_name: str) -> str:
|
||||
@self.route(
|
||||
'/<tool_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(tool_name: str, request_context: RequestContext) -> str:
|
||||
"""获取特定工具详情"""
|
||||
tools = await self.ap.tool_mgr.get_all_tools(include_skill_authoring=True)
|
||||
tools = await self.ap.tool_mgr.get_all_tools(request_context, include_skill_authoring=True)
|
||||
|
||||
for tool in tools:
|
||||
if tool.name == tool_name:
|
||||
|
||||
@@ -2,8 +2,11 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from langbot.pkg.cloud.entitlements import EntitlementFeatureUnavailableError
|
||||
from langbot_plugin.box.errors import BoxError
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@@ -12,58 +15,91 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
"""Skills management API endpoints."""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_or_create_skills() -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
skills = await self.ap.skill_service.list_skills()
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'skills': skills})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_skills(request_context: RequestContext) -> quart.Response:
|
||||
try:
|
||||
skills = await self.ap.skill_service.list_skills(request_context)
|
||||
except EntitlementFeatureUnavailableError:
|
||||
# Plans without managed sandbox support have no runnable skills.
|
||||
# Treat that capability absence as an empty collection so the
|
||||
# shared UI can render normally instead of surfacing a 500.
|
||||
return self.success(data={'skills': []})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'skills': skills})
|
||||
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def create_skill(request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
if 'name' not in data or not data['name']:
|
||||
return self.http_status(400, -1, 'Missing required field: name')
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.create_skill(data)
|
||||
skill = await self.ap.skill_service.create_skill(request_context, data)
|
||||
return self.success(data={'skill': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def get_update_delete_skill(skill_name: str) -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
skill = await self.ap.skill_service.get_skill(skill_name)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'skill': skill})
|
||||
@self.route(
|
||||
'/<skill_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_skill(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
try:
|
||||
skill = await self.ap.skill_service.get_skill(request_context, skill_name)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'skill': skill})
|
||||
|
||||
@self.route(
|
||||
'/<skill_name>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def update_delete_skill(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
if quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
try:
|
||||
skill = await self.ap.skill_service.update_skill(skill_name, data)
|
||||
skill = await self.ap.skill_service.update_skill(request_context, skill_name, data)
|
||||
return self.success(data={'skill': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
try:
|
||||
await self.ap.skill_service.delete_skill(skill_name)
|
||||
await self.ap.skill_service.delete_skill(request_context, skill_name)
|
||||
return self.success()
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>/files', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_skill_files(skill_name: str) -> quart.Response:
|
||||
@self.route(
|
||||
'/<skill_name>/files',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_skill_files(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
"""List files in skill package directory."""
|
||||
path = quart.request.args.get('path', '.').strip()
|
||||
include_hidden = quart.request.args.get('include_hidden', 'false').lower() == 'true'
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.list_skill_files(
|
||||
request_context,
|
||||
skill_name,
|
||||
path=path,
|
||||
include_hidden=include_hidden,
|
||||
@@ -73,38 +109,55 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route(
|
||||
'/<skill_name>/files/<path:path>', methods=['GET', 'PUT'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<skill_name>/files/<path:path>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def read_or_write_skill_file(skill_name: str, path: str) -> quart.Response:
|
||||
"""Read or write a file in skill package."""
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
result = await self.ap.skill_service.read_skill_file(skill_name, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
async def read_skill_file(skill_name: str, path: str, request_context: RequestContext) -> quart.Response:
|
||||
try:
|
||||
result = await self.ap.skill_service.read_skill_file(request_context, skill_name, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
# PUT - write file
|
||||
@self.route(
|
||||
'/<skill_name>/files/<path:path>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def write_skill_file(skill_name: str, path: str, request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
content = data.get('content', '')
|
||||
if content is None:
|
||||
return self.http_status(400, -1, 'Missing required field: content')
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.write_skill_file(skill_name, path, content)
|
||||
result = await self.ap.skill_service.write_skill_file(request_context, skill_name, path, content)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill(skill_name: str) -> quart.Response:
|
||||
skill = self.ap.skill_mgr.get_skill_by_name(skill_name)
|
||||
@self.route(
|
||||
'/<skill_name>/preview',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def preview_skill(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
skill = await self.ap.skill_service.get_skill(request_context, skill_name)
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'instructions': skill.get('instructions', '')})
|
||||
|
||||
@self.route('/install/github', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def install_skill_from_github() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/github',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def install_skill_from_github(request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
required_fields = ['asset_url', 'owner', 'repo']
|
||||
for field in required_fields:
|
||||
@@ -115,15 +168,20 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Missing required field: release_tag')
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.install_from_github(data)
|
||||
skill = await self.ap.skill_service.install_from_github(request_context, data)
|
||||
return self.success(data={'skills': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to install skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/install/github/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill_from_github() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/github/preview',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def preview_skill_from_github(request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
required_fields = ['asset_url', 'owner', 'repo']
|
||||
for field in required_fields:
|
||||
@@ -134,15 +192,20 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Missing required field: release_tag')
|
||||
|
||||
try:
|
||||
preview = await self.ap.skill_service.preview_install_from_github(data)
|
||||
preview = await self.ap.skill_service.preview_install_from_github(request_context, data)
|
||||
return self.success(data={'skills': preview})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/install/upload', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def install_skill_from_upload() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/upload',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def install_skill_from_upload(request_context: RequestContext) -> quart.Response:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
@@ -150,6 +213,7 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.install_from_zip_upload(
|
||||
request_context,
|
||||
file_bytes=file.read(),
|
||||
filename=file.filename or '',
|
||||
source_paths=form.getlist('source_paths'),
|
||||
@@ -157,34 +221,45 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.success(data={'skills': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to install skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/install/upload/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill_from_upload() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/upload/preview',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def preview_skill_from_upload(request_context: RequestContext) -> quart.Response:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
try:
|
||||
preview = await self.ap.skill_service.preview_install_from_zip_upload(
|
||||
request_context,
|
||||
file_bytes=file.read(),
|
||||
filename=file.filename or '',
|
||||
)
|
||||
return self.success(data={'skills': preview})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/scan', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def scan_skill_directory() -> quart.Response:
|
||||
@self.route(
|
||||
'/scan',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def scan_skill_directory(request_context: RequestContext) -> quart.Response:
|
||||
path = quart.request.args.get('path', '').strip()
|
||||
if not path:
|
||||
return self.http_status(400, -1, 'Missing required parameter: path')
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.scan_directory_async(path)
|
||||
result = await self.ap.skill_service.scan_directory_async(request_context, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@@ -1,19 +1,39 @@
|
||||
from .. import group
|
||||
from ...authz import Permission
|
||||
from ...context import ExecutionContext, RequestContext
|
||||
|
||||
|
||||
def collect_basic_stats(ap, request_context: RequestContext) -> dict[str, int]:
|
||||
"""Collect runtime counters only from the selected Workspace placement."""
|
||||
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
sessions = [
|
||||
session
|
||||
for session in ap.sess_mgr.session_list
|
||||
if (
|
||||
getattr(session, 'instance_uuid', None) == execution_context.instance_uuid
|
||||
and getattr(session, 'workspace_uuid', None) == execution_context.workspace_uuid
|
||||
and getattr(session, 'placement_generation', None) == execution_context.placement_generation
|
||||
)
|
||||
]
|
||||
conversation_count = sum(
|
||||
len(session.conversations if session.conversations is not None else []) for session in sessions
|
||||
)
|
||||
return {
|
||||
'active_session_count': len(sessions),
|
||||
'conversation_count': conversation_count,
|
||||
'query_count': ap.query_pool.get_query_count(execution_context),
|
||||
}
|
||||
|
||||
|
||||
@group.group_class('stats', '/api/v1/stats')
|
||||
class StatsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/basic', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
conv_count = 0
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
conv_count += len(session.conversations if session.conversations is not None else [])
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'active_session_count': len(self.ap.sess_mgr.session_list),
|
||||
'conversation_count': conv_count,
|
||||
'query_count': self.ap.query_pool.query_id_counter,
|
||||
}
|
||||
)
|
||||
@self.route(
|
||||
'/basic',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data=collect_basic_stats(self.ap, request_context))
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import base64
|
||||
|
||||
import quart
|
||||
@@ -59,7 +60,14 @@ class SurveyRouterGroup(group.RouterGroup):
|
||||
continue
|
||||
try:
|
||||
payload = data_url.split(',', 1)[1]
|
||||
if len(base64.b64decode(payload, validate=True)) > 1024 * 1024:
|
||||
if len(payload) > 4 * ((1024 * 1024 + 2) // 3) + 4:
|
||||
return self.fail(5, 'attachment too large')
|
||||
decoded = await asyncio.to_thread(
|
||||
base64.b64decode,
|
||||
payload,
|
||||
validate=True,
|
||||
)
|
||||
if len(decoded) > 1024 * 1024:
|
||||
return self.fail(5, 'attachment too large')
|
||||
except Exception:
|
||||
return self.fail(5, 'attachment too large')
|
||||
|
||||
@@ -5,7 +5,11 @@ import sqlalchemy
|
||||
|
||||
from .. import group
|
||||
from .....utils import constants
|
||||
from .....entity.persistence.metadata import Metadata
|
||||
from .....entity.persistence.metadata import WorkspaceMetadata
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .....provider.tools.loaders.mcp_policy import stdio_mcp_enabled
|
||||
from .....workspace.invitation_delivery import InvitationDeliveryService
|
||||
|
||||
|
||||
@group.group_class('system', '/api/v1/system')
|
||||
@@ -17,17 +21,46 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
wizard_status = 'none'
|
||||
wizard_progress = None
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key.in_(['wizard_status', 'wizard_progress']))
|
||||
)
|
||||
for row in result:
|
||||
if row.key == 'wizard_status':
|
||||
wizard_status = row.value
|
||||
elif row.key == 'wizard_progress':
|
||||
try:
|
||||
wizard_progress = json.loads(row.value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
wizard_progress = None
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if authorization.startswith('Bearer '):
|
||||
account, _ = await self._authenticate_account(authorization.removeprefix('Bearer '))
|
||||
request_context = await self._resolve_account_context(account, group.AuthType.USER_TOKEN)
|
||||
if request_context is not None:
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
|
||||
async def load_workspace_metadata():
|
||||
return await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
WorkspaceMetadata.key,
|
||||
WorkspaceMetadata.value,
|
||||
).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key.in_(['wizard_status', 'wizard_progress']),
|
||||
)
|
||||
)
|
||||
|
||||
cloud_runtime = (
|
||||
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
)
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud system metadata requires an explicit tenant UoW')
|
||||
async with tenant_uow(request_context.workspace_uuid):
|
||||
result = await load_workspace_metadata()
|
||||
else:
|
||||
result = await load_workspace_metadata()
|
||||
# ``execute_async`` deliberately preserves its historical
|
||||
# AsyncConnection result shape. Selecting the two fields
|
||||
# explicitly keeps this reader independent of ORM Session
|
||||
# scalar semantics inside a tenant UoW.
|
||||
for row in result:
|
||||
if row.key == 'wizard_status':
|
||||
wizard_status = row.value
|
||||
elif row.key == 'wizard_progress':
|
||||
try:
|
||||
wizard_progress = json.loads(row.value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
wizard_progress = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -43,6 +76,10 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
else:
|
||||
outbound_ips = []
|
||||
|
||||
invitation_delivery_service = getattr(self.ap, 'invitation_delivery_service', None)
|
||||
if invitation_delivery_service is None:
|
||||
invitation_delivery_service = InvitationDeliveryService(self.ap)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'version': constants.semantic_version,
|
||||
@@ -60,15 +97,24 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
'disable_models_service': self.ap.instance_config.data.get('space', {}).get(
|
||||
'disable_models_service', False
|
||||
),
|
||||
# Exposed independently of Box status so the WebUI cannot
|
||||
# infer stdio permission from sandbox availability.
|
||||
'mcp_stdio_enabled': stdio_mcp_enabled(self.ap),
|
||||
'limitation': self.ap.instance_config.data.get('system', {}).get('limitation', {}),
|
||||
'outbound_ips': outbound_ips,
|
||||
'invitation_delivery': invitation_delivery_service.capability(),
|
||||
'wizard_status': wizard_status,
|
||||
'wizard_progress': wizard_progress,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/wizard/completed', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/wizard/completed',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.WORKSPACE_UPDATE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Mark wizard status in metadata table and clear progress.
|
||||
|
||||
Accepts JSON body: { "status": "skipped" | "completed" }
|
||||
@@ -80,28 +126,48 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == 'wizard_status')
|
||||
sqlalchemy.select(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_status',
|
||||
)
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == 'wizard_status').values(value=status)
|
||||
sqlalchemy.update(WorkspaceMetadata)
|
||||
.where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_status',
|
||||
)
|
||||
.values(value=status)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key='wizard_status', value=status)
|
||||
sqlalchemy.insert(WorkspaceMetadata).values(
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
key='wizard_status',
|
||||
value=status,
|
||||
)
|
||||
)
|
||||
|
||||
# Clear wizard progress when wizard is completed/skipped
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(Metadata).where(Metadata.key == 'wizard_progress')
|
||||
sqlalchemy.delete(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_progress',
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, 500, f'Failed to update wizard status: {e}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
return self.success(data={})
|
||||
|
||||
@self.route('/wizard/progress', methods=['PUT'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/wizard/progress',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.WORKSPACE_UPDATE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Save wizard progress to metadata table.
|
||||
|
||||
Accepts JSON body with wizard state fields:
|
||||
@@ -113,23 +179,40 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == 'wizard_progress')
|
||||
sqlalchemy.select(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_progress',
|
||||
)
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == 'wizard_progress').values(value=progress_json)
|
||||
sqlalchemy.update(WorkspaceMetadata)
|
||||
.where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_progress',
|
||||
)
|
||||
.values(value=progress_json)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key='wizard_progress', value=progress_json)
|
||||
sqlalchemy.insert(WorkspaceMetadata).values(
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
key='wizard_progress',
|
||||
value=progress_json,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, 500, f'Failed to save wizard progress: {e}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
return self.success(data={})
|
||||
|
||||
@self.route('/tasks', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/tasks',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
task_type = quart.request.args.get('type')
|
||||
task_kind = quart.request.args.get('kind')
|
||||
|
||||
@@ -138,30 +221,56 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
if task_kind == '':
|
||||
task_kind = None
|
||||
|
||||
return self.success(data=self.ap.task_mgr.get_tasks_dict(task_type, task_kind))
|
||||
return self.success(
|
||||
data=self.ap.task_mgr.get_tasks_dict(
|
||||
task_type,
|
||||
task_kind,
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
)
|
||||
)
|
||||
|
||||
@self.route('/tasks/<task_id>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(task_id: str) -> str:
|
||||
task = self.ap.task_mgr.get_task_by_id(int(task_id))
|
||||
@self.route(
|
||||
'/tasks/<task_id>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(task_id: str, request_context: RequestContext) -> str:
|
||||
task = self.ap.task_mgr.get_task_by_id(
|
||||
int(task_id),
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
)
|
||||
|
||||
if task is None:
|
||||
return self.http_status(404, 404, 'Task not found')
|
||||
|
||||
return self.success(data=task.to_dict())
|
||||
|
||||
@self.route('/storage-analysis', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
return self.success(data=await self.ap.maintenance_service.get_storage_analysis())
|
||||
@self.route(
|
||||
'/storage-analysis',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data=await self.ap.maintenance_service.get_storage_analysis(request_context))
|
||||
|
||||
@self.route(
|
||||
'/debug/plugin/action',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def _() -> str:
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
if not constants.debug_mode:
|
||||
return self.http_status(403, 403, 'Forbidden')
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
data = await quart.request.json
|
||||
|
||||
class AnoymousAction:
|
||||
@@ -174,6 +283,7 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
AnoymousAction(data['action']),
|
||||
data['data'],
|
||||
timeout=data.get('timeout', 10),
|
||||
action_context=self.ap.plugin_connector.handler.require_bound_action_context().without_installation(),
|
||||
)
|
||||
|
||||
return self.success(data=resp)
|
||||
@@ -182,8 +292,10 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
'/status/plugin-system',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _() -> str:
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin_connector_error = 'ok'
|
||||
is_connected = True
|
||||
|
||||
|
||||
@@ -1,14 +1,55 @@
|
||||
import quart
|
||||
import argon2
|
||||
import asyncio
|
||||
import traceback
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from .. import group
|
||||
from .....entity.errors import account as account_errors
|
||||
from ...context import RequestContext
|
||||
from .....cloud.launch import SpaceLaunchError
|
||||
from ...service.user import ControlPlaneDirectoryRequiredError, PublicRegistrationClosedError
|
||||
|
||||
|
||||
@group.group_class('user', '/api/v1/user')
|
||||
class UserRouterGroup(group.RouterGroup):
|
||||
@staticmethod
|
||||
def _origin(value: str) -> tuple[str, str, int | None] | None:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {'http', 'https'} or not parsed.hostname:
|
||||
return None
|
||||
return parsed.scheme, parsed.hostname.casefold(), parsed.port
|
||||
|
||||
def _validate_space_redirect_uri(self, redirect_uri: str, *, bind: bool) -> str:
|
||||
parsed = urlsplit(redirect_uri)
|
||||
if (
|
||||
parsed.scheme not in {'http', 'https'}
|
||||
or not parsed.hostname
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.fragment
|
||||
or parsed.path != '/auth/space/callback'
|
||||
):
|
||||
raise ValueError('Invalid redirect_uri parameter')
|
||||
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if bind:
|
||||
if query != {'mode': ['bind']}:
|
||||
raise ValueError('Invalid Space binding redirect_uri')
|
||||
elif query:
|
||||
raise ValueError('Invalid Space login redirect_uri')
|
||||
|
||||
redirect_origin = self._origin(redirect_uri)
|
||||
api_config = self.ap.instance_config.data.get('api', {})
|
||||
trusted_origins = {
|
||||
self._origin(str(api_config.get(config_key, '') or '').strip())
|
||||
for config_key in ('webui_url', 'webhook_prefix')
|
||||
}
|
||||
trusted_origins.discard(None)
|
||||
if redirect_origin not in trusted_origins:
|
||||
raise ValueError('Untrusted redirect_uri origin')
|
||||
return redirect_uri
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/init', methods=['GET', 'POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
@@ -23,12 +64,19 @@ class UserRouterGroup(group.RouterGroup):
|
||||
user_email = json_data['user']
|
||||
password = json_data['password']
|
||||
|
||||
await self.ap.user_service.create_user(user_email, password)
|
||||
try:
|
||||
await self.ap.user_service.create_user(user_email, password)
|
||||
except ControlPlaneDirectoryRequiredError as exc:
|
||||
return self.http_status(409, exc.code, str(exc))
|
||||
except PublicRegistrationClosedError:
|
||||
return self.http_status(409, 'registration_closed', 'System already initialized')
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/auth', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud':
|
||||
return self.http_status(403, 'password_login_disabled', 'Password login is disabled on LangBot Cloud')
|
||||
json_data = await quart.request.json
|
||||
|
||||
try:
|
||||
@@ -40,9 +88,9 @@ class UserRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data={'token': token})
|
||||
|
||||
@self.route('/check-token', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
token = await self.ap.user_service.generate_jwt_token(user_email)
|
||||
@self.route('/check-token', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> str:
|
||||
token = await self.ap.user_service.generate_jwt_token(account)
|
||||
|
||||
return self.success(data={'token': token})
|
||||
|
||||
@@ -101,15 +149,50 @@ class UserRouterGroup(group.RouterGroup):
|
||||
async def _() -> str:
|
||||
"""Get Space OAuth authorization URL for redirect"""
|
||||
redirect_uri = quart.request.args.get('redirect_uri', '')
|
||||
state = quart.request.args.get('state', '')
|
||||
|
||||
if not redirect_uri:
|
||||
return self.fail(1, 'Missing redirect_uri parameter')
|
||||
if 'state' in quart.request.args:
|
||||
return self.fail(1, 'Caller-supplied OAuth state is not allowed')
|
||||
|
||||
try:
|
||||
redirect_uri = self._validate_space_redirect_uri(redirect_uri, bind=False)
|
||||
launch_workspace_uuid = quart.request.args.get('launch_workspace_uuid')
|
||||
if launch_workspace_uuid:
|
||||
if not getattr(getattr(self.ap, 'deployment', None), 'multi_workspace_enabled', False):
|
||||
return self.fail(1, 'Space launch requires Cloud mode')
|
||||
try:
|
||||
uuid.UUID(launch_workspace_uuid)
|
||||
except ValueError:
|
||||
return self.fail(1, 'Invalid launch Workspace')
|
||||
state = await self.ap.user_service.issue_space_oauth_state(
|
||||
'login',
|
||||
launch_workspace_uuid=launch_workspace_uuid,
|
||||
)
|
||||
else:
|
||||
state = await self.ap.user_service.issue_space_oauth_state('login')
|
||||
authorize_url = self.ap.space_service.get_oauth_authorize_url(redirect_uri, state)
|
||||
return self.success(data={'authorize_url': authorize_url})
|
||||
except Exception as e:
|
||||
except ValueError as e:
|
||||
return self.fail(1, str(e))
|
||||
|
||||
@self.route('/space/bind-authorize-url', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Issue an account-bound, one-time Space OAuth redirect."""
|
||||
redirect_uri = quart.request.args.get('redirect_uri', '')
|
||||
if not redirect_uri:
|
||||
return self.fail(1, 'Missing redirect_uri parameter')
|
||||
if not request_context.account_uuid:
|
||||
return self.http_status(403, 'account_required', 'An Account is required')
|
||||
try:
|
||||
redirect_uri = self._validate_space_redirect_uri(redirect_uri, bind=True)
|
||||
state = await self.ap.user_service.issue_space_oauth_state(
|
||||
'bind',
|
||||
account_uuid=request_context.account_uuid,
|
||||
)
|
||||
authorize_url = self.ap.space_service.get_oauth_authorize_url(redirect_uri, state)
|
||||
return self.success(data={'authorize_url': authorize_url})
|
||||
except ValueError as e:
|
||||
return self.fail(1, str(e))
|
||||
|
||||
@self.route('/space/callback', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
@@ -117,11 +200,23 @@ class UserRouterGroup(group.RouterGroup):
|
||||
"""Handle OAuth callback - exchange code for tokens and authenticate"""
|
||||
json_data = await quart.request.json
|
||||
code = json_data.get('code')
|
||||
state = json_data.get('state')
|
||||
launch_assertion = json_data.get('launch_assertion')
|
||||
workspace_uuid = json_data.get('workspace_uuid')
|
||||
|
||||
if launch_assertion:
|
||||
return await self._handle_space_direct_launch(
|
||||
str(launch_assertion),
|
||||
str(workspace_uuid or '') or None,
|
||||
)
|
||||
|
||||
if not code:
|
||||
return self.fail(1, 'Missing authorization code')
|
||||
if not state:
|
||||
return self.fail(1, 'Missing state parameter')
|
||||
|
||||
try:
|
||||
consumed_state = await self.ap.user_service.consume_space_oauth_state_details(state, 'login')
|
||||
# Exchange code for tokens
|
||||
token_data = await self.ap.space_service.exchange_oauth_code(code)
|
||||
access_token = token_data.get('access_token')
|
||||
@@ -136,61 +231,80 @@ class UserRouterGroup(group.RouterGroup):
|
||||
access_token, refresh_token, expires_in
|
||||
)
|
||||
|
||||
launch_workspace_uuid = consumed_state.launch_workspace_uuid
|
||||
if launch_workspace_uuid:
|
||||
try:
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
user_obj.uuid,
|
||||
launch_workspace_uuid,
|
||||
)
|
||||
except Exception:
|
||||
self.ap.logger.warning('Rejected Space OAuth launch for unauthorized Workspace')
|
||||
return self.fail(1, 'Space OAuth failed')
|
||||
return self.success(
|
||||
data={
|
||||
'token': jwt_token,
|
||||
'user': user_obj.user,
|
||||
'workspace_uuid': access.workspace.uuid,
|
||||
}
|
||||
)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'token': jwt_token,
|
||||
'user': user_obj.user,
|
||||
}
|
||||
)
|
||||
except ControlPlaneDirectoryRequiredError as e:
|
||||
return self.http_status(409, e.code, str(e))
|
||||
except account_errors.AccountEmailMismatchError as e:
|
||||
return self.fail(3, str(e))
|
||||
except ValueError as e:
|
||||
traceback.print_exc()
|
||||
self.ap.logger.warning(f'Space OAuth callback failed: {e}')
|
||||
return self.fail(1, str(e))
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return self.fail(2, f'OAuth callback failed: {str(e)}')
|
||||
|
||||
@self.route('/info', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
"""Get current user information including account type"""
|
||||
user_obj = await self.ap.user_service.get_user_by_email(user_email)
|
||||
|
||||
if user_obj is None:
|
||||
return self.http_status(404, -1, 'User not found')
|
||||
return self.fail(getattr(e, 'code', 3), str(e))
|
||||
except ValueError:
|
||||
self.ap.logger.exception('Space OAuth callback failed')
|
||||
return self.fail(1, 'Space OAuth failed')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/info', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> str:
|
||||
"""Get current Account information without re-querying under Workspace RLS."""
|
||||
return self.success(
|
||||
data={
|
||||
'user': user_obj.user,
|
||||
'account_type': user_obj.account_type,
|
||||
'has_password': bool(user_obj.password and user_obj.password.strip()),
|
||||
'account_uuid': account.uuid,
|
||||
'user': account.user,
|
||||
'account_type': account.account_type,
|
||||
'has_password': bool(account.password and account.password.strip()),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/space-credits', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
"""Get Space credits balance for current user"""
|
||||
credits = await self.ap.space_service.get_credits(user_email)
|
||||
return self.success(data={'credits': credits})
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get Space credits using only the selected Workspace owner's credentials."""
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
request_context.account_uuid,
|
||||
request_context.workspace_uuid,
|
||||
)
|
||||
owner = await self.ap.user_service.get_workspace_owner(access.workspace.uuid)
|
||||
owner_space_bound = bool(owner and owner.space_account_uuid)
|
||||
credits = await self.ap.space_service.get_credits(owner.user) if owner_space_bound else None
|
||||
return self.success(
|
||||
data={
|
||||
'credits': credits,
|
||||
'owner_space_bound': owner_space_bound,
|
||||
'is_workspace_owner': access.membership.role == 'owner',
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/account-info', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
"""Get account info for login page (account type and has_password)"""
|
||||
"""Return instance login capabilities without disclosing an account."""
|
||||
if not await self.ap.user_service.is_initialized():
|
||||
return self.success(data={'initialized': False})
|
||||
|
||||
user_obj = await self.ap.user_service.get_first_user()
|
||||
if user_obj is None:
|
||||
return self.success(data={'initialized': False})
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'initialized': True,
|
||||
'account_type': user_obj.account_type,
|
||||
'has_password': bool(user_obj.password and user_obj.password.strip()),
|
||||
}
|
||||
)
|
||||
capabilities = await self.ap.user_service.get_login_capabilities()
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud':
|
||||
capabilities['password_login_enabled'] = False
|
||||
return self.success(data={'initialized': True, **capabilities})
|
||||
|
||||
@self.route('/set-password', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
@@ -233,7 +347,7 @@ class UserRouterGroup(group.RouterGroup):
|
||||
|
||||
json_data = await quart.request.json
|
||||
code = json_data.get('code')
|
||||
state = json_data.get('state') # JWT token passed as state
|
||||
state = json_data.get('state')
|
||||
|
||||
if not code:
|
||||
return self.http_status(400, -1, 'Missing authorization code')
|
||||
@@ -241,13 +355,10 @@ class UserRouterGroup(group.RouterGroup):
|
||||
if not state:
|
||||
return self.http_status(400, -1, 'Missing state parameter')
|
||||
|
||||
# Verify state is a valid JWT token
|
||||
try:
|
||||
user_email = await self.ap.user_service.verify_jwt_token(state)
|
||||
user_obj = await self.ap.user_service.consume_space_oauth_state(state, 'bind')
|
||||
except Exception:
|
||||
return self.http_status(401, -1, 'Invalid or expired state')
|
||||
|
||||
user_obj = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if user_obj is None:
|
||||
return self.http_status(404, -1, 'User not found')
|
||||
|
||||
@@ -255,8 +366,8 @@ class UserRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Only local accounts can bind to Space')
|
||||
|
||||
try:
|
||||
updated_user = await self.ap.user_service.bind_space_account(user_email, code)
|
||||
jwt_token = await self.ap.user_service.generate_jwt_token(updated_user.user)
|
||||
updated_user = await self.ap.user_service.bind_space_account(user_obj.user, code)
|
||||
jwt_token = await self.ap.user_service.generate_jwt_token(updated_user)
|
||||
return self.success(
|
||||
data={
|
||||
'token': jwt_token,
|
||||
@@ -264,7 +375,46 @@ class UserRouterGroup(group.RouterGroup):
|
||||
'account_type': updated_user.account_type,
|
||||
}
|
||||
)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to bind Space account: {str(e)}')
|
||||
except account_errors.AccountEmailMismatchError:
|
||||
return self.http_status(
|
||||
409,
|
||||
'space_account_email_mismatch',
|
||||
'Bind the LangBot Account with the same email as this local Account',
|
||||
)
|
||||
except ValueError:
|
||||
return self.http_status(400, -1, 'Space account binding failed')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
async def _handle_space_direct_launch(
|
||||
self,
|
||||
launch_assertion: str,
|
||||
workspace_uuid: str | None,
|
||||
) -> str:
|
||||
try:
|
||||
launch = await self.ap.space_launch_service.consume_assertion(
|
||||
launch_assertion,
|
||||
expected_workspace_uuid=workspace_uuid,
|
||||
)
|
||||
account = await self.ap.user_service.get_user_by_uuid(launch['account_uuid'])
|
||||
if account is None:
|
||||
raise SpaceLaunchError('Launch Account is not projected into Core')
|
||||
self.ap.user_service._require_active_account(account)
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
account.uuid,
|
||||
launch['workspace_uuid'],
|
||||
)
|
||||
token = await self.ap.user_service.generate_jwt_token(account)
|
||||
return self.success(
|
||||
data={
|
||||
'token': token,
|
||||
'user': account.user,
|
||||
'workspace_uuid': access.workspace.uuid,
|
||||
}
|
||||
)
|
||||
except SpaceLaunchError:
|
||||
self.ap.logger.warning('Rejected Space direct-launch assertion')
|
||||
return self.fail(1, 'Space launch failed')
|
||||
except Exception:
|
||||
self.ap.logger.exception('Space direct launch failed')
|
||||
return self.fail(1, 'Space launch failed')
|
||||
|
||||
@@ -1,49 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission, has_permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('webhook_mgmt', '/api/v1/webhooks')
|
||||
class WebhookManagementRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'])
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
webhooks = await self.ap.webhook_service.get_webhooks()
|
||||
return self.success(data={'webhooks': webhooks})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name', '')
|
||||
url = json_data.get('url', '')
|
||||
description = json_data.get('description', '')
|
||||
enabled = json_data.get('enabled', True)
|
||||
@self.route('', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
webhooks = await self.ap.webhook_service.get_webhooks(
|
||||
request_context,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
return self.success(data={'webhooks': webhooks})
|
||||
|
||||
if not name:
|
||||
return self.http_status(400, -1, 'Name is required')
|
||||
if not url:
|
||||
return self.http_status(400, -1, 'URL is required')
|
||||
@self.route('', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.get_json(silent=True) or {}
|
||||
name = json_data.get('name', '')
|
||||
url = json_data.get('url', '')
|
||||
description = json_data.get('description', '')
|
||||
enabled = json_data.get('enabled', True)
|
||||
|
||||
webhook = await self.ap.webhook_service.create_webhook(name, url, description, enabled)
|
||||
return self.success(data={'webhook': webhook})
|
||||
if not name:
|
||||
return self.http_status(400, -1, 'Name is required')
|
||||
if not url:
|
||||
return self.http_status(400, -1, 'URL is required')
|
||||
|
||||
@self.route('/<int:webhook_id>', methods=['GET', 'PUT', 'DELETE'])
|
||||
async def _(webhook_id: int) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
webhook = await self.ap.webhook_service.get_webhook(webhook_id)
|
||||
if webhook is None:
|
||||
try:
|
||||
webhook = await self.ap.webhook_service.create_webhook(
|
||||
request_context,
|
||||
name,
|
||||
url,
|
||||
description,
|
||||
enabled,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'webhook': webhook})
|
||||
|
||||
@self.route('/<int:webhook_id>', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def _(webhook_id: int, request_context: RequestContext) -> str:
|
||||
webhook = await self.ap.webhook_service.get_webhook(
|
||||
request_context,
|
||||
webhook_id,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
if webhook is None:
|
||||
return self.http_status(404, -1, 'Webhook not found')
|
||||
return self.success(data={'webhook': webhook})
|
||||
|
||||
@self.route(
|
||||
'/<int:webhook_id>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(webhook_id: int, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.get_json(silent=True) or {}
|
||||
updated = await self.ap.webhook_service.update_webhook(
|
||||
request_context,
|
||||
webhook_id,
|
||||
json_data.get('name'),
|
||||
json_data.get('url'),
|
||||
json_data.get('description'),
|
||||
json_data.get('enabled'),
|
||||
)
|
||||
if not updated:
|
||||
return self.http_status(404, -1, 'Webhook not found')
|
||||
return self.success(data={'webhook': webhook})
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name')
|
||||
url = json_data.get('url')
|
||||
description = json_data.get('description')
|
||||
enabled = json_data.get('enabled')
|
||||
|
||||
await self.ap.webhook_service.update_webhook(webhook_id, name, url, description, enabled)
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.webhook_service.delete_webhook(webhook_id)
|
||||
return self.success()
|
||||
deleted = await self.ap.webhook_service.delete_webhook(request_context, webhook_id)
|
||||
if not deleted:
|
||||
return self.http_status(404, -1, 'Webhook not found')
|
||||
return self.success()
|
||||
|
||||
@@ -4,6 +4,7 @@ import quart
|
||||
import traceback
|
||||
|
||||
from .. import group
|
||||
from .....utils import bounded_executor
|
||||
|
||||
|
||||
@group.group_class('webhooks', '/bots')
|
||||
@@ -30,7 +31,10 @@ class WebhookRouterGroup(group.RouterGroup):
|
||||
适配器返回的响应
|
||||
"""
|
||||
try:
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
# Public ingress never accepts X-Workspace-Id. The opaque bot UUID
|
||||
# is resolved against the already-bound runtime resource, which
|
||||
# carries the trusted Workspace and placement generation.
|
||||
runtime_bot = await self.ap.platform_mgr.resolve_public_bot(bot_uuid)
|
||||
|
||||
if not runtime_bot:
|
||||
return quart.jsonify({'error': 'Bot not found'}), 404
|
||||
@@ -41,14 +45,40 @@ class WebhookRouterGroup(group.RouterGroup):
|
||||
if not hasattr(runtime_bot.adapter, 'handle_unified_webhook'):
|
||||
return quart.jsonify({'error': 'Adapter does not support unified webhook'}), 501
|
||||
|
||||
response = await runtime_bot.adapter.handle_unified_webhook(
|
||||
bot_uuid=bot_uuid,
|
||||
path=path,
|
||||
request=quart.request,
|
||||
)
|
||||
async def dispatch():
|
||||
await self.ap.workspace_service.get_execution_binding(
|
||||
runtime_bot.workspace_uuid,
|
||||
expected_generation=runtime_bot.placement_generation,
|
||||
)
|
||||
return await runtime_bot.adapter.handle_unified_webhook(
|
||||
bot_uuid=bot_uuid,
|
||||
path=path,
|
||||
request=quart.request,
|
||||
)
|
||||
|
||||
with bounded_executor.blocking_work_scope(runtime_bot.workspace_uuid):
|
||||
persistence_mgr = self.ap.persistence_mgr
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud webhook dispatch requires an explicit tenant scope')
|
||||
async with tenant_scope(runtime_bot.workspace_uuid):
|
||||
response = await dispatch()
|
||||
else:
|
||||
response = await dispatch()
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Webhook dispatch error for bot {bot_uuid}: {traceback.format_exc()}')
|
||||
return quart.jsonify({'error': str(e)}), 500
|
||||
except bounded_executor.BlockingWorkCapacityError as exc:
|
||||
return self.http_status(
|
||||
429,
|
||||
'blocking_work_capacity_exceeded',
|
||||
str(exc),
|
||||
)
|
||||
except Exception:
|
||||
request_id = self.request_id()
|
||||
self.ap.logger.error(
|
||||
f'Webhook dispatch error request_id={request_id} bot={bot_uuid}: {traceback.format_exc()}'
|
||||
)
|
||||
return self.internal_error_response(request_id)
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission, permissions_for_role
|
||||
from ...context import RequestContext
|
||||
from ...service.user import AccountExistsLoginRequiredError, ControlPlaneDirectoryRequiredError
|
||||
from .....entity.persistence.workspace import Workspace, WorkspaceInvitation, WorkspaceMembership
|
||||
from .....entity.persistence.workspace import WorkspaceSource
|
||||
from .....workspace.collaboration import WorkspaceMemberView
|
||||
from .....workspace.errors import WorkspaceNotFoundError
|
||||
from .....workspace.invitation_delivery import InvitationDeliveryService
|
||||
from .. import group
|
||||
|
||||
|
||||
def _workspace_payload(workspace: Workspace) -> dict[str, typing.Any]:
|
||||
return {
|
||||
'uuid': workspace.uuid,
|
||||
'instance_uuid': workspace.instance_uuid,
|
||||
'name': workspace.name,
|
||||
'slug': workspace.slug,
|
||||
'type': workspace.type,
|
||||
'status': workspace.status,
|
||||
'source': workspace.source,
|
||||
}
|
||||
|
||||
|
||||
def _membership_payload(
|
||||
membership: WorkspaceMembership,
|
||||
*,
|
||||
email: str,
|
||||
) -> dict[str, typing.Any]:
|
||||
return {
|
||||
'uuid': membership.uuid,
|
||||
'workspace_uuid': membership.workspace_uuid,
|
||||
'account_uuid': membership.account_uuid,
|
||||
'email': email,
|
||||
'role': membership.role,
|
||||
'status': membership.status,
|
||||
'joined_at': membership.joined_at.isoformat() if membership.joined_at else None,
|
||||
'created_at': membership.created_at.isoformat() if membership.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _invitation_payload(invitation: WorkspaceInvitation) -> dict[str, typing.Any]:
|
||||
"""Serialize an invitation without its bearer-secret hash."""
|
||||
|
||||
return {
|
||||
'uuid': invitation.uuid,
|
||||
'workspace_uuid': invitation.workspace_uuid,
|
||||
'normalized_email': invitation.normalized_email,
|
||||
'role': invitation.role,
|
||||
'status': invitation.status,
|
||||
'expires_at': invitation.expires_at.isoformat(),
|
||||
'created_at': invitation.created_at.isoformat() if invitation.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
@group.group_class('workspaces', '/api/v1/workspaces')
|
||||
class WorkspacesRouterGroup(group.RouterGroup):
|
||||
async def _run_in_workspace_uow(
|
||||
self, workspace_uuid: str, operation: typing.Callable[[], typing.Awaitable[typing.Any]]
|
||||
):
|
||||
"""Bind collaboration persistence to the selected tenant in Cloud."""
|
||||
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
async with self.ap.persistence_mgr.tenant_uow(workspace_uuid):
|
||||
return await operation()
|
||||
return await operation()
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/bootstrap', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> typing.Any:
|
||||
"""List the active Workspaces available to an authenticated Account.
|
||||
|
||||
This account-only endpoint intentionally runs before Workspace
|
||||
selection. It never accepts a selector as authority and does not
|
||||
choose a default Workspace for a multi-membership Account.
|
||||
"""
|
||||
|
||||
accesses = await self.ap.workspace_collaboration_service.list_account_workspaces(account.uuid)
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
workspaces: list[dict[str, typing.Any]] = []
|
||||
for access in accesses:
|
||||
plan_name: str | None = None
|
||||
if access.workspace.source == WorkspaceSource.CLOUD_PROJECTION.value and resolver is not None:
|
||||
entitlement = await resolver.resolve(
|
||||
access.workspace.uuid,
|
||||
minimum_revision=access.membership.projection_revision,
|
||||
)
|
||||
plan_name = entitlement.plan_name
|
||||
workspaces.append(
|
||||
{
|
||||
'workspace': _workspace_payload(access.workspace),
|
||||
'membership': _membership_payload(access.membership, email=account.user),
|
||||
'permissions': sorted(permissions_for_role(access.membership.role)),
|
||||
'placement_generation': access.execution.placement_generation,
|
||||
'plan_name': plan_name,
|
||||
}
|
||||
)
|
||||
return self.success(data={'workspaces': workspaces})
|
||||
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> typing.Any:
|
||||
accesses = await self.ap.workspace_collaboration_service.list_account_workspaces(account.uuid)
|
||||
return self.success(data={'workspaces': [_workspace_payload(access.workspace) for access in accesses]})
|
||||
|
||||
@self.route('', methods=['POST'], permission=Permission.WORKSPACE_VIEW)
|
||||
async def _(request_context: RequestContext) -> typing.Any:
|
||||
if self.ap.workspace_service.policy.multi_workspace_enabled:
|
||||
return self.http_status(
|
||||
409,
|
||||
'control_plane_required',
|
||||
'Cloud Workspaces are created by the SaaS control plane',
|
||||
)
|
||||
return self.http_status(403, 'edition_limit', 'This edition supports one Workspace per instance')
|
||||
|
||||
@self.route('/current', methods=['GET'], permission=Permission.WORKSPACE_VIEW)
|
||||
async def _(request_context: RequestContext) -> typing.Any:
|
||||
membership = quart.g.workspace_membership
|
||||
account = await self.ap.user_service.get_user_by_uuid(request_context.account_uuid)
|
||||
if account is None:
|
||||
return self.http_status(401, 'invalid_authentication', 'Account not found')
|
||||
workspace = await self.ap.workspace_service.get_workspace(request_context.workspace_uuid)
|
||||
plan_name: str | None = None
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if workspace.source == WorkspaceSource.CLOUD_PROJECTION.value and resolver is not None:
|
||||
entitlement = await resolver.resolve(
|
||||
workspace.uuid,
|
||||
minimum_revision=request_context.entitlement_revision,
|
||||
)
|
||||
plan_name = entitlement.plan_name
|
||||
return self.success(
|
||||
data={
|
||||
'workspace': _workspace_payload(workspace),
|
||||
'membership': _membership_payload(membership, email=account.user),
|
||||
'permissions': sorted(request_context.workspace.permissions),
|
||||
'placement_generation': request_context.placement_generation,
|
||||
'plan_name': plan_name,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<workspace_uuid>', methods=['GET'], permission=Permission.WORKSPACE_VIEW)
|
||||
async def _(workspace_uuid: str, request_context: RequestContext) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
workspace = await self.ap.workspace_service.get_workspace(workspace_uuid)
|
||||
return self.success(data={'workspace': _workspace_payload(workspace)})
|
||||
|
||||
@self.route('/<workspace_uuid>/members', methods=['GET'], permission=Permission.MEMBER_VIEW)
|
||||
async def _(workspace_uuid: str, request_context: RequestContext) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
|
||||
async def list_members():
|
||||
return await self.ap.workspace_collaboration_service.list_members(
|
||||
workspace_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
members = await self._run_in_workspace_uow(workspace_uuid, list_members)
|
||||
return self.success(data={'members': [self._member_view_payload(item) for item in members]})
|
||||
|
||||
@self.route(
|
||||
'/<workspace_uuid>/invitations',
|
||||
methods=['GET', 'POST'],
|
||||
permission=Permission.MEMBER_INVITE,
|
||||
)
|
||||
async def _(workspace_uuid: str, request_context: RequestContext) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
if quart.request.method == 'GET':
|
||||
|
||||
async def list_invitations():
|
||||
return await self.ap.workspace_collaboration_service.list_invitations(
|
||||
workspace_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
invitations = await self._run_in_workspace_uow(workspace_uuid, list_invitations)
|
||||
return self.success(data={'invitations': [_invitation_payload(item) for item in invitations]})
|
||||
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
|
||||
async def create_invitation():
|
||||
return await self.ap.workspace_collaboration_service.create_invitation(
|
||||
workspace_uuid,
|
||||
quart.g.workspace_membership,
|
||||
str(data.get('email', '')),
|
||||
str(data.get('role', 'viewer')),
|
||||
)
|
||||
|
||||
created = await self._run_in_workspace_uow(workspace_uuid, create_invitation)
|
||||
delivery_service = self._invitation_delivery_service()
|
||||
link = delivery_service.build_invitation_link(created.token)
|
||||
workspace = await self.ap.workspace_service.get_workspace(workspace_uuid)
|
||||
delivery = await delivery_service.deliver_invitation(
|
||||
recipient_email=created.invitation.normalized_email,
|
||||
workspace_name=workspace.name,
|
||||
invitation_link=link,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'invitation': _invitation_payload(created.invitation),
|
||||
'token': created.token,
|
||||
'link': link,
|
||||
'delivery': delivery.to_public_dict(),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route(
|
||||
'/<workspace_uuid>/invitations/<invitation_uuid>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.MEMBER_INVITE,
|
||||
)
|
||||
async def _(
|
||||
workspace_uuid: str,
|
||||
invitation_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
|
||||
async def revoke_invitation():
|
||||
return await self.ap.workspace_collaboration_service.revoke_invitation(
|
||||
workspace_uuid, invitation_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
invitation = await self._run_in_workspace_uow(workspace_uuid, revoke_invitation)
|
||||
return self.success(data={'invitation': _invitation_payload(invitation)})
|
||||
|
||||
@self.route(
|
||||
'/<workspace_uuid>/members/<account_uuid>',
|
||||
methods=['PATCH', 'DELETE'],
|
||||
permission=Permission.MEMBER_UPDATE_ROLE,
|
||||
)
|
||||
async def _(
|
||||
workspace_uuid: str,
|
||||
account_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
if quart.request.method == 'DELETE':
|
||||
if Permission.MEMBER_REMOVE.value not in request_context.workspace.permissions:
|
||||
return self.http_status(403, 'permission_denied', 'Member removal permission is required')
|
||||
|
||||
async def remove_member():
|
||||
return await self.ap.workspace_collaboration_service.remove_member(
|
||||
workspace_uuid, account_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
member = await self._run_in_workspace_uow(workspace_uuid, remove_member)
|
||||
return self.success(data={'account_uuid': member.account_uuid})
|
||||
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
|
||||
async def update_member_role():
|
||||
return await self.ap.workspace_collaboration_service.update_member_role(
|
||||
workspace_uuid,
|
||||
account_uuid,
|
||||
str(data.get('role', '')),
|
||||
quart.g.workspace_membership,
|
||||
)
|
||||
|
||||
member = await self._run_in_workspace_uow(workspace_uuid, update_member_role)
|
||||
account = await self.ap.user_service.get_user_by_uuid(member.account_uuid)
|
||||
return self.success(
|
||||
data={
|
||||
'member': _membership_payload(
|
||||
member,
|
||||
email=account.user if account is not None else '',
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _require_current_workspace(workspace_uuid: str, request_context: RequestContext) -> None:
|
||||
if workspace_uuid != request_context.workspace_uuid:
|
||||
raise WorkspaceNotFoundError('Workspace not found')
|
||||
|
||||
def _invitation_delivery_service(self) -> InvitationDeliveryService:
|
||||
service = getattr(self.ap, 'invitation_delivery_service', None)
|
||||
if service is None:
|
||||
service = InvitationDeliveryService(self.ap)
|
||||
self.ap.invitation_delivery_service = service
|
||||
return service
|
||||
|
||||
@staticmethod
|
||||
def _member_view_payload(view: WorkspaceMemberView) -> dict[str, typing.Any]:
|
||||
return _membership_payload(view.membership, email=view.email)
|
||||
|
||||
|
||||
@group.group_class('invitations', '/api/v1/invitations')
|
||||
class InvitationsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/inspect', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> typing.Any:
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
invitation, workspace = await self.ap.workspace_collaboration_service.inspect_invitation(
|
||||
str(data.get('token', ''))
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'invitation': _invitation_payload(invitation),
|
||||
'workspace': _workspace_payload(workspace),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/accept', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> typing.Any:
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
invitation_token = str(data.get('token', ''))
|
||||
if not invitation_token:
|
||||
return self.http_status(400, 'invitation_invalid', 'Invitation token is required')
|
||||
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if authorization.startswith('Bearer '):
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') != 'cloud':
|
||||
return self.http_status(
|
||||
409,
|
||||
'invitation_logout_required',
|
||||
'Sign out before creating the invited local Account',
|
||||
)
|
||||
try:
|
||||
account = await self.ap.user_service.get_authenticated_account(
|
||||
authorization.removeprefix('Bearer ')
|
||||
)
|
||||
if isinstance(account, str):
|
||||
account = await self.ap.user_service.get_user_by_email(account)
|
||||
except Exception as exc:
|
||||
return self._auth_error_response(exc)
|
||||
if account is None:
|
||||
return self.http_status(401, 'invalid_authentication', 'Account not found')
|
||||
membership = await self.ap.workspace_collaboration_service.accept_invitation(
|
||||
invitation_token,
|
||||
account.uuid,
|
||||
)
|
||||
token = await self.ap.user_service.generate_jwt_token(account)
|
||||
return self.success(data={'token': token, 'workspace_uuid': membership.workspace_uuid})
|
||||
|
||||
registration = data.get('registration')
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud':
|
||||
return self.http_status(
|
||||
401,
|
||||
'account_exists_login_required',
|
||||
'Login with your LangBot Account to accept this invitation',
|
||||
)
|
||||
if not isinstance(registration, dict):
|
||||
return self.http_status(
|
||||
401,
|
||||
'account_exists_login_required',
|
||||
'Sign in or provide registration details to accept this invitation',
|
||||
)
|
||||
password = registration.get('password')
|
||||
if not isinstance(password, str) or len(password) < 8:
|
||||
return self.http_status(400, 'invalid_password', 'Password must contain at least 8 characters')
|
||||
try:
|
||||
_, membership = await self.ap.user_service.register_invited_account(
|
||||
invitation_token,
|
||||
str(registration.get('email', '')),
|
||||
password,
|
||||
)
|
||||
except ControlPlaneDirectoryRequiredError as exc:
|
||||
return self.http_status(409, exc.code, str(exc))
|
||||
except AccountExistsLoginRequiredError as exc:
|
||||
return self.http_status(409, exc.code, str(exc))
|
||||
return self.success(data={'workspace_uuid': membership.workspace_uuid, 'login_required': True})
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import typing
|
||||
|
||||
import quart
|
||||
import quart_cors
|
||||
@@ -27,6 +28,37 @@ importutil.import_modules_in_pkg(groups_knowledge)
|
||||
importutil.import_modules_in_pkg(groups_resources)
|
||||
|
||||
|
||||
class BoundedJSONRequest(quart.Request):
|
||||
"""Parse bounded HTTP JSON bodies outside the shared event loop."""
|
||||
|
||||
async def get_json(
|
||||
self,
|
||||
force: bool = False,
|
||||
silent: bool = False,
|
||||
cache: bool = True,
|
||||
) -> typing.Any:
|
||||
# Keep Quart's cache and error semantics, changing only where the
|
||||
# potentially 10 MiB JSON decoder runs. The RouterGroup establishes a
|
||||
# trusted Workspace blocking-work scope before calling route handlers.
|
||||
if cache and self._cached_json[silent] is not Ellipsis:
|
||||
return self._cached_json[silent]
|
||||
if not (force or self.is_json):
|
||||
return None
|
||||
|
||||
data = await self.get_data(cache=cache, as_text=False)
|
||||
try:
|
||||
result = await asyncio.to_thread(self.json_module.loads, data)
|
||||
except ValueError as error:
|
||||
if silent:
|
||||
result = None
|
||||
else:
|
||||
result = self.on_json_loading_failed(error)
|
||||
|
||||
if cache:
|
||||
self._cached_json[silent] = result
|
||||
return result
|
||||
|
||||
|
||||
class HTTPController:
|
||||
ap: app.Application
|
||||
|
||||
@@ -35,6 +67,7 @@ class HTTPController:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
self.quart_app = quart.Quart(__name__)
|
||||
self.quart_app.request_class = BoundedJSONRequest
|
||||
quart_cors.cors(self.quart_app, allow_origin='*')
|
||||
|
||||
# Set maximum content length to prevent large file uploads
|
||||
@@ -103,6 +136,7 @@ class HTTPController:
|
||||
config.accesslog = '-'
|
||||
config.bind = [f'{host}:{port}']
|
||||
config.errorlog = config.accesslog
|
||||
config.websocket_max_message_size = group.MAX_FILE_SIZE
|
||||
|
||||
asgi_app = self.quart_app
|
||||
if self.mcp_mount is not None:
|
||||
@@ -113,7 +147,16 @@ class HTTPController:
|
||||
async def register_routes(self) -> None:
|
||||
@self.quart_app.route('/healthz')
|
||||
async def healthz():
|
||||
return {'code': 0, 'msg': 'ok'}
|
||||
get_resource_stats = getattr(
|
||||
self.ap,
|
||||
'get_runtime_resource_stats',
|
||||
None,
|
||||
)
|
||||
return {
|
||||
'code': 0,
|
||||
'msg': 'ok',
|
||||
'resources': (get_resource_stats() if callable(get_resource_stats) else {}),
|
||||
}
|
||||
|
||||
for g in group.preregistered_groups:
|
||||
ginst = g(self.ap, self.quart_app)
|
||||
|
||||
@@ -1,97 +1,304 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import datetime
|
||||
import hashlib
|
||||
import secrets
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import apikey
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..authz import Permission, PermissionDeniedError
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class ApiKeyIdentity:
|
||||
"""Trusted Workspace identity derived from an API-key secret."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
api_key_uuid: str
|
||||
permissions: frozenset[str]
|
||||
|
||||
|
||||
class ApiKeyService:
|
||||
ap: app.Application
|
||||
"""Manage hashed, Workspace-bound API keys."""
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(self, ap: Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_api_keys(self) -> list[dict]:
|
||||
"""Get all API keys"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(apikey.ApiKey))
|
||||
@staticmethod
|
||||
def _hash_secret(secret: str) -> str:
|
||||
return hashlib.sha256(secret.encode('utf-8')).hexdigest()
|
||||
|
||||
keys = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(apikey.ApiKey, key) for key in keys]
|
||||
@staticmethod
|
||||
def _utcnow() -> datetime.datetime:
|
||||
return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
|
||||
async def create_api_key(self, name: str, description: str = '') -> dict:
|
||||
"""Create a new API key"""
|
||||
# Generate a secure random API key
|
||||
key = f'lbk_{secrets.token_urlsafe(32)}'
|
||||
@staticmethod
|
||||
def _normalize_scopes(
|
||||
scopes: typing.Iterable[str] | None,
|
||||
*,
|
||||
default: typing.Iterable[str] = (),
|
||||
) -> list[str]:
|
||||
requested = list(default if scopes is None else scopes)
|
||||
valid = {permission.value for permission in Permission}
|
||||
normalized: list[str] = []
|
||||
for scope in requested:
|
||||
if not isinstance(scope, str):
|
||||
raise ValueError('API key scopes must be strings')
|
||||
value = scope.strip()
|
||||
if value not in valid:
|
||||
raise ValueError(f'Unknown API key scope: {value}')
|
||||
if value not in normalized:
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
key_data = {'name': name, 'key': key, 'description': description}
|
||||
def _serialize(self, row: typing.Any) -> dict[str, typing.Any]:
|
||||
value = self.ap.persistence_mgr.serialize_model(apikey.ApiKey, row)
|
||||
value.pop('key_hash', None)
|
||||
# The secret is deliberately unrecoverable after creation.
|
||||
value['secret_available'] = False
|
||||
return value
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(apikey.ApiKey).values(**key_data))
|
||||
|
||||
# Retrieve the created key
|
||||
async def get_api_keys(self, context: TenantContext) -> list[dict[str, typing.Any]]:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).order_by(apikey.ApiKey.created_at, apikey.ApiKey.id),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
created_key = result.first()
|
||||
return [self._serialize(key) for key in result.all()]
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(apikey.ApiKey, created_key)
|
||||
async def create_api_key(
|
||||
self,
|
||||
context: TenantContext,
|
||||
name: str,
|
||||
description: str = '',
|
||||
*,
|
||||
scopes: typing.Iterable[str] | None = None,
|
||||
expires_at: datetime.datetime | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
normalized_name = name.strip()
|
||||
if not normalized_name:
|
||||
raise ValueError('Name is required')
|
||||
if expires_at is not None:
|
||||
if expires_at.tzinfo is not None:
|
||||
expires_at = expires_at.astimezone(datetime.UTC).replace(tzinfo=None)
|
||||
if expires_at <= self._utcnow():
|
||||
raise ValueError('API key expiry must be in the future')
|
||||
|
||||
async def get_api_key(self, key_id: int) -> dict | None:
|
||||
"""Get a specific API key by ID"""
|
||||
default_scopes = getattr(getattr(context, 'workspace', None), 'permissions', frozenset())
|
||||
normalized_scopes = self._normalize_scopes(scopes, default=default_scopes)
|
||||
allowed_scopes = frozenset(default_scopes)
|
||||
unauthorized_scopes = sorted(set(normalized_scopes) - allowed_scopes)
|
||||
if unauthorized_scopes:
|
||||
# API-key management delegates the caller's authority; it must not
|
||||
# become a path for minting a stronger principal.
|
||||
raise PermissionDeniedError(unauthorized_scopes[0])
|
||||
secret = f'lbk_{secrets.token_urlsafe(32)}'
|
||||
key_uuid = str(uuid.uuid4())
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(apikey.ApiKey).values(
|
||||
uuid=key_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
created_by_account_uuid=getattr(context, 'account_uuid', None),
|
||||
name=normalized_name,
|
||||
key_hash=self._hash_secret(secret),
|
||||
scopes=normalized_scopes,
|
||||
status=apikey.ApiKeyStatus.ACTIVE.value,
|
||||
expires_at=expires_at,
|
||||
description=description.strip(),
|
||||
)
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.id == key_id)
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.uuid == key_uuid),
|
||||
apikey.ApiKey,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
created = result.first()
|
||||
if created is None:
|
||||
raise RuntimeError('Created API key could not be loaded')
|
||||
value = self._serialize(created)
|
||||
value['key'] = secret
|
||||
value['secret_available'] = True
|
||||
return value
|
||||
|
||||
async def get_api_key(self, context: TenantContext, key_id: int) -> dict[str, typing.Any] | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.id == key_id),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
key = result.first()
|
||||
return None if key is None else self._serialize(key)
|
||||
|
||||
if key is None:
|
||||
async def authenticate_api_key(self, secret: str) -> ApiKeyIdentity | None:
|
||||
"""Authenticate a secret and derive its Workspace without trusting headers."""
|
||||
|
||||
if not isinstance(secret, str) or not secret.strip():
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(apikey.ApiKey, key)
|
||||
global_secret = self.ap.instance_config.data.get('api', {}).get('global_api_key', '')
|
||||
if global_secret and secrets.compare_digest(secret, global_secret):
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
if workspace_service is None or workspace_service.policy.multi_workspace_enabled:
|
||||
return None
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid='global-oss-api-key',
|
||||
permissions=frozenset(permission.value for permission in Permission),
|
||||
)
|
||||
|
||||
async def verify_api_key(self, key: str) -> bool:
|
||||
"""Verify if an API key is valid.
|
||||
if not secret.startswith('lbk_'):
|
||||
return None
|
||||
secret_hash = self._hash_secret(secret)
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
discovery_uow = getattr(self.ap.persistence_mgr, 'api_key_discovery_uow', None)
|
||||
if current_session() is None and callable(discovery_uow):
|
||||
async with discovery_uow(secret_hash) as discovery:
|
||||
key = await discovery.session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
key = result.first()
|
||||
if key is None:
|
||||
return None
|
||||
discovered_workspace_uuid = key.workspace_uuid
|
||||
discovered_key_id = key.id
|
||||
now = self._utcnow()
|
||||
|
||||
A key is accepted if it matches the global API key configured in
|
||||
``config.yaml`` (``api.global_api_key``) — which requires no login
|
||||
session and no database record — or if it matches a key created via
|
||||
the web UI (stored in the database, prefixed with ``lbk_``).
|
||||
"""
|
||||
if not isinstance(key, str) or not key:
|
||||
return False
|
||||
async def bind_and_record_use() -> tuple[typing.Any, typing.Any] | None:
|
||||
# Re-read inside the tenant transaction. A revoke/expiry racing
|
||||
# discovery must not result in an authenticated identity.
|
||||
active_session = current_session()
|
||||
if active_session is not None:
|
||||
scoped_key = await active_session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
else: # compatibility for isolated service tests
|
||||
scoped_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
scoped_key = scoped_result.first()
|
||||
if scoped_key is None or scoped_key.status != apikey.ApiKeyStatus.ACTIVE.value:
|
||||
return None
|
||||
if scoped_key.expires_at is not None and scoped_key.expires_at <= now:
|
||||
return None
|
||||
|
||||
# 1. Global API key from config.yaml (no DB lookup, no login state).
|
||||
# Note: config completion only backfills top-level keys, so existing
|
||||
# installs may not have this key — access it defensively.
|
||||
global_api_key = self.ap.instance_config.data.get('api', {}).get('global_api_key', '')
|
||||
if global_api_key and secrets.compare_digest(key, global_api_key):
|
||||
return True
|
||||
binding = await self.ap.workspace_service.get_execution_binding(discovered_workspace_uuid)
|
||||
updated = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(
|
||||
apikey.ApiKey.id == scoped_key.id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
apikey.ApiKey.status == apikey.ApiKeyStatus.ACTIVE.value,
|
||||
)
|
||||
.values(last_used_at=now)
|
||||
.returning(apikey.ApiKey.id)
|
||||
)
|
||||
# Authentication and revocation race on this atomic predicate. If
|
||||
# revoke won, no active row is returned and the stale object read
|
||||
# above must never become an authenticated identity.
|
||||
if updated.scalar_one_or_none() is None:
|
||||
return None
|
||||
return binding, scoped_key
|
||||
|
||||
# 2. Web-UI-created keys are stored in the database and prefixed lbk_.
|
||||
if not key.startswith('lbk_'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if current_session() is None and callable(tenant_uow):
|
||||
async with tenant_uow(discovered_workspace_uuid):
|
||||
bound = await bind_and_record_use()
|
||||
else:
|
||||
bound = await bind_and_record_use()
|
||||
if bound is None:
|
||||
return None
|
||||
binding, scoped_key = bound
|
||||
raw_scopes = list(scoped_key.scopes or [])
|
||||
permissions = (
|
||||
frozenset(permission.value for permission in Permission)
|
||||
if '*' in raw_scopes
|
||||
else frozenset(self._normalize_scopes(raw_scopes))
|
||||
)
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid=scoped_key.uuid,
|
||||
permissions=permissions,
|
||||
)
|
||||
|
||||
key_obj = result.first()
|
||||
return key_obj is not None
|
||||
async def verify_api_key(self, secret: str) -> bool:
|
||||
try:
|
||||
return await self.authenticate_api_key(secret) is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def delete_api_key(self, key_id: int) -> None:
|
||||
"""Delete an API key"""
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.delete(apikey.ApiKey).where(apikey.ApiKey.id == key_id))
|
||||
|
||||
async def update_api_key(self, key_id: int, name: str = None, description: str = None) -> None:
|
||||
"""Update an API key's metadata (name, description)"""
|
||||
update_data = {}
|
||||
if name is not None:
|
||||
update_data['name'] = name
|
||||
if description is not None:
|
||||
update_data['description'] = description
|
||||
|
||||
if update_data:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey).where(apikey.ApiKey.id == key_id).values(**update_data)
|
||||
async def delete_api_key(self, context: TenantContext, key_id: int) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(apikey.ApiKey.id == key_id)
|
||||
.values(status=apikey.ApiKeyStatus.REVOKED.value),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', 0) == 0:
|
||||
raise WorkspaceNotFoundError('API key not found')
|
||||
|
||||
async def update_api_key(
|
||||
self,
|
||||
context: TenantContext,
|
||||
key_id: int,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
) -> None:
|
||||
update_data: dict[str, typing.Any] = {}
|
||||
if name is not None:
|
||||
normalized_name = name.strip()
|
||||
if not normalized_name:
|
||||
raise ValueError('Name is required')
|
||||
update_data['name'] = normalized_name
|
||||
if description is not None:
|
||||
update_data['description'] = description.strip()
|
||||
if not update_data:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(apikey.ApiKey).where(apikey.ApiKey.id == key_id).values(**update_data),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', 0) == 0:
|
||||
raise WorkspaceNotFoundError('API key not found')
|
||||
|
||||
@@ -2,11 +2,12 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import sqlalchemy
|
||||
import typing
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
class BotService:
|
||||
@@ -17,9 +18,11 @@ class BotService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_bots(self, include_secret: bool = True) -> list[dict]:
|
||||
async def get_bots(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_bot.Bot), persistence_bot.Bot, context)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
@@ -29,10 +32,14 @@ class BotService:
|
||||
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
|
||||
|
||||
async def get_bot(self, bot_uuid: str, include_secret: bool = True) -> dict | None:
|
||||
async def get_bot(self, context: TenantContext, bot_uuid: str, include_secret: bool = False) -> dict | None:
|
||||
"""获取机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
bot = result.first()
|
||||
@@ -46,15 +53,20 @@ class BotService:
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
|
||||
|
||||
async def get_runtime_bot_info(self, bot_uuid: str, include_secret: bool = True) -> dict:
|
||||
async def get_runtime_bot_info(
|
||||
self,
|
||||
context: TenantContext,
|
||||
bot_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict:
|
||||
"""获取机器人运行时信息"""
|
||||
persistence_bot = await self.get_bot(bot_uuid, include_secret)
|
||||
persistence_bot = await self.get_bot(context, bot_uuid, include_secret)
|
||||
if persistence_bot is None:
|
||||
raise Exception('Bot not found')
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
|
||||
adapter_runtime_values = {}
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is not None:
|
||||
adapter_runtime_values['bot_account_id'] = runtime_bot.adapter.bot_account_id
|
||||
|
||||
@@ -86,22 +98,29 @@ class BotService:
|
||||
|
||||
return persistence_bot
|
||||
|
||||
async def create_bot(self, bot_data: dict) -> str:
|
||||
async def create_bot(self, context: TenantContext, bot_data: dict) -> str:
|
||||
"""Create bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_bots = limitation.get('max_bots', -1)
|
||||
if max_bots >= 0:
|
||||
existing_bots = await self.get_bots()
|
||||
existing_bots = await self.get_bots(context)
|
||||
if len(existing_bots) >= max_bots:
|
||||
raise ValueError(f'Maximum number of bots ({max_bots}) reached')
|
||||
|
||||
# TODO: 检查配置信息格式
|
||||
bot_data = bot_data.copy()
|
||||
bot_data['uuid'] = str(uuid.uuid4())
|
||||
bot_data['workspace_uuid'] = workspace_uuid
|
||||
|
||||
# bind the most recently updated pipeline if any exist
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -112,61 +131,84 @@ class BotService:
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(bot_data))
|
||||
|
||||
bot = await self.get_bot(bot_data['uuid'])
|
||||
bot = await self.get_bot(context, bot_data['uuid'], include_secret=True)
|
||||
|
||||
await self.ap.platform_mgr.load_bot(bot)
|
||||
await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
return bot_data['uuid']
|
||||
|
||||
async def update_bot(self, bot_uuid: str, bot_data: dict) -> None:
|
||||
async def update_bot(self, context: TenantContext, bot_uuid: str, bot_data: dict) -> None:
|
||||
"""Update bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
update_data = bot_data.copy()
|
||||
|
||||
if 'uuid' in update_data:
|
||||
del update_data['uuid']
|
||||
update_data.pop('uuid', None)
|
||||
update_data.pop('workspace_uuid', None)
|
||||
|
||||
# set use_pipeline_name
|
||||
if 'use_pipeline_uuid' in update_data:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
update_data['use_pipeline_name'] = pipeline.name
|
||||
else:
|
||||
raise Exception('Pipeline not found')
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
# select from db
|
||||
bot = await self.get_bot(bot_uuid)
|
||||
bot = await self.get_bot(context, bot_uuid, include_secret=True)
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(bot)
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
if runtime_bot.enable:
|
||||
await runtime_bot.run()
|
||||
|
||||
# update all conversation that use this bot
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if session.using_conversation is not None and session.using_conversation.bot_uuid == bot_uuid:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
and session.using_conversation.bot_uuid == bot_uuid
|
||||
and getattr(session, 'workspace_uuid', workspace_uuid) == workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
|
||||
async def delete_bot(self, bot_uuid: str) -> None:
|
||||
async def delete_bot(self, context: TenantContext, bot_uuid: str) -> None:
|
||||
"""Delete bot"""
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
async def list_event_logs(
|
||||
self, bot_uuid: str, from_index: int, max_count: int
|
||||
) -> typing.Tuple[list[dict], int, int, int]:
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
self, context: TenantContext, bot_uuid: str, from_index: int, max_count: int
|
||||
) -> tuple[list[dict], int]:
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is None:
|
||||
raise Exception('Bot not found')
|
||||
|
||||
@@ -174,7 +216,14 @@ class BotService:
|
||||
|
||||
return [log.to_json() for log in logs], total_count
|
||||
|
||||
async def send_message(self, bot_uuid: str, target_type: str, target_id: str, message_chain_data: dict) -> None:
|
||||
async def send_message(
|
||||
self,
|
||||
context: TenantContext,
|
||||
bot_uuid: str,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
message_chain_data: dict,
|
||||
) -> None:
|
||||
"""Send message to a specific target via bot
|
||||
|
||||
Args:
|
||||
@@ -183,11 +232,14 @@ class BotService:
|
||||
target_id: The ID of the target
|
||||
message_chain_data: The message chain data in dict format
|
||||
"""
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
|
||||
# Import here to avoid circular imports
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
|
||||
# Get runtime bot
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is None:
|
||||
raise Exception(f'Bot not found: {bot_uuid}')
|
||||
|
||||
@@ -202,19 +254,29 @@ class BotService:
|
||||
|
||||
# ============ Bot Admins ============
|
||||
|
||||
async def get_bot_admins(self, bot_uuid: str) -> list[dict]:
|
||||
async def get_bot_admins(self, context: TenantContext, bot_uuid: str) -> list[dict]:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid),
|
||||
persistence_bot.BotAdmin,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return [{'id': r.id, 'launcher_type': r.launcher_type, 'launcher_id': r.launcher_id} for r in result.all()]
|
||||
|
||||
async def add_bot_admin(self, bot_uuid: str, launcher_type: str, launcher_id: str) -> int:
|
||||
async def add_bot_admin(self, context: TenantContext, bot_uuid: str, launcher_type: str, launcher_id: str) -> int:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_bot.BotAdmin).values(
|
||||
workspace_uuid=workspace_uuid,
|
||||
bot_uuid=bot_uuid,
|
||||
launcher_type=launcher_type,
|
||||
launcher_id=launcher_id,
|
||||
@@ -222,12 +284,18 @@ class BotService:
|
||||
)
|
||||
return result.inserted_primary_key[0]
|
||||
|
||||
async def delete_bot_admin(self, bot_uuid: str, admin_id: int) -> None:
|
||||
async def delete_bot_admin(self, context: TenantContext, bot_uuid: str, admin_id: int) -> None:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.BotAdmin).where(
|
||||
persistence_bot.BotAdmin.bot_uuid == bot_uuid,
|
||||
persistence_bot.BotAdmin.id == admin_id,
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_bot.BotAdmin).where(
|
||||
persistence_bot.BotAdmin.bot_uuid == bot_uuid,
|
||||
persistence_bot.BotAdmin.id == admin_id,
|
||||
),
|
||||
persistence_bot.BotAdmin,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2,8 +2,13 @@ from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....api.http.authz import WorkspaceRequiredError
|
||||
from ....api.http.context import ExecutionContext, RequestContext
|
||||
from ....core import app
|
||||
from ....entity.persistence import rag as persistence_rag
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
class KnowledgeService:
|
||||
@@ -14,34 +19,69 @@ class KnowledgeService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_knowledge_bases(self) -> list[dict]:
|
||||
@staticmethod
|
||||
def _execution_context(context: RequestContext | ExecutionContext) -> ExecutionContext:
|
||||
if isinstance(context, RequestContext):
|
||||
return ExecutionContext.from_request(context)
|
||||
if isinstance(context, ExecutionContext):
|
||||
return context
|
||||
raise WorkspaceRequiredError('RequestContext or ExecutionContext is required')
|
||||
|
||||
async def get_knowledge_bases(self, context: TenantContext, *, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有知识库"""
|
||||
return await self.ap.rag_mgr.get_all_knowledge_base_details()
|
||||
require_workspace_uuid(context)
|
||||
knowledge_bases = await self.ap.rag_mgr.get_all_knowledge_base_details(context)
|
||||
return knowledge_bases if include_secret else [redact_secrets(base) for base in knowledge_bases]
|
||||
|
||||
async def get_knowledge_base(self, kb_uuid: str) -> dict | None:
|
||||
async def get_knowledge_base(
|
||||
self,
|
||||
context: TenantContext,
|
||||
kb_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""获取知识库"""
|
||||
return await self.ap.rag_mgr.get_knowledge_base_details(kb_uuid)
|
||||
require_workspace_uuid(context)
|
||||
knowledge_base = await self.ap.rag_mgr.get_knowledge_base_details(context, kb_uuid)
|
||||
if knowledge_base is None or include_secret:
|
||||
return knowledge_base
|
||||
return redact_secrets(knowledge_base)
|
||||
|
||||
async def create_knowledge_base(self, kb_data: dict) -> str:
|
||||
async def create_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_data: dict,
|
||||
) -> str:
|
||||
"""创建知识库"""
|
||||
require_workspace_uuid(context)
|
||||
# In new architecture, we delegate entirely to RAGManager which uses plugins.
|
||||
# Legacy internal KB creation is removed.
|
||||
limitation = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('system', {}).get('limitation', {})
|
||||
)
|
||||
max_knowledge_bases = limitation.get('max_knowledge_bases', -1)
|
||||
if max_knowledge_bases >= 0:
|
||||
knowledge_bases = await self.ap.rag_mgr.get_all_knowledge_base_details(context)
|
||||
if len(knowledge_bases) >= max_knowledge_bases:
|
||||
raise ValueError(f'Maximum number of knowledge bases ({max_knowledge_bases}) reached')
|
||||
|
||||
knowledge_engine_plugin_id = kb_data.get('knowledge_engine_plugin_id')
|
||||
if not knowledge_engine_plugin_id:
|
||||
raise ValueError('knowledge_engine_plugin_id is required')
|
||||
|
||||
creation_settings = kb_data.get('creation_settings', {})
|
||||
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
|
||||
retrieval_settings = kb_data.get('retrieval_settings', {})
|
||||
|
||||
# Validate required fields based on plugin's creation_schema and retrieval_schema
|
||||
await self._validate_schema_required_fields(
|
||||
context,
|
||||
knowledge_engine_plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
|
||||
kb = await self.ap.rag_mgr.create_knowledge_base(
|
||||
context,
|
||||
name=kb_data.get('name', 'Untitled'),
|
||||
knowledge_engine_plugin_id=knowledge_engine_plugin_id,
|
||||
creation_settings=creation_settings,
|
||||
@@ -52,6 +92,7 @@ class KnowledgeService:
|
||||
|
||||
async def _validate_schema_required_fields(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
plugin_id: str,
|
||||
creation_settings: dict,
|
||||
retrieval_settings: dict,
|
||||
@@ -69,7 +110,11 @@ class KnowledgeService:
|
||||
Raises:
|
||||
ValueError: If any required field is missing or empty.
|
||||
"""
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return
|
||||
|
||||
# Validate creation_schema
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
creation_schema = await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
self._check_required_fields(creation_schema, creation_settings, 'creation_settings')
|
||||
@@ -79,6 +124,7 @@ class KnowledgeService:
|
||||
self.ap.logger.warning(f'Failed to get creation_schema for validation: {e}')
|
||||
|
||||
# Validate retrieval_schema
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
retrieval_schema = await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
self._check_required_fields(retrieval_schema, retrieval_settings, 'retrieval_settings')
|
||||
@@ -151,8 +197,16 @@ class KnowledgeService:
|
||||
)
|
||||
raise ValueError(f'{field_label} is required ({context}.{field_name})')
|
||||
|
||||
async def update_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
|
||||
async def update_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
kb_data: dict,
|
||||
) -> None:
|
||||
"""更新知识库"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
# Filter to only mutable fields
|
||||
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
|
||||
|
||||
@@ -162,17 +216,18 @@ class KnowledgeService:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(filtered_data)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(kb_uuid)
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(context, kb_uuid)
|
||||
|
||||
kb = await self.get_knowledge_base(kb_uuid)
|
||||
kb = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
|
||||
if kb is None:
|
||||
raise Exception('Knowledge base not found after update')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self.ap.rag_mgr.load_knowledge_base(kb)
|
||||
await self.ap.rag_mgr.load_knowledge_base(context, kb)
|
||||
|
||||
async def _check_doc_capability(self, kb_uuid: str, operation: str) -> None:
|
||||
async def _check_doc_capability(self, context: TenantContext, kb_uuid: str, operation: str) -> None:
|
||||
"""Check if the KB's Knowledge Engine supports document operations.
|
||||
|
||||
Args:
|
||||
@@ -182,104 +237,145 @@ class KnowledgeService:
|
||||
Raises:
|
||||
Exception: If the KB does not support doc_ingestion.
|
||||
"""
|
||||
kb_info = await self.ap.rag_mgr.get_knowledge_base_details(kb_uuid)
|
||||
kb_info = await self.ap.rag_mgr.get_knowledge_base_details(context, kb_uuid)
|
||||
if not kb_info:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
capabilities = kb_info.get('knowledge_engine', {}).get('capabilities', [])
|
||||
if 'doc_ingestion' not in capabilities:
|
||||
raise Exception(f'This knowledge base does not support {operation}')
|
||||
|
||||
async def store_file(self, kb_uuid: str, file_id: str, parser_plugin_id: str | None = None) -> str:
|
||||
async def store_file(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
file_id: str,
|
||||
parser_plugin_id: str | None = None,
|
||||
) -> str:
|
||||
"""存储文件"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self._check_doc_capability(kb_uuid, 'document upload')
|
||||
await self._check_doc_capability(context, kb_uuid, 'document upload')
|
||||
|
||||
result = await runtime_kb.store_file(file_id, parser_plugin_id=parser_plugin_id)
|
||||
result = await runtime_kb.store_file(execution_context, file_id, parser_plugin_id=parser_plugin_id)
|
||||
|
||||
# Update the KB's updated_at timestamp
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(updated_at=sqlalchemy.func.now())
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def retrieve_knowledge_base(
|
||||
self, kb_uuid: str, query: str, retrieval_settings: dict | None = None
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
query: str,
|
||||
retrieval_settings: dict | None = None,
|
||||
) -> list[dict]:
|
||||
"""检索知识库"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# Pass retrieval_settings
|
||||
results = await runtime_kb.retrieve(query, settings=retrieval_settings)
|
||||
results = await runtime_kb.retrieve(execution_context, query, settings=retrieval_settings)
|
||||
|
||||
return [result.model_dump() for result in results]
|
||||
|
||||
async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]:
|
||||
async def get_files_by_knowledge_base(self, context: TenantContext, kb_uuid: str) -> list[dict]:
|
||||
"""获取知识库文件"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
files = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_rag.File, file) for file in files]
|
||||
|
||||
async def delete_file(self, kb_uuid: str, file_id: str) -> None:
|
||||
async def delete_file(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
file_id: str,
|
||||
) -> None:
|
||||
"""删除文件"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self._check_doc_capability(kb_uuid, 'document deletion')
|
||||
await self._check_doc_capability(context, kb_uuid, 'document deletion')
|
||||
|
||||
await runtime_kb.delete_file(file_id)
|
||||
await runtime_kb.delete_file(execution_context, file_id)
|
||||
|
||||
# Update the KB's updated_at timestamp
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(updated_at=sqlalchemy.func.now())
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
async def delete_knowledge_base(self, kb_uuid: str) -> None:
|
||||
async def delete_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
) -> None:
|
||||
"""删除知识库"""
|
||||
# Delete from DB first to commit the deletion, then clean up runtime/plugin (best-effort)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.KnowledgeBase).where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# delete files
|
||||
# NOTE: Chunk cleanup is for legacy (pre-plugin) KBs that stored chunks locally.
|
||||
# For plugin-based Knowledge Engines, the Chunk table is not populated, so this is a no-op.
|
||||
files = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
for file in files:
|
||||
# delete chunks
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.Chunk).where(persistence_rag.Chunk.file_id == file.uuid)
|
||||
sqlalchemy.delete(persistence_rag.Chunk)
|
||||
.where(persistence_rag.Chunk.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.Chunk.file_id == file.uuid)
|
||||
)
|
||||
# delete file
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.File).where(persistence_rag.File.uuid == file.uuid)
|
||||
sqlalchemy.delete(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file.uuid)
|
||||
)
|
||||
|
||||
# Remove from runtime and notify plugin (best-effort, DB is already cleaned up)
|
||||
await self.ap.rag_mgr.delete_knowledge_base(kb_uuid)
|
||||
# Remove from runtime and notify plugin before deleting the owning row.
|
||||
await self.ap.rag_mgr.delete_knowledge_base(context, kb_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.KnowledgeBase)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
# ================= Knowledge Engine Discovery =================
|
||||
|
||||
async def list_knowledge_engines(self) -> list[dict]:
|
||||
async def list_knowledge_engines(self, context: TenantContext) -> list[dict]:
|
||||
"""List all available Knowledge Engines from plugins."""
|
||||
require_workspace_uuid(context)
|
||||
engines = []
|
||||
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return engines
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
|
||||
# Get KnowledgeEngine plugins
|
||||
try:
|
||||
@@ -290,10 +386,12 @@ class KnowledgeService:
|
||||
|
||||
return engines
|
||||
|
||||
async def list_parsers(self, mime_type: str | None = None) -> list[dict]:
|
||||
async def list_parsers(self, context: TenantContext, mime_type: str | None = None) -> list[dict]:
|
||||
"""List available parsers, optionally filtered by MIME type."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return []
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
parsers = await self.ap.plugin_connector.list_parsers()
|
||||
if mime_type:
|
||||
@@ -303,16 +401,24 @@ class KnowledgeService:
|
||||
self.ap.logger.warning(f'Failed to list parsers: {e}')
|
||||
return []
|
||||
|
||||
async def get_engine_creation_schema(self, plugin_id: str) -> dict:
|
||||
async def get_engine_creation_schema(self, context: TenantContext, plugin_id: str) -> dict:
|
||||
"""Get creation settings schema for a specific Knowledge Engine."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return {}
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
return await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to get creation schema for {plugin_id}: {e}')
|
||||
return {}
|
||||
|
||||
async def get_engine_retrieval_schema(self, plugin_id: str) -> dict:
|
||||
async def get_engine_retrieval_schema(self, context: TenantContext, plugin_id: str) -> dict:
|
||||
"""Get retrieval settings schema for a specific Knowledge Engine."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return {}
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
return await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -11,11 +13,36 @@ import sqlalchemy
|
||||
from ....core import app
|
||||
from ....entity.persistence import bstorage as persistence_bstorage
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
LOG_FILE_PATTERN = re.compile(r'^langbot-(\d{4}-\d{2}-\d{2})\.log(?:\.\d+)?$')
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS = 7
|
||||
DEFAULT_LOG_RETENTION_DAYS = 3
|
||||
DEFAULT_MAX_FILES_PER_RUN = 1000
|
||||
HARD_MAX_FILES_PER_RUN = 10000
|
||||
UPLOAD_OWNER_TYPES = ('upload_image', 'upload_document', 'upload')
|
||||
|
||||
|
||||
def _workspace_scope(method):
|
||||
"""Bind maintenance work to a Workspace without spanning external I/O."""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapped(self, context, *args, **kwargs):
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud maintenance requires an explicit tenant scope')
|
||||
async with tenant_scope(workspace_uuid):
|
||||
return await method(self, context, *args, **kwargs)
|
||||
return await method(self, context, *args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class MaintenanceService:
|
||||
@@ -26,7 +53,22 @@ class MaintenanceService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def cleanup_expired_files(self) -> dict[str, int]:
|
||||
def _max_files_per_run(self) -> int:
|
||||
cleanup_cfg = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('storage', {}).get('cleanup', {})
|
||||
)
|
||||
value = self._positive_int(
|
||||
cleanup_cfg.get('max_files_per_run', DEFAULT_MAX_FILES_PER_RUN),
|
||||
DEFAULT_MAX_FILES_PER_RUN,
|
||||
'storage.cleanup.max_files_per_run',
|
||||
)
|
||||
return min(value, HARD_MAX_FILES_PER_RUN)
|
||||
|
||||
@_workspace_scope
|
||||
async def cleanup_expired_files(self, context: ExecutionContext) -> dict[str, int]:
|
||||
if not isinstance(context, ExecutionContext):
|
||||
raise WorkspaceRequiredError('Storage cleanup requires an ExecutionContext')
|
||||
require_workspace_uuid(context)
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
@@ -40,11 +82,17 @@ class MaintenanceService:
|
||||
)
|
||||
|
||||
return {
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(upload_retention_days),
|
||||
'log_files': self._cleanup_expired_log_files(log_retention_days),
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
|
||||
'log_files': await asyncio.to_thread(
|
||||
self._cleanup_expired_log_files,
|
||||
log_retention_days,
|
||||
)
|
||||
if await self._is_oss_singleton(context)
|
||||
else 0,
|
||||
}
|
||||
|
||||
async def get_storage_analysis(self) -> dict[str, Any]:
|
||||
async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:
|
||||
require_workspace_uuid(context)
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
@@ -62,32 +110,34 @@ class MaintenanceService:
|
||||
database_path = (
|
||||
Path(database_cfg.get('sqlite', {}).get('path', 'data/langbot.db')) if database_type == 'sqlite' else None
|
||||
)
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
is_oss_singleton = await self._is_oss_singleton(context)
|
||||
if is_oss_singleton:
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
else:
|
||||
scoped_storage_path = Path('data/storage') / self.ap.storage_mgr.scoped_prefix(context)
|
||||
roots = [('storage', scoped_storage_path)]
|
||||
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
sections = await asyncio.to_thread(self._collect_sections, roots)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts(context)
|
||||
binary_storage = await self._binary_storage_stats(context)
|
||||
upload_candidates = await self._expired_uploaded_candidates(context, upload_retention_days)
|
||||
log_candidates = (
|
||||
await asyncio.to_thread(
|
||||
self._expired_log_candidates,
|
||||
log_retention_days,
|
||||
)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts()
|
||||
binary_storage = await self._binary_storage_stats()
|
||||
upload_candidates = await self._expired_uploaded_candidates(upload_retention_days)
|
||||
log_candidates = self._expired_log_candidates(log_retention_days)
|
||||
if is_oss_singleton
|
||||
else []
|
||||
)
|
||||
|
||||
return {
|
||||
'generated_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
@@ -105,70 +155,156 @@ class MaintenanceService:
|
||||
'uploaded_files': upload_candidates,
|
||||
'log_files': log_candidates,
|
||||
},
|
||||
'tasks': self.ap.task_mgr.get_stats() if self.ap.task_mgr else {},
|
||||
'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
|
||||
}
|
||||
|
||||
async def _cleanup_expired_uploaded_files(self, retention_days: int) -> int:
|
||||
def _collect_sections(
|
||||
self,
|
||||
roots: list[tuple[str, Path | None]],
|
||||
) -> list[dict[str, Any]]:
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
)
|
||||
return sections
|
||||
|
||||
async def _is_oss_singleton(self, context: TenantContext) -> bool:
|
||||
try:
|
||||
await self.ap.workspace_service.get_local_execution_binding(
|
||||
require_workspace_uuid(context),
|
||||
expected_generation=getattr(context, 'placement_generation', None),
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _cleanup_expired_uploaded_files(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
provider_name = provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
candidates = self._expired_local_upload_candidates(retention_days, include_paths=True)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
candidates = await asyncio.to_thread(
|
||||
self._expired_local_upload_candidates,
|
||||
context,
|
||||
retention_days,
|
||||
True,
|
||||
)
|
||||
return await asyncio.to_thread(
|
||||
self._delete_local_candidates,
|
||||
candidates,
|
||||
)
|
||||
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._cleanup_expired_s3_uploaded_files(retention_days)
|
||||
return await self._cleanup_expired_s3_uploaded_files(context, retention_days)
|
||||
|
||||
return 0
|
||||
|
||||
async def _expired_uploaded_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
async def _expired_uploaded_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider_name = self.ap.storage_mgr.storage_provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
return self._expired_local_upload_candidates(retention_days)
|
||||
return await asyncio.to_thread(
|
||||
self._expired_local_upload_candidates,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._expired_s3_upload_candidates(retention_days)
|
||||
return await self._expired_s3_upload_candidates(context, retention_days)
|
||||
return []
|
||||
|
||||
async def _cleanup_expired_s3_uploaded_files(self, retention_days: int) -> int:
|
||||
async def _cleanup_expired_s3_uploaded_files(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
candidates = await self._expired_s3_upload_candidates(retention_days)
|
||||
candidates = await self._expired_s3_upload_candidates(context, retention_days)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
await provider.delete(item['key'])
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
async def _expired_s3_upload_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
async def _expired_s3_upload_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
run_io = getattr(provider, '_run_io', None)
|
||||
if callable(run_io):
|
||||
return await run_io(
|
||||
self._expired_s3_upload_candidates_sync,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
return await asyncio.to_thread(
|
||||
self._expired_s3_upload_candidates_sync,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
|
||||
def _expired_s3_upload_candidates_sync(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
candidates = []
|
||||
max_candidates = self._max_files_per_run()
|
||||
paginator = provider.s3_client.get_paginator('list_objects_v2')
|
||||
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
seen_prefixes: set[str] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
prefix = self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
if prefix in seen_prefixes:
|
||||
continue
|
||||
seen_prefixes.add(prefix)
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name, Prefix=prefix):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(context, key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
|
||||
return candidates
|
||||
|
||||
def _delete_local_candidates(self, candidates: list[dict[str, Any]]) -> int:
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
|
||||
def _cleanup_expired_log_files(self, retention_days: int) -> int:
|
||||
deleted = 0
|
||||
for item in self._expired_log_candidates(retention_days, include_paths=True):
|
||||
@@ -182,28 +318,42 @@ class MaintenanceService:
|
||||
return deleted
|
||||
|
||||
def _expired_local_upload_candidates(
|
||||
self, retention_days: int, include_paths: bool = False
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
include_paths: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
storage_root = Path('data/storage')
|
||||
if not storage_root.exists():
|
||||
return []
|
||||
|
||||
cutoff = datetime.datetime.now().timestamp() - retention_days * 86400
|
||||
candidates = []
|
||||
for entry in storage_root.iterdir():
|
||||
if not entry.is_file() or not self._is_uploaded_file_key(entry.name):
|
||||
max_candidates = self._max_files_per_run()
|
||||
seen_roots: set[Path] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
scoped_root = storage_root / self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
if scoped_root in seen_roots:
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
seen_roots.add(scoped_root)
|
||||
if not scoped_root.exists():
|
||||
continue
|
||||
item = {
|
||||
'key': entry.name,
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(stat.st_mtime, datetime.timezone.utc).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
for entry in scoped_root.rglob('*'):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
continue
|
||||
item = {
|
||||
'key': entry.relative_to(storage_root).as_posix(),
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(
|
||||
stat.st_mtime,
|
||||
datetime.timezone.utc,
|
||||
).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
return candidates
|
||||
|
||||
def _expired_log_candidates(self, retention_days: int, include_paths: bool = False) -> list[dict[str, Any]]:
|
||||
@@ -236,33 +386,51 @@ class MaintenanceService:
|
||||
candidates.append(item)
|
||||
return candidates
|
||||
|
||||
def _is_uploaded_file_key(self, key: str) -> bool:
|
||||
return '/' not in key and not key.startswith('plugin_config_')
|
||||
def _is_uploaded_file_key(self, context: TenantContext, key: str) -> bool:
|
||||
return any(
|
||||
key.startswith(self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type))
|
||||
and self.ap.storage_mgr.is_scoped_object_key(key, expected_owner_type=owner_type)
|
||||
for owner_type in UPLOAD_OWNER_TYPES
|
||||
)
|
||||
|
||||
async def _monitoring_counts(self) -> dict[str, int]:
|
||||
async def _monitoring_counts(self, context: TenantContext) -> dict[str, int]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
tables = {
|
||||
'messages': persistence_monitoring.MonitoringMessage.id,
|
||||
'llm_calls': persistence_monitoring.MonitoringLLMCall.id,
|
||||
'tool_calls': persistence_monitoring.MonitoringToolCall.id,
|
||||
'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
'errors': persistence_monitoring.MonitoringError.id,
|
||||
'sessions': persistence_monitoring.MonitoringSession.session_id,
|
||||
'feedback': persistence_monitoring.MonitoringFeedback.id,
|
||||
'messages': (persistence_monitoring.MonitoringMessage, persistence_monitoring.MonitoringMessage.id),
|
||||
'llm_calls': (persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.id),
|
||||
'tool_calls': (persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.id),
|
||||
'embedding_calls': (
|
||||
persistence_monitoring.MonitoringEmbeddingCall,
|
||||
persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
),
|
||||
'errors': (persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.id),
|
||||
'sessions': (
|
||||
persistence_monitoring.MonitoringSession,
|
||||
persistence_monitoring.MonitoringSession.session_id,
|
||||
),
|
||||
'feedback': (persistence_monitoring.MonitoringFeedback, persistence_monitoring.MonitoringFeedback.id),
|
||||
}
|
||||
counts: dict[str, int] = {}
|
||||
for key, column in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.func.count(column)))
|
||||
for key, (model, column) in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(column)).where(model.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
counts[key] = result.scalar() or 0
|
||||
return counts
|
||||
|
||||
async def _binary_storage_stats(self) -> dict[str, Any]:
|
||||
async def _binary_storage_stats(self, context: TenantContext) -> dict[str, Any]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key))
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key)).where(
|
||||
persistence_bstorage.BinaryStorage.workspace_uuid == workspace_uuid
|
||||
)
|
||||
)
|
||||
size_bytes = None
|
||||
try:
|
||||
size_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value)))
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value))
|
||||
).where(persistence_bstorage.BinaryStorage.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
size_bytes = size_result.scalar() or 0
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,198 +1,451 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
import copy
|
||||
import re
|
||||
import uuid
|
||||
import asyncio
|
||||
|
||||
from ....core import app
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app, taskmgr
|
||||
from ....core.task_boundary import create_detached_task
|
||||
from ....entity.persistence import mcp as persistence_mcp
|
||||
from ....core import taskmgr
|
||||
from ....provider.tools.loaders.mcp import RuntimeMCPSession, MCPSessionStatus
|
||||
from ....entity.persistence import plugin as persistence_plugin
|
||||
from ....provider.tools.loaders.mcp import MCPSessionStatus, RuntimeMCPSession
|
||||
from ....provider.tools.loaders.mcp_policy import require_stdio_mcp_enabled
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..context import ExecutionContext
|
||||
from .secrets import is_url_key, redact_url_secrets, restore_url_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
_SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
_SENSITIVE_CONFIG_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'apikey',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_CONFIG_TOKENS = frozenset(
|
||||
{
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def _is_sensitive_config_key(key: object) -> bool:
|
||||
normalized = _normalize_config_key(key)
|
||||
if normalized in _SENSITIVE_CONFIG_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_CONFIG_TOKENS:
|
||||
return True
|
||||
return 'key' in tokens and bool(tokens & _SENSITIVE_KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def _mask_secret_structure(value):
|
||||
if isinstance(value, dict):
|
||||
return {key: _mask_secret_structure(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_mask_secret_structure(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_mask_secret_structure(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return _SECRET_MASK
|
||||
|
||||
|
||||
def redact_mcp_secrets(value):
|
||||
"""Return a recursively redacted copy of MCP configuration data."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
_mask_secret_structure(item)
|
||||
if _is_sensitive_config_key(key)
|
||||
else redact_url_secrets(item)
|
||||
if is_url_key(key)
|
||||
else redact_mcp_secrets(item)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_mcp_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_mcp_secrets(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def restore_mcp_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from the current MCP config before a write."""
|
||||
|
||||
if sensitive and value == _SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked MCP secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: (
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
)
|
||||
if not sensitive and not _is_sensitive_config_key(key) and is_url_key(key)
|
||||
else restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or _is_sensitive_config_key(key),
|
||||
)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class MCPService:
|
||||
"""Workspace-scoped MCP configuration and runtime facade."""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_runtime_info(self, server_name: str) -> dict | None:
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
if session:
|
||||
return session.get_runtime_info_dict()
|
||||
return None
|
||||
async def _execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip()
|
||||
generation = getattr(context, 'placement_generation', None)
|
||||
if not instance_uuid or not isinstance(generation, int) or isinstance(generation, bool) or generation <= 0:
|
||||
raise ValueError('MCP operations require an explicit fenced execution context')
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=generation,
|
||||
)
|
||||
if binding.instance_uuid != instance_uuid:
|
||||
raise ValueError('MCP execution context belongs to another LangBot instance')
|
||||
return ExecutionContext(
|
||||
instance_uuid=instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
placement_generation=generation,
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
)
|
||||
|
||||
async def get_mcp_servers(self, contain_runtime_info: bool = False) -> list[dict]:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
|
||||
async def get_runtime_info(self, context: TenantContext, server_name: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
return session.get_runtime_info_dict() if session else None
|
||||
|
||||
servers = result.all()
|
||||
async def get_mcp_servers(self, context: TenantContext, contain_runtime_info: bool = False) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_mcp.MCPServer), persistence_mcp.MCPServer, context)
|
||||
)
|
||||
serialized_servers = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server) for server in servers
|
||||
redact_mcp_secrets(self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server))
|
||||
for server in result.all()
|
||||
]
|
||||
if contain_runtime_info:
|
||||
for server in serialized_servers:
|
||||
runtime_info = await self.get_runtime_info(server['name'])
|
||||
|
||||
server['runtime_info'] = runtime_info if runtime_info else None
|
||||
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server['name'])
|
||||
server['runtime_info'] = session.get_runtime_info_dict() if session else None
|
||||
return serialized_servers
|
||||
|
||||
async def create_mcp_server(self, server_data: dict) -> str:
|
||||
# Check limitation (extensions = MCP servers + plugins)
|
||||
async def create_mcp_server(self, context: TenantContext, server_data: dict) -> str:
|
||||
execution_context = await self._execution_context(context)
|
||||
workspace_uuid = execution_context.workspace_uuid
|
||||
|
||||
# This gate is independent of Box availability. Cloud v2 disables
|
||||
# stdio MCP even though Box Runtime itself remains available.
|
||||
require_stdio_mcp_enabled(self.ap, server_data)
|
||||
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_extensions = limitation.get('max_extensions', -1)
|
||||
if max_extensions >= 0:
|
||||
existing_mcp_servers = await self.get_mcp_servers()
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
total_extensions = len(existing_mcp_servers) + len(plugins)
|
||||
if total_extensions >= max_extensions:
|
||||
mcp_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_mcp.MCPServer.uuid)).where(
|
||||
persistence_mcp.MCPServer.workspace_uuid == workspace_uuid
|
||||
)
|
||||
)
|
||||
plugin_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
if (mcp_count_result.scalar() or 0) + (plugin_count_result.scalar() or 0) >= max_extensions:
|
||||
raise ValueError(f'Maximum number of extensions ({max_extensions}) reached')
|
||||
|
||||
server_name = str(server_data.get('name') or '').strip()
|
||||
payload = dict(server_data)
|
||||
payload.pop('workspace_uuid', None)
|
||||
server_name = str(payload.get('name') or '').strip()
|
||||
if not server_name:
|
||||
raise ValueError('MCP server name is required')
|
||||
server_data['name'] = server_name
|
||||
payload['name'] = server_name
|
||||
payload['workspace_uuid'] = workspace_uuid
|
||||
payload['uuid'] = str(uuid.uuid4())
|
||||
|
||||
existing_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name)
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(
|
||||
persistence_mcp.MCPServer.workspace_uuid == workspace_uuid,
|
||||
persistence_mcp.MCPServer.name == server_name,
|
||||
)
|
||||
)
|
||||
if existing_result.first() is not None:
|
||||
raise ValueError(f'MCP server already exists: {server_name}')
|
||||
|
||||
server_data['uuid'] = str(uuid.uuid4())
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(server_data))
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_data['uuid'])
|
||||
)
|
||||
server_entity = result.first()
|
||||
if server_entity:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server_entity)
|
||||
if self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(payload))
|
||||
created = await self._get_mcp_server_by_uuid_raw(execution_context, payload['uuid'])
|
||||
if created and self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = create_detached_task(
|
||||
self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
)
|
||||
tracker = getattr(
|
||||
self.ap.tool_mgr.mcp_tool_loader,
|
||||
'track_hosted_task',
|
||||
None,
|
||||
)
|
||||
if callable(tracker):
|
||||
tracker(task, execution_context)
|
||||
else:
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
return payload['uuid']
|
||||
|
||||
return server_data['uuid']
|
||||
async def get_mcp_server_by_uuid(self, context: TenantContext, server_uuid: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server_data = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
return redact_mcp_secrets(server_data) if server_data is not None else None
|
||||
|
||||
async def get_mcp_server_by_name(self, server_name: str) -> dict | None:
|
||||
async def _get_mcp_server_by_uuid_raw(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
server_uuid: str,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server) if server else None
|
||||
|
||||
async def get_mcp_server_by_name(self, context: TenantContext, server_name: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server_data = await self._get_mcp_server_by_name_raw(execution_context, server_name)
|
||||
if server_data is None:
|
||||
return None
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
response_data = {
|
||||
**server_data,
|
||||
'runtime_info': session.get_runtime_info_dict() if session else None,
|
||||
}
|
||||
return redact_mcp_secrets(response_data)
|
||||
|
||||
async def _get_mcp_server_by_name_raw(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
server_name: str,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
if server is None:
|
||||
return None
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
|
||||
runtime_info = await self.get_runtime_info(server.name)
|
||||
server_data = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
server_data['runtime_info'] = runtime_info if runtime_info else None
|
||||
return server_data
|
||||
async def update_mcp_server(self, context: TenantContext, server_uuid: str, server_data: dict) -> None:
|
||||
execution_context = await self._execution_context(context)
|
||||
old_server = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if old_server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
|
||||
payload = dict(server_data)
|
||||
payload.pop('uuid', None)
|
||||
payload.pop('workspace_uuid', None)
|
||||
payload = restore_mcp_secret_placeholders(payload, old_server)
|
||||
if 'name' in payload:
|
||||
payload['name'] = str(payload['name'] or '').strip()
|
||||
if not payload['name']:
|
||||
raise ValueError('MCP server name is required')
|
||||
duplicate = await self._get_mcp_server_by_name_raw(execution_context, payload['name'])
|
||||
if duplicate is not None and duplicate['uuid'] != server_uuid:
|
||||
raise ValueError(f'MCP server already exists: {payload["name"]}')
|
||||
|
||||
effective_server = {**old_server, **payload}
|
||||
# Existing disabled rows remain readable/deletable. Switching away
|
||||
# from stdio or explicitly disabling one is also allowed, but an
|
||||
# update may never leave a disabled stdio server enabled.
|
||||
if bool(effective_server.get('enable', True)):
|
||||
require_stdio_mcp_enabled(self.ap, effective_server)
|
||||
|
||||
async def update_mcp_server(self, server_uuid: str, server_data: dict) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
.values(payload),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
old_server = result.first()
|
||||
old_server_name = old_server.name if old_server else None
|
||||
old_enable = old_server.enable if old_server else False
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
.values(server_data)
|
||||
)
|
||||
loader = self.ap.tool_mgr.mcp_tool_loader
|
||||
if loader is None:
|
||||
return
|
||||
old_name = old_server['name']
|
||||
old_enable = bool(old_server['enable'])
|
||||
updated = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if updated is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
new_enable = bool(updated['enable'])
|
||||
if old_enable and loader.has_session(execution_context, old_name):
|
||||
await loader.remove_mcp_server(execution_context, old_name)
|
||||
if new_enable:
|
||||
task = create_detached_task(
|
||||
loader.host_mcp_server(execution_context, updated),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
)
|
||||
tracker = getattr(loader, 'track_hosted_task', None)
|
||||
if callable(tracker):
|
||||
tracker(task, execution_context)
|
||||
else:
|
||||
loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
if self.ap.tool_mgr.mcp_tool_loader:
|
||||
new_enable = server_data.get('enable', False)
|
||||
|
||||
need_remove = old_server_name and old_server_name in self.ap.tool_mgr.mcp_tool_loader.sessions
|
||||
|
||||
if old_enable and not new_enable:
|
||||
if need_remove:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(old_server_name)
|
||||
|
||||
elif not old_enable and new_enable:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
updated_server = result.first()
|
||||
if updated_server:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, updated_server)
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
elif old_enable and new_enable:
|
||||
if need_remove:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(old_server_name)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
updated_server = result.first()
|
||||
if updated_server:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, updated_server)
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
async def delete_mcp_server(self, server_uuid: str) -> None:
|
||||
async def delete_mcp_server(self, context: TenantContext, server_uuid: str) -> None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
server_name = server.name if server else None
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
loader = self.ap.tool_mgr.mcp_tool_loader
|
||||
if loader and loader.has_session(execution_context, server['name']):
|
||||
await loader.remove_mcp_server(execution_context, server['name'])
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
async def _require_server(self, context: TenantContext, server_name: str) -> tuple[ExecutionContext, dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
server = await self._get_mcp_server_by_name_raw(execution_context, server_name)
|
||||
if server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
return execution_context, server
|
||||
|
||||
if server_name and self.ap.tool_mgr.mcp_tool_loader:
|
||||
if server_name in self.ap.tool_mgr.mcp_tool_loader.sessions:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(server_name)
|
||||
async def get_mcp_server_resources(self, context: TenantContext, server_name: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resources(execution_context, server_name)
|
||||
|
||||
async def get_mcp_server_resources(self, server_name: str) -> list[dict]:
|
||||
"""Get resources from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resources(server_name)
|
||||
|
||||
async def get_mcp_server_resource_templates(self, server_name: str) -> list[dict]:
|
||||
"""Get resource templates from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(server_name)
|
||||
async def get_mcp_server_resource_templates(self, context: TenantContext, server_name: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(execution_context, server_name)
|
||||
|
||||
async def read_mcp_server_resource_envelope(
|
||||
self,
|
||||
context: TenantContext,
|
||||
server_name: str,
|
||||
uri: str,
|
||||
*,
|
||||
max_bytes: int | None = None,
|
||||
include_blob: bool = False,
|
||||
) -> dict:
|
||||
"""Read a resource from a specific MCP server with metadata."""
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
kwargs = {'include_blob': include_blob, 'source': 'ui_preview'}
|
||||
if max_bytes is not None:
|
||||
kwargs['max_bytes'] = max_bytes
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(server_name, uri, **kwargs)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(
|
||||
execution_context,
|
||||
server_name,
|
||||
uri,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def read_mcp_server_resource(self, server_name: str, uri: str) -> list[dict]:
|
||||
"""Read a resource from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource(server_name, uri)
|
||||
|
||||
async def test_mcp_server(self, server_name: str, server_data: dict) -> int:
|
||||
"""测试 MCP 服务器连接并返回任务 ID"""
|
||||
async def read_mcp_server_resource(self, context: TenantContext, server_name: str, uri: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource(execution_context, server_name, uri)
|
||||
|
||||
async def test_mcp_server(self, context: TenantContext, server_name: str, server_data: dict) -> int:
|
||||
execution_context = await self._execution_context(context)
|
||||
runtime_mcp_session: RuntimeMCPSession | None = None
|
||||
|
||||
test_session: RuntimeMCPSession | None = None
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
|
||||
if server_name != '_':
|
||||
runtime_mcp_session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
_, persisted_server = await self._require_server(execution_context, server_name)
|
||||
require_stdio_mcp_enabled(self.ap, persisted_server)
|
||||
runtime_mcp_session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
if runtime_mcp_session is None:
|
||||
raise ValueError(f'Server not found: {server_name}')
|
||||
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
persisted_session = runtime_mcp_session
|
||||
|
||||
async def _refresh_and_report() -> None:
|
||||
# Testing a persisted server should REUSE its live shared-session
|
||||
# process, not rebuild it. Try a lightweight refresh (a real
|
||||
# list_tools probe over the existing connection) first; only fall
|
||||
# back to a full start() when the session has no live connection
|
||||
# to probe (never connected, or the process is actually gone).
|
||||
needs_start = persisted_session.status == MCPSessionStatus.ERROR or persisted_session.session is None
|
||||
if needs_start:
|
||||
await persisted_session.start()
|
||||
@@ -200,30 +453,24 @@ class MCPService:
|
||||
try:
|
||||
await persisted_session.refresh()
|
||||
except Exception:
|
||||
# The live connection was stale/dropped: reconnect once
|
||||
# (reusing the live managed process where possible) and
|
||||
# re-probe, instead of reporting a false failure.
|
||||
await persisted_session.start()
|
||||
# Surface the discovered tools so the config page can render them
|
||||
# even for an already-hosted server.
|
||||
ctx.metadata['runtime_info'] = persisted_session.get_runtime_info_dict()
|
||||
|
||||
coroutine = _refresh_and_report()
|
||||
else:
|
||||
runtime_mcp_session = await self.ap.tool_mgr.mcp_tool_loader.load_mcp_server(server_config=server_data)
|
||||
|
||||
# A transient test owns an isolated Box session. Always tear it down
|
||||
# after the test completes (success or failure) so it does not leak.
|
||||
payload = dict(server_data)
|
||||
payload.pop('workspace_uuid', None)
|
||||
payload['workspace_uuid'] = execution_context.workspace_uuid
|
||||
require_stdio_mcp_enabled(self.ap, payload)
|
||||
runtime_mcp_session = await self.ap.tool_mgr.mcp_tool_loader.load_mcp_server(
|
||||
execution_context,
|
||||
payload,
|
||||
)
|
||||
test_session = runtime_mcp_session
|
||||
|
||||
async def _run_and_cleanup() -> None:
|
||||
try:
|
||||
await test_session.start()
|
||||
# Capture the runtime info (status + discovered tools) BEFORE
|
||||
# shutting the transient session down. The create/edit config
|
||||
# page has no persisted server to reload from, so without this
|
||||
# a successful test could only show "no tools found". The
|
||||
# frontend reads ctx.metadata.runtime_info to render the tools.
|
||||
ctx.metadata['runtime_info'] = test_session.get_runtime_info_dict()
|
||||
finally:
|
||||
try:
|
||||
@@ -236,27 +483,41 @@ class MCPService:
|
||||
|
||||
coroutine = _run_and_cleanup()
|
||||
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
coroutine,
|
||||
kind='mcp-operation',
|
||||
name=f'mcp-test-{server_name}',
|
||||
label=f'Testing MCP server {server_name}',
|
||||
context=ctx,
|
||||
)
|
||||
try:
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
coroutine,
|
||||
kind='mcp-operation',
|
||||
name=f'mcp-test-{execution_context.workspace_uuid}-{server_name}',
|
||||
label=f'Testing MCP server {server_name}',
|
||||
context=ctx,
|
||||
instance_uuid=execution_context.instance_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
placement_generation=execution_context.placement_generation,
|
||||
)
|
||||
except taskmgr.TaskCapacityError:
|
||||
if test_session is not None:
|
||||
try:
|
||||
await test_session.shutdown()
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(
|
||||
f'Failed to tear down rejected transient MCP test session '
|
||||
f'{test_session.server_name}: {type(exc).__name__}: {exc}'
|
||||
)
|
||||
raise
|
||||
return wrapper.id
|
||||
|
||||
async def get_mcp_server_logs(self, server_name: str, limit: int = 200, level: str | None = None) -> list[dict]:
|
||||
"""Get recent log lines captured from the MCP server's stderr."""
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
async def get_mcp_server_logs(
|
||||
self,
|
||||
context: TenantContext,
|
||||
server_name: str,
|
||||
limit: int = 200,
|
||||
level: str | None = None,
|
||||
) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
if not session:
|
||||
return []
|
||||
|
||||
# Get logs from the session's buffer
|
||||
logs = list(session._log_buffer)
|
||||
|
||||
# Filter by level if specified
|
||||
if level:
|
||||
logs = [log for log in logs if log.get('level') == level]
|
||||
|
||||
# Return the most recent 'limit' logs
|
||||
return logs[-limit:]
|
||||
|
||||
@@ -9,6 +9,9 @@ from ....core import app
|
||||
from ....entity.persistence import model as persistence_model
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....provider.modelmgr import requester as model_requester
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import mask_secret_value, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
def _parse_provider_api_keys(provider_dict: dict) -> dict:
|
||||
@@ -34,7 +37,29 @@ def _runtime_model_data(model_uuid: str, model_data: dict) -> dict:
|
||||
return {**model_data, 'uuid': model_uuid}
|
||||
|
||||
|
||||
async def _validate_provider_supports(ap: app.Application, provider_uuid: str, model_type: str) -> None:
|
||||
def _redact_model_secrets(model_data: dict) -> dict:
|
||||
"""Return a copy with model args and embedded provider credentials masked."""
|
||||
|
||||
redacted = model_data.copy()
|
||||
if 'extra_args' in redacted:
|
||||
redacted['extra_args'] = redact_secrets(redacted['extra_args'])
|
||||
if isinstance(redacted.get('provider'), dict):
|
||||
provider = redacted['provider'].copy()
|
||||
# ModelProvider never contains another provider. Dropping this key also
|
||||
# makes the serializer robust to a reused/self-referential test double.
|
||||
provider.pop('provider', None)
|
||||
if 'api_keys' in provider:
|
||||
provider['api_keys'] = mask_secret_value(provider['api_keys'])
|
||||
redacted['provider'] = provider
|
||||
return redacted
|
||||
|
||||
|
||||
async def _validate_provider_supports(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
model_type: str,
|
||||
) -> None:
|
||||
"""Validate that the provider's requester declares support for ``model_type``.
|
||||
|
||||
``model_type`` is one of the manifest ``support_type`` values:
|
||||
@@ -47,11 +72,12 @@ async def _validate_provider_supports(ap: app.Application, provider_uuid: str, m
|
||||
if model_mgr is None:
|
||||
return
|
||||
|
||||
provider_dict = getattr(model_mgr, 'provider_dict', None)
|
||||
if not provider_dict:
|
||||
get_provider = getattr(model_mgr, 'get_provider_by_uuid', None)
|
||||
if not callable(get_provider):
|
||||
return
|
||||
runtime_provider = provider_dict.get(provider_uuid)
|
||||
if runtime_provider is None:
|
||||
try:
|
||||
runtime_provider = await get_provider(context, provider_uuid)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
requester_name = getattr(getattr(runtime_provider, 'provider_entity', None), 'requester', None)
|
||||
@@ -74,20 +100,48 @@ async def _validate_provider_supports(ap: app.Application, provider_uuid: str, m
|
||||
raise ValueError(f'Provider requester "{requester_name}" does not support {model_type} models')
|
||||
|
||||
|
||||
async def _require_workspace_provider(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
) -> dict:
|
||||
"""Require the referenced provider to belong to the active Workspace."""
|
||||
|
||||
provider = await ap.provider_service.get_provider(context, provider_uuid)
|
||||
if provider is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
return provider
|
||||
|
||||
|
||||
async def _require_runtime_provider(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
) -> model_requester.RuntimeProvider:
|
||||
try:
|
||||
return await ap.model_mgr.get_provider_by_uuid(context, provider_uuid)
|
||||
except ValueError as exc:
|
||||
raise Exception('provider not found') from exc
|
||||
|
||||
|
||||
class LLMModelsService:
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_llm_models(self, include_secret: bool = True) -> list[dict]:
|
||||
async def get_llm_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all LLM models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.LLMModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_model.LLMModel), persistence_model.LLMModel, context)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
# Get all providers for lookup
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -98,29 +152,50 @@ class LLMModelsService:
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
if not include_secret:
|
||||
provider_dict['api_keys'] = ['***'] * len(provider_dict.get('api_keys', []))
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_llm_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_llm_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get LLM models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.LLMModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.LLMModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_llm_model(
|
||||
self, model_data: dict, preserve_uuid: bool = False, auto_set_to_default_pipeline: bool = True
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_data: dict,
|
||||
preserve_uuid: bool = False,
|
||||
auto_set_to_default_pipeline: bool = True,
|
||||
) -> str:
|
||||
"""Create a new LLM model"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = workspace_uuid
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
# Handle provider creation if needed
|
||||
if 'provider' in model_data:
|
||||
@@ -130,31 +205,35 @@ class LLMModelsService:
|
||||
else:
|
||||
# Create new provider
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'llm')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'llm')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.LLMModel).values(**model_data))
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
context,
|
||||
persistence_model.LLMModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
await self.ap.model_mgr.cache_llm_model(context, runtime_llm_model)
|
||||
|
||||
if auto_set_to_default_pipeline:
|
||||
# set the default pipeline model to this model
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
@@ -167,14 +246,23 @@ class LLMModelsService:
|
||||
'fallbacks': [],
|
||||
}
|
||||
pipeline_data = {'config': pipeline_config}
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline.uuid, pipeline_data)
|
||||
await self.ap.pipeline_service.update_pipeline(context, pipeline.uuid, pipeline_data)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_llm_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_llm_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single LLM model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
@@ -184,21 +272,38 @@ class LLMModelsService:
|
||||
|
||||
# Get provider
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_llm_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_llm_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing LLM model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_llm_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
# Handle provider update if needed
|
||||
if 'provider' in model_data:
|
||||
@@ -207,50 +312,71 @@ class LLMModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'llm')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
await self.ap.model_mgr.remove_llm_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_llm_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
persistence_model.LLMModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
context,
|
||||
persistence_model.LLMModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
await self.ap.model_mgr.cache_llm_model(context, runtime_llm_model)
|
||||
|
||||
async def delete_llm_model(self, model_uuid: str) -> None:
|
||||
async def delete_llm_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete an LLM model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_llm_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_llm_model(context, model_uuid)
|
||||
|
||||
async def test_llm_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def test_llm_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test an LLM model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_llm_model: model_requester.RuntimeLLMModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.llm_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_llm_model = model
|
||||
break
|
||||
if runtime_llm_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_llm_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_llm_model = await self.ap.model_mgr.get_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_llm_model = await self.ap.model_mgr.init_temporary_runtime_llm_model(model_data)
|
||||
runtime_llm_model = await self.ap.model_mgr.init_temporary_runtime_llm_model(context, model_data)
|
||||
|
||||
extra_args = model_data.get('extra_args', {})
|
||||
await runtime_llm_model.provider.invoke_llm(
|
||||
@@ -259,6 +385,7 @@ class LLMModelsService:
|
||||
messages=[provider_message.Message(role='user', content='Hello, world! Please just reply a "Hello".')],
|
||||
funcs=[],
|
||||
extra_args=extra_args,
|
||||
execution_context=runtime_llm_model.execution_context,
|
||||
)
|
||||
|
||||
|
||||
@@ -268,13 +395,19 @@ class EmbeddingModelsService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_embedding_models(self) -> list[dict]:
|
||||
async def get_embedding_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all embedding models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.EmbeddingModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel), persistence_model.EmbeddingModel, context
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -284,25 +417,46 @@ class EmbeddingModelsService:
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_embedding_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_embedding_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get embedding models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_embedding_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
async def create_embedding_model(
|
||||
self, context: TenantContext, model_data: dict, preserve_uuid: bool = False
|
||||
) -> str:
|
||||
"""Create a new embedding model"""
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -310,35 +464,44 @@ class EmbeddingModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'text-embedding')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'text-embedding')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.EmbeddingModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
context,
|
||||
persistence_model.EmbeddingModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
await self.ap.model_mgr.cache_embedding_model(context, runtime_embedding_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_embedding_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_embedding_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single embedding model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
@@ -348,21 +511,38 @@ class EmbeddingModelsService:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_embedding_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_embedding_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing embedding model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_embedding_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -370,57 +550,82 @@ class EmbeddingModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'text-embedding')
|
||||
|
||||
await self.ap.model_mgr.remove_embedding_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
persistence_model.EmbeddingModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
|
||||
async def delete_embedding_model(self, model_uuid: str) -> None:
|
||||
"""Delete an embedding model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_embedding_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
async def test_embedding_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
await self.ap.model_mgr.remove_embedding_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
context,
|
||||
persistence_model.EmbeddingModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
await self.ap.model_mgr.cache_embedding_model(context, runtime_embedding_model)
|
||||
|
||||
async def delete_embedding_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete an embedding model"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_embedding_model(context, model_uuid)
|
||||
|
||||
async def test_embedding_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test an embedding model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_embedding_model: model_requester.RuntimeEmbeddingModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.embedding_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_embedding_model = model
|
||||
break
|
||||
if runtime_embedding_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_embedding_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_embedding_model = await self.ap.model_mgr.get_embedding_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_embedding_model = await self.ap.model_mgr.init_temporary_runtime_embedding_model(model_data)
|
||||
runtime_embedding_model = await self.ap.model_mgr.init_temporary_runtime_embedding_model(
|
||||
context,
|
||||
model_data,
|
||||
)
|
||||
|
||||
await runtime_embedding_model.provider.invoke_embedding(
|
||||
model=runtime_embedding_model,
|
||||
input_text=['Hello, world!'],
|
||||
extra_args={},
|
||||
execution_context=runtime_embedding_model.execution_context,
|
||||
)
|
||||
|
||||
|
||||
@@ -430,13 +635,17 @@ class RerankModelsService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_rerank_models(self) -> list[dict]:
|
||||
async def get_rerank_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all rerank models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.RerankModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_model.RerankModel), persistence_model.RerankModel, context)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -446,25 +655,44 @@ class RerankModelsService:
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_rerank_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_rerank_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get rerank models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_rerank_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
async def create_rerank_model(self, context: TenantContext, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
"""Create a new rerank model"""
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -472,34 +700,45 @@ class RerankModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'rerank')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'rerank')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.RerankModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
context,
|
||||
persistence_model.RerankModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
await self.ap.model_mgr.cache_rerank_model(context, runtime_rerank_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_rerank_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_rerank_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single rerank model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
@@ -508,21 +747,38 @@ class RerankModelsService:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_rerank_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing rerank model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_rerank_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -530,50 +786,76 @@ class RerankModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'rerank')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
persistence_model.RerankModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
context,
|
||||
persistence_model.RerankModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
await self.ap.model_mgr.cache_rerank_model(context, runtime_rerank_model)
|
||||
|
||||
async def delete_rerank_model(self, model_uuid: str) -> None:
|
||||
async def delete_rerank_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete a rerank model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_rerank_model(context, model_uuid)
|
||||
|
||||
async def test_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def test_rerank_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test a rerank model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_rerank_model: model_requester.RuntimeRerankModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.rerank_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_rerank_model = model
|
||||
break
|
||||
if runtime_rerank_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_rerank_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_rerank_model = await self.ap.model_mgr.get_rerank_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(model_data)
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(
|
||||
context,
|
||||
model_data,
|
||||
)
|
||||
|
||||
await runtime_rerank_model.provider.invoke_rerank(
|
||||
model=runtime_rerank_model,
|
||||
@@ -582,4 +864,5 @@ class RerankModelsService:
|
||||
'Artificial intelligence is a branch of computer science.',
|
||||
'The weather is nice today.',
|
||||
],
|
||||
execution_context=runtime_rerank_model.execution_context,
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,9 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
default_stage_order = [
|
||||
@@ -30,7 +33,8 @@ class PipelineService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_pipeline_metadata(self) -> list[dict]:
|
||||
async def get_pipeline_metadata(self, context: TenantContext) -> list[dict]:
|
||||
require_workspace_uuid(context)
|
||||
return [
|
||||
self.ap.pipeline_config_meta_trigger,
|
||||
self.ap.pipeline_config_meta_safety,
|
||||
@@ -38,8 +42,19 @@ class PipelineService:
|
||||
self.ap.pipeline_config_meta_output,
|
||||
]
|
||||
|
||||
async def get_pipelines(self, sort_by: str = 'created_at', sort_order: str = 'DESC') -> list[dict]:
|
||||
query = sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
async def get_pipelines(
|
||||
self,
|
||||
context: TenantContext,
|
||||
sort_by: str = 'created_at',
|
||||
sort_order: str = 'DESC',
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
query = scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
|
||||
if sort_by == 'created_at':
|
||||
if sort_order == 'DESC':
|
||||
@@ -54,15 +69,26 @@ class PipelineService:
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(query)
|
||||
pipelines = result.all()
|
||||
return [
|
||||
serialized = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
for pipeline in pipelines
|
||||
]
|
||||
return serialized if include_secret else [redact_secrets(pipeline) for pipeline in serialized]
|
||||
|
||||
async def get_pipeline(self, pipeline_uuid: str) -> dict | None:
|
||||
async def get_pipeline(
|
||||
self,
|
||||
context: TenantContext,
|
||||
pipeline_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -71,20 +97,24 @@ class PipelineService:
|
||||
if pipeline is None:
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
serialized = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
return serialized if include_secret else redact_secrets(serialized)
|
||||
|
||||
async def create_pipeline(self, pipeline_data: dict, default: bool = False) -> str:
|
||||
async def create_pipeline(self, context: TenantContext, pipeline_data: dict, default: bool = False) -> str:
|
||||
from ....utils import paths as path_utils
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
if max_pipelines >= 0:
|
||||
existing_pipelines = await self.get_pipelines()
|
||||
existing_pipelines = await self.get_pipelines(context)
|
||||
if len(existing_pipelines) >= max_pipelines:
|
||||
raise ValueError(f'Maximum number of pipelines ({max_pipelines}) reached')
|
||||
|
||||
pipeline_data = pipeline_data.copy()
|
||||
pipeline_data['uuid'] = str(uuid.uuid4())
|
||||
pipeline_data['workspace_uuid'] = workspace_uuid
|
||||
pipeline_data['for_version'] = self.ap.ver_mgr.get_current_version()
|
||||
pipeline_data['stages'] = default_stage_order.copy()
|
||||
pipeline_data['is_default'] = default
|
||||
@@ -108,79 +138,122 @@ class PipelineService:
|
||||
sqlalchemy.insert(persistence_pipeline.LegacyPipeline).values(**pipeline_data)
|
||||
)
|
||||
|
||||
pipeline = await self.get_pipeline(pipeline_data['uuid'])
|
||||
pipeline = await self.get_pipeline(context, pipeline_data['uuid'], include_secret=True)
|
||||
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
return pipeline_data['uuid']
|
||||
|
||||
async def update_pipeline(self, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
async def update_pipeline(self, context: TenantContext, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
pipeline_data = pipeline_data.copy()
|
||||
for protected_field in ('uuid', 'for_version', 'stages', 'is_default'):
|
||||
for protected_field in ('uuid', 'workspace_uuid', 'for_version', 'stages', 'is_default'):
|
||||
pipeline_data.pop(protected_field, None)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(**pipeline_data)
|
||||
)
|
||||
if 'config' in pipeline_data:
|
||||
current_config = None
|
||||
if contains_secret_placeholder(pipeline_data['config']):
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
pipeline_data['config'] = restore_secret_placeholders(
|
||||
pipeline_data['config'],
|
||||
current_config if current_config is not None else {},
|
||||
)
|
||||
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(**pipeline_data),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
if 'name' in pipeline_data:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(
|
||||
persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid
|
||||
),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
for bot in bots:
|
||||
bot_data = {'use_pipeline_name': pipeline_data['name']}
|
||||
await self.ap.bot_service.update_bot(bot.uuid, bot_data)
|
||||
await self.ap.bot_service.update_bot(context, bot.uuid, bot_data)
|
||||
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
# update all conversation that use this pipeline
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if session.using_conversation is not None and session.using_conversation.pipeline_uuid == pipeline_uuid:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
and session.using_conversation.pipeline_uuid == pipeline_uuid
|
||||
and getattr(session, 'workspace_uuid', workspace_uuid) == workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
|
||||
async def delete_pipeline(self, pipeline_uuid: str) -> None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
async def delete_pipeline(self, context: TenantContext, pipeline_uuid: str) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
|
||||
async def copy_pipeline(self, pipeline_uuid: str) -> str:
|
||||
async def copy_pipeline(self, context: TenantContext, pipeline_uuid: str) -> str:
|
||||
"""Copy a pipeline with all its configurations"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
if max_pipelines >= 0:
|
||||
existing_pipelines = await self.get_pipelines()
|
||||
existing_pipelines = await self.get_pipelines(context)
|
||||
if len(existing_pipelines) >= max_pipelines:
|
||||
raise ValueError(f'Maximum number of pipelines ({max_pipelines}) reached')
|
||||
|
||||
# Get the original pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
original_pipeline = result.first()
|
||||
if original_pipeline is None:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Create new pipeline data
|
||||
new_uuid = str(uuid.uuid4())
|
||||
new_pipeline_data = {
|
||||
'uuid': new_uuid,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'name': f'{original_pipeline.name} (Copy)',
|
||||
'description': original_pipeline.description,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
@@ -207,13 +280,14 @@ class PipelineService:
|
||||
)
|
||||
|
||||
# Load the new pipeline
|
||||
pipeline = await self.get_pipeline(new_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
pipeline = await self.get_pipeline(context, new_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
return new_uuid
|
||||
|
||||
async def update_pipeline_extensions(
|
||||
self,
|
||||
context: TenantContext,
|
||||
pipeline_uuid: str,
|
||||
bound_plugins: list[dict],
|
||||
bound_mcp_servers: list[str] = None,
|
||||
@@ -225,16 +299,21 @@ class PipelineService:
|
||||
mcp_resource_agent_read_enabled: bool | None = None,
|
||||
) -> None:
|
||||
"""Update the bound plugins and MCP servers for a pipeline"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Get current pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
pipeline = result.first()
|
||||
if pipeline is None:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Update extensions_preferences
|
||||
extensions_preferences = pipeline.extensions_preferences or {}
|
||||
@@ -252,12 +331,16 @@ class PipelineService:
|
||||
extensions_preferences['mcp_resources'] = bound_mcp_resources
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(extensions_preferences=extensions_preferences)
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(extensions_preferences=extensions_preferences),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
# Reload pipeline to apply changes
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
@@ -7,6 +7,9 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import model as persistence_model
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
class ModelProviderService:
|
||||
@@ -35,9 +38,15 @@ class ModelProviderService:
|
||||
|
||||
return normalized_keys
|
||||
|
||||
async def get_providers(self) -> list[dict]:
|
||||
async def get_providers(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all providers"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.ModelProvider))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
providers = result.all()
|
||||
providers_list = []
|
||||
for p in providers:
|
||||
@@ -50,14 +59,25 @@ class ModelProviderService:
|
||||
provider_dict['api_keys'] = json.loads(provider_dict['api_keys'])
|
||||
except Exception:
|
||||
provider_dict['api_keys'] = []
|
||||
if not include_secret:
|
||||
provider_dict = redact_secrets(provider_dict)
|
||||
providers_list.append(provider_dict)
|
||||
return providers_list
|
||||
|
||||
async def get_provider(self, provider_uuid: str) -> dict | None:
|
||||
async def get_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single provider by UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = result.first()
|
||||
@@ -72,103 +92,171 @@ class ModelProviderService:
|
||||
provider_dict['api_keys'] = json.loads(provider_dict['api_keys'])
|
||||
except Exception:
|
||||
provider_dict['api_keys'] = []
|
||||
if not include_secret:
|
||||
provider_dict = redact_secrets(provider_dict)
|
||||
return provider_dict
|
||||
|
||||
async def create_provider(self, provider_data: dict) -> str:
|
||||
async def create_provider(self, context: TenantContext, provider_data: dict) -> str:
|
||||
"""Create a new provider"""
|
||||
provider_data = provider_data.copy()
|
||||
provider_data['uuid'] = str(uuid.uuid4())
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
provider_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
provider_data['api_keys'] = self._normalize_api_keys(
|
||||
restore_secret_placeholders(provider_data.get('api_keys'), sensitive=True)
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
|
||||
)
|
||||
|
||||
# load to runtime
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider_data)
|
||||
self.ap.model_mgr.provider_dict[runtime_provider.provider_entity.uuid] = runtime_provider
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(context, provider_data)
|
||||
await self.ap.model_mgr.cache_provider(context, runtime_provider)
|
||||
return provider_data['uuid']
|
||||
|
||||
async def update_provider(self, provider_uuid: str, provider_data: dict) -> None:
|
||||
async def update_provider(self, context: TenantContext, provider_uuid: str, provider_data: dict) -> None:
|
||||
"""Update an existing provider"""
|
||||
if 'uuid' in provider_data:
|
||||
del provider_data['uuid']
|
||||
provider_data = provider_data.copy()
|
||||
provider_data.pop('uuid', None)
|
||||
provider_data.pop('workspace_uuid', None)
|
||||
if 'api_keys' in provider_data:
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
.values(**provider_data)
|
||||
submitted_keys = provider_data.get('api_keys')
|
||||
if contains_secret_placeholder(submitted_keys, sensitive=True):
|
||||
current_provider = await self.get_provider(context, provider_uuid, include_secret=True)
|
||||
if current_provider is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
submitted_keys = restore_secret_placeholders(
|
||||
submitted_keys,
|
||||
current_provider.get('api_keys', []),
|
||||
sensitive=True,
|
||||
)
|
||||
provider_data['api_keys'] = self._normalize_api_keys(submitted_keys)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
.values(**provider_data),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider(provider_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
await self.ap.model_mgr.reload_provider(context, provider_uuid)
|
||||
|
||||
async def delete_provider(self, provider_uuid: str) -> None:
|
||||
async def delete_provider(self, context: TenantContext, provider_uuid: str) -> None:
|
||||
"""Delete a provider (only if no models reference it)"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check if any models use this provider
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.LLMModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if llm_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: LLM models still reference it')
|
||||
|
||||
embedding_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if embedding_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Embedding models still reference it')
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if rerank_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Rerank models still reference it')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_provider(provider_uuid)
|
||||
await self.ap.model_mgr.remove_provider(context, provider_uuid)
|
||||
|
||||
async def get_provider_model_counts(self, provider_uuid: str) -> dict:
|
||||
async def get_provider_model_counts(self, context: TenantContext, provider_uuid: str) -> dict:
|
||||
"""Get count of models using this provider"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_provider(context, provider_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.provider_uuid == provider_uuid),
|
||||
persistence_model.LLMModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
llm_count = llm_result.scalar() or 0
|
||||
|
||||
embedding_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.provider_uuid == provider_uuid),
|
||||
persistence_model.EmbeddingModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
embedding_count = embedding_result.scalar() or 0
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid),
|
||||
persistence_model.RerankModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
rerank_count = rerank_result.scalar() or 0
|
||||
|
||||
return {'llm_count': llm_count, 'embedding_count': embedding_count, 'rerank_count': rerank_count}
|
||||
|
||||
async def find_or_create_provider(self, requester: str, base_url: str, api_keys: list) -> str:
|
||||
async def find_or_create_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
requester: str,
|
||||
base_url: str,
|
||||
api_keys: list,
|
||||
) -> str:
|
||||
"""Find existing provider or create new one"""
|
||||
api_keys = self._normalize_api_keys(api_keys)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
api_keys = self._normalize_api_keys(restore_secret_placeholders(api_keys, sensitive=True))
|
||||
|
||||
# Try to find existing provider with same config
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.requester == requester,
|
||||
persistence_model.ModelProvider.base_url == base_url,
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.requester == requester,
|
||||
persistence_model.ModelProvider.base_url == base_url,
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
for provider in result.all():
|
||||
@@ -187,29 +275,38 @@ class ModelProviderService:
|
||||
pass
|
||||
|
||||
return await self.create_provider(
|
||||
context,
|
||||
{
|
||||
'name': provider_name,
|
||||
'requester': requester,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
async def update_space_model_provider_api_keys(self, api_key: str) -> None:
|
||||
async def update_space_model_provider_api_keys(self, context: TenantContext, api_key: str) -> None:
|
||||
"""Update Space model provider API keys"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key)),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider('00000000-0000-0000-0000-000000000000')
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
await self.ap.model_mgr.reload_provider(context, '00000000-0000-0000-0000-000000000000')
|
||||
|
||||
async def scan_provider_models(self, provider_uuid: str, model_type: str | None = None) -> dict:
|
||||
provider = await self.get_provider(provider_uuid)
|
||||
async def scan_provider_models(
|
||||
self, context: TenantContext, provider_uuid: str, model_type: str | None = None
|
||||
) -> dict:
|
||||
provider = await self.get_provider(context, provider_uuid, include_secret=True)
|
||||
if provider is None:
|
||||
raise ValueError('provider not found')
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider)
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(context, provider)
|
||||
|
||||
try:
|
||||
scan_result = await runtime_provider.requester.scan_models(
|
||||
@@ -230,11 +327,15 @@ class ModelProviderService:
|
||||
scanned_models = scan_result
|
||||
debug_info = None
|
||||
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(provider_uuid)
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(context, provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
context, provider_uuid
|
||||
)
|
||||
rerank_service = getattr(self.ap, 'rerank_models_service', None)
|
||||
rerank_models = (
|
||||
await rerank_service.get_rerank_models_by_provider(provider_uuid) if rerank_service is not None else []
|
||||
await rerank_service.get_rerank_models_by_provider(context, provider_uuid)
|
||||
if rerank_service is not None
|
||||
else []
|
||||
)
|
||||
existing_llm_names = {model['name'] for model in llm_models}
|
||||
existing_embedding_names = {model['name'] for model in embedding_models}
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
|
||||
SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
|
||||
_SENSITIVE_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'api_keys',
|
||||
'apikey',
|
||||
'apikeys',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'header_value',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
'webhook_url',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_TOKENS = frozenset(
|
||||
{
|
||||
'apikey',
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_URL_QUERY_NAMES = frozenset(
|
||||
{
|
||||
'code',
|
||||
'credential',
|
||||
'credentials',
|
||||
'password',
|
||||
'passwd',
|
||||
'sig',
|
||||
'signature',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def is_sensitive_key(key: object) -> bool:
|
||||
"""Return whether a configuration key conventionally carries a secret."""
|
||||
|
||||
normalized = _normalize_key(key)
|
||||
if normalized in _SENSITIVE_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_TOKENS:
|
||||
return True
|
||||
return bool(tokens & {'key', 'keys'}) and bool(tokens & _KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def is_url_key(key: object) -> bool:
|
||||
"""Return whether a configuration field conventionally carries a URL."""
|
||||
|
||||
normalized = _normalize_key(key)
|
||||
return normalized == 'url' or normalized.endswith('_url')
|
||||
|
||||
|
||||
def _is_sensitive_url_query_key(key: object) -> bool:
|
||||
normalized = _normalize_key(key)
|
||||
return (
|
||||
is_sensitive_key(key) or normalized in _SENSITIVE_URL_QUERY_NAMES or normalized.endswith(('_sig', '_signature'))
|
||||
)
|
||||
|
||||
|
||||
def _redact_url_string(value: str) -> str:
|
||||
if not value:
|
||||
return value
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
netloc = parsed.netloc
|
||||
if '@' in netloc:
|
||||
_, host = netloc.rsplit('@', 1)
|
||||
netloc = f'{SECRET_MASK}@{host}'
|
||||
query = urlencode(
|
||||
[
|
||||
(key, SECRET_MASK if _is_sensitive_url_query_key(key) and item else item)
|
||||
for key, item in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
],
|
||||
doseq=True,
|
||||
safe='*',
|
||||
)
|
||||
return urlunsplit((parsed.scheme, netloc, parsed.path, query, parsed.fragment))
|
||||
except (TypeError, ValueError):
|
||||
# A malformed URL cannot be safely decomposed, so fail closed.
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def redact_url_secrets(value):
|
||||
"""Redact URL userinfo and credential-like query values."""
|
||||
|
||||
if isinstance(value, str):
|
||||
return _redact_url_string(value)
|
||||
if isinstance(value, list):
|
||||
return [redact_url_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_url_secrets(item) for item in value)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def _contains_url_secret_placeholder(value) -> bool:
|
||||
if isinstance(value, str):
|
||||
if value == SECRET_MASK:
|
||||
return True
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
if '@' in parsed.netloc and SECRET_MASK in parsed.netloc.rsplit('@', 1)[0]:
|
||||
return True
|
||||
return any(
|
||||
item == SECRET_MASK and _is_sensitive_url_query_key(key)
|
||||
for key, item in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(_contains_url_secret_placeholder(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def _restore_url_string(value: str, current_value) -> str:
|
||||
if value == SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked URL secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
|
||||
try:
|
||||
submitted = urlsplit(value)
|
||||
except (TypeError, ValueError):
|
||||
return value
|
||||
|
||||
current = None
|
||||
if isinstance(current_value, str):
|
||||
try:
|
||||
current = urlsplit(current_value)
|
||||
except (TypeError, ValueError):
|
||||
current = None
|
||||
|
||||
netloc = submitted.netloc
|
||||
if '@' in netloc:
|
||||
submitted_userinfo, host = netloc.rsplit('@', 1)
|
||||
if SECRET_MASK in submitted_userinfo:
|
||||
if current is None or '@' not in current.netloc:
|
||||
raise ValueError('Masked URL userinfo has no existing value')
|
||||
current_userinfo, _ = current.netloc.rsplit('@', 1)
|
||||
netloc = f'{current_userinfo}@{host}'
|
||||
|
||||
current_query: dict[str, list[str]] = {}
|
||||
if current is not None:
|
||||
for key, item in parse_qsl(current.query, keep_blank_values=True):
|
||||
current_query.setdefault(_normalize_key(key), []).append(item)
|
||||
consumed: dict[str, int] = {}
|
||||
restored_query: list[tuple[str, str]] = []
|
||||
for key, item in parse_qsl(submitted.query, keep_blank_values=True):
|
||||
normalized = _normalize_key(key)
|
||||
if item == SECRET_MASK and _is_sensitive_url_query_key(key):
|
||||
index = consumed.get(normalized, 0)
|
||||
candidates = current_query.get(normalized, [])
|
||||
if index >= len(candidates):
|
||||
raise ValueError('Masked URL query secret has no existing value')
|
||||
item = candidates[index]
|
||||
consumed[normalized] = index + 1
|
||||
restored_query.append((key, item))
|
||||
|
||||
return urlunsplit(
|
||||
(
|
||||
submitted.scheme,
|
||||
netloc,
|
||||
submitted.path,
|
||||
urlencode(restored_query, doseq=True, safe='*'),
|
||||
submitted.fragment,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def restore_url_secret_placeholders(value, current_value=_MISSING_SECRET):
|
||||
"""Restore URL placeholders from the corresponding persisted URL."""
|
||||
|
||||
if isinstance(value, str):
|
||||
return _restore_url_string(value, current_value)
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def mask_secret_value(value):
|
||||
"""Return a shape-preserving copy whose non-empty leaves are masked."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: mask_secret_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [mask_secret_value(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(mask_secret_value(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def redact_secrets(value):
|
||||
"""Return a recursively redacted copy without mutating the source value."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
mask_secret_value(item)
|
||||
if is_sensitive_key(key)
|
||||
else redact_url_secrets(item)
|
||||
if is_url_key(key)
|
||||
else redact_secrets(item)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_secrets(item) for item in value)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def restore_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from existing data before a management write.
|
||||
|
||||
``***`` is a reserved placeholder only inside a sensitive field. A masked
|
||||
leaf without an existing counterpart is rejected so it can never become a
|
||||
persisted credential. Empty values and explicit replacements pass through.
|
||||
"""
|
||||
|
||||
if sensitive and value == SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: (
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
)
|
||||
if not sensitive and not is_sensitive_key(key) and is_url_key(key)
|
||||
else restore_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or is_sensitive_key(key),
|
||||
)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def contains_secret_placeholder(value, *, sensitive: bool = False) -> bool:
|
||||
"""Return whether ``value`` contains a meaningful masked secret leaf."""
|
||||
|
||||
if sensitive and value == SECRET_MASK:
|
||||
return True
|
||||
if isinstance(value, dict):
|
||||
return any(
|
||||
(
|
||||
_contains_url_secret_placeholder(item)
|
||||
if not sensitive and not is_sensitive_key(key) and is_url_key(key)
|
||||
else contains_secret_placeholder(item, sensitive=sensitive or is_sensitive_key(key))
|
||||
)
|
||||
for key, item in value.items()
|
||||
)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(contains_secret_placeholder(item, sensitive=sensitive) for item in value)
|
||||
return False
|
||||
@@ -1,9 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import inspect
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
import zipfile
|
||||
from typing import Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
@@ -12,6 +14,9 @@ import httpx
|
||||
|
||||
from ....core import app
|
||||
from ....skill.utils import parse_frontmatter
|
||||
from ....utils import httpclient
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
_PUBLIC_SKILL_FIELDS = (
|
||||
@@ -32,6 +37,12 @@ _GITHUB_ASSET_HOSTS = {
|
||||
'raw.githubusercontent.com',
|
||||
'codeload.github.com',
|
||||
}
|
||||
_MAX_GITHUB_ARCHIVE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_GITHUB_ARCHIVE_ENTRIES = 4096
|
||||
_MAX_SKILL_ARCHIVE_FILES = 1024
|
||||
_MAX_SKILL_FILE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_SKILL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
|
||||
_MAX_SKILL_COMPRESSION_RATIO = 200
|
||||
|
||||
|
||||
class SkillService:
|
||||
@@ -75,75 +86,112 @@ class SkillService:
|
||||
"""Backwards-compatible alias preserved for clarity at call sites."""
|
||||
self._require_box(action)
|
||||
|
||||
async def _execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip()
|
||||
generation = getattr(context, 'placement_generation', None)
|
||||
if not instance_uuid or isinstance(generation, bool) or not isinstance(generation, int) or generation <= 0:
|
||||
raise ValueError('Skill operations require an explicit fenced execution context')
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=generation,
|
||||
)
|
||||
if binding.instance_uuid != instance_uuid:
|
||||
raise ValueError('Skill execution context belongs to another LangBot instance')
|
||||
return ExecutionContext(
|
||||
instance_uuid=instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
placement_generation=generation,
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _serialize_skill(skill: dict) -> dict:
|
||||
return {field: skill.get(field) for field in _PUBLIC_SKILL_FIELDS if field in skill}
|
||||
|
||||
async def list_skills(self) -> list[dict]:
|
||||
async def list_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
# When Box is unavailable, surface an empty list rather than raising —
|
||||
# the skills page should render cleanly, and the UI separately renders
|
||||
# a "Box disabled / unavailable" banner via useBoxStatus.
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return []
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills()]
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills(execution_context)]
|
||||
|
||||
async def get_skill(self, skill_name: str) -> Optional[dict]:
|
||||
async def get_skill(self, context: TenantContext, skill_name: str) -> Optional[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return None
|
||||
skill = await box_service.get_skill(skill_name)
|
||||
skill = await box_service.get_skill(execution_context, skill_name)
|
||||
return self._serialize_skill(skill) if skill else None
|
||||
|
||||
async def get_skill_by_name(self, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(name)
|
||||
async def get_skill_by_name(self, context: TenantContext, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(context, name)
|
||||
|
||||
async def create_skill(self, data: dict) -> dict:
|
||||
async def create_skill(self, context: TenantContext, data: dict) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Creating a skill')
|
||||
created = await box_service.create_skill(data)
|
||||
await self._reload_skills()
|
||||
created = await box_service.create_skill(execution_context, data)
|
||||
await self._reload_skills(execution_context)
|
||||
return self._serialize_skill(created)
|
||||
|
||||
async def update_skill(self, skill_name: str, data: dict) -> dict:
|
||||
async def update_skill(self, context: TenantContext, skill_name: str, data: dict) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Editing a skill')
|
||||
updated = await box_service.update_skill(skill_name, data)
|
||||
await self._reload_skills()
|
||||
updated = await box_service.update_skill(execution_context, skill_name, data)
|
||||
await self._reload_skills(execution_context)
|
||||
return self._serialize_skill(updated)
|
||||
|
||||
async def delete_skill(self, skill_name: str) -> bool:
|
||||
async def delete_skill(self, context: TenantContext, skill_name: str) -> bool:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Deleting a skill')
|
||||
await box_service.delete_skill(skill_name)
|
||||
await self._reload_skills()
|
||||
await box_service.delete_skill(execution_context, skill_name)
|
||||
await self._reload_skills(execution_context)
|
||||
return True
|
||||
|
||||
async def list_skill_files(
|
||||
self,
|
||||
context: TenantContext,
|
||||
skill_name: str,
|
||||
path: str = '.',
|
||||
include_hidden: bool = False,
|
||||
max_entries: int = 200,
|
||||
) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Browsing skill files')
|
||||
return await box_service.list_skill_files(skill_name, path, include_hidden, max_entries)
|
||||
return await box_service.list_skill_files(execution_context, skill_name, path, include_hidden, max_entries)
|
||||
|
||||
async def read_skill_file(self, skill_name: str, path: str) -> dict:
|
||||
async def read_skill_file(self, context: TenantContext, skill_name: str, path: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Reading a skill file')
|
||||
return await box_service.read_skill_file(skill_name, path)
|
||||
return await box_service.read_skill_file(execution_context, skill_name, path)
|
||||
|
||||
async def write_skill_file(self, skill_name: str, path: str, content: str) -> dict:
|
||||
async def write_skill_file(self, context: TenantContext, skill_name: str, path: str, content: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Editing skill files')
|
||||
result = await box_service.write_skill_file(skill_name, path, content)
|
||||
await self._reload_skills()
|
||||
result = await box_service.write_skill_file(execution_context, skill_name, path, content)
|
||||
await self._reload_skills(execution_context)
|
||||
return result
|
||||
|
||||
async def install_from_github(self, data: dict) -> list[dict]:
|
||||
async def install_from_github(self, context: TenantContext, data: dict) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._install_github_skill_md(raw_asset_url, owner=owner, repo=repo, data=data)
|
||||
return await self._install_github_skill_md(
|
||||
execution_context,
|
||||
raw_asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
data=data,
|
||||
)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
@@ -151,29 +199,37 @@ class SkillService:
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
filename = f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip'
|
||||
installed = await box_service.install_skill_zip(
|
||||
execution_context,
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
source_subdir=source_subdir,
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(execution_context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_github(self, data: dict) -> list[dict]:
|
||||
async def preview_install_from_github(self, context: TenantContext, data: dict) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._preview_github_skill_md(raw_asset_url, owner=owner, repo=repo)
|
||||
return await self._preview_github_skill_md(
|
||||
execution_context,
|
||||
raw_asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
return await box_service.preview_skill_zip(
|
||||
execution_context,
|
||||
zip_bytes,
|
||||
f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip',
|
||||
source_subdir=source_subdir,
|
||||
@@ -181,27 +237,45 @@ class SkillService:
|
||||
|
||||
async def install_from_zip_upload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
source_paths: list[str] | None = None,
|
||||
source_path: str = '',
|
||||
) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Installing a skill from upload')
|
||||
installed = await box_service.install_skill_zip(
|
||||
execution_context,
|
||||
file_bytes,
|
||||
filename,
|
||||
source_paths=source_paths or [],
|
||||
source_path=source_path,
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(execution_context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_zip_upload(self, *, file_bytes: bytes, filename: str) -> list[dict]:
|
||||
async def preview_install_from_zip_upload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Previewing a skill upload')
|
||||
return await box_service.preview_skill_zip(file_bytes, filename)
|
||||
return await box_service.preview_skill_zip(execution_context, file_bytes, filename)
|
||||
|
||||
async def _install_github_skill_md(self, asset_url: str, *, owner: str, repo: str, data: dict) -> list[dict]:
|
||||
async def _install_github_skill_md(
|
||||
self,
|
||||
context: TenantContext,
|
||||
asset_url: str,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
data: dict,
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
zip_bytes, filename, _package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
@@ -210,46 +284,73 @@ class SkillService:
|
||||
)
|
||||
|
||||
installed = await box_service.install_skill_zip(
|
||||
context,
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
target_suffix='',
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def _preview_github_skill_md(self, asset_url: str, *, owner: str, repo: str) -> list[dict]:
|
||||
async def _preview_github_skill_md(
|
||||
self,
|
||||
context: TenantContext,
|
||||
asset_url: str,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
zip_bytes, _filename, package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
return await box_service.preview_skill_zip(zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
return await box_service.preview_skill_zip(context, zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
|
||||
async def reload_skills(self) -> list[dict]:
|
||||
await self._reload_skills()
|
||||
return await self.list_skills()
|
||||
async def reload_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
await self._reload_skills(execution_context)
|
||||
return await self.list_skills(execution_context)
|
||||
|
||||
async def scan_directory_async(self, path: str) -> dict:
|
||||
async def scan_directory_async(self, context: TenantContext, path: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Scanning a skill directory')
|
||||
return await box_service.scan_skill_directory(path)
|
||||
return await box_service.scan_skill_directory(execution_context, path)
|
||||
|
||||
async def _reload_skills(self) -> None:
|
||||
async def _reload_skills(self, context: TenantContext) -> None:
|
||||
skill_mgr = getattr(self.ap, 'skill_mgr', None)
|
||||
reload_skills = getattr(skill_mgr, 'reload_skills', None)
|
||||
if not callable(reload_skills):
|
||||
return
|
||||
result = reload_skills()
|
||||
result = reload_skills(context)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _download_github_asset(self, asset_url: str) -> bytes:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
|
||||
resp = await client.get(asset_url)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=True,
|
||||
timeout=120,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(_MAX_GITHUB_ARCHIVE_BYTES),
|
||||
) as client:
|
||||
async with client.stream('GET', asset_url) as resp:
|
||||
resp.raise_for_status()
|
||||
content_length = resp.headers.get('content-length')
|
||||
if content_length is not None:
|
||||
try:
|
||||
if int(content_length) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
except ValueError as exc:
|
||||
if 'exceeds' in str(exc):
|
||||
raise
|
||||
content = bytearray()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
content.extend(chunk)
|
||||
if len(content) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
return bytes(content)
|
||||
|
||||
async def _download_github_skill_directory_as_zip(
|
||||
self, asset_url: str, *, owner: str, repo: str
|
||||
@@ -257,14 +358,25 @@ class SkillService:
|
||||
info = self._parse_github_skill_md_url(asset_url, owner=owner, repo=repo)
|
||||
archive_url = f'https://codeload.github.com/{owner}/{repo}/zip/{quote(info["ref"], safe="/")}'
|
||||
archive_bytes = await self._download_github_asset(archive_url)
|
||||
return await asyncio.to_thread(self._build_github_skill_directory_zip, archive_bytes, info)
|
||||
|
||||
def _build_github_skill_directory_zip(
|
||||
self,
|
||||
archive_bytes: bytes,
|
||||
info: dict[str, str],
|
||||
) -> tuple[bytes, str, str]:
|
||||
"""Validate and repack a GitHub skill archive outside the event loop."""
|
||||
try:
|
||||
source_archive = zipfile.ZipFile(io.BytesIO(archive_bytes), 'r')
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError('GitHub repository archive must be a valid .zip archive') from exc
|
||||
|
||||
with source_archive as source_zip:
|
||||
if len(source_zip.infolist()) > _MAX_GITHUB_ARCHIVE_ENTRIES:
|
||||
raise ValueError('GitHub repository archive contains too many entries')
|
||||
skill_entry = self._find_github_skill_archive_entry(source_zip, info['file_path'])
|
||||
if skill_entry.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError('GitHub SKILL.md exceeds the file size limit')
|
||||
try:
|
||||
skill_md_content = source_zip.read(skill_entry).decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
@@ -302,6 +414,7 @@ class SkillService:
|
||||
normalized_source_dir = posixpath.normpath(source_skill_dir)
|
||||
source_prefix = f'{normalized_source_dir}/'
|
||||
copied_files = 0
|
||||
copied_bytes = 0
|
||||
|
||||
for member in source_zip.infolist():
|
||||
normalized_member = posixpath.normpath(member.filename)
|
||||
@@ -324,10 +437,33 @@ class SkillService:
|
||||
if member.is_dir():
|
||||
target_zip.writestr(target_info, b'')
|
||||
continue
|
||||
|
||||
target_zip.writestr(target_info, source_zip.read(member))
|
||||
if member.flag_bits & 0x1:
|
||||
raise ValueError('Encrypted GitHub skill archive entries are not supported')
|
||||
unix_mode = member.external_attr >> 16
|
||||
if stat.S_IFMT(unix_mode) == stat.S_IFLNK:
|
||||
raise ValueError(f'GitHub archive contains a symbolic link: {member.filename}')
|
||||
if member.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError(f'GitHub skill file exceeds the size limit: {member.filename}')
|
||||
if member.file_size and member.file_size > max(member.compress_size, 1) * _MAX_SKILL_COMPRESSION_RATIO:
|
||||
raise ValueError(f'GitHub skill file exceeds the compression-ratio limit: {member.filename}')
|
||||
copied_files += 1
|
||||
copied_bytes += member.file_size
|
||||
if copied_files > _MAX_SKILL_ARCHIVE_FILES:
|
||||
raise ValueError('GitHub skill directory contains too many files')
|
||||
if copied_bytes > _MAX_SKILL_UNCOMPRESSED_BYTES:
|
||||
raise ValueError('GitHub skill directory exceeds the uncompressed size limit')
|
||||
|
||||
# Copy in bounded chunks instead of materialising a potentially
|
||||
# large member in Core memory. The Box Runtime independently
|
||||
# revalidates the resulting archive before installation.
|
||||
with source_zip.open(member, 'r') as source_file, target_zip.open(target_info, 'w') as target_file:
|
||||
remaining = member.file_size
|
||||
while remaining:
|
||||
chunk = source_file.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise ValueError(f'GitHub skill file is truncated: {member.filename}')
|
||||
target_file.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if copied_files == 0:
|
||||
raise ValueError('GitHub skill directory is empty')
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from langbot.pkg.utils import httpclient
|
||||
import typing
|
||||
import datetime
|
||||
@@ -11,6 +13,10 @@ from ....entity.persistence import user
|
||||
from ....entity.dto.space_model import SpaceModel
|
||||
|
||||
|
||||
_CREDITS_CACHE_TTL_SECONDS = 60
|
||||
_CREDITS_CACHE_MAX_ENTRIES = 4096
|
||||
|
||||
|
||||
class SpaceService:
|
||||
"""Service for interacting with LangBot Space API"""
|
||||
|
||||
@@ -19,7 +25,24 @@ class SpaceService:
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
self._credits_cache = {}
|
||||
self._credits_cache = OrderedDict()
|
||||
|
||||
def _ordered_credits_cache(
|
||||
self,
|
||||
) -> OrderedDict[str, tuple[int, float]]:
|
||||
if not isinstance(self._credits_cache, OrderedDict):
|
||||
# Preserve compatibility with tests and callers that seed the cache.
|
||||
self._credits_cache = OrderedDict(self._credits_cache)
|
||||
return self._credits_cache
|
||||
|
||||
def _prune_credits_cache(self, now: float) -> None:
|
||||
cache = self._ordered_credits_cache()
|
||||
while cache:
|
||||
email = next(iter(cache))
|
||||
_, cached_at = cache[email]
|
||||
if now - cached_at < _CREDITS_CACHE_TTL_SECONDS:
|
||||
break
|
||||
cache.pop(email, None)
|
||||
|
||||
def _get_space_config(self) -> typing.Dict[str, str]:
|
||||
"""Get Space configuration from config file"""
|
||||
@@ -85,12 +108,14 @@ class SpaceService:
|
||||
|
||||
def get_oauth_authorize_url(self, redirect_uri: str, state: str = '') -> str:
|
||||
"""Get the Space OAuth authorization URL for redirect"""
|
||||
from urllib.parse import urlencode
|
||||
|
||||
space_config = self._get_space_config()
|
||||
authorize_url = space_config['oauth_authorize_url']
|
||||
params = f'redirect_uri={redirect_uri}'
|
||||
params = {'redirect_uri': redirect_uri}
|
||||
if state:
|
||||
params += f'&state={state}'
|
||||
return f'{authorize_url}?{params}'
|
||||
params['state'] = state
|
||||
return f'{authorize_url}?{urlencode(params)}'
|
||||
|
||||
async def exchange_oauth_code(self, code: str) -> typing.Dict:
|
||||
"""Exchange OAuth authorization code for tokens"""
|
||||
@@ -105,8 +130,9 @@ class SpaceService:
|
||||
json={'code': code, 'instance_id': constants.instance_id},
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to exchange OAuth code: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to exchange OAuth code: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to exchange OAuth code: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -121,8 +147,9 @@ class SpaceService:
|
||||
f'{space_url}/api/v1/accounts/token/refresh', json={'refresh_token': refresh_token}
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to refresh token: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to refresh token: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to refresh token: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -137,8 +164,9 @@ class SpaceService:
|
||||
f'{space_url}/api/v1/accounts/me', headers={'Authorization': f'Bearer {access_token}'}
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get user info: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to get user info: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to get user info: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -154,11 +182,13 @@ class SpaceService:
|
||||
|
||||
async def get_credits(self, user_email: str, force_refresh: bool = False) -> int | None:
|
||||
"""Get Space credits for user with caching (60s TTL)"""
|
||||
cache_ttl = 60
|
||||
now = time.time()
|
||||
cached_fallback = self._credits_cache.get(user_email)
|
||||
self._prune_credits_cache(now)
|
||||
|
||||
if not force_refresh and user_email in self._credits_cache:
|
||||
credits, ts = self._credits_cache[user_email]
|
||||
if time.time() - ts < cache_ttl:
|
||||
if now - ts < _CREDITS_CACHE_TTL_SECONDS:
|
||||
return credits
|
||||
|
||||
try:
|
||||
@@ -167,10 +197,14 @@ class SpaceService:
|
||||
return None
|
||||
credits = info.get('credits')
|
||||
if credits is not None:
|
||||
self._credits_cache[user_email] = (credits, time.time())
|
||||
cache = self._ordered_credits_cache()
|
||||
cache.pop(user_email, None)
|
||||
if len(cache) >= _CREDITS_CACHE_MAX_ENTRIES:
|
||||
cache.popitem(last=False)
|
||||
cache[user_email] = (credits, time.time())
|
||||
return credits
|
||||
except Exception:
|
||||
return self._credits_cache.get(user_email, (None, 0))[0]
|
||||
return cached_fallback[0] if cached_fallback is not None else None
|
||||
|
||||
async def get_models(self) -> typing.List[SpaceModel]:
|
||||
"""Get models from Space"""
|
||||
@@ -181,8 +215,9 @@ class SpaceService:
|
||||
session = httpclient.get_session()
|
||||
async with session.get(f'{space_url}/api/v1/models', params={'page_size': 100}) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get models: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to get models: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to get models: {data.get("msg")}')
|
||||
models_data = data.get('data', {}).get('models', [])
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext, RequestContext, WorkspaceContext
|
||||
|
||||
TenantContext: typing.TypeAlias = RequestContext | ExecutionContext | WorkspaceContext | str
|
||||
|
||||
|
||||
def require_workspace_uuid(context: TenantContext | None) -> str:
|
||||
"""Resolve an explicit Workspace UUID without allowing a global fallback."""
|
||||
|
||||
if isinstance(context, str):
|
||||
workspace_uuid = context
|
||||
elif isinstance(context, RequestContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
elif isinstance(context, ExecutionContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
elif isinstance(context, WorkspaceContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
else:
|
||||
raise WorkspaceRequiredError('Workspace context is required')
|
||||
|
||||
normalized = workspace_uuid.strip()
|
||||
if not normalized:
|
||||
raise WorkspaceRequiredError('Workspace context is required')
|
||||
return normalized
|
||||
|
||||
|
||||
def scope_statement(statement: typing.Any, model: typing.Any, context: TenantContext) -> typing.Any:
|
||||
"""Add the mandatory Workspace predicate to a SQLAlchemy statement."""
|
||||
|
||||
return statement.where(model.workspace_uuid == require_workspace_uuid(context))
|
||||
@@ -6,71 +6,391 @@ import jwt
|
||||
import datetime
|
||||
import typing
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import heapq
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import user
|
||||
from ....entity.persistence.workspace import MembershipRole, MembershipStatus, WorkspaceMembership
|
||||
from ....utils import constants
|
||||
from ....entity.errors import account as account_errors
|
||||
from ....workspace.collaboration import normalize_email
|
||||
from ....utils import bounded_executor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
|
||||
_SPACE_OAUTH_STATE_MAX_ENTRIES = 4096
|
||||
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR = 64
|
||||
_SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER = 4
|
||||
|
||||
|
||||
class AccountExistsLoginRequiredError(ValueError):
|
||||
code = 'account_exists_login_required'
|
||||
|
||||
|
||||
class PublicRegistrationClosedError(ValueError):
|
||||
code = 'registration_closed'
|
||||
|
||||
|
||||
class ControlPlaneDirectoryRequiredError(PublicRegistrationClosedError):
|
||||
code = 'control_plane_required'
|
||||
|
||||
|
||||
class AccountDisabledError(ValueError):
|
||||
code = 'account_disabled'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class SpaceOAuthStateConsumption:
|
||||
purpose: typing.Literal['login', 'bind']
|
||||
account: user.User | None
|
||||
launch_workspace_uuid: str | None = None
|
||||
|
||||
|
||||
class UserService:
|
||||
ap: app.Application
|
||||
ap: Application
|
||||
_create_user_lock: asyncio.Lock
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(self, ap: Application) -> None:
|
||||
self.ap = ap
|
||||
self._create_user_lock = asyncio.Lock()
|
||||
self._password_hash_lock = asyncio.Semaphore(1)
|
||||
self._password_hash_lock = asyncio.Lock()
|
||||
self._space_oauth_state_lock = asyncio.Lock()
|
||||
self._space_oauth_states: dict[str, tuple[str, str | None, float, str | None]] = {}
|
||||
self._space_oauth_state_expiry_heap: list[tuple[float, str]] = []
|
||||
|
||||
@staticmethod
|
||||
def _space_oauth_state_digest(state: str) -> str:
|
||||
return hashlib.sha256(state.encode('utf-8')).hexdigest()
|
||||
|
||||
def _prune_space_oauth_states(self, now: float) -> None:
|
||||
while self._space_oauth_state_expiry_heap:
|
||||
expires_at, digest = self._space_oauth_state_expiry_heap[0]
|
||||
entry = self._space_oauth_states.get(digest)
|
||||
if entry is None or entry[2] != expires_at:
|
||||
heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
continue
|
||||
if expires_at > now:
|
||||
break
|
||||
heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
self._space_oauth_states.pop(digest, None)
|
||||
|
||||
max_heap_entries = max(
|
||||
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR,
|
||||
len(self._space_oauth_states) * _SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER,
|
||||
)
|
||||
if len(self._space_oauth_state_expiry_heap) > max_heap_entries:
|
||||
self._space_oauth_state_expiry_heap[:] = [
|
||||
(entry[2], digest) for digest, entry in self._space_oauth_states.items()
|
||||
]
|
||||
heapq.heapify(self._space_oauth_state_expiry_heap)
|
||||
|
||||
def _evict_earliest_space_oauth_state(self) -> None:
|
||||
while self._space_oauth_state_expiry_heap:
|
||||
expires_at, digest = heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
entry = self._space_oauth_states.get(digest)
|
||||
if entry is not None and entry[2] == expires_at:
|
||||
self._space_oauth_states.pop(digest, None)
|
||||
return
|
||||
|
||||
async def issue_space_oauth_state(
|
||||
self,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
*,
|
||||
account_uuid: str | None = None,
|
||||
launch_workspace_uuid: str | None = None,
|
||||
ttl_seconds: int = 600,
|
||||
) -> str:
|
||||
"""Issue an opaque, single-use OAuth state without exposing a JWT."""
|
||||
if purpose == 'bind' and not account_uuid:
|
||||
raise ValueError('An Account is required for Space binding')
|
||||
if purpose == 'login' and account_uuid is not None:
|
||||
raise ValueError('Login state cannot be bound to an Account')
|
||||
if purpose != 'login' and launch_workspace_uuid is not None:
|
||||
raise ValueError('Launch Workspace state is only valid for Space login')
|
||||
if ttl_seconds <= 0:
|
||||
raise ValueError('OAuth state lifetime must be positive')
|
||||
|
||||
raw_state = secrets.token_urlsafe(32)
|
||||
digest = self._space_oauth_state_digest(raw_state)
|
||||
expires_at = time.monotonic() + min(ttl_seconds, 600)
|
||||
async with self._space_oauth_state_lock:
|
||||
now = time.monotonic()
|
||||
self._prune_space_oauth_states(now)
|
||||
if len(self._space_oauth_states) >= _SPACE_OAUTH_STATE_MAX_ENTRIES:
|
||||
self._evict_earliest_space_oauth_state()
|
||||
self._space_oauth_states[digest] = (purpose, account_uuid, expires_at, launch_workspace_uuid)
|
||||
heapq.heappush(
|
||||
self._space_oauth_state_expiry_heap,
|
||||
(expires_at, digest),
|
||||
)
|
||||
return raw_state
|
||||
|
||||
async def consume_space_oauth_state_details(
|
||||
self,
|
||||
raw_state: str,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
) -> SpaceOAuthStateConsumption:
|
||||
"""Atomically consume OAuth state and return any bound launch intent."""
|
||||
if not isinstance(raw_state, str) or not raw_state:
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
digest = self._space_oauth_state_digest(raw_state)
|
||||
async with self._space_oauth_state_lock:
|
||||
entry = self._space_oauth_states.pop(digest, None)
|
||||
if entry is None or entry[0] != purpose or entry[2] <= time.monotonic():
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
if purpose == 'login':
|
||||
return SpaceOAuthStateConsumption(
|
||||
purpose='login',
|
||||
account=None,
|
||||
launch_workspace_uuid=entry[3],
|
||||
)
|
||||
|
||||
account_uuid = entry[1]
|
||||
account = await self.get_user_by_uuid(account_uuid or '')
|
||||
if account is None:
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
self._require_active_account(account)
|
||||
return SpaceOAuthStateConsumption(purpose='bind', account=account)
|
||||
|
||||
async def consume_space_oauth_state(
|
||||
self,
|
||||
raw_state: str,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
) -> user.User | None:
|
||||
"""Atomically consume OAuth state and resolve its active bind Account."""
|
||||
consumed = await self.consume_space_oauth_state_details(raw_state, purpose)
|
||||
return consumed.account
|
||||
|
||||
async def _hash_password(self, password: str) -> str:
|
||||
if self._password_hash_lock.locked():
|
||||
raise bounded_executor.BlockingWorkCapacityError(
|
||||
'Password hashing capacity reached',
|
||||
scope='system:authentication',
|
||||
)
|
||||
async with self._password_hash_lock:
|
||||
return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
|
||||
with bounded_executor.blocking_work_scope('system:authentication'):
|
||||
return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
|
||||
|
||||
def _require_local_directory(self) -> None:
|
||||
if self._uses_control_plane_directory():
|
||||
raise ControlPlaneDirectoryRequiredError(
|
||||
'Cloud Accounts and directory changes are managed by the SaaS control plane'
|
||||
)
|
||||
|
||||
def _uses_control_plane_directory(self) -> bool:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
return bool(workspace_service is not None and workspace_service.policy.multi_workspace_enabled)
|
||||
|
||||
async def _verify_password(self, hashed_password: str, password: str) -> None:
|
||||
if self._password_hash_lock.locked():
|
||||
raise bounded_executor.BlockingWorkCapacityError(
|
||||
'Password hashing capacity reached',
|
||||
scope='system:authentication',
|
||||
)
|
||||
async with self._password_hash_lock:
|
||||
await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
|
||||
with bounded_executor.blocking_work_scope('system:authentication'):
|
||||
await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
|
||||
|
||||
async def _update_space_provider_for_account(self, account: typing.Any, api_key: str) -> None:
|
||||
"""Refresh the OSS Workspace Space provider without guessing a SaaS Workspace.
|
||||
|
||||
Space OAuth credentials belong to an Account, while model-provider secrets
|
||||
belong to a Workspace. Community edition has one unambiguous Workspace, so
|
||||
the historical automatic refresh remains available only to the Workspace owner.
|
||||
In multi-Workspace SaaS mode the OAuth callback has
|
||||
no trusted Workspace selector; the closed control plane or an explicit
|
||||
Workspace settings action must perform that linkage instead.
|
||||
"""
|
||||
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
if workspace_service is None or collaboration_service is None or not isinstance(account_uuid, str):
|
||||
# Never turn a missing tenant kernel into a global secret mutation.
|
||||
return
|
||||
if workspace_service.policy.multi_workspace_enabled:
|
||||
return
|
||||
|
||||
accesses = await collaboration_service.list_account_workspaces(account_uuid)
|
||||
if len(accesses) != 1:
|
||||
return
|
||||
access = accesses[0]
|
||||
if access.membership.role != MembershipRole.OWNER.value:
|
||||
return
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(
|
||||
access.workspace.uuid,
|
||||
api_key,
|
||||
)
|
||||
|
||||
async def is_initialized(self) -> bool:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
account = await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
return account is not None
|
||||
|
||||
result_list = result.all()
|
||||
return result_list is not None and len(result_list) > 0
|
||||
async def get_login_capabilities(self) -> dict[str, bool]:
|
||||
"""Derive enabled public login methods in an explicit discovery scope."""
|
||||
password_count = sqlalchemy.func.count().filter(user.User.password.is_not(None), user.User.password != '')
|
||||
space_count = sqlalchemy.func.count().filter(user.User.space_account_uuid.is_not(None))
|
||||
statement = sqlalchemy.select(password_count, space_count).where(
|
||||
user.User.status == user.AccountStatus.ACTIVE.value
|
||||
)
|
||||
digest = hashlib.sha256(f'login-capabilities:{self._jwt_identity()[1]}'.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
result = await discovery.session.execute(statement)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
password_accounts, space_accounts = result.one()
|
||||
return {
|
||||
'password_login_enabled': bool(password_accounts),
|
||||
'space_login_enabled': bool(space_accounts),
|
||||
}
|
||||
|
||||
async def get_workspace_owner(self, workspace_uuid: str) -> user.User | None:
|
||||
"""Resolve the active owner Account for a Workspace."""
|
||||
statement = (
|
||||
sqlalchemy.select(user.User)
|
||||
.join(WorkspaceMembership, WorkspaceMembership.account_uuid == user.User.uuid)
|
||||
.where(
|
||||
WorkspaceMembership.workspace_uuid == workspace_uuid,
|
||||
WorkspaceMembership.role == MembershipRole.OWNER.value,
|
||||
WorkspaceMembership.status == MembershipStatus.ACTIVE.value,
|
||||
user.User.status == user.AccountStatus.ACTIVE.value,
|
||||
)
|
||||
)
|
||||
current_session = self.ap.persistence_mgr.current_session()
|
||||
if current_session is not None:
|
||||
return await current_session.scalar(statement)
|
||||
return await self._identity_scalar(statement, f'workspace-owner:{workspace_uuid}')
|
||||
|
||||
def _session_factory(self) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(self.ap.persistence_mgr.get_db_engine(), expire_on_commit=False)
|
||||
|
||||
def _jwt_identity(self) -> tuple[str, str]:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
instance_uuid = str(getattr(workspace_service, 'instance_uuid', '') or constants.instance_id).strip()
|
||||
# UserService is constructed only after config/bootstrap in production.
|
||||
# The fallback keeps lightweight isolated unit tests deterministic.
|
||||
if not instance_uuid:
|
||||
instance_uuid = 'uninitialized-test-instance'
|
||||
return 'langbot-core', f'langbot-instance:{instance_uuid}'
|
||||
|
||||
def _legacy_local_tokens_allowed(self) -> bool:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
policy = getattr(workspace_service, 'policy', None)
|
||||
return getattr(policy, 'multi_workspace_enabled', False) is not True
|
||||
|
||||
async def create_user(self, user_email: str, password: str) -> None:
|
||||
"""Create the first local Account and Workspace owner atomically."""
|
||||
|
||||
await self.create_initial_account(user_email, password)
|
||||
|
||||
async def create_initial_account(self, user_email: str, password: str) -> user.User:
|
||||
self._require_local_directory()
|
||||
normalized_email = normalize_email(user_email)
|
||||
hashed_password = await self._hash_password(password)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(user.User).values(user=user_email, password=hashed_password, account_type='local')
|
||||
async with self._create_user_lock:
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
existing_count = int(
|
||||
(await session.scalar(sqlalchemy.select(sqlalchemy.func.count()).select_from(user.User))) or 0
|
||||
)
|
||||
if existing_count:
|
||||
raise PublicRegistrationClosedError('System already initialized')
|
||||
account = self._new_account(normalized_email, hashed_password)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
await self.ap.workspace_service.bootstrap_local_account(account.uuid, session=session)
|
||||
return account
|
||||
|
||||
async def register_invited_account(
|
||||
self,
|
||||
invitation_token: str,
|
||||
user_email: str,
|
||||
password: str,
|
||||
) -> tuple[user.User, typing.Any]:
|
||||
"""Create an invited Account and accept its Membership in one transaction."""
|
||||
|
||||
normalized_email = normalize_email(user_email)
|
||||
if self._uses_control_plane_directory():
|
||||
raise ControlPlaneDirectoryRequiredError(
|
||||
'Cloud invitation registration must use a Space account to preserve control-plane identity'
|
||||
)
|
||||
invitation, _ = await self.ap.workspace_collaboration_service.inspect_invitation(invitation_token)
|
||||
if invitation.normalized_email != normalized_email:
|
||||
from ....workspace.collaboration import InvitationEmailMismatchError
|
||||
|
||||
raise InvitationEmailMismatchError('Invitation email does not match the Account')
|
||||
hashed_password = await self._hash_password(password)
|
||||
|
||||
async with self._create_user_lock:
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
existing = await session.scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email)
|
||||
)
|
||||
if existing is not None:
|
||||
raise AccountExistsLoginRequiredError('An Account already exists for this email')
|
||||
account = self._new_account(normalized_email, hashed_password)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
membership = await self.ap.workspace_collaboration_service.accept_invitation(
|
||||
invitation_token,
|
||||
account.uuid,
|
||||
session=session,
|
||||
)
|
||||
return account, membership
|
||||
|
||||
def _new_account(self, normalized_email: str, hashed_password: str) -> user.User:
|
||||
return user.User(
|
||||
uuid=str(uuid.uuid4()),
|
||||
user=normalized_email,
|
||||
normalized_email=normalized_email,
|
||||
password=hashed_password,
|
||||
account_type='local',
|
||||
status=user.AccountStatus.ACTIVE.value,
|
||||
source=user.AccountSource.LOCAL.value,
|
||||
projection_revision=0,
|
||||
)
|
||||
|
||||
async def get_user_by_email(self, user_email: str) -> user.User | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.user == user_email)
|
||||
normalized_email = user_email.strip().casefold()
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
async def get_user_by_uuid(self, account_uuid: str) -> user.User | None:
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid),
|
||||
f'uuid:{account_uuid}',
|
||||
)
|
||||
|
||||
async def get_user_by_space_account_uuid(self, space_account_uuid: str) -> user.User | None:
|
||||
"""Get user by Space account UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid)
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
|
||||
async def authenticate(self, user_email: str, password: str) -> str | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.user == user_email)
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
|
||||
if result_list is None or len(result_list) == 0:
|
||||
user_obj = await self.get_user_by_email(user_email)
|
||||
if user_obj is None:
|
||||
raise ValueError('用户不存在')
|
||||
|
||||
user_obj = result_list[0]
|
||||
self._require_active_account(user_obj)
|
||||
|
||||
# Check if this user has a local password set
|
||||
if not user_obj.password:
|
||||
@@ -78,30 +398,121 @@ class UserService:
|
||||
|
||||
await self._verify_password(user_obj.password, password)
|
||||
|
||||
return await self.generate_jwt_token(user_email)
|
||||
return await self.generate_jwt_token(user_obj)
|
||||
|
||||
async def generate_jwt_token(self, user_email: str) -> str:
|
||||
async def generate_jwt_token(self, account: user.User | str) -> str:
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
jwt_expire = self.ap.instance_config.data['system']['jwt']['expire']
|
||||
|
||||
account_obj: user.User | None = account if not isinstance(account, str) and hasattr(account, 'user') else None
|
||||
user_email = account_obj.user if account_obj is not None else account
|
||||
if account_obj is None and hasattr(self.ap, 'persistence_mgr'):
|
||||
try:
|
||||
account_obj = await self.get_user_by_email(user_email)
|
||||
except (AttributeError, TypeError):
|
||||
# Lightweight unit-test and bootstrap callers may not have persistence wired.
|
||||
account_obj = None
|
||||
|
||||
payload = {
|
||||
'user': user_email,
|
||||
'iss': 'LangBot-' + constants.edition,
|
||||
'iss': self._jwt_identity()[0],
|
||||
'aud': self._jwt_identity()[1],
|
||||
'exp': datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=jwt_expire),
|
||||
}
|
||||
if account_obj is not None:
|
||||
self._require_active_account(account_obj)
|
||||
payload.update(
|
||||
{
|
||||
'sub': account_obj.uuid,
|
||||
'account_revision': account_obj.projection_revision,
|
||||
}
|
||||
)
|
||||
|
||||
return jwt.encode(payload, jwt_secret, algorithm='HS256')
|
||||
|
||||
async def verify_jwt_token(self, token: str) -> str:
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
account = await self.get_authenticated_account(token, allow_unresolved_legacy=True)
|
||||
if isinstance(account, str):
|
||||
return account
|
||||
return account.user
|
||||
|
||||
return jwt.decode(token, jwt_secret, algorithms=['HS256'])['user']
|
||||
async def get_authenticated_account(
|
||||
self,
|
||||
token: str,
|
||||
*,
|
||||
allow_unresolved_legacy: bool = False,
|
||||
) -> user.User | str:
|
||||
"""Resolve a JWT to an active Account, accepting bounded legacy email tokens."""
|
||||
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
issuer, audience = self._jwt_identity()
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
jwt_secret,
|
||||
algorithms=['HS256'],
|
||||
issuer=issuer,
|
||||
audience=audience,
|
||||
options={'require': ['exp', 'iss', 'aud']},
|
||||
)
|
||||
except jwt.MissingRequiredClaimError:
|
||||
# Preserve one bounded OSS upgrade path for previously issued
|
||||
# community tokens. SaaS/Cloud policy never accepts these tokens,
|
||||
# and a token carrying a new-style or foreign audience cannot fall
|
||||
# back into the legacy decoder.
|
||||
unverified = jwt.decode(token, options={'verify_signature': False})
|
||||
if (
|
||||
not self._legacy_local_tokens_allowed()
|
||||
or 'aud' in unverified
|
||||
or unverified.get('iss') != 'LangBot-community'
|
||||
):
|
||||
raise
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
jwt_secret,
|
||||
algorithms=['HS256'],
|
||||
options={'require': ['exp'], 'verify_aud': False, 'verify_iss': False},
|
||||
)
|
||||
account_obj: user.User | None = None
|
||||
account_uuid = payload.get('sub')
|
||||
if isinstance(account_uuid, str) and account_uuid:
|
||||
try:
|
||||
account_obj = await self.get_user_by_uuid(account_uuid)
|
||||
except AttributeError:
|
||||
account_obj = None
|
||||
if account_obj is None:
|
||||
legacy_email = payload.get('user')
|
||||
if not isinstance(legacy_email, str) or not legacy_email:
|
||||
raise ValueError('JWT Account identity is missing')
|
||||
try:
|
||||
account_obj = await self.get_user_by_email(legacy_email)
|
||||
except AttributeError:
|
||||
account_obj = None
|
||||
if account_obj is None and allow_unresolved_legacy:
|
||||
return legacy_email
|
||||
if account_obj is None:
|
||||
raise ValueError('Account not found')
|
||||
self._require_active_account(account_obj)
|
||||
token_revision = payload.get('account_revision')
|
||||
if token_revision is not None and int(token_revision) != account_obj.projection_revision:
|
||||
raise ValueError('Account token revision is stale')
|
||||
return account_obj
|
||||
|
||||
@staticmethod
|
||||
def _require_active_account(account: user.User) -> None:
|
||||
status = getattr(account, 'status', user.AccountStatus.ACTIVE.value)
|
||||
if isinstance(status, str) and status != user.AccountStatus.ACTIVE.value:
|
||||
raise AccountDisabledError('Account is disabled')
|
||||
|
||||
async def reset_password(self, user_email: str, new_password: str) -> None:
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def change_password(self, user_email: str, current_password: str, new_password: str) -> None:
|
||||
@@ -115,9 +526,13 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Space user management
|
||||
@@ -132,6 +547,16 @@ class UserService:
|
||||
expires_in: int = 0,
|
||||
) -> user.User:
|
||||
"""Create or update a Space user account (only if system not initialized or user exists)"""
|
||||
if self._uses_control_plane_directory():
|
||||
return await self._update_projected_space_user(
|
||||
space_account_uuid=space_account_uuid,
|
||||
email=email,
|
||||
access_token=access_token,
|
||||
refresh_token=refresh_token,
|
||||
api_key=api_key,
|
||||
expires_in=expires_in,
|
||||
)
|
||||
self._require_local_directory()
|
||||
expires_at = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) if expires_in > 0 else None
|
||||
|
||||
async with self._create_user_lock:
|
||||
@@ -140,7 +565,7 @@ class UserService:
|
||||
|
||||
if existing_user:
|
||||
# Update existing user's tokens
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.space_account_uuid == space_account_uuid)
|
||||
.values(
|
||||
@@ -148,19 +573,56 @@ class UserService:
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
await self._update_space_provider_for_account(existing_user, api_key)
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
|
||||
# Check if user with same email exists
|
||||
existing_email_user = await self.get_user_by_email(email)
|
||||
if existing_email_user:
|
||||
# Update existing user to link with Space account
|
||||
# Email is display/contact identity, not an OAuth subject. An
|
||||
# unknown Space subject must never take over an existing local
|
||||
# Account merely by presenting the same email. The Account
|
||||
# owner must first authenticate locally and use the explicit,
|
||||
# account-bound bind flow.
|
||||
raise account_errors.SpaceAccountBindingRequiredError()
|
||||
|
||||
# Check if system is already initialized
|
||||
is_initialized = await self.is_initialized()
|
||||
if is_initialized:
|
||||
raise account_errors.SpaceAccountNotRegisteredError()
|
||||
|
||||
# Create new Space user (first time initialization)
|
||||
if hasattr(self.ap.persistence_mgr, 'get_db_engine') and hasattr(self.ap, 'workspace_service'):
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
account = user.User(
|
||||
uuid=str(uuid.uuid4()),
|
||||
user=normalize_email(email),
|
||||
normalized_email=normalize_email(email),
|
||||
password='',
|
||||
account_type='space',
|
||||
status=user.AccountStatus.ACTIVE.value,
|
||||
source=user.AccountSource.LOCAL.value,
|
||||
projection_revision=0,
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
await self.ap.workspace_service.bootstrap_local_account(account.uuid, session=session)
|
||||
else:
|
||||
# Compatibility path for lightweight service tests without a real engine.
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.user == email)
|
||||
.values(
|
||||
sqlalchemy.insert(user.User).values(
|
||||
user=normalize_email(email),
|
||||
normalized_email=normalize_email(email),
|
||||
password='',
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
@@ -169,30 +631,56 @@ class UserService:
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
return await self.get_user_by_email(email)
|
||||
created_user = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if created_user is not None:
|
||||
await self._update_space_provider_for_account(created_user, api_key)
|
||||
return created_user
|
||||
|
||||
# Check if system is already initialized
|
||||
is_initialized = await self.is_initialized()
|
||||
if is_initialized:
|
||||
raise account_errors.AccountEmailMismatchError()
|
||||
async def _update_projected_space_user(
|
||||
self,
|
||||
*,
|
||||
space_account_uuid: str,
|
||||
email: str,
|
||||
access_token: str,
|
||||
refresh_token: str,
|
||||
api_key: str,
|
||||
expires_in: int,
|
||||
) -> user.User:
|
||||
"""Attach OAuth credentials to an already projected Cloud Account."""
|
||||
|
||||
# Create new Space user (first time initialization)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(user.User).values(
|
||||
user=email,
|
||||
password='', # Space users don't have local password
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
normalized_email = normalize_email(email)
|
||||
expires_at = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) if expires_in > 0 else None
|
||||
async with self._create_user_lock:
|
||||
projected = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if (
|
||||
projected is None
|
||||
or projected.uuid != space_account_uuid
|
||||
or projected.normalized_email != normalized_email
|
||||
or projected.source != user.AccountSource.CLOUD_PROJECTION.value
|
||||
or projected.account_type != 'space'
|
||||
):
|
||||
raise ControlPlaneDirectoryRequiredError('Space Account is not present in the verified Cloud directory')
|
||||
self._require_active_account(projected)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(
|
||||
user.User.uuid == projected.uuid,
|
||||
user.User.space_account_uuid == space_account_uuid,
|
||||
user.User.source == user.AccountSource.CLOUD_PROJECTION.value,
|
||||
)
|
||||
.values(
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
refreshed = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if refreshed is None:
|
||||
raise ControlPlaneDirectoryRequiredError('Space Account disappeared from the verified Cloud directory')
|
||||
self._require_active_account(refreshed)
|
||||
return refreshed
|
||||
|
||||
async def authenticate_space_user(
|
||||
self, access_token: str, refresh_token: str, expires_in: int = 0
|
||||
@@ -221,15 +709,44 @@ class UserService:
|
||||
)
|
||||
|
||||
# Generate JWT token
|
||||
jwt_token = await self.generate_jwt_token(email)
|
||||
jwt_token = await self.generate_jwt_token(user_obj)
|
||||
|
||||
return jwt_token, user_obj
|
||||
|
||||
async def get_first_user(self) -> user.User | None:
|
||||
"""Get the first user (for single-user mode)"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list else None
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
|
||||
async def _identity_scalar(
|
||||
self,
|
||||
statement: typing.Any,
|
||||
identity: str,
|
||||
) -> user.User | None:
|
||||
"""Execute one exact Account lookup in an explicit discovery transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.scalar(statement)
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
rows = result.all()
|
||||
return rows[0] if rows else None
|
||||
|
||||
async def _identity_execute(self, statement: typing.Any, identity: str) -> typing.Any:
|
||||
"""Execute one exact Account mutation in an explicit transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.execute(statement)
|
||||
return await self.ap.persistence_mgr.execute_async(statement)
|
||||
|
||||
async def set_password(self, user_email: str, new_password: str, current_password: str | None = None) -> None:
|
||||
"""Set or change password for a user"""
|
||||
@@ -246,12 +763,19 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def bind_space_account(self, user_email: str, code: str) -> user.User:
|
||||
"""Bind Space account to existing local account"""
|
||||
local_account = await self.get_user_by_email(user_email)
|
||||
if local_account is None:
|
||||
raise ValueError('User not found')
|
||||
# Exchange code for tokens
|
||||
token_data = await self.ap.space_service.exchange_oauth_code(code)
|
||||
access_token = token_data.get('access_token')
|
||||
@@ -273,28 +797,33 @@ class UserService:
|
||||
|
||||
if not space_account_uuid or not space_email:
|
||||
raise ValueError('Invalid Space user info')
|
||||
if normalize_email(space_email) != normalize_email(user_email):
|
||||
raise account_errors.AccountEmailMismatchError()
|
||||
|
||||
# Check if this Space account is already bound to another user
|
||||
existing_space_user = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if existing_space_user and existing_space_user.user != user_email:
|
||||
if existing_space_user and existing_space_user.normalized_email != normalize_email(user_email):
|
||||
raise ValueError('This Space account is already bound to another user')
|
||||
|
||||
# Update local account to Space account
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.user == user_email)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(
|
||||
user=space_email, # Update email to Space email
|
||||
user=normalize_email(space_email), # Update email to Space email
|
||||
normalized_email=normalize_email(space_email),
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Update Space model provider API keys
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
await self._update_space_provider_for_account(local_account, api_key)
|
||||
|
||||
return await self.get_user_by_email(space_email)
|
||||
|
||||
@@ -4,6 +4,12 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import webhook
|
||||
from .secrets import SECRET_MASK, mask_secret_value, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
_DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE = 16
|
||||
_HARD_MAX_WEBHOOKS_PER_WORKSPACE = 64
|
||||
|
||||
|
||||
class WebhookService:
|
||||
@@ -12,31 +18,99 @@ class WebhookService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_webhooks(self) -> list[dict]:
|
||||
def max_per_workspace(self) -> int:
|
||||
"""Return the configured webhook cap within the process hard limit."""
|
||||
|
||||
config = getattr(getattr(self.ap, 'instance_config', None), 'data', {})
|
||||
try:
|
||||
value = int(
|
||||
config.get('webhooks', {}).get(
|
||||
'max_per_workspace',
|
||||
_DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE,
|
||||
)
|
||||
)
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
value = _DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE
|
||||
return min(max(value, 1), _HARD_MAX_WEBHOOKS_PER_WORKSPACE)
|
||||
|
||||
def _serialize_webhook(self, entity, *, include_secret: bool) -> dict:
|
||||
serialized = self.ap.persistence_mgr.serialize_model(webhook.Webhook, entity)
|
||||
if not include_secret:
|
||||
serialized = serialized.copy()
|
||||
serialized['url'] = mask_secret_value(serialized.get('url'))
|
||||
return serialized
|
||||
|
||||
async def get_webhooks(self, context: TenantContext, *, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all webhooks"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(webhook.Webhook))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).order_by(webhook.Webhook.id).limit(_HARD_MAX_WEBHOOKS_PER_WORKSPACE),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
webhooks = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(webhook.Webhook, wh) for wh in webhooks]
|
||||
return [self._serialize_webhook(wh, include_secret=include_secret) for wh in webhooks]
|
||||
|
||||
async def create_webhook(self, name: str, url: str, description: str = '', enabled: bool = True) -> dict:
|
||||
async def create_webhook(
|
||||
self,
|
||||
context: TenantContext,
|
||||
name: str,
|
||||
url: str,
|
||||
description: str = '',
|
||||
enabled: bool = True,
|
||||
) -> dict:
|
||||
"""Create a new webhook"""
|
||||
webhook_data = {'name': name, 'url': url, 'description': description, 'enabled': enabled}
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
max_webhooks = self.max_per_workspace()
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(webhook.Webhook)
|
||||
.where(webhook.Webhook.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
if (count_result.scalar() or 0) >= max_webhooks:
|
||||
raise ValueError(f'Maximum number of webhooks ({max_webhooks}) reached')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(webhook.Webhook).values(**webhook_data))
|
||||
url = restore_secret_placeholders(url, sensitive=True)
|
||||
webhook_data = {
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'name': name,
|
||||
'url': url,
|
||||
'description': description,
|
||||
'enabled': enabled,
|
||||
}
|
||||
|
||||
insert_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(webhook.Webhook).values(**webhook_data)
|
||||
)
|
||||
|
||||
# Retrieve the created webhook
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.url == url).order_by(webhook.Webhook.id.desc())
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == insert_result.inserted_primary_key[0]),
|
||||
webhook.Webhook,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
created_webhook = result.first()
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(webhook.Webhook, created_webhook)
|
||||
|
||||
async def get_webhook(self, webhook_id: int) -> dict | None:
|
||||
async def get_webhook(
|
||||
self,
|
||||
context: TenantContext,
|
||||
webhook_id: int,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a specific webhook by ID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == webhook_id)
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == webhook_id),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
wh = result.first()
|
||||
@@ -44,16 +118,27 @@ class WebhookService:
|
||||
if wh is None:
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(webhook.Webhook, wh)
|
||||
return self._serialize_webhook(wh, include_secret=include_secret)
|
||||
|
||||
async def update_webhook(
|
||||
self, webhook_id: int, name: str = None, url: str = None, description: str = None, enabled: bool = None
|
||||
) -> None:
|
||||
self,
|
||||
context: TenantContext,
|
||||
webhook_id: int,
|
||||
name: str | None = None,
|
||||
url: str | None = None,
|
||||
description: str | None = None,
|
||||
enabled: bool | None = None,
|
||||
) -> bool:
|
||||
"""Update a webhook's metadata"""
|
||||
update_data = {}
|
||||
if name is not None:
|
||||
update_data['name'] = name
|
||||
if url is not None:
|
||||
if url == SECRET_MASK:
|
||||
current = await self.get_webhook(context, webhook_id, include_secret=True)
|
||||
if current is None:
|
||||
return False
|
||||
url = restore_secret_placeholders(url, current.get('url'), sensitive=True)
|
||||
update_data['url'] = url
|
||||
if description is not None:
|
||||
update_data['description'] = description
|
||||
@@ -61,20 +146,37 @@ class WebhookService:
|
||||
update_data['enabled'] = enabled
|
||||
|
||||
if update_data:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(webhook.Webhook).where(webhook.Webhook.id == webhook_id).values(**update_data)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(webhook.Webhook).where(webhook.Webhook.id == webhook_id).values(**update_data),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return (result.rowcount or 0) > 0
|
||||
return await self.get_webhook(context, webhook_id) is not None
|
||||
|
||||
async def delete_webhook(self, webhook_id: int) -> None:
|
||||
async def delete_webhook(self, context: TenantContext, webhook_id: int) -> bool:
|
||||
"""Delete a webhook"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(webhook.Webhook).where(webhook.Webhook.id == webhook_id)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(webhook.Webhook).where(webhook.Webhook.id == webhook_id),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def get_enabled_webhooks(self) -> list[dict]:
|
||||
async def get_enabled_webhooks(self, context: TenantContext) -> list[dict]:
|
||||
"""Get all enabled webhooks"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.enabled == True)
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.enabled == True),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
.order_by(webhook.Webhook.id)
|
||||
.limit(self.max_per_workspace())
|
||||
)
|
||||
|
||||
webhooks = result.all()
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
|
||||
from ..http.context import RequestContext
|
||||
|
||||
|
||||
_request_context: contextvars.ContextVar[RequestContext | None] = contextvars.ContextVar(
|
||||
'langbot_mcp_request_context',
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def bind_request_context(context: RequestContext) -> contextvars.Token[RequestContext | None]:
|
||||
"""Bind the authenticated MCP request while its ASGI request is executing."""
|
||||
|
||||
return _request_context.set(context)
|
||||
|
||||
|
||||
def reset_request_context(token: contextvars.Token[RequestContext | None]) -> None:
|
||||
_request_context.reset(token)
|
||||
|
||||
|
||||
def get_request_context() -> RequestContext:
|
||||
"""Return the current trusted MCP context or fail closed."""
|
||||
|
||||
context = _request_context.get()
|
||||
if context is None:
|
||||
raise RuntimeError('MCP Workspace context is unavailable')
|
||||
return context
|
||||
@@ -19,7 +19,10 @@ from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
from ..http.context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from .context import bind_request_context, reset_request_context
|
||||
from .server import LangBotMCPServer
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -28,6 +31,9 @@ if typing.TYPE_CHECKING:
|
||||
|
||||
# JSON-RPC-ish 401 body returned before the MCP app is reached.
|
||||
_UNAUTHORIZED_BODY = b'{"error":"unauthorized","message":"A valid LangBot API key is required for MCP access."}'
|
||||
_ENTITLEMENT_UNAVAILABLE_BODY = (
|
||||
b'{"error":"entitlement_unavailable","message":"Workspace entitlement is unavailable for MCP access."}'
|
||||
)
|
||||
|
||||
|
||||
def _extract_api_key(headers: list[tuple[bytes, bytes]]) -> str:
|
||||
@@ -76,7 +82,7 @@ class MCPMount:
|
||||
def wrap(self, quart_asgi: typing.Callable) -> typing.Callable:
|
||||
"""Return a dispatcher ASGI app fronting ``quart_asgi``."""
|
||||
mcp_asgi = self._mcp_asgi
|
||||
verify_api_key = self.ap.apikey_service.verify_api_key
|
||||
authenticate_api_key = self.ap.apikey_service.authenticate_api_key
|
||||
is_mcp_path = self._is_mcp_path
|
||||
|
||||
async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def]
|
||||
@@ -88,12 +94,12 @@ class MCPMount:
|
||||
|
||||
# Authenticate MCP HTTP requests with a LangBot API key.
|
||||
api_key = _extract_api_key(scope.get('headers', []))
|
||||
authorized = False
|
||||
identity = None
|
||||
if api_key:
|
||||
with contextlib.suppress(Exception):
|
||||
authorized = await verify_api_key(api_key)
|
||||
identity = await authenticate_api_key(api_key)
|
||||
|
||||
if not authorized:
|
||||
if identity is None:
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
@@ -107,6 +113,56 @@ class MCPMount:
|
||||
await send({'type': 'http.response.body', 'body': _UNAUTHORIZED_BODY})
|
||||
return
|
||||
|
||||
await mcp_asgi(scope, receive, send)
|
||||
deployment_admission = getattr(self.ap, 'deployment_admission', None)
|
||||
try:
|
||||
if deployment_admission is not None:
|
||||
deployment_admission.require_active()
|
||||
entitlement_revision = 0
|
||||
deployment = getattr(self.ap, 'deployment', None)
|
||||
if deployment is not None and getattr(deployment, 'multi_workspace_enabled', False):
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None or identity.instance_uuid != resolver.instance_uuid:
|
||||
raise RuntimeError('Workspace entitlement resolver is unavailable')
|
||||
entitlement = await resolver.resolve(identity.workspace_uuid)
|
||||
entitlement_revision = entitlement.entitlement_revision
|
||||
except Exception:
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
'status': 403,
|
||||
'headers': [(b'content-type', b'application/json')],
|
||||
}
|
||||
)
|
||||
await send({'type': 'http.response.body', 'body': _ENTITLEMENT_UNAVAILABLE_BODY})
|
||||
return
|
||||
|
||||
request_context = RequestContext(
|
||||
instance_uuid=identity.instance_uuid,
|
||||
placement_generation=identity.placement_generation,
|
||||
request_id=str(uuid.uuid4()),
|
||||
auth_type='api-key',
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.API_KEY,
|
||||
api_key_uuid=identity.api_key_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=identity.workspace_uuid,
|
||||
membership_uuid=None,
|
||||
role=None,
|
||||
permissions=identity.permissions,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('MCP request persistence scope is unavailable')
|
||||
async with tenant_scope(identity.workspace_uuid):
|
||||
token = bind_request_context(request_context)
|
||||
try:
|
||||
await mcp_asgi(scope, receive, send)
|
||||
if deployment_admission is not None:
|
||||
deployment_admission.require_active()
|
||||
finally:
|
||||
reset_request_context(token)
|
||||
|
||||
return dispatcher
|
||||
|
||||
@@ -22,6 +22,9 @@ import typing
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
from ..http.authz import Permission, require_permission
|
||||
from .context import get_request_context
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ...core import app as app_module
|
||||
|
||||
@@ -46,6 +49,12 @@ def _dump(value: typing.Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _authorized(permission: Permission):
|
||||
context = get_request_context()
|
||||
require_permission(context, permission)
|
||||
return context
|
||||
|
||||
|
||||
class LangBotMCPServer:
|
||||
"""Builds and owns the FastMCP instance for LangBot."""
|
||||
|
||||
@@ -72,6 +81,7 @@ class LangBotMCPServer:
|
||||
# ----- System (read-only) -------------------------------------- #
|
||||
@mcp.tool(description='Get basic LangBot system/runtime information (version, edition).')
|
||||
async def get_system_info() -> str:
|
||||
_authorized(Permission.WORKSPACE_VIEW)
|
||||
version = None
|
||||
try:
|
||||
version = ap.ver_mgr.get_current_version()
|
||||
@@ -87,11 +97,13 @@ class LangBotMCPServer:
|
||||
# ----- Bots ---------------------------------------------------- #
|
||||
@mcp.tool(description='List all messaging-platform bots. Secrets are redacted.')
|
||||
async def list_bots() -> str:
|
||||
return _dump(await ap.bot_service.get_bots(include_secret=False))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.bot_service.get_bots(context, include_secret=False))
|
||||
|
||||
@mcp.tool(description='Get a single bot by its UUID. Secrets are redacted.')
|
||||
async def get_bot(bot_uuid: str) -> str:
|
||||
return _dump(await ap.bot_service.get_bot(bot_uuid, include_secret=False))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.bot_service.get_bot(context, bot_uuid, include_secret=False))
|
||||
|
||||
@mcp.tool(
|
||||
description=(
|
||||
@@ -101,26 +113,31 @@ class LangBotMCPServer:
|
||||
)
|
||||
)
|
||||
async def create_bot(bot_data: dict) -> str:
|
||||
return _dump({'uuid': await ap.bot_service.create_bot(bot_data)})
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
return _dump({'uuid': await ap.bot_service.create_bot(context, bot_data)})
|
||||
|
||||
@mcp.tool(description='Update a bot by UUID. `bot_data` matches the PUT bot body.')
|
||||
async def update_bot(bot_uuid: str, bot_data: dict) -> str:
|
||||
await ap.bot_service.update_bot(bot_uuid, bot_data)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.bot_service.update_bot(context, bot_uuid, bot_data)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(description='Delete a bot by UUID.')
|
||||
async def delete_bot(bot_uuid: str) -> str:
|
||||
await ap.bot_service.delete_bot(bot_uuid)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.bot_service.delete_bot(context, bot_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
# ----- Pipelines ----------------------------------------------- #
|
||||
@mcp.tool(description='List all pipelines.')
|
||||
async def list_pipelines() -> str:
|
||||
return _dump(await ap.pipeline_service.get_pipelines())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.pipeline_service.get_pipelines(context))
|
||||
|
||||
@mcp.tool(description='Get a single pipeline by UUID.')
|
||||
async def get_pipeline(pipeline_uuid: str) -> str:
|
||||
return _dump(await ap.pipeline_service.get_pipeline(pipeline_uuid))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.pipeline_service.get_pipeline(context, pipeline_uuid))
|
||||
|
||||
@mcp.tool(
|
||||
description=(
|
||||
@@ -129,49 +146,59 @@ class LangBotMCPServer:
|
||||
)
|
||||
)
|
||||
async def create_pipeline(pipeline_data: dict) -> str:
|
||||
return _dump({'uuid': await ap.pipeline_service.create_pipeline(pipeline_data)})
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
return _dump({'uuid': await ap.pipeline_service.create_pipeline(context, pipeline_data)})
|
||||
|
||||
@mcp.tool(description='Update a pipeline by UUID. `pipeline_data` matches the PUT body.')
|
||||
async def update_pipeline(pipeline_uuid: str, pipeline_data: dict) -> str:
|
||||
await ap.pipeline_service.update_pipeline(pipeline_uuid, pipeline_data)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.pipeline_service.update_pipeline(context, pipeline_uuid, pipeline_data)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(description='Delete a pipeline by UUID.')
|
||||
async def delete_pipeline(pipeline_uuid: str) -> str:
|
||||
await ap.pipeline_service.delete_pipeline(pipeline_uuid)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.pipeline_service.delete_pipeline(context, pipeline_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
# ----- Models -------------------------------------------------- #
|
||||
@mcp.tool(description='List all configured LLM models. Secrets are redacted.')
|
||||
async def list_llm_models() -> str:
|
||||
return _dump(await ap.llm_model_service.get_llm_models(include_secret=False))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.llm_model_service.get_llm_models(context, include_secret=False))
|
||||
|
||||
@mcp.tool(description='Get a single LLM model by UUID.')
|
||||
async def get_llm_model(model_uuid: str) -> str:
|
||||
return _dump(await ap.llm_model_service.get_llm_model(model_uuid))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.llm_model_service.get_llm_model(context, model_uuid, include_secret=False))
|
||||
|
||||
@mcp.tool(description='List all configured embedding models.')
|
||||
async def list_embedding_models() -> str:
|
||||
return _dump(await ap.embedding_models_service.get_embedding_models())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.embedding_models_service.get_embedding_models(context, include_secret=False))
|
||||
|
||||
@mcp.tool(description='List all model providers (OpenAI-compatible, Anthropic, etc.).')
|
||||
async def list_model_providers() -> str:
|
||||
return _dump(await ap.provider_service.get_providers())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.provider_service.get_providers(context, include_secret=False))
|
||||
|
||||
# ----- Knowledge bases ----------------------------------------- #
|
||||
@mcp.tool(description='List all knowledge bases (RAG).')
|
||||
async def list_knowledge_bases() -> str:
|
||||
return _dump(await ap.knowledge_service.get_knowledge_bases())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.knowledge_service.get_knowledge_bases(context))
|
||||
|
||||
@mcp.tool(description='Get a single knowledge base by UUID.')
|
||||
async def get_knowledge_base(kb_uuid: str) -> str:
|
||||
return _dump(await ap.knowledge_service.get_knowledge_base(kb_uuid))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.knowledge_service.get_knowledge_base(context, kb_uuid))
|
||||
|
||||
@mcp.tool(
|
||||
description=('Retrieve (semantic search) from a knowledge base. Returns the matched chunks for `query`.')
|
||||
)
|
||||
async def retrieve_knowledge_base(kb_uuid: str, query: str) -> str:
|
||||
return _dump(await ap.knowledge_service.retrieve_knowledge_base(kb_uuid, query))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.knowledge_service.retrieve_knowledge_base(context, kb_uuid, query))
|
||||
|
||||
# ----- MCP servers (LangBot as MCP client) --------------------- #
|
||||
@mcp.tool(
|
||||
@@ -180,16 +207,19 @@ class LangBotMCPServer:
|
||||
)
|
||||
)
|
||||
async def list_mcp_servers() -> str:
|
||||
return _dump(await ap.mcp_service.get_mcp_servers())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.mcp_service.get_mcp_servers(context))
|
||||
|
||||
# ----- Skills -------------------------------------------------- #
|
||||
@mcp.tool(description='List installed skills.')
|
||||
async def list_skills() -> str:
|
||||
return _dump(await ap.skill_service.list_skills())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.skill_service.list_skills(context))
|
||||
|
||||
@mcp.tool(description='Get a single skill by name.')
|
||||
async def get_skill(skill_name: str) -> str:
|
||||
return _dump(await ap.skill_service.get_skill(skill_name))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.skill_service.get_skill(context, skill_name))
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# ASGI app
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from langbot_plugin.box.errors import BoxAdmissionError, BoxRuntimeUnavailableError
|
||||
from langbot_plugin.box.models import (
|
||||
SandboxAdmissionGrant,
|
||||
SandboxAdmissionPolicy,
|
||||
SandboxAdmissionRevocation,
|
||||
)
|
||||
|
||||
from ..api.http.context import ExecutionContext
|
||||
from ..cloud.entitlements import EntitlementSnapshot, EntitlementUnavailableError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langbot_plugin.box.client import BoxRuntimeClient
|
||||
|
||||
from ..core.app import Application
|
||||
|
||||
|
||||
_UTC = dt.timezone.utc
|
||||
_MANAGED_SANDBOX_FEATURE = 'managed_sandbox'
|
||||
_MANAGED_SANDBOX_SESSION_LIMIT = 'managed_sandbox_sessions'
|
||||
_MAX_GRANT_TTL_SEC = 300
|
||||
|
||||
|
||||
class SandboxAdmissionController:
|
||||
"""Project Cloud entitlements into short-lived Box Runtime grants.
|
||||
|
||||
Product and plan names intentionally never cross this boundary. The
|
||||
closed Control Plane supplies a versioned generic entitlement, while Core
|
||||
installs only the numeric authority understood by the shared Box Runtime.
|
||||
|
||||
No state is allocated for a Workspace until it attempts to use the
|
||||
managed sandbox. Per-Workspace locks serialize renewal/revocation so a
|
||||
concurrent first use cannot install conflicting grants.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: Application,
|
||||
client: BoxRuntimeClient,
|
||||
*,
|
||||
policy: SandboxAdmissionPolicy,
|
||||
wall_time: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self.client = client
|
||||
self.policy = policy
|
||||
self._wall_time = wall_time
|
||||
self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary()
|
||||
self._highest_revisions: dict[str, int] = {}
|
||||
|
||||
def _workspace_lock(self, workspace_uuid: str) -> asyncio.Lock:
|
||||
lock = self._locks.get(workspace_uuid)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
self._locks[workspace_uuid] = lock
|
||||
return lock
|
||||
|
||||
@staticmethod
|
||||
def _context_revision(context: ExecutionContext) -> int:
|
||||
revision = getattr(context, 'entitlement_revision', 0)
|
||||
if isinstance(revision, bool) or not isinstance(revision, int):
|
||||
return 0
|
||||
return max(revision, 0)
|
||||
|
||||
def _revocation_revision(self, context: ExecutionContext, candidate_revision: int = 0) -> int:
|
||||
return max(
|
||||
1,
|
||||
self._highest_revisions.get(context.workspace_uuid, 0),
|
||||
self._context_revision(context),
|
||||
candidate_revision,
|
||||
)
|
||||
|
||||
async def _revoke_locked(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
*,
|
||||
candidate_revision: int = 0,
|
||||
) -> None:
|
||||
revision = self._revocation_revision(context, candidate_revision)
|
||||
revocation = SandboxAdmissionRevocation(
|
||||
instance_uuid=context.instance_uuid,
|
||||
workspace_uuid=context.workspace_uuid,
|
||||
entitlement_revision=revision,
|
||||
)
|
||||
try:
|
||||
result = await self.client.revoke_sandbox_admission_grant(revocation)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or result.get('revoked') is not True
|
||||
or result.get('workspace_uuid') != context.workspace_uuid
|
||||
or result.get('entitlement_revision') != revision
|
||||
):
|
||||
raise BoxRuntimeUnavailableError('Box Runtime returned an invalid sandbox revocation receipt')
|
||||
except Exception as exc:
|
||||
# The caller still fails closed even if the control connection is
|
||||
# unavailable. A previously installed grant expires independently
|
||||
# in at most five minutes inside the Runtime.
|
||||
self.ap.logger.warning(
|
||||
'Failed to install Box sandbox admission revocation: '
|
||||
f'workspace_uuid={context.workspace_uuid} revision={revision} error={exc}'
|
||||
)
|
||||
self._highest_revisions[context.workspace_uuid] = revision
|
||||
|
||||
@staticmethod
|
||||
def _require_managed_sandbox(snapshot: EntitlementSnapshot) -> None:
|
||||
snapshot.require_feature(_MANAGED_SANDBOX_FEATURE)
|
||||
sessions = snapshot.limit(_MANAGED_SANDBOX_SESSION_LIMIT)
|
||||
if sessions != 1:
|
||||
raise EntitlementUnavailableError('Workspace entitlement must grant exactly one managed sandbox session')
|
||||
|
||||
def _grant_expiry(self, snapshot: EntitlementSnapshot) -> dt.datetime:
|
||||
now_epoch = int(self._wall_time())
|
||||
ttl_sec = min(self.policy.max_grant_ttl_sec, _MAX_GRANT_TTL_SEC)
|
||||
expires_epoch = min(snapshot.expires_at, now_epoch + ttl_sec)
|
||||
if expires_epoch <= now_epoch:
|
||||
raise EntitlementUnavailableError('Workspace entitlement expired before sandbox admission')
|
||||
return dt.datetime.fromtimestamp(expires_epoch, tz=_UTC)
|
||||
|
||||
async def require(self, context: ExecutionContext) -> SandboxAdmissionGrant:
|
||||
"""Validate entitlement freshness and install/renew one Runtime grant."""
|
||||
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None:
|
||||
raise EntitlementUnavailableError('Workspace entitlement resolver is unavailable')
|
||||
if context.instance_uuid != resolver.instance_uuid:
|
||||
raise EntitlementUnavailableError('Workspace entitlement targets another LangBot instance')
|
||||
|
||||
lock = self._workspace_lock(context.workspace_uuid)
|
||||
async with lock:
|
||||
try:
|
||||
snapshot = await resolver.resolve(
|
||||
context.workspace_uuid,
|
||||
minimum_revision=self._context_revision(context),
|
||||
now=int(self._wall_time()),
|
||||
)
|
||||
except EntitlementUnavailableError as exc:
|
||||
# Only a verified, scoped snapshot can authoritatively revoke
|
||||
# a revision. Provider timeouts, malformed responses, and
|
||||
# rollback/equivocation errors fail this request closed but do
|
||||
# not tombstone a still-valid revision forever.
|
||||
authoritative_revision = exc.entitlement_revision
|
||||
if authoritative_revision is not None:
|
||||
await self._revoke_locked(
|
||||
context,
|
||||
candidate_revision=authoritative_revision,
|
||||
)
|
||||
raise
|
||||
|
||||
try:
|
||||
self._require_managed_sandbox(snapshot)
|
||||
except EntitlementUnavailableError:
|
||||
await self._revoke_locked(
|
||||
context,
|
||||
candidate_revision=snapshot.entitlement_revision,
|
||||
)
|
||||
raise
|
||||
|
||||
grant = SandboxAdmissionGrant(
|
||||
instance_uuid=context.instance_uuid,
|
||||
workspace_uuid=context.workspace_uuid,
|
||||
execution_generation=context.placement_generation,
|
||||
entitlement_revision=snapshot.entitlement_revision,
|
||||
expires_at=self._grant_expiry(snapshot),
|
||||
max_sessions=1,
|
||||
max_managed_processes=0,
|
||||
)
|
||||
result = await self.client.upsert_sandbox_admission_grant(grant)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or result.get('installed') is not True
|
||||
or result.get('workspace_uuid') != context.workspace_uuid
|
||||
or result.get('execution_generation') != context.placement_generation
|
||||
or result.get('entitlement_revision') != snapshot.entitlement_revision
|
||||
or result.get('max_sessions') != 1
|
||||
or result.get('max_managed_processes') != 0
|
||||
):
|
||||
raise BoxRuntimeUnavailableError('Box Runtime returned an invalid sandbox admission receipt')
|
||||
self._highest_revisions[context.workspace_uuid] = max(
|
||||
self._highest_revisions.get(context.workspace_uuid, 0),
|
||||
snapshot.entitlement_revision,
|
||||
)
|
||||
return grant
|
||||
|
||||
async def revoke(self, context: ExecutionContext, *, entitlement_revision: int = 0) -> None:
|
||||
"""Explicitly revoke a Workspace grant using a monotonic tombstone."""
|
||||
|
||||
async with self._workspace_lock(context.workspace_uuid):
|
||||
await self._revoke_locked(context, candidate_revision=entitlement_revision)
|
||||
|
||||
|
||||
def require_cloud_admission_policy(raw_policy: object) -> SandboxAdmissionPolicy:
|
||||
"""Parse the Cloud Box policy without permitting an OSS downgrade."""
|
||||
|
||||
try:
|
||||
policy = SandboxAdmissionPolicy.model_validate(raw_policy)
|
||||
except Exception as exc:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission policy is invalid') from exc
|
||||
if not policy.required:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission must be required')
|
||||
if policy.logical_session_id != 'global':
|
||||
raise BoxAdmissionError('Cloud Box sandbox session ID must be global')
|
||||
if policy.required_backend != 'nsjail':
|
||||
raise BoxAdmissionError('Cloud Box sandbox backend must be nsjail')
|
||||
if policy.max_sessions != 1 or policy.max_managed_processes != 0:
|
||||
raise BoxAdmissionError('Cloud Box sandbox policy must allow one session and zero managed processes')
|
||||
if policy.max_grant_ttl_sec > _MAX_GRANT_TTL_SEC:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission grant TTL must not exceed 300 seconds')
|
||||
if policy.workspace_quota_mb <= 0:
|
||||
raise BoxAdmissionError('Cloud Box sandbox workspace quota must be a positive integer')
|
||||
return policy
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
import typing
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -16,6 +17,17 @@ from langbot_plugin.runtime.io.connection import Connection
|
||||
from langbot_plugin.box.client import ActionRPCBoxClient
|
||||
from langbot_plugin.box.errors import BoxRuntimeUnavailableError
|
||||
from langbot_plugin.box.actions import LangBotToBoxAction
|
||||
from langbot_plugin.box.security import (
|
||||
BOX_CONTROL_TOKEN_ENV,
|
||||
BOX_CONTROL_TOKEN_HEADER,
|
||||
BOX_INSTANCE_HEADER,
|
||||
BOX_PLACEMENT_GENERATION_HEADER,
|
||||
BOX_TRUSTED_INSTANCE_ENV,
|
||||
BOX_WORKSPACE_HEADER,
|
||||
normalize_instance_uuid,
|
||||
validate_control_token,
|
||||
)
|
||||
from langbot_plugin.entities.io.context import ActionContext
|
||||
|
||||
from ..utils import platform
|
||||
from ..utils.managed_runtime import ManagedRuntimeConnector
|
||||
@@ -123,6 +135,8 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
self._relay_host = parsed.hostname or '127.0.0.1'
|
||||
self._relay_port = parsed.port or _DEFAULT_PORT
|
||||
self._filtered_box_config = _filter_config_for_runtime(_get_box_config(ap))
|
||||
self._trusted_instance_uuid = normalize_instance_uuid(self.ap.workspace_service.instance_uuid)
|
||||
self._control_token = str(os.environ.get(BOX_CONTROL_TOKEN_ENV) or '').strip()
|
||||
|
||||
def uses_websocket(self) -> bool:
|
||||
"""Whether the connector should use WebSocket to reach the Box runtime.
|
||||
@@ -223,8 +237,11 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
from langbot_plugin.runtime.io.controllers.stdio.client import StdioClientController
|
||||
|
||||
self.ap.logger.info('Use stdio to connect to box runtime')
|
||||
self._ensure_control_token(allow_generate=True)
|
||||
python_path = sys.executable
|
||||
env = os.environ.copy()
|
||||
env[BOX_CONTROL_TOKEN_ENV] = self._control_token
|
||||
env[BOX_TRUSTED_INSTANCE_ENV] = self._trusted_instance_uuid
|
||||
if self._filtered_box_config:
|
||||
env['LANGBOT_BOX_CONFIG'] = json.dumps(self._filtered_box_config)
|
||||
|
||||
@@ -259,7 +276,10 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
"""Launch box server as detached subprocess, then connect via WS (Windows)."""
|
||||
self.ap.logger.info('(windows) Use cmd to launch box runtime and communicate via ws')
|
||||
|
||||
self._ensure_control_token(allow_generate=True)
|
||||
env = os.environ.copy()
|
||||
env[BOX_CONTROL_TOKEN_ENV] = self._control_token
|
||||
env[BOX_TRUSTED_INSTANCE_ENV] = self._trusted_instance_uuid
|
||||
if self._filtered_box_config:
|
||||
env['LANGBOT_BOX_CONFIG'] = json.dumps(self._filtered_box_config)
|
||||
|
||||
@@ -282,6 +302,7 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
|
||||
async def _connect_remote_ws(self) -> None:
|
||||
"""Connect to a remote (or Docker) box server via WebSocket."""
|
||||
self._ensure_control_token(allow_generate=False)
|
||||
ws_url = self._resolve_rpc_ws_url()
|
||||
self.ap.logger.info(f'Use WebSocket to connect to box runtime ({ws_url})')
|
||||
await self._connect_ws(ws_url, 'WebSocket')
|
||||
@@ -325,7 +346,11 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
if self.runtime_disconnect_callback is not None:
|
||||
await self.runtime_disconnect_callback(self)
|
||||
|
||||
ctrl = WebSocketClientController(ws_url=ws_url, make_connection_failed_callback=on_connect_failed)
|
||||
ctrl = WebSocketClientController(
|
||||
ws_url=ws_url,
|
||||
make_connection_failed_callback=on_connect_failed,
|
||||
additional_headers=self.get_control_headers(),
|
||||
)
|
||||
self._ctrl = ctrl
|
||||
self._ctrl_task = asyncio.create_task(
|
||||
ctrl.run(self._make_connection_callback(transport_name, connected, connect_error, self._generation))
|
||||
@@ -339,6 +364,41 @@ class BoxRuntimeConnector(ManagedRuntimeConnector):
|
||||
if connect_error:
|
||||
raise BoxRuntimeUnavailableError(f'box runtime connection failed: {connect_error[0]}')
|
||||
|
||||
def _ensure_control_token(self, *, allow_generate: bool) -> str:
|
||||
if not self._control_token and allow_generate:
|
||||
self._control_token = secrets.token_urlsafe(48)
|
||||
try:
|
||||
self._control_token = validate_control_token(self._control_token)
|
||||
except ValueError as exc:
|
||||
raise BoxRuntimeUnavailableError(
|
||||
f'{BOX_CONTROL_TOKEN_ENV} must be configured with a strong shared secret for an external Box runtime'
|
||||
) from exc
|
||||
return self._control_token
|
||||
|
||||
def get_control_headers(self) -> dict[str, str]:
|
||||
"""Headers for the instance-authenticated RPC control handshake."""
|
||||
|
||||
self._ensure_control_token(allow_generate=False)
|
||||
return {
|
||||
BOX_CONTROL_TOKEN_HEADER: self._control_token,
|
||||
BOX_INSTANCE_HEADER: self._trusted_instance_uuid,
|
||||
}
|
||||
|
||||
def get_relay_headers(
|
||||
self,
|
||||
action_context: ActionContext,
|
||||
) -> dict[str, str]:
|
||||
"""Return authenticated, placement-scoped relay handshake headers."""
|
||||
|
||||
context = ActionContext.model_validate(action_context).without_installation()
|
||||
if context.instance_uuid != self._trusted_instance_uuid:
|
||||
raise BoxRuntimeUnavailableError('Box relay context belongs to another LangBot instance')
|
||||
return {
|
||||
**self.get_control_headers(),
|
||||
BOX_WORKSPACE_HEADER: context.workspace_uuid,
|
||||
BOX_PLACEMENT_GENERATION_HEADER: str(context.placement_generation),
|
||||
}
|
||||
|
||||
def _make_connection_callback(
|
||||
self,
|
||||
transport_name: str,
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class UnsafeWorkspacePathError(OSError):
|
||||
"""A tenant-controlled path could not be opened without following links."""
|
||||
|
||||
|
||||
_DIRECTORY_FLAGS = (
|
||||
os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
)
|
||||
_FILE_READ_FLAGS = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
_FILE_WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
_MAX_REMOVAL_ENTRIES = 4096
|
||||
_MAX_REMOVAL_DEPTH = 16
|
||||
|
||||
|
||||
def _component(value: str) -> str:
|
||||
normalized = str(value or '').strip()
|
||||
if (
|
||||
not normalized
|
||||
or normalized in {'.', '..'}
|
||||
or '/' in normalized
|
||||
or '\\' in normalized
|
||||
or '\x00' in normalized
|
||||
or len(os.fsencode(normalized)) > 240
|
||||
):
|
||||
raise UnsafeWorkspacePathError('Unsafe Workspace path component')
|
||||
return normalized
|
||||
|
||||
|
||||
def _unsafe(path: str, exc: BaseException | None = None) -> UnsafeWorkspacePathError:
|
||||
error = UnsafeWorkspacePathError(f'Workspace path is not a link-free directory: {path}')
|
||||
if exc is not None:
|
||||
error.__cause__ = exc
|
||||
return error
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _root_fd(root: str):
|
||||
try:
|
||||
fd = os.open(root, _DIRECTORY_FLAGS)
|
||||
except OSError as exc:
|
||||
raise _unsafe(root, exc)
|
||||
try:
|
||||
if not stat.S_ISDIR(os.fstat(fd).st_mode):
|
||||
raise _unsafe(root)
|
||||
yield fd
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _open_dir_at(parent_fd: int, name: str, *, create: bool) -> int:
|
||||
name = _component(name)
|
||||
if create:
|
||||
try:
|
||||
os.mkdir(name, mode=0o700, dir_fd=parent_fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
try:
|
||||
fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
||||
except OSError as exc:
|
||||
raise _unsafe(name, exc)
|
||||
if not stat.S_ISDIR(os.fstat(fd).st_mode):
|
||||
os.close(fd)
|
||||
raise _unsafe(name)
|
||||
return fd
|
||||
|
||||
|
||||
def _remove_entry(
|
||||
parent_fd: int,
|
||||
name: str,
|
||||
*,
|
||||
budget: list[int] | None = None,
|
||||
depth: int = 0,
|
||||
) -> None:
|
||||
"""Remove an entry recursively without following a symlink at any depth."""
|
||||
|
||||
name = _component(name)
|
||||
budget = budget if budget is not None else [_MAX_REMOVAL_ENTRIES]
|
||||
if depth > _MAX_REMOVAL_DEPTH or budget[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Workspace cleanup exceeded its inode budget')
|
||||
budget[0] -= 1
|
||||
for _ in range(4):
|
||||
try:
|
||||
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except OSError as exc:
|
||||
if exc.errno not in {errno.ELOOP, errno.ENOTDIR, errno.EACCES}:
|
||||
raise
|
||||
try:
|
||||
os.unlink(name, dir_fd=parent_fd)
|
||||
return
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except IsADirectoryError:
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
_clear_dir(child_fd, budget=budget, depth=depth + 1)
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
try:
|
||||
os.rmdir(name, dir_fd=parent_fd)
|
||||
return
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except NotADirectoryError:
|
||||
continue
|
||||
raise UnsafeWorkspacePathError('Workspace entry changed while it was being removed')
|
||||
|
||||
|
||||
def _clear_dir(directory_fd: int, *, budget: list[int], depth: int) -> None:
|
||||
# ``scandir(fd)`` enumerates the already-open directory. Names are then
|
||||
# resolved relative to the same fd, so a tenant cannot redirect the walk by
|
||||
# swapping an ancestor symlink between validation and use.
|
||||
# Do not materialize the whole directory: an attacker-controlled outbox
|
||||
# may contain an inode bomb even when its byte size is tiny. Removal is
|
||||
# deliberately budgeted and fails closed once the per-operation cap is
|
||||
# reached; hard filesystem/inode quota remains a Cloud readiness gate.
|
||||
with os.scandir(directory_fd) as iterator:
|
||||
for entry in iterator:
|
||||
_remove_entry(directory_fd, entry.name, budget=budget, depth=depth)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _query_fd(root: str, subdir: str, query_key: str, *, create: bool, reset: bool = False):
|
||||
subdir = _component(subdir)
|
||||
query_key = _component(query_key)
|
||||
with _root_fd(root) as root_fd:
|
||||
subdir_fd = _open_dir_at(root_fd, subdir, create=create)
|
||||
try:
|
||||
if reset:
|
||||
_remove_entry(subdir_fd, query_key)
|
||||
query_fd = _open_dir_at(subdir_fd, query_key, create=create)
|
||||
try:
|
||||
yield query_fd
|
||||
finally:
|
||||
os.close(query_fd)
|
||||
finally:
|
||||
os.close(subdir_fd)
|
||||
|
||||
|
||||
def write_files(
|
||||
root: str,
|
||||
subdir: str,
|
||||
query_key: str,
|
||||
files: Iterable[tuple[str, bytes]],
|
||||
) -> None:
|
||||
"""Atomically recreate one query directory and write regular files only."""
|
||||
|
||||
with _query_fd(root, subdir, query_key, create=True, reset=True) as query_fd:
|
||||
for raw_name, data in files:
|
||||
name = _component(raw_name)
|
||||
try:
|
||||
file_fd = os.open(name, _FILE_WRITE_FLAGS, 0o600, dir_fd=query_fd)
|
||||
except OSError as exc:
|
||||
raise UnsafeWorkspacePathError(f'Could not create a link-free Workspace file: {name}') from exc
|
||||
with os.fdopen(file_fd, 'wb') as file_obj:
|
||||
file_obj.write(data)
|
||||
|
||||
|
||||
def _read_directory(
|
||||
directory_fd: int,
|
||||
*,
|
||||
prefix: str,
|
||||
max_file_bytes: int,
|
||||
max_files: int,
|
||||
max_total_bytes: int,
|
||||
output: list[tuple[str, bytes]],
|
||||
total: list[int],
|
||||
remaining_entries: list[int],
|
||||
remaining_directories: list[int],
|
||||
depth: int,
|
||||
) -> None:
|
||||
if depth > 8:
|
||||
return
|
||||
with os.scandir(directory_fd) as iterator:
|
||||
for entry in iterator:
|
||||
if len(output) >= max_files or total[0] >= max_total_bytes:
|
||||
return
|
||||
if remaining_entries[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory-entry limit')
|
||||
remaining_entries[0] -= 1
|
||||
name = _component(entry.name)
|
||||
relative = f'{prefix}/{name}' if prefix else name
|
||||
if entry.is_symlink():
|
||||
continue
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if remaining_directories[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory limit')
|
||||
remaining_directories[0] -= 1
|
||||
try:
|
||||
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=directory_fd)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
_read_directory(
|
||||
child_fd,
|
||||
prefix=relative,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_files=max_files,
|
||||
max_total_bytes=max_total_bytes,
|
||||
output=output,
|
||||
total=total,
|
||||
remaining_entries=remaining_entries,
|
||||
remaining_directories=remaining_directories,
|
||||
depth=depth + 1,
|
||||
)
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
continue
|
||||
try:
|
||||
file_fd = os.open(name, _FILE_READ_FLAGS, dir_fd=directory_fd)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
metadata = os.fstat(file_fd)
|
||||
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > max_file_bytes:
|
||||
continue
|
||||
remaining = max_total_bytes - total[0]
|
||||
if metadata.st_size > remaining:
|
||||
continue
|
||||
with os.fdopen(file_fd, 'rb', closefd=False) as file_obj:
|
||||
data = file_obj.read(max_file_bytes + 1)
|
||||
if len(data) > max_file_bytes or len(data) > remaining:
|
||||
continue
|
||||
output.append((relative, data))
|
||||
total[0] += len(data)
|
||||
finally:
|
||||
os.close(file_fd)
|
||||
|
||||
|
||||
def read_regular_files(
|
||||
root: str,
|
||||
subdir: str,
|
||||
query_key: str,
|
||||
*,
|
||||
max_file_bytes: int,
|
||||
max_files: int,
|
||||
max_total_bytes: int,
|
||||
max_entries: int = 512,
|
||||
max_directories: int = 64,
|
||||
) -> list[tuple[str, bytes]]:
|
||||
"""Read bounded regular files without following tenant-created links."""
|
||||
|
||||
output: list[tuple[str, bytes]] = []
|
||||
try:
|
||||
with _query_fd(root, subdir, query_key, create=False) as query_fd:
|
||||
_read_directory(
|
||||
query_fd,
|
||||
prefix='',
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_files=max_files,
|
||||
max_total_bytes=max_total_bytes,
|
||||
output=output,
|
||||
total=[0],
|
||||
remaining_entries=[max_entries],
|
||||
remaining_directories=[max_directories],
|
||||
depth=0,
|
||||
)
|
||||
except (FileNotFoundError, UnsafeWorkspacePathError):
|
||||
# A missing directory is an empty outbox. An unsafe existing path is
|
||||
# deliberately surfaced to the caller rather than followed.
|
||||
if os.path.lexists(os.path.join(root, subdir, query_key)):
|
||||
raise
|
||||
return output
|
||||
|
||||
|
||||
def reset_directory(root: str, subdir: str, query_key: str) -> None:
|
||||
with _query_fd(root, subdir, query_key, create=True, reset=True):
|
||||
return
|
||||
|
||||
|
||||
def purge_subdirectory(root: str, subdir: str) -> None:
|
||||
"""Remove one known subtree without following a hostile replacement link."""
|
||||
|
||||
with _root_fd(root) as root_fd:
|
||||
_remove_entry(root_fd, _component(subdir))
|
||||
+871
-147
File diff suppressed because it is too large
Load Diff
@@ -126,24 +126,30 @@ def should_prepare_python_env(host_path: str | None) -> bool:
|
||||
return bool(list_python_manifest_files(normalized_root))
|
||||
|
||||
|
||||
def wrap_python_command_with_env(command: str, *, mount_path: str = '/workspace') -> str:
|
||||
def wrap_python_command_with_env(
|
||||
command: str,
|
||||
*,
|
||||
mount_path: str = '/workspace',
|
||||
state_path: str | None = None,
|
||||
) -> str:
|
||||
"""Wrap a command with a reusable sandbox-local Python env bootstrap.
|
||||
|
||||
This is the generic "workspace is a Python project" path used by mutable
|
||||
workspaces such as skills. Read-only installation strategies stay in the
|
||||
higher-level caller because they are application policy, not workspace
|
||||
semantics.
|
||||
``mount_path`` is always the source tree used for manifest hashing and
|
||||
installation. ``state_path`` may point at a separate writable directory
|
||||
for read-only source mounts; when omitted, legacy mutable-workspace behavior
|
||||
stores the environment beside the source.
|
||||
"""
|
||||
writable_state_path = state_path or mount_path
|
||||
bootstrap = textwrap.dedent(
|
||||
f"""
|
||||
set -e
|
||||
|
||||
_LB_VENV_DIR="{mount_path}/.venv"
|
||||
_LB_META_DIR="{mount_path}/.langbot"
|
||||
_LB_VENV_DIR="{writable_state_path}/.venv"
|
||||
_LB_META_DIR="{writable_state_path}/.langbot"
|
||||
_LB_META_FILE="$_LB_META_DIR/python-env.json"
|
||||
_LB_LOCK_DIR="$_LB_META_DIR/python-env.lock"
|
||||
_LB_TMP_DIR="{mount_path}/.tmp"
|
||||
_LB_PIP_CACHE_DIR="{mount_path}/.cache/pip"
|
||||
_LB_TMP_DIR="{writable_state_path}/.tmp"
|
||||
_LB_PIP_CACHE_DIR="{writable_state_path}/.cache/pip"
|
||||
|
||||
mkdir -p "$_LB_META_DIR" "$_LB_TMP_DIR" "$_LB_PIP_CACHE_DIR"
|
||||
_LB_SYSTEM_PYTHON="$(command -v python3 || command -v python || true)"
|
||||
@@ -165,17 +171,23 @@ def wrap_python_command_with_env(command: str, *, mount_path: str = '/workspace'
|
||||
import sys
|
||||
|
||||
root = "{mount_path}"
|
||||
max_manifest_bytes = 10 * 1024 * 1024
|
||||
digest = hashlib.sha256()
|
||||
manifest_files = []
|
||||
for rel in ("requirements.txt", "pyproject.toml", "setup.py", "setup.cfg"):
|
||||
path = os.path.join(root, rel)
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
if os.path.getsize(path) > max_manifest_bytes:
|
||||
raise RuntimeError(
|
||||
f"Python project manifest exceeds {{max_manifest_bytes}} bytes: {{rel}}"
|
||||
)
|
||||
manifest_files.append(rel)
|
||||
with open(path, "rb") as handle:
|
||||
digest.update(rel.encode("utf-8"))
|
||||
digest.update(b"\\0")
|
||||
digest.update(handle.read())
|
||||
while chunk := handle.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
digest.update(b"\\0")
|
||||
|
||||
print(
|
||||
@@ -274,6 +286,7 @@ class BoxWorkspaceSession:
|
||||
def __init__(
|
||||
self,
|
||||
box_service,
|
||||
execution_context,
|
||||
session_id: str,
|
||||
*,
|
||||
host_path: str | None = None,
|
||||
@@ -290,6 +303,7 @@ class BoxWorkspaceSession:
|
||||
persistent: bool = False,
|
||||
):
|
||||
self.box_service = box_service
|
||||
self.execution_context = execution_context
|
||||
self.session_id = session_id
|
||||
self.host_path = host_path
|
||||
self.host_path_mode = host_path_mode
|
||||
@@ -363,7 +377,7 @@ class BoxWorkspaceSession:
|
||||
timeout_sec: int | None = None,
|
||||
):
|
||||
payload = self.build_exec_payload(cmd, workdir=workdir, env=env, timeout_sec=timeout_sec)
|
||||
return await self.box_service.client.execute(self.box_service.build_spec(payload))
|
||||
return await self.box_service.execute_in_context(self.execution_context, payload)
|
||||
|
||||
async def execute_for_query(
|
||||
self,
|
||||
@@ -378,7 +392,7 @@ class BoxWorkspaceSession:
|
||||
return await self.box_service.execute_spec_payload(payload, query)
|
||||
|
||||
async def create_session(self):
|
||||
return await self.box_service.create_session(self.build_session_payload())
|
||||
return await self.box_service.create_session(self.execution_context, self.build_session_payload())
|
||||
|
||||
def build_process_payload(
|
||||
self,
|
||||
@@ -415,16 +429,26 @@ class BoxWorkspaceSession:
|
||||
):
|
||||
payload = self.build_process_payload(command, args, env=env, cwd=cwd)
|
||||
payload['process_id'] = process_id
|
||||
return await self.box_service.start_managed_process(self.session_id, payload)
|
||||
return await self.box_service.start_managed_process(self.execution_context, self.session_id, payload)
|
||||
|
||||
async def get_managed_process(self, process_id: str = 'default'):
|
||||
return await self.box_service.get_managed_process(self.session_id, process_id)
|
||||
return await self.box_service.get_managed_process(self.execution_context, self.session_id, process_id)
|
||||
|
||||
async def stop_managed_process(self, process_id: str = 'default') -> None:
|
||||
await self.box_service.stop_managed_process(self.session_id, process_id)
|
||||
await self.box_service.stop_managed_process(self.execution_context, self.session_id, process_id)
|
||||
|
||||
def get_managed_process_websocket_url(self, process_id: str = 'default') -> str:
|
||||
return self.box_service.get_managed_process_websocket_url(self.session_id, process_id)
|
||||
async def get_managed_process_websocket_connection(
|
||||
self,
|
||||
process_id: str = 'default',
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
return await self.box_service.get_managed_process_websocket_connection(
|
||||
self.execution_context,
|
||||
self.session_id,
|
||||
process_id,
|
||||
)
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
await self.box_service.client.delete_session(self.session_id)
|
||||
await self.box_service.client.delete_session(
|
||||
self.session_id,
|
||||
action_context=self.box_service._action_context(self.execution_context),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Contracts used by the optional closed Cloud control-plane bootstrap."""
|
||||
|
||||
from .bootstrap import (
|
||||
CloudBootstrapError,
|
||||
CloudManifestProvider,
|
||||
CloudManifestRefreshService,
|
||||
OpenSourceDeployment,
|
||||
VerifiedCloudDeployment,
|
||||
resolve_deployment,
|
||||
)
|
||||
from .directory import (
|
||||
DirectoryDelta,
|
||||
DirectoryEvent,
|
||||
DirectoryEventBatch,
|
||||
DirectoryMember,
|
||||
DirectoryProjectionProvider,
|
||||
DirectoryProjectionUnavailableError,
|
||||
DirectorySnapshot,
|
||||
DirectoryWorkspace,
|
||||
)
|
||||
from .directory_projection import DirectoryProjectionService
|
||||
from .entitlements import (
|
||||
EntitlementProvider,
|
||||
EntitlementResolver,
|
||||
EntitlementSnapshot,
|
||||
EntitlementUnavailableError,
|
||||
OpenSourceEntitlementProvider,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'CloudBootstrapError',
|
||||
'CloudManifestProvider',
|
||||
'CloudManifestRefreshService',
|
||||
'DirectoryDelta',
|
||||
'DirectoryEvent',
|
||||
'DirectoryEventBatch',
|
||||
'DirectoryMember',
|
||||
'DirectoryProjectionProvider',
|
||||
'DirectoryProjectionService',
|
||||
'DirectoryProjectionUnavailableError',
|
||||
'DirectorySnapshot',
|
||||
'DirectoryWorkspace',
|
||||
'EntitlementProvider',
|
||||
'EntitlementResolver',
|
||||
'EntitlementSnapshot',
|
||||
'EntitlementUnavailableError',
|
||||
'OpenSourceDeployment',
|
||||
'OpenSourceEntitlementProvider',
|
||||
'VerifiedCloudDeployment',
|
||||
'resolve_deployment',
|
||||
]
|
||||
@@ -0,0 +1,408 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import importlib.metadata
|
||||
import inspect
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from ..workspace.policy import CloudWorkspacePolicy, SingleWorkspacePolicy
|
||||
from .directory import DirectoryProjectionProvider, directory_projection_limits_from_config
|
||||
from .entitlements import EntitlementProvider, OpenSourceEntitlementProvider
|
||||
|
||||
|
||||
CLOUD_BOOTSTRAP_ENTRY_POINT = 'langbot.cloud_bootstrap'
|
||||
REQUIRED_TENANT_ISOLATION_VERSION = 2
|
||||
SUPPORTED_PGVECTOR_DIMENSIONS = frozenset({384, 512, 768, 1024, 1536})
|
||||
|
||||
|
||||
class CloudBootstrapError(RuntimeError):
|
||||
"""Fail-closed Cloud bootstrap validation error."""
|
||||
|
||||
|
||||
class CloudRuntimeUnavailableError(CloudBootstrapError):
|
||||
"""The verified Cloud receipt no longer admits runtime work."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class CloudManifestProvider(Protocol):
|
||||
"""Closed adapter responsible for renewing the signed deployment receipt."""
|
||||
|
||||
async def refresh_manifest(self) -> VerifiedCloudDeployment:
|
||||
"""Fetch, verify, and return the newest deployment receipt."""
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Release control-plane transport resources."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class OpenSourceDeployment:
|
||||
"""Default deployment selected when no closed bootstrap is installed."""
|
||||
|
||||
mode: str = 'oss'
|
||||
workspace_policy: SingleWorkspacePolicy = dataclasses.field(default_factory=SingleWorkspacePolicy)
|
||||
entitlement_provider: OpenSourceEntitlementProvider = dataclasses.field(
|
||||
default_factory=OpenSourceEntitlementProvider
|
||||
)
|
||||
directory_provider: None = None
|
||||
manifest_provider: None = None
|
||||
persistence_mode: str = 'oss_compat'
|
||||
required_vector_backend: str | None = None
|
||||
|
||||
@property
|
||||
def multi_workspace_enabled(self) -> bool:
|
||||
return False
|
||||
|
||||
def validate_instance_config(self, config: dict[str, Any]) -> None:
|
||||
del config
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class VerifiedCloudDeployment:
|
||||
"""Receipt returned only after the closed package verifies a Manifest.
|
||||
|
||||
Core deliberately does not accept a config flag as a substitute for this
|
||||
object. The closed entry point owns root-key/JWS verification and the
|
||||
entitlement adapter; open Core validates the receipt's runtime invariants.
|
||||
"""
|
||||
|
||||
instance_uuid: str
|
||||
manifest_jti: str
|
||||
manifest_generation: int
|
||||
expires_at: int
|
||||
release: str
|
||||
capabilities: frozenset[str]
|
||||
tenant_isolation_version: int
|
||||
entitlement_provider: EntitlementProvider
|
||||
directory_provider: DirectoryProjectionProvider
|
||||
manifest_provider: CloudManifestProvider
|
||||
verification_key_id: str
|
||||
mode: str = dataclasses.field(default='cloud', init=False)
|
||||
workspace_policy: CloudWorkspacePolicy = dataclasses.field(default_factory=CloudWorkspacePolicy, init=False)
|
||||
persistence_mode: str = dataclasses.field(default='cloud_runtime', init=False)
|
||||
required_vector_backend: str = dataclasses.field(default='pgvector', init=False)
|
||||
|
||||
@property
|
||||
def multi_workspace_enabled(self) -> bool:
|
||||
return True
|
||||
|
||||
def validate(self, expected_instance_uuid: str, *, now: int | None = None) -> None:
|
||||
current_time = int(time.time()) if now is None else now
|
||||
if not self.instance_uuid or self.instance_uuid != expected_instance_uuid:
|
||||
raise CloudBootstrapError('Verified Cloud Manifest targets another LangBot instance')
|
||||
if not self.manifest_jti or not self.verification_key_id:
|
||||
raise CloudBootstrapError('Verified Cloud Manifest receipt is incomplete')
|
||||
if isinstance(self.manifest_generation, bool) or self.manifest_generation <= 0:
|
||||
raise CloudBootstrapError('Verified Cloud Manifest generation must be positive')
|
||||
if self.expires_at <= current_time:
|
||||
raise CloudBootstrapError('Verified Cloud Manifest is expired')
|
||||
if self.tenant_isolation_version < REQUIRED_TENANT_ISOLATION_VERSION:
|
||||
raise CloudBootstrapError('Verified Cloud Manifest requires an unsupported tenant isolation version')
|
||||
if 'multi_workspace_v2' not in self.capabilities:
|
||||
raise CloudBootstrapError('Verified Cloud Manifest does not grant multi_workspace_v2')
|
||||
if not isinstance(self.entitlement_provider, EntitlementProvider):
|
||||
raise CloudBootstrapError('Verified Cloud bootstrap did not provide an entitlement adapter')
|
||||
if not isinstance(self.directory_provider, DirectoryProjectionProvider):
|
||||
raise CloudBootstrapError('Verified Cloud bootstrap did not provide a directory adapter')
|
||||
if not isinstance(self.manifest_provider, CloudManifestProvider):
|
||||
raise CloudBootstrapError('Verified Cloud bootstrap did not provide a Manifest renewal adapter')
|
||||
|
||||
def validate_instance_config(self, config: dict[str, Any]) -> None:
|
||||
try:
|
||||
directory_projection_limits_from_config(config)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise CloudBootstrapError(f'Cloud directory limits are invalid: {exc}') from exc
|
||||
if config.get('database', {}).get('use') != 'postgresql':
|
||||
raise CloudBootstrapError('Cloud runtime requires database.use=postgresql')
|
||||
if config.get('vdb', {}).get('use') != self.required_vector_backend:
|
||||
raise CloudBootstrapError('Cloud runtime requires vdb.use=pgvector')
|
||||
pgvector_config = config.get('vdb', {}).get('pgvector', {})
|
||||
if pgvector_config.get('use_business_database') is not True:
|
||||
raise CloudBootstrapError('Cloud runtime requires vdb.pgvector.use_business_database=true')
|
||||
dimensions = pgvector_config.get('allowed_dimensions')
|
||||
if (
|
||||
not isinstance(dimensions, list)
|
||||
or not dimensions
|
||||
or any(isinstance(item, bool) or not isinstance(item, int) for item in dimensions)
|
||||
or not set(dimensions).issubset(SUPPORTED_PGVECTOR_DIMENSIONS)
|
||||
):
|
||||
supported = ', '.join(str(item) for item in sorted(SUPPORTED_PGVECTOR_DIMENSIONS))
|
||||
raise CloudBootstrapError(f'Cloud pgvector allowed_dimensions must be a non-empty subset of: {supported}')
|
||||
if config.get('mcp', {}).get('stdio', {}).get('enabled', True) is not False:
|
||||
raise CloudBootstrapError('Cloud runtime requires mcp.stdio.enabled=false')
|
||||
plugin_worker = config.get('plugin', {}).get('worker', {})
|
||||
if plugin_worker.get('require_hard_limits') is not True:
|
||||
raise CloudBootstrapError('Cloud Runtime requires plugin.worker.require_hard_limits=true')
|
||||
box_config = config.get('box', {})
|
||||
if box_config.get('enabled') is not True:
|
||||
raise CloudBootstrapError('Cloud runtime requires box.enabled=true')
|
||||
if box_config.get('backend') != 'nsjail':
|
||||
raise CloudBootstrapError('Cloud runtime requires box.backend=nsjail')
|
||||
runtime_endpoint = str(box_config.get('runtime', {}).get('endpoint', '') or '').strip()
|
||||
if not runtime_endpoint:
|
||||
raise CloudBootstrapError('Cloud runtime requires a shared external box.runtime.endpoint')
|
||||
admission = box_config.get('admission', {})
|
||||
required_admission = {
|
||||
'required': True,
|
||||
'logical_session_id': 'global',
|
||||
'required_backend': 'nsjail',
|
||||
'max_sessions': 1,
|
||||
'max_managed_processes': 0,
|
||||
}
|
||||
if any(admission.get(name) != value for name, value in required_admission.items()):
|
||||
raise CloudBootstrapError(
|
||||
'Cloud runtime requires grant-enforced Box admission with one global session and zero managed processes'
|
||||
)
|
||||
grant_ttl = admission.get('max_grant_ttl_sec')
|
||||
if isinstance(grant_ttl, bool) or not isinstance(grant_ttl, int) or not 1 <= grant_ttl <= 300:
|
||||
raise CloudBootstrapError('Cloud Box admission max_grant_ttl_sec must be between 1 and 300')
|
||||
workspace_quota_mb = admission.get('workspace_quota_mb')
|
||||
if isinstance(workspace_quota_mb, bool) or not isinstance(workspace_quota_mb, int) or workspace_quota_mb <= 0:
|
||||
raise CloudBootstrapError('Cloud Box admission workspace_quota_mb must be a positive integer')
|
||||
local_config = box_config.get('local', {})
|
||||
host_root = str(local_config.get('host_root', '') or '').strip()
|
||||
default_workspace = str(local_config.get('default_workspace', '') or '').strip()
|
||||
allowed_mount_roots = local_config.get('allowed_mount_roots')
|
||||
if not host_root or not os.path.isabs(host_root):
|
||||
raise CloudBootstrapError('Cloud Box local.host_root must be an absolute shared-volume path')
|
||||
if not default_workspace or not os.path.isabs(default_workspace):
|
||||
raise CloudBootstrapError('Cloud Box local.default_workspace must be an absolute shared-volume path')
|
||||
if (
|
||||
not isinstance(allowed_mount_roots, list)
|
||||
or not allowed_mount_roots
|
||||
or any(not isinstance(root, str) or not os.path.isabs(root) for root in allowed_mount_roots)
|
||||
):
|
||||
raise CloudBootstrapError('Cloud Box local.allowed_mount_roots must contain absolute shared-volume paths')
|
||||
resolved_workspace = os.path.realpath(default_workspace)
|
||||
if not any(
|
||||
resolved_workspace == os.path.realpath(root)
|
||||
or resolved_workspace.startswith(f'{os.path.realpath(root)}{os.sep}')
|
||||
for root in allowed_mount_roots
|
||||
):
|
||||
raise CloudBootstrapError('Cloud Box local.default_workspace must be under allowed_mount_roots')
|
||||
|
||||
|
||||
class CloudBootstrapProvider(Protocol):
|
||||
def bootstrap(
|
||||
self,
|
||||
*,
|
||||
instance_uuid: str,
|
||||
instance_config: dict[str, Any],
|
||||
) -> VerifiedCloudDeployment | Awaitable[VerifiedCloudDeployment]: ...
|
||||
|
||||
|
||||
class DeploymentAdmissionGuard:
|
||||
"""Continuously enforce one verified deployment receipt.
|
||||
|
||||
Startup verification alone is insufficient because a long-running process
|
||||
could otherwise keep serving after the signed Manifest expires. The guard
|
||||
tracks both wall-clock expiry and a monotonic deadline so moving the system
|
||||
clock backwards cannot extend an already admitted receipt.
|
||||
|
||||
A closed bootstrap may atomically replace the receipt with a strictly newer
|
||||
Manifest generation after performing its own signature verification. The
|
||||
logical instance and deployment mode cannot change during the process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instance_uuid: str,
|
||||
deployment: OpenSourceDeployment | VerifiedCloudDeployment,
|
||||
*,
|
||||
wall_time: Callable[[], float] = time.time,
|
||||
monotonic_time: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.instance_uuid = instance_uuid
|
||||
self._wall_time = wall_time
|
||||
self._monotonic_time = monotonic_time
|
||||
self._lock = threading.Lock()
|
||||
self._deployment = deployment
|
||||
self._deadline: float | None = None
|
||||
self._install_initial(deployment)
|
||||
|
||||
@property
|
||||
def deployment(self) -> OpenSourceDeployment | VerifiedCloudDeployment:
|
||||
with self._lock:
|
||||
return self._deployment
|
||||
|
||||
def _install_initial(self, deployment: OpenSourceDeployment | VerifiedCloudDeployment) -> None:
|
||||
now = int(self._wall_time())
|
||||
if isinstance(deployment, VerifiedCloudDeployment):
|
||||
deployment.validate(self.instance_uuid, now=now)
|
||||
self._deadline = self._monotonic_time() + (deployment.expires_at - now)
|
||||
elif not isinstance(deployment, OpenSourceDeployment):
|
||||
raise TypeError('Deployment admission requires a verified deployment object')
|
||||
|
||||
@staticmethod
|
||||
def _receipt_identity(deployment: VerifiedCloudDeployment) -> tuple[Any, ...]:
|
||||
return (
|
||||
deployment.instance_uuid,
|
||||
deployment.manifest_jti,
|
||||
deployment.manifest_generation,
|
||||
deployment.expires_at,
|
||||
deployment.release,
|
||||
tuple(sorted(deployment.capabilities)),
|
||||
deployment.tenant_isolation_version,
|
||||
deployment.verification_key_id,
|
||||
)
|
||||
|
||||
def replace(self, deployment: VerifiedCloudDeployment) -> None:
|
||||
"""Atomically install a verified, non-rollback Cloud receipt."""
|
||||
|
||||
now = int(self._wall_time())
|
||||
deployment.validate(self.instance_uuid, now=now)
|
||||
with self._lock:
|
||||
current = self._deployment
|
||||
if not isinstance(current, VerifiedCloudDeployment):
|
||||
raise CloudRuntimeUnavailableError('Deployment mode cannot change while LangBot is running')
|
||||
if deployment.manifest_generation < current.manifest_generation:
|
||||
raise CloudRuntimeUnavailableError('Cloud Manifest generation rolled back')
|
||||
if deployment.manifest_generation == current.manifest_generation and self._receipt_identity(
|
||||
deployment
|
||||
) != self._receipt_identity(current):
|
||||
raise CloudRuntimeUnavailableError('Cloud Manifest generation has conflicting contents')
|
||||
self._deployment = deployment
|
||||
self._deadline = self._monotonic_time() + (deployment.expires_at - now)
|
||||
|
||||
def require_active(self) -> OpenSourceDeployment | VerifiedCloudDeployment:
|
||||
"""Return the active deployment or fail closed after Manifest expiry."""
|
||||
|
||||
now = int(self._wall_time())
|
||||
monotonic_now = self._monotonic_time()
|
||||
with self._lock:
|
||||
deployment = self._deployment
|
||||
deadline = self._deadline
|
||||
if isinstance(deployment, OpenSourceDeployment):
|
||||
return deployment
|
||||
try:
|
||||
deployment.validate(self.instance_uuid, now=now)
|
||||
except CloudBootstrapError as exc:
|
||||
raise CloudRuntimeUnavailableError(str(exc)) from exc
|
||||
if deadline is None or monotonic_now >= deadline:
|
||||
raise CloudRuntimeUnavailableError('Verified Cloud Manifest is expired')
|
||||
return deployment
|
||||
|
||||
|
||||
class CloudManifestRefreshService:
|
||||
"""Renew a short-lived verified Manifest before runtime admission expires."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
admission: DeploymentAdmissionGuard,
|
||||
provider: CloudManifestProvider,
|
||||
logger: Any,
|
||||
*,
|
||||
wall_time: Callable[[], float] = time.time,
|
||||
refresh_margin_seconds: int = 180,
|
||||
maximum_sleep_seconds: int = 300,
|
||||
) -> None:
|
||||
if not isinstance(provider, CloudManifestProvider):
|
||||
raise TypeError('Cloud Manifest refresh requires a CloudManifestProvider')
|
||||
if refresh_margin_seconds < 120:
|
||||
raise ValueError('Cloud Manifest refresh margin must be at least 120 seconds')
|
||||
if maximum_sleep_seconds <= 0:
|
||||
raise ValueError('Cloud Manifest refresh maximum sleep must be positive')
|
||||
self.admission = admission
|
||||
self.provider = provider
|
||||
self.logger = logger
|
||||
self._wall_time = wall_time
|
||||
self.refresh_margin_seconds = refresh_margin_seconds
|
||||
self.maximum_sleep_seconds = maximum_sleep_seconds
|
||||
|
||||
def next_refresh_delay(self) -> float:
|
||||
deployment = self.admission.deployment
|
||||
if not isinstance(deployment, VerifiedCloudDeployment):
|
||||
return float(self.maximum_sleep_seconds)
|
||||
remaining = deployment.expires_at - self._wall_time()
|
||||
return max(
|
||||
5.0,
|
||||
min(
|
||||
float(self.maximum_sleep_seconds),
|
||||
remaining - self.refresh_margin_seconds,
|
||||
),
|
||||
)
|
||||
|
||||
async def refresh_once(self) -> VerifiedCloudDeployment:
|
||||
candidate = await self.provider.refresh_manifest()
|
||||
if not isinstance(candidate, VerifiedCloudDeployment):
|
||||
raise CloudBootstrapError('Cloud Manifest provider returned an invalid deployment receipt')
|
||||
self.admission.replace(candidate)
|
||||
return candidate
|
||||
|
||||
async def run(self) -> None:
|
||||
retry_delay = 5.0
|
||||
while True:
|
||||
try:
|
||||
await asyncio.sleep(self.next_refresh_delay())
|
||||
await self.refresh_once()
|
||||
retry_delay = 5.0
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception:
|
||||
self.logger.exception('Cloud Manifest refresh failed')
|
||||
await asyncio.sleep(retry_delay)
|
||||
retry_delay = min(retry_delay * 2, 30.0)
|
||||
|
||||
|
||||
async def _invoke_provider(
|
||||
loaded: Any,
|
||||
*,
|
||||
instance_uuid: str,
|
||||
instance_config: dict[str, Any],
|
||||
) -> VerifiedCloudDeployment:
|
||||
provider = loaded() if inspect.isclass(loaded) else loaded
|
||||
bootstrap = getattr(provider, 'bootstrap', None)
|
||||
if not callable(bootstrap):
|
||||
raise CloudBootstrapError('Cloud bootstrap entry point must expose bootstrap()')
|
||||
result = bootstrap(instance_uuid=instance_uuid, instance_config=instance_config)
|
||||
if inspect.isawaitable(result):
|
||||
result = await result
|
||||
if not isinstance(result, VerifiedCloudDeployment):
|
||||
raise CloudBootstrapError('Cloud bootstrap must return VerifiedCloudDeployment')
|
||||
return result
|
||||
|
||||
|
||||
async def resolve_deployment(
|
||||
*,
|
||||
instance_uuid: str,
|
||||
instance_config: dict[str, Any],
|
||||
entry_points: Callable[[], Any] | None = None,
|
||||
now: int | None = None,
|
||||
) -> OpenSourceDeployment | VerifiedCloudDeployment:
|
||||
"""Discover the optional closed bootstrap and validate its receipt.
|
||||
|
||||
Absence selects OSS singleton mode. Presence is fail-closed: duplicate,
|
||||
broken, invalid, or expired providers never fall back to an OSS Workspace.
|
||||
"""
|
||||
|
||||
discover = entry_points or importlib.metadata.entry_points
|
||||
discovered = discover()
|
||||
if hasattr(discovered, 'select'):
|
||||
candidates = list(discovered.select(group=CLOUD_BOOTSTRAP_ENTRY_POINT))
|
||||
else: # Python/importlib compatibility for dict-like EntryPoints
|
||||
candidates = list(discovered.get(CLOUD_BOOTSTRAP_ENTRY_POINT, ()))
|
||||
if not candidates:
|
||||
deployment = OpenSourceDeployment()
|
||||
deployment.validate_instance_config(instance_config)
|
||||
return deployment
|
||||
if len(candidates) != 1:
|
||||
raise CloudBootstrapError('Exactly one Cloud bootstrap provider may be installed')
|
||||
|
||||
try:
|
||||
loaded = candidates[0].load()
|
||||
deployment = await _invoke_provider(
|
||||
loaded,
|
||||
instance_uuid=instance_uuid,
|
||||
instance_config=instance_config,
|
||||
)
|
||||
deployment.validate(instance_uuid, now=now)
|
||||
deployment.validate_instance_config(instance_config)
|
||||
return deployment
|
||||
except CloudBootstrapError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise CloudBootstrapError('Closed Cloud bootstrap failed') from exc
|
||||
@@ -0,0 +1,311 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
import pydantic
|
||||
|
||||
|
||||
DEFAULT_MAX_ACTIVE_WORKSPACES = 1_000
|
||||
HARD_MAX_ACTIVE_WORKSPACES = 5_000
|
||||
DEFAULT_MAX_SNAPSHOT_WORKSPACES = 1_000
|
||||
HARD_MAX_SNAPSHOT_WORKSPACES = 5_000
|
||||
DEFAULT_MAX_SNAPSHOT_MEMBERSHIPS = 20_000
|
||||
HARD_MAX_SNAPSHOT_MEMBERSHIPS = 100_000
|
||||
DEFAULT_MAX_CONTROL_PLANE_RESPONSE_BYTES = 32 * 1024 * 1024
|
||||
HARD_MAX_CONTROL_PLANE_RESPONSE_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
class DirectoryProjectionUnavailableError(RuntimeError):
|
||||
"""Raised when the verified Cloud directory cannot safely admit work."""
|
||||
|
||||
|
||||
class DirectoryProjectionLimits(pydantic.BaseModel):
|
||||
"""Instance-owned cardinality limits for verified Cloud directory data.
|
||||
|
||||
These are operational safety limits, not subscription entitlements. Core
|
||||
fails the complete projection transaction when a limit is exceeded instead
|
||||
of truncating an authoritative directory and accidentally hiding tenants.
|
||||
The closed adapter consumes the same limits before priming entitlement
|
||||
caches, and additionally bounds the HTTP response buffered for signature
|
||||
verification.
|
||||
"""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
max_active_workspaces: int = pydantic.Field(
|
||||
default=DEFAULT_MAX_ACTIVE_WORKSPACES,
|
||||
ge=1,
|
||||
le=HARD_MAX_ACTIVE_WORKSPACES,
|
||||
)
|
||||
max_snapshot_workspaces: int = pydantic.Field(
|
||||
default=DEFAULT_MAX_SNAPSHOT_WORKSPACES,
|
||||
ge=1,
|
||||
le=HARD_MAX_SNAPSHOT_WORKSPACES,
|
||||
)
|
||||
max_snapshot_memberships: int = pydantic.Field(
|
||||
default=DEFAULT_MAX_SNAPSHOT_MEMBERSHIPS,
|
||||
ge=1,
|
||||
le=HARD_MAX_SNAPSHOT_MEMBERSHIPS,
|
||||
)
|
||||
max_response_bytes: int = pydantic.Field(
|
||||
default=DEFAULT_MAX_CONTROL_PLANE_RESPONSE_BYTES,
|
||||
ge=1024 * 1024,
|
||||
le=HARD_MAX_CONTROL_PLANE_RESPONSE_BYTES,
|
||||
)
|
||||
|
||||
@pydantic.field_validator(
|
||||
'max_active_workspaces',
|
||||
'max_snapshot_workspaces',
|
||||
'max_snapshot_memberships',
|
||||
'max_response_bytes',
|
||||
mode='before',
|
||||
)
|
||||
@classmethod
|
||||
def _reject_boolean_limits(cls, value: object) -> object:
|
||||
if isinstance(value, bool):
|
||||
raise ValueError('must be an integer')
|
||||
return value
|
||||
|
||||
@pydantic.model_validator(mode='after')
|
||||
def _validate_workspace_limits(self) -> DirectoryProjectionLimits:
|
||||
if self.max_snapshot_workspaces < self.max_active_workspaces:
|
||||
raise ValueError('max_snapshot_workspaces must be greater than or equal to max_active_workspaces')
|
||||
return self
|
||||
|
||||
|
||||
def directory_projection_limits_from_config(config: dict[str, Any]) -> DirectoryProjectionLimits:
|
||||
"""Parse typed Cloud directory limits from the instance configuration."""
|
||||
|
||||
cloud_config = config.get('cloud', {})
|
||||
if not isinstance(cloud_config, dict):
|
||||
raise ValueError('cloud must be a mapping')
|
||||
directory_config = cloud_config.get('directory', {})
|
||||
if not isinstance(directory_config, dict):
|
||||
raise ValueError('cloud.directory must be a mapping')
|
||||
return DirectoryProjectionLimits.model_validate(directory_config)
|
||||
|
||||
|
||||
class DirectoryMember(pydantic.BaseModel):
|
||||
"""One account membership published by the SaaS control plane."""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
membership_uuid: str = pydantic.Field(min_length=1, max_length=36)
|
||||
account_uuid: str = pydantic.Field(min_length=1, max_length=36)
|
||||
normalized_email: str = pydantic.Field(min_length=1, max_length=320)
|
||||
display_name: str = pydantic.Field(min_length=1, max_length=255)
|
||||
account_status: str = pydantic.Field(pattern=r'^(active|blocked|disabled|deleted)$')
|
||||
role: str = pydantic.Field(pattern=r'^(owner|admin|member|developer|operator|viewer)$')
|
||||
membership_status: str = pydantic.Field(pattern=r'^(active|invited|disabled|removed)$')
|
||||
projection_revision: int = pydantic.Field(ge=1)
|
||||
joined_at: datetime.datetime | None = None
|
||||
|
||||
@pydantic.field_validator('normalized_email')
|
||||
@classmethod
|
||||
def _normalize_email(cls, value: str) -> str:
|
||||
normalized = value.strip().casefold()
|
||||
if normalized != value:
|
||||
raise ValueError('Directory email must already be normalized')
|
||||
return normalized
|
||||
|
||||
|
||||
class DirectoryWorkspace(pydantic.BaseModel):
|
||||
"""One Workspace and its authoritative membership projection."""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
uuid: str = pydantic.Field(min_length=1, max_length=36)
|
||||
name: str = pydantic.Field(min_length=1, max_length=255)
|
||||
slug: str = pydantic.Field(min_length=1, max_length=255)
|
||||
type: str = pydantic.Field(pattern=r'^(personal|team)$')
|
||||
status: str = pydantic.Field(pattern=r'^(provisioning|active|suspended|archived|deleted)$')
|
||||
created_by_account_uuid: str = pydantic.Field(min_length=1, max_length=36)
|
||||
projection_revision: int = pydantic.Field(ge=1)
|
||||
execution_generation: int = pydantic.Field(ge=1)
|
||||
members: tuple[DirectoryMember, ...] = pydantic.Field(
|
||||
default=(),
|
||||
max_length=HARD_MAX_SNAPSHOT_MEMBERSHIPS,
|
||||
)
|
||||
|
||||
@pydantic.field_validator('members', mode='before')
|
||||
@classmethod
|
||||
def _copy_members(cls, value: Sequence[DirectoryMember] | None) -> tuple[DirectoryMember, ...]:
|
||||
return tuple(value or ())
|
||||
|
||||
@pydantic.model_validator(mode='after')
|
||||
def _validate_members(self) -> DirectoryWorkspace:
|
||||
membership_uuids: set[str] = set()
|
||||
account_uuids: set[str] = set()
|
||||
for member in self.members:
|
||||
if member.membership_uuid in membership_uuids:
|
||||
raise ValueError('Directory Workspace contains duplicate membership UUIDs')
|
||||
if member.account_uuid in account_uuids:
|
||||
raise ValueError('Directory Workspace contains duplicate account UUIDs')
|
||||
membership_uuids.add(member.membership_uuid)
|
||||
account_uuids.add(member.account_uuid)
|
||||
if self.created_by_account_uuid not in account_uuids:
|
||||
raise ValueError('Directory Workspace must include its creator')
|
||||
return self
|
||||
|
||||
|
||||
class DirectorySnapshot(pydantic.BaseModel):
|
||||
"""Full signed directory state at one monotonic outbox cursor."""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
instance_uuid: str = pydantic.Field(min_length=1, max_length=255)
|
||||
cursor: int = pydantic.Field(ge=0)
|
||||
generated_at: datetime.datetime
|
||||
workspaces: tuple[DirectoryWorkspace, ...] = pydantic.Field(
|
||||
default=(),
|
||||
max_length=HARD_MAX_SNAPSHOT_WORKSPACES,
|
||||
)
|
||||
|
||||
@pydantic.field_validator('workspaces', mode='before')
|
||||
@classmethod
|
||||
def _copy_workspaces(cls, value: Sequence[DirectoryWorkspace] | None) -> tuple[DirectoryWorkspace, ...]:
|
||||
return tuple(value or ())
|
||||
|
||||
@pydantic.model_validator(mode='after')
|
||||
def _validate_workspaces(self) -> DirectorySnapshot:
|
||||
workspace_uuids: set[str] = set()
|
||||
slugs: set[str] = set()
|
||||
membership_uuids: set[str] = set()
|
||||
for workspace in self.workspaces:
|
||||
if workspace.uuid in workspace_uuids:
|
||||
raise ValueError('Directory snapshot contains duplicate Workspace UUIDs')
|
||||
if workspace.slug in slugs:
|
||||
raise ValueError('Directory snapshot contains duplicate Workspace slugs')
|
||||
workspace_uuids.add(workspace.uuid)
|
||||
slugs.add(workspace.slug)
|
||||
for member in workspace.members:
|
||||
if member.membership_uuid in membership_uuids:
|
||||
raise ValueError('Directory snapshot contains duplicate membership UUIDs')
|
||||
membership_uuids.add(member.membership_uuid)
|
||||
return self
|
||||
|
||||
|
||||
class DirectoryDelta(pydantic.BaseModel):
|
||||
"""Signed authoritative state for an explicitly requested Workspace set."""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
instance_uuid: str = pydantic.Field(min_length=1, max_length=255)
|
||||
requested_workspace_uuids: tuple[str, ...]
|
||||
generated_at: datetime.datetime
|
||||
workspaces: tuple[DirectoryWorkspace, ...] = ()
|
||||
|
||||
@pydantic.field_validator('requested_workspace_uuids', mode='before')
|
||||
@classmethod
|
||||
def _copy_requested_workspace_uuids(cls, value: Sequence[str]) -> tuple[str, ...]:
|
||||
return tuple(value)
|
||||
|
||||
@pydantic.field_validator('workspaces', mode='before')
|
||||
@classmethod
|
||||
def _copy_workspaces(cls, value: Sequence[DirectoryWorkspace] | None) -> tuple[DirectoryWorkspace, ...]:
|
||||
return tuple(value or ())
|
||||
|
||||
@pydantic.model_validator(mode='after')
|
||||
def _validate_workspaces(self) -> DirectoryDelta:
|
||||
requested = self.requested_workspace_uuids
|
||||
if not requested or len(requested) > 100:
|
||||
raise ValueError('Directory delta must request between 1 and 100 Workspaces')
|
||||
if any(not workspace_uuid or len(workspace_uuid) > 36 for workspace_uuid in requested):
|
||||
raise ValueError('Directory delta contains an invalid requested Workspace UUID')
|
||||
if len(requested) != len(set(requested)):
|
||||
raise ValueError('Directory delta contains duplicate requested Workspace UUIDs')
|
||||
|
||||
requested_set = set(requested)
|
||||
workspace_uuids: set[str] = set()
|
||||
slugs: set[str] = set()
|
||||
membership_uuids: set[str] = set()
|
||||
for workspace in self.workspaces:
|
||||
if workspace.uuid in workspace_uuids:
|
||||
raise ValueError('Directory delta contains duplicate Workspace UUIDs')
|
||||
if workspace.uuid not in requested_set:
|
||||
raise ValueError('Directory delta returned an unrequested Workspace')
|
||||
if workspace.slug in slugs:
|
||||
raise ValueError('Directory delta contains duplicate Workspace slugs')
|
||||
workspace_uuids.add(workspace.uuid)
|
||||
slugs.add(workspace.slug)
|
||||
for member in workspace.members:
|
||||
if member.membership_uuid in membership_uuids:
|
||||
raise ValueError('Directory delta contains duplicate membership UUIDs')
|
||||
membership_uuids.add(member.membership_uuid)
|
||||
return self
|
||||
|
||||
|
||||
class DirectoryEvent(pydantic.BaseModel):
|
||||
"""One signed control-plane outbox notification."""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
cursor: int = pydantic.Field(ge=1)
|
||||
uuid: str = pydantic.Field(min_length=1, max_length=36)
|
||||
aggregate_uuid: str = pydantic.Field(min_length=1, max_length=36)
|
||||
event_type: str = pydantic.Field(min_length=1, max_length=128)
|
||||
revision: int = pydantic.Field(ge=1)
|
||||
payload: dict[str, Any] = pydantic.Field(default_factory=dict)
|
||||
created_at: datetime.datetime
|
||||
|
||||
|
||||
class DirectoryEventBatch(pydantic.BaseModel):
|
||||
"""Signed events returned after a caller-supplied directory cursor."""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
instance_uuid: str = pydantic.Field(min_length=1, max_length=255)
|
||||
after_cursor: int = pydantic.Field(ge=0)
|
||||
cursor: int = pydantic.Field(ge=0)
|
||||
high_water_cursor: int = pydantic.Field(ge=0)
|
||||
events: tuple[DirectoryEvent, ...] = ()
|
||||
|
||||
@pydantic.field_validator('events', mode='before')
|
||||
@classmethod
|
||||
def _copy_events(cls, value: Sequence[DirectoryEvent] | None) -> tuple[DirectoryEvent, ...]:
|
||||
return tuple(value or ())
|
||||
|
||||
@pydantic.model_validator(mode='after')
|
||||
def _validate_events(self) -> DirectoryEventBatch:
|
||||
if self.cursor < self.after_cursor:
|
||||
raise ValueError('Directory event cursor rolled back')
|
||||
if self.high_water_cursor < self.cursor:
|
||||
raise ValueError('Directory event high-water mark rolled back')
|
||||
event_cursors = [event.cursor for event in self.events]
|
||||
event_uuids = [event.uuid for event in self.events]
|
||||
if event_cursors != sorted(event_cursors) or len(event_cursors) != len(set(event_cursors)):
|
||||
raise ValueError('Directory events must have strictly increasing cursors')
|
||||
if len(event_uuids) != len(set(event_uuids)):
|
||||
raise ValueError('Directory event batch contains duplicate UUIDs')
|
||||
if any(cursor <= self.after_cursor or cursor > self.cursor for cursor in event_cursors):
|
||||
raise ValueError('Directory event falls outside the requested cursor window')
|
||||
if not self.events and (self.cursor != self.after_cursor or self.high_water_cursor != self.after_cursor):
|
||||
raise ValueError('Empty Directory event batch cannot advance or trail the high-water mark')
|
||||
if self.events and self.cursor != self.events[-1].cursor:
|
||||
raise ValueError('Directory event batch cursor must equal its final event cursor')
|
||||
return self
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DirectoryProjectionProvider(Protocol):
|
||||
"""Closed adapter that returns signature-verified control-plane data."""
|
||||
|
||||
async def fetch_snapshot(self, instance_uuid: str) -> DirectorySnapshot:
|
||||
"""Fetch and verify an authoritative full snapshot."""
|
||||
|
||||
async def fetch_events(
|
||||
self,
|
||||
instance_uuid: str,
|
||||
after_cursor: int,
|
||||
limit: int,
|
||||
) -> DirectoryEventBatch:
|
||||
"""Fetch and verify directory events after one process-local cursor."""
|
||||
|
||||
async def fetch_workspaces(
|
||||
self,
|
||||
instance_uuid: str,
|
||||
workspace_uuids: tuple[str, ...],
|
||||
) -> DirectoryDelta:
|
||||
"""Fetch and verify authoritative state for an explicit Workspace set."""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,251 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import pydantic
|
||||
|
||||
|
||||
class EntitlementUnavailableError(RuntimeError):
|
||||
"""Raised when a trusted, currently-active entitlement is unavailable."""
|
||||
|
||||
def __init__(self, message: str, *, entitlement_revision: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.entitlement_revision = entitlement_revision
|
||||
|
||||
|
||||
class EntitlementFeatureUnavailableError(EntitlementUnavailableError):
|
||||
"""Raised only when an active entitlement does not grant one feature."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
feature: str,
|
||||
*,
|
||||
entitlement_revision: int | None = None,
|
||||
) -> None:
|
||||
self.feature = feature
|
||||
super().__init__(
|
||||
f'Workspace entitlement does not grant {feature}',
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
|
||||
|
||||
class EntitlementSnapshot(pydantic.BaseModel):
|
||||
"""Capability projection consumed by open-source Core.
|
||||
|
||||
Admission and quota decisions use normalized features/limits rather than
|
||||
product plan names. ``plan_name`` is signed display metadata for Cloud UI
|
||||
only and must never drive authorization or quota enforcement.
|
||||
"""
|
||||
|
||||
model_config = pydantic.ConfigDict(frozen=True, extra='forbid')
|
||||
|
||||
instance_uuid: str = pydantic.Field(min_length=1, max_length=256)
|
||||
workspace_uuid: str = pydantic.Field(min_length=1, max_length=256)
|
||||
entitlement_revision: int = pydantic.Field(ge=1)
|
||||
status: str = pydantic.Field(pattern=r'^(active|suspended|cancelled)$')
|
||||
not_before: int = pydantic.Field(ge=0)
|
||||
expires_at: int = pydantic.Field(gt=0)
|
||||
features: dict[str, bool] = pydantic.Field(default_factory=dict)
|
||||
limits: dict[str, int] = pydantic.Field(default_factory=dict)
|
||||
# Signed display metadata for Cloud UI only. Admission and quota decisions
|
||||
# must continue to use generic ``features`` and ``limits`` exclusively.
|
||||
plan_name: str | None = pydantic.Field(default=None, min_length=1, max_length=128)
|
||||
|
||||
@pydantic.field_validator('features')
|
||||
@classmethod
|
||||
def _validate_feature_names(cls, value: dict[str, bool]) -> dict[str, bool]:
|
||||
if any(not str(name).strip() for name in value):
|
||||
raise ValueError('Entitlement feature names must be non-empty')
|
||||
return dict(value)
|
||||
|
||||
@pydantic.field_validator('limits')
|
||||
@classmethod
|
||||
def _validate_limits(cls, value: dict[str, int]) -> dict[str, int]:
|
||||
normalized: dict[str, int] = {}
|
||||
for name, limit in value.items():
|
||||
if not str(name).strip():
|
||||
raise ValueError('Entitlement limit names must be non-empty')
|
||||
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 0:
|
||||
raise ValueError(f'Entitlement limit {name!r} must be a non-negative integer')
|
||||
normalized[str(name)] = limit
|
||||
return normalized
|
||||
|
||||
def require_active(
|
||||
self,
|
||||
*,
|
||||
instance_uuid: str,
|
||||
workspace_uuid: str,
|
||||
now: int | None = None,
|
||||
) -> EntitlementSnapshot:
|
||||
current_time = int(time.time()) if now is None else now
|
||||
if self.instance_uuid != instance_uuid or self.workspace_uuid != workspace_uuid:
|
||||
raise EntitlementUnavailableError('Entitlement scope does not match the Workspace execution context')
|
||||
if self.status != 'active':
|
||||
raise EntitlementUnavailableError(
|
||||
'Workspace entitlement is not active',
|
||||
entitlement_revision=self.entitlement_revision,
|
||||
)
|
||||
if current_time < self.not_before or current_time >= self.expires_at:
|
||||
raise EntitlementUnavailableError(
|
||||
'Workspace entitlement is not currently valid',
|
||||
entitlement_revision=self.entitlement_revision,
|
||||
)
|
||||
return self
|
||||
|
||||
def require_feature(self, feature: str) -> None:
|
||||
if self.features.get(feature) is not True:
|
||||
raise EntitlementFeatureUnavailableError(
|
||||
feature,
|
||||
entitlement_revision=self.entitlement_revision,
|
||||
)
|
||||
|
||||
def limit(self, name: str) -> int:
|
||||
value = self.limits.get(name)
|
||||
if value is None:
|
||||
raise EntitlementUnavailableError(f'Workspace entitlement does not define limit {name}')
|
||||
return value
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EntitlementProvider(Protocol):
|
||||
"""Closed Control Plane adapter injected by a verified Cloud bootstrap."""
|
||||
|
||||
async def get_workspace_entitlement(self, workspace_uuid: str) -> EntitlementSnapshot:
|
||||
"""Return the newest verified snapshot for one Workspace."""
|
||||
|
||||
|
||||
class OpenSourceEntitlementProvider:
|
||||
"""Marker provider for OSS; Cloud admission grants never use this class."""
|
||||
|
||||
async def get_workspace_entitlement(self, workspace_uuid: str) -> EntitlementSnapshot:
|
||||
del workspace_uuid
|
||||
raise EntitlementUnavailableError('Signed Workspace entitlements are only available in Cloud mode')
|
||||
|
||||
|
||||
class EntitlementResolver:
|
||||
"""Validate scope/freshness and reject revision rollback or equivocation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instance_uuid: str,
|
||||
provider: EntitlementProvider,
|
||||
*,
|
||||
deployment_admission: Callable[[], object] | None = None,
|
||||
) -> None:
|
||||
self.instance_uuid = instance_uuid
|
||||
self.provider = provider
|
||||
self._deployment_admission = deployment_admission
|
||||
self._lock = asyncio.Lock()
|
||||
self._snapshots: dict[str, tuple[int, str, EntitlementSnapshot]] = {}
|
||||
self._active_workspace_uuids: frozenset[str] | None = None
|
||||
|
||||
@staticmethod
|
||||
def _fingerprint(snapshot: EntitlementSnapshot) -> str:
|
||||
return json.dumps(snapshot.model_dump(mode='json'), sort_keys=True, separators=(',', ':'))
|
||||
|
||||
async def resolve(
|
||||
self,
|
||||
workspace_uuid: str,
|
||||
*,
|
||||
minimum_revision: int = 0,
|
||||
now: int | None = None,
|
||||
) -> EntitlementSnapshot:
|
||||
if self._deployment_admission is not None:
|
||||
self._deployment_admission()
|
||||
async with self._lock:
|
||||
self._require_projected_workspace_locked(workspace_uuid)
|
||||
candidate = await self.provider.get_workspace_entitlement(workspace_uuid)
|
||||
if self._deployment_admission is not None:
|
||||
# A provider call may cross the Manifest expiry boundary.
|
||||
self._deployment_admission()
|
||||
if not isinstance(candidate, EntitlementSnapshot):
|
||||
raise EntitlementUnavailableError('Entitlement provider returned an invalid snapshot')
|
||||
# Deep-copy untrusted provider-owned containers before caching them.
|
||||
candidate = EntitlementSnapshot.model_validate(candidate.model_dump())
|
||||
candidate.require_active(
|
||||
instance_uuid=self.instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
now=now,
|
||||
)
|
||||
if candidate.entitlement_revision < minimum_revision:
|
||||
raise EntitlementUnavailableError('Workspace entitlement revision rolled back')
|
||||
|
||||
fingerprint = self._fingerprint(candidate)
|
||||
async with self._lock:
|
||||
# The directory may fence a Workspace while the provider call is
|
||||
# in flight. Recheck before retaining or returning its snapshot.
|
||||
self._require_projected_workspace_locked(workspace_uuid)
|
||||
previous = self._snapshots.get(workspace_uuid)
|
||||
if previous is not None:
|
||||
previous_revision, previous_fingerprint, _ = previous
|
||||
if candidate.entitlement_revision < previous_revision:
|
||||
raise EntitlementUnavailableError('Workspace entitlement revision rolled back')
|
||||
if candidate.entitlement_revision == previous_revision and fingerprint != previous_fingerprint:
|
||||
raise EntitlementUnavailableError('Workspace entitlement revision has conflicting contents')
|
||||
self._snapshots[workspace_uuid] = (
|
||||
candidate.entitlement_revision,
|
||||
fingerprint,
|
||||
candidate,
|
||||
)
|
||||
return candidate.model_copy(deep=True)
|
||||
|
||||
def _require_projected_workspace_locked(self, workspace_uuid: str) -> None:
|
||||
active_workspace_uuids = self._active_workspace_uuids
|
||||
if active_workspace_uuids is not None and workspace_uuid not in active_workspace_uuids:
|
||||
raise EntitlementUnavailableError('Workspace is not active in the Cloud directory projection')
|
||||
|
||||
async def reconcile_active_workspaces(
|
||||
self,
|
||||
workspace_uuids: set[str] | frozenset[str],
|
||||
) -> None:
|
||||
"""Drop entitlement history for Workspaces fenced by the directory."""
|
||||
|
||||
active = frozenset(workspace_uuids)
|
||||
async with self._lock:
|
||||
self._active_workspace_uuids = active
|
||||
self._snapshots = {
|
||||
workspace_uuid: cached for workspace_uuid, cached in self._snapshots.items() if workspace_uuid in active
|
||||
}
|
||||
|
||||
async def set_workspace_active(
|
||||
self,
|
||||
workspace_uuid: str,
|
||||
*,
|
||||
active: bool,
|
||||
) -> None:
|
||||
"""Apply one incremental directory activity change."""
|
||||
|
||||
await self.update_workspace_activity(
|
||||
active_workspace_uuids={workspace_uuid} if active else set(),
|
||||
inactive_workspace_uuids=set() if active else {workspace_uuid},
|
||||
)
|
||||
|
||||
async def update_workspace_activity(
|
||||
self,
|
||||
*,
|
||||
active_workspace_uuids: set[str] | frozenset[str],
|
||||
inactive_workspace_uuids: set[str] | frozenset[str],
|
||||
) -> None:
|
||||
"""Apply one directory delta without copying the active set per item."""
|
||||
|
||||
active_updates = set(active_workspace_uuids)
|
||||
inactive_updates = set(inactive_workspace_uuids)
|
||||
if active_updates & inactive_updates:
|
||||
raise ValueError('Workspace activity update contains conflicting entries')
|
||||
async with self._lock:
|
||||
current = set(self._active_workspace_uuids or ())
|
||||
current.update(active_updates)
|
||||
current.difference_update(inactive_updates)
|
||||
for workspace_uuid in inactive_updates:
|
||||
self._snapshots.pop(workspace_uuid, None)
|
||||
self._active_workspace_uuids = frozenset(current)
|
||||
|
||||
def snapshot_counts(self) -> dict[str, int]:
|
||||
return {
|
||||
'active_workspaces': len(self._active_workspace_uuids or ()),
|
||||
'cached_snapshots': len(self._snapshots),
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import heapq
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ..core.app import Application
|
||||
|
||||
|
||||
CONTROL_PLANE_TYP = 'langbot-control-plane+jwt'
|
||||
LAUNCH_KIND = 'workspace.launch'
|
||||
EXPECTED_ISSUER = 'langbot-space'
|
||||
EXPECTED_AUDIENCE = 'langbot-cloud-runtime'
|
||||
_CONSUMED_JTI_MAX_ENTRIES = 4096
|
||||
_CONSUMED_JTI_HEAP_COMPACT_FLOOR = 64
|
||||
_CONSUMED_JTI_HEAP_MAX_MULTIPLIER = 4
|
||||
|
||||
|
||||
class SpaceLaunchError(ValueError):
|
||||
"""Raised when a Space-issued Cloud launch assertion is not admissible."""
|
||||
|
||||
|
||||
def _decode_base64url(value: str, *, label: str) -> bytes:
|
||||
if not value or any(character.isspace() for character in value):
|
||||
raise SpaceLaunchError(f'Launch assertion {label} is not canonical base64url')
|
||||
try:
|
||||
raw = base64.b64decode(value + ('=' * (-len(value) % 4)), altchars=b'-_', validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise SpaceLaunchError(f'Launch assertion {label} is not valid base64url') from exc
|
||||
if base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii') != value:
|
||||
raise SpaceLaunchError(f'Launch assertion {label} is not canonical base64url')
|
||||
return raw
|
||||
|
||||
|
||||
def _strict_json_object(value: bytes, *, label: str) -> dict[str, typing.Any]:
|
||||
def reject_duplicate_keys(pairs: Iterable[tuple[str, typing.Any]]) -> dict[str, typing.Any]:
|
||||
result: dict[str, typing.Any] = {}
|
||||
for key, item in pairs:
|
||||
if key in result:
|
||||
raise SpaceLaunchError(f'Launch assertion {label} contains duplicate key {key!r}')
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
try:
|
||||
decoded = json.loads(value, object_pairs_hook=reject_duplicate_keys)
|
||||
except SpaceLaunchError:
|
||||
raise
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise SpaceLaunchError(f'Launch assertion {label} is not valid JSON') from exc
|
||||
if not isinstance(decoded, dict):
|
||||
raise SpaceLaunchError(f'Launch assertion {label} must be a JSON object')
|
||||
return decoded
|
||||
|
||||
|
||||
def _required_string(claims: dict[str, typing.Any], name: str) -> str:
|
||||
value = claims.get(name)
|
||||
if not isinstance(value, str) or not value or value != value.strip():
|
||||
raise SpaceLaunchError(f'Launch assertion claim {name} must be a non-empty string')
|
||||
return value
|
||||
|
||||
|
||||
def _required_int(claims: dict[str, typing.Any], name: str, *, minimum: int = 0) -> int:
|
||||
value = claims.get(name)
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
|
||||
raise SpaceLaunchError(f'Launch assertion claim {name} must be an integer >= {minimum}')
|
||||
return value
|
||||
|
||||
|
||||
def _load_ed25519_public_key(encoded: str) -> Ed25519PublicKey:
|
||||
value = encoded.strip()
|
||||
if value.startswith('-----BEGIN'):
|
||||
try:
|
||||
key = serialization.load_pem_public_key(value.encode('ascii'))
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise SpaceLaunchError('Space launch public key is not valid PEM') from exc
|
||||
if not isinstance(key, Ed25519PublicKey):
|
||||
raise SpaceLaunchError('Space launch public key must be Ed25519')
|
||||
return key
|
||||
|
||||
try:
|
||||
raw = base64.b64decode(value + ('=' * (-len(value) % 4)), altchars=b'-_', validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise SpaceLaunchError('Space launch public key must be base64 encoded') from exc
|
||||
if len(raw) != 32:
|
||||
raise SpaceLaunchError('Space launch Ed25519 public key must contain 32 bytes')
|
||||
return Ed25519PublicKey.from_public_bytes(raw)
|
||||
|
||||
|
||||
class SpaceLaunchService:
|
||||
"""Verify and single-use consume Space Cloud direct-launch assertions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: Application,
|
||||
*,
|
||||
wall_time: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self._wall_time = wall_time
|
||||
self._replay_lock = asyncio.Lock()
|
||||
self._consumed_jtis: dict[str, int] = {}
|
||||
self._consumed_jti_expiry_heap: list[tuple[int, str]] = []
|
||||
|
||||
async def consume_assertion(
|
||||
self,
|
||||
assertion: str,
|
||||
*,
|
||||
expected_workspace_uuid: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
claims = self._verify_assertion(assertion)
|
||||
payload = claims.get('payload')
|
||||
if not isinstance(payload, dict):
|
||||
raise SpaceLaunchError('Launch assertion payload must be a JSON object')
|
||||
account_uuid = _required_string(payload, 'account_uuid')
|
||||
workspace_uuid = _required_string(payload, 'workspace_uuid')
|
||||
if expected_workspace_uuid is not None and workspace_uuid != expected_workspace_uuid:
|
||||
raise SpaceLaunchError('Launch assertion targets another Workspace')
|
||||
await self._consume_jti(_required_string(claims, 'jti'), _required_int(claims, 'exp', minimum=1))
|
||||
return {
|
||||
'account_uuid': account_uuid,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
}
|
||||
|
||||
def _verify_assertion(self, token: str) -> dict[str, typing.Any]:
|
||||
if not getattr(getattr(self.ap, 'deployment', None), 'multi_workspace_enabled', False):
|
||||
raise SpaceLaunchError('Space direct launch requires verified Cloud mode')
|
||||
public_key, key_id, clock_skew_seconds = self._trust_config()
|
||||
segments = token.split('.')
|
||||
if len(segments) != 3:
|
||||
raise SpaceLaunchError('Launch assertion must be a compact JWS')
|
||||
encoded_header, encoded_claims, encoded_signature = segments
|
||||
header = _strict_json_object(_decode_base64url(encoded_header, label='header'), label='header')
|
||||
if set(header) != {'alg', 'kid', 'typ'}:
|
||||
raise SpaceLaunchError('Launch assertion header contains unsupported fields')
|
||||
if header.get('alg') != 'EdDSA':
|
||||
raise SpaceLaunchError('Launch assertion algorithm must be EdDSA')
|
||||
if header.get('kid') != key_id:
|
||||
raise SpaceLaunchError('Launch assertion key ID does not match Cloud trust')
|
||||
if header.get('typ') != CONTROL_PLANE_TYP:
|
||||
raise SpaceLaunchError('Launch assertion type is not a control-plane payload')
|
||||
|
||||
signature = _decode_base64url(encoded_signature, label='signature')
|
||||
if len(signature) != 64:
|
||||
raise SpaceLaunchError('Launch assertion signature must contain 64 bytes')
|
||||
try:
|
||||
public_key.verify(signature, f'{encoded_header}.{encoded_claims}'.encode('ascii'))
|
||||
except InvalidSignature as exc:
|
||||
raise SpaceLaunchError('Launch assertion signature is invalid') from exc
|
||||
|
||||
claims = _strict_json_object(_decode_base64url(encoded_claims, label='claims'), label='claims')
|
||||
instance_uuid = self.ap.workspace_service.instance_uuid
|
||||
if _required_string(claims, 'iss') != EXPECTED_ISSUER:
|
||||
raise SpaceLaunchError('Launch assertion issuer is not LangBot Space')
|
||||
if _required_string(claims, 'aud') != EXPECTED_AUDIENCE:
|
||||
raise SpaceLaunchError('Launch assertion audience does not target Cloud runtime')
|
||||
if _required_string(claims, 'sub') != f'langbot-instance:{instance_uuid}':
|
||||
raise SpaceLaunchError('Launch assertion subject targets another instance')
|
||||
if _required_string(claims, 'instance_uuid') != instance_uuid:
|
||||
raise SpaceLaunchError('Launch assertion instance UUID does not match this Core')
|
||||
if _required_string(claims, 'kind') != LAUNCH_KIND:
|
||||
raise SpaceLaunchError('Launch assertion kind is not workspace.launch')
|
||||
|
||||
issued_at = _required_int(claims, 'iat')
|
||||
not_before = _required_int(claims, 'nbf')
|
||||
expires_at = _required_int(claims, 'exp', minimum=1)
|
||||
now = self._wall_time()
|
||||
if issued_at > now + clock_skew_seconds:
|
||||
raise SpaceLaunchError('Launch assertion was issued in the future')
|
||||
if not_before > now + clock_skew_seconds:
|
||||
raise SpaceLaunchError('Launch assertion is not active yet')
|
||||
if expires_at <= now - clock_skew_seconds:
|
||||
raise SpaceLaunchError('Launch assertion is expired')
|
||||
if expires_at <= max(issued_at, not_before):
|
||||
raise SpaceLaunchError('Launch assertion expiry must follow issue time')
|
||||
return claims
|
||||
|
||||
def _trust_config(self) -> tuple[Ed25519PublicKey, str, float]:
|
||||
data = getattr(getattr(self.ap, 'instance_config', None), 'data', {}) or {}
|
||||
space_config = data.get('space', {})
|
||||
launch_config = space_config.get('launch', {}) if isinstance(space_config, dict) else {}
|
||||
if not isinstance(launch_config, dict):
|
||||
launch_config = {}
|
||||
public_key_value = (
|
||||
os.environ.get('LANGBOT_SPACE_CONTROL_PLANE_PUBLIC_KEY', '').strip()
|
||||
or str(launch_config.get('control_plane_public_key', '') or '').strip()
|
||||
)
|
||||
key_id = (
|
||||
os.environ.get('LANGBOT_SPACE_CONTROL_PLANE_KEY_ID', '').strip()
|
||||
or str(launch_config.get('control_plane_key_id', '') or '').strip()
|
||||
or str(getattr(getattr(self.ap, 'deployment', None), 'verification_key_id', '') or '').strip()
|
||||
)
|
||||
if not public_key_value or not key_id:
|
||||
raise SpaceLaunchError('Space launch control-plane trust is not configured')
|
||||
clock_skew = self._bounded_float(
|
||||
os.environ.get('LANGBOT_SPACE_CONTROL_PLANE_CLOCK_SKEW_SECONDS') or launch_config.get('clock_skew_seconds'),
|
||||
default=30.0,
|
||||
minimum=0.0,
|
||||
maximum=300.0,
|
||||
)
|
||||
return _load_ed25519_public_key(public_key_value), key_id, clock_skew
|
||||
|
||||
async def _consume_jti(self, jti: str, expires_at: int) -> None:
|
||||
digest = hashlib.sha256(jti.encode('utf-8')).hexdigest()
|
||||
now = int(self._wall_time())
|
||||
async with self._replay_lock:
|
||||
self._prune_consumed_jtis(now)
|
||||
if digest in self._consumed_jtis:
|
||||
raise SpaceLaunchError('Launch assertion has already been consumed')
|
||||
if len(self._consumed_jtis) >= _CONSUMED_JTI_MAX_ENTRIES:
|
||||
# Evicting a still-valid digest would make a signed launch
|
||||
# assertion replayable. Bound memory by failing closed instead.
|
||||
raise SpaceLaunchError('Launch assertion replay cache capacity reached')
|
||||
self._consumed_jtis[digest] = expires_at
|
||||
heapq.heappush(
|
||||
self._consumed_jti_expiry_heap,
|
||||
(expires_at, digest),
|
||||
)
|
||||
|
||||
def _prune_consumed_jtis(self, now: int) -> None:
|
||||
while self._consumed_jti_expiry_heap:
|
||||
expires_at, digest = self._consumed_jti_expiry_heap[0]
|
||||
current_expiry = self._consumed_jtis.get(digest)
|
||||
if current_expiry != expires_at:
|
||||
heapq.heappop(self._consumed_jti_expiry_heap)
|
||||
continue
|
||||
if expires_at > now:
|
||||
break
|
||||
heapq.heappop(self._consumed_jti_expiry_heap)
|
||||
self._consumed_jtis.pop(digest, None)
|
||||
|
||||
max_heap_entries = max(
|
||||
_CONSUMED_JTI_HEAP_COMPACT_FLOOR,
|
||||
len(self._consumed_jtis) * _CONSUMED_JTI_HEAP_MAX_MULTIPLIER,
|
||||
)
|
||||
if len(self._consumed_jti_expiry_heap) > max_heap_entries:
|
||||
self._consumed_jti_expiry_heap[:] = [(expiry, digest) for digest, expiry in self._consumed_jtis.items()]
|
||||
heapq.heapify(self._consumed_jti_expiry_heap)
|
||||
|
||||
@staticmethod
|
||||
def _bounded_float(
|
||||
value: typing.Any,
|
||||
*,
|
||||
default: float,
|
||||
minimum: float,
|
||||
maximum: float,
|
||||
) -> float:
|
||||
try:
|
||||
result = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
if not minimum <= result <= maximum:
|
||||
return default
|
||||
return result
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
import inspect
|
||||
|
||||
from ..core import app
|
||||
from . import operator
|
||||
@@ -63,6 +64,12 @@ class CommandManager:
|
||||
) -> typing.AsyncGenerator[command_context.CommandReturn, None]:
|
||||
"""执行命令"""
|
||||
|
||||
require_context = getattr(self.ap.plugin_connector, 'require_workspace_context', None)
|
||||
if require_context is not None:
|
||||
result = require_context(context)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
command_list = await self.ap.plugin_connector.list_commands(bound_plugins)
|
||||
|
||||
for command in command_list:
|
||||
@@ -89,6 +96,7 @@ class CommandManager:
|
||||
|
||||
_admins = await self.ap.persistence_mgr.execute_async(
|
||||
_sa.select(_BotAdmin).where(
|
||||
_BotAdmin.workspace_uuid == query.workspace_uuid,
|
||||
_BotAdmin.bot_uuid == (query.bot_uuid or ''),
|
||||
_BotAdmin.launcher_type == query.launcher_type.value,
|
||||
_BotAdmin.launcher_id == str(query.launcher_id),
|
||||
@@ -98,7 +106,11 @@ class CommandManager:
|
||||
privilege = 2
|
||||
|
||||
ctx = command_context.ExecuteContext(
|
||||
instance_uuid=query.instance_uuid,
|
||||
workspace_uuid=query.workspace_uuid,
|
||||
placement_generation=query.placement_generation,
|
||||
query_id=query.query_id,
|
||||
query_uuid=query.query_uuid,
|
||||
session=session,
|
||||
command_text=command_text,
|
||||
full_command_text=full_command_text,
|
||||
|
||||
+281
-55
@@ -2,9 +2,9 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import asyncio
|
||||
import contextlib
|
||||
import traceback
|
||||
import os
|
||||
import contextlib
|
||||
|
||||
from ..platform import botmgr as im_mgr
|
||||
from ..platform.webhook_pusher import WebhookPusher
|
||||
@@ -19,7 +19,7 @@ from ..plugin import connector as plugin_connector
|
||||
from ..pipeline import pool
|
||||
from ..pipeline import controller, pipelinemgr
|
||||
from ..pipeline import aggregator as message_aggregator
|
||||
from ..utils import version as version_mgr, proxy as proxy_mgr
|
||||
from ..utils import version as version_mgr, proxy as proxy_mgr, httpclient
|
||||
from ..persistence import mgr as persistencemgr
|
||||
from ..api.http.controller import main as http_controller
|
||||
from ..api.http.service import user as user_service
|
||||
@@ -37,7 +37,7 @@ from ..api.http.service import skill as skill_service
|
||||
from ..api.http.service import maintenance as maintenance_service
|
||||
from ..discover import engine as discover_engine
|
||||
from ..storage import mgr as storagemgr
|
||||
from ..utils import logcache
|
||||
from ..utils import bounded_executor, event_loop_monitor, logcache
|
||||
from . import taskmgr
|
||||
from . import entities as core_entities
|
||||
from ..rag.knowledge import kbmgr as rag_mgr
|
||||
@@ -46,6 +46,14 @@ from ..vector import mgr as vectordb_mgr
|
||||
from ..telemetry import telemetry as telemetry_module
|
||||
from ..survey import manager as survey_module
|
||||
from ..skill import manager as skill_mgr
|
||||
from ..workspace import service as workspace_service_module
|
||||
from ..workspace import collaboration as workspace_collaboration_module
|
||||
from ..workspace import invitation_delivery as invitation_delivery_module
|
||||
from ..cloud import bootstrap as cloud_bootstrap_module
|
||||
from ..cloud import launch as cloud_launch_module
|
||||
from ..cloud import directory_projection as cloud_directory_projection_module
|
||||
from ..cloud import entitlements as cloud_entitlements_module
|
||||
from ..api.http.context import ExecutionContext, PrincipalContext, PrincipalType
|
||||
|
||||
|
||||
class Application:
|
||||
@@ -120,6 +128,24 @@ class Application:
|
||||
|
||||
persistence_mgr: persistencemgr.PersistenceManager = None
|
||||
|
||||
workspace_service: workspace_service_module.WorkspaceService = None
|
||||
|
||||
workspace_collaboration_service: workspace_collaboration_module.WorkspaceCollaborationService = None
|
||||
|
||||
invitation_delivery_service: invitation_delivery_module.InvitationDeliveryService = None
|
||||
|
||||
space_launch_service: cloud_launch_module.SpaceLaunchService = None
|
||||
|
||||
deployment: cloud_bootstrap_module.OpenSourceDeployment | cloud_bootstrap_module.VerifiedCloudDeployment = None
|
||||
|
||||
deployment_admission: cloud_bootstrap_module.DeploymentAdmissionGuard = None
|
||||
|
||||
manifest_refresh_service: cloud_bootstrap_module.CloudManifestRefreshService | None = None
|
||||
|
||||
entitlement_resolver: cloud_entitlements_module.EntitlementResolver | None = None
|
||||
|
||||
directory_projection_service: cloud_directory_projection_module.DirectoryProjectionService | None = None
|
||||
|
||||
vector_db_mgr: vectordb_mgr.VectorDBManager = None
|
||||
|
||||
http_ctrl: http_controller.HTTPController = None
|
||||
@@ -166,15 +192,123 @@ class Application:
|
||||
|
||||
maintenance_service: maintenance_service.MaintenanceService = None
|
||||
|
||||
blocking_executor: bounded_executor.BoundedThreadPoolExecutor | None = None
|
||||
event_loop_monitor: event_loop_monitor.EventLoopLagMonitor
|
||||
|
||||
def __init__(self):
|
||||
self._shutdown_lock = asyncio.Lock()
|
||||
self._shutdown_complete = False
|
||||
self._shutdown_task: asyncio.Task | None = None
|
||||
self.event_loop_monitor = event_loop_monitor.EventLoopLagMonitor()
|
||||
|
||||
def get_runtime_resource_stats(self) -> dict[str, object]:
|
||||
"""Return aggregate O(1) counters for liveness and soak validation."""
|
||||
|
||||
try:
|
||||
asyncio_tasks = len(asyncio.all_tasks(self.event_loop))
|
||||
except (RuntimeError, TypeError):
|
||||
asyncio_tasks = 0
|
||||
|
||||
task_stats = self.task_mgr.get_stats() if self.task_mgr is not None else {}
|
||||
query_pool_stats = {}
|
||||
if self.query_pool is not None:
|
||||
query_pool_stats = {
|
||||
'queued': len(self.query_pool.queries),
|
||||
'cached': len(self.query_pool.cached_queries),
|
||||
'active_workspaces': len(self.query_pool.active_query_count_by_workspace),
|
||||
}
|
||||
|
||||
model_stats = {}
|
||||
if self.model_mgr is not None:
|
||||
model_stats = {
|
||||
'providers': len(self.model_mgr.provider_dict),
|
||||
'llms': len(self.model_mgr.llm_model_dict),
|
||||
'embeddings': len(self.model_mgr.embedding_model_dict),
|
||||
'rerankers': len(self.model_mgr.rerank_model_dict),
|
||||
}
|
||||
|
||||
runtime_stats = {
|
||||
'bots': len(getattr(self.platform_mgr, '_bots_by_key', {})),
|
||||
'pipelines': len(getattr(self.pipeline_mgr, '_pipelines_by_key', {})),
|
||||
'knowledge_bases': len(getattr(self.rag_mgr, 'knowledge_bases', {})),
|
||||
'message_aggregation_buffers': len(getattr(self.msg_aggregator, 'buffers', {})),
|
||||
'message_aggregation_scopes': len(
|
||||
getattr(
|
||||
self.msg_aggregator,
|
||||
'_buffer_counts_by_scope',
|
||||
{},
|
||||
)
|
||||
),
|
||||
'plugin_installations': len(
|
||||
getattr(
|
||||
self.plugin_connector,
|
||||
'_known_desired_states',
|
||||
{},
|
||||
)
|
||||
),
|
||||
}
|
||||
mcp_loader = getattr(self.tool_mgr, 'mcp_tool_loader', None)
|
||||
runtime_stats.update(
|
||||
{
|
||||
'mcp_sessions': len(getattr(mcp_loader, '_sessions', {})),
|
||||
'mcp_host_tasks': len(getattr(mcp_loader, '_hosted_mcp_tasks', ())),
|
||||
'mcp_dispatch_tasks': len(getattr(mcp_loader, '_host_dispatch_tasks', ())),
|
||||
'mcp_projection_retirements': len(getattr(mcp_loader, '_pending_projection_retirements', ())),
|
||||
'mcp_projection_reconcile_active': int(
|
||||
(
|
||||
projection_task := getattr(
|
||||
mcp_loader,
|
||||
'_projection_reconcile_task',
|
||||
None,
|
||||
)
|
||||
)
|
||||
is not None
|
||||
and not projection_task.done()
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
directory_stats = {}
|
||||
directory_snapshot = getattr(self.directory_projection_service, 'resource_snapshot', None)
|
||||
if callable(directory_snapshot):
|
||||
directory_stats = directory_snapshot()
|
||||
|
||||
database_stats = {}
|
||||
database_snapshot = getattr(self.persistence_mgr, 'get_resource_stats', None)
|
||||
if callable(database_snapshot):
|
||||
database_stats = database_snapshot()
|
||||
|
||||
return {
|
||||
'asyncio_tasks': asyncio_tasks,
|
||||
'event_loop': self.event_loop_monitor.snapshot(),
|
||||
'blocking_executor': (self.blocking_executor.snapshot() if self.blocking_executor is not None else {}),
|
||||
'application_tasks': task_stats,
|
||||
'database_pool': database_stats,
|
||||
'directory': directory_stats,
|
||||
'query_pool': query_pool_stats,
|
||||
'models': model_stats,
|
||||
'runtimes': runtime_stats,
|
||||
'telemetry_tasks': len(getattr(self.telemetry, 'send_tasks', ())),
|
||||
}
|
||||
|
||||
async def initialize(self):
|
||||
pass
|
||||
|
||||
async def run(self):
|
||||
self.event_loop_monitor.start()
|
||||
try:
|
||||
if self.directory_projection_service is not None:
|
||||
self.task_mgr.create_task(
|
||||
self.directory_projection_service.run(),
|
||||
name='cloud-directory-projection',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
if self.manifest_refresh_service is not None:
|
||||
self.task_mgr.create_task(
|
||||
self.manifest_refresh_service.run(),
|
||||
name='cloud-manifest-refresh',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
await self.plugin_connector.initialize_plugins()
|
||||
|
||||
# 后续可能会允许动态重启其他任务
|
||||
@@ -213,74 +347,128 @@ class Application:
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
|
||||
# Start monitoring data cleanup task if enabled
|
||||
monitoring_cfg = self.instance_config.data.get('monitoring', {})
|
||||
auto_cleanup_cfg = monitoring_cfg.get('auto_cleanup', {})
|
||||
if auto_cleanup_cfg.get('enabled', True):
|
||||
retention_days = self._get_positive_int_config(
|
||||
auto_cleanup_cfg.get('retention_days', 30),
|
||||
default=30,
|
||||
name='monitoring.auto_cleanup.retention_days',
|
||||
)
|
||||
delete_batch_size = self._get_positive_int_config(
|
||||
auto_cleanup_cfg.get('delete_batch_size', 1000),
|
||||
default=1000,
|
||||
name='monitoring.auto_cleanup.delete_batch_size',
|
||||
)
|
||||
check_interval_hours = self._get_positive_float_config(
|
||||
monitoring_enabled = auto_cleanup_cfg.get('enabled', True)
|
||||
retention_days = self._get_positive_int_config(
|
||||
auto_cleanup_cfg.get('retention_days', 30),
|
||||
default=30,
|
||||
name='monitoring.auto_cleanup.retention_days',
|
||||
)
|
||||
delete_batch_size = self._get_positive_int_config(
|
||||
auto_cleanup_cfg.get('delete_batch_size', 1000),
|
||||
default=1000,
|
||||
name='monitoring.auto_cleanup.delete_batch_size',
|
||||
)
|
||||
monitoring_interval_seconds = (
|
||||
self._get_positive_float_config(
|
||||
auto_cleanup_cfg.get('check_interval_hours', 1),
|
||||
default=1,
|
||||
name='monitoring.auto_cleanup.check_interval_hours',
|
||||
)
|
||||
* 3600
|
||||
)
|
||||
|
||||
async def monitoring_cleanup_loop():
|
||||
check_interval_seconds = check_interval_hours * 3600
|
||||
while True:
|
||||
try:
|
||||
deleted = await self.monitoring_service.cleanup_expired_records(
|
||||
retention_days,
|
||||
batch_size=delete_batch_size,
|
||||
)
|
||||
total_deleted = sum(deleted.values())
|
||||
if total_deleted > 0:
|
||||
self.logger.info(
|
||||
f'Monitoring auto-cleanup: deleted {total_deleted} expired records '
|
||||
f'(retention={retention_days}d): {deleted}'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Monitoring auto-cleanup error: {e}')
|
||||
await asyncio.sleep(check_interval_seconds)
|
||||
|
||||
self.task_mgr.create_task(
|
||||
monitoring_cleanup_loop(),
|
||||
name='monitoring-cleanup',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
|
||||
# Start storage/log maintenance task if enabled
|
||||
storage_cleanup_cfg = self.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
if storage_cleanup_cfg.get('enabled', True) and self.maintenance_service is not None:
|
||||
check_interval_hours = self._get_positive_float_config(
|
||||
storage_enabled = storage_cleanup_cfg.get('enabled', True) and self.maintenance_service is not None
|
||||
storage_interval_seconds = (
|
||||
self._get_positive_float_config(
|
||||
storage_cleanup_cfg.get('check_interval_hours', 1),
|
||||
default=1,
|
||||
name='storage.cleanup.check_interval_hours',
|
||||
)
|
||||
* 3600
|
||||
)
|
||||
|
||||
async def storage_cleanup_loop():
|
||||
check_interval_seconds = check_interval_hours * 3600
|
||||
maintenance_intervals: dict[str, float] = {}
|
||||
if monitoring_enabled:
|
||||
maintenance_intervals['monitoring'] = monitoring_interval_seconds
|
||||
if storage_enabled:
|
||||
maintenance_intervals['storage'] = storage_interval_seconds
|
||||
if self.workspace_collaboration_service is not None:
|
||||
maintenance_intervals['invitations'] = 3600.0
|
||||
|
||||
if maintenance_intervals:
|
||||
|
||||
async def resource_maintenance_loop():
|
||||
"""Share tenant discovery and serialize periodic maintenance."""
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
started_at = loop.time()
|
||||
next_due = {name: started_at + interval for name, interval in maintenance_intervals.items()}
|
||||
while True:
|
||||
await asyncio.sleep(max(min(next_due.values()) - loop.time(), 0.0))
|
||||
observed_at = loop.time()
|
||||
due = {name for name, due_at in next_due.items() if due_at <= observed_at}
|
||||
if not due:
|
||||
continue
|
||||
try:
|
||||
deleted = await self.maintenance_service.cleanup_expired_files()
|
||||
total_deleted = sum(deleted.values())
|
||||
if total_deleted > 0:
|
||||
self.logger.info(f'Storage maintenance: deleted expired files: {deleted}')
|
||||
except Exception as e:
|
||||
self.logger.warning(f'Storage maintenance error: {e}')
|
||||
await asyncio.sleep(check_interval_seconds)
|
||||
bindings = await self.workspace_service.list_active_execution_bindings()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.logger.warning(f'Resource maintenance Workspace discovery failed: {exc}')
|
||||
else:
|
||||
for binding in bindings:
|
||||
context = ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
|
||||
)
|
||||
if 'monitoring' in due:
|
||||
try:
|
||||
deleted = await self.monitoring_service.cleanup_expired_records(
|
||||
context,
|
||||
retention_days,
|
||||
batch_size=delete_batch_size,
|
||||
)
|
||||
total_deleted = sum(deleted.values())
|
||||
if total_deleted > 0:
|
||||
self.logger.info(
|
||||
f'Monitoring auto-cleanup: deleted {total_deleted} expired records '
|
||||
f'for Workspace {context.workspace_uuid} '
|
||||
f'(retention={retention_days}d): {deleted}'
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.logger.warning(
|
||||
f'Monitoring auto-cleanup failed for '
|
||||
f'Workspace {context.workspace_uuid}: {exc}'
|
||||
)
|
||||
if 'storage' in due:
|
||||
try:
|
||||
deleted = await self.maintenance_service.cleanup_expired_files(context)
|
||||
total_deleted = sum(deleted.values())
|
||||
if total_deleted > 0:
|
||||
self.logger.info(
|
||||
f'Storage maintenance for Workspace {context.workspace_uuid}: '
|
||||
f'deleted expired files: {deleted}'
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.logger.warning(
|
||||
f'Storage maintenance failed for Workspace {context.workspace_uuid}: {exc}'
|
||||
)
|
||||
if 'invitations' in due:
|
||||
try:
|
||||
await self.workspace_collaboration_service.cleanup_expired_invitations(
|
||||
active_bindings=bindings,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
self.logger.warning(f'Expired Workspace invitation cleanup failed: {exc}')
|
||||
|
||||
completed_at = loop.time()
|
||||
for name in due:
|
||||
next_due[name] = completed_at + maintenance_intervals[name]
|
||||
|
||||
self.task_mgr.create_task(
|
||||
storage_cleanup_loop(),
|
||||
name='storage-maintenance',
|
||||
resource_maintenance_loop(),
|
||||
name='resource-maintenance',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
)
|
||||
|
||||
@@ -328,30 +516,68 @@ class Application:
|
||||
|
||||
if self.task_mgr is not None:
|
||||
self.task_mgr.cancel_by_scope(core_entities.LifecycleControlScope.APPLICATION)
|
||||
with contextlib.suppress(Exception):
|
||||
await self.event_loop_monitor.stop()
|
||||
mcp_mount = getattr(self.http_ctrl, 'mcp_mount', None)
|
||||
if mcp_mount is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await mcp_mount.stop_session_manager()
|
||||
if self.platform_mgr is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.platform_mgr.shutdown()
|
||||
if self.tool_mgr is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.tool_mgr.shutdown()
|
||||
if self.model_mgr is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.model_mgr.shutdown()
|
||||
if self.box_service is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.box_service.shutdown()
|
||||
if self.plugin_connector is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.plugin_connector.aclose()
|
||||
if self.telemetry is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.telemetry.shutdown()
|
||||
if self.vector_db_mgr is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.vector_db_mgr.shutdown()
|
||||
if self.storage_mgr is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self.storage_mgr.shutdown()
|
||||
manifest_provider = getattr(self.deployment, 'manifest_provider', None)
|
||||
if manifest_provider is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await manifest_provider.aclose()
|
||||
|
||||
if self.task_mgr is not None:
|
||||
tasks = [wrapper.task for wrapper in self.task_mgr.tasks if not wrapper.task.done()]
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
with contextlib.suppress(Exception):
|
||||
await httpclient.close_all()
|
||||
persistence_shutdown = getattr(self.persistence_mgr, 'shutdown', None)
|
||||
if callable(persistence_shutdown):
|
||||
with contextlib.suppress(Exception):
|
||||
await persistence_shutdown()
|
||||
else:
|
||||
# Compatibility for lightweight test/application doubles.
|
||||
persistence_db = getattr(self.persistence_mgr, 'db', None)
|
||||
persistence_engine = getattr(persistence_db, 'engine', None)
|
||||
if persistence_engine is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await persistence_engine.dispose()
|
||||
self._shutdown_complete = True
|
||||
|
||||
def dispose(self):
|
||||
"""Compatibility wrapper for callers that cannot await shutdown."""
|
||||
if self._shutdown_complete:
|
||||
return
|
||||
loop = self.event_loop
|
||||
if loop is not None and not loop.is_closed():
|
||||
loop.create_task(self.shutdown())
|
||||
if self._shutdown_task is None or self._shutdown_task.done():
|
||||
self._shutdown_task = loop.create_task(self.shutdown())
|
||||
return
|
||||
if self.plugin_connector is not None:
|
||||
self.plugin_connector.dispose()
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import traceback
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
|
||||
from . import app
|
||||
@@ -32,14 +33,22 @@ async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
|
||||
|
||||
ap.event_loop = loop
|
||||
|
||||
# Execute startup stage
|
||||
for stage_name in stage_order:
|
||||
stage_cls = stage.preregistered_stages[stage_name]
|
||||
stage_inst = stage_cls()
|
||||
try:
|
||||
# Execute startup stage
|
||||
for stage_name in stage_order:
|
||||
stage_cls = stage.preregistered_stages[stage_name]
|
||||
stage_inst = stage_cls()
|
||||
|
||||
await stage_inst.run(ap)
|
||||
await stage_inst.run(ap)
|
||||
|
||||
await ap.initialize()
|
||||
await ap.initialize()
|
||||
except BaseException:
|
||||
# ``main()`` cannot clean up a partially built application because
|
||||
# ``make_app()`` has not returned it yet. Release managers, pools and
|
||||
# child processes that earlier startup stages already attached.
|
||||
with contextlib.suppress(BaseException):
|
||||
await ap.shutdown()
|
||||
raise
|
||||
|
||||
return ap
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
class TaskCapacityError(RuntimeError):
|
||||
"""Raised when the configured user-task admission limit is exhausted."""
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .. import stage, app
|
||||
from ...utils import version, proxy
|
||||
from ...utils import version, proxy, constants
|
||||
from ...pipeline import pool, controller, pipelinemgr
|
||||
from ...pipeline import aggregator as message_aggregator
|
||||
from ...box import service as box_service
|
||||
@@ -37,6 +37,16 @@ from ...vector import mgr as vectordb_mgr
|
||||
from .. import taskmgr
|
||||
from ...telemetry import telemetry as telemetry_module
|
||||
from ...survey import manager as survey_module
|
||||
from ...workspace import service as workspace_service_module
|
||||
from ...workspace import collaboration as workspace_collaboration_module
|
||||
from ...workspace import invitation_delivery as invitation_delivery_module
|
||||
from ...cloud import bootstrap as cloud_bootstrap
|
||||
from ...cloud import launch as cloud_launch_module
|
||||
from ...cloud.directory import directory_projection_limits_from_config
|
||||
from ...cloud.directory_projection import DirectoryProjectionService
|
||||
from ...cloud.entitlements import EntitlementResolver
|
||||
from ...api.http.context import ExecutionContext, PrincipalContext, PrincipalType
|
||||
from ...api.http.authz import WorkspaceRequiredError
|
||||
|
||||
|
||||
@stage.stage_class('BuildAppStage')
|
||||
@@ -45,15 +55,43 @@ class BuildAppStage(stage.BootingStage):
|
||||
|
||||
async def run(self, ap: app.Application):
|
||||
"""Build LangBot application"""
|
||||
# Multi-Workspace mode is selected only by an installed closed
|
||||
# bootstrap that returns a verified Manifest receipt. Mutable values
|
||||
# such as system.edition are intentionally absent from this boundary.
|
||||
deployment = await cloud_bootstrap.resolve_deployment(
|
||||
instance_uuid=constants.instance_id,
|
||||
instance_config=ap.instance_config.data,
|
||||
)
|
||||
ap.deployment = deployment
|
||||
ap.deployment_admission = cloud_bootstrap.DeploymentAdmissionGuard(
|
||||
constants.instance_id,
|
||||
deployment,
|
||||
)
|
||||
ap.manifest_refresh_service = (
|
||||
cloud_bootstrap.CloudManifestRefreshService(
|
||||
ap.deployment_admission,
|
||||
deployment.manifest_provider,
|
||||
ap.logger,
|
||||
)
|
||||
if deployment.multi_workspace_enabled
|
||||
else None
|
||||
)
|
||||
ap.entitlement_resolver = (
|
||||
EntitlementResolver(
|
||||
constants.instance_id,
|
||||
deployment.entitlement_provider,
|
||||
deployment_admission=ap.deployment_admission.require_active,
|
||||
)
|
||||
if deployment.multi_workspace_enabled
|
||||
else None
|
||||
)
|
||||
|
||||
ap.task_mgr = taskmgr.AsyncTaskManager(ap)
|
||||
|
||||
discover = discover_engine.ComponentDiscoveryEngine(ap)
|
||||
discover.discover_blueprint('templates/components.yaml')
|
||||
ap.discover = discover
|
||||
|
||||
user_service_inst = user_service.UserService(ap)
|
||||
ap.user_service = user_service_inst
|
||||
|
||||
space_service_inst = space_service.SpaceService(ap)
|
||||
ap.space_service = space_service_inst
|
||||
|
||||
@@ -98,23 +136,77 @@ class BuildAppStage(stage.BootingStage):
|
||||
await ver_mgr.initialize()
|
||||
ap.ver_mgr = ver_mgr
|
||||
|
||||
ap.query_pool = pool.QueryPool()
|
||||
|
||||
log_cache = logcache.LogCache()
|
||||
ap.log_cache = log_cache
|
||||
|
||||
storage_mgr_inst = storagemgr.StorageMgr(ap)
|
||||
await storage_mgr_inst.initialize()
|
||||
ap.storage_mgr = storage_mgr_inst
|
||||
await storage_mgr_inst.initialize()
|
||||
|
||||
persistence_mgr_inst = persistencemgr.PersistenceManager(ap)
|
||||
persistence_mgr_inst = persistencemgr.PersistenceManager(
|
||||
ap,
|
||||
mode=persistencemgr.PersistenceMode(deployment.persistence_mode),
|
||||
)
|
||||
ap.persistence_mgr = persistence_mgr_inst
|
||||
await persistence_mgr_inst.initialize()
|
||||
|
||||
if deployment.multi_workspace_enabled:
|
||||
directory_projection_service = DirectoryProjectionService(
|
||||
ap,
|
||||
deployment.directory_provider,
|
||||
constants.instance_id,
|
||||
limits=directory_projection_limits_from_config(ap.instance_config.data),
|
||||
)
|
||||
await directory_projection_service.initialize()
|
||||
ap.directory_projection_service = directory_projection_service
|
||||
|
||||
workspace_policy = deployment.workspace_policy
|
||||
workspace_service_inst = workspace_service_module.WorkspaceService(
|
||||
ap,
|
||||
policy=workspace_policy,
|
||||
)
|
||||
if not workspace_policy.multi_workspace_enabled:
|
||||
await workspace_service_inst.ensure_singleton_workspace()
|
||||
ap.workspace_service = workspace_service_inst
|
||||
if workspace_policy.multi_workspace_enabled:
|
||||
# Directory refresh starts in Application.run(), after this serial
|
||||
# build graph. Share one validated immutable binding snapshot
|
||||
# across model/platform/pipeline/RAG/plugin initialization instead
|
||||
# of repeating tenant validation for every manager.
|
||||
await workspace_service_inst.prime_startup_execution_bindings()
|
||||
|
||||
ap.workspace_collaboration_service = workspace_collaboration_module.WorkspaceCollaborationService(
|
||||
ap,
|
||||
workspace_service_inst,
|
||||
)
|
||||
ap.invitation_delivery_service = invitation_delivery_module.InvitationDeliveryService(ap)
|
||||
ap.space_launch_service = cloud_launch_module.SpaceLaunchService(ap)
|
||||
|
||||
user_service_inst = user_service.UserService(ap)
|
||||
ap.user_service = user_service_inst
|
||||
|
||||
async def resolve_singleton_execution_context() -> ExecutionContext:
|
||||
if workspace_policy.multi_workspace_enabled:
|
||||
raise WorkspaceRequiredError('Cloud runtime work requires an explicit Workspace context')
|
||||
binding = await workspace_service_inst.get_local_execution_binding()
|
||||
return ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
|
||||
)
|
||||
|
||||
concurrency_config = ap.instance_config.data.get('concurrency', {})
|
||||
ap.query_pool = pool.QueryPool(
|
||||
singleton_context_resolver=resolve_singleton_execution_context,
|
||||
max_queries=int(concurrency_config.get('pending_queries', 1000)),
|
||||
max_queries_per_workspace=int(concurrency_config.get('pending_queries_per_workspace', 100)),
|
||||
)
|
||||
|
||||
# Telemetry manager: attach to app so other components can call via self.ap.telemetry
|
||||
telemetry_inst = telemetry_module.TelemetryManager(ap)
|
||||
await telemetry_inst.initialize()
|
||||
ap.telemetry = telemetry_inst
|
||||
await telemetry_inst.initialize()
|
||||
|
||||
# Survey manager
|
||||
survey_inst = survey_module.SurveyManager(ap)
|
||||
@@ -134,16 +226,16 @@ class BuildAppStage(stage.BootingStage):
|
||||
ap.sess_mgr = llm_session_mgr_inst
|
||||
|
||||
box_service_inst = box_service.BoxService(ap)
|
||||
await box_service_inst.initialize()
|
||||
ap.box_service = box_service_inst
|
||||
await box_service_inst.initialize()
|
||||
|
||||
llm_tool_mgr_inst = llm_tool_mgr.ToolManager(ap)
|
||||
await llm_tool_mgr_inst.initialize()
|
||||
ap.tool_mgr = llm_tool_mgr_inst
|
||||
await llm_tool_mgr_inst.initialize()
|
||||
|
||||
im_mgr_inst = im_mgr.PlatformManager(ap=ap)
|
||||
await im_mgr_inst.initialize()
|
||||
ap.platform_mgr = im_mgr_inst
|
||||
await im_mgr_inst.initialize()
|
||||
|
||||
# Initialize webhook pusher
|
||||
webhook_pusher_inst = WebhookPusher(ap)
|
||||
@@ -171,12 +263,12 @@ class BuildAppStage(stage.BootingStage):
|
||||
|
||||
# 初始化向量数据库管理器
|
||||
vectordb_mgr_inst = vectordb_mgr.VectorDBManager(ap)
|
||||
await vectordb_mgr_inst.initialize()
|
||||
ap.vector_db_mgr = vectordb_mgr_inst
|
||||
await vectordb_mgr_inst.initialize()
|
||||
|
||||
http_ctrl = http_controller.HTTPController(ap)
|
||||
await http_ctrl.initialize()
|
||||
ap.http_ctrl = http_ctrl
|
||||
await http_ctrl.initialize()
|
||||
|
||||
monitoring_service_inst = monitoring_service.MonitoringService(ap)
|
||||
ap.monitoring_service = monitoring_service_inst
|
||||
@@ -196,6 +288,7 @@ class BuildAppStage(stage.BootingStage):
|
||||
ap.logger.warning(f'Plugin runtime unavailable during startup; reconnecting in background: {exc}')
|
||||
plugin_connector_inst.schedule_reconnect()
|
||||
ap.plugin_connector = plugin_connector_inst
|
||||
workspace_service_inst.release_startup_execution_bindings()
|
||||
|
||||
ctrl = controller.Controller(ap)
|
||||
ap.ctrl = ctrl
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import copy
|
||||
from typing import Any
|
||||
from langbot.pkg.utils import constants
|
||||
from langbot.pkg.utils import bounded_executor, constants
|
||||
import yaml
|
||||
import importlib.resources as resources
|
||||
import uuid
|
||||
@@ -12,6 +13,102 @@ from .. import stage, app
|
||||
from ..bootutils import config
|
||||
|
||||
|
||||
_RUNTIME_POLICY_DEFAULTS = {
|
||||
'cloud': {
|
||||
'directory': {
|
||||
'max_active_workspaces': 1000,
|
||||
'max_snapshot_workspaces': 1000,
|
||||
'max_snapshot_memberships': 20000,
|
||||
'max_response_bytes': 33554432,
|
||||
}
|
||||
},
|
||||
'database': {
|
||||
'postgresql': {
|
||||
'pool_size': 10,
|
||||
'max_overflow': 10,
|
||||
'pool_timeout_seconds': 30,
|
||||
'pool_recycle_seconds': 1800,
|
||||
'statement_timeout_ms': 60000,
|
||||
'lock_timeout_ms': 5000,
|
||||
'idle_in_transaction_session_timeout_ms': 60000,
|
||||
}
|
||||
},
|
||||
'system': {
|
||||
'blocking_executor': {
|
||||
'max_workers': bounded_executor.DEFAULT_MAX_WORKERS,
|
||||
'max_pending': bounded_executor.DEFAULT_MAX_PENDING,
|
||||
'max_inflight_per_scope': (bounded_executor.DEFAULT_MAX_INFLIGHT_PER_SCOPE),
|
||||
}
|
||||
},
|
||||
'plugin': {
|
||||
'worker': {
|
||||
'max_cpus': 1.0,
|
||||
'max_memory_mb': 512,
|
||||
'max_pids': 128,
|
||||
'max_open_files': 256,
|
||||
'max_file_size_mb': 512,
|
||||
'max_workers': 16,
|
||||
'max_total_cpus': 8.0,
|
||||
'max_total_memory_mb': 8192,
|
||||
'max_installations': 10000,
|
||||
'max_concurrent_restarts': 1,
|
||||
'restart_failure_threshold': 8,
|
||||
'restart_failure_window_seconds': 30.0,
|
||||
'restart_circuit_open_seconds': 60.0,
|
||||
'require_hard_limits': False,
|
||||
}
|
||||
},
|
||||
'mcp': {'stdio': {'enabled': True}},
|
||||
'monitoring': {
|
||||
'query_limits': {
|
||||
'page_rows': 1000,
|
||||
'export_rows': 10000,
|
||||
'detail_rows': 2000,
|
||||
'timeseries_buckets': 1000,
|
||||
'max_offset': 1000000,
|
||||
},
|
||||
'auto_cleanup': {'max_batches_per_table_per_run': 4},
|
||||
},
|
||||
'storage': {
|
||||
'max_object_read_bytes': 10485760,
|
||||
'cleanup': {'max_files_per_run': 1000},
|
||||
},
|
||||
'webhooks': {
|
||||
'max_per_workspace': 16,
|
||||
'max_inflight_requests': 16,
|
||||
},
|
||||
'box': {
|
||||
'limits': {
|
||||
'max_workspace_entries': 100000,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _complete_runtime_policy_defaults(cfg: dict) -> dict:
|
||||
"""Backfill typed security-policy leaves before applying env overrides.
|
||||
|
||||
The historic config loader intentionally does not deep-complete the whole
|
||||
template. These fields are different: their native env overrides must
|
||||
retain boolean/numeric types on upgraded instances, so their defaults must
|
||||
exist before ``CLOUD__...``, ``PLUGIN__...`` and ``MCP__...`` are parsed.
|
||||
"""
|
||||
|
||||
def merge(target: dict, defaults: dict, path: tuple[str, ...] = ()) -> None:
|
||||
for key, default in defaults.items():
|
||||
if key not in target:
|
||||
target[key] = copy.deepcopy(default)
|
||||
continue
|
||||
if isinstance(default, dict):
|
||||
if not isinstance(target[key], dict):
|
||||
dotted_path = '.'.join((*path, key))
|
||||
raise ValueError(f'{dotted_path} must be a mapping')
|
||||
merge(target[key], default, (*path, key))
|
||||
|
||||
merge(cfg, _RUNTIME_POLICY_DEFAULTS)
|
||||
return cfg
|
||||
|
||||
|
||||
def _apply_env_overrides_to_config(cfg: dict) -> dict:
|
||||
"""Apply environment variable overrides to data/config.yaml
|
||||
|
||||
@@ -64,11 +161,19 @@ def _apply_env_overrides_to_config(cfg: dict) -> dict:
|
||||
if '__' not in env_key:
|
||||
continue
|
||||
|
||||
print(f'apply env overrides to config: env_key: {env_key}, env_value: {env_value}')
|
||||
|
||||
# Convert environment variable name to config path
|
||||
# e.g., CONCURRENCY__PIPELINE -> ['concurrency', 'pipeline']
|
||||
keys = [key.lower() for key in env_key.split('__')]
|
||||
# macOS and some launchers expose variables such as
|
||||
# ``__CF_USER_TEXT_ENCODING``. They are not LangBot config paths and
|
||||
# must not create an empty top-level YAML key when config is dumped.
|
||||
if any(not key for key in keys):
|
||||
continue
|
||||
|
||||
# Values may contain database passwords, runtime control tokens, or
|
||||
# provider credentials. Keep the useful audit breadcrumb without ever
|
||||
# copying the secret into startup logs.
|
||||
print(f'apply env override to config: env_key: {env_key}')
|
||||
|
||||
# Navigate to the target value and validate the path
|
||||
current = cfg
|
||||
@@ -150,9 +255,21 @@ class LoadConfigStage(stage.BootingStage):
|
||||
|
||||
ap.instance_config = await config.load_yaml_config('data/config.yaml', 'config.yaml', completion=False)
|
||||
|
||||
# Deep-complete only typed execution-policy fields. This keeps native
|
||||
# env coercion reliable for existing data/config.yaml files.
|
||||
ap.instance_config.data = _complete_runtime_policy_defaults(ap.instance_config.data)
|
||||
|
||||
# Apply environment variable overrides to data/config.yaml
|
||||
ap.instance_config.data = _apply_env_overrides_to_config(ap.instance_config.data)
|
||||
|
||||
blocking_config = ap.instance_config.data['system']['blocking_executor']
|
||||
ap.blocking_executor = bounded_executor.configure_bounded_default_executor(
|
||||
ap.event_loop,
|
||||
max_workers=blocking_config['max_workers'],
|
||||
max_pending=blocking_config['max_pending'],
|
||||
max_inflight_per_scope=blocking_config['max_inflight_per_scope'],
|
||||
)
|
||||
|
||||
await ap.instance_config.dump_config()
|
||||
|
||||
# load or generate instance id
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from .. import stage, app, note
|
||||
from .. import entities as core_entities
|
||||
from ...utils import importutil
|
||||
|
||||
from .. import notes
|
||||
@@ -31,6 +30,12 @@ class ShowNotesStage(stage.BootingStage):
|
||||
if msg:
|
||||
ap.logger.log(level, msg)
|
||||
|
||||
asyncio.create_task(ayield_note(note_inst))
|
||||
ap.task_mgr.create_task(
|
||||
ayield_note(note_inst),
|
||||
kind='launch-note',
|
||||
name=f'launch-note-{note_cls.__name__}',
|
||||
scopes=[core_entities.LifecycleControlScope.APPLICATION],
|
||||
instance_uuid=ap.workspace_service.instance_uuid,
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import typing
|
||||
|
||||
from ..utils import bounded_executor
|
||||
|
||||
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
|
||||
def create_detached_task(
|
||||
coro: typing.Coroutine[typing.Any, typing.Any, T],
|
||||
*,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
name: str | None = None,
|
||||
after_commit_manager: typing.Any | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
) -> asyncio.Task[T]:
|
||||
"""Create a task that inherits no request-local ContextVars.
|
||||
|
||||
A normal ``asyncio.create_task`` copies the caller's context. That is
|
||||
unsafe for work which outlives an HTTP request because it can copy the
|
||||
request's active database transaction or trusted tenant scope into a
|
||||
different asyncio task. Detached work must receive durable identity such
|
||||
as ``ExecutionContext`` through explicit arguments and establish its own
|
||||
tenant scope or unit of work whenever it accesses persistence.
|
||||
"""
|
||||
|
||||
task_loop = loop or asyncio.get_running_loop()
|
||||
gate: asyncio.Future[None] | None = None
|
||||
# Inspect the type so dynamic Mock/AsyncMock attributes do not turn into a
|
||||
# fake gate in lightweight tests or embedders.
|
||||
gate_factory = getattr(type(after_commit_manager), 'create_after_commit_gate', None)
|
||||
if callable(gate_factory):
|
||||
gate = gate_factory(after_commit_manager)
|
||||
task_coro = _wait_for_commit(coro, gate) if gate is not None else coro
|
||||
if workspace_uuid is not None:
|
||||
task_coro = bounded_executor.run_in_blocking_work_scope(
|
||||
task_coro,
|
||||
workspace_uuid,
|
||||
)
|
||||
return task_loop.create_task(task_coro, name=name, context=contextvars.Context())
|
||||
|
||||
|
||||
async def _wait_for_commit(
|
||||
coro: typing.Coroutine[typing.Any, typing.Any, T],
|
||||
gate: asyncio.Future[None],
|
||||
) -> T:
|
||||
try:
|
||||
await gate
|
||||
except BaseException:
|
||||
coro.close()
|
||||
raise
|
||||
return await coro
|
||||
|
||||
|
||||
async def run_in_workspace_uow(
|
||||
ap: typing.Any,
|
||||
workspace_uuid: str,
|
||||
operation: typing.Callable[[], typing.Awaitable[T]],
|
||||
) -> T:
|
||||
"""Run one short persistence section in a detached Cloud task scope.
|
||||
|
||||
This helper deliberately scopes only the supplied operation. Callers
|
||||
should not wrap long-running network or runtime work in a database
|
||||
transaction.
|
||||
"""
|
||||
|
||||
persistence_mgr = getattr(ap, 'persistence_mgr', None)
|
||||
if persistence_mgr is None:
|
||||
return await operation()
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if not cloud_runtime:
|
||||
return await operation()
|
||||
|
||||
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Detached Cloud tasks require an explicit tenant unit of work')
|
||||
async with tenant_uow(workspace_uuid):
|
||||
return await operation()
|
||||
@@ -7,6 +7,8 @@ import time
|
||||
|
||||
from . import app
|
||||
from . import entities as core_entities
|
||||
from .errors import TaskCapacityError
|
||||
from .task_boundary import create_detached_task
|
||||
|
||||
|
||||
class TaskContext:
|
||||
@@ -21,13 +23,18 @@ class TaskContext:
|
||||
metadata: dict
|
||||
"""Structured metadata for progress reporting"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, max_log_chars: int = 200000):
|
||||
self.current_action = 'default'
|
||||
self.log = ''
|
||||
self.metadata = {}
|
||||
self.max_log_chars = max(int(max_log_chars), 1)
|
||||
|
||||
def _log(self, msg: str):
|
||||
self.log += msg + '\n'
|
||||
if len(self.log) > self.max_log_chars:
|
||||
marker = '[older task output truncated]\n'
|
||||
keep = max(self.max_log_chars - len(marker), 0)
|
||||
self.log = marker + (self.log[-keep:] if keep else '')
|
||||
|
||||
def set_current_action(self, action: str):
|
||||
self.current_action = action
|
||||
@@ -98,6 +105,15 @@ class TaskWrapper:
|
||||
scopes: list[core_entities.LifecycleControlScope]
|
||||
"""Task scope"""
|
||||
|
||||
instance_uuid: str | None
|
||||
"""Owning LangBot instance for a tenant user task."""
|
||||
|
||||
workspace_uuid: str | None
|
||||
"""Owning Workspace for a tenant user task."""
|
||||
|
||||
placement_generation: int | None
|
||||
"""Workspace execution fence captured when the task was created."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: app.Application,
|
||||
@@ -108,18 +124,30 @@ class TaskWrapper:
|
||||
label: str = '',
|
||||
context: TaskContext = None,
|
||||
scopes: list[core_entities.LifecycleControlScope] = [core_entities.LifecycleControlScope.APPLICATION],
|
||||
instance_uuid: str | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
placement_generation: int | None = None,
|
||||
):
|
||||
self.id = TaskWrapper._id_index
|
||||
TaskWrapper._id_index += 1
|
||||
self.ap = ap
|
||||
self.task_context = context or TaskContext()
|
||||
self.task = self.ap.event_loop.create_task(coro)
|
||||
self.task = create_detached_task(
|
||||
coro,
|
||||
loop=self.ap.event_loop,
|
||||
name=name or None,
|
||||
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
|
||||
workspace_uuid=workspace_uuid,
|
||||
)
|
||||
self.task_type = task_type
|
||||
self.kind = kind
|
||||
self.name = name
|
||||
self.label = label if label != '' else name
|
||||
self.task.set_name(name)
|
||||
self.scopes = scopes
|
||||
self.instance_uuid = instance_uuid
|
||||
self.workspace_uuid = workspace_uuid
|
||||
self.placement_generation = placement_generation
|
||||
self.created_at = time.time()
|
||||
|
||||
def assume_exception(self):
|
||||
@@ -155,6 +183,8 @@ class TaskWrapper:
|
||||
'kind': self.kind,
|
||||
'name': self.name,
|
||||
'label': self.label,
|
||||
'workspace_uuid': self.workspace_uuid,
|
||||
'placement_generation': self.placement_generation,
|
||||
'scopes': [scope.value for scope in self.scopes],
|
||||
'created_at': self.created_at,
|
||||
'task_context': self.task_context.to_dict(),
|
||||
@@ -184,6 +214,39 @@ class AsyncTaskManager:
|
||||
self.ap = ap
|
||||
self.tasks = []
|
||||
|
||||
def _task_log_limit(self) -> int:
|
||||
value = self.ap.instance_config.data.get('system', {}).get('task_retention', {}).get('max_log_chars', 200000)
|
||||
try:
|
||||
value = int(value)
|
||||
except (TypeError, ValueError):
|
||||
value = 200000
|
||||
return max(value, 1)
|
||||
|
||||
def _user_task_limit(self, name: str, default: int) -> int:
|
||||
value = self.ap.instance_config.data.get('system', {}).get('task_retention', {}).get(name, default)
|
||||
try:
|
||||
value = int(value)
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return max(value, 1)
|
||||
|
||||
def _admit_user_task(self, coro: typing.Coroutine, workspace_uuid: str | None) -> None:
|
||||
active_user_tasks = [
|
||||
wrapper for wrapper in self.tasks if wrapper.task_type == 'user' and not wrapper.task.done()
|
||||
]
|
||||
global_limit = self._user_task_limit('max_active_user_tasks', 256)
|
||||
if len(active_user_tasks) >= global_limit:
|
||||
coro.close()
|
||||
raise TaskCapacityError('The instance has too many active user operations')
|
||||
|
||||
if workspace_uuid is None:
|
||||
return
|
||||
workspace_limit = self._user_task_limit('max_active_user_tasks_per_workspace', 8)
|
||||
active_workspace_tasks = sum(1 for wrapper in active_user_tasks if wrapper.workspace_uuid == workspace_uuid)
|
||||
if active_workspace_tasks >= workspace_limit:
|
||||
coro.close()
|
||||
raise TaskCapacityError('The Workspace has too many active user operations')
|
||||
|
||||
def create_task(
|
||||
self,
|
||||
coro: typing.Coroutine,
|
||||
@@ -193,8 +256,30 @@ class AsyncTaskManager:
|
||||
label: str = '',
|
||||
context: TaskContext = None,
|
||||
scopes: list[core_entities.LifecycleControlScope] = [core_entities.LifecycleControlScope.APPLICATION],
|
||||
instance_uuid: str | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
placement_generation: int | None = None,
|
||||
) -> TaskWrapper:
|
||||
wrapper = TaskWrapper(self.ap, coro, task_type, kind, name, label, context, scopes)
|
||||
if context is None:
|
||||
context = TaskContext(max_log_chars=self._task_log_limit())
|
||||
else:
|
||||
context.max_log_chars = self._task_log_limit()
|
||||
if len(context.log) > context.max_log_chars:
|
||||
context.log = context.log[-context.max_log_chars :]
|
||||
|
||||
wrapper = TaskWrapper(
|
||||
self.ap,
|
||||
coro,
|
||||
task_type,
|
||||
kind,
|
||||
name,
|
||||
label,
|
||||
context,
|
||||
scopes,
|
||||
instance_uuid,
|
||||
workspace_uuid,
|
||||
placement_generation,
|
||||
)
|
||||
self.tasks.append(wrapper)
|
||||
wrapper.task.add_done_callback(lambda _: self._prune_completed_tasks())
|
||||
self._prune_completed_tasks()
|
||||
@@ -208,8 +293,23 @@ class AsyncTaskManager:
|
||||
label: str = '',
|
||||
context: TaskContext = None,
|
||||
scopes: list[core_entities.LifecycleControlScope] = [core_entities.LifecycleControlScope.APPLICATION],
|
||||
instance_uuid: str | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
placement_generation: int | None = None,
|
||||
) -> TaskWrapper:
|
||||
return self.create_task(coro, 'user', kind, name, label, context, scopes)
|
||||
self._admit_user_task(coro, workspace_uuid)
|
||||
return self.create_task(
|
||||
coro,
|
||||
'user',
|
||||
kind,
|
||||
name,
|
||||
label,
|
||||
context,
|
||||
scopes,
|
||||
instance_uuid,
|
||||
workspace_uuid,
|
||||
placement_generation,
|
||||
)
|
||||
|
||||
async def wait_all(self):
|
||||
await asyncio.gather(*[t.task for t in self.tasks], return_exceptions=True)
|
||||
@@ -221,12 +321,20 @@ class AsyncTaskManager:
|
||||
self,
|
||||
type: str = None,
|
||||
kind: str = None,
|
||||
*,
|
||||
instance_uuid: str | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
placement_generation: int | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
'tasks': [
|
||||
t.to_dict()
|
||||
for t in self.tasks
|
||||
if (type is None or t.task_type == type) and (kind is None or t.kind == kind)
|
||||
if (type is None or t.task_type == type)
|
||||
and (kind is None or t.kind == kind)
|
||||
and (instance_uuid is None or t.instance_uuid == instance_uuid)
|
||||
and (workspace_uuid is None or t.workspace_uuid == workspace_uuid)
|
||||
and (placement_generation is None or t.placement_generation == placement_generation)
|
||||
],
|
||||
'id_index': TaskWrapper._id_index,
|
||||
}
|
||||
@@ -240,9 +348,21 @@ class AsyncTaskManager:
|
||||
'id_index': TaskWrapper._id_index,
|
||||
}
|
||||
|
||||
def get_task_by_id(self, id: int) -> TaskWrapper | None:
|
||||
def get_task_by_id(
|
||||
self,
|
||||
id: int,
|
||||
*,
|
||||
instance_uuid: str | None = None,
|
||||
workspace_uuid: str | None = None,
|
||||
placement_generation: int | None = None,
|
||||
) -> TaskWrapper | None:
|
||||
for t in self.tasks:
|
||||
if t.id == id:
|
||||
if (
|
||||
t.id == id
|
||||
and (instance_uuid is None or t.instance_uuid == instance_uuid)
|
||||
and (workspace_uuid is None or t.workspace_uuid == workspace_uuid)
|
||||
and (placement_generation is None or t.placement_generation == placement_generation)
|
||||
):
|
||||
return t
|
||||
return None
|
||||
|
||||
|
||||
@@ -211,6 +211,7 @@ class ComponentDiscoveryEngine:
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
self.components = {}
|
||||
|
||||
def load_component_manifest(self, path: str, owner: str = 'builtin', no_save: bool = False) -> Component | None:
|
||||
"""加载组件清单"""
|
||||
|
||||
@@ -2,5 +2,19 @@ from __future__ import annotations
|
||||
|
||||
|
||||
class AccountEmailMismatchError(Exception):
|
||||
def __str__(self):
|
||||
def __str__(self) -> str:
|
||||
return 'Account email mismatch'
|
||||
|
||||
|
||||
class SpaceAccountNotRegisteredError(AccountEmailMismatchError):
|
||||
code = 'space_account_not_registered'
|
||||
|
||||
def __str__(self) -> str:
|
||||
return 'No Account is registered for this Space email'
|
||||
|
||||
|
||||
class SpaceAccountBindingRequiredError(AccountEmailMismatchError):
|
||||
code = 'space_account_binding_required'
|
||||
|
||||
def __str__(self) -> str:
|
||||
return 'This local Account must bind Space from Account settings before Space login'
|
||||
|
||||
@@ -1,16 +1,47 @@
|
||||
import enum
|
||||
import uuid as uuid_lib
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class ApiKeyStatus(enum.StrEnum):
|
||||
ACTIVE = 'active'
|
||||
REVOKED = 'revoked'
|
||||
|
||||
|
||||
def _new_uuid() -> str:
|
||||
return str(uuid_lib.uuid4())
|
||||
|
||||
|
||||
class ApiKey(Base):
|
||||
"""API Key for external service authentication"""
|
||||
|
||||
__tablename__ = 'api_keys'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(36), nullable=False, default=_new_uuid)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
created_by_account_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('users.uuid', ondelete='SET NULL'),
|
||||
nullable=True,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
key = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, unique=True)
|
||||
key_hash = sqlalchemy.Column(sqlalchemy.String(64), nullable=False)
|
||||
scopes = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=list, server_default='[]')
|
||||
status = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=ApiKeyStatus.ACTIVE.value,
|
||||
)
|
||||
expires_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True)
|
||||
last_used_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True)
|
||||
description = sqlalchemy.Column(sqlalchemy.String(512), nullable=True, default='')
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
updated_at = sqlalchemy.Column(
|
||||
@@ -19,3 +50,16 @@ class ApiKey(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('uq_api_keys_uuid', 'uuid', unique=True),
|
||||
# Authentication begins with the presented secret, before a Workspace
|
||||
# can be trusted, so hashes remain globally unique.
|
||||
sqlalchemy.Index('uq_api_keys_key_hash', 'key_hash', unique=True),
|
||||
sqlalchemy.Index('ix_api_keys_workspace_name', 'workspace_uuid', 'name'),
|
||||
sqlalchemy.Index('ix_api_keys_workspace_status', 'workspace_uuid', 'status'),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"status IN ('active', 'revoked')",
|
||||
name='ck_api_keys_status',
|
||||
),
|
||||
)
|
||||
|
||||
@@ -9,12 +9,31 @@ class BotAdmin(Base):
|
||||
__tablename__ = 'bot_admins'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
bot_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
launcher_type = sqlalchemy.Column(sqlalchemy.String(64), nullable=False)
|
||||
launcher_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
|
||||
__table_args__ = (sqlalchemy.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'),)
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint(
|
||||
'workspace_uuid',
|
||||
'bot_uuid',
|
||||
'launcher_type',
|
||||
'launcher_id',
|
||||
name='uq_bot_admin',
|
||||
),
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'bot_uuid'],
|
||||
['bots.workspace_uuid', 'bots.uuid'],
|
||||
name='fk_bot_admins_workspace_bot',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class Bot(Base):
|
||||
@@ -23,6 +42,11 @@ class Bot(Base):
|
||||
__tablename__ = 'bots'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
description = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
adapter = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
@@ -38,3 +62,8 @@ class Bot(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('workspace_uuid', 'uuid', name='uq_bots_workspace_uuid'),
|
||||
sqlalchemy.Index('ix_bots_workspace_name', 'workspace_uuid', 'name'),
|
||||
)
|
||||
|
||||
@@ -8,6 +8,11 @@ class BinaryStorage(Base):
|
||||
|
||||
__tablename__ = 'binary_storages'
|
||||
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
primary_key=True,
|
||||
)
|
||||
unique_key = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
key = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
owner_type = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
@@ -20,3 +25,12 @@ class BinaryStorage(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index(
|
||||
'ix_binary_storages_workspace_owner',
|
||||
'workspace_uuid',
|
||||
'owner_type',
|
||||
'owner',
|
||||
),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class DirectoryProjectionState(Base):
|
||||
"""Durable cursor and lease for one verified Cloud directory."""
|
||||
|
||||
__tablename__ = 'directory_projection_states'
|
||||
|
||||
instance_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
cursor = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='0')
|
||||
snapshot_coverage_cursor = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='0')
|
||||
snapshot_fingerprint = sqlalchemy.Column(sqlalchemy.Text, nullable=False)
|
||||
last_applied_at = sqlalchemy.Column(sqlalchemy.DateTime(timezone=True), nullable=False)
|
||||
lease_expires_at = sqlalchemy.Column(sqlalchemy.DateTime(timezone=True), nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.CheckConstraint('cursor >= 0', name='ck_directory_projection_state_cursor'),
|
||||
sqlalchemy.CheckConstraint(
|
||||
'snapshot_coverage_cursor >= 0 AND snapshot_coverage_cursor <= cursor',
|
||||
name='ck_directory_projection_state_snapshot_coverage',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
'length(snapshot_fingerprint) = 64',
|
||||
name='ck_directory_projection_state_fingerprint',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class DirectoryProjectionInbox(Base):
|
||||
"""Idempotency ledger for signed control-plane directory events."""
|
||||
|
||||
__tablename__ = 'directory_projection_inbox'
|
||||
|
||||
instance_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
event_uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True)
|
||||
cursor = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False)
|
||||
event_type = sqlalchemy.Column(sqlalchemy.String(128), nullable=False)
|
||||
revision = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False)
|
||||
fingerprint = sqlalchemy.Column(sqlalchemy.Text, nullable=False)
|
||||
received_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sqlalchemy.func.now(),
|
||||
)
|
||||
applied_at = sqlalchemy.Column(sqlalchemy.DateTime(timezone=True), nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint(
|
||||
'instance_uuid',
|
||||
'cursor',
|
||||
name='uq_directory_projection_inbox_cursor',
|
||||
),
|
||||
sqlalchemy.Index(
|
||||
'ix_directory_projection_inbox_pending',
|
||||
'instance_uuid',
|
||||
'applied_at',
|
||||
'cursor',
|
||||
),
|
||||
sqlalchemy.CheckConstraint('cursor > 0', name='ck_directory_projection_inbox_cursor'),
|
||||
sqlalchemy.CheckConstraint('revision > 0', name='ck_directory_projection_inbox_revision'),
|
||||
sqlalchemy.CheckConstraint(
|
||||
'length(fingerprint) = 64',
|
||||
name='ck_directory_projection_inbox_fingerprint',
|
||||
),
|
||||
)
|
||||
@@ -7,6 +7,11 @@ class MCPServer(Base):
|
||||
__tablename__ = 'mcp_servers'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
enable = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False)
|
||||
mode = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) # stdio, remote (legacy: sse, http)
|
||||
@@ -22,3 +27,8 @@ class MCPServer(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('workspace_uuid', 'name', name='uq_mcp_servers_workspace_name'),
|
||||
sqlalchemy.Index('ix_mcp_servers_workspace_enable', 'workspace_uuid', 'enable'),
|
||||
)
|
||||
|
||||
@@ -19,3 +19,17 @@ class Metadata(Base):
|
||||
|
||||
key = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
value = sqlalchemy.Column(sqlalchemy.String(255))
|
||||
|
||||
|
||||
class WorkspaceMetadata(Base):
|
||||
"""Metadata owned by one workspace rather than by the LangBot instance."""
|
||||
|
||||
__tablename__ = 'workspace_metadata'
|
||||
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
primary_key=True,
|
||||
)
|
||||
key = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
value = sqlalchemy.Column(sqlalchemy.String(255))
|
||||
|
||||
@@ -9,6 +9,11 @@ class ModelProvider(Base):
|
||||
__tablename__ = 'model_providers'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
requester = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
base_url = sqlalchemy.Column(sqlalchemy.String(512), nullable=False)
|
||||
@@ -21,6 +26,12 @@ class ModelProvider(Base):
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('workspace_uuid', 'uuid', name='uq_model_providers_workspace_uuid'),
|
||||
sqlalchemy.Index('ix_model_providers_workspace_name', 'workspace_uuid', 'name'),
|
||||
sqlalchemy.Index('ix_model_providers_workspace_requester', 'workspace_uuid', 'requester'),
|
||||
)
|
||||
|
||||
|
||||
class LLMModel(Base):
|
||||
"""LLM model"""
|
||||
@@ -28,6 +39,11 @@ class LLMModel(Base):
|
||||
__tablename__ = 'llm_models'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
abilities = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=[])
|
||||
@@ -42,6 +58,16 @@ class LLMModel(Base):
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'provider_uuid'],
|
||||
['model_providers.workspace_uuid', 'model_providers.uuid'],
|
||||
name='fk_llm_models_workspace_provider',
|
||||
),
|
||||
sqlalchemy.Index('ix_llm_models_workspace_provider', 'workspace_uuid', 'provider_uuid'),
|
||||
sqlalchemy.Index('ix_llm_models_workspace_name', 'workspace_uuid', 'name'),
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingModel(Base):
|
||||
"""Embedding model"""
|
||||
@@ -49,6 +75,11 @@ class EmbeddingModel(Base):
|
||||
__tablename__ = 'embedding_models'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
|
||||
@@ -61,6 +92,16 @@ class EmbeddingModel(Base):
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'provider_uuid'],
|
||||
['model_providers.workspace_uuid', 'model_providers.uuid'],
|
||||
name='fk_embedding_models_workspace_provider',
|
||||
),
|
||||
sqlalchemy.Index('ix_embedding_models_workspace_provider', 'workspace_uuid', 'provider_uuid'),
|
||||
sqlalchemy.Index('ix_embedding_models_workspace_name', 'workspace_uuid', 'name'),
|
||||
)
|
||||
|
||||
|
||||
class RerankModel(Base):
|
||||
"""Rerank model"""
|
||||
@@ -68,6 +109,11 @@ class RerankModel(Base):
|
||||
__tablename__ = 'rerank_models'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
|
||||
@@ -79,3 +125,13 @@ class RerankModel(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'provider_uuid'],
|
||||
['model_providers.workspace_uuid', 'model_providers.uuid'],
|
||||
name='fk_rerank_models_workspace_provider',
|
||||
),
|
||||
sqlalchemy.Index('ix_rerank_models_workspace_provider', 'workspace_uuid', 'provider_uuid'),
|
||||
sqlalchemy.Index('ix_rerank_models_workspace_name', 'workspace_uuid', 'name'),
|
||||
)
|
||||
|
||||
@@ -9,6 +9,11 @@ class MonitoringMessage(Base):
|
||||
__tablename__ = 'monitoring_messages'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
bot_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
|
||||
bot_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
@@ -25,6 +30,11 @@ class MonitoringMessage(Base):
|
||||
variables = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # Query variables as JSON string
|
||||
role = sqlalchemy.Column(sqlalchemy.String(50), nullable=True, default='user') # user, assistant
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('ix_monitoring_messages_workspace_timestamp', 'workspace_uuid', 'timestamp'),
|
||||
sqlalchemy.Index('ix_monitoring_messages_workspace_session', 'workspace_uuid', 'session_id'),
|
||||
)
|
||||
|
||||
|
||||
class MonitoringLLMCall(Base):
|
||||
"""LLM call records"""
|
||||
@@ -32,6 +42,11 @@ class MonitoringLLMCall(Base):
|
||||
__tablename__ = 'monitoring_llm_calls'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
model_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
input_tokens = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
|
||||
@@ -48,6 +63,11 @@ class MonitoringLLMCall(Base):
|
||||
error_message = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) # Associated message ID
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('ix_monitoring_llm_calls_workspace_timestamp', 'workspace_uuid', 'timestamp'),
|
||||
sqlalchemy.Index('ix_monitoring_llm_calls_workspace_session', 'workspace_uuid', 'session_id'),
|
||||
)
|
||||
|
||||
|
||||
class MonitoringToolCall(Base):
|
||||
"""Tool call records"""
|
||||
@@ -55,6 +75,11 @@ class MonitoringToolCall(Base):
|
||||
__tablename__ = 'monitoring_tool_calls'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
tool_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
tool_source = sqlalchemy.Column(sqlalchemy.String(50), nullable=False) # native, plugin, mcp, skill
|
||||
@@ -70,12 +95,22 @@ class MonitoringToolCall(Base):
|
||||
result = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||
error_message = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('ix_monitoring_tool_calls_workspace_timestamp', 'workspace_uuid', 'timestamp'),
|
||||
sqlalchemy.Index('ix_monitoring_tool_calls_workspace_session', 'workspace_uuid', 'session_id'),
|
||||
)
|
||||
|
||||
|
||||
class MonitoringSession(Base):
|
||||
"""Session tracking records"""
|
||||
|
||||
__tablename__ = 'monitoring_sessions'
|
||||
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
primary_key=True,
|
||||
)
|
||||
session_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
bot_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
|
||||
bot_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
@@ -89,6 +124,11 @@ class MonitoringSession(Base):
|
||||
user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
user_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # User display name
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('ix_monitoring_sessions_workspace_activity', 'workspace_uuid', 'last_activity'),
|
||||
sqlalchemy.Index('ix_monitoring_sessions_workspace_active', 'workspace_uuid', 'is_active'),
|
||||
)
|
||||
|
||||
|
||||
class MonitoringError(Base):
|
||||
"""Error log records"""
|
||||
@@ -96,6 +136,11 @@ class MonitoringError(Base):
|
||||
__tablename__ = 'monitoring_errors'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
error_type = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
error_message = sqlalchemy.Column(sqlalchemy.Text, nullable=False)
|
||||
@@ -107,6 +152,11 @@ class MonitoringError(Base):
|
||||
stack_trace = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
|
||||
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True) # Associated message ID
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('ix_monitoring_errors_workspace_timestamp', 'workspace_uuid', 'timestamp'),
|
||||
sqlalchemy.Index('ix_monitoring_errors_workspace_session', 'workspace_uuid', 'session_id'),
|
||||
)
|
||||
|
||||
|
||||
class MonitoringEmbeddingCall(Base):
|
||||
"""Embedding call records"""
|
||||
@@ -114,6 +164,11 @@ class MonitoringEmbeddingCall(Base):
|
||||
__tablename__ = 'monitoring_embedding_calls'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
model_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
prompt_tokens = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
|
||||
@@ -129,6 +184,19 @@ class MonitoringEmbeddingCall(Base):
|
||||
message_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
call_type = sqlalchemy.Column(sqlalchemy.String(50), nullable=True) # embedding, retrieve
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index(
|
||||
'ix_monitoring_embedding_calls_workspace_timestamp',
|
||||
'workspace_uuid',
|
||||
'timestamp',
|
||||
),
|
||||
sqlalchemy.Index(
|
||||
'ix_monitoring_embedding_calls_workspace_kb',
|
||||
'workspace_uuid',
|
||||
'knowledge_base_id',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MonitoringFeedback(Base):
|
||||
"""User feedback records (like/dislike) from AI Bot conversations"""
|
||||
@@ -136,8 +204,13 @@ class MonitoringFeedback(Base):
|
||||
__tablename__ = 'monitoring_feedback'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
timestamp = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, index=True)
|
||||
feedback_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, unique=True, index=True)
|
||||
feedback_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
|
||||
feedback_type = sqlalchemy.Column(sqlalchemy.Integer, nullable=False) # 1=like, 2=dislike
|
||||
feedback_content = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # User feedback text
|
||||
inaccurate_reasons = sqlalchemy.Column(sqlalchemy.Text, nullable=True) # JSON list of inaccurate reasons
|
||||
@@ -151,3 +224,13 @@ class MonitoringFeedback(Base):
|
||||
stream_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True, index=True)
|
||||
user_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
platform = sqlalchemy.Column(sqlalchemy.String(255), nullable=True) # e.g., wecom
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint(
|
||||
'workspace_uuid',
|
||||
'feedback_id',
|
||||
name='uq_monitoring_feedback_workspace_feedback_id',
|
||||
),
|
||||
sqlalchemy.Index('ix_monitoring_feedback_workspace_timestamp', 'workspace_uuid', 'timestamp'),
|
||||
sqlalchemy.Index('ix_monitoring_feedback_workspace_session', 'workspace_uuid', 'session_id'),
|
||||
)
|
||||
|
||||
@@ -9,6 +9,11 @@ class LegacyPipeline(Base):
|
||||
__tablename__ = 'legacy_pipelines'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
description = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
emoji = sqlalchemy.Column(sqlalchemy.String(10), nullable=True, default='⚙️')
|
||||
@@ -36,6 +41,16 @@ class LegacyPipeline(Base):
|
||||
},
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint(
|
||||
'workspace_uuid',
|
||||
'uuid',
|
||||
name='uq_legacy_pipelines_workspace_uuid',
|
||||
),
|
||||
sqlalchemy.Index('ix_legacy_pipelines_workspace_name', 'workspace_uuid', 'name'),
|
||||
sqlalchemy.Index('ix_legacy_pipelines_workspace_default', 'workspace_uuid', 'is_default'),
|
||||
)
|
||||
|
||||
|
||||
class PipelineRunRecord(Base):
|
||||
"""Pipeline run record"""
|
||||
@@ -43,6 +58,11 @@ class PipelineRunRecord(Base):
|
||||
__tablename__ = 'pipeline_run_records'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
pipeline_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
status = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
@@ -56,3 +76,22 @@ class PipelineRunRecord(Base):
|
||||
finished_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False)
|
||||
result = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
|
||||
knowledge_base_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'pipeline_uuid'],
|
||||
['legacy_pipelines.workspace_uuid', 'legacy_pipelines.uuid'],
|
||||
name='fk_pipeline_run_records_workspace_pipeline',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sqlalchemy.Index(
|
||||
'ix_pipeline_run_records_workspace_pipeline',
|
||||
'workspace_uuid',
|
||||
'pipeline_uuid',
|
||||
),
|
||||
sqlalchemy.Index(
|
||||
'ix_pipeline_run_records_workspace_created',
|
||||
'workspace_uuid',
|
||||
'created_at',
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
@@ -8,8 +11,24 @@ class PluginSetting(Base):
|
||||
|
||||
__tablename__ = 'plugin_settings'
|
||||
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
primary_key=True,
|
||||
)
|
||||
plugin_author = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
plugin_name = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
installation_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
nullable=False,
|
||||
default=lambda: str(uuid.uuid4()),
|
||||
)
|
||||
artifact_digest = sqlalchemy.Column(
|
||||
sqlalchemy.String(64),
|
||||
nullable=False,
|
||||
default=lambda: hashlib.sha256(f'pending:{uuid.uuid4()}'.encode()).hexdigest(),
|
||||
)
|
||||
runtime_revision = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=1)
|
||||
enabled = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True)
|
||||
priority = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
|
||||
config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=dict)
|
||||
@@ -22,3 +41,16 @@ class PluginSetting(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('installation_uuid', name='uq_plugin_settings_installation_uuid'),
|
||||
sqlalchemy.CheckConstraint('runtime_revision >= 1', name='ck_plugin_settings_runtime_revision_positive'),
|
||||
sqlalchemy.CheckConstraint('length(artifact_digest) = 64', name='ck_plugin_settings_artifact_digest_length'),
|
||||
sqlalchemy.Index('ix_plugin_settings_workspace_enabled', 'workspace_uuid', 'enabled'),
|
||||
sqlalchemy.Index(
|
||||
'ix_plugin_settings_workspace_installation',
|
||||
'workspace_uuid',
|
||||
'installation_uuid',
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -5,6 +5,11 @@ from .base import Base
|
||||
class KnowledgeBase(Base):
|
||||
__tablename__ = 'knowledge_bases'
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String, index=True)
|
||||
description = sqlalchemy.Column(sqlalchemy.Text)
|
||||
emoji = sqlalchemy.Column(sqlalchemy.String(10), nullable=True, default='📚')
|
||||
@@ -13,8 +18,20 @@ class KnowledgeBase(Base):
|
||||
# New fields for plugin-based RAG
|
||||
knowledge_engine_plugin_id = sqlalchemy.Column(sqlalchemy.String, nullable=True)
|
||||
collection_id = sqlalchemy.Column(sqlalchemy.String, nullable=True)
|
||||
# Server-managed compatibility marker. Pre-tenancy installations stored
|
||||
# vectors directly under ``collection_id``; new knowledge bases use a
|
||||
# tenant-derived opaque physical collection instead.
|
||||
legacy_vector_collection = sqlalchemy.Column(
|
||||
sqlalchemy.Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=sqlalchemy.false(),
|
||||
)
|
||||
creation_settings = sqlalchemy.Column(sqlalchemy.JSON, nullable=True, default=None)
|
||||
retrieval_settings = sqlalchemy.Column(sqlalchemy.JSON, nullable=True, default=None)
|
||||
# Server-selected pgvector dimension. ``None`` means no embedding has been
|
||||
# written yet; the first pgvector upsert binds it atomically.
|
||||
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
|
||||
|
||||
# Field sets for different operations
|
||||
MUTABLE_FIELDS = {'name', 'description', 'retrieval_settings'}
|
||||
@@ -23,22 +40,77 @@ class KnowledgeBase(Base):
|
||||
CREATE_FIELDS = MUTABLE_FIELDS | {'uuid', 'knowledge_engine_plugin_id', 'collection_id', 'creation_settings'}
|
||||
"""Fields used when creating a new knowledge base."""
|
||||
|
||||
ALL_DB_FIELDS = CREATE_FIELDS | {'emoji', 'created_at', 'updated_at'}
|
||||
ALL_DB_FIELDS = CREATE_FIELDS | {
|
||||
'workspace_uuid',
|
||||
'legacy_vector_collection',
|
||||
'embedding_dimension',
|
||||
'emoji',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
}
|
||||
"""All fields stored in database (for loading from DB row)."""
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('workspace_uuid', 'uuid', name='uq_knowledge_bases_workspace_uuid'),
|
||||
sqlalchemy.Index('ix_knowledge_bases_workspace_name', 'workspace_uuid', 'name'),
|
||||
sqlalchemy.Index(
|
||||
'uq_knowledge_bases_workspace_collection',
|
||||
'workspace_uuid',
|
||||
'collection_id',
|
||||
unique=True,
|
||||
sqlite_where=sqlalchemy.text('collection_id IS NOT NULL'),
|
||||
postgresql_where=sqlalchemy.text('collection_id IS NOT NULL'),
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
'embedding_dimension IS NULL OR embedding_dimension > 0',
|
||||
name='ck_knowledge_bases_embedding_dimension_positive',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class File(Base):
|
||||
__tablename__ = 'knowledge_base_files'
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
kb_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
file_name = sqlalchemy.Column(sqlalchemy.String)
|
||||
extension = sqlalchemy.Column(sqlalchemy.String)
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.func.now())
|
||||
status = sqlalchemy.Column(sqlalchemy.String, default='pending') # pending, processing, completed, failed
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('workspace_uuid', 'uuid', name='uq_knowledge_base_files_workspace_uuid'),
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'kb_id'],
|
||||
['knowledge_bases.workspace_uuid', 'knowledge_bases.uuid'],
|
||||
name='fk_knowledge_base_files_workspace_kb',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sqlalchemy.Index('ix_knowledge_base_files_workspace_kb', 'workspace_uuid', 'kb_id'),
|
||||
)
|
||||
|
||||
|
||||
class Chunk(Base):
|
||||
__tablename__ = 'knowledge_base_chunks'
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
file_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
|
||||
text = sqlalchemy.Column(sqlalchemy.Text)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'file_id'],
|
||||
['knowledge_base_files.workspace_uuid', 'knowledge_base_files.uuid'],
|
||||
name='fk_knowledge_base_chunks_workspace_file',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sqlalchemy.Index('ix_knowledge_base_chunks_workspace_file', 'workspace_uuid', 'file_id'),
|
||||
)
|
||||
|
||||
@@ -1,15 +1,47 @@
|
||||
import enum
|
||||
import uuid as uuid_lib
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class AccountStatus(enum.StrEnum):
|
||||
ACTIVE = 'active'
|
||||
DISABLED = 'disabled'
|
||||
DELETED = 'deleted'
|
||||
|
||||
|
||||
class AccountSource(enum.StrEnum):
|
||||
LOCAL = 'local'
|
||||
CLOUD_PROJECTION = 'cloud_projection'
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'users'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
|
||||
uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
nullable=False,
|
||||
default=lambda: str(uuid_lib.uuid4()),
|
||||
)
|
||||
user = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
normalized_email = sqlalchemy.Column(sqlalchemy.String(320), nullable=False)
|
||||
password = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
|
||||
status = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=AccountStatus.ACTIVE.value,
|
||||
)
|
||||
source = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=AccountSource.LOCAL.value,
|
||||
)
|
||||
projection_revision = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='0')
|
||||
|
||||
# Account type: 'local' (default) or 'space'
|
||||
account_type = sqlalchemy.Column(sqlalchemy.String(32), nullable=False, server_default='local')
|
||||
|
||||
@@ -27,3 +59,22 @@ class User(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('uq_users_uuid', 'uuid', unique=True),
|
||||
sqlalchemy.Index('uq_users_normalized_email', 'normalized_email', unique=True),
|
||||
sqlalchemy.CheckConstraint(
|
||||
'normalized_email = trim(normalized_email) '
|
||||
'AND length(normalized_email) > 0 '
|
||||
'AND length(normalized_email) <= 320',
|
||||
name='ck_users_normalized_email',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"status IN ('active', 'disabled', 'deleted')",
|
||||
name='ck_users_status',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"source IN ('local', 'cloud_projection')",
|
||||
name='ck_users_source',
|
||||
),
|
||||
)
|
||||
|
||||
@@ -9,6 +9,11 @@ class Webhook(Base):
|
||||
__tablename__ = 'webhooks'
|
||||
|
||||
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
url = sqlalchemy.Column(sqlalchemy.String(1024), nullable=False)
|
||||
description = sqlalchemy.Column(sqlalchemy.String(512), nullable=True, default='')
|
||||
@@ -20,3 +25,5 @@ class Webhook(Base):
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (sqlalchemy.Index('ix_webhooks_workspace_name', 'workspace_uuid', 'name'),)
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import uuid as uuid_lib
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class WorkspaceType(enum.StrEnum):
|
||||
PERSONAL = 'personal'
|
||||
TEAM = 'team'
|
||||
|
||||
|
||||
class WorkspaceStatus(enum.StrEnum):
|
||||
PROVISIONING = 'provisioning'
|
||||
ACTIVE = 'active'
|
||||
SUSPENDED = 'suspended'
|
||||
ARCHIVED = 'archived'
|
||||
DELETED = 'deleted'
|
||||
|
||||
|
||||
class WorkspaceSource(enum.StrEnum):
|
||||
LOCAL = 'local'
|
||||
CLOUD_PROJECTION = 'cloud_projection'
|
||||
|
||||
|
||||
class MembershipRole(enum.StrEnum):
|
||||
OWNER = 'owner'
|
||||
ADMIN = 'admin'
|
||||
DEVELOPER = 'developer'
|
||||
OPERATOR = 'operator'
|
||||
VIEWER = 'viewer'
|
||||
|
||||
|
||||
class MembershipStatus(enum.StrEnum):
|
||||
ACTIVE = 'active'
|
||||
DISABLED = 'disabled'
|
||||
REMOVED = 'removed'
|
||||
|
||||
|
||||
class InvitationStatus(enum.StrEnum):
|
||||
PENDING = 'pending'
|
||||
ACCEPTED = 'accepted'
|
||||
REVOKED = 'revoked'
|
||||
EXPIRED = 'expired'
|
||||
|
||||
|
||||
class WorkspaceExecutionStatus(enum.StrEnum):
|
||||
PROVISIONING = 'provisioning'
|
||||
ACTIVE = 'active'
|
||||
MIGRATING = 'migrating'
|
||||
DRAINING = 'draining'
|
||||
INACTIVE = 'inactive'
|
||||
|
||||
|
||||
class WorkspaceExecutionSource(enum.StrEnum):
|
||||
LOCAL = 'local'
|
||||
CLOUD = 'cloud'
|
||||
|
||||
|
||||
def _new_uuid() -> str:
|
||||
return str(uuid_lib.uuid4())
|
||||
|
||||
|
||||
class Workspace(Base):
|
||||
__tablename__ = 'workspaces'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True, default=_new_uuid)
|
||||
instance_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
slug = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
type = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=WorkspaceType.TEAM.value,
|
||||
)
|
||||
status = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=WorkspaceStatus.ACTIVE.value,
|
||||
)
|
||||
created_by_account_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('users.uuid', ondelete='SET NULL'),
|
||||
nullable=True,
|
||||
)
|
||||
source = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=WorkspaceSource.LOCAL.value,
|
||||
)
|
||||
projection_revision = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='0')
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
updated_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime,
|
||||
nullable=False,
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('instance_uuid', 'slug', name='uq_workspaces_instance_slug'),
|
||||
sqlalchemy.Index('ix_workspaces_instance_status', 'instance_uuid', 'status'),
|
||||
sqlalchemy.Index(
|
||||
'uq_workspaces_local_instance',
|
||||
'instance_uuid',
|
||||
unique=True,
|
||||
sqlite_where=sqlalchemy.text("source = 'local'"),
|
||||
postgresql_where=sqlalchemy.text("source = 'local'"),
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"type IN ('personal', 'team')",
|
||||
name='ck_workspaces_type',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"status IN ('provisioning', 'active', 'suspended', 'archived', 'deleted')",
|
||||
name='ck_workspaces_status',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"source IN ('local', 'cloud_projection')",
|
||||
name='ck_workspaces_source',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class WorkspaceMembership(Base):
|
||||
__tablename__ = 'workspace_memberships'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True, default=_new_uuid)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
account_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('users.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
role = sqlalchemy.Column(sqlalchemy.String(32), nullable=False)
|
||||
status = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=MembershipStatus.ACTIVE.value,
|
||||
)
|
||||
invited_by_account_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('users.uuid', ondelete='SET NULL'),
|
||||
nullable=True,
|
||||
)
|
||||
joined_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True)
|
||||
projection_revision = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='0')
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
updated_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime,
|
||||
nullable=False,
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('workspace_uuid', 'account_uuid', name='uq_workspace_membership_account'),
|
||||
sqlalchemy.Index('ix_workspace_memberships_account_status', 'account_uuid', 'status'),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"role IN ('owner', 'admin', 'developer', 'operator', 'viewer')",
|
||||
name='ck_workspace_memberships_role',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"status IN ('active', 'disabled', 'removed')",
|
||||
name='ck_workspace_memberships_status',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class WorkspaceInvitation(Base):
|
||||
__tablename__ = 'workspace_invitations'
|
||||
|
||||
uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True, default=_new_uuid)
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
normalized_email = sqlalchemy.Column(sqlalchemy.String(320), nullable=False)
|
||||
role = sqlalchemy.Column(sqlalchemy.String(32), nullable=False)
|
||||
token_hash = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
status = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=InvitationStatus.PENDING.value,
|
||||
)
|
||||
expires_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False)
|
||||
accepted_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True)
|
||||
revoked_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True)
|
||||
created_by_account_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('users.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
)
|
||||
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
|
||||
updated_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime,
|
||||
nullable=False,
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('uq_workspace_invitations_token_hash', 'token_hash', unique=True),
|
||||
sqlalchemy.Index(
|
||||
'uq_workspace_invitations_pending_email',
|
||||
'workspace_uuid',
|
||||
'normalized_email',
|
||||
unique=True,
|
||||
sqlite_where=sqlalchemy.text("status = 'pending'"),
|
||||
postgresql_where=sqlalchemy.text("status = 'pending'"),
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"role IN ('admin', 'developer', 'operator', 'viewer')",
|
||||
name='ck_workspace_invitations_role',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"status IN ('pending', 'accepted', 'revoked', 'expired')",
|
||||
name='ck_workspace_invitations_status',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class WorkspaceExecutionState(Base):
|
||||
__tablename__ = 'workspace_execution_states'
|
||||
|
||||
workspace_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
|
||||
primary_key=True,
|
||||
)
|
||||
instance_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
|
||||
active_generation = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='1')
|
||||
state = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=WorkspaceExecutionStatus.ACTIVE.value,
|
||||
)
|
||||
write_fenced = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, server_default=sqlalchemy.false())
|
||||
source = sqlalchemy.Column(
|
||||
sqlalchemy.String(32),
|
||||
nullable=False,
|
||||
server_default=WorkspaceExecutionSource.LOCAL.value,
|
||||
)
|
||||
desired_state_revision = sqlalchemy.Column(sqlalchemy.BigInteger, nullable=False, server_default='0')
|
||||
updated_at = sqlalchemy.Column(
|
||||
sqlalchemy.DateTime,
|
||||
nullable=False,
|
||||
server_default=sqlalchemy.func.now(),
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.Index('ix_workspace_execution_states_instance_state', 'instance_uuid', 'state'),
|
||||
sqlalchemy.CheckConstraint('active_generation > 0', name='ck_workspace_execution_generation'),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"state IN ('provisioning', 'active', 'migrating', 'draining', 'inactive')",
|
||||
name='ck_workspace_execution_state',
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
"source IN ('local', 'cloud')",
|
||||
name='ck_workspace_execution_source',
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,542 @@
|
||||
"""add the workspace tenancy persistence kernel
|
||||
|
||||
Revision ID: 0009_workspace_tenancy
|
||||
Revises: 0008_mcp_resource_prefs
|
||||
Create Date: 2026-07-18
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0009_workspace_tenancy'
|
||||
down_revision = '0008_mcp_resource_prefs'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _table_names(conn: sa.Connection) -> set[str]:
|
||||
return set(sa.inspect(conn).get_table_names())
|
||||
|
||||
|
||||
def _column_map(conn: sa.Connection, table_name: str) -> dict[str, dict]:
|
||||
return {column['name']: column for column in sa.inspect(conn).get_columns(table_name)}
|
||||
|
||||
|
||||
def _constraint_names(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
names = {
|
||||
constraint['name']
|
||||
for constraint in inspector.get_check_constraints(table_name)
|
||||
if constraint.get('name') is not None
|
||||
}
|
||||
names.update(
|
||||
constraint['name']
|
||||
for constraint in inspector.get_unique_constraints(table_name)
|
||||
if constraint.get('name') is not None
|
||||
)
|
||||
return names
|
||||
|
||||
|
||||
def _index_names(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
return {index['name'] for index in sa.inspect(conn).get_indexes(table_name)}
|
||||
|
||||
|
||||
def _upgrade_users(conn: sa.Connection) -> None:
|
||||
if 'users' not in _table_names(conn):
|
||||
return
|
||||
|
||||
columns = _column_map(conn, 'users')
|
||||
if 'uuid' not in columns:
|
||||
op.add_column('users', sa.Column('uuid', sa.String(36), nullable=True))
|
||||
if 'status' not in columns:
|
||||
op.add_column('users', sa.Column('status', sa.String(32), nullable=True, server_default='active'))
|
||||
if 'source' not in columns:
|
||||
op.add_column('users', sa.Column('source', sa.String(32), nullable=True, server_default='local'))
|
||||
if 'projection_revision' not in columns:
|
||||
op.add_column(
|
||||
'users',
|
||||
sa.Column('projection_revision', sa.BigInteger(), nullable=True, server_default='0'),
|
||||
)
|
||||
|
||||
users = sa.table(
|
||||
'users',
|
||||
sa.column('id', sa.Integer()),
|
||||
sa.column('uuid', sa.String(36)),
|
||||
sa.column('status', sa.String(32)),
|
||||
sa.column('source', sa.String(32)),
|
||||
sa.column('projection_revision', sa.BigInteger()),
|
||||
)
|
||||
|
||||
seen_uuids: set[str] = set()
|
||||
for user_id, account_uuid in conn.execute(sa.select(users.c.id, users.c.uuid).order_by(users.c.id)).all():
|
||||
normalized_uuid = account_uuid.strip() if isinstance(account_uuid, str) else ''
|
||||
try:
|
||||
normalized_uuid = str(uuid.UUID(normalized_uuid))
|
||||
except (ValueError, AttributeError):
|
||||
normalized_uuid = ''
|
||||
if not normalized_uuid or normalized_uuid in seen_uuids:
|
||||
normalized_uuid = str(uuid.uuid4())
|
||||
if normalized_uuid != account_uuid:
|
||||
conn.execute(users.update().where(users.c.id == user_id).values(uuid=normalized_uuid))
|
||||
seen_uuids.add(normalized_uuid)
|
||||
|
||||
conn.execute(users.update().where(users.c.status.is_(None)).values(status='active'))
|
||||
conn.execute(users.update().where(users.c.source.is_(None)).values(source='local'))
|
||||
conn.execute(users.update().where(users.c.projection_revision.is_(None)).values(projection_revision=0))
|
||||
|
||||
columns = _column_map(conn, 'users')
|
||||
constraint_names = _constraint_names(conn, 'users')
|
||||
needs_batch_alter = any(
|
||||
columns[column_name]['nullable'] for column_name in ('uuid', 'status', 'source', 'projection_revision')
|
||||
) or not {'ck_users_status', 'ck_users_source'}.issubset(constraint_names)
|
||||
|
||||
if needs_batch_alter:
|
||||
with op.batch_alter_table('users') as batch_op:
|
||||
if columns['uuid']['nullable']:
|
||||
batch_op.alter_column('uuid', existing_type=sa.String(36), nullable=False)
|
||||
if columns['status']['nullable']:
|
||||
batch_op.alter_column(
|
||||
'status',
|
||||
existing_type=sa.String(32),
|
||||
nullable=False,
|
||||
server_default='active',
|
||||
)
|
||||
if columns['source']['nullable']:
|
||||
batch_op.alter_column(
|
||||
'source',
|
||||
existing_type=sa.String(32),
|
||||
nullable=False,
|
||||
server_default='local',
|
||||
)
|
||||
if columns['projection_revision']['nullable']:
|
||||
batch_op.alter_column(
|
||||
'projection_revision',
|
||||
existing_type=sa.BigInteger(),
|
||||
nullable=False,
|
||||
server_default='0',
|
||||
)
|
||||
if 'ck_users_status' not in constraint_names:
|
||||
batch_op.create_check_constraint(
|
||||
'ck_users_status',
|
||||
"status IN ('active', 'disabled', 'deleted')",
|
||||
)
|
||||
if 'ck_users_source' not in constraint_names:
|
||||
batch_op.create_check_constraint(
|
||||
'ck_users_source',
|
||||
"source IN ('local', 'cloud_projection')",
|
||||
)
|
||||
|
||||
if 'uq_users_uuid' not in _index_names(conn, 'users'):
|
||||
op.create_index('uq_users_uuid', 'users', ['uuid'], unique=True)
|
||||
|
||||
|
||||
def _create_workspace_tables(conn: sa.Connection) -> None:
|
||||
tables = _table_names(conn)
|
||||
if 'users' not in tables:
|
||||
# LangBot's supported startup path creates the baseline schema before
|
||||
# Alembic runs. Keep direct Alembic probes on an empty database safe.
|
||||
return
|
||||
|
||||
if 'workspaces' not in tables:
|
||||
op.create_table(
|
||||
'workspaces',
|
||||
sa.Column('uuid', sa.String(36), primary_key=True),
|
||||
sa.Column('instance_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('name', sa.String(255), nullable=False),
|
||||
sa.Column('slug', sa.String(255), nullable=False),
|
||||
sa.Column('type', sa.String(32), nullable=False, server_default='team'),
|
||||
sa.Column('status', sa.String(32), nullable=False, server_default='active'),
|
||||
sa.Column('created_by_account_uuid', sa.String(36), nullable=True),
|
||||
sa.Column('source', sa.String(32), nullable=False, server_default='local'),
|
||||
sa.Column('projection_revision', sa.BigInteger(), nullable=False, server_default='0'),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(
|
||||
['created_by_account_uuid'],
|
||||
['users.uuid'],
|
||||
name='fk_workspaces_created_by_account',
|
||||
ondelete='SET NULL',
|
||||
),
|
||||
sa.UniqueConstraint('instance_uuid', 'slug', name='uq_workspaces_instance_slug'),
|
||||
sa.CheckConstraint("type IN ('personal', 'team')", name='ck_workspaces_type'),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('provisioning', 'active', 'suspended', 'archived', 'deleted')",
|
||||
name='ck_workspaces_status',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"source IN ('local', 'cloud_projection')",
|
||||
name='ck_workspaces_source',
|
||||
),
|
||||
)
|
||||
workspace_indexes = _index_names(conn, 'workspaces')
|
||||
if 'ix_workspaces_instance_status' not in workspace_indexes:
|
||||
op.create_index(
|
||||
'ix_workspaces_instance_status',
|
||||
'workspaces',
|
||||
['instance_uuid', 'status'],
|
||||
)
|
||||
if 'uq_workspaces_local_instance' not in workspace_indexes:
|
||||
op.create_index(
|
||||
'uq_workspaces_local_instance',
|
||||
'workspaces',
|
||||
['instance_uuid'],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("source = 'local'"),
|
||||
postgresql_where=sa.text("source = 'local'"),
|
||||
)
|
||||
|
||||
tables = _table_names(conn)
|
||||
if 'workspace_memberships' not in tables:
|
||||
op.create_table(
|
||||
'workspace_memberships',
|
||||
sa.Column('uuid', sa.String(36), primary_key=True),
|
||||
sa.Column('workspace_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('account_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('role', sa.String(32), nullable=False),
|
||||
sa.Column('status', sa.String(32), nullable=False, server_default='active'),
|
||||
sa.Column('invited_by_account_uuid', sa.String(36), nullable=True),
|
||||
sa.Column('joined_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('projection_revision', sa.BigInteger(), nullable=False, server_default='0'),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid'],
|
||||
['workspaces.uuid'],
|
||||
name='fk_workspace_memberships_workspace',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
['account_uuid'],
|
||||
['users.uuid'],
|
||||
name='fk_workspace_memberships_account',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
['invited_by_account_uuid'],
|
||||
['users.uuid'],
|
||||
name='fk_workspace_memberships_invited_by_account',
|
||||
ondelete='SET NULL',
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
'workspace_uuid',
|
||||
'account_uuid',
|
||||
name='uq_workspace_membership_account',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"role IN ('owner', 'admin', 'developer', 'operator', 'viewer')",
|
||||
name='ck_workspace_memberships_role',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('active', 'disabled', 'removed')",
|
||||
name='ck_workspace_memberships_status',
|
||||
),
|
||||
)
|
||||
membership_indexes = _index_names(conn, 'workspace_memberships')
|
||||
if 'ix_workspace_memberships_account_status' not in membership_indexes:
|
||||
op.create_index(
|
||||
'ix_workspace_memberships_account_status',
|
||||
'workspace_memberships',
|
||||
['account_uuid', 'status'],
|
||||
)
|
||||
|
||||
tables = _table_names(conn)
|
||||
if 'workspace_invitations' not in tables:
|
||||
op.create_table(
|
||||
'workspace_invitations',
|
||||
sa.Column('uuid', sa.String(36), primary_key=True),
|
||||
sa.Column('workspace_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('normalized_email', sa.String(320), nullable=False),
|
||||
sa.Column('role', sa.String(32), nullable=False),
|
||||
sa.Column('token_hash', sa.String(255), nullable=False),
|
||||
sa.Column('status', sa.String(32), nullable=False, server_default='pending'),
|
||||
sa.Column('expires_at', sa.DateTime(), nullable=False),
|
||||
sa.Column('accepted_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('revoked_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_by_account_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid'],
|
||||
['workspaces.uuid'],
|
||||
name='fk_workspace_invitations_workspace',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
['created_by_account_uuid'],
|
||||
['users.uuid'],
|
||||
name='fk_workspace_invitations_created_by_account',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"role IN ('admin', 'developer', 'operator', 'viewer')",
|
||||
name='ck_workspace_invitations_role',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('pending', 'accepted', 'revoked', 'expired')",
|
||||
name='ck_workspace_invitations_status',
|
||||
),
|
||||
)
|
||||
invitation_indexes = _index_names(conn, 'workspace_invitations')
|
||||
if 'uq_workspace_invitations_token_hash' not in invitation_indexes:
|
||||
op.create_index(
|
||||
'uq_workspace_invitations_token_hash',
|
||||
'workspace_invitations',
|
||||
['token_hash'],
|
||||
unique=True,
|
||||
)
|
||||
if 'uq_workspace_invitations_pending_email' not in invitation_indexes:
|
||||
op.create_index(
|
||||
'uq_workspace_invitations_pending_email',
|
||||
'workspace_invitations',
|
||||
['workspace_uuid', 'normalized_email'],
|
||||
unique=True,
|
||||
sqlite_where=sa.text("status = 'pending'"),
|
||||
postgresql_where=sa.text("status = 'pending'"),
|
||||
)
|
||||
|
||||
tables = _table_names(conn)
|
||||
if 'workspace_execution_states' not in tables:
|
||||
op.create_table(
|
||||
'workspace_execution_states',
|
||||
sa.Column('workspace_uuid', sa.String(36), primary_key=True),
|
||||
sa.Column('instance_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('active_generation', sa.BigInteger(), nullable=False, server_default='1'),
|
||||
sa.Column('state', sa.String(32), nullable=False, server_default='active'),
|
||||
sa.Column('write_fenced', sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column('source', sa.String(32), nullable=False, server_default='local'),
|
||||
sa.Column('desired_state_revision', sa.BigInteger(), nullable=False, server_default='0'),
|
||||
sa.Column('updated_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid'],
|
||||
['workspaces.uuid'],
|
||||
name='fk_workspace_execution_states_workspace',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.CheckConstraint('active_generation > 0', name='ck_workspace_execution_generation'),
|
||||
sa.CheckConstraint(
|
||||
"state IN ('provisioning', 'active', 'migrating', 'draining', 'inactive')",
|
||||
name='ck_workspace_execution_state',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"source IN ('local', 'cloud')",
|
||||
name='ck_workspace_execution_source',
|
||||
),
|
||||
)
|
||||
execution_indexes = _index_names(conn, 'workspace_execution_states')
|
||||
if 'ix_workspace_execution_states_instance_state' not in execution_indexes:
|
||||
op.create_index(
|
||||
'ix_workspace_execution_states_instance_state',
|
||||
'workspace_execution_states',
|
||||
['instance_uuid', 'state'],
|
||||
)
|
||||
|
||||
|
||||
def _load_instance_uuid(conn: sa.Connection) -> str | None:
|
||||
if 'metadata' not in _table_names(conn):
|
||||
return None
|
||||
|
||||
metadata = sa.table(
|
||||
'metadata',
|
||||
sa.column('key', sa.String(255)),
|
||||
sa.column('value', sa.String(255)),
|
||||
)
|
||||
value = conn.execute(sa.select(metadata.c.value).where(metadata.c.key == 'instance_uuid')).scalar_one_or_none()
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _bootstrap_default_workspace(conn: sa.Connection) -> None:
|
||||
required_tables = {'users', 'workspaces', 'workspace_memberships', 'workspace_execution_states'}
|
||||
if not required_tables.issubset(_table_names(conn)):
|
||||
return
|
||||
|
||||
instance_uuid = _load_instance_uuid(conn)
|
||||
users_exist = 'users' in _table_names(conn) and bool(conn.execute(sa.text('SELECT 1 FROM users LIMIT 1')).first())
|
||||
if instance_uuid is None:
|
||||
if users_exist:
|
||||
raise RuntimeError("Cannot bootstrap the default workspace without metadata['instance_uuid']")
|
||||
return
|
||||
|
||||
workspaces = sa.table(
|
||||
'workspaces',
|
||||
sa.column('uuid', sa.String(36)),
|
||||
sa.column('instance_uuid', sa.String(255)),
|
||||
sa.column('name', sa.String(255)),
|
||||
sa.column('slug', sa.String(255)),
|
||||
sa.column('type', sa.String(32)),
|
||||
sa.column('status', sa.String(32)),
|
||||
sa.column('created_by_account_uuid', sa.String(36)),
|
||||
sa.column('source', sa.String(32)),
|
||||
sa.column('projection_revision', sa.BigInteger()),
|
||||
)
|
||||
local_rows = conn.execute(
|
||||
sa.select(workspaces.c.uuid).where(
|
||||
workspaces.c.instance_uuid == instance_uuid,
|
||||
workspaces.c.source == 'local',
|
||||
)
|
||||
).all()
|
||||
if len(local_rows) > 1:
|
||||
raise RuntimeError(f'Multiple local workspaces already exist for instance {instance_uuid!r}')
|
||||
|
||||
users = sa.table(
|
||||
'users',
|
||||
sa.column('id', sa.Integer()),
|
||||
sa.column('uuid', sa.String(36)),
|
||||
)
|
||||
owner_account_uuid = None
|
||||
if 'users' in _table_names(conn):
|
||||
owner_account_uuid = conn.execute(sa.select(users.c.uuid).order_by(users.c.id).limit(1)).scalar_one_or_none()
|
||||
|
||||
if local_rows:
|
||||
workspace_uuid = local_rows[0][0]
|
||||
if owner_account_uuid is not None:
|
||||
conn.execute(
|
||||
workspaces.update()
|
||||
.where(workspaces.c.uuid == workspace_uuid)
|
||||
.where(workspaces.c.created_by_account_uuid.is_(None))
|
||||
.values(created_by_account_uuid=owner_account_uuid)
|
||||
)
|
||||
else:
|
||||
workspace_uuid = str(uuid.uuid4())
|
||||
conn.execute(
|
||||
workspaces.insert().values(
|
||||
uuid=workspace_uuid,
|
||||
instance_uuid=instance_uuid,
|
||||
name='Default Workspace',
|
||||
slug='default',
|
||||
type='team',
|
||||
status='active',
|
||||
created_by_account_uuid=owner_account_uuid,
|
||||
source='local',
|
||||
projection_revision=0,
|
||||
)
|
||||
)
|
||||
|
||||
execution_states = sa.table(
|
||||
'workspace_execution_states',
|
||||
sa.column('workspace_uuid', sa.String(36)),
|
||||
sa.column('instance_uuid', sa.String(255)),
|
||||
sa.column('active_generation', sa.BigInteger()),
|
||||
sa.column('state', sa.String(32)),
|
||||
sa.column('write_fenced', sa.Boolean()),
|
||||
sa.column('source', sa.String(32)),
|
||||
sa.column('desired_state_revision', sa.BigInteger()),
|
||||
)
|
||||
execution_state = conn.execute(
|
||||
sa.select(
|
||||
execution_states.c.instance_uuid,
|
||||
execution_states.c.active_generation,
|
||||
execution_states.c.state,
|
||||
execution_states.c.write_fenced,
|
||||
execution_states.c.source,
|
||||
).where(execution_states.c.workspace_uuid == workspace_uuid)
|
||||
).first()
|
||||
if execution_state is None:
|
||||
conn.execute(
|
||||
execution_states.insert().values(
|
||||
workspace_uuid=workspace_uuid,
|
||||
instance_uuid=instance_uuid,
|
||||
active_generation=1,
|
||||
state='active',
|
||||
write_fenced=False,
|
||||
source='local',
|
||||
desired_state_revision=0,
|
||||
)
|
||||
)
|
||||
elif (
|
||||
execution_state.instance_uuid != instance_uuid
|
||||
or execution_state.active_generation != 1
|
||||
or execution_state.state != 'active'
|
||||
or execution_state.write_fenced
|
||||
or execution_state.source != 'local'
|
||||
):
|
||||
raise RuntimeError(f'Default workspace {workspace_uuid!r} has an invalid local execution state')
|
||||
|
||||
if owner_account_uuid is None:
|
||||
return
|
||||
|
||||
memberships = sa.table(
|
||||
'workspace_memberships',
|
||||
sa.column('uuid', sa.String(36)),
|
||||
sa.column('workspace_uuid', sa.String(36)),
|
||||
sa.column('account_uuid', sa.String(36)),
|
||||
sa.column('role', sa.String(32)),
|
||||
sa.column('status', sa.String(32)),
|
||||
sa.column('joined_at', sa.DateTime()),
|
||||
sa.column('projection_revision', sa.BigInteger()),
|
||||
)
|
||||
membership = conn.execute(
|
||||
sa.select(memberships.c.uuid, memberships.c.joined_at).where(
|
||||
memberships.c.workspace_uuid == workspace_uuid,
|
||||
memberships.c.account_uuid == owner_account_uuid,
|
||||
)
|
||||
).first()
|
||||
now = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
if membership is None:
|
||||
conn.execute(
|
||||
memberships.insert().values(
|
||||
uuid=str(uuid.uuid4()),
|
||||
workspace_uuid=workspace_uuid,
|
||||
account_uuid=owner_account_uuid,
|
||||
role='owner',
|
||||
status='active',
|
||||
joined_at=now,
|
||||
projection_revision=0,
|
||||
)
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
memberships.update()
|
||||
.where(memberships.c.uuid == membership.uuid)
|
||||
.values(
|
||||
role='owner',
|
||||
status='active',
|
||||
joined_at=membership.joined_at or now,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
_upgrade_users(conn)
|
||||
_create_workspace_tables(conn)
|
||||
_bootstrap_default_workspace(conn)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
tables = _table_names(conn)
|
||||
for table_name in (
|
||||
'workspace_execution_states',
|
||||
'workspace_invitations',
|
||||
'workspace_memberships',
|
||||
'workspaces',
|
||||
):
|
||||
if table_name in tables:
|
||||
op.drop_table(table_name)
|
||||
|
||||
if 'users' not in _table_names(conn):
|
||||
return
|
||||
|
||||
indexes = _index_names(conn, 'users')
|
||||
if 'uq_users_uuid' in indexes:
|
||||
op.drop_index('uq_users_uuid', table_name='users')
|
||||
|
||||
columns = _column_map(conn, 'users')
|
||||
constraint_names = _constraint_names(conn, 'users')
|
||||
with op.batch_alter_table('users') as batch_op:
|
||||
# SQLite batch recreation otherwise preserves the named checks while
|
||||
# dropping their referenced columns, producing ``no such column`` only
|
||||
# after the Workspace directory tables have already been removed.
|
||||
for constraint_name in ('ck_users_source', 'ck_users_status'):
|
||||
if constraint_name in constraint_names:
|
||||
batch_op.drop_constraint(constraint_name, type_='check')
|
||||
for column_name in ('projection_revision', 'source', 'status', 'uuid'):
|
||||
if column_name in columns:
|
||||
batch_op.drop_column(column_name)
|
||||
@@ -0,0 +1,884 @@
|
||||
"""scope every tenant-owned resource to a workspace
|
||||
|
||||
Revision ID: 0010_scope_resources
|
||||
Revises: 0009_workspace_tenancy
|
||||
Create Date: 2026-07-19
|
||||
|
||||
This migration is intentionally expand/backfill/contract. Existing rows are
|
||||
bound to the single local Workspace created by revision 0009 before any
|
||||
non-null or scoped-key constraint is installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0010_scope_resources'
|
||||
down_revision = '0009_workspace_tenancy'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_TENANT_TABLES = (
|
||||
'api_keys',
|
||||
'bots',
|
||||
'bot_admins',
|
||||
'binary_storages',
|
||||
'mcp_servers',
|
||||
'model_providers',
|
||||
'llm_models',
|
||||
'embedding_models',
|
||||
'rerank_models',
|
||||
'legacy_pipelines',
|
||||
'pipeline_run_records',
|
||||
'plugin_settings',
|
||||
'knowledge_bases',
|
||||
'knowledge_base_files',
|
||||
'knowledge_base_chunks',
|
||||
'webhooks',
|
||||
'monitoring_messages',
|
||||
'monitoring_llm_calls',
|
||||
'monitoring_tool_calls',
|
||||
'monitoring_sessions',
|
||||
'monitoring_errors',
|
||||
'monitoring_embedding_calls',
|
||||
'monitoring_feedback',
|
||||
)
|
||||
|
||||
_COMPOSITE_PRIMARY_KEYS = {
|
||||
'binary_storages': ('workspace_uuid', 'unique_key'),
|
||||
'plugin_settings': ('workspace_uuid', 'plugin_author', 'plugin_name'),
|
||||
'monitoring_sessions': ('workspace_uuid', 'session_id'),
|
||||
}
|
||||
|
||||
_COMPOSITE_FOREIGN_KEYS = {
|
||||
'bot_admins': (
|
||||
(
|
||||
'fk_bot_admins_workspace_bot',
|
||||
('workspace_uuid', 'bot_uuid'),
|
||||
'bots',
|
||||
('workspace_uuid', 'uuid'),
|
||||
'CASCADE',
|
||||
),
|
||||
),
|
||||
'llm_models': (
|
||||
(
|
||||
'fk_llm_models_workspace_provider',
|
||||
('workspace_uuid', 'provider_uuid'),
|
||||
'model_providers',
|
||||
('workspace_uuid', 'uuid'),
|
||||
None,
|
||||
),
|
||||
),
|
||||
'embedding_models': (
|
||||
(
|
||||
'fk_embedding_models_workspace_provider',
|
||||
('workspace_uuid', 'provider_uuid'),
|
||||
'model_providers',
|
||||
('workspace_uuid', 'uuid'),
|
||||
None,
|
||||
),
|
||||
),
|
||||
'rerank_models': (
|
||||
(
|
||||
'fk_rerank_models_workspace_provider',
|
||||
('workspace_uuid', 'provider_uuid'),
|
||||
'model_providers',
|
||||
('workspace_uuid', 'uuid'),
|
||||
None,
|
||||
),
|
||||
),
|
||||
'pipeline_run_records': (
|
||||
(
|
||||
'fk_pipeline_run_records_workspace_pipeline',
|
||||
('workspace_uuid', 'pipeline_uuid'),
|
||||
'legacy_pipelines',
|
||||
('workspace_uuid', 'uuid'),
|
||||
'CASCADE',
|
||||
),
|
||||
),
|
||||
'knowledge_base_files': (
|
||||
(
|
||||
'fk_knowledge_base_files_workspace_kb',
|
||||
('workspace_uuid', 'kb_id'),
|
||||
'knowledge_bases',
|
||||
('workspace_uuid', 'uuid'),
|
||||
'CASCADE',
|
||||
),
|
||||
),
|
||||
'knowledge_base_chunks': (
|
||||
(
|
||||
'fk_knowledge_base_chunks_workspace_file',
|
||||
('workspace_uuid', 'file_id'),
|
||||
'knowledge_base_files',
|
||||
('workspace_uuid', 'uuid'),
|
||||
'CASCADE',
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
_SCOPED_INDEXES: dict[str, tuple[tuple[str, tuple[str, ...], bool, sa.TextClause | None], ...]] = {
|
||||
'api_keys': (
|
||||
('uq_api_keys_uuid', ('uuid',), True, None),
|
||||
('uq_api_keys_key_hash', ('key_hash',), True, None),
|
||||
('ix_api_keys_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
('ix_api_keys_workspace_status', ('workspace_uuid', 'status'), False, None),
|
||||
),
|
||||
'bots': (
|
||||
('uq_bots_workspace_uuid', ('workspace_uuid', 'uuid'), True, None),
|
||||
('ix_bots_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
('ix_bots_workspace_updated', ('workspace_uuid', 'updated_at'), False, None),
|
||||
),
|
||||
'bot_admins': (
|
||||
(
|
||||
'uq_bot_admin',
|
||||
('workspace_uuid', 'bot_uuid', 'launcher_type', 'launcher_id'),
|
||||
True,
|
||||
None,
|
||||
),
|
||||
('ix_bot_admins_workspace_bot', ('workspace_uuid', 'bot_uuid'), False, None),
|
||||
),
|
||||
'binary_storages': (
|
||||
(
|
||||
'ix_binary_storages_workspace_owner',
|
||||
('workspace_uuid', 'owner_type', 'owner'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
),
|
||||
'mcp_servers': (
|
||||
('uq_mcp_servers_workspace_name', ('workspace_uuid', 'name'), True, None),
|
||||
('ix_mcp_servers_workspace_enable', ('workspace_uuid', 'enable'), False, None),
|
||||
('ix_mcp_servers_workspace_updated', ('workspace_uuid', 'updated_at'), False, None),
|
||||
),
|
||||
'model_providers': (
|
||||
('uq_model_providers_workspace_uuid', ('workspace_uuid', 'uuid'), True, None),
|
||||
('ix_model_providers_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
('ix_model_providers_workspace_requester', ('workspace_uuid', 'requester'), False, None),
|
||||
),
|
||||
'llm_models': (
|
||||
('ix_llm_models_workspace_provider', ('workspace_uuid', 'provider_uuid'), False, None),
|
||||
('ix_llm_models_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
),
|
||||
'embedding_models': (
|
||||
('ix_embedding_models_workspace_provider', ('workspace_uuid', 'provider_uuid'), False, None),
|
||||
('ix_embedding_models_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
),
|
||||
'rerank_models': (
|
||||
('ix_rerank_models_workspace_provider', ('workspace_uuid', 'provider_uuid'), False, None),
|
||||
('ix_rerank_models_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
),
|
||||
'legacy_pipelines': (
|
||||
('uq_legacy_pipelines_workspace_uuid', ('workspace_uuid', 'uuid'), True, None),
|
||||
('ix_legacy_pipelines_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
('ix_legacy_pipelines_workspace_default', ('workspace_uuid', 'is_default'), False, None),
|
||||
('ix_legacy_pipelines_workspace_updated', ('workspace_uuid', 'updated_at'), False, None),
|
||||
),
|
||||
'pipeline_run_records': (
|
||||
(
|
||||
'ix_pipeline_run_records_workspace_pipeline',
|
||||
('workspace_uuid', 'pipeline_uuid'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
(
|
||||
'ix_pipeline_run_records_workspace_created',
|
||||
('workspace_uuid', 'created_at'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
),
|
||||
'plugin_settings': (('ix_plugin_settings_workspace_enabled', ('workspace_uuid', 'enabled'), False, None),),
|
||||
'knowledge_bases': (
|
||||
('uq_knowledge_bases_workspace_uuid', ('workspace_uuid', 'uuid'), True, None),
|
||||
('ix_knowledge_bases_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
(
|
||||
'uq_knowledge_bases_workspace_collection',
|
||||
('workspace_uuid', 'collection_id'),
|
||||
True,
|
||||
sa.text('collection_id IS NOT NULL'),
|
||||
),
|
||||
),
|
||||
'knowledge_base_files': (
|
||||
('uq_knowledge_base_files_workspace_uuid', ('workspace_uuid', 'uuid'), True, None),
|
||||
('ix_knowledge_base_files_workspace_kb', ('workspace_uuid', 'kb_id'), False, None),
|
||||
),
|
||||
'knowledge_base_chunks': (('ix_knowledge_base_chunks_workspace_file', ('workspace_uuid', 'file_id'), False, None),),
|
||||
'webhooks': (
|
||||
('ix_webhooks_workspace_name', ('workspace_uuid', 'name'), False, None),
|
||||
('ix_webhooks_workspace_enabled', ('workspace_uuid', 'enabled'), False, None),
|
||||
('ix_webhooks_workspace_created', ('workspace_uuid', 'created_at'), False, None),
|
||||
),
|
||||
'monitoring_messages': (
|
||||
('ix_monitoring_messages_workspace_timestamp', ('workspace_uuid', 'timestamp'), False, None),
|
||||
('ix_monitoring_messages_workspace_bot', ('workspace_uuid', 'bot_id', 'timestamp'), False, None),
|
||||
(
|
||||
'ix_monitoring_messages_workspace_pipeline',
|
||||
('workspace_uuid', 'pipeline_id', 'timestamp'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
('ix_monitoring_messages_workspace_session', ('workspace_uuid', 'session_id'), False, None),
|
||||
),
|
||||
'monitoring_llm_calls': (
|
||||
('ix_monitoring_llm_calls_workspace_timestamp', ('workspace_uuid', 'timestamp'), False, None),
|
||||
('ix_monitoring_llm_calls_workspace_session', ('workspace_uuid', 'session_id'), False, None),
|
||||
('ix_monitoring_llm_calls_workspace_message', ('workspace_uuid', 'message_id'), False, None),
|
||||
),
|
||||
'monitoring_tool_calls': (
|
||||
('ix_monitoring_tool_calls_workspace_timestamp', ('workspace_uuid', 'timestamp'), False, None),
|
||||
('ix_monitoring_tool_calls_workspace_session', ('workspace_uuid', 'session_id'), False, None),
|
||||
('ix_monitoring_tool_calls_workspace_message', ('workspace_uuid', 'message_id'), False, None),
|
||||
),
|
||||
'monitoring_sessions': (
|
||||
('ix_monitoring_sessions_workspace_activity', ('workspace_uuid', 'last_activity'), False, None),
|
||||
('ix_monitoring_sessions_workspace_active', ('workspace_uuid', 'is_active'), False, None),
|
||||
('ix_monitoring_sessions_workspace_bot', ('workspace_uuid', 'bot_id', 'last_activity'), False, None),
|
||||
),
|
||||
'monitoring_errors': (
|
||||
('ix_monitoring_errors_workspace_timestamp', ('workspace_uuid', 'timestamp'), False, None),
|
||||
('ix_monitoring_errors_workspace_session', ('workspace_uuid', 'session_id'), False, None),
|
||||
('ix_monitoring_errors_workspace_message', ('workspace_uuid', 'message_id'), False, None),
|
||||
),
|
||||
'monitoring_embedding_calls': (
|
||||
(
|
||||
'ix_monitoring_embedding_calls_workspace_timestamp',
|
||||
('workspace_uuid', 'timestamp'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
(
|
||||
'ix_monitoring_embedding_calls_workspace_kb',
|
||||
('workspace_uuid', 'knowledge_base_id'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
(
|
||||
'ix_monitoring_embedding_calls_workspace_session',
|
||||
('workspace_uuid', 'session_id'),
|
||||
False,
|
||||
None,
|
||||
),
|
||||
),
|
||||
'monitoring_feedback': (
|
||||
(
|
||||
'uq_monitoring_feedback_workspace_feedback_id',
|
||||
('workspace_uuid', 'feedback_id'),
|
||||
True,
|
||||
None,
|
||||
),
|
||||
('ix_monitoring_feedback_workspace_timestamp', ('workspace_uuid', 'timestamp'), False, None),
|
||||
('ix_monitoring_feedback_workspace_session', ('workspace_uuid', 'session_id'), False, None),
|
||||
('ix_monitoring_feedback_workspace_message', ('workspace_uuid', 'message_id'), False, None),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _inspector(conn: sa.Connection) -> sa.Inspector:
|
||||
return sa.inspect(conn)
|
||||
|
||||
|
||||
def _table_names(conn: sa.Connection) -> set[str]:
|
||||
return set(_inspector(conn).get_table_names())
|
||||
|
||||
|
||||
def _columns(conn: sa.Connection, table_name: str) -> dict[str, dict]:
|
||||
return {column['name']: column for column in _inspector(conn).get_columns(table_name)}
|
||||
|
||||
|
||||
def _index_names(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
return {index['name'] for index in _inspector(conn).get_indexes(table_name)}
|
||||
|
||||
|
||||
def _unique_column_sets(conn: sa.Connection, table_name: str) -> set[tuple[str, ...]]:
|
||||
inspector = _inspector(conn)
|
||||
result = {
|
||||
tuple(constraint.get('column_names') or ()) for constraint in inspector.get_unique_constraints(table_name)
|
||||
}
|
||||
result.update(
|
||||
tuple(index.get('column_names') or ()) for index in inspector.get_indexes(table_name) if index.get('unique')
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _foreign_key_exists(
|
||||
conn: sa.Connection,
|
||||
table_name: str,
|
||||
local_columns: tuple[str, ...],
|
||||
referred_table: str,
|
||||
referred_columns: tuple[str, ...],
|
||||
) -> bool:
|
||||
return any(
|
||||
tuple(foreign_key.get('constrained_columns') or ()) == local_columns
|
||||
and foreign_key.get('referred_table') == referred_table
|
||||
and tuple(foreign_key.get('referred_columns') or ()) == referred_columns
|
||||
for foreign_key in _inspector(conn).get_foreign_keys(table_name)
|
||||
)
|
||||
|
||||
|
||||
def _metadata_value(conn: sa.Connection, key: str) -> str | None:
|
||||
if 'metadata' not in _table_names(conn):
|
||||
return None
|
||||
metadata = sa.table(
|
||||
'metadata',
|
||||
sa.column('key', sa.String(255)),
|
||||
sa.column('value', sa.String(255)),
|
||||
)
|
||||
value = conn.execute(sa.select(metadata.c.value).where(metadata.c.key == key)).scalar_one_or_none()
|
||||
return value.strip() if isinstance(value, str) and value.strip() else None
|
||||
|
||||
|
||||
def _default_workspace_uuid(conn: sa.Connection) -> str | None:
|
||||
if 'workspaces' not in _table_names(conn):
|
||||
return None
|
||||
workspaces = sa.table(
|
||||
'workspaces',
|
||||
sa.column('uuid', sa.String(36)),
|
||||
sa.column('instance_uuid', sa.String(255)),
|
||||
sa.column('source', sa.String(32)),
|
||||
)
|
||||
instance_uuid = _metadata_value(conn, 'instance_uuid')
|
||||
query = sa.select(workspaces.c.uuid).where(workspaces.c.source == 'local')
|
||||
if instance_uuid is not None:
|
||||
query = query.where(workspaces.c.instance_uuid == instance_uuid)
|
||||
rows = conn.execute(query).all()
|
||||
if len(rows) > 1:
|
||||
raise RuntimeError('Cannot backfill tenant resources: multiple local Workspaces exist')
|
||||
return rows[0][0] if rows else None
|
||||
|
||||
|
||||
def _upgrade_normalized_email(conn: sa.Connection) -> None:
|
||||
if 'users' not in _table_names(conn):
|
||||
return
|
||||
columns = _columns(conn, 'users')
|
||||
if 'normalized_email' not in columns:
|
||||
op.add_column('users', sa.Column('normalized_email', sa.String(320), nullable=True))
|
||||
|
||||
users = sa.table(
|
||||
'users',
|
||||
sa.column('id', sa.Integer()),
|
||||
sa.column('user', sa.String(255)),
|
||||
sa.column('normalized_email', sa.String(320)),
|
||||
)
|
||||
# Use the exact same normalization algorithm as the runtime. Database
|
||||
# ``lower()`` is ASCII-only on SQLite and is not equivalent to Python
|
||||
# ``casefold()`` (for example, Straße -> strasse). Recompute every row so
|
||||
# an interrupted expand/backfill attempt using an older migration body can
|
||||
# be resumed safely.
|
||||
seen_emails: dict[str, int] = {}
|
||||
for user_id, email in conn.execute(sa.select(users.c.id, users.c.user).order_by(users.c.id)).all():
|
||||
normalized_email = str(email or '').strip().casefold()
|
||||
if not normalized_email:
|
||||
raise RuntimeError(f'Cannot normalize empty account identity for user row {user_id}')
|
||||
if len(normalized_email) > 320:
|
||||
raise RuntimeError(
|
||||
f'Cannot normalize account identity for user row {user_id}: canonical value exceeds 320 characters'
|
||||
)
|
||||
duplicate_user_id = seen_emails.get(normalized_email)
|
||||
if duplicate_user_id is not None:
|
||||
raise RuntimeError(
|
||||
f'Cannot create normalized account identity: user rows '
|
||||
f'{duplicate_user_id} and {user_id} both normalize to {normalized_email!r}'
|
||||
)
|
||||
seen_emails[normalized_email] = user_id
|
||||
conn.execute(users.update().where(users.c.id == user_id).values(normalized_email=normalized_email))
|
||||
|
||||
columns = _columns(conn, 'users')
|
||||
checks = {
|
||||
constraint.get('name'): constraint
|
||||
for constraint in _inspector(conn).get_check_constraints('users')
|
||||
if constraint.get('name') is not None
|
||||
}
|
||||
identity_check = checks.get('ck_users_normalized_email')
|
||||
identity_check_sql = str((identity_check or {}).get('sqltext') or '').casefold()
|
||||
# Python casefold is the canonical identity algorithm. SQL ``lower`` is
|
||||
# dialect/locale dependent (notably Cherokee folds to uppercase in Python
|
||||
# but PostgreSQL lowercases it), so the database validates only portable
|
||||
# structural invariants and uniqueness.
|
||||
replace_legacy_identity_check = identity_check is not None and 'lower' in identity_check_sql
|
||||
needs_contract = columns['normalized_email']['nullable'] or identity_check is None or replace_legacy_identity_check
|
||||
if needs_contract:
|
||||
with op.batch_alter_table('users') as batch_op:
|
||||
if replace_legacy_identity_check:
|
||||
batch_op.drop_constraint('ck_users_normalized_email', type_='check')
|
||||
if columns['normalized_email']['nullable']:
|
||||
batch_op.alter_column(
|
||||
'normalized_email',
|
||||
existing_type=columns['normalized_email']['type'],
|
||||
nullable=False,
|
||||
)
|
||||
if identity_check is None or replace_legacy_identity_check:
|
||||
batch_op.create_check_constraint(
|
||||
'ck_users_normalized_email',
|
||||
'normalized_email = trim(normalized_email) '
|
||||
'AND length(normalized_email) > 0 '
|
||||
'AND length(normalized_email) <= 320',
|
||||
)
|
||||
if 'uq_users_normalized_email' not in _index_names(conn, 'users'):
|
||||
op.create_index('uq_users_normalized_email', 'users', ['normalized_email'], unique=True)
|
||||
|
||||
|
||||
def _api_key_owner(conn: sa.Connection, workspace_uuid: str | None) -> str | None:
|
||||
if workspace_uuid is None or 'workspace_memberships' not in _table_names(conn):
|
||||
return None
|
||||
memberships = sa.table(
|
||||
'workspace_memberships',
|
||||
sa.column('workspace_uuid', sa.String(36)),
|
||||
sa.column('account_uuid', sa.String(36)),
|
||||
sa.column('role', sa.String(32)),
|
||||
)
|
||||
return conn.execute(
|
||||
sa.select(memberships.c.account_uuid)
|
||||
.where(
|
||||
memberships.c.workspace_uuid == workspace_uuid,
|
||||
memberships.c.role == 'owner',
|
||||
)
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def _expand_and_hash_api_keys(conn: sa.Connection, workspace_uuid: str | None) -> None:
|
||||
if 'api_keys' not in _table_names(conn):
|
||||
return
|
||||
columns = _columns(conn, 'api_keys')
|
||||
additions = (
|
||||
('uuid', sa.Column('uuid', sa.String(36), nullable=True)),
|
||||
('created_by_account_uuid', sa.Column('created_by_account_uuid', sa.String(36), nullable=True)),
|
||||
('key_hash', sa.Column('key_hash', sa.String(64), nullable=True)),
|
||||
('scopes', sa.Column('scopes', sa.JSON(), nullable=True)),
|
||||
('status', sa.Column('status', sa.String(32), nullable=True, server_default='active')),
|
||||
('expires_at', sa.Column('expires_at', sa.DateTime(), nullable=True)),
|
||||
('last_used_at', sa.Column('last_used_at', sa.DateTime(), nullable=True)),
|
||||
)
|
||||
for name, column in additions:
|
||||
if name not in columns:
|
||||
op.add_column('api_keys', column)
|
||||
|
||||
columns = _columns(conn, 'api_keys')
|
||||
api_key_columns = [sa.column('id', sa.Integer())]
|
||||
for name in ('uuid', 'created_by_account_uuid', 'key_hash', 'scopes', 'status', 'key'):
|
||||
if name in columns:
|
||||
api_key_columns.append(sa.column(name, columns[name]['type']))
|
||||
api_keys = sa.table('api_keys', *api_key_columns)
|
||||
owner_uuid = _api_key_owner(conn, workspace_uuid)
|
||||
selected_columns = [api_keys.c.id, api_keys.c.uuid, api_keys.c.key_hash]
|
||||
if 'key' in api_keys.c:
|
||||
selected_columns.append(api_keys.c.key)
|
||||
rows = conn.execute(sa.select(*selected_columns).order_by(api_keys.c.id)).mappings().all()
|
||||
for row in rows:
|
||||
values: dict[str, object] = {}
|
||||
if not row['uuid']:
|
||||
values['uuid'] = str(uuid.uuid4())
|
||||
if not row['key_hash']:
|
||||
plaintext = row.get('key')
|
||||
if not isinstance(plaintext, str) or not plaintext:
|
||||
raise RuntimeError(f'API key row {row["id"]} has no secret to hash')
|
||||
values['key_hash'] = hashlib.sha256(plaintext.encode()).hexdigest()
|
||||
if values:
|
||||
conn.execute(api_keys.update().where(api_keys.c.id == row['id']).values(**values))
|
||||
# Pre-tenancy API keys historically had unrestricted instance access. A
|
||||
# wildcard preserves that behavior while binding it to the backfilled
|
||||
# Workspace; new keys must store their requested explicit scopes.
|
||||
conn.execute(api_keys.update().where(api_keys.c.scopes.is_(None)).values(scopes=['*']))
|
||||
conn.execute(api_keys.update().where(api_keys.c.status.is_(None)).values(status='active'))
|
||||
if owner_uuid is not None:
|
||||
conn.execute(
|
||||
api_keys.update()
|
||||
.where(api_keys.c.created_by_account_uuid.is_(None))
|
||||
.values(created_by_account_uuid=owner_uuid)
|
||||
)
|
||||
|
||||
columns = _columns(conn, 'api_keys')
|
||||
check_names = {constraint.get('name') for constraint in _inspector(conn).get_check_constraints('api_keys')}
|
||||
existing_fks = _inspector(conn).get_foreign_keys('api_keys')
|
||||
creator_fk_exists = any(
|
||||
tuple(foreign_key.get('constrained_columns') or ()) == ('created_by_account_uuid',)
|
||||
and foreign_key.get('referred_table') == 'users'
|
||||
and tuple(foreign_key.get('referred_columns') or ()) == ('uuid',)
|
||||
for foreign_key in existing_fks
|
||||
)
|
||||
has_legacy_key = 'key' in columns
|
||||
needs_contract = (
|
||||
any(columns[name]['nullable'] for name in ('uuid', 'key_hash', 'scopes', 'status'))
|
||||
or 'ck_api_keys_status' not in check_names
|
||||
or not creator_fk_exists
|
||||
or has_legacy_key
|
||||
)
|
||||
if needs_contract:
|
||||
naming = {'uq': 'uq_%(table_name)s_%(column_0_name)s'}
|
||||
with op.batch_alter_table('api_keys', naming_convention=naming) as batch_op:
|
||||
for name in ('uuid', 'key_hash', 'scopes', 'status'):
|
||||
if columns[name]['nullable']:
|
||||
batch_op.alter_column(name, existing_type=columns[name]['type'], nullable=False)
|
||||
if 'ck_api_keys_status' not in check_names:
|
||||
batch_op.create_check_constraint('ck_api_keys_status', "status IN ('active', 'revoked')")
|
||||
if not creator_fk_exists:
|
||||
batch_op.create_foreign_key(
|
||||
'fk_api_keys_created_by_account',
|
||||
'users',
|
||||
['created_by_account_uuid'],
|
||||
['uuid'],
|
||||
ondelete='SET NULL',
|
||||
)
|
||||
if has_legacy_key:
|
||||
# Dropping the column also removes its old global plaintext
|
||||
# unique constraint/index during SQLite's batch rebuild.
|
||||
batch_op.drop_column('key')
|
||||
|
||||
|
||||
def _expand_workspace_columns(conn: sa.Connection, workspace_uuid: str | None) -> None:
|
||||
tables = _table_names(conn)
|
||||
for table_name in _TENANT_TABLES:
|
||||
if table_name not in tables:
|
||||
continue
|
||||
columns = _columns(conn, table_name)
|
||||
if 'workspace_uuid' not in columns:
|
||||
op.add_column(table_name, sa.Column('workspace_uuid', sa.String(36), nullable=True))
|
||||
tenant_table = sa.table(table_name, sa.column('workspace_uuid', sa.String(36)))
|
||||
null_count = conn.scalar(
|
||||
sa.select(sa.func.count()).select_from(tenant_table).where(tenant_table.c.workspace_uuid.is_(None))
|
||||
)
|
||||
if null_count:
|
||||
if workspace_uuid is None:
|
||||
raise RuntimeError(f'Cannot backfill {table_name}: the instance has no unique local Workspace')
|
||||
conn.execute(
|
||||
tenant_table.update()
|
||||
.where(tenant_table.c.workspace_uuid.is_(None))
|
||||
.values(workspace_uuid=workspace_uuid)
|
||||
)
|
||||
|
||||
|
||||
def _mark_legacy_vector_collections(conn: sa.Connection, workspace_uuid: str | None) -> None:
|
||||
"""Persist which pre-tenancy KBs must keep using ``collection_id``.
|
||||
|
||||
The marker is backfilled only when this migration introduces the column,
|
||||
or resumes while that newly added column is still nullable. A fresh
|
||||
schema already contains the non-null column with ``false`` as its default,
|
||||
so knowledge bases created under the scoped-vector contract can never be
|
||||
mistaken for legacy data during a later migration retry.
|
||||
"""
|
||||
|
||||
if 'knowledge_bases' not in _table_names(conn):
|
||||
return
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
introduced = 'legacy_vector_collection' not in columns
|
||||
if introduced:
|
||||
op.add_column(
|
||||
'knowledge_bases',
|
||||
sa.Column('legacy_vector_collection', sa.Boolean(), nullable=True),
|
||||
)
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
needs_legacy_backfill = introduced or columns['legacy_vector_collection']['nullable']
|
||||
|
||||
knowledge_bases = sa.table(
|
||||
'knowledge_bases',
|
||||
sa.column('collection_id', columns['collection_id']['type']),
|
||||
sa.column('legacy_vector_collection', sa.Boolean()),
|
||||
*((sa.column('workspace_uuid', columns['workspace_uuid']['type']),) if 'workspace_uuid' in columns else ()),
|
||||
)
|
||||
if needs_legacy_backfill and workspace_uuid is not None:
|
||||
legacy_filter = sa.and_(
|
||||
knowledge_bases.c.collection_id.is_not(None),
|
||||
sa.func.length(sa.func.trim(knowledge_bases.c.collection_id)) > 0,
|
||||
)
|
||||
if 'workspace_uuid' in knowledge_bases.c:
|
||||
# A partially migrated database may already have Workspace
|
||||
# columns. Never mark a projected cloud row as legacy.
|
||||
legacy_filter = sa.and_(
|
||||
legacy_filter,
|
||||
sa.or_(
|
||||
knowledge_bases.c.workspace_uuid.is_(None),
|
||||
knowledge_bases.c.workspace_uuid == workspace_uuid,
|
||||
),
|
||||
)
|
||||
conn.execute(knowledge_bases.update().where(legacy_filter).values(legacy_vector_collection=True))
|
||||
|
||||
conn.execute(
|
||||
knowledge_bases.update()
|
||||
.where(knowledge_bases.c.legacy_vector_collection.is_(None))
|
||||
.values(legacy_vector_collection=False)
|
||||
)
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
if columns['legacy_vector_collection']['nullable']:
|
||||
with op.batch_alter_table('knowledge_bases') as batch_op:
|
||||
batch_op.alter_column(
|
||||
'legacy_vector_collection',
|
||||
existing_type=columns['legacy_vector_collection']['type'],
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
)
|
||||
|
||||
|
||||
def _drop_legacy_uniqueness(conn: sa.Connection) -> None:
|
||||
if 'bot_admins' in _table_names(conn):
|
||||
for constraint in _inspector(conn).get_unique_constraints('bot_admins'):
|
||||
if tuple(constraint.get('column_names') or ()) == ('bot_uuid', 'launcher_type', 'launcher_id'):
|
||||
with op.batch_alter_table('bot_admins') as batch_op:
|
||||
batch_op.drop_constraint(constraint['name'], type_='unique')
|
||||
break
|
||||
|
||||
if 'monitoring_feedback' in _table_names(conn):
|
||||
dropped_constraint = False
|
||||
for constraint in _inspector(conn).get_unique_constraints('monitoring_feedback'):
|
||||
if tuple(constraint.get('column_names') or ()) == ('feedback_id',):
|
||||
convention = {'uq': 'uq_%(table_name)s_%(column_0_name)s'}
|
||||
constraint_name = constraint.get('name') or 'uq_monitoring_feedback_feedback_id'
|
||||
with op.batch_alter_table(
|
||||
'monitoring_feedback',
|
||||
naming_convention=convention,
|
||||
) as batch_op:
|
||||
batch_op.drop_constraint(constraint_name, type_='unique')
|
||||
dropped_constraint = True
|
||||
break
|
||||
if not dropped_constraint:
|
||||
for index in _inspector(conn).get_indexes('monitoring_feedback'):
|
||||
if index.get('unique') and tuple(index.get('column_names') or ()) == ('feedback_id',):
|
||||
op.drop_index(index['name'], table_name='monitoring_feedback')
|
||||
|
||||
|
||||
def _create_index_if_missing(
|
||||
conn: sa.Connection,
|
||||
table_name: str,
|
||||
name: str,
|
||||
columns: tuple[str, ...],
|
||||
unique: bool,
|
||||
predicate: sa.TextClause | None,
|
||||
) -> None:
|
||||
if name in _index_names(conn, table_name):
|
||||
return
|
||||
if unique and predicate is None and columns in _unique_column_sets(conn, table_name):
|
||||
return
|
||||
kwargs = {}
|
||||
if predicate is not None:
|
||||
kwargs = {'sqlite_where': predicate, 'postgresql_where': predicate}
|
||||
op.create_index(name, table_name, list(columns), unique=unique, **kwargs)
|
||||
|
||||
|
||||
def _validate_scoped_unique_data(conn: sa.Connection) -> None:
|
||||
checks = (
|
||||
('mcp_servers', ('workspace_uuid', 'name')),
|
||||
('knowledge_bases', ('workspace_uuid', 'collection_id')),
|
||||
)
|
||||
for table_name, column_names in checks:
|
||||
if table_name not in _table_names(conn):
|
||||
continue
|
||||
columns = _columns(conn, table_name)
|
||||
if not all(column_name in columns for column_name in column_names):
|
||||
continue
|
||||
table = sa.table(
|
||||
table_name,
|
||||
*(sa.column(column_name, columns[column_name]['type']) for column_name in column_names),
|
||||
)
|
||||
group_columns = [table.c[column_name] for column_name in column_names]
|
||||
query = sa.select(*group_columns, sa.func.count()).group_by(*group_columns).having(sa.func.count() > 1)
|
||||
if column_names[-1] == 'collection_id':
|
||||
query = query.where(group_columns[-1].is_not(None))
|
||||
duplicate = conn.execute(query.limit(1)).first()
|
||||
if duplicate is not None:
|
||||
raise RuntimeError(
|
||||
f'Cannot create scoped unique key on {table_name}{column_names}: duplicate {duplicate!r}'
|
||||
)
|
||||
|
||||
|
||||
def _create_parent_and_scoped_indexes(conn: sa.Connection) -> None:
|
||||
_validate_scoped_unique_data(conn)
|
||||
tables = _table_names(conn)
|
||||
for table_name, indexes in _SCOPED_INDEXES.items():
|
||||
if table_name not in tables:
|
||||
continue
|
||||
available_columns = _columns(conn, table_name)
|
||||
for name, columns, unique, predicate in indexes:
|
||||
if all(column in available_columns for column in columns):
|
||||
_create_index_if_missing(conn, table_name, name, columns, unique, predicate)
|
||||
|
||||
|
||||
def _contract_table(conn: sa.Connection, table_name: str) -> None:
|
||||
columns = _columns(conn, table_name)
|
||||
if 'workspace_uuid' not in columns:
|
||||
return
|
||||
direct_workspace_fk = _foreign_key_exists(
|
||||
conn,
|
||||
table_name,
|
||||
('workspace_uuid',),
|
||||
'workspaces',
|
||||
('uuid',),
|
||||
)
|
||||
current_pk = tuple(_inspector(conn).get_pk_constraint(table_name).get('constrained_columns') or ())
|
||||
desired_pk = _COMPOSITE_PRIMARY_KEYS.get(table_name)
|
||||
missing_composite_fks = [
|
||||
foreign_key
|
||||
for foreign_key in _COMPOSITE_FOREIGN_KEYS.get(table_name, ())
|
||||
if not _foreign_key_exists(
|
||||
conn,
|
||||
table_name,
|
||||
foreign_key[1],
|
||||
foreign_key[2],
|
||||
foreign_key[3],
|
||||
)
|
||||
]
|
||||
needs_contract = (
|
||||
columns['workspace_uuid']['nullable']
|
||||
or not direct_workspace_fk
|
||||
or (desired_pk is not None and current_pk != desired_pk)
|
||||
or bool(missing_composite_fks)
|
||||
)
|
||||
if not needs_contract:
|
||||
return
|
||||
|
||||
naming = {
|
||||
'pk': 'pk_%(table_name)s',
|
||||
'fk': 'fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s',
|
||||
}
|
||||
pk_name = _inspector(conn).get_pk_constraint(table_name).get('name') or f'pk_{table_name}'
|
||||
with op.batch_alter_table(table_name, naming_convention=naming) as batch_op:
|
||||
if columns['workspace_uuid']['nullable']:
|
||||
batch_op.alter_column(
|
||||
'workspace_uuid',
|
||||
existing_type=columns['workspace_uuid']['type'],
|
||||
nullable=False,
|
||||
)
|
||||
if desired_pk is not None and current_pk != desired_pk:
|
||||
batch_op.drop_constraint(pk_name, type_='primary')
|
||||
batch_op.create_primary_key(f'pk_{table_name}', list(desired_pk))
|
||||
if not direct_workspace_fk:
|
||||
batch_op.create_foreign_key(
|
||||
f'fk_{table_name}_workspace',
|
||||
'workspaces',
|
||||
['workspace_uuid'],
|
||||
['uuid'],
|
||||
ondelete='CASCADE',
|
||||
)
|
||||
for name, local_columns, referred_table, referred_columns, ondelete in missing_composite_fks:
|
||||
batch_op.create_foreign_key(
|
||||
name,
|
||||
referred_table,
|
||||
list(local_columns),
|
||||
list(referred_columns),
|
||||
ondelete=ondelete,
|
||||
)
|
||||
|
||||
|
||||
def _contract_workspace_columns(conn: sa.Connection) -> None:
|
||||
tables = _table_names(conn)
|
||||
# Parents must be contracted before their children so SQLite can validate
|
||||
# the exact composite target key during a batch-table rebuild.
|
||||
order = (
|
||||
'api_keys',
|
||||
'bots',
|
||||
'bot_admins',
|
||||
'binary_storages',
|
||||
'mcp_servers',
|
||||
'model_providers',
|
||||
'llm_models',
|
||||
'embedding_models',
|
||||
'rerank_models',
|
||||
'legacy_pipelines',
|
||||
'pipeline_run_records',
|
||||
'plugin_settings',
|
||||
'knowledge_bases',
|
||||
'knowledge_base_files',
|
||||
'knowledge_base_chunks',
|
||||
'webhooks',
|
||||
'monitoring_messages',
|
||||
'monitoring_llm_calls',
|
||||
'monitoring_tool_calls',
|
||||
'monitoring_sessions',
|
||||
'monitoring_errors',
|
||||
'monitoring_embedding_calls',
|
||||
'monitoring_feedback',
|
||||
)
|
||||
for table_name in order:
|
||||
if table_name in tables:
|
||||
_contract_table(conn, table_name)
|
||||
|
||||
|
||||
def _migrate_workspace_metadata(conn: sa.Connection, workspace_uuid: str | None) -> None:
|
||||
tables = _table_names(conn)
|
||||
if 'workspaces' not in tables:
|
||||
return
|
||||
if 'workspace_metadata' not in tables:
|
||||
op.create_table(
|
||||
'workspace_metadata',
|
||||
sa.Column('workspace_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('key', sa.String(255), nullable=False),
|
||||
sa.Column('value', sa.String(255), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid'],
|
||||
['workspaces.uuid'],
|
||||
name='fk_workspace_metadata_workspace',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.PrimaryKeyConstraint('workspace_uuid', 'key', name='pk_workspace_metadata'),
|
||||
)
|
||||
if workspace_uuid is None or 'metadata' not in tables:
|
||||
return
|
||||
metadata = sa.table(
|
||||
'metadata',
|
||||
sa.column('key', sa.String(255)),
|
||||
sa.column('value', sa.String(255)),
|
||||
)
|
||||
workspace_metadata = sa.table(
|
||||
'workspace_metadata',
|
||||
sa.column('workspace_uuid', sa.String(36)),
|
||||
sa.column('key', sa.String(255)),
|
||||
sa.column('value', sa.String(255)),
|
||||
)
|
||||
tenant_keys = ('wizard_status', 'wizard_progress', 'rag_plugin_migration_needed')
|
||||
rows = conn.execute(sa.select(metadata.c.key, metadata.c.value).where(metadata.c.key.in_(tenant_keys))).all()
|
||||
for key, value in rows:
|
||||
exists = conn.execute(
|
||||
sa.select(workspace_metadata.c.key).where(
|
||||
workspace_metadata.c.workspace_uuid == workspace_uuid,
|
||||
workspace_metadata.c.key == key,
|
||||
)
|
||||
).first()
|
||||
if exists is None:
|
||||
conn.execute(
|
||||
workspace_metadata.insert().values(
|
||||
workspace_uuid=workspace_uuid,
|
||||
key=key,
|
||||
value=value,
|
||||
)
|
||||
)
|
||||
if rows:
|
||||
conn.execute(metadata.delete().where(metadata.c.key.in_(tenant_keys)))
|
||||
|
||||
|
||||
def _validate_contract(conn: sa.Connection) -> None:
|
||||
for table_name in _TENANT_TABLES:
|
||||
if table_name not in _table_names(conn):
|
||||
continue
|
||||
columns = _columns(conn, table_name)
|
||||
if 'workspace_uuid' not in columns or columns['workspace_uuid']['nullable']:
|
||||
raise RuntimeError(f'{table_name}.workspace_uuid was not contracted to NOT NULL')
|
||||
table = sa.table(table_name, sa.column('workspace_uuid', sa.String(36)))
|
||||
if conn.scalar(sa.select(sa.func.count()).select_from(table).where(table.c.workspace_uuid.is_(None))):
|
||||
raise RuntimeError(f'{table_name} still contains unscoped rows')
|
||||
if conn.dialect.name == 'sqlite':
|
||||
violations = conn.execute(sa.text('PRAGMA foreign_key_check')).all()
|
||||
if violations:
|
||||
raise RuntimeError(f'SQLite foreign key validation failed: {violations[:5]!r}')
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
_upgrade_normalized_email(conn)
|
||||
workspace_uuid = _default_workspace_uuid(conn)
|
||||
_mark_legacy_vector_collections(conn, workspace_uuid)
|
||||
_expand_workspace_columns(conn, workspace_uuid)
|
||||
_expand_and_hash_api_keys(conn, workspace_uuid)
|
||||
_drop_legacy_uniqueness(conn)
|
||||
_create_parent_and_scoped_indexes(conn)
|
||||
_contract_workspace_columns(conn)
|
||||
_migrate_workspace_metadata(conn, workspace_uuid)
|
||||
_validate_contract(conn)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise RuntimeError(
|
||||
'0010_scope_resources is intentionally irreversible because plaintext API key secrets were securely removed'
|
||||
)
|
||||
@@ -0,0 +1,201 @@
|
||||
"""enforce PostgreSQL tenant isolation with exact discovery contracts
|
||||
|
||||
Revision ID: 0011_postgres_tenant_rls
|
||||
Revises: 0010_scope_resources
|
||||
Create Date: 2026-07-19
|
||||
|
||||
The table and policy lists are deliberately duplicated from the runtime
|
||||
contract. Alembic revisions must remain self-contained after application code
|
||||
evolves. Discovery policies are SELECT-only and reveal the minimum rows needed
|
||||
to turn an authenticated credential into one Workspace transaction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0011_postgres_tenant_rls'
|
||||
down_revision = '0010_scope_resources'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_POLICY_NAME = 'langbot_workspace_isolation'
|
||||
_ACCOUNT_POLICY_NAME = 'langbot_account_discovery'
|
||||
_API_KEY_POLICY_NAME = 'langbot_api_key_discovery'
|
||||
_INVITATION_POLICY_NAME = 'langbot_invitation_discovery'
|
||||
_INSTANCE_POLICY_NAME = 'langbot_instance_discovery'
|
||||
|
||||
_TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
_ACCOUNT_SETTING = 'langbot.account_uuid'
|
||||
_API_KEY_HASH_SETTING = 'langbot.api_key_hash'
|
||||
_INVITATION_HASH_SETTING = 'langbot.invitation_hash'
|
||||
_INSTANCE_SETTING = 'langbot.instance_uuid'
|
||||
_OSS_WORKSPACE_METADATA_KEY = 'oss_workspace_uuid'
|
||||
|
||||
_TENANT_TABLE_COLUMNS: dict[str, str] = {
|
||||
'workspaces': 'uuid',
|
||||
'workspace_memberships': 'workspace_uuid',
|
||||
'workspace_invitations': 'workspace_uuid',
|
||||
'workspace_execution_states': 'workspace_uuid',
|
||||
'workspace_metadata': 'workspace_uuid',
|
||||
'api_keys': 'workspace_uuid',
|
||||
'bots': 'workspace_uuid',
|
||||
'bot_admins': 'workspace_uuid',
|
||||
'binary_storages': 'workspace_uuid',
|
||||
'mcp_servers': 'workspace_uuid',
|
||||
'model_providers': 'workspace_uuid',
|
||||
'llm_models': 'workspace_uuid',
|
||||
'embedding_models': 'workspace_uuid',
|
||||
'rerank_models': 'workspace_uuid',
|
||||
'legacy_pipelines': 'workspace_uuid',
|
||||
'pipeline_run_records': 'workspace_uuid',
|
||||
'plugin_settings': 'workspace_uuid',
|
||||
'knowledge_bases': 'workspace_uuid',
|
||||
'knowledge_base_files': 'workspace_uuid',
|
||||
'knowledge_base_chunks': 'workspace_uuid',
|
||||
'webhooks': 'workspace_uuid',
|
||||
'monitoring_messages': 'workspace_uuid',
|
||||
'monitoring_llm_calls': 'workspace_uuid',
|
||||
'monitoring_tool_calls': 'workspace_uuid',
|
||||
'monitoring_sessions': 'workspace_uuid',
|
||||
'monitoring_errors': 'workspace_uuid',
|
||||
'monitoring_embedding_calls': 'workspace_uuid',
|
||||
'monitoring_feedback': 'workspace_uuid',
|
||||
}
|
||||
|
||||
|
||||
def _setting(name: str) -> str:
|
||||
return f"NULLIF(current_setting('{name}', true), '')"
|
||||
|
||||
|
||||
def _tenant_expression(column: str) -> str:
|
||||
return f'{column}::text = {_setting(_TENANT_SETTING)}'
|
||||
|
||||
|
||||
_DISCOVERY_POLICIES: dict[str, dict[str, str]] = {
|
||||
'workspace_memberships': {
|
||||
_ACCOUNT_POLICY_NAME: (f"account_uuid::text = {_setting(_ACCOUNT_SETTING)} AND status = 'active'"),
|
||||
},
|
||||
'workspace_execution_states': {
|
||||
_INSTANCE_POLICY_NAME: (
|
||||
f"instance_uuid = {_setting(_INSTANCE_SETTING)} AND state = 'active' AND write_fenced = false"
|
||||
),
|
||||
},
|
||||
'api_keys': {
|
||||
_API_KEY_POLICY_NAME: (
|
||||
f"key_hash = {_setting(_API_KEY_HASH_SETTING)} AND status = 'active' "
|
||||
'AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)'
|
||||
),
|
||||
},
|
||||
'workspace_invitations': {
|
||||
_INVITATION_POLICY_NAME: f'token_hash = {_setting(_INVITATION_HASH_SETTING)}',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _quote_identifier(conn: sa.Connection, identifier: str) -> str:
|
||||
return conn.dialect.identifier_preparer.quote(identifier)
|
||||
|
||||
|
||||
def _record_oss_workspace_scope(conn: sa.Connection) -> None:
|
||||
"""Keep PostgreSQL OSS usable after FORCE RLS is enabled."""
|
||||
|
||||
local_workspaces = (
|
||||
conn.execute(sa.text("SELECT uuid FROM workspaces WHERE source = 'local' ORDER BY uuid")).scalars().all()
|
||||
)
|
||||
if len(local_workspaces) != 1:
|
||||
return
|
||||
|
||||
existing = conn.execute(
|
||||
sa.text('SELECT value FROM metadata WHERE key = :key'),
|
||||
{'key': _OSS_WORKSPACE_METADATA_KEY},
|
||||
).scalar_one_or_none()
|
||||
if existing is None:
|
||||
conn.execute(
|
||||
sa.text('INSERT INTO metadata (key, value) VALUES (:key, :value)'),
|
||||
{'key': _OSS_WORKSPACE_METADATA_KEY, 'value': local_workspaces[0]},
|
||||
)
|
||||
elif existing != local_workspaces[0]:
|
||||
raise RuntimeError('Stored OSS Workspace scope does not match the local Workspace')
|
||||
|
||||
|
||||
def _drop_all_policies(conn: sa.Connection, table_name: str) -> None:
|
||||
policy_names = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT p.polname
|
||||
FROM pg_policy p
|
||||
JOIN pg_class c ON c.oid = p.polrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema() AND c.relname = :table_name
|
||||
"""
|
||||
),
|
||||
{'table_name': table_name},
|
||||
).scalars()
|
||||
table = _quote_identifier(conn, table_name)
|
||||
for policy in policy_names:
|
||||
op.execute(sa.text(f'DROP POLICY {_quote_identifier(conn, policy)} ON {table}'))
|
||||
|
||||
|
||||
def _create_policy(
|
||||
conn: sa.Connection,
|
||||
table_name: str,
|
||||
policy_name: str,
|
||||
expression: str,
|
||||
*,
|
||||
command: str,
|
||||
) -> None:
|
||||
table = _quote_identifier(conn, table_name)
|
||||
policy = _quote_identifier(conn, policy_name)
|
||||
if command == 'ALL':
|
||||
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC USING ({expression}) WITH CHECK ({expression})'
|
||||
elif command == 'SELECT':
|
||||
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR SELECT TO PUBLIC USING ({expression})'
|
||||
else: # pragma: no cover - migration-local invariant
|
||||
raise AssertionError(f'Unsupported RLS command: {command}')
|
||||
op.execute(sa.text(sql))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
missing_tables = set(_TENANT_TABLE_COLUMNS) - existing_tables
|
||||
if missing_tables:
|
||||
raise RuntimeError(f'Cannot enable tenant RLS before all tenant-owned tables exist: {sorted(missing_tables)!r}')
|
||||
|
||||
_record_oss_workspace_scope(conn)
|
||||
|
||||
for table_name, tenant_column in _TENANT_TABLE_COLUMNS.items():
|
||||
table = _quote_identifier(conn, table_name)
|
||||
_drop_all_policies(conn, table_name)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_POLICY_NAME,
|
||||
_tenant_expression(_quote_identifier(conn, tenant_column)),
|
||||
command='ALL',
|
||||
)
|
||||
for policy_name, expression in _DISCOVERY_POLICIES.get(table_name, {}).items():
|
||||
_create_policy(conn, table_name, policy_name, expression, command='SELECT')
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
for table_name in _TENANT_TABLE_COLUMNS:
|
||||
if table_name not in existing_tables:
|
||||
continue
|
||||
table = _quote_identifier(conn, table_name)
|
||||
_drop_all_policies(conn, table_name)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
|
||||
@@ -0,0 +1,179 @@
|
||||
"""add immutable plugin installation identity
|
||||
|
||||
Revision ID: 0012_plugin_identity
|
||||
Revises: 0011_postgres_tenant_rls
|
||||
Create Date: 2026-07-19
|
||||
|
||||
The migration gives every legacy row a random, stable installation UUID. A
|
||||
legacy artifact digest is only a valid SHA-256-shaped recovery marker; Core
|
||||
replaces it with the package digest and increments ``runtime_revision`` before
|
||||
the next package apply.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = '0012_plugin_identity'
|
||||
down_revision = '0011_postgres_tenant_rls'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_TABLE = 'plugin_settings'
|
||||
_INSTALLATION_INDEX = 'ix_plugin_settings_workspace_installation'
|
||||
_INSTALLATION_UNIQUE = 'uq_plugin_settings_installation_uuid'
|
||||
_REVISION_CHECK = 'ck_plugin_settings_runtime_revision_positive'
|
||||
_DIGEST_CHECK = 'ck_plugin_settings_artifact_digest_length'
|
||||
|
||||
|
||||
def _column_names(conn: sa.Connection) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return set()
|
||||
return {column['name'] for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def _legacy_digest(installation_uuid: str) -> str:
|
||||
return hashlib.sha256(f'legacy-installation:{installation_uuid}'.encode()).hexdigest()
|
||||
|
||||
|
||||
def _suspend_postgres_rls(conn: sa.Connection) -> tuple[bool, bool]:
|
||||
"""Let the release migration backfill every tenant row after revision 0011."""
|
||||
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return False, False
|
||||
row = conn.execute(
|
||||
sa.text('SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = to_regclass(:table_name)'),
|
||||
{'table_name': _TABLE},
|
||||
).one()
|
||||
rls_enabled, rls_forced = bool(row.relrowsecurity), bool(row.relforcerowsecurity)
|
||||
if rls_forced:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} NO FORCE ROW LEVEL SECURITY'))
|
||||
if rls_enabled:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} DISABLE ROW LEVEL SECURITY'))
|
||||
return rls_enabled, rls_forced
|
||||
|
||||
|
||||
def _restore_postgres_rls(conn: sa.Connection, state: tuple[bool, bool]) -> None:
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
rls_enabled, rls_forced = state
|
||||
if rls_enabled:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} ENABLE ROW LEVEL SECURITY'))
|
||||
if rls_forced:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} FORCE ROW LEVEL SECURITY'))
|
||||
|
||||
|
||||
def _backfill(conn: sa.Connection) -> None:
|
||||
table = sa.table(
|
||||
_TABLE,
|
||||
sa.column('workspace_uuid', sa.String(36)),
|
||||
sa.column('plugin_author', sa.String(255)),
|
||||
sa.column('plugin_name', sa.String(255)),
|
||||
sa.column('installation_uuid', sa.String(36)),
|
||||
sa.column('artifact_digest', sa.String(64)),
|
||||
sa.column('runtime_revision', sa.Integer()),
|
||||
)
|
||||
rows = conn.execute(
|
||||
sa.select(
|
||||
table.c.workspace_uuid,
|
||||
table.c.plugin_author,
|
||||
table.c.plugin_name,
|
||||
table.c.installation_uuid,
|
||||
table.c.artifact_digest,
|
||||
table.c.runtime_revision,
|
||||
)
|
||||
).all()
|
||||
for row in rows:
|
||||
installation_uuid = str(row.installation_uuid or uuid.uuid4())
|
||||
values: dict[str, object] = {}
|
||||
if not row.installation_uuid:
|
||||
values['installation_uuid'] = installation_uuid
|
||||
if not row.artifact_digest or len(str(row.artifact_digest)) != 64:
|
||||
values['artifact_digest'] = _legacy_digest(installation_uuid)
|
||||
if row.runtime_revision is None or row.runtime_revision < 1:
|
||||
values['runtime_revision'] = 1
|
||||
if values:
|
||||
conn.execute(
|
||||
table.update()
|
||||
.where(table.c.workspace_uuid == row.workspace_uuid)
|
||||
.where(table.c.plugin_author == row.plugin_author)
|
||||
.where(table.c.plugin_name == row.plugin_name)
|
||||
.values(**values)
|
||||
)
|
||||
|
||||
|
||||
def _constraint_names(conn: sa.Connection, kind: str) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
getter = inspector.get_unique_constraints if kind == 'unique' else inspector.get_check_constraints
|
||||
return {str(item.get('name')) for item in getter(_TABLE) if item.get('name')}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
columns = _column_names(conn)
|
||||
if not columns:
|
||||
return
|
||||
|
||||
if 'installation_uuid' not in columns:
|
||||
op.add_column(_TABLE, sa.Column('installation_uuid', sa.String(36), nullable=True))
|
||||
if 'artifact_digest' not in columns:
|
||||
op.add_column(_TABLE, sa.Column('artifact_digest', sa.String(64), nullable=True))
|
||||
if 'runtime_revision' not in columns:
|
||||
op.add_column(_TABLE, sa.Column('runtime_revision', sa.Integer(), nullable=True))
|
||||
|
||||
rls_state = _suspend_postgres_rls(conn)
|
||||
try:
|
||||
_backfill(conn)
|
||||
finally:
|
||||
_restore_postgres_rls(conn, rls_state)
|
||||
|
||||
# Fresh databases are created from current metadata before Alembic runs;
|
||||
# guards make the revision safe for that path and for interrupted upgrades.
|
||||
indexes = {index['name'] for index in sa.inspect(conn).get_indexes(_TABLE)}
|
||||
unique_constraints = _constraint_names(conn, 'unique')
|
||||
check_constraints = _constraint_names(conn, 'check')
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
batch.alter_column('installation_uuid', existing_type=sa.String(36), nullable=False)
|
||||
batch.alter_column('artifact_digest', existing_type=sa.String(64), nullable=False)
|
||||
batch.alter_column('runtime_revision', existing_type=sa.Integer(), nullable=False)
|
||||
if _REVISION_CHECK not in check_constraints:
|
||||
batch.create_check_constraint(_REVISION_CHECK, 'runtime_revision >= 1')
|
||||
if _DIGEST_CHECK not in check_constraints:
|
||||
batch.create_check_constraint(_DIGEST_CHECK, 'length(artifact_digest) = 64')
|
||||
if _INSTALLATION_UNIQUE not in unique_constraints:
|
||||
batch.create_unique_constraint(_INSTALLATION_UNIQUE, ['installation_uuid'])
|
||||
if _INSTALLATION_INDEX not in indexes and _INSTALLATION_INDEX not in unique_constraints:
|
||||
batch.create_index(
|
||||
_INSTALLATION_INDEX,
|
||||
['workspace_uuid', 'installation_uuid'],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
columns = _column_names(conn)
|
||||
if not columns:
|
||||
return
|
||||
indexes = {index['name'] for index in sa.inspect(conn).get_indexes(_TABLE)}
|
||||
checks = _constraint_names(conn, 'check')
|
||||
uniques = _constraint_names(conn, 'unique')
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
if _INSTALLATION_INDEX in indexes:
|
||||
batch.drop_index(_INSTALLATION_INDEX)
|
||||
if _DIGEST_CHECK in checks:
|
||||
batch.drop_constraint(_DIGEST_CHECK, type_='check')
|
||||
if _REVISION_CHECK in checks:
|
||||
batch.drop_constraint(_REVISION_CHECK, type_='check')
|
||||
if _INSTALLATION_UNIQUE in uniques:
|
||||
batch.drop_constraint(_INSTALLATION_UNIQUE, type_='unique')
|
||||
for column_name in ('runtime_revision', 'artifact_digest', 'installation_uuid'):
|
||||
if column_name in columns:
|
||||
batch.drop_column(column_name)
|
||||
@@ -0,0 +1,382 @@
|
||||
"""create tenant-scoped pgvector storage in the business database
|
||||
|
||||
Revision ID: 0013_tenant_pgvector
|
||||
Revises: 0012_plugin_identity
|
||||
Create Date: 2026-07-19
|
||||
|
||||
The Cloud application role never executes this DDL. A release migration role
|
||||
installs pgvector once, creates an untyped vector column, and builds a bounded
|
||||
set of expression/partial ANN indexes. Existing legacy rows are migrated only
|
||||
when each row maps unambiguously to one knowledge base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import typing
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
|
||||
revision = '0013_tenant_pgvector'
|
||||
down_revision = '0012_plugin_identity'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_VECTOR_TABLE = 'langbot_vectors'
|
||||
_LEGACY_TABLE = 'langbot_vectors_legacy_0013'
|
||||
_TENANT_POLICY = 'langbot_workspace_isolation'
|
||||
_TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
_KB_DIMENSION_CHECK = 'ck_knowledge_bases_embedding_dimension_positive'
|
||||
_VECTOR_DIMENSION_CHECK = 'ck_langbot_vectors_embedding_dimension'
|
||||
_VECTOR_ALLOWED_CHECK = 'ck_langbot_vectors_embedding_dimension_enabled'
|
||||
_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
|
||||
_LEGACY_SOURCE_TABLES = (
|
||||
'knowledge_bases',
|
||||
'knowledge_base_files',
|
||||
'knowledge_base_chunks',
|
||||
)
|
||||
|
||||
|
||||
def _quote(conn: sa.Connection, identifier: str) -> str:
|
||||
return conn.dialect.identifier_preparer.quote(identifier)
|
||||
|
||||
|
||||
def _columns(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return set()
|
||||
return {column['name'] for column in inspector.get_columns(table_name)}
|
||||
|
||||
|
||||
def _checks(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
return {str(item['name']) for item in sa.inspect(conn).get_check_constraints(table_name) if item.get('name')}
|
||||
|
||||
|
||||
def _ensure_knowledge_base_dimension(conn: sa.Connection) -> None:
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
if 'embedding_dimension' not in columns:
|
||||
op.add_column('knowledge_bases', sa.Column('embedding_dimension', sa.Integer(), nullable=True))
|
||||
if _KB_DIMENSION_CHECK not in _checks(conn, 'knowledge_bases'):
|
||||
with op.batch_alter_table('knowledge_bases') as batch:
|
||||
batch.create_check_constraint(
|
||||
_KB_DIMENSION_CHECK,
|
||||
'embedding_dimension IS NULL OR embedding_dimension > 0',
|
||||
)
|
||||
|
||||
|
||||
def _create_vector_table() -> None:
|
||||
enabled = ', '.join(str(item) for item in _ALLOWED_DIMENSIONS)
|
||||
op.create_table(
|
||||
_VECTOR_TABLE,
|
||||
sa.Column('workspace_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('knowledge_base_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('vector_id', sa.String(255), nullable=False),
|
||||
sa.Column('embedding_dimension', sa.Integer(), nullable=False),
|
||||
sa.Column('embedding', Vector(), nullable=False),
|
||||
sa.Column('text', sa.Text(), nullable=True),
|
||||
sa.Column('file_id', sa.String(255), nullable=True),
|
||||
sa.Column('chunk_uuid', sa.String(255), nullable=True),
|
||||
sa.PrimaryKeyConstraint(
|
||||
'workspace_uuid',
|
||||
'knowledge_base_uuid',
|
||||
'vector_id',
|
||||
name='pk_langbot_vectors',
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'knowledge_base_uuid'],
|
||||
['knowledge_bases.workspace_uuid', 'knowledge_bases.uuid'],
|
||||
name='fk_langbot_vectors_workspace_kb',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
'vector_dims(embedding) = embedding_dimension',
|
||||
name=_VECTOR_DIMENSION_CHECK,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
f'embedding_dimension IN ({enabled})',
|
||||
name=_VECTOR_ALLOWED_CHECK,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'ix_langbot_vectors_workspace_kb_file',
|
||||
_VECTOR_TABLE,
|
||||
['workspace_uuid', 'knowledge_base_uuid', 'file_id'],
|
||||
)
|
||||
op.create_index(
|
||||
'ix_langbot_vectors_workspace_kb_chunk',
|
||||
_VECTOR_TABLE,
|
||||
['workspace_uuid', 'knowledge_base_uuid', 'chunk_uuid'],
|
||||
)
|
||||
|
||||
|
||||
def _legacy_mapping_predicate() -> str:
|
||||
return """
|
||||
kb.collection_id = legacy.collection
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM knowledge_base_files AS files
|
||||
LEFT JOIN knowledge_base_chunks AS chunks
|
||||
ON chunks.workspace_uuid = files.workspace_uuid
|
||||
AND chunks.file_id = files.uuid
|
||||
WHERE files.workspace_uuid = kb.workspace_uuid
|
||||
AND files.kb_id = kb.uuid
|
||||
AND (files.uuid = legacy.file_id OR chunks.uuid = legacy.chunk_uuid)
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def _legacy_source_rls_states(conn: sa.Connection) -> dict[str, tuple[bool, bool]]:
|
||||
rows = (
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT c.relname, c.relrowsecurity, c.relforcerowsecurity
|
||||
FROM pg_class AS c
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.relname IN :table_names
|
||||
AND c.relkind IN ('r', 'p')
|
||||
"""
|
||||
).bindparams(sa.bindparam('table_names', expanding=True)),
|
||||
{'table_names': _LEGACY_SOURCE_TABLES},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
states = {str(row['relname']): (bool(row['relrowsecurity']), bool(row['relforcerowsecurity'])) for row in rows}
|
||||
missing = set(_LEGACY_SOURCE_TABLES) - set(states)
|
||||
if missing:
|
||||
raise RuntimeError(f'Legacy pgvector source tables are missing: {sorted(missing)!r}')
|
||||
return states
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _suspend_legacy_source_rls(conn: sa.Connection) -> typing.Iterator[None]:
|
||||
"""Temporarily let the table-owning migrator map all legacy tenant rows.
|
||||
|
||||
Revision 0011 enables and forces RLS on each source table. The release
|
||||
migrator intentionally has neither superuser nor BYPASSRLS, so even a table
|
||||
owner cannot read those rows until FORCE RLS is paused. Preserve both flags
|
||||
independently and restore them in ``finally`` so mixed pre-existing states
|
||||
survive successful, rejected, and interrupted legacy migrations.
|
||||
"""
|
||||
|
||||
states = _legacy_source_rls_states(conn)
|
||||
try:
|
||||
for table_name in _LEGACY_SOURCE_TABLES:
|
||||
table = _quote(conn, table_name)
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
|
||||
yield
|
||||
finally:
|
||||
for table_name in _LEGACY_SOURCE_TABLES:
|
||||
table = _quote(conn, table_name)
|
||||
rls_enabled, rls_forced = states[table_name]
|
||||
enabled_clause = 'ENABLE' if rls_enabled else 'DISABLE'
|
||||
forced_clause = 'FORCE' if rls_forced else 'NO FORCE'
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} {enabled_clause} ROW LEVEL SECURITY'))
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} {forced_clause} ROW LEVEL SECURITY'))
|
||||
|
||||
|
||||
def _migrate_legacy_rows(conn: sa.Connection) -> None:
|
||||
predicate = _legacy_mapping_predicate()
|
||||
ambiguous = conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
WITH candidates AS (
|
||||
SELECT legacy.id, kb.workspace_uuid, kb.uuid AS knowledge_base_uuid
|
||||
FROM {_LEGACY_TABLE} AS legacy
|
||||
JOIN knowledge_bases AS kb ON ({predicate})
|
||||
), candidate_counts AS (
|
||||
SELECT id, COUNT(*) AS count
|
||||
FROM candidates
|
||||
GROUP BY id
|
||||
)
|
||||
SELECT legacy.id, COALESCE(candidate_counts.count, 0) AS candidate_count
|
||||
FROM {_LEGACY_TABLE} AS legacy
|
||||
LEFT JOIN candidate_counts ON candidate_counts.id = legacy.id
|
||||
WHERE COALESCE(candidate_counts.count, 0) <> 1
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
).first()
|
||||
if ambiguous is not None:
|
||||
raise RuntimeError(
|
||||
'Legacy pgvector row cannot be mapped to exactly one Workspace/knowledge base: '
|
||||
f'{ambiguous.id!r} has {ambiguous.candidate_count} candidates'
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
INSERT INTO {_VECTOR_TABLE} (
|
||||
workspace_uuid,
|
||||
knowledge_base_uuid,
|
||||
vector_id,
|
||||
embedding_dimension,
|
||||
embedding,
|
||||
text,
|
||||
file_id,
|
||||
chunk_uuid
|
||||
)
|
||||
SELECT
|
||||
kb.workspace_uuid,
|
||||
kb.uuid,
|
||||
legacy.id,
|
||||
vector_dims(legacy.embedding),
|
||||
legacy.embedding,
|
||||
legacy.text,
|
||||
legacy.file_id,
|
||||
legacy.chunk_uuid
|
||||
FROM {_LEGACY_TABLE} AS legacy
|
||||
JOIN knowledge_bases AS kb ON ({predicate})
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
mixed_dimension = conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
SELECT workspace_uuid, knowledge_base_uuid
|
||||
FROM {_VECTOR_TABLE}
|
||||
GROUP BY workspace_uuid, knowledge_base_uuid
|
||||
HAVING MIN(embedding_dimension) <> MAX(embedding_dimension)
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
).first()
|
||||
if mixed_dimension is not None:
|
||||
raise RuntimeError('Legacy knowledge base contains mixed embedding dimensions')
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
UPDATE knowledge_bases AS kb
|
||||
SET embedding_dimension = dimensions.embedding_dimension
|
||||
FROM (
|
||||
SELECT workspace_uuid, knowledge_base_uuid, MIN(embedding_dimension) AS embedding_dimension
|
||||
FROM {_VECTOR_TABLE}
|
||||
GROUP BY workspace_uuid, knowledge_base_uuid
|
||||
) AS dimensions
|
||||
WHERE kb.workspace_uuid = dimensions.workspace_uuid
|
||||
AND kb.uuid = dimensions.knowledge_base_uuid
|
||||
AND kb.embedding_dimension IS NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _drop_all_policies(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _VECTOR_TABLE)
|
||||
policies = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT p.polname
|
||||
FROM pg_policy p
|
||||
JOIN pg_class c ON c.oid = p.polrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema() AND c.relname = :table_name
|
||||
"""
|
||||
),
|
||||
{'table_name': _VECTOR_TABLE},
|
||||
).scalars()
|
||||
for policy_name in policies:
|
||||
op.execute(sa.text(f'DROP POLICY {_quote(conn, policy_name)} ON {table}'))
|
||||
|
||||
|
||||
def _enable_rls(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _VECTOR_TABLE)
|
||||
policy = _quote(conn, _TENANT_POLICY)
|
||||
expression = f"workspace_uuid::text = NULLIF(current_setting('{_TENANT_SETTING}', true), '')"
|
||||
_drop_all_policies(conn)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
|
||||
op.execute(
|
||||
sa.text(
|
||||
f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC '
|
||||
f'USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _create_ann_indexes(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _VECTOR_TABLE)
|
||||
for dimension in _ALLOWED_DIMENSIONS:
|
||||
index = _quote(conn, f'ix_langbot_vectors_hnsw_cosine_{dimension}')
|
||||
op.execute(
|
||||
sa.text(
|
||||
f'CREATE INDEX {index} ON {table} USING hnsw '
|
||||
f'((embedding::vector({dimension})) vector_cosine_ops) '
|
||||
f'WHERE embedding_dimension = {dimension}'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if 'knowledge_bases' not in sa.inspect(conn).get_table_names():
|
||||
if conn.dialect.name == 'postgresql':
|
||||
# The supported PostgreSQL release path creates the business
|
||||
# tables before stamping 0010 and reaching this migration. Missing
|
||||
# knowledge_bases therefore means the operator bypassed the
|
||||
# release bootstrap or the schema is incomplete; stamping head in
|
||||
# that state would make Cloud runtime validation unrecoverable.
|
||||
raise RuntimeError('PostgreSQL release migration requires the knowledge_bases table')
|
||||
# A direct empty SQLite Alembic walk is still used by migration tooling;
|
||||
# Core creates the complete ORM schema on its following compatibility
|
||||
# pass, including the portable embedding_dimension field.
|
||||
return
|
||||
|
||||
_ensure_knowledge_base_dimension(conn)
|
||||
|
||||
# pgvector storage is PostgreSQL-only, but ``embedding_dimension`` is an
|
||||
# ORM field used by both deployment modes. Existing OSS SQLite databases
|
||||
# must receive the column before this revision becomes a no-op.
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
|
||||
op.execute(sa.text('CREATE EXTENSION IF NOT EXISTS vector'))
|
||||
columns = _columns(conn, _VECTOR_TABLE)
|
||||
if columns:
|
||||
scoped_columns = {
|
||||
'workspace_uuid',
|
||||
'knowledge_base_uuid',
|
||||
'vector_id',
|
||||
'embedding_dimension',
|
||||
'embedding',
|
||||
}
|
||||
if scoped_columns.issubset(columns):
|
||||
raise RuntimeError('Tenant pgvector table exists before its owning release migration')
|
||||
legacy_columns = {'id', 'collection', 'embedding'}
|
||||
if not legacy_columns.issubset(columns):
|
||||
raise RuntimeError('Existing pgvector table has an unsupported schema')
|
||||
if _LEGACY_TABLE in sa.inspect(conn).get_table_names():
|
||||
raise RuntimeError(f'Interrupted pgvector migration left {_LEGACY_TABLE!r} behind')
|
||||
op.rename_table(_VECTOR_TABLE, _LEGACY_TABLE)
|
||||
|
||||
_create_vector_table()
|
||||
if _LEGACY_TABLE in sa.inspect(conn).get_table_names():
|
||||
with _suspend_legacy_source_rls(conn):
|
||||
_migrate_legacy_rows(conn)
|
||||
op.drop_table(_LEGACY_TABLE)
|
||||
_create_ann_indexes(conn)
|
||||
_enable_rls(conn)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name == 'postgresql' and _VECTOR_TABLE in sa.inspect(conn).get_table_names():
|
||||
_drop_all_policies(conn)
|
||||
op.drop_table(_VECTOR_TABLE)
|
||||
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
if 'embedding_dimension' in columns:
|
||||
checks = _checks(conn, 'knowledge_bases')
|
||||
with op.batch_alter_table('knowledge_bases') as batch:
|
||||
if _KB_DIMENSION_CHECK in checks:
|
||||
batch.drop_constraint(_KB_DIMENSION_CHECK, type_='check')
|
||||
batch.drop_column('embedding_dimension')
|
||||
@@ -0,0 +1,268 @@
|
||||
"""add the Cloud directory projection persistence boundary
|
||||
|
||||
Revision ID: 0014_cloud_directory
|
||||
Revises: 0013_tenant_pgvector
|
||||
Create Date: 2026-07-24
|
||||
|
||||
The open Core projector receives already-verified control-plane data and is the
|
||||
only runtime path allowed to mutate projected Workspace directory rows. Its
|
||||
transaction-local instance setting is intentionally distinct from both normal
|
||||
Workspace scope and the read-only instance discovery scope.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = '0014_cloud_directory'
|
||||
down_revision = '0013_tenant_pgvector'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_STATE_TABLE = 'directory_projection_states'
|
||||
_INBOX_TABLE = 'directory_projection_inbox'
|
||||
_DIRECTORY_POLICY_NAME = 'langbot_directory_projection'
|
||||
_TENANT_POLICY_NAME = 'langbot_workspace_isolation'
|
||||
_LOCAL_WRITE_POLICY_NAME = 'langbot_workspace_local_directory_write'
|
||||
_DIRECTORY_SETTING = 'langbot.directory_instance_uuid'
|
||||
_TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
_PROJECTED_TENANT_TABLES = (
|
||||
'workspaces',
|
||||
'workspace_memberships',
|
||||
'workspace_execution_states',
|
||||
)
|
||||
|
||||
|
||||
def _setting(name: str) -> str:
|
||||
return f"NULLIF(current_setting('{name}', true), '')"
|
||||
|
||||
|
||||
def _quote(conn: sa.Connection, identifier: str) -> str:
|
||||
return conn.dialect.identifier_preparer.quote(identifier)
|
||||
|
||||
|
||||
def _create_tables(conn: sa.Connection) -> None:
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
if _STATE_TABLE not in existing_tables:
|
||||
op.create_table(
|
||||
_STATE_TABLE,
|
||||
sa.Column('instance_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('cursor', sa.BigInteger(), server_default='0', nullable=False),
|
||||
sa.Column('snapshot_coverage_cursor', sa.BigInteger(), server_default='0', nullable=False),
|
||||
sa.Column('snapshot_fingerprint', sa.Text(), nullable=False),
|
||||
sa.Column('last_applied_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('lease_expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint(
|
||||
'cursor >= 0',
|
||||
name='ck_directory_projection_state_cursor',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
'snapshot_coverage_cursor >= 0 AND snapshot_coverage_cursor <= cursor',
|
||||
name='ck_directory_projection_state_snapshot_coverage',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
'length(snapshot_fingerprint) = 64',
|
||||
name='ck_directory_projection_state_fingerprint',
|
||||
),
|
||||
sa.PrimaryKeyConstraint('instance_uuid'),
|
||||
)
|
||||
if _INBOX_TABLE not in existing_tables:
|
||||
op.create_table(
|
||||
_INBOX_TABLE,
|
||||
sa.Column('instance_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('event_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('cursor', sa.BigInteger(), nullable=False),
|
||||
sa.Column('event_type', sa.String(128), nullable=False),
|
||||
sa.Column('revision', sa.BigInteger(), nullable=False),
|
||||
sa.Column('fingerprint', sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
'received_at',
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column('applied_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.CheckConstraint(
|
||||
'cursor > 0',
|
||||
name='ck_directory_projection_inbox_cursor',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
'revision > 0',
|
||||
name='ck_directory_projection_inbox_revision',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
'length(fingerprint) = 64',
|
||||
name='ck_directory_projection_inbox_fingerprint',
|
||||
),
|
||||
sa.PrimaryKeyConstraint('instance_uuid', 'event_uuid'),
|
||||
sa.UniqueConstraint(
|
||||
'instance_uuid',
|
||||
'cursor',
|
||||
name='uq_directory_projection_inbox_cursor',
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'ix_directory_projection_inbox_pending',
|
||||
_INBOX_TABLE,
|
||||
['instance_uuid', 'applied_at', 'cursor'],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def _drop_policy(conn: sa.Connection, table_name: str, policy_name: str) -> None:
|
||||
table = _quote(conn, table_name)
|
||||
policy = _quote(conn, policy_name)
|
||||
op.execute(sa.text(f'DROP POLICY IF EXISTS {policy} ON {table}'))
|
||||
|
||||
|
||||
def _create_policy(
|
||||
conn: sa.Connection,
|
||||
table_name: str,
|
||||
policy_name: str,
|
||||
expression: str,
|
||||
*,
|
||||
command: str = 'ALL',
|
||||
) -> None:
|
||||
table = _quote(conn, table_name)
|
||||
policy = _quote(conn, policy_name)
|
||||
_drop_policy(conn, table_name, policy_name)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
|
||||
if command == 'SELECT':
|
||||
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR SELECT TO PUBLIC USING ({expression})'
|
||||
elif command == 'ALL':
|
||||
sql = (
|
||||
f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC '
|
||||
f'USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
else: # pragma: no cover - migration-local invariant.
|
||||
raise AssertionError(f'Unsupported RLS policy command: {command}')
|
||||
op.execute(sa.text(sql))
|
||||
|
||||
|
||||
def _install_postgres_policies(conn: sa.Connection) -> None:
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
required_tables = set(_PROJECTED_TENANT_TABLES) | {_STATE_TABLE, _INBOX_TABLE}
|
||||
missing_tables = required_tables - existing_tables
|
||||
if missing_tables:
|
||||
raise RuntimeError(
|
||||
f'Cannot enable Cloud directory projection RLS before all required tables exist: {sorted(missing_tables)!r}'
|
||||
)
|
||||
|
||||
directory_setting = _setting(_DIRECTORY_SETTING)
|
||||
tenant_setting = _setting(_TENANT_SETTING)
|
||||
directory_expressions = {
|
||||
'workspaces': (f"instance_uuid::text = {directory_setting} AND source = 'cloud_projection'"),
|
||||
'workspace_memberships': (
|
||||
'EXISTS ('
|
||||
'SELECT 1 FROM workspaces AS directory_workspace '
|
||||
'WHERE directory_workspace.uuid = workspace_memberships.workspace_uuid '
|
||||
f'AND directory_workspace.instance_uuid::text = {directory_setting} '
|
||||
"AND directory_workspace.source = 'cloud_projection'"
|
||||
')'
|
||||
),
|
||||
'workspace_execution_states': (
|
||||
f"instance_uuid::text = {directory_setting} AND source = 'cloud' AND EXISTS ("
|
||||
'SELECT 1 FROM workspaces AS directory_workspace '
|
||||
'WHERE directory_workspace.uuid = workspace_execution_states.workspace_uuid '
|
||||
f'AND directory_workspace.instance_uuid::text = {directory_setting} '
|
||||
"AND directory_workspace.source = 'cloud_projection'"
|
||||
')'
|
||||
),
|
||||
_STATE_TABLE: f'instance_uuid::text = {directory_setting}',
|
||||
_INBOX_TABLE: f'instance_uuid::text = {directory_setting}',
|
||||
}
|
||||
tenant_expressions = {
|
||||
'workspaces': f'uuid::text = {tenant_setting}',
|
||||
'workspace_memberships': f'workspace_uuid::text = {tenant_setting}',
|
||||
'workspace_execution_states': f'workspace_uuid::text = {tenant_setting}',
|
||||
}
|
||||
local_write_expressions = {
|
||||
'workspaces': f"uuid::text = {tenant_setting} AND source = 'local'",
|
||||
'workspace_memberships': (
|
||||
f'workspace_uuid::text = {tenant_setting} AND EXISTS ('
|
||||
'SELECT 1 FROM workspaces AS local_workspace '
|
||||
'WHERE local_workspace.uuid = workspace_memberships.workspace_uuid '
|
||||
"AND local_workspace.source = 'local'"
|
||||
')'
|
||||
),
|
||||
'workspace_execution_states': (
|
||||
f'workspace_uuid::text = {tenant_setting} AND EXISTS ('
|
||||
'SELECT 1 FROM workspaces AS local_workspace '
|
||||
'WHERE local_workspace.uuid = workspace_execution_states.workspace_uuid '
|
||||
"AND local_workspace.source = 'local'"
|
||||
')'
|
||||
),
|
||||
}
|
||||
for table_name in _PROJECTED_TENANT_TABLES:
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_TENANT_POLICY_NAME,
|
||||
tenant_expressions[table_name],
|
||||
command='SELECT',
|
||||
)
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_LOCAL_WRITE_POLICY_NAME,
|
||||
local_write_expressions[table_name],
|
||||
)
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_DIRECTORY_POLICY_NAME,
|
||||
directory_expressions[table_name],
|
||||
)
|
||||
for table_name in (_STATE_TABLE, _INBOX_TABLE):
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_DIRECTORY_POLICY_NAME,
|
||||
directory_expressions[table_name],
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
_create_tables(conn)
|
||||
if conn.dialect.name == 'postgresql':
|
||||
_install_postgres_policies(conn)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
if conn.dialect.name == 'postgresql':
|
||||
tenant_setting = _setting(_TENANT_SETTING)
|
||||
tenant_columns = {
|
||||
'workspaces': 'uuid',
|
||||
'workspace_memberships': 'workspace_uuid',
|
||||
'workspace_execution_states': 'workspace_uuid',
|
||||
}
|
||||
for table_name in _PROJECTED_TENANT_TABLES:
|
||||
if table_name not in existing_tables:
|
||||
continue
|
||||
_drop_policy(conn, table_name, _DIRECTORY_POLICY_NAME)
|
||||
_drop_policy(conn, table_name, _LOCAL_WRITE_POLICY_NAME)
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_TENANT_POLICY_NAME,
|
||||
f'{tenant_columns[table_name]}::text = {tenant_setting}',
|
||||
)
|
||||
for table_name in (_STATE_TABLE, _INBOX_TABLE):
|
||||
if table_name not in existing_tables:
|
||||
continue
|
||||
_drop_policy(conn, table_name, _DIRECTORY_POLICY_NAME)
|
||||
table = _quote(conn, table_name)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
|
||||
|
||||
if _INBOX_TABLE in existing_tables:
|
||||
op.drop_table(_INBOX_TABLE)
|
||||
if _STATE_TABLE in existing_tables:
|
||||
op.drop_table(_STATE_TABLE)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""allow Core-owned collaboration writes on Cloud Workspaces
|
||||
|
||||
Revision ID: 0015_cloud_core_collab
|
||||
Revises: 0014_cloud_directory
|
||||
Create Date: 2026-07-26
|
||||
|
||||
Cloud-projected Workspace identity remains projected by the directory
|
||||
boundary, but membership role/remove and invitation acceptance are now owned
|
||||
by Core tenant scope.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = '0015_cloud_core_collab'
|
||||
down_revision = '0014_cloud_directory'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_TABLE_NAME = 'workspace_memberships'
|
||||
_POLICY_NAME = 'langbot_workspace_local_directory_write'
|
||||
_TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
|
||||
|
||||
def _setting(name: str) -> str:
|
||||
return f"NULLIF(current_setting('{name}', true), '')"
|
||||
|
||||
|
||||
def _quote(conn: sa.Connection, identifier: str) -> str:
|
||||
return conn.dialect.identifier_preparer.quote(identifier)
|
||||
|
||||
|
||||
def _drop_policy(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _TABLE_NAME)
|
||||
policy = _quote(conn, _POLICY_NAME)
|
||||
op.execute(sa.text(f'DROP POLICY IF EXISTS {policy} ON {table}'))
|
||||
|
||||
|
||||
def _create_policy(conn: sa.Connection, expression: str) -> None:
|
||||
table = _quote(conn, _TABLE_NAME)
|
||||
policy = _quote(conn, _POLICY_NAME)
|
||||
op.execute(
|
||||
sa.text(
|
||||
f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC '
|
||||
f'USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
expression = f'workspace_uuid::text = {_setting(_TENANT_SETTING)}'
|
||||
_drop_policy(conn)
|
||||
_create_policy(conn, expression)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
expression = (
|
||||
f'workspace_uuid::text = {_setting(_TENANT_SETTING)} AND EXISTS ('
|
||||
'SELECT 1 FROM workspaces AS local_workspace '
|
||||
'WHERE local_workspace.uuid = workspace_memberships.workspace_uuid '
|
||||
"AND local_workspace.source = 'local'"
|
||||
')'
|
||||
)
|
||||
_drop_policy(conn)
|
||||
_create_policy(conn, expression)
|
||||
@@ -18,6 +18,7 @@ from typing import TYPE_CHECKING
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
@@ -47,12 +48,28 @@ def _do_stamp(connection: Connection, revision: str = 'head') -> None:
|
||||
command.stamp(cfg, revision)
|
||||
|
||||
|
||||
def _do_downgrade(connection: Connection, revision: str) -> None:
|
||||
"""Synchronous downgrade — runs inside run_sync."""
|
||||
cfg = _build_config(connection)
|
||||
command.downgrade(cfg, revision)
|
||||
|
||||
|
||||
def _do_get_current(connection: Connection) -> str | None:
|
||||
"""Get current alembic revision synchronously."""
|
||||
ctx = MigrationContext.configure(connection)
|
||||
return ctx.get_current_revision()
|
||||
|
||||
|
||||
def get_alembic_head() -> str:
|
||||
"""Resolve the single release head without opening a database connection."""
|
||||
cfg = Config()
|
||||
cfg.set_main_option('script_location', _ALEMBIC_DIR)
|
||||
head = ScriptDirectory.from_config(cfg).get_current_head()
|
||||
if head is None:
|
||||
raise RuntimeError('Alembic has no migration head')
|
||||
return head
|
||||
|
||||
|
||||
def _do_autogenerate(connection: Connection, message: str = 'auto migration') -> None:
|
||||
"""Synchronous autogenerate — runs inside run_sync."""
|
||||
cfg = _build_config(connection)
|
||||
@@ -73,6 +90,13 @@ async def run_alembic_stamp(async_engine: AsyncEngine, revision: str = 'head') -
|
||||
await conn.commit()
|
||||
|
||||
|
||||
async def run_alembic_downgrade(async_engine: AsyncEngine, revision: str) -> None:
|
||||
"""Run Alembic downgrade to the given revision."""
|
||||
async with async_engine.connect() as conn:
|
||||
await conn.run_sync(_do_downgrade, revision)
|
||||
await conn.commit()
|
||||
|
||||
|
||||
async def get_alembic_current(async_engine: AsyncEngine) -> str | None:
|
||||
"""Get current alembic revision, or None if not stamped."""
|
||||
async with async_engine.connect() as conn:
|
||||
@@ -121,6 +145,7 @@ if __name__ == '__main__':
|
||||
print('Commands:')
|
||||
print(' autogenerate "message" — Generate migration from ORM model diff')
|
||||
print(' upgrade [revision] — Upgrade database (default: head)')
|
||||
print(' downgrade <revision> — Downgrade database to a revision')
|
||||
print(' stamp [revision] — Stamp revision without running (default: head)')
|
||||
print(' current — Show current revision')
|
||||
sys.exit(1)
|
||||
@@ -140,6 +165,13 @@ if __name__ == '__main__':
|
||||
rev = sys.argv[2] if len(sys.argv) > 2 else 'head'
|
||||
asyncio.run(run_alembic_stamp(engine, rev))
|
||||
print(f'Stamped: {rev}')
|
||||
elif cmd == 'downgrade':
|
||||
if len(sys.argv) < 3:
|
||||
print('Usage: python -m langbot.pkg.persistence.alembic_runner downgrade <revision>')
|
||||
sys.exit(1)
|
||||
rev = sys.argv[2]
|
||||
asyncio.run(run_alembic_downgrade(engine, rev))
|
||||
print(f'Downgraded to: {rev}')
|
||||
elif cmd == 'current':
|
||||
rev = asyncio.run(get_alembic_current(engine))
|
||||
print(f'Current revision: {rev}')
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import abc
|
||||
|
||||
import sqlalchemy
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from ..core import app
|
||||
@@ -30,8 +31,15 @@ class BaseDatabaseManager(abc.ABC):
|
||||
|
||||
engine: sqlalchemy_asyncio.AsyncEngine
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
ap: app.Application,
|
||||
*,
|
||||
url_override: sqlalchemy.engine.URL | None = None,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self.url_override = url_override
|
||||
self.persistence_mode: str | None = None
|
||||
|
||||
@abc.abstractmethod
|
||||
async def initialize(self) -> None:
|
||||
|
||||
@@ -1,21 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from .. import database
|
||||
from ..postgresql_url import normalize_asyncpg_url
|
||||
|
||||
|
||||
MAX_POOL_CONNECTIONS = 100
|
||||
MAX_POOL_TIMEOUT_SECONDS = 300
|
||||
MAX_POOL_RECYCLE_SECONDS = 86_400
|
||||
MAX_STATEMENT_TIMEOUT_MS = 300_000
|
||||
MAX_LOCK_TIMEOUT_MS = 60_000
|
||||
MAX_IDLE_TRANSACTION_TIMEOUT_MS = 300_000
|
||||
|
||||
|
||||
@database.manager_class('postgresql')
|
||||
class PostgreSQLDatabaseManager(database.BaseDatabaseManager):
|
||||
"""PostgreSQL database manager"""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
|
||||
@staticmethod
|
||||
def _pool_integer(
|
||||
config: dict,
|
||||
name: str,
|
||||
default: int,
|
||||
*,
|
||||
minimum: int,
|
||||
maximum: int,
|
||||
) -> int:
|
||||
value = config.get(name, default)
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
|
||||
comparator = 'non-negative' if minimum == 0 else 'positive'
|
||||
raise ValueError(f'database.postgresql.{name} must be a {comparator} integer no greater than {maximum}')
|
||||
return value
|
||||
|
||||
host = postgresql_config.get('host', '127.0.0.1')
|
||||
port = postgresql_config.get('port', 5432)
|
||||
user = postgresql_config.get('user', 'postgres')
|
||||
password = postgresql_config.get('password', 'postgres')
|
||||
database = postgresql_config.get('database', 'postgres')
|
||||
engine_url = f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{database}'
|
||||
self.engine = sqlalchemy_asyncio.create_async_engine(engine_url)
|
||||
async def initialize(self) -> None:
|
||||
self._pool_timeouts_total = 0
|
||||
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
|
||||
if not isinstance(postgresql_config, dict):
|
||||
raise ValueError('database.postgresql must be an object')
|
||||
if self.url_override is not None:
|
||||
engine_url = self.url_override
|
||||
else:
|
||||
explicit_url = postgresql_config.get('url')
|
||||
if explicit_url:
|
||||
if not isinstance(explicit_url, str):
|
||||
raise ValueError('database.postgresql.url must be a string')
|
||||
try:
|
||||
engine_url = sqlalchemy.engine.make_url(explicit_url)
|
||||
except Exception:
|
||||
raise ValueError('database.postgresql.url is invalid') from None
|
||||
try:
|
||||
engine_url = normalize_asyncpg_url(engine_url)
|
||||
except ValueError:
|
||||
raise ValueError('database.postgresql.url must use valid PostgreSQL asyncpg options') from None
|
||||
else:
|
||||
engine_url = sqlalchemy.URL.create(
|
||||
'postgresql+asyncpg',
|
||||
username=postgresql_config.get('user', 'postgres'),
|
||||
password=postgresql_config.get('password', 'postgres'),
|
||||
host=postgresql_config.get('host', '127.0.0.1'),
|
||||
port=postgresql_config.get('port', 5432),
|
||||
database=postgresql_config.get('database', 'postgres'),
|
||||
)
|
||||
self.pool_size = self._pool_integer(
|
||||
postgresql_config,
|
||||
'pool_size',
|
||||
10,
|
||||
minimum=1,
|
||||
maximum=MAX_POOL_CONNECTIONS,
|
||||
)
|
||||
self.max_overflow = self._pool_integer(
|
||||
postgresql_config,
|
||||
'max_overflow',
|
||||
10,
|
||||
minimum=0,
|
||||
maximum=MAX_POOL_CONNECTIONS,
|
||||
)
|
||||
if self.pool_size + self.max_overflow > MAX_POOL_CONNECTIONS:
|
||||
raise ValueError(f'database.postgresql pool_size + max_overflow must not exceed {MAX_POOL_CONNECTIONS}')
|
||||
self.pool_timeout_seconds = self._pool_integer(
|
||||
postgresql_config,
|
||||
'pool_timeout_seconds',
|
||||
30,
|
||||
minimum=1,
|
||||
maximum=MAX_POOL_TIMEOUT_SECONDS,
|
||||
)
|
||||
self.pool_recycle_seconds = self._pool_integer(
|
||||
postgresql_config,
|
||||
'pool_recycle_seconds',
|
||||
1800,
|
||||
minimum=1,
|
||||
maximum=MAX_POOL_RECYCLE_SECONDS,
|
||||
)
|
||||
connect_args = {}
|
||||
self.statement_timeout_ms = 0
|
||||
self.lock_timeout_ms = 0
|
||||
self.idle_transaction_timeout_ms = 0
|
||||
if self.persistence_mode == 'cloud_runtime':
|
||||
self.statement_timeout_ms = self._pool_integer(
|
||||
postgresql_config,
|
||||
'statement_timeout_ms',
|
||||
60_000,
|
||||
minimum=1,
|
||||
maximum=MAX_STATEMENT_TIMEOUT_MS,
|
||||
)
|
||||
self.lock_timeout_ms = self._pool_integer(
|
||||
postgresql_config,
|
||||
'lock_timeout_ms',
|
||||
5_000,
|
||||
minimum=1,
|
||||
maximum=MAX_LOCK_TIMEOUT_MS,
|
||||
)
|
||||
self.idle_transaction_timeout_ms = self._pool_integer(
|
||||
postgresql_config,
|
||||
'idle_in_transaction_session_timeout_ms',
|
||||
60_000,
|
||||
minimum=1,
|
||||
maximum=MAX_IDLE_TRANSACTION_TIMEOUT_MS,
|
||||
)
|
||||
connect_args = {
|
||||
'server_settings': {
|
||||
'statement_timeout': str(self.statement_timeout_ms),
|
||||
'lock_timeout': str(self.lock_timeout_ms),
|
||||
'idle_in_transaction_session_timeout': str(self.idle_transaction_timeout_ms),
|
||||
}
|
||||
}
|
||||
self.engine = sqlalchemy_asyncio.create_async_engine(
|
||||
engine_url,
|
||||
pool_size=self.pool_size,
|
||||
max_overflow=self.max_overflow,
|
||||
pool_timeout=self.pool_timeout_seconds,
|
||||
pool_recycle=self.pool_recycle_seconds,
|
||||
pool_pre_ping=True,
|
||||
**({'connect_args': connect_args} if connect_args else {}),
|
||||
)
|
||||
|
||||
def resource_stats(self) -> dict[str, int]:
|
||||
"""Return aggregate pool gauges without exposing connection details."""
|
||||
|
||||
pool = self.engine.pool
|
||||
|
||||
def read(name: str) -> int:
|
||||
method = getattr(pool, name, None)
|
||||
if not callable(method):
|
||||
return 0
|
||||
try:
|
||||
return int(method())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
return {
|
||||
'configured_size': self.pool_size,
|
||||
'configured_max_overflow': self.max_overflow,
|
||||
'configured_capacity': self.pool_size + self.max_overflow,
|
||||
'statement_timeout_ms': self.statement_timeout_ms,
|
||||
'lock_timeout_ms': self.lock_timeout_ms,
|
||||
'idle_in_transaction_session_timeout_ms': self.idle_transaction_timeout_ms,
|
||||
'timeouts_total': self._pool_timeouts_total,
|
||||
'checked_in': read('checkedin'),
|
||||
'checked_out': read('checkedout'),
|
||||
'overflow': max(read('overflow'), 0),
|
||||
}
|
||||
|
||||
def record_pool_timeout(self) -> None:
|
||||
self._pool_timeouts_total += 1
|
||||
|
||||
+1751
-15
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,25 @@
|
||||
"""Safe PostgreSQL URL normalization shared by runtime and migration jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
|
||||
def normalize_asyncpg_url(url: sqlalchemy.engine.URL) -> sqlalchemy.engine.URL:
|
||||
"""Select asyncpg and translate the common libpq TLS query spelling."""
|
||||
|
||||
if url.drivername == 'postgresql':
|
||||
url = url.set(drivername='postgresql+asyncpg')
|
||||
elif url.drivername != 'postgresql+asyncpg':
|
||||
raise ValueError('PostgreSQL URL must use PostgreSQL with the asyncpg driver')
|
||||
|
||||
query = dict(url.query)
|
||||
sslmode = query.pop('sslmode', None)
|
||||
if sslmode is not None:
|
||||
if 'ssl' in query and query['ssl'] != sslmode:
|
||||
raise ValueError('PostgreSQL URL cannot specify conflicting ssl and sslmode options')
|
||||
# SQLAlchemy expands URL query keys into asyncpg keyword arguments.
|
||||
# asyncpg calls this keyword ``ssl`` even though PostgreSQL DSNs
|
||||
# conventionally spell the same mode ``sslmode``.
|
||||
query['ssl'] = sslmode
|
||||
return url.set(query=query)
|
||||
@@ -0,0 +1,190 @@
|
||||
"""One-shot, operator-only Cloud PostgreSQL release migration entrypoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ..cloud.bootstrap import SUPPORTED_PGVECTOR_DIMENSIONS
|
||||
from ..core import app as core_app
|
||||
from ..core.stages.load_config import LoadConfigStage
|
||||
from ..core.stages.setup_logger import SetupLoggerStage
|
||||
from .mgr import PersistenceManager, PersistenceMode
|
||||
from .postgresql_url import normalize_asyncpg_url
|
||||
|
||||
|
||||
DEFAULT_OPERATOR_DSN_ENV = 'LANGBOT_CLOUD_MIGRATION_DSN'
|
||||
_ENV_NAME = re.compile(r'^[A-Z_][A-Z0-9_]*$')
|
||||
|
||||
|
||||
class CloudReleaseMigrationConfigurationError(RuntimeError):
|
||||
"""Raised before any database operation when migration input is unsafe."""
|
||||
|
||||
|
||||
def _url_endpoint(url: sqlalchemy.engine.URL, *, label: str) -> tuple[str, int]:
|
||||
"""Return a comparison-safe PostgreSQL endpoint without leaking its DSN."""
|
||||
|
||||
try:
|
||||
host = (url.host or '').strip().casefold()
|
||||
port = url.port or 5432
|
||||
except (TypeError, ValueError):
|
||||
raise CloudReleaseMigrationConfigurationError(f'{label} PostgreSQL host or port is invalid') from None
|
||||
if not host or isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
|
||||
raise CloudReleaseMigrationConfigurationError(f'{label} PostgreSQL host or port is invalid')
|
||||
return host, port
|
||||
|
||||
|
||||
def _operator_database_url(
|
||||
instance_config: dict,
|
||||
*,
|
||||
environ: Mapping[str, str],
|
||||
) -> sqlalchemy.engine.URL:
|
||||
database_config = instance_config.get('database')
|
||||
if not isinstance(database_config, dict) or database_config.get('use') != 'postgresql':
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration requires explicit database.use=postgresql; SQLite fallback is forbidden'
|
||||
)
|
||||
|
||||
runtime_config = database_config.get('postgresql')
|
||||
if not isinstance(runtime_config, dict):
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL configuration is missing')
|
||||
|
||||
migration_config = database_config.get('cloud_migration', {})
|
||||
if not isinstance(migration_config, dict):
|
||||
raise CloudReleaseMigrationConfigurationError('database.cloud_migration must be a mapping')
|
||||
dsn_env = migration_config.get('operator_dsn_env', DEFAULT_OPERATOR_DSN_ENV)
|
||||
if not isinstance(dsn_env, str) or not _ENV_NAME.fullmatch(dsn_env):
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'database.cloud_migration.operator_dsn_env must name an uppercase environment variable'
|
||||
)
|
||||
|
||||
raw_dsn = environ.get(dsn_env, '').strip()
|
||||
if not raw_dsn:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
f'Cloud release migration requires the operator DSN in environment variable {dsn_env}'
|
||||
)
|
||||
try:
|
||||
operator_url = sqlalchemy.engine.make_url(raw_dsn)
|
||||
except Exception:
|
||||
# Never echo a malformed DSN because it may contain an unescaped secret.
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud release migration operator DSN is invalid') from None
|
||||
|
||||
try:
|
||||
operator_url = normalize_asyncpg_url(operator_url)
|
||||
except ValueError:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must use valid PostgreSQL asyncpg options'
|
||||
) from None
|
||||
|
||||
operator_user = (operator_url.username or '').strip()
|
||||
operator_database = (operator_url.database or '').strip()
|
||||
operator_host, operator_port = _url_endpoint(operator_url, label='Cloud release migration operator')
|
||||
runtime_url_value = runtime_config.get('url')
|
||||
if runtime_url_value:
|
||||
if not isinstance(runtime_url_value, str):
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL URL must be a string')
|
||||
try:
|
||||
runtime_url = sqlalchemy.engine.make_url(runtime_url_value)
|
||||
except Exception:
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL URL is invalid') from None
|
||||
if runtime_url.drivername not in {'postgresql', 'postgresql+asyncpg'}:
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime database URL must use PostgreSQL')
|
||||
runtime_user = (runtime_url.username or '').strip()
|
||||
runtime_database = (runtime_url.database or '').strip()
|
||||
runtime_host, runtime_port = _url_endpoint(runtime_url, label='Cloud runtime')
|
||||
else:
|
||||
runtime_user = str(runtime_config.get('user', 'postgres') or '').strip()
|
||||
runtime_database = str(runtime_config.get('database', 'postgres') or '').strip()
|
||||
runtime_host = str(runtime_config.get('host', '') or '').strip().casefold()
|
||||
runtime_port = runtime_config.get('port', 5432)
|
||||
if not operator_user or not operator_database:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must include a user, host, and database'
|
||||
)
|
||||
if (
|
||||
not runtime_user
|
||||
or not runtime_database
|
||||
or not runtime_host
|
||||
or isinstance(runtime_port, bool)
|
||||
or not isinstance(runtime_port, int)
|
||||
or not 1 <= runtime_port <= 65535
|
||||
):
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud runtime PostgreSQL user, host, port, and database are required'
|
||||
)
|
||||
if operator_user == runtime_user:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration requires a distinct operator role from the runtime PostgreSQL role'
|
||||
)
|
||||
if operator_database != runtime_database:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must target the configured runtime database'
|
||||
)
|
||||
if operator_host != runtime_host or operator_port != runtime_port:
|
||||
# The first Cloud release intentionally requires the migrator and
|
||||
# runtime to use the same PostgreSQL endpoint. Supporting a direct
|
||||
# operator endpoint plus a runtime pooler requires a database-backed
|
||||
# immutable cluster identity check; accepting aliases here would turn a
|
||||
# same-named database on another cluster into a silent migration target.
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must target the configured runtime PostgreSQL endpoint'
|
||||
)
|
||||
|
||||
vdb_config = instance_config.get('vdb')
|
||||
if not isinstance(vdb_config, dict) or vdb_config.get('use') != 'pgvector':
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud release migration requires vdb.use=pgvector')
|
||||
pgvector_config = vdb_config.get('pgvector')
|
||||
if not isinstance(pgvector_config, dict) or pgvector_config.get('use_business_database') is not True:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration requires vdb.pgvector.use_business_database=true'
|
||||
)
|
||||
dimensions = pgvector_config.get('allowed_dimensions')
|
||||
if (
|
||||
not isinstance(dimensions, list)
|
||||
or not dimensions
|
||||
or any(isinstance(item, bool) or not isinstance(item, int) for item in dimensions)
|
||||
or not set(dimensions).issubset(SUPPORTED_PGVECTOR_DIMENSIONS)
|
||||
):
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration pgvector dimensions are outside the release-created index set'
|
||||
)
|
||||
|
||||
return operator_url
|
||||
|
||||
|
||||
async def run_cloud_release_migration(
|
||||
ap: core_app.Application,
|
||||
*,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Run and validate one release migration with an isolated operator DSN."""
|
||||
|
||||
operator_url = _operator_database_url(
|
||||
ap.instance_config.data,
|
||||
environ=os.environ if environ is None else environ,
|
||||
)
|
||||
manager = PersistenceManager(
|
||||
ap,
|
||||
mode=PersistenceMode.RELEASE_MIGRATION,
|
||||
database_url=operator_url,
|
||||
)
|
||||
ap.persistence_mgr = manager
|
||||
try:
|
||||
await manager.initialize()
|
||||
ap.logger.info('Cloud PostgreSQL release migration reached and validated the exact release head.')
|
||||
finally:
|
||||
await manager.shutdown()
|
||||
|
||||
|
||||
async def run_cloud_release_migration_from_config(loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""Load only process configuration/logging, then run the one-shot job."""
|
||||
|
||||
ap = core_app.Application()
|
||||
ap.event_loop = loop
|
||||
await LoadConfigStage().run(ap)
|
||||
await SetupLoggerStage().run(ap)
|
||||
await run_cloud_release_migration(ap)
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Durable SQLite backups for destructive Alembic migration boundaries."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import secrets
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import typing
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
|
||||
class SQLiteMigrationBackupError(RuntimeError):
|
||||
"""A verified migration backup could not be created or restored."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class SQLiteMigrationBackup:
|
||||
database_path: pathlib.Path
|
||||
backup_path: pathlib.Path
|
||||
manifest_path: pathlib.Path
|
||||
source_revision: str
|
||||
target_revision: str
|
||||
created_at: str
|
||||
|
||||
|
||||
def _safe_label(value: str) -> str:
|
||||
label = re.sub(r'[^A-Za-z0-9_.-]+', '-', value).strip('-')
|
||||
return label or 'unknown'
|
||||
|
||||
|
||||
def _database_path(engine: AsyncEngine) -> pathlib.Path:
|
||||
if engine.dialect.name != 'sqlite':
|
||||
raise SQLiteMigrationBackupError('SQLite migration backups require a SQLite engine')
|
||||
database = engine.url.database
|
||||
if not database or database == ':memory:' or engine.url.query.get('mode') == 'memory':
|
||||
raise SQLiteMigrationBackupError('Tenant schema migrations require a file-backed SQLite database for recovery')
|
||||
database_path = pathlib.Path(database).expanduser()
|
||||
if not database_path.is_absolute():
|
||||
database_path = pathlib.Path.cwd() / database_path
|
||||
database_path = database_path.resolve()
|
||||
if not database_path.is_file():
|
||||
raise SQLiteMigrationBackupError(f'SQLite database does not exist: {database_path}')
|
||||
return database_path
|
||||
|
||||
|
||||
def _open_read_only(path: pathlib.Path) -> sqlite3.Connection:
|
||||
return sqlite3.connect(f'{path.as_uri()}?mode=ro', uri=True, timeout=30)
|
||||
|
||||
|
||||
def _read_revision(connection: sqlite3.Connection) -> str | None:
|
||||
has_version_table = connection.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'alembic_version'"
|
||||
).fetchone()
|
||||
if has_version_table is None:
|
||||
return None
|
||||
rows = connection.execute('SELECT version_num FROM alembic_version').fetchall()
|
||||
if not rows:
|
||||
return None
|
||||
if len(rows) != 1 or not isinstance(rows[0][0], str):
|
||||
raise SQLiteMigrationBackupError('SQLite backup has an invalid Alembic revision table')
|
||||
return rows[0][0]
|
||||
|
||||
|
||||
def _verify_connection(connection: sqlite3.Connection, expected_revision: str) -> None:
|
||||
quick_check = connection.execute('PRAGMA quick_check').fetchall()
|
||||
if quick_check != [('ok',)]:
|
||||
raise SQLiteMigrationBackupError(f'SQLite quick_check failed: {quick_check[:5]!r}')
|
||||
actual_revision = _read_revision(connection)
|
||||
if actual_revision != expected_revision:
|
||||
raise SQLiteMigrationBackupError(
|
||||
f'SQLite backup revision mismatch: {actual_revision!r} != {expected_revision!r}'
|
||||
)
|
||||
|
||||
|
||||
def _verify_file(path: pathlib.Path, expected_revision: str) -> None:
|
||||
with _open_read_only(path) as connection:
|
||||
_verify_connection(connection, expected_revision)
|
||||
|
||||
|
||||
def _write_manifest(backup: SQLiteMigrationBackup, status: str, **extra: typing.Any) -> None:
|
||||
payload: dict[str, typing.Any] = {
|
||||
'version': 1,
|
||||
'status': status,
|
||||
'created_at': backup.created_at,
|
||||
'database_path': str(backup.database_path),
|
||||
'backup_path': str(backup.backup_path),
|
||||
'source_revision': backup.source_revision,
|
||||
'target_revision': backup.target_revision,
|
||||
'quick_check': 'ok',
|
||||
**extra,
|
||||
}
|
||||
backup.manifest_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f'.{backup.manifest_path.name}.',
|
||||
suffix='.tmp',
|
||||
dir=backup.manifest_path.parent,
|
||||
)
|
||||
temporary_path = pathlib.Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, 'w', encoding='utf-8') as file:
|
||||
json.dump(payload, file, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
file.write('\n')
|
||||
file.flush()
|
||||
os.fsync(file.fileno())
|
||||
os.chmod(temporary_path, 0o600)
|
||||
os.replace(temporary_path, backup.manifest_path)
|
||||
_fsync_directory(backup.manifest_path.parent)
|
||||
finally:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _fsync_file(path: pathlib.Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _fsync_directory(path: pathlib.Path) -> None:
|
||||
descriptor = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _create_backup(
|
||||
database_path: pathlib.Path,
|
||||
source_revision: str,
|
||||
target_revision: str,
|
||||
) -> SQLiteMigrationBackup:
|
||||
backup_directory = database_path.parent / 'migration-backups'
|
||||
backup_directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
os.chmod(backup_directory, 0o700)
|
||||
created_at = datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%dT%H-%M-%S.%fZ')
|
||||
stem = (
|
||||
f'{database_path.stem}-pre-{_safe_label(target_revision)}-'
|
||||
f'from-{_safe_label(source_revision)}-{created_at}-{secrets.token_hex(4)}'
|
||||
)
|
||||
backup_path = backup_directory / f'{stem}.sqlite3'
|
||||
manifest_path = backup_directory / f'{stem}.json'
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f'.{stem}.',
|
||||
suffix='.creating',
|
||||
dir=backup_directory,
|
||||
)
|
||||
os.close(descriptor)
|
||||
temporary_path = pathlib.Path(temporary_name)
|
||||
try:
|
||||
with (
|
||||
_open_read_only(database_path) as source,
|
||||
sqlite3.connect(
|
||||
temporary_path,
|
||||
timeout=30,
|
||||
) as destination,
|
||||
):
|
||||
source.execute('PRAGMA busy_timeout = 30000')
|
||||
source.backup(destination)
|
||||
destination.commit()
|
||||
_verify_connection(destination, source_revision)
|
||||
os.chmod(temporary_path, 0o600)
|
||||
_fsync_file(temporary_path)
|
||||
os.replace(temporary_path, backup_path)
|
||||
_fsync_file(backup_path)
|
||||
_fsync_directory(backup_directory)
|
||||
backup = SQLiteMigrationBackup(
|
||||
database_path=database_path,
|
||||
backup_path=backup_path,
|
||||
manifest_path=manifest_path,
|
||||
source_revision=source_revision,
|
||||
target_revision=target_revision,
|
||||
created_at=created_at,
|
||||
)
|
||||
_write_manifest(backup, 'verified')
|
||||
return backup
|
||||
except Exception:
|
||||
backup_path.unlink(missing_ok=True)
|
||||
manifest_path.unlink(missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
async def create_verified_backup(
|
||||
engine: AsyncEngine,
|
||||
*,
|
||||
source_revision: str,
|
||||
target_revision: str,
|
||||
) -> SQLiteMigrationBackup:
|
||||
"""Create and verify an online-consistent backup next to instance data."""
|
||||
|
||||
database_path = _database_path(engine)
|
||||
return await asyncio.to_thread(
|
||||
_create_backup,
|
||||
database_path,
|
||||
source_revision,
|
||||
target_revision,
|
||||
)
|
||||
|
||||
|
||||
def _restore_backup(backup: SQLiteMigrationBackup) -> None:
|
||||
_verify_file(backup.backup_path, backup.source_revision)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f'.{backup.database_path.name}.',
|
||||
suffix='.restoring',
|
||||
dir=backup.database_path.parent,
|
||||
)
|
||||
os.close(descriptor)
|
||||
temporary_path = pathlib.Path(temporary_name)
|
||||
try:
|
||||
with (
|
||||
_open_read_only(backup.backup_path) as source,
|
||||
sqlite3.connect(
|
||||
temporary_path,
|
||||
timeout=30,
|
||||
) as destination,
|
||||
):
|
||||
source.backup(destination)
|
||||
destination.commit()
|
||||
_verify_connection(destination, backup.source_revision)
|
||||
os.chmod(temporary_path, 0o600)
|
||||
_fsync_file(temporary_path)
|
||||
|
||||
# A stale WAL could replay pages from the failed migration after the
|
||||
# main database file is replaced. The engine is disposed before this
|
||||
# function runs, so these exact sidecars are safe to remove.
|
||||
for suffix in ('-wal', '-shm', '-journal'):
|
||||
pathlib.Path(f'{backup.database_path}{suffix}').unlink(missing_ok=True)
|
||||
os.replace(temporary_path, backup.database_path)
|
||||
_fsync_file(backup.database_path)
|
||||
_fsync_directory(backup.database_path.parent)
|
||||
_verify_file(backup.database_path, backup.source_revision)
|
||||
finally:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
async def restore_verified_backup(engine: AsyncEngine, backup: SQLiteMigrationBackup) -> None:
|
||||
"""Atomically restore a verified backup after a migration failure."""
|
||||
|
||||
await engine.dispose()
|
||||
await asyncio.to_thread(_restore_backup, backup)
|
||||
await asyncio.to_thread(
|
||||
_write_manifest,
|
||||
backup,
|
||||
'restored_after_failure',
|
||||
restored_at=datetime.datetime.now(datetime.UTC).isoformat(),
|
||||
)
|
||||
|
||||
|
||||
async def mark_migration_succeeded(
|
||||
backup: SQLiteMigrationBackup,
|
||||
*,
|
||||
completed_revision: str,
|
||||
) -> None:
|
||||
"""Mark a retained verified backup after its migration boundary succeeds."""
|
||||
|
||||
await asyncio.to_thread(
|
||||
_write_manifest,
|
||||
backup,
|
||||
'migration_succeeded',
|
||||
completed_at=datetime.datetime.now(datetime.UTC).isoformat(),
|
||||
completed_revision=completed_revision,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user