mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 04:40:57 +00:00
feat(tenancy): add Workspace multi-tenant foundation (#2353)
* Document multi-tenant workspace architecture * Add OSS and commercial workspace boundaries * docs: redesign multi-tenant workspace architecture * feat(tenancy): implement workspace isolation * docs(tenancy): record verification evidence * docs(tenancy): revise single-instance SaaS topology * docs(tenancy): refine architecture options * docs: finalize cloud v2 multi-tenant decisions * feat(tenancy): establish cloud isolation foundations * feat(tenancy): harden shared cloud runtime boundaries * docs(tenancy): record final isolation verification * fix(tenancy): close isolation and permission gaps * docs(tenancy): record final isolation verification * feat(tenancy): connect cloud workspace control plane * fix(build): install git for pinned SDK * docs(cloud): update control plane verification * chore: update multi-tenant SDK pin * fix(cloud): skip legacy model sync during startup * test(cloud): preserve minimal model manager fixtures * fix(cloud): preserve authenticated account context * fix(cloud): reuse authenticated account for user info * feat(cloud): complete Workspace settings navigation * test(web): cover Workspace dropdown menu * feat(web): place workspace controls in sidebar * refactor(web): streamline workspace controls * style(web): format workspace layout test * fix(cloud): surface runtime and workspace plan status * fix(plugin): keep runtime identity stable across restarts * fix(ui): widen and center workspace switcher * fix(ui): hide roles from workspace switcher * fix(ui): align workspace switcher with sidebar entries * feat(workspace): add in-product collaboration and direct Cloud launch * style: format collaboration changes * fix(workspace): bind collaboration APIs to tenant UoW * fix(cloud): preserve Core-owned collaboration state * test(cloud): require Space identity for invite registration * feat(cloud): complete secure invitation experience * style(web): format invitation flows * fix(cloud): recover box runtime without unscoped skill reload * feat(oss): enforce invitation account and owner billing flows * style: format OSS account service * test(oss): cover invitation logout handoff * fix(oss): resolve workspace owner in scoped session * feat(cloud): harden multi-tenant runtime resources * fix(cloud): bound runtime restart storms * fix(cloud): eliminate periodic runtime CPU spikes * fix(cloud): enforce instance capacity ceilings * fix(cloud): scope public login capability discovery * fix(cloud): bound tenant maintenance and monitoring work * fix(runtime): bound tenant resource amplification * fix(deps): pin green multi-tenant plugin SDK * fix(cloud): handle unavailable skill capability * fix(security): require authentication for image file endpoint (H-2) - Changed /api/v1/files/image from AuthType.NONE to USER_TOKEN_OR_API_KEY - Added Permission.RESOURCE_VIEW requirement - Prevents unauthenticated cross-tenant file access via leaked keys - Fixes HIGH severity finding from multi-tenant security review docs: add comprehensive database migration guide - Complete migration steps for OSS → multi-tenant - Backup, execution, verification procedures - Rollback scenarios and recovery plans - Performance tuning recommendations * test: add comprehensive cross-tenant isolation tests Added 7 critical test scenarios for multi-tenant boundaries: - Cross-tenant bot access prevention - Viewer role read-only enforcement - Removed member immediate access revocation - Model provider credential isolation - WebSocket message isolation - Invitation token workspace scoping - Multi-workspace context validation These tests address P0-2 coverage gaps for: - workspaces.py (membership & invitation flows) - user.py (authentication & authorization) - websocket_chat.py (real-time isolation) - plugins.py (resource access control) docs: finalize database migration guide * fix(security): resolve M-1, M-2, M-3 security findings M-1: WebSocket authorization TOCTOU race (FIXED) - Changed _revalidate_websocket_authorization to return RequestContext - Ensures validated context is used immediately without race window - Prevents removed members from sending messages during revalidation gap M-2: Model Manager cache workspace isolation (VERIFIED) - Confirmed _CacheKey already uses 4-tuple: (instance, workspace, generation, resource) - Cache is properly scoped per workspace, no cross-tenant leakage possible - No code change needed, documented as working correctly M-3: Invitation lock workspace scoping (FIXED) - Changed lock key from token_digest to workspace_uuid:token_digest - Prevents DoS where attacker locks token in Workspace A to block Workspace B - Locks now isolated per workspace All MEDIUM severity findings from security review now resolved. * fix(cloud): unblock tenant CI and enforce knowledge quotas * fix(tenancy): scope rerank model sync --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import types
|
||||
import typing
|
||||
|
||||
from .context import RequestContext
|
||||
|
||||
|
||||
class WorkspaceRole(enum.StrEnum):
|
||||
OWNER = 'owner'
|
||||
ADMIN = 'admin'
|
||||
DEVELOPER = 'developer'
|
||||
OPERATOR = 'operator'
|
||||
VIEWER = 'viewer'
|
||||
|
||||
|
||||
class Permission(enum.StrEnum):
|
||||
WORKSPACE_VIEW = 'workspace.view'
|
||||
WORKSPACE_UPDATE = 'workspace.update'
|
||||
WORKSPACE_DELETE = 'workspace.delete'
|
||||
OWNER_TRANSFER = 'owner.transfer'
|
||||
MEMBER_VIEW = 'member.view'
|
||||
MEMBER_INVITE = 'member.invite'
|
||||
MEMBER_UPDATE_ROLE = 'member.update_role'
|
||||
MEMBER_REMOVE = 'member.remove'
|
||||
RESOURCE_VIEW = 'resource.view'
|
||||
RESOURCE_MANAGE = 'resource.manage'
|
||||
RUNTIME_OPERATE = 'runtime.operate'
|
||||
PROVIDER_SECRET_MANAGE = 'provider_secret.manage'
|
||||
API_KEY_MANAGE = 'api_key.manage'
|
||||
AUDIT_VIEW = 'audit.view'
|
||||
DATA_EXPORT = 'data.export'
|
||||
BILLING_LINK_MANAGE = 'billing_link.manage'
|
||||
|
||||
|
||||
_VIEW_PERMISSIONS = {
|
||||
Permission.WORKSPACE_VIEW,
|
||||
Permission.MEMBER_VIEW,
|
||||
Permission.RESOURCE_VIEW,
|
||||
}
|
||||
|
||||
_ROLE_PERMISSIONS: typing.Final = types.MappingProxyType(
|
||||
{
|
||||
WorkspaceRole.OWNER: frozenset(Permission),
|
||||
WorkspaceRole.ADMIN: frozenset(
|
||||
permission
|
||||
for permission in Permission
|
||||
if permission
|
||||
not in {
|
||||
Permission.WORKSPACE_DELETE,
|
||||
Permission.OWNER_TRANSFER,
|
||||
Permission.BILLING_LINK_MANAGE,
|
||||
}
|
||||
),
|
||||
WorkspaceRole.DEVELOPER: frozenset(
|
||||
_VIEW_PERMISSIONS
|
||||
| {
|
||||
Permission.RESOURCE_MANAGE,
|
||||
Permission.RUNTIME_OPERATE,
|
||||
Permission.PROVIDER_SECRET_MANAGE,
|
||||
}
|
||||
),
|
||||
WorkspaceRole.OPERATOR: frozenset(_VIEW_PERMISSIONS | {Permission.RUNTIME_OPERATE}),
|
||||
WorkspaceRole.VIEWER: frozenset(_VIEW_PERMISSIONS),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class AuthorizationError(Exception):
|
||||
"""Base class for errors that map to an HTTP authorization response."""
|
||||
|
||||
status_code = 403
|
||||
error_code = 'forbidden'
|
||||
|
||||
|
||||
class WorkspaceRequiredError(AuthorizationError):
|
||||
status_code = 400
|
||||
error_code = 'workspace_required'
|
||||
|
||||
|
||||
class PermissionDeniedError(AuthorizationError):
|
||||
error_code = 'permission_denied'
|
||||
|
||||
def __init__(self, permission: str) -> None:
|
||||
super().__init__(f'Missing Workspace permission: {permission}')
|
||||
self.permission = permission
|
||||
|
||||
|
||||
class EditionLimitError(AuthorizationError):
|
||||
error_code = 'edition_limit'
|
||||
|
||||
|
||||
def permissions_for_role(role: str | WorkspaceRole) -> frozenset[str]:
|
||||
"""Return the canonical fixed permissions for a Workspace role."""
|
||||
|
||||
try:
|
||||
parsed_role = WorkspaceRole(role)
|
||||
except ValueError:
|
||||
return frozenset()
|
||||
return frozenset(permission.value for permission in _ROLE_PERMISSIONS[parsed_role])
|
||||
|
||||
|
||||
def has_permission(ctx: RequestContext, permission: str | Permission) -> bool:
|
||||
"""Return whether the context contains one effective permission."""
|
||||
|
||||
permission_value = permission.value if isinstance(permission, Permission) else permission
|
||||
return permission_value in ctx.workspace.permissions
|
||||
|
||||
|
||||
def require_permission(ctx: RequestContext, permission: str | Permission) -> None:
|
||||
"""Raise a stable authorization error when a permission is missing."""
|
||||
|
||||
permission_value = permission.value if isinstance(permission, Permission) else permission
|
||||
if not has_permission(ctx, permission_value):
|
||||
raise PermissionDeniedError(permission_value)
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import enum
|
||||
|
||||
|
||||
class PrincipalType(enum.StrEnum):
|
||||
"""Kinds of authenticated principals accepted by LangBot."""
|
||||
|
||||
ACCOUNT = 'account'
|
||||
API_KEY = 'api_key'
|
||||
SYSTEM = 'system'
|
||||
PUBLIC_BOT = 'public_bot'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class PrincipalContext:
|
||||
"""Authenticated identity before Workspace authorization is applied."""
|
||||
|
||||
principal_type: PrincipalType
|
||||
account_uuid: str | None = None
|
||||
api_key_uuid: str | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class WorkspaceContext:
|
||||
"""Workspace membership and effective permissions for one request."""
|
||||
|
||||
workspace_uuid: str
|
||||
membership_uuid: str | None
|
||||
role: str | None
|
||||
permissions: frozenset[str]
|
||||
membership_revision: int = 0
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class RequestContext:
|
||||
"""Trusted authorization context passed to HTTP services."""
|
||||
|
||||
instance_uuid: str
|
||||
placement_generation: int
|
||||
request_id: str
|
||||
auth_type: str
|
||||
principal: PrincipalContext
|
||||
workspace: WorkspaceContext
|
||||
entitlement_revision: int = 0
|
||||
|
||||
@property
|
||||
def workspace_uuid(self) -> str:
|
||||
"""Return the selected Workspace UUID."""
|
||||
|
||||
return self.workspace.workspace_uuid
|
||||
|
||||
@property
|
||||
def account_uuid(self) -> str | None:
|
||||
"""Return the Account UUID when the principal is an Account."""
|
||||
|
||||
return self.principal.account_uuid
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class ExecutionContext:
|
||||
"""Workspace context propagated to asynchronous and runtime work."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
bot_uuid: str | None = None
|
||||
pipeline_uuid: str | None = None
|
||||
query_uuid: str | None = None
|
||||
trigger_principal: PrincipalContext | None = None
|
||||
entitlement_revision: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_request(
|
||||
cls,
|
||||
ctx: RequestContext,
|
||||
*,
|
||||
bot_uuid: str | None = None,
|
||||
pipeline_uuid: str | None = None,
|
||||
query_uuid: str | None = None,
|
||||
) -> ExecutionContext:
|
||||
"""Create a runtime context without losing the tenant generation."""
|
||||
|
||||
return cls(
|
||||
instance_uuid=ctx.instance_uuid,
|
||||
workspace_uuid=ctx.workspace_uuid,
|
||||
placement_generation=ctx.placement_generation,
|
||||
bot_uuid=bot_uuid,
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
query_uuid=query_uuid,
|
||||
trigger_principal=ctx.principal,
|
||||
entitlement_revision=ctx.entitlement_revision,
|
||||
)
|
||||
@@ -5,9 +5,21 @@ import typing
|
||||
import enum
|
||||
import quart
|
||||
import traceback
|
||||
import inspect
|
||||
import uuid
|
||||
from quart.typing import RouteCallable
|
||||
|
||||
from ....core import app
|
||||
from ....utils import constants
|
||||
from ....utils import bounded_executor
|
||||
from ....workspace.collaboration import MembershipPermissionError, WorkspaceCollaborationError
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ....cloud.entitlements import EntitlementUnavailableError
|
||||
from ....core.errors import TaskCapacityError
|
||||
from ..authz import AuthorizationError, Permission, permissions_for_role, require_permission
|
||||
from ..context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
# Maximum file upload size limit (10MB)
|
||||
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
||||
@@ -33,6 +45,7 @@ class AuthType(enum.Enum):
|
||||
"""Authentication type"""
|
||||
|
||||
NONE = 'none'
|
||||
ACCOUNT_TOKEN = 'account-token'
|
||||
USER_TOKEN = 'user-token'
|
||||
API_KEY = 'api-key'
|
||||
USER_TOKEN_OR_API_KEY = 'user-token-or-api-key'
|
||||
@@ -43,11 +56,11 @@ class RouterGroup(abc.ABC):
|
||||
|
||||
path: str
|
||||
|
||||
ap: app.Application
|
||||
ap: Application
|
||||
|
||||
quart_app: quart.Quart
|
||||
|
||||
def __init__(self, ap: app.Application, quart_app: quart.Quart) -> None:
|
||||
def __init__(self, ap: Application, quart_app: quart.Quart) -> None:
|
||||
self.ap = ap
|
||||
self.quart_app = quart_app
|
||||
|
||||
@@ -59,16 +72,38 @@ class RouterGroup(abc.ABC):
|
||||
self,
|
||||
rule: str,
|
||||
auth_type: AuthType = AuthType.USER_TOKEN,
|
||||
permission: Permission | str | None = None,
|
||||
**options: typing.Any,
|
||||
) -> typing.Callable[[RouteCallable], RouteCallable]: # decorator
|
||||
"""Register a route"""
|
||||
|
||||
if auth_type == AuthType.ACCOUNT_TOKEN and permission is not None:
|
||||
raise ValueError('Account-token routes cannot declare Workspace permissions')
|
||||
|
||||
def decorator(f: RouteCallable) -> RouteCallable:
|
||||
nonlocal rule
|
||||
rule = self.path + rule
|
||||
|
||||
async def handler_error(*args, **kwargs):
|
||||
if auth_type == AuthType.USER_TOKEN:
|
||||
request_context: RequestContext | None = None
|
||||
if auth_type == AuthType.ACCOUNT_TOKEN:
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if not authorization.startswith('Bearer '):
|
||||
return self.http_status(401, -1, 'No valid user token provided')
|
||||
token = authorization.removeprefix('Bearer ')
|
||||
if not token:
|
||||
return self.http_status(401, -1, 'No valid user token provided')
|
||||
|
||||
try:
|
||||
account, user_email = await self._authenticate_account(token)
|
||||
# Account-token routes deliberately stop before Workspace
|
||||
# selection. They may bootstrap a selector, but cannot
|
||||
# receive RequestContext or enforce Workspace permissions.
|
||||
self._inject_handler_context(f, kwargs, user_email, None, account)
|
||||
except Exception as e:
|
||||
return self._auth_error_response(e)
|
||||
|
||||
elif auth_type == AuthType.USER_TOKEN:
|
||||
# get token from Authorization header
|
||||
token = quart.request.headers.get('Authorization', '').replace('Bearer ', '')
|
||||
|
||||
@@ -76,18 +111,15 @@ class RouterGroup(abc.ABC):
|
||||
return self.http_status(401, -1, 'No valid user token provided')
|
||||
|
||||
try:
|
||||
user_email = await self.ap.user_service.verify_jwt_token(token)
|
||||
|
||||
# check if this account exists
|
||||
user = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if not user:
|
||||
return self.http_status(401, -1, 'User not found')
|
||||
|
||||
# check if f accepts user_email parameter
|
||||
if 'user_email' in f.__code__.co_varnames:
|
||||
kwargs['user_email'] = user_email
|
||||
account, user_email = await self._authenticate_account(token)
|
||||
request_context = await self._resolve_account_context(account, auth_type)
|
||||
if permission is not None:
|
||||
if request_context is None:
|
||||
raise AuthorizationError('Workspace authorization is unavailable')
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, user_email, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
|
||||
elif auth_type == AuthType.API_KEY:
|
||||
# get API key from Authorization header or X-API-Key header
|
||||
@@ -101,11 +133,12 @@ class RouterGroup(abc.ABC):
|
||||
return self.http_status(401, -1, 'No valid API key provided')
|
||||
|
||||
try:
|
||||
is_valid = await self.ap.apikey_service.verify_api_key(api_key)
|
||||
if not is_valid:
|
||||
return self.http_status(401, -1, 'Invalid API key')
|
||||
request_context = await self._authenticate_api_key(api_key, auth_type)
|
||||
if permission is not None:
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, None, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
|
||||
elif auth_type == AuthType.USER_TOKEN_OR_API_KEY:
|
||||
# Try API key first (check X-API-Key header)
|
||||
@@ -114,11 +147,12 @@ class RouterGroup(abc.ABC):
|
||||
if api_key:
|
||||
# API key authentication
|
||||
try:
|
||||
is_valid = await self.ap.apikey_service.verify_api_key(api_key)
|
||||
if not is_valid:
|
||||
return self.http_status(401, -1, 'Invalid API key')
|
||||
request_context = await self._authenticate_api_key(api_key, auth_type)
|
||||
if permission is not None:
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, None, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
else:
|
||||
# Try user token authentication (Authorization header)
|
||||
token = quart.request.headers.get('Authorization', '').replace('Bearer ', '')
|
||||
@@ -129,35 +163,89 @@ class RouterGroup(abc.ABC):
|
||||
)
|
||||
|
||||
try:
|
||||
user_email = await self.ap.user_service.verify_jwt_token(token)
|
||||
|
||||
# check if this account exists
|
||||
user = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if not user:
|
||||
return self.http_status(401, -1, 'User not found')
|
||||
|
||||
# check if f accepts user_email parameter
|
||||
if 'user_email' in f.__code__.co_varnames:
|
||||
kwargs['user_email'] = user_email
|
||||
account, user_email = await self._authenticate_account(token)
|
||||
request_context = await self._resolve_account_context(account, auth_type)
|
||||
if permission is not None:
|
||||
if request_context is None:
|
||||
raise AuthorizationError('Workspace authorization is unavailable')
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, user_email, request_context)
|
||||
except (AuthorizationError, WorkspaceNotFoundError, MembershipPermissionError) as e:
|
||||
# Authentication succeeded and authorization was
|
||||
# evaluated. Do not reinterpret a denied user token
|
||||
# as an API key, which would mask the stable 403/404.
|
||||
return self._auth_error_response(e)
|
||||
except Exception:
|
||||
# If user token fails, maybe it's an API key in Authorization header
|
||||
try:
|
||||
is_valid = await self.ap.apikey_service.verify_api_key(token)
|
||||
if not is_valid:
|
||||
return self.http_status(401, -1, 'Invalid authentication credentials')
|
||||
request_context = await self._authenticate_api_key(token, auth_type)
|
||||
if permission is not None:
|
||||
require_permission(request_context, permission)
|
||||
self._inject_handler_context(f, kwargs, None, request_context)
|
||||
except Exception as e:
|
||||
return self.http_status(401, -1, str(e))
|
||||
return self._auth_error_response(e)
|
||||
|
||||
try:
|
||||
if request_context is not None:
|
||||
with bounded_executor.blocking_work_scope(request_context.workspace_uuid):
|
||||
persistence_mgr = getattr(
|
||||
self.ap,
|
||||
'persistence_mgr',
|
||||
None,
|
||||
)
|
||||
tenant_scope_descriptor = getattr(
|
||||
type(persistence_mgr),
|
||||
'tenant_scope',
|
||||
None,
|
||||
)
|
||||
if callable(tenant_scope_descriptor):
|
||||
# Authorization discovery is complete. Carry
|
||||
# the trusted Workspace identity across the
|
||||
# handler, but do not reserve a database
|
||||
# connection while it waits on providers,
|
||||
# runtimes, uploads, or streamed clients.
|
||||
# Services that need atomic writes open a UoW.
|
||||
async with persistence_mgr.tenant_scope(request_context.workspace_uuid):
|
||||
return await f(*args, **kwargs)
|
||||
return await f(*args, **kwargs)
|
||||
return await f(*args, **kwargs)
|
||||
|
||||
except Exception as e: # 自动 500
|
||||
traceback.print_exc()
|
||||
# return self.http_status(500, -2, str(e))
|
||||
return self.http_status(500, -2, str(e))
|
||||
if isinstance(e, AuthorizationError):
|
||||
return self.http_status(e.status_code, e.error_code, str(e))
|
||||
if isinstance(e, WorkspaceNotFoundError):
|
||||
return self.http_status(404, 'resource_not_found', 'Resource not found')
|
||||
if isinstance(e, MembershipPermissionError):
|
||||
return self.http_status(403, e.code, str(e))
|
||||
if isinstance(e, WorkspaceCollaborationError):
|
||||
return self.http_status(400, e.code, str(e))
|
||||
if isinstance(e, TaskCapacityError):
|
||||
return self.http_status(429, 'task_capacity_exceeded', str(e))
|
||||
if isinstance(
|
||||
e,
|
||||
bounded_executor.BlockingWorkCapacityError,
|
||||
):
|
||||
return self.http_status(
|
||||
429,
|
||||
'blocking_work_capacity_exceeded',
|
||||
str(e),
|
||||
)
|
||||
request_id = self.request_id()
|
||||
logger = getattr(self.ap, 'logger', self.quart_app.logger)
|
||||
logger.error(
|
||||
f'Unhandled HTTP error request_id={request_id} '
|
||||
f'method={quart.request.method} path={quart.request.path}\n{traceback.format_exc()}'
|
||||
)
|
||||
return self.internal_error_response(request_id)
|
||||
|
||||
new_f = handler_error
|
||||
new_f.__name__ = (self.name + rule).replace('/', '__')
|
||||
# Quart/Flask requires a unique endpoint name even when the same URL
|
||||
# intentionally has separate handlers for different HTTP methods.
|
||||
# Include the method set so CRUD routes can declare distinct
|
||||
# permissions without colliding during application startup.
|
||||
methods = options.get('methods') or ['GET']
|
||||
method_suffix = '__'.join(sorted(str(method).upper() for method in methods))
|
||||
new_f.__name__ = (self.name + rule + '__' + method_suffix).replace('/', '__')
|
||||
new_f.__doc__ = f.__doc__
|
||||
|
||||
self.quart_app.route(rule, **options)(new_f)
|
||||
@@ -165,6 +253,192 @@ class RouterGroup(abc.ABC):
|
||||
|
||||
return decorator
|
||||
|
||||
async def _authenticate_account(self, token: str) -> tuple[typing.Any, str]:
|
||||
account: typing.Any = None
|
||||
resolver = getattr(self.ap.user_service, 'get_authenticated_account', None)
|
||||
if callable(resolver):
|
||||
resolved = resolver(token)
|
||||
if inspect.isawaitable(resolved):
|
||||
account = await resolved
|
||||
|
||||
if isinstance(account, str) or account is None:
|
||||
user_email = account or await self.ap.user_service.verify_jwt_token(token)
|
||||
account = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if account is None:
|
||||
raise ValueError('User not found')
|
||||
return account, account.user
|
||||
|
||||
async def _resolve_account_context(
|
||||
self,
|
||||
account: typing.Any,
|
||||
auth_type: AuthType,
|
||||
) -> RequestContext | None:
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
# Compatibility for isolated controller tests that do not wire the tenancy kernel.
|
||||
if collaboration_service is None or not isinstance(account_uuid, str):
|
||||
return None
|
||||
|
||||
requested_workspace_uuid = quart.request.headers.get('X-Workspace-Id')
|
||||
access = await collaboration_service.resolve_account_workspace(account_uuid, requested_workspace_uuid)
|
||||
entitlement_revision = await self._resolve_entitlement_revision(
|
||||
access.execution.instance_uuid,
|
||||
access.workspace.uuid,
|
||||
)
|
||||
request_context = RequestContext(
|
||||
instance_uuid=access.execution.instance_uuid,
|
||||
placement_generation=access.execution.placement_generation,
|
||||
request_id=self.request_id(),
|
||||
auth_type=auth_type.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.ACCOUNT,
|
||||
account_uuid=account_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=access.workspace.uuid,
|
||||
membership_uuid=access.membership.uuid,
|
||||
role=access.membership.role,
|
||||
permissions=permissions_for_role(access.membership.role),
|
||||
membership_revision=access.membership.projection_revision,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
quart.g.request_context = request_context
|
||||
quart.g.workspace_membership = access.membership
|
||||
return request_context
|
||||
|
||||
async def _authenticate_api_key(self, api_key: str, auth_type: AuthType) -> RequestContext:
|
||||
authenticator = getattr(self.ap.apikey_service, 'authenticate_api_key', None)
|
||||
if callable(authenticator):
|
||||
authenticated = authenticator(api_key)
|
||||
if inspect.isawaitable(authenticated):
|
||||
identity = await authenticated
|
||||
if identity is not None:
|
||||
entitlement_revision = await self._resolve_entitlement_revision(
|
||||
identity.instance_uuid,
|
||||
identity.workspace_uuid,
|
||||
)
|
||||
request_context = RequestContext(
|
||||
instance_uuid=identity.instance_uuid,
|
||||
placement_generation=identity.placement_generation,
|
||||
request_id=self.request_id(),
|
||||
auth_type=auth_type.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.API_KEY,
|
||||
api_key_uuid=identity.api_key_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=identity.workspace_uuid,
|
||||
membership_uuid=None,
|
||||
role=None,
|
||||
permissions=identity.permissions,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
quart.g.request_context = request_context
|
||||
return request_context
|
||||
|
||||
if not await self.ap.apikey_service.verify_api_key(api_key):
|
||||
raise ValueError('Invalid API key')
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
if workspace_service is None:
|
||||
raise ValueError('API key Workspace binding is unavailable')
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
request_context = RequestContext(
|
||||
instance_uuid=binding.instance_uuid or constants.instance_id,
|
||||
placement_generation=binding.placement_generation,
|
||||
request_id=self.request_id(),
|
||||
auth_type=auth_type.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.API_KEY,
|
||||
api_key_uuid='legacy-oss-api-key',
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
membership_uuid=None,
|
||||
role=None,
|
||||
permissions=frozenset(item.value for item in Permission),
|
||||
),
|
||||
)
|
||||
quart.g.request_context = request_context
|
||||
return request_context
|
||||
|
||||
async def _resolve_entitlement_revision(self, instance_uuid: str, workspace_uuid: str) -> int:
|
||||
deployment = getattr(self.ap, 'deployment', None)
|
||||
if deployment is None or not getattr(deployment, 'multi_workspace_enabled', False):
|
||||
return 0
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None:
|
||||
raise EntitlementUnavailableError('Workspace entitlement resolver is unavailable')
|
||||
if instance_uuid != resolver.instance_uuid:
|
||||
raise EntitlementUnavailableError('Workspace entitlement targets another LangBot instance')
|
||||
snapshot = await resolver.resolve(workspace_uuid)
|
||||
return snapshot.entitlement_revision
|
||||
|
||||
@staticmethod
|
||||
def _inject_handler_context(
|
||||
handler: RouteCallable,
|
||||
kwargs: dict[str, typing.Any],
|
||||
user_email: str | None,
|
||||
request_context: RequestContext | None,
|
||||
account: typing.Any = None,
|
||||
) -> None:
|
||||
parameters = inspect.signature(handler).parameters
|
||||
if user_email is not None and 'user_email' in parameters:
|
||||
kwargs['user_email'] = user_email
|
||||
if account is not None and 'account' in parameters:
|
||||
kwargs['account'] = account
|
||||
if request_context is not None:
|
||||
if 'request_context' in parameters:
|
||||
kwargs['request_context'] = request_context
|
||||
elif 'ctx' in parameters:
|
||||
kwargs['ctx'] = request_context
|
||||
|
||||
def _auth_error_response(self, error: Exception) -> typing.Any:
|
||||
if isinstance(error, AuthorizationError):
|
||||
return self.http_status(error.status_code, error.error_code, str(error))
|
||||
if isinstance(error, WorkspaceNotFoundError):
|
||||
return self.http_status(404, 'resource_not_found', 'Resource not found')
|
||||
if isinstance(error, MembershipPermissionError):
|
||||
return self.http_status(403, error.code, str(error))
|
||||
if isinstance(error, EntitlementUnavailableError):
|
||||
return self.http_status(403, 'entitlement_unavailable', str(error))
|
||||
request_id = self.request_id()
|
||||
logger = getattr(self.ap, 'logger', self.quart_app.logger)
|
||||
logger.warning(f'Authentication failed request_id={request_id} error_type={type(error).__name__}: {error}')
|
||||
return self.http_status(
|
||||
401,
|
||||
'invalid_authentication',
|
||||
'Invalid authentication credentials',
|
||||
)
|
||||
|
||||
def request_id(self) -> str:
|
||||
"""Return one stable request ID for authentication, logs, and errors."""
|
||||
|
||||
request_context = getattr(quart.g, 'request_context', None)
|
||||
request_id = getattr(request_context, 'request_id', None) or getattr(quart.g, 'request_id', None)
|
||||
if not request_id:
|
||||
candidate = str(quart.request.headers.get('X-Request-Id') or '').strip()
|
||||
if not candidate or len(candidate) > 128 or any(ord(char) < 32 for char in candidate):
|
||||
candidate = str(uuid.uuid4())
|
||||
request_id = candidate
|
||||
quart.g.request_id = request_id
|
||||
return str(request_id)
|
||||
|
||||
def internal_error_response(self, request_id: str | None = None) -> typing.Tuple[quart.Response, int]:
|
||||
"""Return a stable 500 response without exposing the underlying exception."""
|
||||
|
||||
resolved_request_id = request_id or self.request_id()
|
||||
response = quart.jsonify(
|
||||
{
|
||||
'code': 'internal_error',
|
||||
'msg': 'Internal server error',
|
||||
'request_id': resolved_request_id,
|
||||
}
|
||||
)
|
||||
response.headers['X-Request-Id'] = resolved_request_id
|
||||
return response, 500
|
||||
|
||||
def success(self, data: typing.Any = None) -> quart.Response:
|
||||
"""Return a 200 response"""
|
||||
return quart.jsonify(
|
||||
@@ -175,7 +449,7 @@ class RouterGroup(abc.ABC):
|
||||
}
|
||||
)
|
||||
|
||||
def fail(self, code: int, msg: str) -> quart.Response:
|
||||
def fail(self, code: int | str, msg: str) -> quart.Response:
|
||||
"""Return an error response"""
|
||||
|
||||
return quart.jsonify(
|
||||
@@ -185,6 +459,6 @@ class RouterGroup(abc.ABC):
|
||||
}
|
||||
)
|
||||
|
||||
def http_status(self, status: int, code: int, msg: str) -> typing.Tuple[quart.Response, int]:
|
||||
def http_status(self, status: int, code: int | str, msg: str) -> typing.Tuple[quart.Response, int]:
|
||||
"""返回一个指定状态码的响应"""
|
||||
return (self.fail(code, msg), status)
|
||||
|
||||
@@ -1,43 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('apikeys', '/api/v1/apikeys')
|
||||
class ApiKeysRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'])
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
keys = await self.ap.apikey_service.get_api_keys()
|
||||
return self.success(data={'keys': keys})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name', '')
|
||||
description = json_data.get('description', '')
|
||||
@self.route('', methods=['GET'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
keys = await self.ap.apikey_service.get_api_keys(request_context)
|
||||
return self.success(data={'keys': keys})
|
||||
|
||||
if not name:
|
||||
return self.http_status(400, -1, 'Name is required')
|
||||
@self.route('', methods=['POST'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
expires_at = json_data.get('expires_at')
|
||||
parsed_expiry = None
|
||||
if expires_at:
|
||||
try:
|
||||
parsed_expiry = datetime.datetime.fromisoformat(str(expires_at).replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
return self.http_status(400, 'invalid_expiry', 'Invalid API key expiry')
|
||||
try:
|
||||
key = await self.ap.apikey_service.create_api_key(
|
||||
request_context,
|
||||
json_data.get('name', ''),
|
||||
json_data.get('description', ''),
|
||||
scopes=json_data.get('scopes'),
|
||||
expires_at=parsed_expiry,
|
||||
)
|
||||
except ValueError as error:
|
||||
return self.http_status(400, 'invalid_api_key', str(error))
|
||||
return self.success(data={'key': key})
|
||||
|
||||
key = await self.ap.apikey_service.create_api_key(name, description)
|
||||
return self.success(data={'key': key})
|
||||
@self.route('/<int:key_id>', methods=['GET'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(key_id: int, request_context: RequestContext) -> str:
|
||||
key = await self.ap.apikey_service.get_api_key(request_context, key_id)
|
||||
if key is None:
|
||||
return self.http_status(404, 'resource_not_found', 'API key not found')
|
||||
return self.success(data={'key': key})
|
||||
|
||||
@self.route('/<int:key_id>', methods=['GET', 'PUT', 'DELETE'])
|
||||
async def _(key_id: int) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
key = await self.ap.apikey_service.get_api_key(key_id)
|
||||
if key is None:
|
||||
return self.http_status(404, -1, 'API key not found')
|
||||
return self.success(data={'key': key})
|
||||
@self.route('/<int:key_id>', methods=['PUT'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(key_id: int, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
await self.ap.apikey_service.update_api_key(
|
||||
request_context,
|
||||
key_id,
|
||||
json_data.get('name'),
|
||||
json_data.get('description'),
|
||||
)
|
||||
except ValueError as error:
|
||||
return self.http_status(400, 'invalid_api_key', str(error))
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name')
|
||||
description = json_data.get('description')
|
||||
|
||||
await self.ap.apikey_service.update_api_key(key_id, name, description)
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.apikey_service.delete_api_key(key_id)
|
||||
return self.success()
|
||||
@self.route('/<int:key_id>', methods=['DELETE'], permission=Permission.API_KEY_MANAGE)
|
||||
async def _(key_id: int, request_context: RequestContext) -> str:
|
||||
await self.ap.apikey_service.delete_api_key(request_context, key_id)
|
||||
return self.success()
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from langbot.pkg.utils import constants
|
||||
from langbot_plugin.box.errors import BoxAdmissionError
|
||||
|
||||
from langbot.pkg.cloud.entitlements import EntitlementUnavailableError
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
from .box_visibility import should_hide_box_runtime_status
|
||||
|
||||
@@ -9,18 +13,56 @@ from .box_visibility import should_hide_box_runtime_status
|
||||
@group.group_class('box', '/api/v1/box')
|
||||
class BoxRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
status = await self.ap.box_service.get_status()
|
||||
@self.route(
|
||||
'/status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
status = await self.ap.box_service.get_status(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
status['hidden'] = should_hide_box_runtime_status(constants.edition, status.get('enabled'))
|
||||
return self.success(data=status)
|
||||
|
||||
@self.route('/sessions', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
sessions = await self.ap.box_service.get_sessions()
|
||||
@self.route(
|
||||
'/runtime-status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
del request_context
|
||||
status = await self.ap.box_service.get_backend_status()
|
||||
status['hidden'] = should_hide_box_runtime_status(constants.edition, status.get('enabled'))
|
||||
return self.success(data=status)
|
||||
|
||||
@self.route(
|
||||
'/sessions',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
sessions = await self.ap.box_service.get_sessions(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
return self.success(data=sessions)
|
||||
|
||||
@self.route('/errors', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
errors = self.ap.box_service.get_recent_errors()
|
||||
@self.route(
|
||||
'/errors',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
if getattr(self.ap.box_service, 'managed_admission_required', False):
|
||||
await self.ap.box_service.require_workspace_sandbox(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
errors = self.ap.box_service.get_recent_errors(request_context)
|
||||
return self.success(data=errors)
|
||||
|
||||
@@ -3,6 +3,9 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from ...service.secrets import redact_secrets
|
||||
from .. import group
|
||||
|
||||
|
||||
@@ -11,12 +14,29 @@ class ExtensionsRouterGroup(group.RouterGroup):
|
||||
"""Unified API for installed extensions (plugins, MCP servers, skills)."""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> quart.Response:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> quart.Response:
|
||||
if self.ap.plugin_connector.is_enable_plugin:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
async def read_in_task_scope(operation):
|
||||
tenant_scope = getattr(getattr(self.ap, 'persistence_mgr', None), 'tenant_scope', None)
|
||||
if callable(tenant_scope):
|
||||
async with tenant_scope(request_context.workspace_uuid):
|
||||
return await operation()
|
||||
return await operation()
|
||||
|
||||
plugins, mcp_servers, skills = await asyncio.gather(
|
||||
self.ap.plugin_connector.list_plugins(),
|
||||
self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True),
|
||||
self.ap.skill_service.list_skills(),
|
||||
read_in_task_scope(self.ap.plugin_connector.list_plugins),
|
||||
read_in_task_scope(
|
||||
lambda: self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
),
|
||||
read_in_task_scope(lambda: self.ap.skill_service.list_skills(request_context)),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
@@ -39,7 +59,7 @@ class ExtensionsRouterGroup(group.RouterGroup):
|
||||
extensions: list[dict] = []
|
||||
if isinstance(plugins, list):
|
||||
for plugin in plugins:
|
||||
extensions.append({'type': 'plugin', 'plugin': plugin})
|
||||
extensions.append({'type': 'plugin', 'plugin': redact_secrets(plugin)})
|
||||
if isinstance(mcp_servers, list):
|
||||
for server in mcp_servers:
|
||||
extensions.append({'type': 'mcp', 'server': server})
|
||||
|
||||
@@ -7,29 +7,53 @@ import asyncio
|
||||
|
||||
import quart.datastructures
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
def _storage_owner(context: RequestContext) -> str:
|
||||
if context.principal.account_uuid:
|
||||
return f'account:{context.principal.account_uuid}'
|
||||
if context.principal.api_key_uuid:
|
||||
return f'api-key:{context.principal.api_key_uuid}'
|
||||
return f'principal:{context.principal.principal_type.value}'
|
||||
|
||||
|
||||
@group.group_class('files', '/api/v1/files')
|
||||
class FilesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/image/<path:image_key>', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _(image_key: str) -> quart.Response:
|
||||
if '..' in image_key or '\\' in image_key:
|
||||
@self.route(
|
||||
'/image/<path:image_key>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(image_key: str, request_context: RequestContext) -> quart.Response:
|
||||
image_bytes = await self.ap.storage_mgr.resolve_public_object(
|
||||
image_key,
|
||||
expected_owner_type='upload_image',
|
||||
)
|
||||
if image_bytes is None:
|
||||
image_bytes = await self.ap.storage_mgr.resolve_public_object(
|
||||
image_key,
|
||||
expected_owner_type='bot_log',
|
||||
)
|
||||
if image_bytes is None:
|
||||
return quart.Response(status=404)
|
||||
|
||||
if not await self.ap.storage_mgr.storage_provider.exists(image_key):
|
||||
return quart.Response(status=404)
|
||||
|
||||
image_bytes = await self.ap.storage_mgr.storage_provider.load(image_key)
|
||||
mime_type = mimetypes.guess_type(image_key)[0]
|
||||
if mime_type is None:
|
||||
mime_type = 'image/jpeg'
|
||||
|
||||
return quart.Response(image_bytes, mimetype=mime_type)
|
||||
|
||||
@self.route('/images', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def upload_image() -> quart.Response:
|
||||
@self.route(
|
||||
'/images',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def upload_image(request_context: RequestContext) -> quart.Response:
|
||||
request = quart.request
|
||||
|
||||
# Check file size limit before reading the file
|
||||
@@ -66,18 +90,29 @@ class FilesRouterGroup(group.RouterGroup):
|
||||
if '/' in file_name or '\\' in file_name:
|
||||
return self.fail(400, 'File name contains invalid characters')
|
||||
|
||||
file_key = file_name + '_' + str(uuid.uuid4())[:8] + '.' + extension
|
||||
logical_key = f'{uuid.uuid4()}.{extension}'
|
||||
|
||||
# save file to storage
|
||||
await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
|
||||
file_key = await self.ap.storage_mgr.save_scoped(
|
||||
request_context,
|
||||
owner_type='upload_image',
|
||||
owner=_storage_owner(request_context),
|
||||
key=logical_key,
|
||||
value=file_bytes,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'file_key': file_key,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/documents', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def upload_document() -> quart.Response:
|
||||
@self.route(
|
||||
'/documents',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def upload_document(request_context: RequestContext) -> quart.Response:
|
||||
request = quart.request
|
||||
|
||||
# Check file size limit before reading the file
|
||||
@@ -110,12 +145,18 @@ class FilesRouterGroup(group.RouterGroup):
|
||||
if '/' in file_name or '\\' in file_name:
|
||||
return self.fail(400, 'File name contains invalid characters')
|
||||
|
||||
file_key = file_name + '_' + str(uuid.uuid4())[:8]
|
||||
logical_key = str(uuid.uuid4())
|
||||
if extension:
|
||||
file_key += '.' + extension
|
||||
logical_key += '.' + extension
|
||||
|
||||
# save file to storage
|
||||
await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
|
||||
file_key = await self.ap.storage_mgr.save_scoped(
|
||||
request_context,
|
||||
owner_type='upload_document',
|
||||
owner=_storage_owner(request_context),
|
||||
key=logical_key,
|
||||
value=file_bytes,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'file_id': file_key,
|
||||
|
||||
@@ -1,100 +1,146 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('knowledge_base', '/api/v1/knowledge/bases')
|
||||
class KnowledgeBaseRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['POST', 'GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def handle_knowledge_bases() -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
knowledge_bases = await self.ap.knowledge_service.get_knowledge_bases()
|
||||
return self.success(data={'bases': knowledge_bases})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def handle_knowledge_bases(request_context: RequestContext) -> quart.Response:
|
||||
knowledge_bases = await self.ap.knowledge_service.get_knowledge_bases(
|
||||
request_context,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
return self.success(data={'bases': knowledge_bases})
|
||||
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
knowledge_base_uuid = await self.ap.knowledge_service.create_knowledge_base(json_data)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
return self.success(data={'uuid': knowledge_base_uuid})
|
||||
|
||||
return self.http_status(405, -1, 'Method not allowed')
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def create_knowledge_base(request_context: RequestContext) -> quart.Response:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
knowledge_base_uuid = await self.ap.knowledge_service.create_knowledge_base(
|
||||
request_context,
|
||||
json_data,
|
||||
)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
return self.success(data={'uuid': knowledge_base_uuid})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>',
|
||||
methods=['GET', 'DELETE', 'PUT'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def handle_specific_knowledge_base(knowledge_base_uuid: str) -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
knowledge_base = await self.ap.knowledge_service.get_knowledge_base(knowledge_base_uuid)
|
||||
async def get_specific_knowledge_base(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
knowledge_base = await self.ap.knowledge_service.get_knowledge_base(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
if knowledge_base is None:
|
||||
return self.http_status(404, 'resource_not_found', 'knowledge base not found')
|
||||
return self.success(data={'base': knowledge_base})
|
||||
|
||||
if knowledge_base is None:
|
||||
return self.http_status(404, -1, 'knowledge base not found')
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'base': knowledge_base,
|
||||
}
|
||||
)
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>',
|
||||
methods=['DELETE', 'PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def mutate_specific_knowledge_base(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.knowledge_service.update_knowledge_base(knowledge_base_uuid, json_data)
|
||||
await self.ap.knowledge_service.update_knowledge_base(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
json_data,
|
||||
)
|
||||
return self.success(data={'uuid': knowledge_base_uuid})
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.knowledge_service.delete_knowledge_base(knowledge_base_uuid)
|
||||
return self.success({})
|
||||
await self.ap.knowledge_service.delete_knowledge_base(request_context, knowledge_base_uuid)
|
||||
return self.success({})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/files',
|
||||
methods=['GET', 'POST'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_knowledge_base_files(knowledge_base_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
files = await self.ap.knowledge_service.get_files_by_knowledge_base(knowledge_base_uuid)
|
||||
return self.success(
|
||||
data={
|
||||
'files': files,
|
||||
}
|
||||
)
|
||||
async def get_knowledge_base_files(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
files = await self.ap.knowledge_service.get_files_by_knowledge_base(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
return self.success(data={'files': files})
|
||||
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
file_id = json_data.get('file_id')
|
||||
if not file_id:
|
||||
return self.http_status(400, -1, 'File ID is required')
|
||||
|
||||
parser_plugin_id = json_data.get('parser_plugin_id')
|
||||
|
||||
# 调用服务层方法将文件与知识库关联
|
||||
task_id = await self.ap.knowledge_service.store_file(
|
||||
knowledge_base_uuid, file_id, parser_plugin_id=parser_plugin_id
|
||||
)
|
||||
return self.success(
|
||||
{
|
||||
'task_id': task_id,
|
||||
}
|
||||
)
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/files',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def add_knowledge_base_file(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
json_data = await quart.request.json
|
||||
file_id = json_data.get('file_id')
|
||||
if not file_id:
|
||||
return self.http_status(400, -1, 'File ID is required')
|
||||
parser_plugin_id = json_data.get('parser_plugin_id')
|
||||
task_id = await self.ap.knowledge_service.store_file(
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
file_id,
|
||||
parser_plugin_id=parser_plugin_id,
|
||||
)
|
||||
return self.success({'task_id': task_id})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/files/<file_id>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def delete_specific_file_in_kb(file_id: str, knowledge_base_uuid: str) -> str:
|
||||
await self.ap.knowledge_service.delete_file(knowledge_base_uuid, file_id)
|
||||
async def delete_specific_file_in_kb(
|
||||
file_id: str,
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
await self.ap.knowledge_service.delete_file(request_context, knowledge_base_uuid, file_id)
|
||||
return self.success({})
|
||||
|
||||
@self.route(
|
||||
'/<knowledge_base_uuid>/retrieve',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def retrieve_knowledge_base(knowledge_base_uuid: str) -> str:
|
||||
async def retrieve_knowledge_base(
|
||||
knowledge_base_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
json_data = await quart.request.json
|
||||
query = json_data.get('query')
|
||||
|
||||
@@ -104,6 +150,9 @@ class KnowledgeBaseRouterGroup(group.RouterGroup):
|
||||
# Extract retrieval_settings to allow dynamic control over Knowledge Engine behavior (e.g. top_k, filters)
|
||||
retrieval_settings = json_data.get('retrieval_settings', {})
|
||||
results = await self.ap.knowledge_service.retrieve_knowledge_base(
|
||||
knowledge_base_uuid, query, retrieval_settings
|
||||
request_context,
|
||||
knowledge_base_uuid,
|
||||
query,
|
||||
retrieval_settings,
|
||||
)
|
||||
return self.success(data={'results': results})
|
||||
|
||||
@@ -1,25 +1,39 @@
|
||||
import quart
|
||||
from urllib.parse import unquote
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('knowledge_engines', '/api/v1/knowledge/engines')
|
||||
class KnowledgeEnginesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_knowledge_engines() -> quart.Response:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_knowledge_engines(request_context: RequestContext) -> quart.Response:
|
||||
"""List all available Knowledge Engines from plugins.
|
||||
|
||||
Returns a list of Knowledge Engines with their capabilities and configuration schemas.
|
||||
This is used by the frontend to render the knowledge base creation wizard.
|
||||
"""
|
||||
engines = await self.ap.knowledge_service.list_knowledge_engines()
|
||||
engines = await self.ap.knowledge_service.list_knowledge_engines(request_context)
|
||||
return self.success(data={'engines': engines})
|
||||
|
||||
@self.route(
|
||||
'/<path:plugin_id>/creation-schema', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<path:plugin_id>/creation-schema',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_engine_creation_schema(plugin_id: str) -> quart.Response:
|
||||
async def get_engine_creation_schema(
|
||||
plugin_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
"""Get creation settings schema for a specific Knowledge Engine.
|
||||
|
||||
plugin_id is in 'author/name' format, captured via <path:> converter.
|
||||
@@ -27,13 +41,19 @@ class KnowledgeEnginesRouterGroup(group.RouterGroup):
|
||||
plugin_id = unquote(plugin_id)
|
||||
if '/' not in plugin_id:
|
||||
return self.http_status(400, -1, 'Invalid plugin_id format. Expected author/name.')
|
||||
schema = await self.ap.knowledge_service.get_engine_creation_schema(plugin_id)
|
||||
schema = await self.ap.knowledge_service.get_engine_creation_schema(request_context, plugin_id)
|
||||
return self.success(data={'schema': schema})
|
||||
|
||||
@self.route(
|
||||
'/<path:plugin_id>/retrieval-schema', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<path:plugin_id>/retrieval-schema',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_engine_retrieval_schema(plugin_id: str) -> quart.Response:
|
||||
async def get_engine_retrieval_schema(
|
||||
plugin_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> quart.Response:
|
||||
"""Get retrieval settings schema for a specific Knowledge Engine.
|
||||
|
||||
plugin_id is in 'author/name' format, captured via <path:> converter.
|
||||
@@ -41,5 +61,5 @@ class KnowledgeEnginesRouterGroup(group.RouterGroup):
|
||||
plugin_id = unquote(plugin_id)
|
||||
if '/' not in plugin_id:
|
||||
return self.http_status(400, -1, 'Invalid plugin_id format. Expected author/name.')
|
||||
schema = await self.ap.knowledge_service.get_engine_retrieval_schema(plugin_id)
|
||||
schema = await self.ap.knowledge_service.get_engine_retrieval_schema(request_context, plugin_id)
|
||||
return self.success(data={'schema': schema})
|
||||
|
||||
@@ -6,8 +6,12 @@ import quart
|
||||
import sqlalchemy
|
||||
|
||||
from ... import group
|
||||
from ....authz import Permission
|
||||
from ....context import ExecutionContext, RequestContext
|
||||
from ......core import taskmgr
|
||||
from ......entity.persistence import metadata as persistence_metadata
|
||||
from ......workspace.errors import WorkspaceError, WorkspaceNotFoundError
|
||||
from ......utils import httpclient
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
LANGRAG_PLUGIN_AUTHOR = 'langbot-team'
|
||||
@@ -31,24 +35,100 @@ EXTERNAL_PLUGIN_CREATION_FIELDS: dict[str, set[str] | None] = {
|
||||
'langbot-team/FastGPTConnector': None, # all fields -> creation_settings
|
||||
}
|
||||
|
||||
_INFORMATION_SCHEMA_TABLES = sqlalchemy.table(
|
||||
'tables',
|
||||
sqlalchemy.column('table_schema'),
|
||||
sqlalchemy.column('table_name'),
|
||||
schema='information_schema',
|
||||
)
|
||||
_SQLITE_MASTER = sqlalchemy.table(
|
||||
'sqlite_master',
|
||||
sqlalchemy.column('type'),
|
||||
sqlalchemy.column('name'),
|
||||
)
|
||||
_LEGACY_KNOWLEDGE_BASE_BACKUP = sqlalchemy.table(
|
||||
'knowledge_bases_backup',
|
||||
sqlalchemy.column('uuid'),
|
||||
sqlalchemy.column('name'),
|
||||
sqlalchemy.column('description'),
|
||||
sqlalchemy.column('emoji'),
|
||||
sqlalchemy.column('embedding_model_uuid'),
|
||||
sqlalchemy.column('top_k'),
|
||||
sqlalchemy.column('created_at'),
|
||||
sqlalchemy.column('updated_at'),
|
||||
)
|
||||
_LEGACY_EXTERNAL_KNOWLEDGE_BASE = sqlalchemy.table(
|
||||
'external_knowledge_bases',
|
||||
sqlalchemy.column('uuid'),
|
||||
sqlalchemy.column('name'),
|
||||
sqlalchemy.column('description'),
|
||||
sqlalchemy.column('emoji'),
|
||||
sqlalchemy.column('plugin_author'),
|
||||
sqlalchemy.column('plugin_name'),
|
||||
sqlalchemy.column('retriever_config'),
|
||||
sqlalchemy.column('created_at'),
|
||||
)
|
||||
_CURRENT_KNOWLEDGE_BASE = sqlalchemy.table(
|
||||
'knowledge_bases',
|
||||
sqlalchemy.column('uuid'),
|
||||
sqlalchemy.column('workspace_uuid'),
|
||||
sqlalchemy.column('name'),
|
||||
sqlalchemy.column('description'),
|
||||
sqlalchemy.column('emoji'),
|
||||
sqlalchemy.column('created_at'),
|
||||
sqlalchemy.column('updated_at'),
|
||||
sqlalchemy.column('knowledge_engine_plugin_id'),
|
||||
sqlalchemy.column('collection_id'),
|
||||
sqlalchemy.column('creation_settings'),
|
||||
sqlalchemy.column('retrieval_settings'),
|
||||
)
|
||||
|
||||
|
||||
@group.group_class('knowledge/migration', '/api/v1/knowledge/migration')
|
||||
class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
async def _get_migration_flag(self) -> bool:
|
||||
async def _require_local_migration_context(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
) -> ExecutionContext:
|
||||
"""Fence legacy-table migration to the OSS singleton Workspace.
|
||||
|
||||
The backup tables predate Workspace scoping and are deliberately
|
||||
instance-global. A cloud projection must therefore never be allowed
|
||||
to inspect or restore them, even when it has a valid execution lease.
|
||||
"""
|
||||
try:
|
||||
binding = await self.ap.workspace_service.get_local_execution_binding(
|
||||
execution_context.workspace_uuid,
|
||||
expected_generation=execution_context.placement_generation,
|
||||
)
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except WorkspaceError as exc:
|
||||
raise WorkspaceNotFoundError('RAG migration is unavailable') from exc
|
||||
|
||||
if binding.instance_uuid != execution_context.instance_uuid:
|
||||
raise WorkspaceNotFoundError('RAG migration is unavailable')
|
||||
return ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
)
|
||||
|
||||
async def _get_migration_flag(self, execution_context: ExecutionContext) -> bool:
|
||||
"""Check if rag_plugin_migration_needed flag is set."""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_metadata.Metadata).where(
|
||||
persistence_metadata.Metadata.key == 'rag_plugin_migration_needed'
|
||||
)
|
||||
sqlalchemy.select(persistence_metadata.WorkspaceMetadata.value)
|
||||
.where(persistence_metadata.WorkspaceMetadata.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_metadata.WorkspaceMetadata.key == 'rag_plugin_migration_needed')
|
||||
)
|
||||
row = result.first()
|
||||
return row is not None and row.value == 'true'
|
||||
return result.scalar_one_or_none() == 'true'
|
||||
|
||||
async def _set_migration_flag(self, value: str):
|
||||
async def _set_migration_flag(self, execution_context: ExecutionContext, value: str):
|
||||
"""Set rag_plugin_migration_needed flag."""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_metadata.Metadata)
|
||||
.where(persistence_metadata.Metadata.key == 'rag_plugin_migration_needed')
|
||||
sqlalchemy.update(persistence_metadata.WorkspaceMetadata)
|
||||
.where(persistence_metadata.WorkspaceMetadata.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_metadata.WorkspaceMetadata.key == 'rag_plugin_migration_needed')
|
||||
.values(value=value)
|
||||
)
|
||||
|
||||
@@ -56,35 +136,47 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
"""Check if a table exists."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
|
||||
).bindparams(table_name=table_name)
|
||||
sqlalchemy.select(_INFORMATION_SCHEMA_TABLES.c.table_name)
|
||||
.where(_INFORMATION_SCHEMA_TABLES.c.table_schema == 'public')
|
||||
.where(_INFORMATION_SCHEMA_TABLES.c.table_name == table_name)
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar()
|
||||
return result.first() is not None
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
|
||||
table_name=table_name
|
||||
)
|
||||
sqlalchemy.select(_SQLITE_MASTER.c.name)
|
||||
.where(_SQLITE_MASTER.c.type == 'table')
|
||||
.where(_SQLITE_MASTER.c.name == table_name)
|
||||
.limit(1)
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _install_plugin_from_marketplace(
|
||||
self, plugin_id: str, task_context: taskmgr.TaskContext, space_url: str
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
plugin_id: str,
|
||||
task_context: taskmgr.TaskContext,
|
||||
space_url: str,
|
||||
) -> None:
|
||||
"""Install a single plugin from the marketplace."""
|
||||
p_author, p_name = plugin_id.split('/', 1)
|
||||
self.ap.logger.info(f'RAG migration: installing plugin {plugin_id} from marketplace...')
|
||||
task_context.trace(f'Installing plugin {plugin_id} from marketplace...')
|
||||
|
||||
async with httpx.AsyncClient(trust_env=True, timeout=15) as client:
|
||||
async with httpx.AsyncClient(
|
||||
trust_env=True,
|
||||
timeout=15,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(),
|
||||
) as client:
|
||||
resp = await client.get(f'{space_url}/api/v1/marketplace/plugins/{p_author}/{p_name}')
|
||||
resp.raise_for_status()
|
||||
p_data = resp.json().get('data', {}).get('plugin', {})
|
||||
response_data = await httpclient.parse_json_response(resp)
|
||||
p_data = response_data.get('data', {}).get('plugin', {})
|
||||
p_version = p_data.get('latest_version')
|
||||
if not p_version:
|
||||
raise Exception(f'Could not determine latest version for {plugin_id}')
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
await self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE,
|
||||
{
|
||||
@@ -96,8 +188,15 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
)
|
||||
self.ap.logger.info(f'RAG migration: plugin {plugin_id} install request sent.')
|
||||
|
||||
async def _execute_rag_migration(self, task_context: taskmgr.TaskContext, install_plugin: bool = True):
|
||||
async def _execute_rag_migration(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
task_context: taskmgr.TaskContext,
|
||||
install_plugin: bool = True,
|
||||
):
|
||||
"""Execute RAG migration: install required plugins and restore backup data."""
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
warnings = []
|
||||
|
||||
# Collect all plugins we need: LangRAG (always) + connector plugins (from external KBs)
|
||||
@@ -108,7 +207,10 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
has_external = await self._table_exists('external_knowledge_bases')
|
||||
if has_external:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT DISTINCT plugin_author, plugin_name FROM external_knowledge_bases;')
|
||||
sqlalchemy.select(
|
||||
_LEGACY_EXTERNAL_KNOWLEDGE_BASE.c.plugin_author,
|
||||
_LEGACY_EXTERNAL_KNOWLEDGE_BASE.c.plugin_name,
|
||||
).distinct()
|
||||
)
|
||||
for row in result.fetchall():
|
||||
plugin_author = row[0] or ''
|
||||
@@ -127,7 +229,14 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
|
||||
for plugin_id in needed_plugins:
|
||||
try:
|
||||
await self._install_plugin_from_marketplace(plugin_id, task_context, space_url)
|
||||
await self._install_plugin_from_marketplace(
|
||||
execution_context,
|
||||
plugin_id,
|
||||
task_context,
|
||||
space_url,
|
||||
)
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'RAG migration: plugin {plugin_id} install returned: {e}')
|
||||
task_context.trace(f'Plugin install note ({plugin_id}): {e}')
|
||||
@@ -141,8 +250,11 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
engine_id_set: set[str] = set()
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
if all(pid in engine_id_set for pid in needed_plugins):
|
||||
@@ -158,17 +270,18 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
await asyncio.sleep(2)
|
||||
else:
|
||||
try:
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except WorkspaceNotFoundError:
|
||||
raise
|
||||
except Exception:
|
||||
engine_id_set = set()
|
||||
|
||||
# Step 3: Restore internal knowledge bases from backup
|
||||
task_context.trace('Restoring internal knowledge bases...', action='restore-internal')
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM knowledge_bases_backup;')
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(_LEGACY_KNOWLEDGE_BASE_BACKUP))
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
@@ -183,30 +296,30 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
created_at = row_dict.get('created_at')
|
||||
updated_at = row_dict.get('updated_at')
|
||||
|
||||
# DB migration 20 created these columns as TEXT, while a fresh
|
||||
# schema uses SQLAlchemy JSON. Keep the statement structured,
|
||||
# but retain untyped bound values so both physical schemas and
|
||||
# SQLite's string-valued legacy DATETIME rows remain valid.
|
||||
creation_settings = json.dumps({'embedding_model_uuid': embedding_model_uuid})
|
||||
retrieval_settings = json.dumps({'top_k': top_k})
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
sqlalchemy.insert(_CURRENT_KNOWLEDGE_BASE).values(
|
||||
uuid=kb_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
plugin_id=LANGRAG_PLUGIN_ID,
|
||||
knowledge_engine_plugin_id=LANGRAG_PLUGIN_ID,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=retrieval_settings,
|
||||
)
|
||||
)
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
try:
|
||||
config = {'embedding_model_uuid': embedding_model_uuid}
|
||||
await self.ap.plugin_connector.rag_on_kb_create(LANGRAG_PLUGIN_ID, kb_uuid, config)
|
||||
@@ -221,9 +334,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
# Step 4: Restore external knowledge bases
|
||||
task_context.trace('Restoring external knowledge bases...', action='restore-external')
|
||||
if has_external:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(_LEGACY_EXTERNAL_KNOWLEDGE_BASE))
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
@@ -266,20 +377,15 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
retrieval_settings_dict = {k: v for k, v in retriever_config.items() if k not in creation_fields}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
sqlalchemy.insert(_CURRENT_KNOWLEDGE_BASE).values(
|
||||
uuid=kb_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=created_at,
|
||||
plugin_id=external_plugin_id,
|
||||
knowledge_engine_plugin_id=external_plugin_id,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=json.dumps(creation_settings_dict),
|
||||
retrieval_settings=json.dumps(retrieval_settings_dict),
|
||||
@@ -294,6 +400,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
else:
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
try:
|
||||
await self.ap.plugin_connector.rag_on_kb_create(
|
||||
external_plugin_id, kb_uuid, creation_settings_dict
|
||||
@@ -307,16 +414,23 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
await self.ap.rag_mgr.load_knowledge_bases_from_db()
|
||||
|
||||
# Step 5: Clear migration flag
|
||||
await self._set_migration_flag('false')
|
||||
await self._set_migration_flag(execution_context, 'false')
|
||||
task_context.trace('RAG migration completed.', action='done')
|
||||
|
||||
if warnings:
|
||||
task_context.trace(f'Completed with {len(warnings)} warning(s).')
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
@self.route(
|
||||
'/status',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
needed = await self._get_migration_flag(execution_context)
|
||||
|
||||
internal_kb_count = 0
|
||||
external_kb_count = 0
|
||||
@@ -324,13 +438,13 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
if needed:
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases_backup;')
|
||||
sqlalchemy.select(sqlalchemy.func.count()).select_from(_LEGACY_KNOWLEDGE_BASE_BACKUP)
|
||||
)
|
||||
internal_kb_count = result.scalar() or 0
|
||||
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
sqlalchemy.select(sqlalchemy.func.count()).select_from(_LEGACY_EXTERNAL_KNOWLEDGE_BASE)
|
||||
)
|
||||
external_kb_count = result.scalar() or 0
|
||||
|
||||
@@ -342,9 +456,16 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
@self.route(
|
||||
'/execute',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
needed = await self._get_migration_flag(execution_context)
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
@@ -353,20 +474,34 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._execute_rag_migration(task_context=ctx, install_plugin=install_plugin),
|
||||
self._execute_rag_migration(
|
||||
execution_context,
|
||||
task_context=ctx,
|
||||
install_plugin=install_plugin,
|
||||
),
|
||||
kind='rag-migration',
|
||||
name='rag-migration-execute',
|
||||
label='Migrating knowledge bases to plugin architecture',
|
||||
context=ctx,
|
||||
instance_uuid=execution_context.instance_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
placement_generation=execution_context.placement_generation,
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/dismiss', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
@self.route(
|
||||
'/dismiss',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
execution_context = await self._require_local_migration_context(execution_context)
|
||||
needed = await self._get_migration_flag(execution_context)
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
await self._set_migration_flag('false')
|
||||
await self._set_migration_flag(execution_context, 'false')
|
||||
return self.success()
|
||||
|
||||
@@ -1,16 +1,24 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('parsers', '/api/v1/knowledge/parsers')
|
||||
class ParsersRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_parsers() -> quart.Response:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_parsers(request_context: RequestContext) -> quart.Response:
|
||||
"""List all available parsers from plugins.
|
||||
|
||||
Optional query parameter `mime_type` to filter parsers by supported MIME type.
|
||||
"""
|
||||
mime_type = quart.request.args.get('mime_type')
|
||||
parsers = await self.ap.knowledge_service.list_parsers(mime_type)
|
||||
parsers = await self.ap.knowledge_service.list_parsers(request_context, mime_type)
|
||||
return self.success(data={'parsers': parsers})
|
||||
|
||||
@@ -3,14 +3,23 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('logs', '/api/v1/logs')
|
||||
class LogsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route('', methods=['GET'], permission=Permission.AUDIT_VIEW)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
# The process log is instance-global. It is safe to expose only in
|
||||
# the OSS singleton Workspace; SaaS must use Workspace-scoped
|
||||
# observability records instead of leaking another tenant's lines.
|
||||
await self.ap.workspace_service.get_local_execution_binding(
|
||||
request_context.workspace_uuid,
|
||||
expected_generation=request_context.placement_generation,
|
||||
)
|
||||
start_page_number = int(quart.request.args.get('start_page_number', 0))
|
||||
start_offset = int(quart.request.args.get('start_offset', 0))
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import datetime
|
||||
import quart
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@@ -24,8 +26,8 @@ def parse_iso_datetime(datetime_str: str | None) -> datetime.datetime | None:
|
||||
@group.group_class('monitoring', '/api/v1/monitoring')
|
||||
class MonitoringRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/overview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_overview() -> str:
|
||||
@self.route('/overview', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_overview(request_context: RequestContext) -> str:
|
||||
"""Get overview metrics"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -38,6 +40,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
metrics = await self.ap.monitoring_service.get_overview_metrics(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -46,8 +49,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=metrics)
|
||||
|
||||
@self.route('/token-statistics', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_token_statistics() -> str:
|
||||
@self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_token_statistics(request_context: RequestContext) -> str:
|
||||
"""Get detailed token usage statistics (summary, per-model, timeseries)."""
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
@@ -61,6 +64,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
stats = await self.ap.monitoring_service.get_token_statistics(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -70,8 +74,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=stats)
|
||||
|
||||
@self.route('/messages', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_messages() -> str:
|
||||
@self.route('/messages', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_messages(request_context: RequestContext) -> str:
|
||||
"""Get message logs"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -87,6 +91,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
messages, total = await self.ap.monitoring_service.get_messages(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
session_ids=session_ids if session_ids else None,
|
||||
@@ -105,8 +110,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/llm-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_llm_calls() -> str:
|
||||
@self.route('/llm-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_llm_calls(request_context: RequestContext) -> str:
|
||||
"""Get LLM call records"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -121,6 +126,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
llm_calls, total = await self.ap.monitoring_service.get_llm_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -138,8 +144,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/tool-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_tool_calls() -> str:
|
||||
@self.route('/tool-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_tool_calls(request_context: RequestContext) -> str:
|
||||
"""Get tool call records"""
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
pipeline_ids = quart.request.args.getlist('pipelineId')
|
||||
@@ -153,6 +159,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
tool_calls, total = await self.ap.monitoring_service.get_tool_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
session_ids=session_ids if session_ids else None,
|
||||
@@ -171,8 +178,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/embedding-calls', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_embedding_calls() -> str:
|
||||
@self.route('/embedding-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_embedding_calls(request_context: RequestContext) -> str:
|
||||
"""Get embedding call records"""
|
||||
# Parse query parameters
|
||||
start_time_str = quart.request.args.get('startTime')
|
||||
@@ -186,6 +193,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
embedding_calls, total = await self.ap.monitoring_service.get_embedding_calls(
|
||||
request_context,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
knowledge_base_id=knowledge_base_id if knowledge_base_id else None,
|
||||
@@ -202,8 +210,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/sessions', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_sessions() -> str:
|
||||
@self.route('/sessions', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_sessions(request_context: RequestContext) -> str:
|
||||
"""Get session information"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -224,6 +232,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
is_active = is_active_str.lower() == 'true'
|
||||
|
||||
sessions, total = await self.ap.monitoring_service.get_sessions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -242,8 +251,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/errors', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_errors() -> str:
|
||||
@self.route('/errors', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_errors(request_context: RequestContext) -> str:
|
||||
"""Get error logs"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -258,6 +267,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
errors, total = await self.ap.monitoring_service.get_errors(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -275,8 +285,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/data', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_all_data() -> str:
|
||||
@self.route('/data', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_all_data(request_context: RequestContext) -> str:
|
||||
"""Get all monitoring data in a single request"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -291,6 +301,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get overview metrics
|
||||
overview = await self.ap.monitoring_service.get_overview_metrics(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -299,6 +310,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get messages
|
||||
messages, messages_total = await self.ap.monitoring_service.get_messages(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -309,6 +321,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get LLM calls
|
||||
llm_calls, llm_calls_total = await self.ap.monitoring_service.get_llm_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -319,6 +332,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get tool calls
|
||||
tool_calls, tool_calls_total = await self.ap.monitoring_service.get_tool_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -329,6 +343,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get sessions
|
||||
sessions, sessions_total = await self.ap.monitoring_service.get_sessions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -340,6 +355,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get errors
|
||||
errors, errors_total = await self.ap.monitoring_service.get_errors(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -350,6 +366,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
# Get embedding calls
|
||||
embedding_calls, embedding_calls_total = await self.ap.monitoring_service.get_embedding_calls(
|
||||
request_context,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
@@ -376,27 +393,27 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/sessions/<session_id>/analysis', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_session_analysis(session_id: str) -> str:
|
||||
@self.route('/sessions/<session_id>/analysis', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_session_analysis(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Get detailed analysis for a specific session"""
|
||||
analysis = await self.ap.monitoring_service.get_session_analysis(session_id)
|
||||
analysis = await self.ap.monitoring_service.get_session_analysis(request_context, session_id)
|
||||
|
||||
# Always return success with the analysis data
|
||||
# The frontend will handle the 'found: false' case
|
||||
return self.success(data=analysis)
|
||||
|
||||
@self.route('/messages/<message_id>/details', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_message_details(message_id: str) -> str:
|
||||
@self.route('/messages/<message_id>/details', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_message_details(message_id: str, request_context: RequestContext) -> str:
|
||||
"""Get detailed information for a specific message"""
|
||||
details = await self.ap.monitoring_service.get_message_details(message_id)
|
||||
details = await self.ap.monitoring_service.get_message_details(request_context, message_id)
|
||||
|
||||
if not details.get('found'):
|
||||
return self.error(message=f'Message {message_id} not found', code=404)
|
||||
|
||||
return self.success(data=details)
|
||||
|
||||
@self.route('/export', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def export_data() -> tuple[str, int]:
|
||||
@self.route('/export', methods=['GET'], permission=Permission.DATA_EXPORT)
|
||||
async def export_data(request_context: RequestContext) -> tuple[str, int]:
|
||||
"""Export monitoring data as CSV"""
|
||||
# Parse query parameters
|
||||
export_type = quart.request.args.get('type', 'messages')
|
||||
@@ -413,6 +430,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
# Get data based on export type
|
||||
if export_type == 'messages':
|
||||
data = await self.ap.monitoring_service.export_messages(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -437,6 +455,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'llm-calls':
|
||||
data = await self.ap.monitoring_service.export_llm_calls(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -463,6 +482,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'embedding-calls':
|
||||
data = await self.ap.monitoring_service.export_embedding_calls(
|
||||
request_context,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
limit=limit,
|
||||
@@ -485,6 +505,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'errors':
|
||||
data = await self.ap.monitoring_service.export_errors(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -506,6 +527,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'sessions':
|
||||
data = await self.ap.monitoring_service.export_sessions(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -527,6 +549,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
]
|
||||
elif export_type == 'feedback':
|
||||
data = await self.ap.monitoring_service.export_feedback(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -581,8 +604,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return response, 200
|
||||
|
||||
@self.route('/feedback/stats', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_feedback_stats() -> str:
|
||||
@self.route('/feedback/stats', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_feedback_stats(request_context: RequestContext) -> str:
|
||||
"""Get feedback statistics"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -595,6 +618,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
end_time = parse_iso_datetime(end_time_str)
|
||||
|
||||
stats = await self.ap.monitoring_service.get_feedback_stats(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
start_time=start_time,
|
||||
@@ -603,8 +627,8 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=stats)
|
||||
|
||||
@self.route('/feedback', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def get_feedback() -> str:
|
||||
@self.route('/feedback', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def get_feedback(request_context: RequestContext) -> str:
|
||||
"""Get feedback list"""
|
||||
# Parse query parameters
|
||||
bot_ids = quart.request.args.getlist('botId')
|
||||
@@ -623,6 +647,7 @@ class MonitoringRouterGroup(group.RouterGroup):
|
||||
feedback_type = int(feedback_type_str) if feedback_type_str else None
|
||||
|
||||
feedback_list, total = await self.ap.monitoring_service.get_feedback_list(
|
||||
request_context,
|
||||
bot_ids=bot_ids if bot_ids else None,
|
||||
pipeline_ids=pipeline_ids if pipeline_ids else None,
|
||||
feedback_type=feedback_type,
|
||||
|
||||
@@ -20,10 +20,12 @@ import httpx
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ......utils import paths
|
||||
from ......platform.sources.websocket_manager import is_valid_session_id, ws_connection_manager
|
||||
from ......utils import httpclient, paths
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, is_valid_session_id, ws_connection_manager
|
||||
from .websocket_chat import create_scoped_duplex_tasks, wait_for_duplex_tasks
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_AUTH_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# Cache the widget template content
|
||||
_widget_template_cache: str | None = None
|
||||
@@ -58,37 +60,31 @@ def _get_logo_bytes() -> bytes:
|
||||
class EmbedRouterGroup(group.RouterGroup):
|
||||
# -- helpers -------------------------------------------------------------
|
||||
|
||||
def _resolve_bot(self, bot_uuid: str):
|
||||
async def _resolve_bot(self, bot_uuid: str):
|
||||
"""Resolve *bot_uuid* to ``(runtime_bot, pipeline_uuid)``.
|
||||
|
||||
Returns ``(None, None)`` when the bot does not exist, is not a
|
||||
``web_page_bot``, is disabled, or has no pipeline bound.
|
||||
"""
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if (
|
||||
bot.bot_entity.uuid == bot_uuid
|
||||
and bot.bot_entity.adapter == 'web_page_bot'
|
||||
and bot.bot_entity.enable
|
||||
and bot.bot_entity.use_pipeline_uuid
|
||||
):
|
||||
return bot, bot.bot_entity.use_pipeline_uuid
|
||||
bot = await self.ap.platform_mgr.resolve_public_bot(bot_uuid)
|
||||
if (
|
||||
bot is not None
|
||||
and bot.bot_entity.adapter == 'web_page_bot'
|
||||
and bot.bot_entity.enable
|
||||
and bot.bot_entity.use_pipeline_uuid
|
||||
):
|
||||
return bot, bot.bot_entity.use_pipeline_uuid
|
||||
return None, None
|
||||
|
||||
def _get_bot_config(self, bot_uuid: str) -> dict:
|
||||
for bot in self.ap.platform_mgr.bots:
|
||||
if bot.bot_entity.uuid == bot_uuid and bot.bot_entity.adapter == 'web_page_bot':
|
||||
return bot.bot_entity.adapter_config
|
||||
return {}
|
||||
@staticmethod
|
||||
def _get_bot_config(runtime_bot) -> dict:
|
||||
return runtime_bot.bot_entity.adapter_config
|
||||
|
||||
async def _verify_session_token(self, request, bot_uuid: str) -> bool:
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
def _verify_session_token_value(self, token: str, runtime_bot) -> bool:
|
||||
config = self._get_bot_config(runtime_bot)
|
||||
secret = config.get('turnstile_secret_key', '')
|
||||
if not secret:
|
||||
return True
|
||||
auth_header = request.headers.get('Authorization', '')
|
||||
if not auth_header.startswith('Bearer '):
|
||||
return False
|
||||
token = auth_header[7:]
|
||||
try:
|
||||
ts_str, mac = token.split('.', 1)
|
||||
ts = float(ts_str)
|
||||
@@ -99,6 +95,50 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def _verify_session_token(self, request, runtime_bot) -> bool:
|
||||
auth_header = request.headers.get('Authorization', '')
|
||||
token = auth_header[7:] if auth_header.startswith('Bearer ') else ''
|
||||
return self._verify_session_token_value(token, runtime_bot)
|
||||
|
||||
async def _authenticate_websocket(self, runtime_bot) -> None:
|
||||
"""Require the embed session token as the first WebSocket frame."""
|
||||
|
||||
raw_message = await asyncio.wait_for(quart.websocket.receive(), timeout=_AUTH_TIMEOUT_SECONDS)
|
||||
payload = await asyncio.to_thread(json.loads, raw_message)
|
||||
if not isinstance(payload, dict) or payload.get('type') != 'authenticate':
|
||||
raise ValueError('Authentication is required')
|
||||
token = str(payload.get('token') or '')
|
||||
if not self._verify_session_token_value(token, runtime_bot):
|
||||
raise ValueError('Authentication is required')
|
||||
|
||||
async def _assert_execution_active(self, runtime_bot) -> None:
|
||||
context = runtime_bot.execution_context
|
||||
await self.ap.workspace_service.get_execution_binding(
|
||||
context.workspace_uuid,
|
||||
expected_generation=context.placement_generation,
|
||||
)
|
||||
|
||||
async def _resolve_connected_bot(self, owner_bot, pipeline_uuid: str):
|
||||
"""Re-resolve mutable bot state before every public message."""
|
||||
current_bot, current_pipeline_uuid = await self._resolve_bot(owner_bot.bot_entity.uuid)
|
||||
if current_bot is None or current_pipeline_uuid != pipeline_uuid:
|
||||
raise RuntimeError('Bot is unavailable')
|
||||
|
||||
owner_context = owner_bot.execution_context
|
||||
current_context = current_bot.execution_context
|
||||
if (
|
||||
current_context.instance_uuid,
|
||||
current_context.workspace_uuid,
|
||||
current_context.placement_generation,
|
||||
) != (
|
||||
owner_context.instance_uuid,
|
||||
owner_context.workspace_uuid,
|
||||
owner_context.placement_generation,
|
||||
):
|
||||
raise RuntimeError('Bot is unavailable')
|
||||
await self._assert_execution_active(current_bot)
|
||||
return current_bot
|
||||
|
||||
# -- routes --------------------------------------------------------------
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@@ -106,7 +146,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def verify_turnstile(bot_uuid: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
try:
|
||||
@@ -115,18 +155,18 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not token:
|
||||
return self.http_status(400, -1, 'Token is required')
|
||||
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
config = self._get_bot_config(runtime_bot)
|
||||
secret = config.get('turnstile_secret_key', '')
|
||||
if not secret:
|
||||
ts = time.time()
|
||||
return self.success(data={'token': f'{ts}.dummy'})
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with httpx.AsyncClient(event_hooks=httpclient.httpx_response_limit_hooks()) as client:
|
||||
resp = await client.post(
|
||||
'https://challenges.cloudflare.com/turnstile/v0/siteverify',
|
||||
data={'secret': secret, 'response': token},
|
||||
)
|
||||
result = resp.json()
|
||||
result = await httpclient.parse_json_response(resp)
|
||||
|
||||
if not result.get('success'):
|
||||
return self.http_status(403, -1, 'Turnstile verification failed')
|
||||
@@ -146,7 +186,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
"""Serve the embed widget JavaScript with injected configuration."""
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return quart.Response(
|
||||
'// Bot not found or not available', status=404, content_type='application/javascript'
|
||||
@@ -164,7 +204,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not re.match(r'^https?://[a-zA-Z0-9._:/-]+$', base_url):
|
||||
base_url = quart.request.host_url.rstrip('/')
|
||||
|
||||
config = self._get_bot_config(bot_uuid)
|
||||
config = self._get_bot_config(runtime_bot)
|
||||
site_key = config.get('turnstile_site_key', '')
|
||||
locale = config.get('language', 'en_US') or 'en_US'
|
||||
bubble_icon = config.get('bubble_icon', 'logo') or 'logo'
|
||||
@@ -194,10 +234,10 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def get_embed_messages(bot_uuid: str, session_type: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
if not await self._verify_session_token(quart.request, runtime_bot):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
@@ -207,7 +247,8 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not is_valid_session_id(session_id):
|
||||
return self.http_status(400, -1, 'Valid session_id is required')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
@@ -222,10 +263,10 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def reset_embed_session(bot_uuid: str, session_type: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
if not await self._verify_session_token(quart.request, runtime_bot):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
@@ -235,7 +276,8 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
if not is_valid_session_id(session_id):
|
||||
return self.http_status(400, -1, 'Valid session_id is required')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
@@ -250,10 +292,10 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
async def submit_feedback(bot_uuid: str) -> str:
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
return self.http_status(400, -1, 'Invalid bot_uuid format')
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
return self.http_status(404, -1, 'Bot not found or not available')
|
||||
if not await self._verify_session_token(quart.request, bot_uuid):
|
||||
if not await self._verify_session_token(quart.request, runtime_bot):
|
||||
return self.http_status(403, -1, 'Unauthorized or session expired')
|
||||
try:
|
||||
data = await quart.request.get_json()
|
||||
@@ -266,6 +308,7 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
feedback_id = f'embed_{uuid.uuid4().hex[:12]}'
|
||||
|
||||
await self.ap.monitoring_service.record_feedback(
|
||||
runtime_bot.execution_context,
|
||||
feedback_id=feedback_id,
|
||||
feedback_type=feedback_type,
|
||||
bot_id=runtime_bot.bot_entity.uuid,
|
||||
@@ -286,11 +329,12 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
@self.quart_app.websocket(self.path + '/<bot_uuid>/ws/connect')
|
||||
async def embed_websocket_connect(bot_uuid: str):
|
||||
"""WebSocket connection for embed widget, keyed by bot_uuid."""
|
||||
await quart.websocket.accept()
|
||||
if not _is_valid_uuid(bot_uuid):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'}))
|
||||
return
|
||||
|
||||
runtime_bot, pipeline_uuid = self._resolve_bot(bot_uuid)
|
||||
runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
|
||||
if runtime_bot is None:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'}))
|
||||
return
|
||||
@@ -307,18 +351,42 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Valid session_id is required'}))
|
||||
return
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
try:
|
||||
await self._authenticate_websocket(runtime_bot)
|
||||
await self._assert_execution_active(runtime_bot)
|
||||
except Exception:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
|
||||
return
|
||||
|
||||
try:
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
|
||||
websocket_adapter = proxy_bot.adapter
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
return
|
||||
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
websocket=quart.websocket._get_current_object(),
|
||||
scope=WebSocketScope.from_context(runtime_bot.execution_context),
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
session_type=session_type,
|
||||
session_id=session_id,
|
||||
metadata={'user_agent': quart.websocket.headers.get('User-Agent', '')},
|
||||
send_queue_size=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('send_queue_size', 100)
|
||||
),
|
||||
max_connections=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections', 1024)
|
||||
),
|
||||
max_connections_per_workspace=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections_per_workspace', 32)
|
||||
),
|
||||
)
|
||||
|
||||
await quart.websocket.send(
|
||||
@@ -338,11 +406,19 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
f'(bot={bot_uuid}, pipeline={pipeline_uuid}, session_type={session_type})'
|
||||
)
|
||||
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter, runtime_bot))
|
||||
send_task = asyncio.create_task(self._handle_send(connection))
|
||||
receive_task, send_task = create_scoped_duplex_tasks(
|
||||
self._handle_receive(
|
||||
connection,
|
||||
websocket_adapter,
|
||||
runtime_bot,
|
||||
pipeline_uuid,
|
||||
),
|
||||
self._handle_send(connection),
|
||||
runtime_bot.execution_context.workspace_uuid,
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.gather(receive_task, send_task)
|
||||
await wait_for_duplex_tasks(receive_task, send_task)
|
||||
except Exception as e:
|
||||
logger.error(f'Embed WebSocket task error: {e}')
|
||||
finally:
|
||||
@@ -357,14 +433,14 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
|
||||
# -- WebSocket receive/send helpers --------------------------------------
|
||||
|
||||
async def _handle_receive(self, connection, websocket_adapter, owner_bot):
|
||||
async def _handle_receive(self, connection, websocket_adapter, owner_bot, pipeline_uuid: str):
|
||||
try:
|
||||
while connection.is_active:
|
||||
message = await quart.websocket.receive()
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
|
||||
if message_type == 'ping':
|
||||
@@ -372,7 +448,12 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
elif message_type == 'message':
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=owner_bot)
|
||||
try:
|
||||
current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
|
||||
except Exception:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
|
||||
elif message_type == 'disconnect':
|
||||
break
|
||||
|
||||
@@ -383,13 +464,20 @@ class EmbedRouterGroup(group.RouterGroup):
|
||||
logger.error(f'Embed receive error: {e}', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
try:
|
||||
connection.send_queue.put_nowait(None)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
async def _handle_send(self, connection):
|
||||
try:
|
||||
while connection.is_active:
|
||||
while connection.is_active or not connection.send_queue.empty():
|
||||
try:
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
await quart.websocket.send(json.dumps(message))
|
||||
if message is None:
|
||||
break
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except Exception as e:
|
||||
|
||||
@@ -2,120 +2,156 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ....service.secrets import redact_secrets
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('pipelines', '/api/v1/pipelines')
|
||||
class PipelinesRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
sort_by = quart.request.args.get('sort_by', 'created_at')
|
||||
sort_order = quart.request.args.get('sort_order', 'DESC')
|
||||
return self.success(
|
||||
data={'pipelines': await self.ap.pipeline_service.get_pipelines(sort_by, sort_order)}
|
||||
)
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(json_data)
|
||||
|
||||
return self.success(data={'uuid': pipeline_uuid})
|
||||
|
||||
@self.route('/_/metadata', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
return self.success(data={'configs': await self.ap.pipeline_service.get_pipeline_metadata()})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
sort_by = quart.request.args.get('sort_by', 'created_at')
|
||||
sort_order = quart.request.args.get('sort_order', 'DESC')
|
||||
include_secret = has_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
return self.success(
|
||||
data={
|
||||
'pipelines': await self.ap.pipeline_service.get_pipelines(
|
||||
request_context,
|
||||
sort_by,
|
||||
sort_order,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@self.route(
|
||||
'/<pipeline_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(request_context, await quart.request.json)
|
||||
return self.success(data={'uuid': pipeline_uuid})
|
||||
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
@self.route(
|
||||
'/_/metadata',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data={'configs': await self.ap.pipeline_service.get_pipeline_metadata(request_context)})
|
||||
|
||||
return self.success(data={'pipeline': pipeline})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@self.route(
|
||||
'/<pipeline_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
return self.success(data={'pipeline': pipeline})
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline_uuid, json_data)
|
||||
@self.route(
|
||||
'/<pipeline_uuid>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
try:
|
||||
await self.ap.pipeline_service.update_pipeline(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
else:
|
||||
await self.ap.pipeline_service.delete_pipeline(request_context, pipeline_uuid)
|
||||
return self.success()
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.pipeline_service.delete_pipeline(pipeline_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<pipeline_uuid>/copy', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(pipeline_uuid: str) -> str:
|
||||
@self.route(
|
||||
'/<pipeline_uuid>/copy',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
new_uuid = await self.ap.pipeline_service.copy_pipeline(pipeline_uuid)
|
||||
new_uuid = await self.ap.pipeline_service.copy_pipeline(request_context, pipeline_uuid)
|
||||
return self.success(data={'uuid': new_uuid})
|
||||
except ValueError as e:
|
||||
return self.http_status(404, -1, str(e))
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@self.route(
|
||||
'/<pipeline_uuid>/extensions', methods=['GET', 'PUT'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<pipeline_uuid>/extensions',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(pipeline_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
# Get current extensions and available plugins
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
# Only include plugins with pipeline-related components (Command, EventListener, Tool)
|
||||
# Plugins that only have KnowledgeEngine components are not suitable for pipeline extensions
|
||||
pipeline_component_kinds = ['Command', 'EventListener', 'Tool']
|
||||
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True)
|
||||
pipeline_component_kinds = ['Command', 'EventListener', 'Tool']
|
||||
if self.ap.plugin_connector.is_enable_plugin:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugins = await self.ap.plugin_connector.list_plugins(component_kinds=pipeline_component_kinds)
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
available_skills = await self.ap.skill_service.list_skills(request_context)
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
|
||||
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
|
||||
'bound_plugins': extensions_prefs.get('plugins', []),
|
||||
'available_plugins': redact_secrets(plugins),
|
||||
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
|
||||
'available_mcp_servers': mcp_servers,
|
||||
'bound_mcp_resources': extensions_prefs.get('mcp_resources', []),
|
||||
'mcp_resource_agent_read_enabled': extensions_prefs.get('mcp_resource_agent_read_enabled', True),
|
||||
'bound_skills': extensions_prefs.get('skills', []),
|
||||
'available_skills': available_skills,
|
||||
}
|
||||
)
|
||||
|
||||
# Get available skills
|
||||
available_skills = await self.ap.skill_service.list_skills()
|
||||
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
|
||||
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
|
||||
'bound_plugins': extensions_prefs.get('plugins', []),
|
||||
'available_plugins': plugins,
|
||||
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
|
||||
'available_mcp_servers': mcp_servers,
|
||||
'bound_mcp_resources': extensions_prefs.get('mcp_resources', []),
|
||||
'mcp_resource_agent_read_enabled': extensions_prefs.get(
|
||||
'mcp_resource_agent_read_enabled', True
|
||||
),
|
||||
'bound_skills': extensions_prefs.get('skills', []),
|
||||
'available_skills': available_skills,
|
||||
}
|
||||
)
|
||||
elif quart.request.method == 'PUT':
|
||||
# Update bound plugins and MCP servers for this pipeline
|
||||
json_data = await quart.request.json
|
||||
enable_all_plugins = json_data.get('enable_all_plugins', True)
|
||||
enable_all_mcp_servers = json_data.get('enable_all_mcp_servers', True)
|
||||
enable_all_skills = json_data.get('enable_all_skills', True)
|
||||
bound_plugins = json_data.get('bound_plugins', [])
|
||||
bound_mcp_servers = json_data.get('bound_mcp_servers', [])
|
||||
bound_skills = json_data.get('bound_skills', [])
|
||||
bound_mcp_resources = json_data.get('bound_mcp_resources')
|
||||
mcp_resource_agent_read_enabled = json_data.get('mcp_resource_agent_read_enabled')
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
pipeline_uuid,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
enable_all_plugins,
|
||||
enable_all_mcp_servers,
|
||||
bound_skills=bound_skills,
|
||||
enable_all_skills=enable_all_skills,
|
||||
bound_mcp_resources=bound_mcp_resources,
|
||||
mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled,
|
||||
)
|
||||
|
||||
return self.success()
|
||||
@self.route(
|
||||
'/<pipeline_uuid>/extensions',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
request_context,
|
||||
pipeline_uuid,
|
||||
json_data.get('bound_plugins', []),
|
||||
json_data.get('bound_mcp_servers', []),
|
||||
json_data.get('enable_all_plugins', True),
|
||||
json_data.get('enable_all_mcp_servers', True),
|
||||
bound_skills=json_data.get('bound_skills', []),
|
||||
enable_all_skills=json_data.get('enable_all_skills', True),
|
||||
bound_mcp_resources=json_data.get('bound_mcp_resources'),
|
||||
mcp_resource_agent_read_enabled=json_data.get('mcp_resource_agent_read_enabled'),
|
||||
)
|
||||
return self.success()
|
||||
|
||||
@@ -1,64 +1,234 @@
|
||||
"""WebSocket聊天路由 - 支持双向实时通信"""
|
||||
"""Authenticated dashboard WebSocket chat routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, permissions_for_role, require_permission
|
||||
from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from ... import group
|
||||
from ......platform.sources.websocket_manager import ws_connection_manager
|
||||
from ......core.task_boundary import run_in_workspace_uow
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
|
||||
from ......utils import bounded_executor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_AUTH_TIMEOUT_SECONDS = 10.0
|
||||
_DUPLEX_DRAIN_TIMEOUT_SECONDS = 0.25
|
||||
|
||||
|
||||
def create_scoped_duplex_tasks(
|
||||
receive_coro: typing.Coroutine[typing.Any, typing.Any, None],
|
||||
send_coro: typing.Coroutine[typing.Any, typing.Any, None],
|
||||
workspace_uuid: str,
|
||||
) -> tuple[asyncio.Task[None], asyncio.Task[None]]:
|
||||
"""Create both socket directions under one trusted Workspace budget."""
|
||||
|
||||
return (
|
||||
asyncio.create_task(
|
||||
bounded_executor.run_in_blocking_work_scope(
|
||||
receive_coro,
|
||||
workspace_uuid,
|
||||
)
|
||||
),
|
||||
asyncio.create_task(
|
||||
bounded_executor.run_in_blocking_work_scope(
|
||||
send_coro,
|
||||
workspace_uuid,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def wait_for_duplex_tasks(
|
||||
receive_task: asyncio.Task,
|
||||
send_task: asyncio.Task,
|
||||
) -> None:
|
||||
"""Stop the peer direction as soon as either socket task terminates."""
|
||||
|
||||
try:
|
||||
done, _ = await asyncio.wait(
|
||||
{receive_task, send_task},
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
# A receive task may enqueue a terminal authorization/error frame and
|
||||
# then finish. Give the sender a short deterministic drain window
|
||||
# instead of cancelling it before that frame reaches the client.
|
||||
if receive_task in done and not send_task.done():
|
||||
await asyncio.wait(
|
||||
{send_task},
|
||||
timeout=_DUPLEX_DRAIN_TIMEOUT_SECONDS,
|
||||
)
|
||||
finally:
|
||||
for task in (receive_task, send_task):
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(
|
||||
receive_task,
|
||||
send_task,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
|
||||
@group.group_class('websocket_chat', '/api/v1/pipelines/<pipeline_uuid>/ws')
|
||||
class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
async def _authenticate_websocket(self) -> tuple[RequestContext, str]:
|
||||
"""Authenticate the first dashboard WebSocket message.
|
||||
|
||||
Browsers cannot attach the normal Authorization/X-Workspace-Id headers
|
||||
to a WebSocket handshake. The client therefore sends one auth frame
|
||||
immediately after opening the socket; no connection is registered and
|
||||
no runtime object is resolved before this method succeeds.
|
||||
"""
|
||||
|
||||
raw_message = await asyncio.wait_for(quart.websocket.receive(), timeout=_AUTH_TIMEOUT_SECONDS)
|
||||
payload = await asyncio.to_thread(json.loads, raw_message)
|
||||
if not isinstance(payload, dict) or payload.get('type') != 'authenticate':
|
||||
raise ValueError('Authentication is required')
|
||||
|
||||
token = str(payload.get('token') or '').strip()
|
||||
workspace_uuid = str(payload.get('workspace_uuid') or '').strip()
|
||||
if not token or not workspace_uuid:
|
||||
raise ValueError('Authentication is required')
|
||||
|
||||
account, _ = await self._authenticate_account(token)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
if not isinstance(account_uuid, str) or collaboration_service is None:
|
||||
raise ValueError('Workspace authentication is unavailable')
|
||||
|
||||
access = await collaboration_service.resolve_account_workspace(account_uuid, workspace_uuid)
|
||||
request_context = RequestContext(
|
||||
instance_uuid=access.execution.instance_uuid,
|
||||
placement_generation=access.execution.placement_generation,
|
||||
request_id=quart.websocket.headers.get('X-Request-Id') or str(uuid.uuid4()),
|
||||
auth_type=group.AuthType.USER_TOKEN.value,
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.ACCOUNT,
|
||||
account_uuid=account_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=access.workspace.uuid,
|
||||
membership_uuid=access.membership.uuid,
|
||||
role=access.membership.role,
|
||||
permissions=permissions_for_role(access.membership.role),
|
||||
membership_revision=access.membership.projection_revision,
|
||||
),
|
||||
)
|
||||
require_permission(request_context, Permission.RUNTIME_OPERATE)
|
||||
return request_context, token
|
||||
|
||||
async def _revalidate_websocket_authorization(
|
||||
self,
|
||||
request_context: RequestContext,
|
||||
token: str,
|
||||
) -> RequestContext:
|
||||
"""Recheck revocable account, membership, permission, and placement state."""
|
||||
|
||||
account, _ = await self._authenticate_account(token)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
if account_uuid != request_context.account_uuid:
|
||||
raise ValueError('WebSocket account changed')
|
||||
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
if collaboration_service is None or not isinstance(account_uuid, str):
|
||||
raise ValueError('Workspace authentication is unavailable')
|
||||
access = await collaboration_service.resolve_account_workspace(
|
||||
account_uuid,
|
||||
request_context.workspace_uuid,
|
||||
)
|
||||
if (
|
||||
access.workspace.uuid != request_context.workspace_uuid
|
||||
or access.membership.uuid != request_context.workspace.membership_uuid
|
||||
or access.membership.projection_revision != request_context.workspace.membership_revision
|
||||
or access.execution.instance_uuid != request_context.instance_uuid
|
||||
or access.execution.placement_generation != request_context.placement_generation
|
||||
):
|
||||
raise ValueError('WebSocket authorization changed')
|
||||
|
||||
current_context = RequestContext(
|
||||
instance_uuid=access.execution.instance_uuid,
|
||||
placement_generation=access.execution.placement_generation,
|
||||
request_id=request_context.request_id,
|
||||
auth_type=request_context.auth_type,
|
||||
principal=request_context.principal,
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=access.workspace.uuid,
|
||||
membership_uuid=access.membership.uuid,
|
||||
role=access.membership.role,
|
||||
permissions=permissions_for_role(access.membership.role),
|
||||
membership_revision=access.membership.projection_revision,
|
||||
),
|
||||
entitlement_revision=request_context.entitlement_revision,
|
||||
)
|
||||
require_permission(current_context, Permission.RUNTIME_OPERATE)
|
||||
return current_context
|
||||
|
||||
async def _get_scoped_adapter(self, request_context: RequestContext, pipeline_uuid: str):
|
||||
pipeline = await run_in_workspace_uow(
|
||||
self.ap,
|
||||
request_context.workspace_uuid,
|
||||
lambda: self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid),
|
||||
)
|
||||
if pipeline is None:
|
||||
return None
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(request_context)
|
||||
return proxy_bot.adapter
|
||||
|
||||
async def initialize(self) -> None:
|
||||
# 直接使用 quart_app 注册 WebSocket 路由
|
||||
@self.quart_app.websocket(self.path + '/connect')
|
||||
async def websocket_connect(pipeline_uuid: str):
|
||||
"""
|
||||
建立WebSocket连接
|
||||
"""Open one authenticated dashboard debug connection."""
|
||||
|
||||
URL参数:
|
||||
- pipeline_uuid: 流水线UUID
|
||||
- session_type: 会话类型 (person/group)
|
||||
"""
|
||||
await quart.websocket.accept()
|
||||
try:
|
||||
# 获取参数 - 在WebSocket上下文中使用 quart.websocket.args
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
request_context, token = await self._authenticate_websocket()
|
||||
except Exception:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
|
||||
return
|
||||
|
||||
if session_type not in ['person', 'group']:
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
session_type = quart.websocket.args.get('session_type', 'person')
|
||||
if session_type not in ['person', 'group']:
|
||||
await quart.websocket.send(
|
||||
json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Pipeline not found'}))
|
||||
return
|
||||
|
||||
# 获取WebSocket适配器
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
|
||||
if not websocket_adapter:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
|
||||
return
|
||||
|
||||
# Dashboard pipeline-debug sessions must always run under the
|
||||
# built-in websocket_proxy_bot identity. We deliberately do NOT
|
||||
# resolve a web_page_bot owner here — even if one is bound to
|
||||
# the same pipeline, debug requests must not be attributed to
|
||||
# it. The embed widget path (`/api/v1/embed/<bot>/ws/connect`)
|
||||
# is the one that carries the page-bot identity.
|
||||
|
||||
# 注册连接
|
||||
connection = await ws_connection_manager.add_connection(
|
||||
websocket=quart.websocket._get_current_object(),
|
||||
scope=WebSocketScope.from_context(request_context),
|
||||
pipeline_uuid=pipeline_uuid,
|
||||
session_type=session_type,
|
||||
metadata={'user_agent': quart.websocket.headers.get('User-Agent', '')},
|
||||
send_queue_size=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('send_queue_size', 100)
|
||||
),
|
||||
max_connections=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections', 1024)
|
||||
),
|
||||
max_connections_per_workspace=(
|
||||
self.ap.instance_config.data.get('system', {})
|
||||
.get('websocket_retention', {})
|
||||
.get('max_connections_per_workspace', 32)
|
||||
),
|
||||
)
|
||||
|
||||
# 发送连接成功消息
|
||||
await quart.websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
@@ -72,182 +242,188 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f'WebSocket connection established: {connection.connection_id} '
|
||||
f'(pipeline={pipeline_uuid}, session_type={session_type})'
|
||||
f'Dashboard WebSocket connected: {connection.connection_id} '
|
||||
f'(workspace={connection.workspace_uuid}, pipeline={pipeline_uuid}, '
|
||||
f'session_type={session_type})'
|
||||
)
|
||||
|
||||
# 创建接收和发送任务
|
||||
receive_task = asyncio.create_task(self._handle_receive(connection, websocket_adapter))
|
||||
send_task = asyncio.create_task(self._handle_send(connection))
|
||||
|
||||
# 等待任务完成
|
||||
receive_task, send_task = create_scoped_duplex_tasks(
|
||||
self._handle_receive(
|
||||
connection,
|
||||
websocket_adapter,
|
||||
request_context,
|
||||
token,
|
||||
),
|
||||
self._handle_send(connection),
|
||||
request_context.workspace_uuid,
|
||||
)
|
||||
try:
|
||||
await asyncio.gather(receive_task, send_task)
|
||||
except Exception as e:
|
||||
logger.error(f'WebSocket task execution error: {e}')
|
||||
await wait_for_duplex_tasks(receive_task, send_task)
|
||||
except Exception as exc:
|
||||
logger.error(f'WebSocket task execution error: {exc}')
|
||||
finally:
|
||||
# 清理连接
|
||||
await ws_connection_manager.remove_connection(connection.connection_id)
|
||||
logger.debug(f'WebSocket connection cleaned: {connection.connection_id}')
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'WebSocket connection error: {e}', exc_info=True)
|
||||
except Exception:
|
||||
logger.error('Dashboard WebSocket connection error', exc_info=True)
|
||||
try:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': str(e)}))
|
||||
except:
|
||||
await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@self.route('/messages/<session_type>', methods=['GET'])
|
||||
async def get_messages(pipeline_uuid: str, session_type: str) -> str:
|
||||
"""获取消息历史"""
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
@self.route(
|
||||
'/messages/<session_type>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def get_messages(
|
||||
pipeline_uuid: str,
|
||||
session_type: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
messages = websocket_adapter.get_websocket_messages(pipeline_uuid, session_type)
|
||||
return self.success(data={'messages': messages})
|
||||
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
@self.route(
|
||||
'/reset/<session_type>',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def reset_session(
|
||||
pipeline_uuid: str,
|
||||
session_type: str,
|
||||
request_context: RequestContext,
|
||||
) -> str:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
messages = websocket_adapter.get_websocket_messages(pipeline_uuid, session_type)
|
||||
websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
|
||||
if websocket_adapter is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
websocket_adapter.reset_session(pipeline_uuid, session_type)
|
||||
return self.success(data={'message': 'Session reset successfully'})
|
||||
|
||||
return self.success(data={'messages': messages})
|
||||
@self.route(
|
||||
'/connections',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def get_connections(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
if await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid) is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
@self.route('/reset/<session_type>', methods=['POST'])
|
||||
async def reset_session(pipeline_uuid: str, session_type: str) -> str:
|
||||
"""重置会话"""
|
||||
try:
|
||||
if session_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'session_type must be person or group')
|
||||
|
||||
websocket_adapter = self.ap.platform_mgr.websocket_proxy_bot.adapter
|
||||
|
||||
if not websocket_adapter:
|
||||
return self.http_status(404, -1, 'WebSocket adapter not found')
|
||||
|
||||
websocket_adapter.reset_session(pipeline_uuid, session_type)
|
||||
|
||||
return self.success(data={'message': 'Session reset successfully'})
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
@self.route('/connections', methods=['GET'])
|
||||
async def get_connections(pipeline_uuid: str) -> str:
|
||||
"""获取当前连接统计"""
|
||||
try:
|
||||
stats = ws_connection_manager.get_stats()
|
||||
connections = await ws_connection_manager.get_connections_by_pipeline(pipeline_uuid)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'stats': stats,
|
||||
'connections': [
|
||||
{
|
||||
'connection_id': conn.connection_id,
|
||||
'session_type': conn.session_type,
|
||||
'created_at': conn.created_at.isoformat(),
|
||||
'last_active': conn.last_active.isoformat(),
|
||||
'is_active': conn.is_active,
|
||||
}
|
||||
for conn in connections
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
|
||||
@self.route('/broadcast', methods=['POST'])
|
||||
async def broadcast_message(pipeline_uuid: str) -> str:
|
||||
"""向所有连接广播消息(后端主动推送)"""
|
||||
try:
|
||||
data = await quart.request.get_json()
|
||||
message = data.get('message')
|
||||
|
||||
if not message:
|
||||
return self.http_status(400, -1, 'message is required')
|
||||
|
||||
# 广播消息
|
||||
broadcast_data = {
|
||||
'type': 'broadcast',
|
||||
'message': message,
|
||||
'timestamp': datetime.datetime.now().isoformat(),
|
||||
scope = WebSocketScope.from_context(request_context)
|
||||
stats = ws_connection_manager.get_stats(scope=scope)
|
||||
connections = await ws_connection_manager.get_connections_by_pipeline(
|
||||
pipeline_uuid,
|
||||
scope=scope,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'stats': stats,
|
||||
'connections': [
|
||||
{
|
||||
'connection_id': connection.connection_id,
|
||||
'session_type': connection.session_type,
|
||||
'created_at': connection.created_at.isoformat(),
|
||||
'last_active': connection.last_active.isoformat(),
|
||||
'is_active': connection.is_active,
|
||||
}
|
||||
for connection in connections
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
await ws_connection_manager.broadcast_to_pipeline(pipeline_uuid, broadcast_data)
|
||||
@self.route(
|
||||
'/broadcast',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def broadcast_message(pipeline_uuid: str, request_context: RequestContext) -> str:
|
||||
if await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid) is None:
|
||||
return self.http_status(404, -1, 'Pipeline not found')
|
||||
|
||||
return self.success(data={'message': 'Broadcast sent successfully'})
|
||||
data = await quart.request.get_json()
|
||||
message = data.get('message')
|
||||
if not message:
|
||||
return self.http_status(400, -1, 'message is required')
|
||||
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Internal server error: {str(e)}')
|
||||
broadcast_data = {
|
||||
'type': 'broadcast',
|
||||
'message': message,
|
||||
'timestamp': datetime.datetime.now().isoformat(),
|
||||
}
|
||||
await ws_connection_manager.broadcast_to_pipeline(
|
||||
pipeline_uuid,
|
||||
broadcast_data,
|
||||
scope=WebSocketScope.from_context(request_context),
|
||||
)
|
||||
return self.success(data={'message': 'Broadcast sent successfully'})
|
||||
|
||||
async def _handle_receive(self, connection, websocket_adapter):
|
||||
"""处理接收消息的任务"""
|
||||
async def _handle_receive(
|
||||
self,
|
||||
connection,
|
||||
websocket_adapter,
|
||||
request_context: RequestContext,
|
||||
token: str,
|
||||
):
|
||||
try:
|
||||
while connection.is_active:
|
||||
# 接收消息
|
||||
message = await quart.websocket.receive()
|
||||
|
||||
# 更新活跃时间
|
||||
await ws_connection_manager.update_activity(connection.connection_id)
|
||||
|
||||
try:
|
||||
data = json.loads(message)
|
||||
data = await asyncio.to_thread(json.loads, message)
|
||||
message_type = data.get('type', 'message')
|
||||
|
||||
if message_type == 'ping':
|
||||
# 心跳响应
|
||||
await connection.send_queue.put(
|
||||
{'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
|
||||
)
|
||||
|
||||
elif message_type == 'message':
|
||||
# 处理用户消息
|
||||
logger.debug(f'收到消息: {data} from {connection.connection_id}')
|
||||
|
||||
# 处理消息(不等待响应,响应会通过broadcast异步发送)
|
||||
# owner_bot is intentionally NOT passed: the dashboard
|
||||
# debug WebSocket must always run under the proxy bot,
|
||||
# never under a coincidentally-bound web_page_bot.
|
||||
try:
|
||||
await self._revalidate_websocket_authorization(request_context, token)
|
||||
except Exception:
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
|
||||
break
|
||||
await websocket_adapter.handle_websocket_message(connection, data)
|
||||
|
||||
elif message_type == 'disconnect':
|
||||
# 客户端主动断开
|
||||
logger.debug(f'Client disconnected: {connection.connection_id}')
|
||||
break
|
||||
|
||||
else:
|
||||
logger.warning(f'Unknown message type: {message_type}')
|
||||
|
||||
logger.warning(f'Unknown WebSocket message type: {message_type}')
|
||||
except json.JSONDecodeError:
|
||||
logger.error(f'Invalid JSON message: {message}')
|
||||
await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Receive message error: {e}', exc_info=True)
|
||||
except Exception:
|
||||
logger.error('Dashboard WebSocket receive error', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
try:
|
||||
connection.send_queue.put_nowait(None)
|
||||
except asyncio.QueueFull:
|
||||
pass
|
||||
|
||||
async def _handle_send(self, connection):
|
||||
"""处理发送消息的任务"""
|
||||
try:
|
||||
while connection.is_active:
|
||||
# 从队列获取消息
|
||||
while connection.is_active or not connection.send_queue.empty():
|
||||
try:
|
||||
message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
|
||||
|
||||
# 发送消息
|
||||
await quart.websocket.send(json.dumps(message))
|
||||
|
||||
if message is None:
|
||||
break
|
||||
encoded = await asyncio.to_thread(json.dumps, message)
|
||||
await quart.websocket.send(encoded)
|
||||
except asyncio.TimeoutError:
|
||||
# 超时继续循环
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f'Send message error: {e}', exc_info=True)
|
||||
except Exception:
|
||||
logger.error('Dashboard WebSocket send error', exc_info=True)
|
||||
finally:
|
||||
connection.is_active = False
|
||||
|
||||
@@ -1,8 +1,133 @@
|
||||
import quart
|
||||
import mimetypes
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import mimetypes
|
||||
|
||||
import quart
|
||||
|
||||
from langbot.pkg.api.http.authz import Permission
|
||||
from langbot.pkg.api.http.context import RequestContext
|
||||
from langbot.pkg.core.errors import TaskCapacityError
|
||||
from langbot.pkg.utils import httpclient, importutil
|
||||
|
||||
from ... import group
|
||||
from langbot.pkg.utils import importutil
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class _AdapterSessionScope:
|
||||
"""Immutable tenant and principal binding for a credential exchange."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
principal_type: str
|
||||
account_uuid: str | None
|
||||
api_key_uuid: str | None
|
||||
|
||||
@classmethod
|
||||
def from_request_context(cls, request_context: RequestContext) -> '_AdapterSessionScope':
|
||||
principal = request_context.principal
|
||||
return cls(
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
principal_type=principal.principal_type.value,
|
||||
account_uuid=principal.account_uuid,
|
||||
api_key_uuid=principal.api_key_uuid,
|
||||
)
|
||||
|
||||
def matches(self, request_context: RequestContext) -> bool:
|
||||
"""Return whether a request is from the exact initiating tenant principal."""
|
||||
|
||||
return self == self.from_request_context(request_context)
|
||||
|
||||
|
||||
def _bind_session_scope(session: dict, request_context: RequestContext) -> None:
|
||||
session['scope'] = _AdapterSessionScope.from_request_context(request_context)
|
||||
|
||||
|
||||
def _get_owned_session(
|
||||
sessions: dict[str, dict],
|
||||
session_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> dict | None:
|
||||
"""Resolve a session without revealing sessions owned by another scope."""
|
||||
|
||||
session = sessions.get(session_id)
|
||||
scope = session.get('scope') if session is not None else None
|
||||
if not isinstance(scope, _AdapterSessionScope) or not scope.matches(request_context):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def _pop_owned_session(
|
||||
sessions: dict[str, dict],
|
||||
session_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> dict | None:
|
||||
"""Remove an owned session without allowing cross-scope cancellation."""
|
||||
|
||||
session = _get_owned_session(sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return None
|
||||
return sessions.pop(session_id, None)
|
||||
|
||||
|
||||
_MAX_ADAPTER_SESSIONS = 100
|
||||
_MAX_ADAPTER_SESSIONS_PER_WORKSPACE = 10
|
||||
|
||||
|
||||
def _start_adapter_session_task(
|
||||
ap,
|
||||
coro,
|
||||
*,
|
||||
adapter: str,
|
||||
session_id: str,
|
||||
request_context: RequestContext,
|
||||
) -> asyncio.Task | None:
|
||||
"""Attach one credential exchange to tenant admission and app shutdown."""
|
||||
|
||||
try:
|
||||
wrapper = ap.task_mgr.create_user_task(
|
||||
coro,
|
||||
kind='platform-adapter-credential-exchange',
|
||||
name=f'{adapter}-credential-{session_id}',
|
||||
label=f'{adapter} credential exchange',
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
)
|
||||
except TaskCapacityError:
|
||||
coro.close()
|
||||
return None
|
||||
return wrapper.task
|
||||
|
||||
|
||||
def _make_room_for_session(
|
||||
sessions: dict[str, dict],
|
||||
request_context: RequestContext,
|
||||
) -> None:
|
||||
"""Bound credential-exchange sessions globally and per workspace."""
|
||||
|
||||
workspace_uuid = request_context.workspace_uuid
|
||||
owned = [
|
||||
(session_id, session)
|
||||
for session_id, session in sessions.items()
|
||||
if getattr(session.get('scope'), 'workspace_uuid', None) == workspace_uuid
|
||||
]
|
||||
evict_workspace_session = len(owned) >= _MAX_ADAPTER_SESSIONS_PER_WORKSPACE
|
||||
evict_global_session = len(sessions) >= _MAX_ADAPTER_SESSIONS
|
||||
if not evict_workspace_session and not evict_global_session:
|
||||
return
|
||||
|
||||
candidates = owned if evict_workspace_session else list(sessions.items())
|
||||
session_id, _ = min(
|
||||
candidates,
|
||||
key=lambda item: float(item[1].get('created_at', 0.0)),
|
||||
)
|
||||
session = sessions.pop(session_id, None)
|
||||
task = session.get('task') if session is not None else None
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
|
||||
|
||||
def _decrypt_qqofficial_secret(encrypted_b64: str, key: bytes) -> str:
|
||||
@@ -84,8 +209,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/lark/create-app', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/lark/create-app', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start Feishu one-click app registration. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -106,6 +231,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_create_app_sessions, request_context)
|
||||
_create_app_sessions[session_id] = session
|
||||
|
||||
def on_qr_code(info):
|
||||
@@ -137,7 +264,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_registration())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_registration(),
|
||||
adapter='lark',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_create_app_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -160,10 +296,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/lark/create-app/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/lark/create-app/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll registration status."""
|
||||
session = _create_app_sessions.get(session_id)
|
||||
_cleanup_expired_sessions()
|
||||
session = _get_owned_session(_create_app_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -179,10 +320,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/lark/create-app/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/lark/create-app/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a registration session."""
|
||||
session = _create_app_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_create_app_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -206,8 +353,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/weixin/login', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/weixin/login', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start WeChat QR code login. Returns session_id + QR code data URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -229,6 +376,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'error': None,
|
||||
'created_at': time.time(),
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_weixin_login_sessions, request_context)
|
||||
_weixin_login_sessions[session_id] = session
|
||||
|
||||
client = OpenClawWeixinClient(
|
||||
@@ -267,7 +416,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
task = asyncio.create_task(run_login())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_login(),
|
||||
adapter='weixin',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_weixin_login_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -290,10 +448,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/weixin/login/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/weixin/login/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll WeChat login status."""
|
||||
session = _weixin_login_sessions.get(session_id)
|
||||
_cleanup_expired_weixin_sessions()
|
||||
session = _get_owned_session(_weixin_login_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -317,10 +480,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/weixin/login/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/weixin/login/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a WeChat login session."""
|
||||
session = _weixin_login_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_weixin_login_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -344,8 +513,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/dingtalk/create-app', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/dingtalk/create-app', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start DingTalk one-click app creation via Device Flow. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -368,6 +537,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'device_code': None,
|
||||
'interval': 5,
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_dingtalk_sessions, request_context)
|
||||
_dingtalk_sessions[session_id] = session
|
||||
|
||||
async def run_device_flow():
|
||||
@@ -380,7 +551,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
json={'source': 'langbot'},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from DingTalk service'
|
||||
@@ -397,7 +568,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
json={'nonce': nonce},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from DingTalk service'
|
||||
@@ -428,7 +599,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
json={'device_code': device_code},
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json()
|
||||
poll_data = await httpclient.read_json_limited(poll_resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -464,7 +635,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_device_flow())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_device_flow(),
|
||||
adapter='dingtalk',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_dingtalk_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -491,11 +671,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/dingtalk/create-app/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/dingtalk/create-app/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll DingTalk Device Flow status."""
|
||||
_cleanup_expired_dingtalk_sessions()
|
||||
session = _dingtalk_sessions.get(session_id)
|
||||
session = _get_owned_session(_dingtalk_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -511,10 +695,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/dingtalk/create-app/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/dingtalk/create-app/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a DingTalk Device Flow session."""
|
||||
session = _dingtalk_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_dingtalk_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -538,8 +728,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/wecombot/create-bot', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/wecombot/create-bot', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start WeComBot one-click creation via QR code. Returns session_id + QR code URL."""
|
||||
import uuid
|
||||
import time
|
||||
@@ -563,6 +753,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'scode': None,
|
||||
'task': None,
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_wecombot_sessions, request_context)
|
||||
_wecombot_sessions[session_id] = session
|
||||
|
||||
async def run_qr_flow():
|
||||
@@ -574,7 +766,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
f'{WECOM_QC_GENERATE_URL}?source=langbot&plat=0',
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json()
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from WeCom service'
|
||||
@@ -601,7 +793,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
f'{WECOM_QC_QUERY_URL}?scode={scode}',
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json()
|
||||
poll_data = await httpclient.read_json_limited(poll_resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -628,7 +820,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_qr_flow())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_qr_flow(),
|
||||
adapter='wecombot',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_wecombot_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait for QR code to be ready (max 10 seconds)
|
||||
@@ -655,11 +856,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/wecombot/create-bot/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/wecombot/create-bot/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll WeComBot creation status."""
|
||||
_cleanup_expired_wecombot_sessions()
|
||||
session = _wecombot_sessions.get(session_id)
|
||||
session = _get_owned_session(_wecombot_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -675,10 +880,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/wecombot/create-bot/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/wecombot/create-bot/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a WeComBot creation session."""
|
||||
session = _wecombot_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_wecombot_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
@@ -702,8 +913,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
|
||||
@self.route('/qqofficial/bind', methods=['POST'])
|
||||
async def _() -> str:
|
||||
@self.route('/qqofficial/bind', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Start QQ Official QR binding. Returns session_id + QR URL.
|
||||
|
||||
Flow: generate a local AES-256 key, register it with
|
||||
@@ -739,6 +950,8 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
'bind_key_bytes': bind_key_bytes,
|
||||
'interval': 2,
|
||||
}
|
||||
_bind_session_scope(session, request_context)
|
||||
_make_room_for_session(_qqofficial_sessions, request_context)
|
||||
_qqofficial_sessions[session_id] = session
|
||||
|
||||
async def run_qr_binding():
|
||||
@@ -752,7 +965,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
headers={'Accept': 'application/json'},
|
||||
) as resp:
|
||||
try:
|
||||
data = await resp.json(content_type=None)
|
||||
data = await httpclient.read_json_limited(resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
session['status'] = 'error'
|
||||
session['error'] = 'Invalid response from QQ bind service'
|
||||
@@ -790,7 +1003,7 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
headers={'Accept': 'application/json'},
|
||||
) as poll_resp:
|
||||
try:
|
||||
poll_data = await poll_resp.json(content_type=None)
|
||||
poll_data = await httpclient.read_json_limited(poll_resp)
|
||||
except (aiohttp.ContentTypeError, ValueError):
|
||||
continue
|
||||
|
||||
@@ -843,7 +1056,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
session['status'] = 'error'
|
||||
session['error'] = str(e)
|
||||
|
||||
task = asyncio.create_task(run_qr_binding())
|
||||
task = _start_adapter_session_task(
|
||||
self.ap,
|
||||
run_qr_binding(),
|
||||
adapter='qqofficial',
|
||||
session_id=session_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
if task is None:
|
||||
_qqofficial_sessions.pop(session_id, None)
|
||||
return self.http_status(429, -1, 'Too many active credential exchanges')
|
||||
session['task'] = task
|
||||
|
||||
# Wait up to 10s for the QR URL to be ready before responding.
|
||||
@@ -870,11 +1092,15 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/qqofficial/bind/status/<session_id>', methods=['GET'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/qqofficial/bind/status/<session_id>',
|
||||
methods=['GET'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Poll QQ Official QR binding status."""
|
||||
_cleanup_expired_qqofficial_sessions()
|
||||
session = _qqofficial_sessions.get(session_id)
|
||||
session = _get_owned_session(_qqofficial_sessions, session_id, request_context)
|
||||
if not session:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
|
||||
@@ -892,10 +1118,16 @@ class AdaptersRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data=data)
|
||||
|
||||
@self.route('/qqofficial/bind/<session_id>', methods=['DELETE'])
|
||||
async def _(session_id: str) -> str:
|
||||
@self.route(
|
||||
'/qqofficial/bind/<session_id>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(session_id: str, request_context: RequestContext) -> str:
|
||||
"""Cancel and clean up a QQ Official QR binding session."""
|
||||
session = _qqofficial_sessions.pop(session_id, None)
|
||||
session = _pop_owned_session(_qqofficial_sessions, session_id, request_context)
|
||||
if session is None:
|
||||
return self.http_status(404, -1, 'Session not found')
|
||||
if session and session.get('task') and not session['task'].done():
|
||||
session['task'].cancel()
|
||||
return self.success(data={})
|
||||
|
||||
@@ -1,45 +1,95 @@
|
||||
import quart
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('bots', '/api/v1/platform/bots')
|
||||
class BotsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
return self.success(data={'bots': await self.ap.bot_service.get_bots()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
bot_uuid = await self.ap.bot_service.create_bot(json_data)
|
||||
return self.success(data={'uuid': bot_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
include_secret = has_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
return self.success(
|
||||
data={
|
||||
'bots': await self.ap.bot_service.get_bots(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<bot_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
bot = await self.ap.bot_service.get_runtime_bot_info(bot_uuid)
|
||||
if bot is None:
|
||||
return self.http_status(404, -1, 'bot not found')
|
||||
return self.success(data={'bot': bot})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.bot_service.update_bot(bot_uuid, json_data)
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.bot_service.delete_bot(bot_uuid)
|
||||
return self.success()
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
bot_uuid = await self.ap.bot_service.create_bot(request_context, json_data)
|
||||
return self.success(data={'uuid': bot_uuid})
|
||||
|
||||
@self.route('/<bot_uuid>/logs', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
@self.route(
|
||||
'/<bot_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
include_secret = has_permission(request_context, Permission.RESOURCE_MANAGE)
|
||||
bot = await self.ap.bot_service.get_runtime_bot_info(
|
||||
request_context,
|
||||
bot_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
if bot is None:
|
||||
return self.http_status(404, -1, 'bot not found')
|
||||
return self.success(data={'bot': bot})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.bot_service.update_bot(request_context, bot_uuid, json_data)
|
||||
else:
|
||||
await self.ap.bot_service.delete_bot(request_context, bot_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/logs',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
from_index = json_data.get('from_index', -1)
|
||||
max_count = json_data.get('max_count', 10)
|
||||
logs, total_count = await self.ap.bot_service.list_event_logs(bot_uuid, from_index, max_count)
|
||||
logs, total_count = await self.ap.bot_service.list_event_logs(
|
||||
request_context, bot_uuid, from_index, max_count
|
||||
)
|
||||
return self.success(data={'logs': logs, 'total_count': total_count})
|
||||
|
||||
@self.route('/<bot_uuid>/send_message', methods=['POST'], auth_type=group.AuthType.API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
@self.route(
|
||||
'/<bot_uuid>/send_message',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.API_KEY,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
target_type = json_data.get('target_type')
|
||||
target_id = json_data.get('target_id')
|
||||
@@ -54,37 +104,51 @@ class BotsRouterGroup(group.RouterGroup):
|
||||
if target_type not in ['person', 'group']:
|
||||
return self.http_status(400, -1, 'target_type must be either "person" or "group"')
|
||||
|
||||
try:
|
||||
await self.ap.bot_service.send_message(bot_uuid, target_type, target_id, message_chain_data)
|
||||
return self.success(data={'sent': True})
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
return self.http_status(500, -1, f'Failed to send message: {str(e)}')
|
||||
|
||||
# ============ Bot Admins ============
|
||||
|
||||
@self.route('/<bot_uuid>/admins', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(bot_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
admins = await self.ap.bot_service.get_bot_admins(bot_uuid)
|
||||
return self.success(data={'admins': admins})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
launcher_type = json_data.get('launcher_type', '').strip()
|
||||
launcher_id = str(json_data.get('launcher_id', '')).strip()
|
||||
if not launcher_type or not launcher_id:
|
||||
return self.http_status(400, -1, 'launcher_type and launcher_id are required')
|
||||
try:
|
||||
admin_id = await self.ap.bot_service.add_bot_admin(bot_uuid, launcher_type, launcher_id)
|
||||
return self.success(data={'id': admin_id})
|
||||
except Exception as e:
|
||||
return self.http_status(409, -1, str(e))
|
||||
await self.ap.bot_service.send_message(
|
||||
request_context,
|
||||
bot_uuid,
|
||||
target_type,
|
||||
target_id,
|
||||
message_chain_data,
|
||||
)
|
||||
return self.success(data={'sent': True})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/admins/<int:admin_id>', methods=['DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<bot_uuid>/admins',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(bot_uuid: str, admin_id: int) -> str:
|
||||
await self.ap.bot_service.delete_bot_admin(bot_uuid, admin_id)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
admins = await self.ap.bot_service.get_bot_admins(request_context, bot_uuid)
|
||||
return self.success(data={'admins': admins})
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/admins',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(bot_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
launcher_type = json_data.get('launcher_type', '').strip()
|
||||
launcher_id = str(json_data.get('launcher_id', '')).strip()
|
||||
if not launcher_type or not launcher_id:
|
||||
return self.http_status(400, -1, 'launcher_type and launcher_id are required')
|
||||
try:
|
||||
admin_id = await self.ap.bot_service.add_bot_admin(
|
||||
request_context, bot_uuid, launcher_type, launcher_id
|
||||
)
|
||||
return self.success(data={'id': admin_id})
|
||||
except IntegrityError as e:
|
||||
return self.http_status(409, -1, str(e))
|
||||
|
||||
@self.route(
|
||||
'/<bot_uuid>/admins/<int:admin_id>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(bot_uuid: str, admin_id: int, request_context: RequestContext) -> str:
|
||||
await self.ap.bot_service.delete_bot_admin(request_context, bot_uuid, admin_id)
|
||||
return self.success()
|
||||
|
||||
@@ -1,23 +1,158 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import collections.abc
|
||||
import copy
|
||||
import quart
|
||||
import re
|
||||
import httpx
|
||||
import uuid
|
||||
import os
|
||||
import zipfile
|
||||
import yaml
|
||||
from urllib.parse import urlparse
|
||||
import posixpath
|
||||
import sqlalchemy
|
||||
|
||||
from .....core import taskmgr
|
||||
from .....core.task_boundary import run_in_workspace_uow
|
||||
from .....entity.persistence import plugin as persistence_plugin
|
||||
from ...authz import Permission
|
||||
from ...context import ExecutionContext, RequestContext
|
||||
from .. import group
|
||||
from .....workspace.errors import WorkspaceNotFoundError
|
||||
from .....plugin.github import validate_github_plugin_install_info
|
||||
from .....plugin.archive import inspect_plugin_archive_metadata
|
||||
from .....utils import httpclient
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
|
||||
_SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
_SENSITIVE_CONFIG_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'apikey',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_CONFIG_TOKENS = frozenset(
|
||||
{
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def _is_sensitive_config_key(key: object) -> bool:
|
||||
normalized = _normalize_config_key(key)
|
||||
if normalized in _SENSITIVE_CONFIG_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_CONFIG_TOKENS:
|
||||
return True
|
||||
return 'key' in tokens and bool(tokens & _SENSITIVE_KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def _mask_secret_structure(value):
|
||||
"""Mask every non-empty leaf while preserving container structure."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: _mask_secret_structure(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_mask_secret_structure(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_mask_secret_structure(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return _SECRET_MASK
|
||||
|
||||
|
||||
def redact_plugin_secrets(value):
|
||||
"""Return a recursively redacted copy of plugin-facing data."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (_mask_secret_structure(item) if _is_sensitive_config_key(key) else redact_plugin_secrets(item))
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_plugin_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_plugin_secrets(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def restore_plugin_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from the current config before a management write."""
|
||||
|
||||
if sensitive and value == _SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked plugin secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: restore_plugin_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or _is_sensitive_config_key(key),
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_plugin_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_plugin_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
# Resolve the built-in page SDK JS from the langbot_plugin package
|
||||
_PAGE_SDK_PATH = None
|
||||
try:
|
||||
@@ -148,18 +283,78 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'subdir': subdir,
|
||||
}
|
||||
|
||||
async def _check_extensions_limit(self) -> str | None:
|
||||
async def _check_extensions_limit(self, request_context: RequestContext) -> str | None:
|
||||
"""Check if extensions limit is reached. Returns error response if limit exceeded, None otherwise."""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_extensions = limitation.get('max_extensions', -1)
|
||||
if max_extensions >= 0:
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers()
|
||||
mcp_servers = await self.ap.mcp_service.get_mcp_servers(request_context)
|
||||
total_extensions = len(plugins) + len(mcp_servers)
|
||||
if total_extensions >= max_extensions:
|
||||
return self.http_status(400, -1, f'Maximum number of extensions ({max_extensions}) reached')
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _task_scope(request_context: RequestContext) -> dict[str, str | int]:
|
||||
return {
|
||||
'instance_uuid': request_context.instance_uuid,
|
||||
'workspace_uuid': request_context.workspace_uuid,
|
||||
'placement_generation': request_context.placement_generation,
|
||||
}
|
||||
|
||||
async def _run_fenced_plugin_operation(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
operation: collections.abc.Callable[[], collections.abc.Awaitable],
|
||||
):
|
||||
"""Revalidate a captured task context immediately before Runtime I/O."""
|
||||
|
||||
await run_in_workspace_uow(
|
||||
self.ap,
|
||||
execution_context.workspace_uuid,
|
||||
lambda: self.ap.plugin_connector.require_workspace_context(execution_context),
|
||||
)
|
||||
return await operation()
|
||||
|
||||
async def _require_public_plugin_runtime_context(self) -> ExecutionContext:
|
||||
"""Resolve public assets only for the OSS singleton Workspace.
|
||||
|
||||
Public image and iframe requests cannot carry the WebUI bearer token.
|
||||
They therefore remain available for the one-Workspace Core deployment,
|
||||
but fail closed instead of guessing a Workspace when multi-Workspace
|
||||
policy is active.
|
||||
"""
|
||||
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
policy = getattr(workspace_service, 'policy', None)
|
||||
if workspace_service is None or policy is None or getattr(policy, 'multi_workspace_enabled', False):
|
||||
raise WorkspaceNotFoundError('Plugin resource not found')
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
execution_context = ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
)
|
||||
return await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
|
||||
async def _get_stored_plugin_config(
|
||||
self,
|
||||
request_context: RequestContext,
|
||||
author: str,
|
||||
plugin_name: str,
|
||||
plugin: dict,
|
||||
):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting.config)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == request_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
)
|
||||
persisted_config = result.scalar_one_or_none()
|
||||
return persisted_config if persisted_config is not None else plugin['plugin_config']
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/_sdk/page-sdk.js', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> quart.Response:
|
||||
@@ -170,15 +365,27 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return quart.Response(content, mimetype='application/javascript')
|
||||
return quart.Response('// SDK not found', status=404, mimetype='application/javascript')
|
||||
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
|
||||
return self.success(data={'plugins': plugins})
|
||||
return self.success(data={'plugins': redact_plugin_secrets(plugins)})
|
||||
|
||||
@self.route('/debug-info', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/debug-info',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get plugin debug information including debug URL and key"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
debug_info = await self.ap.plugin_connector.get_debug_info()
|
||||
|
||||
# Get debug URL from config
|
||||
@@ -196,77 +403,121 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/<author>/<plugin_name>/upgrade',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.upgrade_plugin(author, plugin_name, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.upgrade_plugin(author, plugin_name, task_context=ctx),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name=f'plugin-upgrade-{plugin_name}',
|
||||
label=f'Upgrading plugin {plugin_name}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>',
|
||||
methods=['GET', 'DELETE'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
return self.success(data={'plugin': plugin})
|
||||
elif quart.request.method == 'DELETE':
|
||||
delete_data = quart.request.args.get('delete_data', 'false').lower() == 'true'
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.delete_plugin(
|
||||
author, plugin_name, delete_data=delete_data, task_context=ctx
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name=f'plugin-remove-{plugin_name}',
|
||||
label=f'Removing plugin {plugin_name}',
|
||||
context=ctx,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
return self.success(data={'plugin': redact_plugin_secrets(plugin)})
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
delete_data = quart.request.args.get('delete_data', 'false').lower() == 'true'
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.delete_plugin(
|
||||
author,
|
||||
plugin_name,
|
||||
delete_data=delete_data,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name=f'plugin-remove-{plugin_name}',
|
||||
label=f'Removing plugin {plugin_name}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/config',
|
||||
methods=['GET', 'PUT'],
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
|
||||
if quart.request.method == 'GET':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting.config)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
config = await self._get_stored_plugin_config(
|
||||
request_context,
|
||||
author,
|
||||
plugin_name,
|
||||
plugin,
|
||||
)
|
||||
return self.success(data={'config': redact_plugin_secrets(config)})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/config',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin = await self.ap.plugin_connector.get_plugin_info(author, plugin_name)
|
||||
if plugin is None:
|
||||
return self.http_status(404, -1, 'plugin not found')
|
||||
current_config = await self._get_stored_plugin_config(
|
||||
request_context,
|
||||
author,
|
||||
plugin_name,
|
||||
plugin,
|
||||
)
|
||||
try:
|
||||
config = restore_plugin_secret_placeholders(
|
||||
await quart.request.json,
|
||||
current_config,
|
||||
)
|
||||
persisted_config = result.scalar_one_or_none()
|
||||
|
||||
config = persisted_config if persisted_config is not None else plugin['plugin_config']
|
||||
return self.success(data={'config': config})
|
||||
elif quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
|
||||
await self.ap.plugin_connector.set_plugin_config(author, plugin_name, data)
|
||||
|
||||
return self.success(data={})
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
await self.ap.plugin_connector.set_plugin_config(author, plugin_name, config)
|
||||
return self.success(data={})
|
||||
|
||||
@self.route(
|
||||
'/<author>/<plugin_name>/readme',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
language = quart.request.args.get('language', 'en')
|
||||
readme = await self.ap.plugin_connector.get_plugin_readme(author, plugin_name, language=language)
|
||||
return self.success(data={'readme': readme})
|
||||
@@ -275,8 +526,10 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/<author>/<plugin_name>/logs',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> quart.Response:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
try:
|
||||
limit = int(quart.request.args.get('limit', 200))
|
||||
except (TypeError, ValueError):
|
||||
@@ -291,11 +544,12 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
auth_type=group.AuthType.NONE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> quart.Response:
|
||||
await self._require_public_plugin_runtime_context()
|
||||
icon_data = await self.ap.plugin_connector.get_plugin_icon(author, plugin_name)
|
||||
icon_base64 = icon_data['plugin_icon_base64']
|
||||
mime_type = icon_data['mime_type']
|
||||
|
||||
icon_data = base64.b64decode(icon_base64)
|
||||
icon_data = await asyncio.to_thread(base64.b64decode, icon_base64)
|
||||
|
||||
return quart.Response(icon_data, mimetype=mime_type)
|
||||
|
||||
@@ -305,6 +559,7 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
auth_type=group.AuthType.NONE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str, filepath: str) -> quart.Response:
|
||||
await self._require_public_plugin_runtime_context()
|
||||
asset_path = _normalize_plugin_asset_path(filepath)
|
||||
if asset_path is None:
|
||||
return quart.Response('Asset not found', status=404)
|
||||
@@ -312,7 +567,10 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
asset_data = await self.ap.plugin_connector.get_plugin_assets(author, plugin_name, asset_path)
|
||||
if not asset_data.get('asset_base64'):
|
||||
return quart.Response('Asset not found', status=404)
|
||||
asset_bytes = base64.b64decode(asset_data['asset_base64'])
|
||||
asset_bytes = await asyncio.to_thread(
|
||||
base64.b64decode,
|
||||
asset_data['asset_base64'],
|
||||
)
|
||||
mime_type = asset_data['mime_type']
|
||||
resp = quart.Response(asset_bytes, mimetype=mime_type)
|
||||
# CSP for HTML pages served to sandboxed iframes (opaque origin).
|
||||
@@ -334,9 +592,11 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/<author>/<plugin_name>/page-api',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(author: str, plugin_name: str) -> str:
|
||||
async def _(author: str, plugin_name: str, request_context: RequestContext) -> str:
|
||||
"""Forward a page API request to the plugin."""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
data = await quart.request.json
|
||||
if not isinstance(data, dict):
|
||||
return self.http_status(400, -1, 'invalid request body')
|
||||
@@ -357,9 +617,15 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, result['error'])
|
||||
return self.success(data=result.get('data'))
|
||||
|
||||
@self.route('/github/releases', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/github/releases',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get releases from a GitHub repository URL"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
data = await quart.request.json
|
||||
repo_url = data.get('repo_url', '')
|
||||
|
||||
@@ -400,10 +666,11 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
trust_env=True,
|
||||
follow_redirects=True,
|
||||
timeout=10,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(),
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
releases = response.json()
|
||||
releases = await httpclient.parse_json_response(response)
|
||||
|
||||
# Format releases data for frontend
|
||||
formatted_releases = []
|
||||
@@ -427,16 +694,18 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'source_subdir': requested_subdir,
|
||||
}
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
return self.http_status(500, -1, f'Failed to fetch releases: {str(e)}')
|
||||
except httpx.RequestError:
|
||||
raise
|
||||
|
||||
@self.route(
|
||||
'/github/release-assets',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _() -> str:
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get assets from a specific GitHub release"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
data = await quart.request.json
|
||||
owner = data.get('owner', '')
|
||||
repo = data.get('repo', '')
|
||||
@@ -452,12 +721,13 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
trust_env=True,
|
||||
follow_redirects=True,
|
||||
timeout=10,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(),
|
||||
) as client:
|
||||
response = await client.get(
|
||||
url,
|
||||
)
|
||||
response.raise_for_status()
|
||||
release = response.json()
|
||||
release = await httpclient.parse_json_response(response)
|
||||
|
||||
# Format assets data for frontend
|
||||
formatted_assets = []
|
||||
@@ -484,42 +754,61 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
# )
|
||||
|
||||
return self.success(data={'assets': formatted_assets})
|
||||
except httpx.RequestError as e:
|
||||
return self.http_status(500, -1, f'Failed to fetch release assets: {str(e)}')
|
||||
except httpx.RequestError:
|
||||
raise
|
||||
|
||||
@self.route('/install/github', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/install/github',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Install plugin from GitHub release asset"""
|
||||
limit_error = await self._check_extensions_limit()
|
||||
limit_error = await self._check_extensions_limit(request_context)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
data = await quart.request.json
|
||||
asset_url = data.get('asset_url', '')
|
||||
owner = data.get('owner', '')
|
||||
repo = data.get('repo', '')
|
||||
release_tag = data.get('release_tag', '')
|
||||
data = await quart.request.json or {}
|
||||
try:
|
||||
install_info = validate_github_plugin_install_info(
|
||||
{
|
||||
'asset_url': data.get('asset_url'),
|
||||
'asset_id': data.get('asset_id'),
|
||||
'release_id': data.get('release_id'),
|
||||
'owner': data.get('owner'),
|
||||
'repo': data.get('repo'),
|
||||
'release_tag': data.get('release_tag'),
|
||||
'github_url': f'https://github.com/{data.get("owner", "")}/{data.get("repo", "")}',
|
||||
}
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
if not asset_url:
|
||||
return self.http_status(400, -1, 'Missing asset_url parameter')
|
||||
owner = install_info['owner']
|
||||
repo = install_info['repo']
|
||||
release_tag = install_info['release_tag']
|
||||
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{owner}/{repo}'
|
||||
ctx.metadata['install_source'] = 'github'
|
||||
install_info = {
|
||||
'asset_url': asset_url,
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'release_tag': release_tag,
|
||||
'github_url': f'https://github.com/{owner}/{repo}',
|
||||
}
|
||||
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.GITHUB, install_info, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.GITHUB,
|
||||
install_info,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-github',
|
||||
label=f'Installing plugin from GitHub {owner}/{repo}@{release_tag}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
@@ -528,9 +817,10 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
'/install/marketplace',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _() -> str:
|
||||
limit_error = await self._check_extensions_limit()
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
limit_error = await self._check_extensions_limit(request_context)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
@@ -538,23 +828,37 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
|
||||
plugin_author = data.get('plugin_author', '')
|
||||
plugin_name = data.get('plugin_name', '')
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{plugin_author}/{plugin_name}'
|
||||
ctx.metadata['install_source'] = 'marketplace'
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.MARKETPLACE, data, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE,
|
||||
data,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-marketplace',
|
||||
label=f'Installing plugin from marketplace {plugin_author}/{plugin_name}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/install/local', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
limit_error = await self._check_extensions_limit()
|
||||
@self.route(
|
||||
'/install/local',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
limit_error = await self._check_extensions_limit(request_context)
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
@@ -563,6 +867,7 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
file_bytes = file.read()
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
data = {
|
||||
'plugin_file': file_bytes,
|
||||
@@ -572,74 +877,72 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
ctx.metadata['plugin_name'] = file.filename or 'local plugin'
|
||||
ctx.metadata['install_source'] = 'local'
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self.ap.plugin_connector.install_plugin(PluginInstallSource.LOCAL, data, task_context=ctx),
|
||||
self._run_fenced_plugin_operation(
|
||||
execution_context,
|
||||
lambda: self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.LOCAL,
|
||||
data,
|
||||
task_context=ctx,
|
||||
),
|
||||
),
|
||||
kind='plugin-operation',
|
||||
name='plugin-install-local',
|
||||
label=f'Installing plugin from local {file.filename}',
|
||||
context=ctx,
|
||||
**self._task_scope(request_context),
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/install/local/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/install/local/preview',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
file_bytes = file.read()
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(file_bytes)) as zf:
|
||||
names = [name for name in zf.namelist() if not name.endswith('/')]
|
||||
manifest_name = next(
|
||||
(
|
||||
name
|
||||
for name in names
|
||||
if name.replace('\\', '/').strip('/').lower() in ('manifest.yaml', 'manifest.yml')
|
||||
),
|
||||
None,
|
||||
)
|
||||
if manifest_name is None:
|
||||
return self.http_status(400, -1, 'manifest.yaml is required')
|
||||
manifest, requirements, names = await asyncio.to_thread(
|
||||
inspect_plugin_archive_metadata,
|
||||
file_bytes,
|
||||
)
|
||||
spec = manifest.get('spec') or {}
|
||||
components = spec.get('components') or {}
|
||||
component_counts = self._count_plugin_components(components, names)
|
||||
component_types = list(component_counts.keys())
|
||||
|
||||
manifest = yaml.safe_load(zf.read(manifest_name).decode('utf-8')) or {}
|
||||
requirements: list[str] = []
|
||||
requirements_name = next(
|
||||
(name for name in names if name.replace('\\', '/').strip('/').lower() == 'requirements.txt'),
|
||||
None,
|
||||
)
|
||||
if requirements_name is not None:
|
||||
requirements = [
|
||||
line.strip()
|
||||
for line in zf.read(requirements_name).decode('utf-8', errors='ignore').splitlines()
|
||||
if line.strip() and not line.strip().startswith('#')
|
||||
]
|
||||
return self.success(
|
||||
data={
|
||||
'filename': file.filename or 'local plugin',
|
||||
'size': len(file_bytes),
|
||||
'manifest': manifest,
|
||||
'metadata': manifest.get('metadata') or {},
|
||||
'component_types': component_types,
|
||||
'component_counts': component_counts,
|
||||
'requirements': requirements,
|
||||
'file_count': len(names),
|
||||
}
|
||||
)
|
||||
except (zipfile.BadZipFile, ValueError) as exc:
|
||||
return self.http_status(400, -1, str(exc) or 'invalid .lbpkg file')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
spec = manifest.get('spec') or {}
|
||||
components = spec.get('components') or {}
|
||||
component_counts = self._count_plugin_components(components, names)
|
||||
component_types = list(component_counts.keys())
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'filename': file.filename or 'local plugin',
|
||||
'size': len(file_bytes),
|
||||
'manifest': manifest,
|
||||
'metadata': manifest.get('metadata') or {},
|
||||
'component_types': component_types,
|
||||
'component_counts': component_counts,
|
||||
'requirements': requirements,
|
||||
'file_count': len(names),
|
||||
}
|
||||
)
|
||||
except zipfile.BadZipFile:
|
||||
return self.http_status(400, -1, 'invalid .lbpkg file')
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview plugin package: {exc}')
|
||||
|
||||
@self.route('/config-files', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/config-files',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Upload a file for plugin configuration"""
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
@@ -650,25 +953,37 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
if len(file_bytes) > MAX_FILE_SIZE:
|
||||
return self.http_status(400, -1, 'file size exceeds 10MB limit')
|
||||
|
||||
# Generate unique file key with original extension
|
||||
original_filename = file.filename
|
||||
original_filename = file.filename or 'config.bin'
|
||||
_, ext = os.path.splitext(original_filename)
|
||||
file_key = f'plugin_config_{uuid.uuid4().hex}{ext}'
|
||||
|
||||
# Save file using storage manager
|
||||
await self.ap.storage_mgr.storage_provider.save(file_key, file_bytes)
|
||||
logical_key = f'plugin_config_{uuid.uuid4().hex}{ext}'
|
||||
file_key = await self.ap.storage_mgr.save_scoped(
|
||||
request_context,
|
||||
owner_type='plugin_config',
|
||||
owner=request_context.workspace_uuid,
|
||||
key=logical_key,
|
||||
value=file_bytes,
|
||||
)
|
||||
|
||||
return self.success(data={'file_key': file_key})
|
||||
|
||||
@self.route('/config-files/<file_key>', methods=['DELETE'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(file_key: str) -> str:
|
||||
@self.route(
|
||||
'/config-files/<path:file_key>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(file_key: str, request_context: RequestContext) -> str:
|
||||
"""Delete a plugin configuration file"""
|
||||
# Only allow deletion of files with plugin_config_ prefix for security
|
||||
if not file_key.startswith('plugin_config_'):
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
if not self.ap.storage_mgr.is_scoped_object_key(file_key, expected_owner_type='plugin_config'):
|
||||
return self.http_status(400, -1, 'invalid file key')
|
||||
|
||||
try:
|
||||
await self.ap.storage_mgr.storage_provider.delete(file_key)
|
||||
await self.ap.storage_mgr.delete_scoped_object_key(
|
||||
request_context,
|
||||
file_key,
|
||||
expected_owner_type='plugin_config',
|
||||
)
|
||||
return self.success(data={'deleted': True})
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'failed to delete file: {str(e)}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@@ -1,147 +1,292 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('models/llm', '/api/v1/provider/models/llm')
|
||||
class LLMModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={'models': await self.ap.llm_model_service.get_llm_models_by_provider(provider_uuid)}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.llm_model_service.get_llm_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.llm_model_service.create_llm_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
|
||||
if provider_uuid:
|
||||
models = await self.ap.llm_model_service.get_llm_models_by_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
else:
|
||||
models = await self.ap.llm_model_service.get_llm_models(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
return self.success(data={'models': models})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.llm_model_service.get_llm_model(model_uuid)
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_uuid = await self.ap.llm_model_service.create_llm_model(
|
||||
request_context,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
model = await self.ap.llm_model_service.get_llm_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
return self.success(data={'model': model})
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.llm_model_service.update_llm_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
await self.ap.llm_model_service.update_llm_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.llm_model_service.delete_llm_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.llm_model_service.test_llm_model(model_uuid, json_data)
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.llm_model_service.delete_llm_model(request_context, model_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.llm_model_service.test_llm_model(request_context, model_uuid, await quart.request.json)
|
||||
return self.success()
|
||||
|
||||
|
||||
@group.group_class('models/embedding', '/api/v1/provider/models/embedding')
|
||||
class EmbeddingModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={
|
||||
'models': await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
provider_uuid
|
||||
)
|
||||
}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.embedding_models_service.get_embedding_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.embedding_models_service.create_embedding_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
|
||||
if provider_uuid:
|
||||
models = await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
else:
|
||||
models = await self.ap.embedding_models_service.get_embedding_models(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
return self.success(data={'models': models})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.embedding_models_service.get_embedding_model(model_uuid)
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_uuid = await self.ap.embedding_models_service.create_embedding_model(
|
||||
request_context,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
model = await self.ap.embedding_models_service.get_embedding_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
return self.success(data={'model': model})
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.embedding_models_service.update_embedding_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
await self.ap.embedding_models_service.update_embedding_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.embedding_models_service.delete_embedding_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.embedding_models_service.test_embedding_model(model_uuid, json_data)
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.embedding_models_service.delete_embedding_model(request_context, model_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.embedding_models_service.test_embedding_model(
|
||||
request_context, model_uuid, await quart.request.json
|
||||
)
|
||||
return self.success()
|
||||
|
||||
|
||||
@group.group_class('models/rerank', '/api/v1/provider/models/rerank')
|
||||
class RerankModelsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
if provider_uuid:
|
||||
return self.success(
|
||||
data={
|
||||
'models': await self.ap.rerank_models_service.get_rerank_models_by_provider(provider_uuid)
|
||||
}
|
||||
)
|
||||
return self.success(data={'models': await self.ap.rerank_models_service.get_rerank_models()})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
model_uuid = await self.ap.rerank_models_service.create_rerank_model(json_data)
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
provider_uuid = quart.request.args.get('provider_uuid')
|
||||
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
|
||||
if provider_uuid:
|
||||
models = await self.ap.rerank_models_service.get_rerank_models_by_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
else:
|
||||
models = await self.ap.rerank_models_service.get_rerank_models(
|
||||
request_context,
|
||||
include_secret=include_secret,
|
||||
)
|
||||
return self.success(data={'models': models})
|
||||
|
||||
@self.route('/<model_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
model = await self.ap.rerank_models_service.get_rerank_model(model_uuid)
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_uuid = await self.ap.rerank_models_service.create_rerank_model(
|
||||
request_context,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': model_uuid})
|
||||
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
|
||||
return self.success(data={'model': model})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.rerank_models_service.update_rerank_model(model_uuid, json_data)
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.rerank_models_service.delete_rerank_model(model_uuid)
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/<model_uuid>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(model_uuid: str) -> str:
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.rerank_models_service.test_rerank_model(model_uuid, json_data)
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
model = await self.ap.rerank_models_service.get_rerank_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if model is None:
|
||||
return self.http_status(404, -1, 'model not found')
|
||||
return self.success(data={'model': model})
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.rerank_models_service.update_rerank_model(
|
||||
request_context,
|
||||
model_uuid,
|
||||
await quart.request.json,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.rerank_models_service.delete_rerank_model(request_context, model_uuid)
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<model_uuid>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(model_uuid: str, request_context: RequestContext) -> str:
|
||||
await self.ap.rerank_models_service.test_rerank_model(request_context, model_uuid, await quart.request.json)
|
||||
return self.success()
|
||||
|
||||
@@ -1,56 +1,102 @@
|
||||
import quart
|
||||
|
||||
from ....authz import Permission, has_permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('models/providers', '/api/v1/provider/providers')
|
||||
class ModelProvidersRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
providers = await self.ap.provider_service.get_providers()
|
||||
# Add model counts
|
||||
for provider in providers:
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(provider['uuid'])
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'providers': providers})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
provider_uuid = await self.ap.provider_service.create_provider(json_data)
|
||||
return self.success(data={'uuid': provider_uuid})
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(provider_uuid: str) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
provider = await self.ap.provider_service.get_provider(provider_uuid)
|
||||
if provider is None:
|
||||
return self.http_status(404, -1, 'provider not found')
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(provider_uuid)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
providers = await self.ap.provider_service.get_providers(
|
||||
request_context,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
for provider in providers:
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider['uuid'])
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'provider': provider})
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.provider_service.update_provider(provider_uuid, json_data)
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
try:
|
||||
await self.ap.provider_service.delete_provider(provider_uuid)
|
||||
return self.success()
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
return self.success(data={'providers': providers})
|
||||
|
||||
@self.route('/<provider_uuid>/scan-models', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def _(provider_uuid: str) -> str:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
provider_uuid = await self.ap.provider_service.create_provider(request_context, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': provider_uuid})
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
provider = await self.ap.provider_service.get_provider(
|
||||
request_context,
|
||||
provider_uuid,
|
||||
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
|
||||
)
|
||||
if provider is None:
|
||||
return self.http_status(404, -1, 'provider not found')
|
||||
counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider_uuid)
|
||||
provider['llm_count'] = counts['llm_count']
|
||||
provider['embedding_count'] = counts['embedding_count']
|
||||
provider['rerank_count'] = counts['rerank_count']
|
||||
return self.success(data={'provider': provider})
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.json
|
||||
try:
|
||||
await self.ap.provider_service.update_provider(request_context, provider_uuid, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>',
|
||||
methods=['DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
await self.ap.provider_service.delete_provider(request_context, provider_uuid)
|
||||
return self.success()
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@self.route(
|
||||
'/<provider_uuid>/scan-models',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.PROVIDER_SECRET_MANAGE,
|
||||
)
|
||||
async def _(provider_uuid: str, request_context: RequestContext) -> str:
|
||||
try:
|
||||
model_type = quart.request.args.get('type')
|
||||
result = await self.ap.provider_service.scan_provider_models(provider_uuid, model_type)
|
||||
result = await self.ap.provider_service.scan_provider_models(request_context, provider_uuid, model_type)
|
||||
return self.success(data=result)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
|
||||
@@ -1,103 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import quart
|
||||
import traceback
|
||||
from urllib.parse import unquote
|
||||
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ......provider.tools.loaders.mcp_policy import MCPStdioDisabledError
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('mcp', '/api/v1/mcp')
|
||||
class MCPRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/servers', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""获取MCP服务器列表"""
|
||||
if quart.request.method == 'GET':
|
||||
servers = await self.ap.mcp_service.get_mcp_servers(contain_runtime_info=True)
|
||||
|
||||
return self.success(data={'servers': servers})
|
||||
|
||||
elif quart.request.method == 'POST':
|
||||
data = await quart.request.json
|
||||
|
||||
try:
|
||||
uuid = await self.ap.mcp_service.create_mcp_server(data)
|
||||
return self.success(data={'uuid': uuid})
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return self.http_status(500, -1, f'Failed to create MCP server: {str(e)}')
|
||||
@self.route(
|
||||
'/servers',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
servers = await self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True)
|
||||
return self.success(data={'servers': servers})
|
||||
|
||||
@self.route(
|
||||
'/servers/<path:server_name>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN
|
||||
'/servers',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(server_name: str) -> str:
|
||||
"""获取、更新或删除MCP服务器配置"""
|
||||
server_name = unquote(server_name)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
data = await quart.request.json
|
||||
try:
|
||||
server_uuid = await self.ap.mcp_service.create_mcp_server(request_context, data)
|
||||
except MCPStdioDisabledError as exc:
|
||||
return self.http_status(403, exc.code, str(exc))
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'uuid': server_uuid})
|
||||
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(server_name)
|
||||
@self.route(
|
||||
'/servers/<path:server_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
server_name = unquote(server_name)
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(request_context, server_name)
|
||||
if server_data is None:
|
||||
return self.http_status(404, -1, 'Server not found')
|
||||
return self.success(data={'server': server_data})
|
||||
|
||||
if quart.request.method == 'GET':
|
||||
return self.success(data={'server': server_data})
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
@self.route(
|
||||
'/servers/<path:server_name>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
server_name = unquote(server_name)
|
||||
server_data = await self.ap.mcp_service.get_mcp_server_by_name(request_context, server_name)
|
||||
if server_data is None:
|
||||
return self.http_status(404, -1, 'Server not found')
|
||||
if quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
try:
|
||||
await self.ap.mcp_service.update_mcp_server(server_data['uuid'], data)
|
||||
return self.success()
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to update MCP server: {str(e)}')
|
||||
await self.ap.mcp_service.update_mcp_server(request_context, server_data['uuid'], data)
|
||||
except MCPStdioDisabledError as exc:
|
||||
return self.http_status(403, exc.code, str(exc))
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
else:
|
||||
await self.ap.mcp_service.delete_mcp_server(request_context, server_data['uuid'])
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
try:
|
||||
await self.ap.mcp_service.delete_mcp_server(server_data['uuid'])
|
||||
return self.success()
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to delete MCP server: {str(e)}')
|
||||
|
||||
@self.route('/servers/<path:server_name>/test', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/test',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""测试MCP服务器连接"""
|
||||
server_name = unquote(server_name)
|
||||
server_data = await quart.request.json
|
||||
task_id = await self.ap.mcp_service.test_mcp_server(server_name=server_name, server_data=server_data)
|
||||
try:
|
||||
task_id = await self.ap.mcp_service.test_mcp_server(
|
||||
request_context,
|
||||
server_name=server_name,
|
||||
server_data=server_data,
|
||||
)
|
||||
except MCPStdioDisabledError as exc:
|
||||
return self.http_status(403, exc.code, str(exc))
|
||||
return self.success(data={'task_id': task_id})
|
||||
|
||||
@self.route('/servers/<path:server_name>/resources', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/resources',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Get resources from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
try:
|
||||
resources = await self.ap.mcp_service.get_mcp_server_resources(server_name)
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(server_name)
|
||||
runtime_info = await self.ap.mcp_service.get_runtime_info(server_name)
|
||||
return self.success(
|
||||
data={
|
||||
'resources': resources,
|
||||
'resource_templates': templates,
|
||||
'resource_capabilities': (runtime_info or {}).get('resource_capabilities', {}),
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to get resources: {str(e)}')
|
||||
resources = await self.ap.mcp_service.get_mcp_server_resources(request_context, server_name)
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(request_context, server_name)
|
||||
runtime_info = await self.ap.mcp_service.get_runtime_info(request_context, server_name)
|
||||
return self.success(
|
||||
data={
|
||||
'resources': resources,
|
||||
'resource_templates': templates,
|
||||
'resource_capabilities': (runtime_info or {}).get('resource_capabilities', {}),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/resource-templates', methods=['GET'], auth_type=group.AuthType.USER_TOKEN
|
||||
'/servers/<path:server_name>/resource-templates',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str) -> str:
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Get resource templates from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
try:
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(server_name)
|
||||
return self.success(data={'resource_templates': templates})
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to get resource templates: {str(e)}')
|
||||
templates = await self.ap.mcp_service.get_mcp_server_resource_templates(request_context, server_name)
|
||||
return self.success(data={'resource_templates': templates})
|
||||
|
||||
@self.route('/servers/<path:server_name>/logs', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/logs',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Get logs from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
try:
|
||||
@@ -106,24 +141,32 @@ class MCPRouterGroup(group.RouterGroup):
|
||||
limit = 200
|
||||
limit = min(limit, 500)
|
||||
level = quart.request.args.get('level') or None
|
||||
logs = await self.ap.mcp_service.get_mcp_server_logs(server_name, limit=limit, level=level)
|
||||
logs = await self.ap.mcp_service.get_mcp_server_logs(
|
||||
request_context,
|
||||
server_name,
|
||||
limit=limit,
|
||||
level=level,
|
||||
)
|
||||
return self.success(data={'logs': logs})
|
||||
|
||||
@self.route('/servers/<path:server_name>/resources/read', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(server_name: str) -> str:
|
||||
@self.route(
|
||||
'/servers/<path:server_name>/resources/read',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(server_name: str, request_context: RequestContext) -> str:
|
||||
"""Read a resource from an MCP server"""
|
||||
server_name = unquote(server_name)
|
||||
data = await quart.request.json
|
||||
uri = data.get('uri')
|
||||
if not uri:
|
||||
return self.http_status(400, -1, 'URI is required')
|
||||
try:
|
||||
envelope = await self.ap.mcp_service.read_mcp_server_resource_envelope(
|
||||
server_name,
|
||||
uri,
|
||||
max_bytes=data.get('max_bytes'),
|
||||
include_blob=bool(data.get('include_blob', False)),
|
||||
)
|
||||
return self.success(data=envelope)
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to read resource: {str(e)}')
|
||||
envelope = await self.ap.mcp_service.read_mcp_server_resource_envelope(
|
||||
request_context,
|
||||
server_name,
|
||||
uri,
|
||||
max_bytes=data.get('max_bytes'),
|
||||
include_blob=bool(data.get('include_blob', False)),
|
||||
)
|
||||
return self.success(data=envelope)
|
||||
|
||||
@@ -2,21 +2,28 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ....authz import Permission
|
||||
from ....context import RequestContext
|
||||
from ... import group
|
||||
|
||||
|
||||
@group.group_class('tools', '/api/v1/tools')
|
||||
class ToolsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""获取所有可用工具列表"""
|
||||
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
|
||||
bound_plugins: list[str] | None = None
|
||||
bound_mcp_servers: list[str] | None = None
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
@@ -35,6 +42,7 @@ class ToolsRouterGroup(group.RouterGroup):
|
||||
return self.success(
|
||||
data={
|
||||
'tools': await self.ap.tool_mgr.get_tool_catalog(
|
||||
request_context,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
@@ -42,10 +50,15 @@ class ToolsRouterGroup(group.RouterGroup):
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(tool_name: str) -> str:
|
||||
@self.route(
|
||||
'/<tool_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(tool_name: str, request_context: RequestContext) -> str:
|
||||
"""获取特定工具详情"""
|
||||
tools = await self.ap.tool_mgr.get_all_tools(include_skill_authoring=True)
|
||||
tools = await self.ap.tool_mgr.get_all_tools(request_context, include_skill_authoring=True)
|
||||
|
||||
for tool in tools:
|
||||
if tool.name == tool_name:
|
||||
|
||||
@@ -2,8 +2,11 @@ from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from langbot.pkg.cloud.entitlements import EntitlementFeatureUnavailableError
|
||||
from langbot_plugin.box.errors import BoxError
|
||||
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@@ -12,58 +15,91 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
"""Skills management API endpoints."""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_or_create_skills() -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
skills = await self.ap.skill_service.list_skills()
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'skills': skills})
|
||||
@self.route(
|
||||
'',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_skills(request_context: RequestContext) -> quart.Response:
|
||||
try:
|
||||
skills = await self.ap.skill_service.list_skills(request_context)
|
||||
except EntitlementFeatureUnavailableError:
|
||||
# Plans without managed sandbox support have no runnable skills.
|
||||
# Treat that capability absence as an empty collection so the
|
||||
# shared UI can render normally instead of surfacing a 500.
|
||||
return self.success(data={'skills': []})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'skills': skills})
|
||||
|
||||
@self.route(
|
||||
'',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def create_skill(request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
if 'name' not in data or not data['name']:
|
||||
return self.http_status(400, -1, 'Missing required field: name')
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.create_skill(data)
|
||||
skill = await self.ap.skill_service.create_skill(request_context, data)
|
||||
return self.success(data={'skill': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def get_update_delete_skill(skill_name: str) -> quart.Response:
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
skill = await self.ap.skill_service.get_skill(skill_name)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'skill': skill})
|
||||
@self.route(
|
||||
'/<skill_name>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def get_skill(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
try:
|
||||
skill = await self.ap.skill_service.get_skill(request_context, skill_name)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'skill': skill})
|
||||
|
||||
@self.route(
|
||||
'/<skill_name>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def update_delete_skill(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
if quart.request.method == 'PUT':
|
||||
data = await quart.request.json
|
||||
try:
|
||||
skill = await self.ap.skill_service.update_skill(skill_name, data)
|
||||
skill = await self.ap.skill_service.update_skill(request_context, skill_name, data)
|
||||
return self.success(data={'skill': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
try:
|
||||
await self.ap.skill_service.delete_skill(skill_name)
|
||||
await self.ap.skill_service.delete_skill(request_context, skill_name)
|
||||
return self.success()
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>/files', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def list_skill_files(skill_name: str) -> quart.Response:
|
||||
@self.route(
|
||||
'/<skill_name>/files',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def list_skill_files(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
"""List files in skill package directory."""
|
||||
path = quart.request.args.get('path', '.').strip()
|
||||
include_hidden = quart.request.args.get('include_hidden', 'false').lower() == 'true'
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.list_skill_files(
|
||||
request_context,
|
||||
skill_name,
|
||||
path=path,
|
||||
include_hidden=include_hidden,
|
||||
@@ -73,38 +109,55 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route(
|
||||
'/<skill_name>/files/<path:path>', methods=['GET', 'PUT'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY
|
||||
'/<skill_name>/files/<path:path>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def read_or_write_skill_file(skill_name: str, path: str) -> quart.Response:
|
||||
"""Read or write a file in skill package."""
|
||||
if quart.request.method == 'GET':
|
||||
try:
|
||||
result = await self.ap.skill_service.read_skill_file(skill_name, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
async def read_skill_file(skill_name: str, path: str, request_context: RequestContext) -> quart.Response:
|
||||
try:
|
||||
result = await self.ap.skill_service.read_skill_file(request_context, skill_name, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
# PUT - write file
|
||||
@self.route(
|
||||
'/<skill_name>/files/<path:path>',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def write_skill_file(skill_name: str, path: str, request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
content = data.get('content', '')
|
||||
if content is None:
|
||||
return self.http_status(400, -1, 'Missing required field: content')
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.write_skill_file(skill_name, path, content)
|
||||
result = await self.ap.skill_service.write_skill_file(request_context, skill_name, path, content)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@self.route('/<skill_name>/preview', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill(skill_name: str) -> quart.Response:
|
||||
skill = self.ap.skill_mgr.get_skill_by_name(skill_name)
|
||||
@self.route(
|
||||
'/<skill_name>/preview',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def preview_skill(skill_name: str, request_context: RequestContext) -> quart.Response:
|
||||
skill = await self.ap.skill_service.get_skill(request_context, skill_name)
|
||||
if not skill:
|
||||
return self.http_status(404, -1, 'Skill not found')
|
||||
return self.success(data={'instructions': skill.get('instructions', '')})
|
||||
|
||||
@self.route('/install/github', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def install_skill_from_github() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/github',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def install_skill_from_github(request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
required_fields = ['asset_url', 'owner', 'repo']
|
||||
for field in required_fields:
|
||||
@@ -115,15 +168,20 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Missing required field: release_tag')
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.install_from_github(data)
|
||||
skill = await self.ap.skill_service.install_from_github(request_context, data)
|
||||
return self.success(data={'skills': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to install skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/install/github/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill_from_github() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/github/preview',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def preview_skill_from_github(request_context: RequestContext) -> quart.Response:
|
||||
data = await quart.request.json
|
||||
required_fields = ['asset_url', 'owner', 'repo']
|
||||
for field in required_fields:
|
||||
@@ -134,15 +192,20 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Missing required field: release_tag')
|
||||
|
||||
try:
|
||||
preview = await self.ap.skill_service.preview_install_from_github(data)
|
||||
preview = await self.ap.skill_service.preview_install_from_github(request_context, data)
|
||||
return self.success(data={'skills': preview})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/install/upload', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def install_skill_from_upload() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/upload',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def install_skill_from_upload(request_context: RequestContext) -> quart.Response:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
@@ -150,6 +213,7 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
|
||||
try:
|
||||
skill = await self.ap.skill_service.install_from_zip_upload(
|
||||
request_context,
|
||||
file_bytes=file.read(),
|
||||
filename=file.filename or '',
|
||||
source_paths=form.getlist('source_paths'),
|
||||
@@ -157,34 +221,45 @@ class SkillsRouterGroup(group.RouterGroup):
|
||||
return self.success(data={'skills': skill})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to install skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/install/upload/preview', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def preview_skill_from_upload() -> quart.Response:
|
||||
@self.route(
|
||||
'/install/upload/preview',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def preview_skill_from_upload(request_context: RequestContext) -> quart.Response:
|
||||
file = (await quart.request.files).get('file')
|
||||
if file is None:
|
||||
return self.http_status(400, -1, 'file is required')
|
||||
|
||||
try:
|
||||
preview = await self.ap.skill_service.preview_install_from_zip_upload(
|
||||
request_context,
|
||||
file_bytes=file.read(),
|
||||
filename=file.filename or '',
|
||||
)
|
||||
return self.success(data={'skills': preview})
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
except Exception as exc:
|
||||
return self.http_status(500, -1, f'Failed to preview skill: {exc}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/scan', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
async def scan_skill_directory() -> quart.Response:
|
||||
@self.route(
|
||||
'/scan',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def scan_skill_directory(request_context: RequestContext) -> quart.Response:
|
||||
path = quart.request.args.get('path', '').strip()
|
||||
if not path:
|
||||
return self.http_status(400, -1, 'Missing required parameter: path')
|
||||
|
||||
try:
|
||||
result = await self.ap.skill_service.scan_directory_async(path)
|
||||
result = await self.ap.skill_service.scan_directory_async(request_context, path)
|
||||
return self.success(data=result)
|
||||
except (ValueError, BoxError) as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
@@ -1,19 +1,39 @@
|
||||
from .. import group
|
||||
from ...authz import Permission
|
||||
from ...context import ExecutionContext, RequestContext
|
||||
|
||||
|
||||
def collect_basic_stats(ap, request_context: RequestContext) -> dict[str, int]:
|
||||
"""Collect runtime counters only from the selected Workspace placement."""
|
||||
|
||||
execution_context = ExecutionContext.from_request(request_context)
|
||||
sessions = [
|
||||
session
|
||||
for session in ap.sess_mgr.session_list
|
||||
if (
|
||||
getattr(session, 'instance_uuid', None) == execution_context.instance_uuid
|
||||
and getattr(session, 'workspace_uuid', None) == execution_context.workspace_uuid
|
||||
and getattr(session, 'placement_generation', None) == execution_context.placement_generation
|
||||
)
|
||||
]
|
||||
conversation_count = sum(
|
||||
len(session.conversations if session.conversations is not None else []) for session in sessions
|
||||
)
|
||||
return {
|
||||
'active_session_count': len(sessions),
|
||||
'conversation_count': conversation_count,
|
||||
'query_count': ap.query_pool.get_query_count(execution_context),
|
||||
}
|
||||
|
||||
|
||||
@group.group_class('stats', '/api/v1/stats')
|
||||
class StatsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/basic', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
conv_count = 0
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
conv_count += len(session.conversations if session.conversations is not None else [])
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'active_session_count': len(self.ap.sess_mgr.session_list),
|
||||
'conversation_count': conv_count,
|
||||
'query_count': self.ap.query_pool.query_id_counter,
|
||||
}
|
||||
)
|
||||
@self.route(
|
||||
'/basic',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data=collect_basic_stats(self.ap, request_context))
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import base64
|
||||
|
||||
import quart
|
||||
@@ -59,7 +60,14 @@ class SurveyRouterGroup(group.RouterGroup):
|
||||
continue
|
||||
try:
|
||||
payload = data_url.split(',', 1)[1]
|
||||
if len(base64.b64decode(payload, validate=True)) > 1024 * 1024:
|
||||
if len(payload) > 4 * ((1024 * 1024 + 2) // 3) + 4:
|
||||
return self.fail(5, 'attachment too large')
|
||||
decoded = await asyncio.to_thread(
|
||||
base64.b64decode,
|
||||
payload,
|
||||
validate=True,
|
||||
)
|
||||
if len(decoded) > 1024 * 1024:
|
||||
return self.fail(5, 'attachment too large')
|
||||
except Exception:
|
||||
return self.fail(5, 'attachment too large')
|
||||
|
||||
@@ -5,7 +5,11 @@ import sqlalchemy
|
||||
|
||||
from .. import group
|
||||
from .....utils import constants
|
||||
from .....entity.persistence.metadata import Metadata
|
||||
from .....entity.persistence.metadata import WorkspaceMetadata
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .....provider.tools.loaders.mcp_policy import stdio_mcp_enabled
|
||||
from .....workspace.invitation_delivery import InvitationDeliveryService
|
||||
|
||||
|
||||
@group.group_class('system', '/api/v1/system')
|
||||
@@ -17,17 +21,46 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
wizard_status = 'none'
|
||||
wizard_progress = None
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key.in_(['wizard_status', 'wizard_progress']))
|
||||
)
|
||||
for row in result:
|
||||
if row.key == 'wizard_status':
|
||||
wizard_status = row.value
|
||||
elif row.key == 'wizard_progress':
|
||||
try:
|
||||
wizard_progress = json.loads(row.value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
wizard_progress = None
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if authorization.startswith('Bearer '):
|
||||
account, _ = await self._authenticate_account(authorization.removeprefix('Bearer '))
|
||||
request_context = await self._resolve_account_context(account, group.AuthType.USER_TOKEN)
|
||||
if request_context is not None:
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
|
||||
async def load_workspace_metadata():
|
||||
return await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
WorkspaceMetadata.key,
|
||||
WorkspaceMetadata.value,
|
||||
).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key.in_(['wizard_status', 'wizard_progress']),
|
||||
)
|
||||
)
|
||||
|
||||
cloud_runtime = (
|
||||
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
)
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud system metadata requires an explicit tenant UoW')
|
||||
async with tenant_uow(request_context.workspace_uuid):
|
||||
result = await load_workspace_metadata()
|
||||
else:
|
||||
result = await load_workspace_metadata()
|
||||
# ``execute_async`` deliberately preserves its historical
|
||||
# AsyncConnection result shape. Selecting the two fields
|
||||
# explicitly keeps this reader independent of ORM Session
|
||||
# scalar semantics inside a tenant UoW.
|
||||
for row in result:
|
||||
if row.key == 'wizard_status':
|
||||
wizard_status = row.value
|
||||
elif row.key == 'wizard_progress':
|
||||
try:
|
||||
wizard_progress = json.loads(row.value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
wizard_progress = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -43,6 +76,10 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
else:
|
||||
outbound_ips = []
|
||||
|
||||
invitation_delivery_service = getattr(self.ap, 'invitation_delivery_service', None)
|
||||
if invitation_delivery_service is None:
|
||||
invitation_delivery_service = InvitationDeliveryService(self.ap)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'version': constants.semantic_version,
|
||||
@@ -60,15 +97,24 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
'disable_models_service': self.ap.instance_config.data.get('space', {}).get(
|
||||
'disable_models_service', False
|
||||
),
|
||||
# Exposed independently of Box status so the WebUI cannot
|
||||
# infer stdio permission from sandbox availability.
|
||||
'mcp_stdio_enabled': stdio_mcp_enabled(self.ap),
|
||||
'limitation': self.ap.instance_config.data.get('system', {}).get('limitation', {}),
|
||||
'outbound_ips': outbound_ips,
|
||||
'invitation_delivery': invitation_delivery_service.capability(),
|
||||
'wizard_status': wizard_status,
|
||||
'wizard_progress': wizard_progress,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/wizard/completed', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/wizard/completed',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.WORKSPACE_UPDATE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Mark wizard status in metadata table and clear progress.
|
||||
|
||||
Accepts JSON body: { "status": "skipped" | "completed" }
|
||||
@@ -80,28 +126,48 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == 'wizard_status')
|
||||
sqlalchemy.select(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_status',
|
||||
)
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == 'wizard_status').values(value=status)
|
||||
sqlalchemy.update(WorkspaceMetadata)
|
||||
.where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_status',
|
||||
)
|
||||
.values(value=status)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key='wizard_status', value=status)
|
||||
sqlalchemy.insert(WorkspaceMetadata).values(
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
key='wizard_status',
|
||||
value=status,
|
||||
)
|
||||
)
|
||||
|
||||
# Clear wizard progress when wizard is completed/skipped
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(Metadata).where(Metadata.key == 'wizard_progress')
|
||||
sqlalchemy.delete(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_progress',
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, 500, f'Failed to update wizard status: {e}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
return self.success(data={})
|
||||
|
||||
@self.route('/wizard/progress', methods=['PUT'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/wizard/progress',
|
||||
methods=['PUT'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.WORKSPACE_UPDATE,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Save wizard progress to metadata table.
|
||||
|
||||
Accepts JSON body with wizard state fields:
|
||||
@@ -113,23 +179,40 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == 'wizard_progress')
|
||||
sqlalchemy.select(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_progress',
|
||||
)
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == 'wizard_progress').values(value=progress_json)
|
||||
sqlalchemy.update(WorkspaceMetadata)
|
||||
.where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key == 'wizard_progress',
|
||||
)
|
||||
.values(value=progress_json)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key='wizard_progress', value=progress_json)
|
||||
sqlalchemy.insert(WorkspaceMetadata).values(
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
key='wizard_progress',
|
||||
value=progress_json,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return self.http_status(500, 500, f'Failed to save wizard progress: {e}')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
return self.success(data={})
|
||||
|
||||
@self.route('/tasks', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
@self.route(
|
||||
'/tasks',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
task_type = quart.request.args.get('type')
|
||||
task_kind = quart.request.args.get('kind')
|
||||
|
||||
@@ -138,30 +221,56 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
if task_kind == '':
|
||||
task_kind = None
|
||||
|
||||
return self.success(data=self.ap.task_mgr.get_tasks_dict(task_type, task_kind))
|
||||
return self.success(
|
||||
data=self.ap.task_mgr.get_tasks_dict(
|
||||
task_type,
|
||||
task_kind,
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
)
|
||||
)
|
||||
|
||||
@self.route('/tasks/<task_id>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(task_id: str) -> str:
|
||||
task = self.ap.task_mgr.get_task_by_id(int(task_id))
|
||||
@self.route(
|
||||
'/tasks/<task_id>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(task_id: str, request_context: RequestContext) -> str:
|
||||
task = self.ap.task_mgr.get_task_by_id(
|
||||
int(task_id),
|
||||
instance_uuid=request_context.instance_uuid,
|
||||
workspace_uuid=request_context.workspace_uuid,
|
||||
placement_generation=request_context.placement_generation,
|
||||
)
|
||||
|
||||
if task is None:
|
||||
return self.http_status(404, 404, 'Task not found')
|
||||
|
||||
return self.success(data=task.to_dict())
|
||||
|
||||
@self.route('/storage-analysis', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
return self.success(data=await self.ap.maintenance_service.get_storage_analysis())
|
||||
@self.route(
|
||||
'/storage-analysis',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
return self.success(data=await self.ap.maintenance_service.get_storage_analysis(request_context))
|
||||
|
||||
@self.route(
|
||||
'/debug/plugin/action',
|
||||
methods=['POST'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RUNTIME_OPERATE,
|
||||
)
|
||||
async def _() -> str:
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
if not constants.debug_mode:
|
||||
return self.http_status(403, 403, 'Forbidden')
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
data = await quart.request.json
|
||||
|
||||
class AnoymousAction:
|
||||
@@ -174,6 +283,7 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
AnoymousAction(data['action']),
|
||||
data['data'],
|
||||
timeout=data.get('timeout', 10),
|
||||
action_context=self.ap.plugin_connector.handler.require_bound_action_context().without_installation(),
|
||||
)
|
||||
|
||||
return self.success(data=resp)
|
||||
@@ -182,8 +292,10 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
'/status/plugin-system',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.USER_TOKEN,
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _() -> str:
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
plugin_connector_error = 'ok'
|
||||
is_connected = True
|
||||
|
||||
|
||||
@@ -1,14 +1,55 @@
|
||||
import quart
|
||||
import argon2
|
||||
import asyncio
|
||||
import traceback
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from .. import group
|
||||
from .....entity.errors import account as account_errors
|
||||
from ...context import RequestContext
|
||||
from .....cloud.launch import SpaceLaunchError
|
||||
from ...service.user import ControlPlaneDirectoryRequiredError, PublicRegistrationClosedError
|
||||
|
||||
|
||||
@group.group_class('user', '/api/v1/user')
|
||||
class UserRouterGroup(group.RouterGroup):
|
||||
@staticmethod
|
||||
def _origin(value: str) -> tuple[str, str, int | None] | None:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {'http', 'https'} or not parsed.hostname:
|
||||
return None
|
||||
return parsed.scheme, parsed.hostname.casefold(), parsed.port
|
||||
|
||||
def _validate_space_redirect_uri(self, redirect_uri: str, *, bind: bool) -> str:
|
||||
parsed = urlsplit(redirect_uri)
|
||||
if (
|
||||
parsed.scheme not in {'http', 'https'}
|
||||
or not parsed.hostname
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.fragment
|
||||
or parsed.path != '/auth/space/callback'
|
||||
):
|
||||
raise ValueError('Invalid redirect_uri parameter')
|
||||
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
if bind:
|
||||
if query != {'mode': ['bind']}:
|
||||
raise ValueError('Invalid Space binding redirect_uri')
|
||||
elif query:
|
||||
raise ValueError('Invalid Space login redirect_uri')
|
||||
|
||||
redirect_origin = self._origin(redirect_uri)
|
||||
api_config = self.ap.instance_config.data.get('api', {})
|
||||
trusted_origins = {
|
||||
self._origin(str(api_config.get(config_key, '') or '').strip())
|
||||
for config_key in ('webui_url', 'webhook_prefix')
|
||||
}
|
||||
trusted_origins.discard(None)
|
||||
if redirect_origin not in trusted_origins:
|
||||
raise ValueError('Untrusted redirect_uri origin')
|
||||
return redirect_uri
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/init', methods=['GET', 'POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
@@ -23,12 +64,19 @@ class UserRouterGroup(group.RouterGroup):
|
||||
user_email = json_data['user']
|
||||
password = json_data['password']
|
||||
|
||||
await self.ap.user_service.create_user(user_email, password)
|
||||
try:
|
||||
await self.ap.user_service.create_user(user_email, password)
|
||||
except ControlPlaneDirectoryRequiredError as exc:
|
||||
return self.http_status(409, exc.code, str(exc))
|
||||
except PublicRegistrationClosedError:
|
||||
return self.http_status(409, 'registration_closed', 'System already initialized')
|
||||
|
||||
return self.success()
|
||||
|
||||
@self.route('/auth', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud':
|
||||
return self.http_status(403, 'password_login_disabled', 'Password login is disabled on LangBot Cloud')
|
||||
json_data = await quart.request.json
|
||||
|
||||
try:
|
||||
@@ -40,9 +88,9 @@ class UserRouterGroup(group.RouterGroup):
|
||||
|
||||
return self.success(data={'token': token})
|
||||
|
||||
@self.route('/check-token', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
token = await self.ap.user_service.generate_jwt_token(user_email)
|
||||
@self.route('/check-token', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> str:
|
||||
token = await self.ap.user_service.generate_jwt_token(account)
|
||||
|
||||
return self.success(data={'token': token})
|
||||
|
||||
@@ -101,15 +149,50 @@ class UserRouterGroup(group.RouterGroup):
|
||||
async def _() -> str:
|
||||
"""Get Space OAuth authorization URL for redirect"""
|
||||
redirect_uri = quart.request.args.get('redirect_uri', '')
|
||||
state = quart.request.args.get('state', '')
|
||||
|
||||
if not redirect_uri:
|
||||
return self.fail(1, 'Missing redirect_uri parameter')
|
||||
if 'state' in quart.request.args:
|
||||
return self.fail(1, 'Caller-supplied OAuth state is not allowed')
|
||||
|
||||
try:
|
||||
redirect_uri = self._validate_space_redirect_uri(redirect_uri, bind=False)
|
||||
launch_workspace_uuid = quart.request.args.get('launch_workspace_uuid')
|
||||
if launch_workspace_uuid:
|
||||
if not getattr(getattr(self.ap, 'deployment', None), 'multi_workspace_enabled', False):
|
||||
return self.fail(1, 'Space launch requires Cloud mode')
|
||||
try:
|
||||
uuid.UUID(launch_workspace_uuid)
|
||||
except ValueError:
|
||||
return self.fail(1, 'Invalid launch Workspace')
|
||||
state = await self.ap.user_service.issue_space_oauth_state(
|
||||
'login',
|
||||
launch_workspace_uuid=launch_workspace_uuid,
|
||||
)
|
||||
else:
|
||||
state = await self.ap.user_service.issue_space_oauth_state('login')
|
||||
authorize_url = self.ap.space_service.get_oauth_authorize_url(redirect_uri, state)
|
||||
return self.success(data={'authorize_url': authorize_url})
|
||||
except Exception as e:
|
||||
except ValueError as e:
|
||||
return self.fail(1, str(e))
|
||||
|
||||
@self.route('/space/bind-authorize-url', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Issue an account-bound, one-time Space OAuth redirect."""
|
||||
redirect_uri = quart.request.args.get('redirect_uri', '')
|
||||
if not redirect_uri:
|
||||
return self.fail(1, 'Missing redirect_uri parameter')
|
||||
if not request_context.account_uuid:
|
||||
return self.http_status(403, 'account_required', 'An Account is required')
|
||||
try:
|
||||
redirect_uri = self._validate_space_redirect_uri(redirect_uri, bind=True)
|
||||
state = await self.ap.user_service.issue_space_oauth_state(
|
||||
'bind',
|
||||
account_uuid=request_context.account_uuid,
|
||||
)
|
||||
authorize_url = self.ap.space_service.get_oauth_authorize_url(redirect_uri, state)
|
||||
return self.success(data={'authorize_url': authorize_url})
|
||||
except ValueError as e:
|
||||
return self.fail(1, str(e))
|
||||
|
||||
@self.route('/space/callback', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
@@ -117,11 +200,23 @@ class UserRouterGroup(group.RouterGroup):
|
||||
"""Handle OAuth callback - exchange code for tokens and authenticate"""
|
||||
json_data = await quart.request.json
|
||||
code = json_data.get('code')
|
||||
state = json_data.get('state')
|
||||
launch_assertion = json_data.get('launch_assertion')
|
||||
workspace_uuid = json_data.get('workspace_uuid')
|
||||
|
||||
if launch_assertion:
|
||||
return await self._handle_space_direct_launch(
|
||||
str(launch_assertion),
|
||||
str(workspace_uuid or '') or None,
|
||||
)
|
||||
|
||||
if not code:
|
||||
return self.fail(1, 'Missing authorization code')
|
||||
if not state:
|
||||
return self.fail(1, 'Missing state parameter')
|
||||
|
||||
try:
|
||||
consumed_state = await self.ap.user_service.consume_space_oauth_state_details(state, 'login')
|
||||
# Exchange code for tokens
|
||||
token_data = await self.ap.space_service.exchange_oauth_code(code)
|
||||
access_token = token_data.get('access_token')
|
||||
@@ -136,61 +231,80 @@ class UserRouterGroup(group.RouterGroup):
|
||||
access_token, refresh_token, expires_in
|
||||
)
|
||||
|
||||
launch_workspace_uuid = consumed_state.launch_workspace_uuid
|
||||
if launch_workspace_uuid:
|
||||
try:
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
user_obj.uuid,
|
||||
launch_workspace_uuid,
|
||||
)
|
||||
except Exception:
|
||||
self.ap.logger.warning('Rejected Space OAuth launch for unauthorized Workspace')
|
||||
return self.fail(1, 'Space OAuth failed')
|
||||
return self.success(
|
||||
data={
|
||||
'token': jwt_token,
|
||||
'user': user_obj.user,
|
||||
'workspace_uuid': access.workspace.uuid,
|
||||
}
|
||||
)
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'token': jwt_token,
|
||||
'user': user_obj.user,
|
||||
}
|
||||
)
|
||||
except ControlPlaneDirectoryRequiredError as e:
|
||||
return self.http_status(409, e.code, str(e))
|
||||
except account_errors.AccountEmailMismatchError as e:
|
||||
return self.fail(3, str(e))
|
||||
except ValueError as e:
|
||||
traceback.print_exc()
|
||||
self.ap.logger.warning(f'Space OAuth callback failed: {e}')
|
||||
return self.fail(1, str(e))
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
return self.fail(2, f'OAuth callback failed: {str(e)}')
|
||||
|
||||
@self.route('/info', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
"""Get current user information including account type"""
|
||||
user_obj = await self.ap.user_service.get_user_by_email(user_email)
|
||||
|
||||
if user_obj is None:
|
||||
return self.http_status(404, -1, 'User not found')
|
||||
return self.fail(getattr(e, 'code', 3), str(e))
|
||||
except ValueError:
|
||||
self.ap.logger.exception('Space OAuth callback failed')
|
||||
return self.fail(1, 'Space OAuth failed')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
@self.route('/info', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> str:
|
||||
"""Get current Account information without re-querying under Workspace RLS."""
|
||||
return self.success(
|
||||
data={
|
||||
'user': user_obj.user,
|
||||
'account_type': user_obj.account_type,
|
||||
'has_password': bool(user_obj.password and user_obj.password.strip()),
|
||||
'account_uuid': account.uuid,
|
||||
'user': account.user,
|
||||
'account_type': account.account_type,
|
||||
'has_password': bool(account.password and account.password.strip()),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/space-credits', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
"""Get Space credits balance for current user"""
|
||||
credits = await self.ap.space_service.get_credits(user_email)
|
||||
return self.success(data={'credits': credits})
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
"""Get Space credits using only the selected Workspace owner's credentials."""
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
request_context.account_uuid,
|
||||
request_context.workspace_uuid,
|
||||
)
|
||||
owner = await self.ap.user_service.get_workspace_owner(access.workspace.uuid)
|
||||
owner_space_bound = bool(owner and owner.space_account_uuid)
|
||||
credits = await self.ap.space_service.get_credits(owner.user) if owner_space_bound else None
|
||||
return self.success(
|
||||
data={
|
||||
'credits': credits,
|
||||
'owner_space_bound': owner_space_bound,
|
||||
'is_workspace_owner': access.membership.role == 'owner',
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/account-info', methods=['GET'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> str:
|
||||
"""Get account info for login page (account type and has_password)"""
|
||||
"""Return instance login capabilities without disclosing an account."""
|
||||
if not await self.ap.user_service.is_initialized():
|
||||
return self.success(data={'initialized': False})
|
||||
|
||||
user_obj = await self.ap.user_service.get_first_user()
|
||||
if user_obj is None:
|
||||
return self.success(data={'initialized': False})
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'initialized': True,
|
||||
'account_type': user_obj.account_type,
|
||||
'has_password': bool(user_obj.password and user_obj.password.strip()),
|
||||
}
|
||||
)
|
||||
capabilities = await self.ap.user_service.get_login_capabilities()
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud':
|
||||
capabilities['password_login_enabled'] = False
|
||||
return self.success(data={'initialized': True, **capabilities})
|
||||
|
||||
@self.route('/set-password', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(user_email: str) -> str:
|
||||
@@ -233,7 +347,7 @@ class UserRouterGroup(group.RouterGroup):
|
||||
|
||||
json_data = await quart.request.json
|
||||
code = json_data.get('code')
|
||||
state = json_data.get('state') # JWT token passed as state
|
||||
state = json_data.get('state')
|
||||
|
||||
if not code:
|
||||
return self.http_status(400, -1, 'Missing authorization code')
|
||||
@@ -241,13 +355,10 @@ class UserRouterGroup(group.RouterGroup):
|
||||
if not state:
|
||||
return self.http_status(400, -1, 'Missing state parameter')
|
||||
|
||||
# Verify state is a valid JWT token
|
||||
try:
|
||||
user_email = await self.ap.user_service.verify_jwt_token(state)
|
||||
user_obj = await self.ap.user_service.consume_space_oauth_state(state, 'bind')
|
||||
except Exception:
|
||||
return self.http_status(401, -1, 'Invalid or expired state')
|
||||
|
||||
user_obj = await self.ap.user_service.get_user_by_email(user_email)
|
||||
if user_obj is None:
|
||||
return self.http_status(404, -1, 'User not found')
|
||||
|
||||
@@ -255,8 +366,8 @@ class UserRouterGroup(group.RouterGroup):
|
||||
return self.http_status(400, -1, 'Only local accounts can bind to Space')
|
||||
|
||||
try:
|
||||
updated_user = await self.ap.user_service.bind_space_account(user_email, code)
|
||||
jwt_token = await self.ap.user_service.generate_jwt_token(updated_user.user)
|
||||
updated_user = await self.ap.user_service.bind_space_account(user_obj.user, code)
|
||||
jwt_token = await self.ap.user_service.generate_jwt_token(updated_user)
|
||||
return self.success(
|
||||
data={
|
||||
'token': jwt_token,
|
||||
@@ -264,7 +375,46 @@ class UserRouterGroup(group.RouterGroup):
|
||||
'account_type': updated_user.account_type,
|
||||
}
|
||||
)
|
||||
except ValueError as e:
|
||||
return self.http_status(400, -1, str(e))
|
||||
except Exception as e:
|
||||
return self.http_status(500, -1, f'Failed to bind Space account: {str(e)}')
|
||||
except account_errors.AccountEmailMismatchError:
|
||||
return self.http_status(
|
||||
409,
|
||||
'space_account_email_mismatch',
|
||||
'Bind the LangBot Account with the same email as this local Account',
|
||||
)
|
||||
except ValueError:
|
||||
return self.http_status(400, -1, 'Space account binding failed')
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
async def _handle_space_direct_launch(
|
||||
self,
|
||||
launch_assertion: str,
|
||||
workspace_uuid: str | None,
|
||||
) -> str:
|
||||
try:
|
||||
launch = await self.ap.space_launch_service.consume_assertion(
|
||||
launch_assertion,
|
||||
expected_workspace_uuid=workspace_uuid,
|
||||
)
|
||||
account = await self.ap.user_service.get_user_by_uuid(launch['account_uuid'])
|
||||
if account is None:
|
||||
raise SpaceLaunchError('Launch Account is not projected into Core')
|
||||
self.ap.user_service._require_active_account(account)
|
||||
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
|
||||
account.uuid,
|
||||
launch['workspace_uuid'],
|
||||
)
|
||||
token = await self.ap.user_service.generate_jwt_token(account)
|
||||
return self.success(
|
||||
data={
|
||||
'token': token,
|
||||
'user': account.user,
|
||||
'workspace_uuid': access.workspace.uuid,
|
||||
}
|
||||
)
|
||||
except SpaceLaunchError:
|
||||
self.ap.logger.warning('Rejected Space direct-launch assertion')
|
||||
return self.fail(1, 'Space launch failed')
|
||||
except Exception:
|
||||
self.ap.logger.exception('Space direct launch failed')
|
||||
return self.fail(1, 'Space launch failed')
|
||||
|
||||
@@ -1,49 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission, has_permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
|
||||
|
||||
@group.group_class('webhook_mgmt', '/api/v1/webhooks')
|
||||
class WebhookManagementRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET', 'POST'])
|
||||
async def _() -> str:
|
||||
if quart.request.method == 'GET':
|
||||
webhooks = await self.ap.webhook_service.get_webhooks()
|
||||
return self.success(data={'webhooks': webhooks})
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name', '')
|
||||
url = json_data.get('url', '')
|
||||
description = json_data.get('description', '')
|
||||
enabled = json_data.get('enabled', True)
|
||||
@self.route('', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
webhooks = await self.ap.webhook_service.get_webhooks(
|
||||
request_context,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
return self.success(data={'webhooks': webhooks})
|
||||
|
||||
if not name:
|
||||
return self.http_status(400, -1, 'Name is required')
|
||||
if not url:
|
||||
return self.http_status(400, -1, 'URL is required')
|
||||
@self.route('', methods=['POST'], permission=Permission.RESOURCE_MANAGE)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
json_data = await quart.request.get_json(silent=True) or {}
|
||||
name = json_data.get('name', '')
|
||||
url = json_data.get('url', '')
|
||||
description = json_data.get('description', '')
|
||||
enabled = json_data.get('enabled', True)
|
||||
|
||||
webhook = await self.ap.webhook_service.create_webhook(name, url, description, enabled)
|
||||
return self.success(data={'webhook': webhook})
|
||||
if not name:
|
||||
return self.http_status(400, -1, 'Name is required')
|
||||
if not url:
|
||||
return self.http_status(400, -1, 'URL is required')
|
||||
|
||||
@self.route('/<int:webhook_id>', methods=['GET', 'PUT', 'DELETE'])
|
||||
async def _(webhook_id: int) -> str:
|
||||
if quart.request.method == 'GET':
|
||||
webhook = await self.ap.webhook_service.get_webhook(webhook_id)
|
||||
if webhook is None:
|
||||
try:
|
||||
webhook = await self.ap.webhook_service.create_webhook(
|
||||
request_context,
|
||||
name,
|
||||
url,
|
||||
description,
|
||||
enabled,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data={'webhook': webhook})
|
||||
|
||||
@self.route('/<int:webhook_id>', methods=['GET'], permission=Permission.RESOURCE_VIEW)
|
||||
async def _(webhook_id: int, request_context: RequestContext) -> str:
|
||||
webhook = await self.ap.webhook_service.get_webhook(
|
||||
request_context,
|
||||
webhook_id,
|
||||
include_secret=has_permission(request_context, Permission.RESOURCE_MANAGE),
|
||||
)
|
||||
if webhook is None:
|
||||
return self.http_status(404, -1, 'Webhook not found')
|
||||
return self.success(data={'webhook': webhook})
|
||||
|
||||
@self.route(
|
||||
'/<int:webhook_id>',
|
||||
methods=['PUT', 'DELETE'],
|
||||
permission=Permission.RESOURCE_MANAGE,
|
||||
)
|
||||
async def _(webhook_id: int, request_context: RequestContext) -> str:
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.get_json(silent=True) or {}
|
||||
updated = await self.ap.webhook_service.update_webhook(
|
||||
request_context,
|
||||
webhook_id,
|
||||
json_data.get('name'),
|
||||
json_data.get('url'),
|
||||
json_data.get('description'),
|
||||
json_data.get('enabled'),
|
||||
)
|
||||
if not updated:
|
||||
return self.http_status(404, -1, 'Webhook not found')
|
||||
return self.success(data={'webhook': webhook})
|
||||
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
name = json_data.get('name')
|
||||
url = json_data.get('url')
|
||||
description = json_data.get('description')
|
||||
enabled = json_data.get('enabled')
|
||||
|
||||
await self.ap.webhook_service.update_webhook(webhook_id, name, url, description, enabled)
|
||||
return self.success()
|
||||
|
||||
elif quart.request.method == 'DELETE':
|
||||
await self.ap.webhook_service.delete_webhook(webhook_id)
|
||||
return self.success()
|
||||
deleted = await self.ap.webhook_service.delete_webhook(request_context, webhook_id)
|
||||
if not deleted:
|
||||
return self.http_status(404, -1, 'Webhook not found')
|
||||
return self.success()
|
||||
|
||||
@@ -4,6 +4,7 @@ import quart
|
||||
import traceback
|
||||
|
||||
from .. import group
|
||||
from .....utils import bounded_executor
|
||||
|
||||
|
||||
@group.group_class('webhooks', '/bots')
|
||||
@@ -30,7 +31,10 @@ class WebhookRouterGroup(group.RouterGroup):
|
||||
适配器返回的响应
|
||||
"""
|
||||
try:
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
# Public ingress never accepts X-Workspace-Id. The opaque bot UUID
|
||||
# is resolved against the already-bound runtime resource, which
|
||||
# carries the trusted Workspace and placement generation.
|
||||
runtime_bot = await self.ap.platform_mgr.resolve_public_bot(bot_uuid)
|
||||
|
||||
if not runtime_bot:
|
||||
return quart.jsonify({'error': 'Bot not found'}), 404
|
||||
@@ -41,14 +45,40 @@ class WebhookRouterGroup(group.RouterGroup):
|
||||
if not hasattr(runtime_bot.adapter, 'handle_unified_webhook'):
|
||||
return quart.jsonify({'error': 'Adapter does not support unified webhook'}), 501
|
||||
|
||||
response = await runtime_bot.adapter.handle_unified_webhook(
|
||||
bot_uuid=bot_uuid,
|
||||
path=path,
|
||||
request=quart.request,
|
||||
)
|
||||
async def dispatch():
|
||||
await self.ap.workspace_service.get_execution_binding(
|
||||
runtime_bot.workspace_uuid,
|
||||
expected_generation=runtime_bot.placement_generation,
|
||||
)
|
||||
return await runtime_bot.adapter.handle_unified_webhook(
|
||||
bot_uuid=bot_uuid,
|
||||
path=path,
|
||||
request=quart.request,
|
||||
)
|
||||
|
||||
with bounded_executor.blocking_work_scope(runtime_bot.workspace_uuid):
|
||||
persistence_mgr = self.ap.persistence_mgr
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud webhook dispatch requires an explicit tenant scope')
|
||||
async with tenant_scope(runtime_bot.workspace_uuid):
|
||||
response = await dispatch()
|
||||
else:
|
||||
response = await dispatch()
|
||||
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Webhook dispatch error for bot {bot_uuid}: {traceback.format_exc()}')
|
||||
return quart.jsonify({'error': str(e)}), 500
|
||||
except bounded_executor.BlockingWorkCapacityError as exc:
|
||||
return self.http_status(
|
||||
429,
|
||||
'blocking_work_capacity_exceeded',
|
||||
str(exc),
|
||||
)
|
||||
except Exception:
|
||||
request_id = self.request_id()
|
||||
self.ap.logger.error(
|
||||
f'Webhook dispatch error request_id={request_id} bot={bot_uuid}: {traceback.format_exc()}'
|
||||
)
|
||||
return self.internal_error_response(request_id)
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
import quart
|
||||
|
||||
from ...authz import Permission, permissions_for_role
|
||||
from ...context import RequestContext
|
||||
from ...service.user import AccountExistsLoginRequiredError, ControlPlaneDirectoryRequiredError
|
||||
from .....entity.persistence.workspace import Workspace, WorkspaceInvitation, WorkspaceMembership
|
||||
from .....entity.persistence.workspace import WorkspaceSource
|
||||
from .....workspace.collaboration import WorkspaceMemberView
|
||||
from .....workspace.errors import WorkspaceNotFoundError
|
||||
from .....workspace.invitation_delivery import InvitationDeliveryService
|
||||
from .. import group
|
||||
|
||||
|
||||
def _workspace_payload(workspace: Workspace) -> dict[str, typing.Any]:
|
||||
return {
|
||||
'uuid': workspace.uuid,
|
||||
'instance_uuid': workspace.instance_uuid,
|
||||
'name': workspace.name,
|
||||
'slug': workspace.slug,
|
||||
'type': workspace.type,
|
||||
'status': workspace.status,
|
||||
'source': workspace.source,
|
||||
}
|
||||
|
||||
|
||||
def _membership_payload(
|
||||
membership: WorkspaceMembership,
|
||||
*,
|
||||
email: str,
|
||||
) -> dict[str, typing.Any]:
|
||||
return {
|
||||
'uuid': membership.uuid,
|
||||
'workspace_uuid': membership.workspace_uuid,
|
||||
'account_uuid': membership.account_uuid,
|
||||
'email': email,
|
||||
'role': membership.role,
|
||||
'status': membership.status,
|
||||
'joined_at': membership.joined_at.isoformat() if membership.joined_at else None,
|
||||
'created_at': membership.created_at.isoformat() if membership.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _invitation_payload(invitation: WorkspaceInvitation) -> dict[str, typing.Any]:
|
||||
"""Serialize an invitation without its bearer-secret hash."""
|
||||
|
||||
return {
|
||||
'uuid': invitation.uuid,
|
||||
'workspace_uuid': invitation.workspace_uuid,
|
||||
'normalized_email': invitation.normalized_email,
|
||||
'role': invitation.role,
|
||||
'status': invitation.status,
|
||||
'expires_at': invitation.expires_at.isoformat(),
|
||||
'created_at': invitation.created_at.isoformat() if invitation.created_at else None,
|
||||
}
|
||||
|
||||
|
||||
@group.group_class('workspaces', '/api/v1/workspaces')
|
||||
class WorkspacesRouterGroup(group.RouterGroup):
|
||||
async def _run_in_workspace_uow(
|
||||
self, workspace_uuid: str, operation: typing.Callable[[], typing.Awaitable[typing.Any]]
|
||||
):
|
||||
"""Bind collaboration persistence to the selected tenant in Cloud."""
|
||||
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
async with self.ap.persistence_mgr.tenant_uow(workspace_uuid):
|
||||
return await operation()
|
||||
return await operation()
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/bootstrap', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> typing.Any:
|
||||
"""List the active Workspaces available to an authenticated Account.
|
||||
|
||||
This account-only endpoint intentionally runs before Workspace
|
||||
selection. It never accepts a selector as authority and does not
|
||||
choose a default Workspace for a multi-membership Account.
|
||||
"""
|
||||
|
||||
accesses = await self.ap.workspace_collaboration_service.list_account_workspaces(account.uuid)
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
workspaces: list[dict[str, typing.Any]] = []
|
||||
for access in accesses:
|
||||
plan_name: str | None = None
|
||||
if access.workspace.source == WorkspaceSource.CLOUD_PROJECTION.value and resolver is not None:
|
||||
entitlement = await resolver.resolve(
|
||||
access.workspace.uuid,
|
||||
minimum_revision=access.membership.projection_revision,
|
||||
)
|
||||
plan_name = entitlement.plan_name
|
||||
workspaces.append(
|
||||
{
|
||||
'workspace': _workspace_payload(access.workspace),
|
||||
'membership': _membership_payload(access.membership, email=account.user),
|
||||
'permissions': sorted(permissions_for_role(access.membership.role)),
|
||||
'placement_generation': access.execution.placement_generation,
|
||||
'plan_name': plan_name,
|
||||
}
|
||||
)
|
||||
return self.success(data={'workspaces': workspaces})
|
||||
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN)
|
||||
async def _(account) -> typing.Any:
|
||||
accesses = await self.ap.workspace_collaboration_service.list_account_workspaces(account.uuid)
|
||||
return self.success(data={'workspaces': [_workspace_payload(access.workspace) for access in accesses]})
|
||||
|
||||
@self.route('', methods=['POST'], permission=Permission.WORKSPACE_VIEW)
|
||||
async def _(request_context: RequestContext) -> typing.Any:
|
||||
if self.ap.workspace_service.policy.multi_workspace_enabled:
|
||||
return self.http_status(
|
||||
409,
|
||||
'control_plane_required',
|
||||
'Cloud Workspaces are created by the SaaS control plane',
|
||||
)
|
||||
return self.http_status(403, 'edition_limit', 'This edition supports one Workspace per instance')
|
||||
|
||||
@self.route('/current', methods=['GET'], permission=Permission.WORKSPACE_VIEW)
|
||||
async def _(request_context: RequestContext) -> typing.Any:
|
||||
membership = quart.g.workspace_membership
|
||||
account = await self.ap.user_service.get_user_by_uuid(request_context.account_uuid)
|
||||
if account is None:
|
||||
return self.http_status(401, 'invalid_authentication', 'Account not found')
|
||||
workspace = await self.ap.workspace_service.get_workspace(request_context.workspace_uuid)
|
||||
plan_name: str | None = None
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if workspace.source == WorkspaceSource.CLOUD_PROJECTION.value and resolver is not None:
|
||||
entitlement = await resolver.resolve(
|
||||
workspace.uuid,
|
||||
minimum_revision=request_context.entitlement_revision,
|
||||
)
|
||||
plan_name = entitlement.plan_name
|
||||
return self.success(
|
||||
data={
|
||||
'workspace': _workspace_payload(workspace),
|
||||
'membership': _membership_payload(membership, email=account.user),
|
||||
'permissions': sorted(request_context.workspace.permissions),
|
||||
'placement_generation': request_context.placement_generation,
|
||||
'plan_name': plan_name,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<workspace_uuid>', methods=['GET'], permission=Permission.WORKSPACE_VIEW)
|
||||
async def _(workspace_uuid: str, request_context: RequestContext) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
workspace = await self.ap.workspace_service.get_workspace(workspace_uuid)
|
||||
return self.success(data={'workspace': _workspace_payload(workspace)})
|
||||
|
||||
@self.route('/<workspace_uuid>/members', methods=['GET'], permission=Permission.MEMBER_VIEW)
|
||||
async def _(workspace_uuid: str, request_context: RequestContext) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
|
||||
async def list_members():
|
||||
return await self.ap.workspace_collaboration_service.list_members(
|
||||
workspace_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
members = await self._run_in_workspace_uow(workspace_uuid, list_members)
|
||||
return self.success(data={'members': [self._member_view_payload(item) for item in members]})
|
||||
|
||||
@self.route(
|
||||
'/<workspace_uuid>/invitations',
|
||||
methods=['GET', 'POST'],
|
||||
permission=Permission.MEMBER_INVITE,
|
||||
)
|
||||
async def _(workspace_uuid: str, request_context: RequestContext) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
if quart.request.method == 'GET':
|
||||
|
||||
async def list_invitations():
|
||||
return await self.ap.workspace_collaboration_service.list_invitations(
|
||||
workspace_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
invitations = await self._run_in_workspace_uow(workspace_uuid, list_invitations)
|
||||
return self.success(data={'invitations': [_invitation_payload(item) for item in invitations]})
|
||||
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
|
||||
async def create_invitation():
|
||||
return await self.ap.workspace_collaboration_service.create_invitation(
|
||||
workspace_uuid,
|
||||
quart.g.workspace_membership,
|
||||
str(data.get('email', '')),
|
||||
str(data.get('role', 'viewer')),
|
||||
)
|
||||
|
||||
created = await self._run_in_workspace_uow(workspace_uuid, create_invitation)
|
||||
delivery_service = self._invitation_delivery_service()
|
||||
link = delivery_service.build_invitation_link(created.token)
|
||||
workspace = await self.ap.workspace_service.get_workspace(workspace_uuid)
|
||||
delivery = await delivery_service.deliver_invitation(
|
||||
recipient_email=created.invitation.normalized_email,
|
||||
workspace_name=workspace.name,
|
||||
invitation_link=link,
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'invitation': _invitation_payload(created.invitation),
|
||||
'token': created.token,
|
||||
'link': link,
|
||||
'delivery': delivery.to_public_dict(),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route(
|
||||
'/<workspace_uuid>/invitations/<invitation_uuid>',
|
||||
methods=['DELETE'],
|
||||
permission=Permission.MEMBER_INVITE,
|
||||
)
|
||||
async def _(
|
||||
workspace_uuid: str,
|
||||
invitation_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
|
||||
async def revoke_invitation():
|
||||
return await self.ap.workspace_collaboration_service.revoke_invitation(
|
||||
workspace_uuid, invitation_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
invitation = await self._run_in_workspace_uow(workspace_uuid, revoke_invitation)
|
||||
return self.success(data={'invitation': _invitation_payload(invitation)})
|
||||
|
||||
@self.route(
|
||||
'/<workspace_uuid>/members/<account_uuid>',
|
||||
methods=['PATCH', 'DELETE'],
|
||||
permission=Permission.MEMBER_UPDATE_ROLE,
|
||||
)
|
||||
async def _(
|
||||
workspace_uuid: str,
|
||||
account_uuid: str,
|
||||
request_context: RequestContext,
|
||||
) -> typing.Any:
|
||||
self._require_current_workspace(workspace_uuid, request_context)
|
||||
if quart.request.method == 'DELETE':
|
||||
if Permission.MEMBER_REMOVE.value not in request_context.workspace.permissions:
|
||||
return self.http_status(403, 'permission_denied', 'Member removal permission is required')
|
||||
|
||||
async def remove_member():
|
||||
return await self.ap.workspace_collaboration_service.remove_member(
|
||||
workspace_uuid, account_uuid, quart.g.workspace_membership
|
||||
)
|
||||
|
||||
member = await self._run_in_workspace_uow(workspace_uuid, remove_member)
|
||||
return self.success(data={'account_uuid': member.account_uuid})
|
||||
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
|
||||
async def update_member_role():
|
||||
return await self.ap.workspace_collaboration_service.update_member_role(
|
||||
workspace_uuid,
|
||||
account_uuid,
|
||||
str(data.get('role', '')),
|
||||
quart.g.workspace_membership,
|
||||
)
|
||||
|
||||
member = await self._run_in_workspace_uow(workspace_uuid, update_member_role)
|
||||
account = await self.ap.user_service.get_user_by_uuid(member.account_uuid)
|
||||
return self.success(
|
||||
data={
|
||||
'member': _membership_payload(
|
||||
member,
|
||||
email=account.user if account is not None else '',
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _require_current_workspace(workspace_uuid: str, request_context: RequestContext) -> None:
|
||||
if workspace_uuid != request_context.workspace_uuid:
|
||||
raise WorkspaceNotFoundError('Workspace not found')
|
||||
|
||||
def _invitation_delivery_service(self) -> InvitationDeliveryService:
|
||||
service = getattr(self.ap, 'invitation_delivery_service', None)
|
||||
if service is None:
|
||||
service = InvitationDeliveryService(self.ap)
|
||||
self.ap.invitation_delivery_service = service
|
||||
return service
|
||||
|
||||
@staticmethod
|
||||
def _member_view_payload(view: WorkspaceMemberView) -> dict[str, typing.Any]:
|
||||
return _membership_payload(view.membership, email=view.email)
|
||||
|
||||
|
||||
@group.group_class('invitations', '/api/v1/invitations')
|
||||
class InvitationsRouterGroup(group.RouterGroup):
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/inspect', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> typing.Any:
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
invitation, workspace = await self.ap.workspace_collaboration_service.inspect_invitation(
|
||||
str(data.get('token', ''))
|
||||
)
|
||||
return self.success(
|
||||
data={
|
||||
'invitation': _invitation_payload(invitation),
|
||||
'workspace': _workspace_payload(workspace),
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/accept', methods=['POST'], auth_type=group.AuthType.NONE)
|
||||
async def _() -> typing.Any:
|
||||
data = await quart.request.get_json(silent=True) or {}
|
||||
invitation_token = str(data.get('token', ''))
|
||||
if not invitation_token:
|
||||
return self.http_status(400, 'invitation_invalid', 'Invitation token is required')
|
||||
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if authorization.startswith('Bearer '):
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') != 'cloud':
|
||||
return self.http_status(
|
||||
409,
|
||||
'invitation_logout_required',
|
||||
'Sign out before creating the invited local Account',
|
||||
)
|
||||
try:
|
||||
account = await self.ap.user_service.get_authenticated_account(
|
||||
authorization.removeprefix('Bearer ')
|
||||
)
|
||||
if isinstance(account, str):
|
||||
account = await self.ap.user_service.get_user_by_email(account)
|
||||
except Exception as exc:
|
||||
return self._auth_error_response(exc)
|
||||
if account is None:
|
||||
return self.http_status(401, 'invalid_authentication', 'Account not found')
|
||||
membership = await self.ap.workspace_collaboration_service.accept_invitation(
|
||||
invitation_token,
|
||||
account.uuid,
|
||||
)
|
||||
token = await self.ap.user_service.generate_jwt_token(account)
|
||||
return self.success(data={'token': token, 'workspace_uuid': membership.workspace_uuid})
|
||||
|
||||
registration = data.get('registration')
|
||||
if getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud':
|
||||
return self.http_status(
|
||||
401,
|
||||
'account_exists_login_required',
|
||||
'Login with your LangBot Account to accept this invitation',
|
||||
)
|
||||
if not isinstance(registration, dict):
|
||||
return self.http_status(
|
||||
401,
|
||||
'account_exists_login_required',
|
||||
'Sign in or provide registration details to accept this invitation',
|
||||
)
|
||||
password = registration.get('password')
|
||||
if not isinstance(password, str) or len(password) < 8:
|
||||
return self.http_status(400, 'invalid_password', 'Password must contain at least 8 characters')
|
||||
try:
|
||||
_, membership = await self.ap.user_service.register_invited_account(
|
||||
invitation_token,
|
||||
str(registration.get('email', '')),
|
||||
password,
|
||||
)
|
||||
except ControlPlaneDirectoryRequiredError as exc:
|
||||
return self.http_status(409, exc.code, str(exc))
|
||||
except AccountExistsLoginRequiredError as exc:
|
||||
return self.http_status(409, exc.code, str(exc))
|
||||
return self.success(data={'workspace_uuid': membership.workspace_uuid, 'login_required': True})
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import typing
|
||||
|
||||
import quart
|
||||
import quart_cors
|
||||
@@ -27,6 +28,37 @@ importutil.import_modules_in_pkg(groups_knowledge)
|
||||
importutil.import_modules_in_pkg(groups_resources)
|
||||
|
||||
|
||||
class BoundedJSONRequest(quart.Request):
|
||||
"""Parse bounded HTTP JSON bodies outside the shared event loop."""
|
||||
|
||||
async def get_json(
|
||||
self,
|
||||
force: bool = False,
|
||||
silent: bool = False,
|
||||
cache: bool = True,
|
||||
) -> typing.Any:
|
||||
# Keep Quart's cache and error semantics, changing only where the
|
||||
# potentially 10 MiB JSON decoder runs. The RouterGroup establishes a
|
||||
# trusted Workspace blocking-work scope before calling route handlers.
|
||||
if cache and self._cached_json[silent] is not Ellipsis:
|
||||
return self._cached_json[silent]
|
||||
if not (force or self.is_json):
|
||||
return None
|
||||
|
||||
data = await self.get_data(cache=cache, as_text=False)
|
||||
try:
|
||||
result = await asyncio.to_thread(self.json_module.loads, data)
|
||||
except ValueError as error:
|
||||
if silent:
|
||||
result = None
|
||||
else:
|
||||
result = self.on_json_loading_failed(error)
|
||||
|
||||
if cache:
|
||||
self._cached_json[silent] = result
|
||||
return result
|
||||
|
||||
|
||||
class HTTPController:
|
||||
ap: app.Application
|
||||
|
||||
@@ -35,6 +67,7 @@ class HTTPController:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
self.quart_app = quart.Quart(__name__)
|
||||
self.quart_app.request_class = BoundedJSONRequest
|
||||
quart_cors.cors(self.quart_app, allow_origin='*')
|
||||
|
||||
# Set maximum content length to prevent large file uploads
|
||||
@@ -103,6 +136,7 @@ class HTTPController:
|
||||
config.accesslog = '-'
|
||||
config.bind = [f'{host}:{port}']
|
||||
config.errorlog = config.accesslog
|
||||
config.websocket_max_message_size = group.MAX_FILE_SIZE
|
||||
|
||||
asgi_app = self.quart_app
|
||||
if self.mcp_mount is not None:
|
||||
@@ -113,7 +147,16 @@ class HTTPController:
|
||||
async def register_routes(self) -> None:
|
||||
@self.quart_app.route('/healthz')
|
||||
async def healthz():
|
||||
return {'code': 0, 'msg': 'ok'}
|
||||
get_resource_stats = getattr(
|
||||
self.ap,
|
||||
'get_runtime_resource_stats',
|
||||
None,
|
||||
)
|
||||
return {
|
||||
'code': 0,
|
||||
'msg': 'ok',
|
||||
'resources': (get_resource_stats() if callable(get_resource_stats) else {}),
|
||||
}
|
||||
|
||||
for g in group.preregistered_groups:
|
||||
ginst = g(self.ap, self.quart_app)
|
||||
|
||||
@@ -1,97 +1,304 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import datetime
|
||||
import hashlib
|
||||
import secrets
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import apikey
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..authz import Permission, PermissionDeniedError
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class ApiKeyIdentity:
|
||||
"""Trusted Workspace identity derived from an API-key secret."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
api_key_uuid: str
|
||||
permissions: frozenset[str]
|
||||
|
||||
|
||||
class ApiKeyService:
|
||||
ap: app.Application
|
||||
"""Manage hashed, Workspace-bound API keys."""
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(self, ap: Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_api_keys(self) -> list[dict]:
|
||||
"""Get all API keys"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(apikey.ApiKey))
|
||||
@staticmethod
|
||||
def _hash_secret(secret: str) -> str:
|
||||
return hashlib.sha256(secret.encode('utf-8')).hexdigest()
|
||||
|
||||
keys = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(apikey.ApiKey, key) for key in keys]
|
||||
@staticmethod
|
||||
def _utcnow() -> datetime.datetime:
|
||||
return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
|
||||
async def create_api_key(self, name: str, description: str = '') -> dict:
|
||||
"""Create a new API key"""
|
||||
# Generate a secure random API key
|
||||
key = f'lbk_{secrets.token_urlsafe(32)}'
|
||||
@staticmethod
|
||||
def _normalize_scopes(
|
||||
scopes: typing.Iterable[str] | None,
|
||||
*,
|
||||
default: typing.Iterable[str] = (),
|
||||
) -> list[str]:
|
||||
requested = list(default if scopes is None else scopes)
|
||||
valid = {permission.value for permission in Permission}
|
||||
normalized: list[str] = []
|
||||
for scope in requested:
|
||||
if not isinstance(scope, str):
|
||||
raise ValueError('API key scopes must be strings')
|
||||
value = scope.strip()
|
||||
if value not in valid:
|
||||
raise ValueError(f'Unknown API key scope: {value}')
|
||||
if value not in normalized:
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
key_data = {'name': name, 'key': key, 'description': description}
|
||||
def _serialize(self, row: typing.Any) -> dict[str, typing.Any]:
|
||||
value = self.ap.persistence_mgr.serialize_model(apikey.ApiKey, row)
|
||||
value.pop('key_hash', None)
|
||||
# The secret is deliberately unrecoverable after creation.
|
||||
value['secret_available'] = False
|
||||
return value
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(apikey.ApiKey).values(**key_data))
|
||||
|
||||
# Retrieve the created key
|
||||
async def get_api_keys(self, context: TenantContext) -> list[dict[str, typing.Any]]:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).order_by(apikey.ApiKey.created_at, apikey.ApiKey.id),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
created_key = result.first()
|
||||
return [self._serialize(key) for key in result.all()]
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(apikey.ApiKey, created_key)
|
||||
async def create_api_key(
|
||||
self,
|
||||
context: TenantContext,
|
||||
name: str,
|
||||
description: str = '',
|
||||
*,
|
||||
scopes: typing.Iterable[str] | None = None,
|
||||
expires_at: datetime.datetime | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
normalized_name = name.strip()
|
||||
if not normalized_name:
|
||||
raise ValueError('Name is required')
|
||||
if expires_at is not None:
|
||||
if expires_at.tzinfo is not None:
|
||||
expires_at = expires_at.astimezone(datetime.UTC).replace(tzinfo=None)
|
||||
if expires_at <= self._utcnow():
|
||||
raise ValueError('API key expiry must be in the future')
|
||||
|
||||
async def get_api_key(self, key_id: int) -> dict | None:
|
||||
"""Get a specific API key by ID"""
|
||||
default_scopes = getattr(getattr(context, 'workspace', None), 'permissions', frozenset())
|
||||
normalized_scopes = self._normalize_scopes(scopes, default=default_scopes)
|
||||
allowed_scopes = frozenset(default_scopes)
|
||||
unauthorized_scopes = sorted(set(normalized_scopes) - allowed_scopes)
|
||||
if unauthorized_scopes:
|
||||
# API-key management delegates the caller's authority; it must not
|
||||
# become a path for minting a stronger principal.
|
||||
raise PermissionDeniedError(unauthorized_scopes[0])
|
||||
secret = f'lbk_{secrets.token_urlsafe(32)}'
|
||||
key_uuid = str(uuid.uuid4())
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(apikey.ApiKey).values(
|
||||
uuid=key_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
created_by_account_uuid=getattr(context, 'account_uuid', None),
|
||||
name=normalized_name,
|
||||
key_hash=self._hash_secret(secret),
|
||||
scopes=normalized_scopes,
|
||||
status=apikey.ApiKeyStatus.ACTIVE.value,
|
||||
expires_at=expires_at,
|
||||
description=description.strip(),
|
||||
)
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.id == key_id)
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.uuid == key_uuid),
|
||||
apikey.ApiKey,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
created = result.first()
|
||||
if created is None:
|
||||
raise RuntimeError('Created API key could not be loaded')
|
||||
value = self._serialize(created)
|
||||
value['key'] = secret
|
||||
value['secret_available'] = True
|
||||
return value
|
||||
|
||||
async def get_api_key(self, context: TenantContext, key_id: int) -> dict[str, typing.Any] | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.id == key_id),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
key = result.first()
|
||||
return None if key is None else self._serialize(key)
|
||||
|
||||
if key is None:
|
||||
async def authenticate_api_key(self, secret: str) -> ApiKeyIdentity | None:
|
||||
"""Authenticate a secret and derive its Workspace without trusting headers."""
|
||||
|
||||
if not isinstance(secret, str) or not secret.strip():
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(apikey.ApiKey, key)
|
||||
global_secret = self.ap.instance_config.data.get('api', {}).get('global_api_key', '')
|
||||
if global_secret and secrets.compare_digest(secret, global_secret):
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
if workspace_service is None or workspace_service.policy.multi_workspace_enabled:
|
||||
return None
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid='global-oss-api-key',
|
||||
permissions=frozenset(permission.value for permission in Permission),
|
||||
)
|
||||
|
||||
async def verify_api_key(self, key: str) -> bool:
|
||||
"""Verify if an API key is valid.
|
||||
if not secret.startswith('lbk_'):
|
||||
return None
|
||||
secret_hash = self._hash_secret(secret)
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
discovery_uow = getattr(self.ap.persistence_mgr, 'api_key_discovery_uow', None)
|
||||
if current_session() is None and callable(discovery_uow):
|
||||
async with discovery_uow(secret_hash) as discovery:
|
||||
key = await discovery.session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
key = result.first()
|
||||
if key is None:
|
||||
return None
|
||||
discovered_workspace_uuid = key.workspace_uuid
|
||||
discovered_key_id = key.id
|
||||
now = self._utcnow()
|
||||
|
||||
A key is accepted if it matches the global API key configured in
|
||||
``config.yaml`` (``api.global_api_key``) — which requires no login
|
||||
session and no database record — or if it matches a key created via
|
||||
the web UI (stored in the database, prefixed with ``lbk_``).
|
||||
"""
|
||||
if not isinstance(key, str) or not key:
|
||||
return False
|
||||
async def bind_and_record_use() -> tuple[typing.Any, typing.Any] | None:
|
||||
# Re-read inside the tenant transaction. A revoke/expiry racing
|
||||
# discovery must not result in an authenticated identity.
|
||||
active_session = current_session()
|
||||
if active_session is not None:
|
||||
scoped_key = await active_session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
else: # compatibility for isolated service tests
|
||||
scoped_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
scoped_key = scoped_result.first()
|
||||
if scoped_key is None or scoped_key.status != apikey.ApiKeyStatus.ACTIVE.value:
|
||||
return None
|
||||
if scoped_key.expires_at is not None and scoped_key.expires_at <= now:
|
||||
return None
|
||||
|
||||
# 1. Global API key from config.yaml (no DB lookup, no login state).
|
||||
# Note: config completion only backfills top-level keys, so existing
|
||||
# installs may not have this key — access it defensively.
|
||||
global_api_key = self.ap.instance_config.data.get('api', {}).get('global_api_key', '')
|
||||
if global_api_key and secrets.compare_digest(key, global_api_key):
|
||||
return True
|
||||
binding = await self.ap.workspace_service.get_execution_binding(discovered_workspace_uuid)
|
||||
updated = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(
|
||||
apikey.ApiKey.id == scoped_key.id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
apikey.ApiKey.status == apikey.ApiKeyStatus.ACTIVE.value,
|
||||
)
|
||||
.values(last_used_at=now)
|
||||
.returning(apikey.ApiKey.id)
|
||||
)
|
||||
# Authentication and revocation race on this atomic predicate. If
|
||||
# revoke won, no active row is returned and the stale object read
|
||||
# above must never become an authenticated identity.
|
||||
if updated.scalar_one_or_none() is None:
|
||||
return None
|
||||
return binding, scoped_key
|
||||
|
||||
# 2. Web-UI-created keys are stored in the database and prefixed lbk_.
|
||||
if not key.startswith('lbk_'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if current_session() is None and callable(tenant_uow):
|
||||
async with tenant_uow(discovered_workspace_uuid):
|
||||
bound = await bind_and_record_use()
|
||||
else:
|
||||
bound = await bind_and_record_use()
|
||||
if bound is None:
|
||||
return None
|
||||
binding, scoped_key = bound
|
||||
raw_scopes = list(scoped_key.scopes or [])
|
||||
permissions = (
|
||||
frozenset(permission.value for permission in Permission)
|
||||
if '*' in raw_scopes
|
||||
else frozenset(self._normalize_scopes(raw_scopes))
|
||||
)
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid=scoped_key.uuid,
|
||||
permissions=permissions,
|
||||
)
|
||||
|
||||
key_obj = result.first()
|
||||
return key_obj is not None
|
||||
async def verify_api_key(self, secret: str) -> bool:
|
||||
try:
|
||||
return await self.authenticate_api_key(secret) is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def delete_api_key(self, key_id: int) -> None:
|
||||
"""Delete an API key"""
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.delete(apikey.ApiKey).where(apikey.ApiKey.id == key_id))
|
||||
|
||||
async def update_api_key(self, key_id: int, name: str = None, description: str = None) -> None:
|
||||
"""Update an API key's metadata (name, description)"""
|
||||
update_data = {}
|
||||
if name is not None:
|
||||
update_data['name'] = name
|
||||
if description is not None:
|
||||
update_data['description'] = description
|
||||
|
||||
if update_data:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey).where(apikey.ApiKey.id == key_id).values(**update_data)
|
||||
async def delete_api_key(self, context: TenantContext, key_id: int) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(apikey.ApiKey.id == key_id)
|
||||
.values(status=apikey.ApiKeyStatus.REVOKED.value),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', 0) == 0:
|
||||
raise WorkspaceNotFoundError('API key not found')
|
||||
|
||||
async def update_api_key(
|
||||
self,
|
||||
context: TenantContext,
|
||||
key_id: int,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
) -> None:
|
||||
update_data: dict[str, typing.Any] = {}
|
||||
if name is not None:
|
||||
normalized_name = name.strip()
|
||||
if not normalized_name:
|
||||
raise ValueError('Name is required')
|
||||
update_data['name'] = normalized_name
|
||||
if description is not None:
|
||||
update_data['description'] = description.strip()
|
||||
if not update_data:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(apikey.ApiKey).where(apikey.ApiKey.id == key_id).values(**update_data),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', 0) == 0:
|
||||
raise WorkspaceNotFoundError('API key not found')
|
||||
|
||||
@@ -2,11 +2,12 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import sqlalchemy
|
||||
import typing
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
class BotService:
|
||||
@@ -17,9 +18,11 @@ class BotService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_bots(self, include_secret: bool = True) -> list[dict]:
|
||||
async def get_bots(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_bot.Bot), persistence_bot.Bot, context)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
@@ -29,10 +32,14 @@ class BotService:
|
||||
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
|
||||
|
||||
async def get_bot(self, bot_uuid: str, include_secret: bool = True) -> dict | None:
|
||||
async def get_bot(self, context: TenantContext, bot_uuid: str, include_secret: bool = False) -> dict | None:
|
||||
"""获取机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
bot = result.first()
|
||||
@@ -46,15 +53,20 @@ class BotService:
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
|
||||
|
||||
async def get_runtime_bot_info(self, bot_uuid: str, include_secret: bool = True) -> dict:
|
||||
async def get_runtime_bot_info(
|
||||
self,
|
||||
context: TenantContext,
|
||||
bot_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict:
|
||||
"""获取机器人运行时信息"""
|
||||
persistence_bot = await self.get_bot(bot_uuid, include_secret)
|
||||
persistence_bot = await self.get_bot(context, bot_uuid, include_secret)
|
||||
if persistence_bot is None:
|
||||
raise Exception('Bot not found')
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
|
||||
adapter_runtime_values = {}
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is not None:
|
||||
adapter_runtime_values['bot_account_id'] = runtime_bot.adapter.bot_account_id
|
||||
|
||||
@@ -86,22 +98,29 @@ class BotService:
|
||||
|
||||
return persistence_bot
|
||||
|
||||
async def create_bot(self, bot_data: dict) -> str:
|
||||
async def create_bot(self, context: TenantContext, bot_data: dict) -> str:
|
||||
"""Create bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_bots = limitation.get('max_bots', -1)
|
||||
if max_bots >= 0:
|
||||
existing_bots = await self.get_bots()
|
||||
existing_bots = await self.get_bots(context)
|
||||
if len(existing_bots) >= max_bots:
|
||||
raise ValueError(f'Maximum number of bots ({max_bots}) reached')
|
||||
|
||||
# TODO: 检查配置信息格式
|
||||
bot_data = bot_data.copy()
|
||||
bot_data['uuid'] = str(uuid.uuid4())
|
||||
bot_data['workspace_uuid'] = workspace_uuid
|
||||
|
||||
# bind the most recently updated pipeline if any exist
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -112,61 +131,84 @@ class BotService:
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(bot_data))
|
||||
|
||||
bot = await self.get_bot(bot_data['uuid'])
|
||||
bot = await self.get_bot(context, bot_data['uuid'], include_secret=True)
|
||||
|
||||
await self.ap.platform_mgr.load_bot(bot)
|
||||
await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
return bot_data['uuid']
|
||||
|
||||
async def update_bot(self, bot_uuid: str, bot_data: dict) -> None:
|
||||
async def update_bot(self, context: TenantContext, bot_uuid: str, bot_data: dict) -> None:
|
||||
"""Update bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
update_data = bot_data.copy()
|
||||
|
||||
if 'uuid' in update_data:
|
||||
del update_data['uuid']
|
||||
update_data.pop('uuid', None)
|
||||
update_data.pop('workspace_uuid', None)
|
||||
|
||||
# set use_pipeline_name
|
||||
if 'use_pipeline_uuid' in update_data:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
update_data['use_pipeline_name'] = pipeline.name
|
||||
else:
|
||||
raise Exception('Pipeline not found')
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
# select from db
|
||||
bot = await self.get_bot(bot_uuid)
|
||||
bot = await self.get_bot(context, bot_uuid, include_secret=True)
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(bot)
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
if runtime_bot.enable:
|
||||
await runtime_bot.run()
|
||||
|
||||
# update all conversation that use this bot
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if session.using_conversation is not None and session.using_conversation.bot_uuid == bot_uuid:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
and session.using_conversation.bot_uuid == bot_uuid
|
||||
and getattr(session, 'workspace_uuid', workspace_uuid) == workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
|
||||
async def delete_bot(self, bot_uuid: str) -> None:
|
||||
async def delete_bot(self, context: TenantContext, bot_uuid: str) -> None:
|
||||
"""Delete bot"""
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
async def list_event_logs(
|
||||
self, bot_uuid: str, from_index: int, max_count: int
|
||||
) -> typing.Tuple[list[dict], int, int, int]:
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
self, context: TenantContext, bot_uuid: str, from_index: int, max_count: int
|
||||
) -> tuple[list[dict], int]:
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is None:
|
||||
raise Exception('Bot not found')
|
||||
|
||||
@@ -174,7 +216,14 @@ class BotService:
|
||||
|
||||
return [log.to_json() for log in logs], total_count
|
||||
|
||||
async def send_message(self, bot_uuid: str, target_type: str, target_id: str, message_chain_data: dict) -> None:
|
||||
async def send_message(
|
||||
self,
|
||||
context: TenantContext,
|
||||
bot_uuid: str,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
message_chain_data: dict,
|
||||
) -> None:
|
||||
"""Send message to a specific target via bot
|
||||
|
||||
Args:
|
||||
@@ -183,11 +232,14 @@ class BotService:
|
||||
target_id: The ID of the target
|
||||
message_chain_data: The message chain data in dict format
|
||||
"""
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
|
||||
# Import here to avoid circular imports
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
|
||||
# Get runtime bot
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is None:
|
||||
raise Exception(f'Bot not found: {bot_uuid}')
|
||||
|
||||
@@ -202,19 +254,29 @@ class BotService:
|
||||
|
||||
# ============ Bot Admins ============
|
||||
|
||||
async def get_bot_admins(self, bot_uuid: str) -> list[dict]:
|
||||
async def get_bot_admins(self, context: TenantContext, bot_uuid: str) -> list[dict]:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid),
|
||||
persistence_bot.BotAdmin,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return [{'id': r.id, 'launcher_type': r.launcher_type, 'launcher_id': r.launcher_id} for r in result.all()]
|
||||
|
||||
async def add_bot_admin(self, bot_uuid: str, launcher_type: str, launcher_id: str) -> int:
|
||||
async def add_bot_admin(self, context: TenantContext, bot_uuid: str, launcher_type: str, launcher_id: str) -> int:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_bot.BotAdmin).values(
|
||||
workspace_uuid=workspace_uuid,
|
||||
bot_uuid=bot_uuid,
|
||||
launcher_type=launcher_type,
|
||||
launcher_id=launcher_id,
|
||||
@@ -222,12 +284,18 @@ class BotService:
|
||||
)
|
||||
return result.inserted_primary_key[0]
|
||||
|
||||
async def delete_bot_admin(self, bot_uuid: str, admin_id: int) -> None:
|
||||
async def delete_bot_admin(self, context: TenantContext, bot_uuid: str, admin_id: int) -> None:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.BotAdmin).where(
|
||||
persistence_bot.BotAdmin.bot_uuid == bot_uuid,
|
||||
persistence_bot.BotAdmin.id == admin_id,
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_bot.BotAdmin).where(
|
||||
persistence_bot.BotAdmin.bot_uuid == bot_uuid,
|
||||
persistence_bot.BotAdmin.id == admin_id,
|
||||
),
|
||||
persistence_bot.BotAdmin,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2,8 +2,13 @@ from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....api.http.authz import WorkspaceRequiredError
|
||||
from ....api.http.context import ExecutionContext, RequestContext
|
||||
from ....core import app
|
||||
from ....entity.persistence import rag as persistence_rag
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
class KnowledgeService:
|
||||
@@ -14,34 +19,69 @@ class KnowledgeService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_knowledge_bases(self) -> list[dict]:
|
||||
@staticmethod
|
||||
def _execution_context(context: RequestContext | ExecutionContext) -> ExecutionContext:
|
||||
if isinstance(context, RequestContext):
|
||||
return ExecutionContext.from_request(context)
|
||||
if isinstance(context, ExecutionContext):
|
||||
return context
|
||||
raise WorkspaceRequiredError('RequestContext or ExecutionContext is required')
|
||||
|
||||
async def get_knowledge_bases(self, context: TenantContext, *, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有知识库"""
|
||||
return await self.ap.rag_mgr.get_all_knowledge_base_details()
|
||||
require_workspace_uuid(context)
|
||||
knowledge_bases = await self.ap.rag_mgr.get_all_knowledge_base_details(context)
|
||||
return knowledge_bases if include_secret else [redact_secrets(base) for base in knowledge_bases]
|
||||
|
||||
async def get_knowledge_base(self, kb_uuid: str) -> dict | None:
|
||||
async def get_knowledge_base(
|
||||
self,
|
||||
context: TenantContext,
|
||||
kb_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""获取知识库"""
|
||||
return await self.ap.rag_mgr.get_knowledge_base_details(kb_uuid)
|
||||
require_workspace_uuid(context)
|
||||
knowledge_base = await self.ap.rag_mgr.get_knowledge_base_details(context, kb_uuid)
|
||||
if knowledge_base is None or include_secret:
|
||||
return knowledge_base
|
||||
return redact_secrets(knowledge_base)
|
||||
|
||||
async def create_knowledge_base(self, kb_data: dict) -> str:
|
||||
async def create_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_data: dict,
|
||||
) -> str:
|
||||
"""创建知识库"""
|
||||
require_workspace_uuid(context)
|
||||
# In new architecture, we delegate entirely to RAGManager which uses plugins.
|
||||
# Legacy internal KB creation is removed.
|
||||
limitation = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('system', {}).get('limitation', {})
|
||||
)
|
||||
max_knowledge_bases = limitation.get('max_knowledge_bases', -1)
|
||||
if max_knowledge_bases >= 0:
|
||||
knowledge_bases = await self.ap.rag_mgr.get_all_knowledge_base_details(context)
|
||||
if len(knowledge_bases) >= max_knowledge_bases:
|
||||
raise ValueError(f'Maximum number of knowledge bases ({max_knowledge_bases}) reached')
|
||||
|
||||
knowledge_engine_plugin_id = kb_data.get('knowledge_engine_plugin_id')
|
||||
if not knowledge_engine_plugin_id:
|
||||
raise ValueError('knowledge_engine_plugin_id is required')
|
||||
|
||||
creation_settings = kb_data.get('creation_settings', {})
|
||||
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
|
||||
retrieval_settings = kb_data.get('retrieval_settings', {})
|
||||
|
||||
# Validate required fields based on plugin's creation_schema and retrieval_schema
|
||||
await self._validate_schema_required_fields(
|
||||
context,
|
||||
knowledge_engine_plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
|
||||
kb = await self.ap.rag_mgr.create_knowledge_base(
|
||||
context,
|
||||
name=kb_data.get('name', 'Untitled'),
|
||||
knowledge_engine_plugin_id=knowledge_engine_plugin_id,
|
||||
creation_settings=creation_settings,
|
||||
@@ -52,6 +92,7 @@ class KnowledgeService:
|
||||
|
||||
async def _validate_schema_required_fields(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
plugin_id: str,
|
||||
creation_settings: dict,
|
||||
retrieval_settings: dict,
|
||||
@@ -69,7 +110,11 @@ class KnowledgeService:
|
||||
Raises:
|
||||
ValueError: If any required field is missing or empty.
|
||||
"""
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return
|
||||
|
||||
# Validate creation_schema
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
creation_schema = await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
self._check_required_fields(creation_schema, creation_settings, 'creation_settings')
|
||||
@@ -79,6 +124,7 @@ class KnowledgeService:
|
||||
self.ap.logger.warning(f'Failed to get creation_schema for validation: {e}')
|
||||
|
||||
# Validate retrieval_schema
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
retrieval_schema = await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
self._check_required_fields(retrieval_schema, retrieval_settings, 'retrieval_settings')
|
||||
@@ -151,8 +197,16 @@ class KnowledgeService:
|
||||
)
|
||||
raise ValueError(f'{field_label} is required ({context}.{field_name})')
|
||||
|
||||
async def update_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
|
||||
async def update_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
kb_data: dict,
|
||||
) -> None:
|
||||
"""更新知识库"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
# Filter to only mutable fields
|
||||
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
|
||||
|
||||
@@ -162,17 +216,18 @@ class KnowledgeService:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(filtered_data)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(kb_uuid)
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(context, kb_uuid)
|
||||
|
||||
kb = await self.get_knowledge_base(kb_uuid)
|
||||
kb = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
|
||||
if kb is None:
|
||||
raise Exception('Knowledge base not found after update')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self.ap.rag_mgr.load_knowledge_base(kb)
|
||||
await self.ap.rag_mgr.load_knowledge_base(context, kb)
|
||||
|
||||
async def _check_doc_capability(self, kb_uuid: str, operation: str) -> None:
|
||||
async def _check_doc_capability(self, context: TenantContext, kb_uuid: str, operation: str) -> None:
|
||||
"""Check if the KB's Knowledge Engine supports document operations.
|
||||
|
||||
Args:
|
||||
@@ -182,104 +237,145 @@ class KnowledgeService:
|
||||
Raises:
|
||||
Exception: If the KB does not support doc_ingestion.
|
||||
"""
|
||||
kb_info = await self.ap.rag_mgr.get_knowledge_base_details(kb_uuid)
|
||||
kb_info = await self.ap.rag_mgr.get_knowledge_base_details(context, kb_uuid)
|
||||
if not kb_info:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
capabilities = kb_info.get('knowledge_engine', {}).get('capabilities', [])
|
||||
if 'doc_ingestion' not in capabilities:
|
||||
raise Exception(f'This knowledge base does not support {operation}')
|
||||
|
||||
async def store_file(self, kb_uuid: str, file_id: str, parser_plugin_id: str | None = None) -> str:
|
||||
async def store_file(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
file_id: str,
|
||||
parser_plugin_id: str | None = None,
|
||||
) -> str:
|
||||
"""存储文件"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self._check_doc_capability(kb_uuid, 'document upload')
|
||||
await self._check_doc_capability(context, kb_uuid, 'document upload')
|
||||
|
||||
result = await runtime_kb.store_file(file_id, parser_plugin_id=parser_plugin_id)
|
||||
result = await runtime_kb.store_file(execution_context, file_id, parser_plugin_id=parser_plugin_id)
|
||||
|
||||
# Update the KB's updated_at timestamp
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(updated_at=sqlalchemy.func.now())
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def retrieve_knowledge_base(
|
||||
self, kb_uuid: str, query: str, retrieval_settings: dict | None = None
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
query: str,
|
||||
retrieval_settings: dict | None = None,
|
||||
) -> list[dict]:
|
||||
"""检索知识库"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# Pass retrieval_settings
|
||||
results = await runtime_kb.retrieve(query, settings=retrieval_settings)
|
||||
results = await runtime_kb.retrieve(execution_context, query, settings=retrieval_settings)
|
||||
|
||||
return [result.model_dump() for result in results]
|
||||
|
||||
async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]:
|
||||
async def get_files_by_knowledge_base(self, context: TenantContext, kb_uuid: str) -> list[dict]:
|
||||
"""获取知识库文件"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
files = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_rag.File, file) for file in files]
|
||||
|
||||
async def delete_file(self, kb_uuid: str, file_id: str) -> None:
|
||||
async def delete_file(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
file_id: str,
|
||||
) -> None:
|
||||
"""删除文件"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self._check_doc_capability(kb_uuid, 'document deletion')
|
||||
await self._check_doc_capability(context, kb_uuid, 'document deletion')
|
||||
|
||||
await runtime_kb.delete_file(file_id)
|
||||
await runtime_kb.delete_file(execution_context, file_id)
|
||||
|
||||
# Update the KB's updated_at timestamp
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(updated_at=sqlalchemy.func.now())
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
async def delete_knowledge_base(self, kb_uuid: str) -> None:
|
||||
async def delete_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
) -> None:
|
||||
"""删除知识库"""
|
||||
# Delete from DB first to commit the deletion, then clean up runtime/plugin (best-effort)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.KnowledgeBase).where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# delete files
|
||||
# NOTE: Chunk cleanup is for legacy (pre-plugin) KBs that stored chunks locally.
|
||||
# For plugin-based Knowledge Engines, the Chunk table is not populated, so this is a no-op.
|
||||
files = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
for file in files:
|
||||
# delete chunks
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.Chunk).where(persistence_rag.Chunk.file_id == file.uuid)
|
||||
sqlalchemy.delete(persistence_rag.Chunk)
|
||||
.where(persistence_rag.Chunk.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.Chunk.file_id == file.uuid)
|
||||
)
|
||||
# delete file
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.File).where(persistence_rag.File.uuid == file.uuid)
|
||||
sqlalchemy.delete(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file.uuid)
|
||||
)
|
||||
|
||||
# Remove from runtime and notify plugin (best-effort, DB is already cleaned up)
|
||||
await self.ap.rag_mgr.delete_knowledge_base(kb_uuid)
|
||||
# Remove from runtime and notify plugin before deleting the owning row.
|
||||
await self.ap.rag_mgr.delete_knowledge_base(context, kb_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.KnowledgeBase)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
# ================= Knowledge Engine Discovery =================
|
||||
|
||||
async def list_knowledge_engines(self) -> list[dict]:
|
||||
async def list_knowledge_engines(self, context: TenantContext) -> list[dict]:
|
||||
"""List all available Knowledge Engines from plugins."""
|
||||
require_workspace_uuid(context)
|
||||
engines = []
|
||||
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return engines
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
|
||||
# Get KnowledgeEngine plugins
|
||||
try:
|
||||
@@ -290,10 +386,12 @@ class KnowledgeService:
|
||||
|
||||
return engines
|
||||
|
||||
async def list_parsers(self, mime_type: str | None = None) -> list[dict]:
|
||||
async def list_parsers(self, context: TenantContext, mime_type: str | None = None) -> list[dict]:
|
||||
"""List available parsers, optionally filtered by MIME type."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return []
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
parsers = await self.ap.plugin_connector.list_parsers()
|
||||
if mime_type:
|
||||
@@ -303,16 +401,24 @@ class KnowledgeService:
|
||||
self.ap.logger.warning(f'Failed to list parsers: {e}')
|
||||
return []
|
||||
|
||||
async def get_engine_creation_schema(self, plugin_id: str) -> dict:
|
||||
async def get_engine_creation_schema(self, context: TenantContext, plugin_id: str) -> dict:
|
||||
"""Get creation settings schema for a specific Knowledge Engine."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return {}
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
return await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to get creation schema for {plugin_id}: {e}')
|
||||
return {}
|
||||
|
||||
async def get_engine_retrieval_schema(self, plugin_id: str) -> dict:
|
||||
async def get_engine_retrieval_schema(self, context: TenantContext, plugin_id: str) -> dict:
|
||||
"""Get retrieval settings schema for a specific Knowledge Engine."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return {}
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
return await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -11,11 +13,36 @@ import sqlalchemy
|
||||
from ....core import app
|
||||
from ....entity.persistence import bstorage as persistence_bstorage
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
LOG_FILE_PATTERN = re.compile(r'^langbot-(\d{4}-\d{2}-\d{2})\.log(?:\.\d+)?$')
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS = 7
|
||||
DEFAULT_LOG_RETENTION_DAYS = 3
|
||||
DEFAULT_MAX_FILES_PER_RUN = 1000
|
||||
HARD_MAX_FILES_PER_RUN = 10000
|
||||
UPLOAD_OWNER_TYPES = ('upload_image', 'upload_document', 'upload')
|
||||
|
||||
|
||||
def _workspace_scope(method):
|
||||
"""Bind maintenance work to a Workspace without spanning external I/O."""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapped(self, context, *args, **kwargs):
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud maintenance requires an explicit tenant scope')
|
||||
async with tenant_scope(workspace_uuid):
|
||||
return await method(self, context, *args, **kwargs)
|
||||
return await method(self, context, *args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class MaintenanceService:
|
||||
@@ -26,7 +53,22 @@ class MaintenanceService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def cleanup_expired_files(self) -> dict[str, int]:
|
||||
def _max_files_per_run(self) -> int:
|
||||
cleanup_cfg = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('storage', {}).get('cleanup', {})
|
||||
)
|
||||
value = self._positive_int(
|
||||
cleanup_cfg.get('max_files_per_run', DEFAULT_MAX_FILES_PER_RUN),
|
||||
DEFAULT_MAX_FILES_PER_RUN,
|
||||
'storage.cleanup.max_files_per_run',
|
||||
)
|
||||
return min(value, HARD_MAX_FILES_PER_RUN)
|
||||
|
||||
@_workspace_scope
|
||||
async def cleanup_expired_files(self, context: ExecutionContext) -> dict[str, int]:
|
||||
if not isinstance(context, ExecutionContext):
|
||||
raise WorkspaceRequiredError('Storage cleanup requires an ExecutionContext')
|
||||
require_workspace_uuid(context)
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
@@ -40,11 +82,17 @@ class MaintenanceService:
|
||||
)
|
||||
|
||||
return {
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(upload_retention_days),
|
||||
'log_files': self._cleanup_expired_log_files(log_retention_days),
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
|
||||
'log_files': await asyncio.to_thread(
|
||||
self._cleanup_expired_log_files,
|
||||
log_retention_days,
|
||||
)
|
||||
if await self._is_oss_singleton(context)
|
||||
else 0,
|
||||
}
|
||||
|
||||
async def get_storage_analysis(self) -> dict[str, Any]:
|
||||
async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:
|
||||
require_workspace_uuid(context)
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
@@ -62,32 +110,34 @@ class MaintenanceService:
|
||||
database_path = (
|
||||
Path(database_cfg.get('sqlite', {}).get('path', 'data/langbot.db')) if database_type == 'sqlite' else None
|
||||
)
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
is_oss_singleton = await self._is_oss_singleton(context)
|
||||
if is_oss_singleton:
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
else:
|
||||
scoped_storage_path = Path('data/storage') / self.ap.storage_mgr.scoped_prefix(context)
|
||||
roots = [('storage', scoped_storage_path)]
|
||||
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
sections = await asyncio.to_thread(self._collect_sections, roots)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts(context)
|
||||
binary_storage = await self._binary_storage_stats(context)
|
||||
upload_candidates = await self._expired_uploaded_candidates(context, upload_retention_days)
|
||||
log_candidates = (
|
||||
await asyncio.to_thread(
|
||||
self._expired_log_candidates,
|
||||
log_retention_days,
|
||||
)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts()
|
||||
binary_storage = await self._binary_storage_stats()
|
||||
upload_candidates = await self._expired_uploaded_candidates(upload_retention_days)
|
||||
log_candidates = self._expired_log_candidates(log_retention_days)
|
||||
if is_oss_singleton
|
||||
else []
|
||||
)
|
||||
|
||||
return {
|
||||
'generated_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
@@ -105,70 +155,156 @@ class MaintenanceService:
|
||||
'uploaded_files': upload_candidates,
|
||||
'log_files': log_candidates,
|
||||
},
|
||||
'tasks': self.ap.task_mgr.get_stats() if self.ap.task_mgr else {},
|
||||
'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
|
||||
}
|
||||
|
||||
async def _cleanup_expired_uploaded_files(self, retention_days: int) -> int:
|
||||
def _collect_sections(
|
||||
self,
|
||||
roots: list[tuple[str, Path | None]],
|
||||
) -> list[dict[str, Any]]:
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
)
|
||||
return sections
|
||||
|
||||
async def _is_oss_singleton(self, context: TenantContext) -> bool:
|
||||
try:
|
||||
await self.ap.workspace_service.get_local_execution_binding(
|
||||
require_workspace_uuid(context),
|
||||
expected_generation=getattr(context, 'placement_generation', None),
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _cleanup_expired_uploaded_files(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
provider_name = provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
candidates = self._expired_local_upload_candidates(retention_days, include_paths=True)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
candidates = await asyncio.to_thread(
|
||||
self._expired_local_upload_candidates,
|
||||
context,
|
||||
retention_days,
|
||||
True,
|
||||
)
|
||||
return await asyncio.to_thread(
|
||||
self._delete_local_candidates,
|
||||
candidates,
|
||||
)
|
||||
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._cleanup_expired_s3_uploaded_files(retention_days)
|
||||
return await self._cleanup_expired_s3_uploaded_files(context, retention_days)
|
||||
|
||||
return 0
|
||||
|
||||
async def _expired_uploaded_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
async def _expired_uploaded_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider_name = self.ap.storage_mgr.storage_provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
return self._expired_local_upload_candidates(retention_days)
|
||||
return await asyncio.to_thread(
|
||||
self._expired_local_upload_candidates,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._expired_s3_upload_candidates(retention_days)
|
||||
return await self._expired_s3_upload_candidates(context, retention_days)
|
||||
return []
|
||||
|
||||
async def _cleanup_expired_s3_uploaded_files(self, retention_days: int) -> int:
|
||||
async def _cleanup_expired_s3_uploaded_files(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
candidates = await self._expired_s3_upload_candidates(retention_days)
|
||||
candidates = await self._expired_s3_upload_candidates(context, retention_days)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
await provider.delete(item['key'])
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
async def _expired_s3_upload_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
async def _expired_s3_upload_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
run_io = getattr(provider, '_run_io', None)
|
||||
if callable(run_io):
|
||||
return await run_io(
|
||||
self._expired_s3_upload_candidates_sync,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
return await asyncio.to_thread(
|
||||
self._expired_s3_upload_candidates_sync,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
|
||||
def _expired_s3_upload_candidates_sync(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
candidates = []
|
||||
max_candidates = self._max_files_per_run()
|
||||
paginator = provider.s3_client.get_paginator('list_objects_v2')
|
||||
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
seen_prefixes: set[str] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
prefix = self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
if prefix in seen_prefixes:
|
||||
continue
|
||||
seen_prefixes.add(prefix)
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name, Prefix=prefix):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(context, key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
|
||||
return candidates
|
||||
|
||||
def _delete_local_candidates(self, candidates: list[dict[str, Any]]) -> int:
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
|
||||
def _cleanup_expired_log_files(self, retention_days: int) -> int:
|
||||
deleted = 0
|
||||
for item in self._expired_log_candidates(retention_days, include_paths=True):
|
||||
@@ -182,28 +318,42 @@ class MaintenanceService:
|
||||
return deleted
|
||||
|
||||
def _expired_local_upload_candidates(
|
||||
self, retention_days: int, include_paths: bool = False
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
include_paths: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
storage_root = Path('data/storage')
|
||||
if not storage_root.exists():
|
||||
return []
|
||||
|
||||
cutoff = datetime.datetime.now().timestamp() - retention_days * 86400
|
||||
candidates = []
|
||||
for entry in storage_root.iterdir():
|
||||
if not entry.is_file() or not self._is_uploaded_file_key(entry.name):
|
||||
max_candidates = self._max_files_per_run()
|
||||
seen_roots: set[Path] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
scoped_root = storage_root / self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
if scoped_root in seen_roots:
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
seen_roots.add(scoped_root)
|
||||
if not scoped_root.exists():
|
||||
continue
|
||||
item = {
|
||||
'key': entry.name,
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(stat.st_mtime, datetime.timezone.utc).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
for entry in scoped_root.rglob('*'):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
continue
|
||||
item = {
|
||||
'key': entry.relative_to(storage_root).as_posix(),
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(
|
||||
stat.st_mtime,
|
||||
datetime.timezone.utc,
|
||||
).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
return candidates
|
||||
|
||||
def _expired_log_candidates(self, retention_days: int, include_paths: bool = False) -> list[dict[str, Any]]:
|
||||
@@ -236,33 +386,51 @@ class MaintenanceService:
|
||||
candidates.append(item)
|
||||
return candidates
|
||||
|
||||
def _is_uploaded_file_key(self, key: str) -> bool:
|
||||
return '/' not in key and not key.startswith('plugin_config_')
|
||||
def _is_uploaded_file_key(self, context: TenantContext, key: str) -> bool:
|
||||
return any(
|
||||
key.startswith(self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type))
|
||||
and self.ap.storage_mgr.is_scoped_object_key(key, expected_owner_type=owner_type)
|
||||
for owner_type in UPLOAD_OWNER_TYPES
|
||||
)
|
||||
|
||||
async def _monitoring_counts(self) -> dict[str, int]:
|
||||
async def _monitoring_counts(self, context: TenantContext) -> dict[str, int]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
tables = {
|
||||
'messages': persistence_monitoring.MonitoringMessage.id,
|
||||
'llm_calls': persistence_monitoring.MonitoringLLMCall.id,
|
||||
'tool_calls': persistence_monitoring.MonitoringToolCall.id,
|
||||
'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
'errors': persistence_monitoring.MonitoringError.id,
|
||||
'sessions': persistence_monitoring.MonitoringSession.session_id,
|
||||
'feedback': persistence_monitoring.MonitoringFeedback.id,
|
||||
'messages': (persistence_monitoring.MonitoringMessage, persistence_monitoring.MonitoringMessage.id),
|
||||
'llm_calls': (persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.id),
|
||||
'tool_calls': (persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.id),
|
||||
'embedding_calls': (
|
||||
persistence_monitoring.MonitoringEmbeddingCall,
|
||||
persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
),
|
||||
'errors': (persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.id),
|
||||
'sessions': (
|
||||
persistence_monitoring.MonitoringSession,
|
||||
persistence_monitoring.MonitoringSession.session_id,
|
||||
),
|
||||
'feedback': (persistence_monitoring.MonitoringFeedback, persistence_monitoring.MonitoringFeedback.id),
|
||||
}
|
||||
counts: dict[str, int] = {}
|
||||
for key, column in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.func.count(column)))
|
||||
for key, (model, column) in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(column)).where(model.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
counts[key] = result.scalar() or 0
|
||||
return counts
|
||||
|
||||
async def _binary_storage_stats(self) -> dict[str, Any]:
|
||||
async def _binary_storage_stats(self, context: TenantContext) -> dict[str, Any]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key))
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key)).where(
|
||||
persistence_bstorage.BinaryStorage.workspace_uuid == workspace_uuid
|
||||
)
|
||||
)
|
||||
size_bytes = None
|
||||
try:
|
||||
size_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value)))
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value))
|
||||
).where(persistence_bstorage.BinaryStorage.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
size_bytes = size_result.scalar() or 0
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,198 +1,451 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
import copy
|
||||
import re
|
||||
import uuid
|
||||
import asyncio
|
||||
|
||||
from ....core import app
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app, taskmgr
|
||||
from ....core.task_boundary import create_detached_task
|
||||
from ....entity.persistence import mcp as persistence_mcp
|
||||
from ....core import taskmgr
|
||||
from ....provider.tools.loaders.mcp import RuntimeMCPSession, MCPSessionStatus
|
||||
from ....entity.persistence import plugin as persistence_plugin
|
||||
from ....provider.tools.loaders.mcp import MCPSessionStatus, RuntimeMCPSession
|
||||
from ....provider.tools.loaders.mcp_policy import require_stdio_mcp_enabled
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..context import ExecutionContext
|
||||
from .secrets import is_url_key, redact_url_secrets, restore_url_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
_SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
_SENSITIVE_CONFIG_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'apikey',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_CONFIG_TOKENS = frozenset(
|
||||
{
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def _is_sensitive_config_key(key: object) -> bool:
|
||||
normalized = _normalize_config_key(key)
|
||||
if normalized in _SENSITIVE_CONFIG_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_CONFIG_TOKENS:
|
||||
return True
|
||||
return 'key' in tokens and bool(tokens & _SENSITIVE_KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def _mask_secret_structure(value):
|
||||
if isinstance(value, dict):
|
||||
return {key: _mask_secret_structure(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_mask_secret_structure(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_mask_secret_structure(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return _SECRET_MASK
|
||||
|
||||
|
||||
def redact_mcp_secrets(value):
|
||||
"""Return a recursively redacted copy of MCP configuration data."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
_mask_secret_structure(item)
|
||||
if _is_sensitive_config_key(key)
|
||||
else redact_url_secrets(item)
|
||||
if is_url_key(key)
|
||||
else redact_mcp_secrets(item)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_mcp_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_mcp_secrets(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def restore_mcp_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from the current MCP config before a write."""
|
||||
|
||||
if sensitive and value == _SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked MCP secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: (
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
)
|
||||
if not sensitive and not _is_sensitive_config_key(key) and is_url_key(key)
|
||||
else restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or _is_sensitive_config_key(key),
|
||||
)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class MCPService:
|
||||
"""Workspace-scoped MCP configuration and runtime facade."""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_runtime_info(self, server_name: str) -> dict | None:
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
if session:
|
||||
return session.get_runtime_info_dict()
|
||||
return None
|
||||
async def _execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip()
|
||||
generation = getattr(context, 'placement_generation', None)
|
||||
if not instance_uuid or not isinstance(generation, int) or isinstance(generation, bool) or generation <= 0:
|
||||
raise ValueError('MCP operations require an explicit fenced execution context')
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=generation,
|
||||
)
|
||||
if binding.instance_uuid != instance_uuid:
|
||||
raise ValueError('MCP execution context belongs to another LangBot instance')
|
||||
return ExecutionContext(
|
||||
instance_uuid=instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
placement_generation=generation,
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
)
|
||||
|
||||
async def get_mcp_servers(self, contain_runtime_info: bool = False) -> list[dict]:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
|
||||
async def get_runtime_info(self, context: TenantContext, server_name: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
return session.get_runtime_info_dict() if session else None
|
||||
|
||||
servers = result.all()
|
||||
async def get_mcp_servers(self, context: TenantContext, contain_runtime_info: bool = False) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_mcp.MCPServer), persistence_mcp.MCPServer, context)
|
||||
)
|
||||
serialized_servers = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server) for server in servers
|
||||
redact_mcp_secrets(self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server))
|
||||
for server in result.all()
|
||||
]
|
||||
if contain_runtime_info:
|
||||
for server in serialized_servers:
|
||||
runtime_info = await self.get_runtime_info(server['name'])
|
||||
|
||||
server['runtime_info'] = runtime_info if runtime_info else None
|
||||
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server['name'])
|
||||
server['runtime_info'] = session.get_runtime_info_dict() if session else None
|
||||
return serialized_servers
|
||||
|
||||
async def create_mcp_server(self, server_data: dict) -> str:
|
||||
# Check limitation (extensions = MCP servers + plugins)
|
||||
async def create_mcp_server(self, context: TenantContext, server_data: dict) -> str:
|
||||
execution_context = await self._execution_context(context)
|
||||
workspace_uuid = execution_context.workspace_uuid
|
||||
|
||||
# This gate is independent of Box availability. Cloud v2 disables
|
||||
# stdio MCP even though Box Runtime itself remains available.
|
||||
require_stdio_mcp_enabled(self.ap, server_data)
|
||||
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_extensions = limitation.get('max_extensions', -1)
|
||||
if max_extensions >= 0:
|
||||
existing_mcp_servers = await self.get_mcp_servers()
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
total_extensions = len(existing_mcp_servers) + len(plugins)
|
||||
if total_extensions >= max_extensions:
|
||||
mcp_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_mcp.MCPServer.uuid)).where(
|
||||
persistence_mcp.MCPServer.workspace_uuid == workspace_uuid
|
||||
)
|
||||
)
|
||||
plugin_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
if (mcp_count_result.scalar() or 0) + (plugin_count_result.scalar() or 0) >= max_extensions:
|
||||
raise ValueError(f'Maximum number of extensions ({max_extensions}) reached')
|
||||
|
||||
server_name = str(server_data.get('name') or '').strip()
|
||||
payload = dict(server_data)
|
||||
payload.pop('workspace_uuid', None)
|
||||
server_name = str(payload.get('name') or '').strip()
|
||||
if not server_name:
|
||||
raise ValueError('MCP server name is required')
|
||||
server_data['name'] = server_name
|
||||
payload['name'] = server_name
|
||||
payload['workspace_uuid'] = workspace_uuid
|
||||
payload['uuid'] = str(uuid.uuid4())
|
||||
|
||||
existing_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name)
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(
|
||||
persistence_mcp.MCPServer.workspace_uuid == workspace_uuid,
|
||||
persistence_mcp.MCPServer.name == server_name,
|
||||
)
|
||||
)
|
||||
if existing_result.first() is not None:
|
||||
raise ValueError(f'MCP server already exists: {server_name}')
|
||||
|
||||
server_data['uuid'] = str(uuid.uuid4())
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(server_data))
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_data['uuid'])
|
||||
)
|
||||
server_entity = result.first()
|
||||
if server_entity:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server_entity)
|
||||
if self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(payload))
|
||||
created = await self._get_mcp_server_by_uuid_raw(execution_context, payload['uuid'])
|
||||
if created and self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = create_detached_task(
|
||||
self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
)
|
||||
tracker = getattr(
|
||||
self.ap.tool_mgr.mcp_tool_loader,
|
||||
'track_hosted_task',
|
||||
None,
|
||||
)
|
||||
if callable(tracker):
|
||||
tracker(task, execution_context)
|
||||
else:
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
return payload['uuid']
|
||||
|
||||
return server_data['uuid']
|
||||
async def get_mcp_server_by_uuid(self, context: TenantContext, server_uuid: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server_data = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
return redact_mcp_secrets(server_data) if server_data is not None else None
|
||||
|
||||
async def get_mcp_server_by_name(self, server_name: str) -> dict | None:
|
||||
async def _get_mcp_server_by_uuid_raw(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
server_uuid: str,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server) if server else None
|
||||
|
||||
async def get_mcp_server_by_name(self, context: TenantContext, server_name: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server_data = await self._get_mcp_server_by_name_raw(execution_context, server_name)
|
||||
if server_data is None:
|
||||
return None
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
response_data = {
|
||||
**server_data,
|
||||
'runtime_info': session.get_runtime_info_dict() if session else None,
|
||||
}
|
||||
return redact_mcp_secrets(response_data)
|
||||
|
||||
async def _get_mcp_server_by_name_raw(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
server_name: str,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
if server is None:
|
||||
return None
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
|
||||
runtime_info = await self.get_runtime_info(server.name)
|
||||
server_data = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
server_data['runtime_info'] = runtime_info if runtime_info else None
|
||||
return server_data
|
||||
async def update_mcp_server(self, context: TenantContext, server_uuid: str, server_data: dict) -> None:
|
||||
execution_context = await self._execution_context(context)
|
||||
old_server = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if old_server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
|
||||
payload = dict(server_data)
|
||||
payload.pop('uuid', None)
|
||||
payload.pop('workspace_uuid', None)
|
||||
payload = restore_mcp_secret_placeholders(payload, old_server)
|
||||
if 'name' in payload:
|
||||
payload['name'] = str(payload['name'] or '').strip()
|
||||
if not payload['name']:
|
||||
raise ValueError('MCP server name is required')
|
||||
duplicate = await self._get_mcp_server_by_name_raw(execution_context, payload['name'])
|
||||
if duplicate is not None and duplicate['uuid'] != server_uuid:
|
||||
raise ValueError(f'MCP server already exists: {payload["name"]}')
|
||||
|
||||
effective_server = {**old_server, **payload}
|
||||
# Existing disabled rows remain readable/deletable. Switching away
|
||||
# from stdio or explicitly disabling one is also allowed, but an
|
||||
# update may never leave a disabled stdio server enabled.
|
||||
if bool(effective_server.get('enable', True)):
|
||||
require_stdio_mcp_enabled(self.ap, effective_server)
|
||||
|
||||
async def update_mcp_server(self, server_uuid: str, server_data: dict) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
.values(payload),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
old_server = result.first()
|
||||
old_server_name = old_server.name if old_server else None
|
||||
old_enable = old_server.enable if old_server else False
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
.values(server_data)
|
||||
)
|
||||
loader = self.ap.tool_mgr.mcp_tool_loader
|
||||
if loader is None:
|
||||
return
|
||||
old_name = old_server['name']
|
||||
old_enable = bool(old_server['enable'])
|
||||
updated = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if updated is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
new_enable = bool(updated['enable'])
|
||||
if old_enable and loader.has_session(execution_context, old_name):
|
||||
await loader.remove_mcp_server(execution_context, old_name)
|
||||
if new_enable:
|
||||
task = create_detached_task(
|
||||
loader.host_mcp_server(execution_context, updated),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
)
|
||||
tracker = getattr(loader, 'track_hosted_task', None)
|
||||
if callable(tracker):
|
||||
tracker(task, execution_context)
|
||||
else:
|
||||
loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
if self.ap.tool_mgr.mcp_tool_loader:
|
||||
new_enable = server_data.get('enable', False)
|
||||
|
||||
need_remove = old_server_name and old_server_name in self.ap.tool_mgr.mcp_tool_loader.sessions
|
||||
|
||||
if old_enable and not new_enable:
|
||||
if need_remove:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(old_server_name)
|
||||
|
||||
elif not old_enable and new_enable:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
updated_server = result.first()
|
||||
if updated_server:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, updated_server)
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
elif old_enable and new_enable:
|
||||
if need_remove:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(old_server_name)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
updated_server = result.first()
|
||||
if updated_server:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, updated_server)
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
async def delete_mcp_server(self, server_uuid: str) -> None:
|
||||
async def delete_mcp_server(self, context: TenantContext, server_uuid: str) -> None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
server_name = server.name if server else None
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
loader = self.ap.tool_mgr.mcp_tool_loader
|
||||
if loader and loader.has_session(execution_context, server['name']):
|
||||
await loader.remove_mcp_server(execution_context, server['name'])
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
async def _require_server(self, context: TenantContext, server_name: str) -> tuple[ExecutionContext, dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
server = await self._get_mcp_server_by_name_raw(execution_context, server_name)
|
||||
if server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
return execution_context, server
|
||||
|
||||
if server_name and self.ap.tool_mgr.mcp_tool_loader:
|
||||
if server_name in self.ap.tool_mgr.mcp_tool_loader.sessions:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(server_name)
|
||||
async def get_mcp_server_resources(self, context: TenantContext, server_name: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resources(execution_context, server_name)
|
||||
|
||||
async def get_mcp_server_resources(self, server_name: str) -> list[dict]:
|
||||
"""Get resources from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resources(server_name)
|
||||
|
||||
async def get_mcp_server_resource_templates(self, server_name: str) -> list[dict]:
|
||||
"""Get resource templates from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(server_name)
|
||||
async def get_mcp_server_resource_templates(self, context: TenantContext, server_name: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(execution_context, server_name)
|
||||
|
||||
async def read_mcp_server_resource_envelope(
|
||||
self,
|
||||
context: TenantContext,
|
||||
server_name: str,
|
||||
uri: str,
|
||||
*,
|
||||
max_bytes: int | None = None,
|
||||
include_blob: bool = False,
|
||||
) -> dict:
|
||||
"""Read a resource from a specific MCP server with metadata."""
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
kwargs = {'include_blob': include_blob, 'source': 'ui_preview'}
|
||||
if max_bytes is not None:
|
||||
kwargs['max_bytes'] = max_bytes
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(server_name, uri, **kwargs)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(
|
||||
execution_context,
|
||||
server_name,
|
||||
uri,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def read_mcp_server_resource(self, server_name: str, uri: str) -> list[dict]:
|
||||
"""Read a resource from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource(server_name, uri)
|
||||
|
||||
async def test_mcp_server(self, server_name: str, server_data: dict) -> int:
|
||||
"""测试 MCP 服务器连接并返回任务 ID"""
|
||||
async def read_mcp_server_resource(self, context: TenantContext, server_name: str, uri: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource(execution_context, server_name, uri)
|
||||
|
||||
async def test_mcp_server(self, context: TenantContext, server_name: str, server_data: dict) -> int:
|
||||
execution_context = await self._execution_context(context)
|
||||
runtime_mcp_session: RuntimeMCPSession | None = None
|
||||
|
||||
test_session: RuntimeMCPSession | None = None
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
|
||||
if server_name != '_':
|
||||
runtime_mcp_session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
_, persisted_server = await self._require_server(execution_context, server_name)
|
||||
require_stdio_mcp_enabled(self.ap, persisted_server)
|
||||
runtime_mcp_session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
if runtime_mcp_session is None:
|
||||
raise ValueError(f'Server not found: {server_name}')
|
||||
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
persisted_session = runtime_mcp_session
|
||||
|
||||
async def _refresh_and_report() -> None:
|
||||
# Testing a persisted server should REUSE its live shared-session
|
||||
# process, not rebuild it. Try a lightweight refresh (a real
|
||||
# list_tools probe over the existing connection) first; only fall
|
||||
# back to a full start() when the session has no live connection
|
||||
# to probe (never connected, or the process is actually gone).
|
||||
needs_start = persisted_session.status == MCPSessionStatus.ERROR or persisted_session.session is None
|
||||
if needs_start:
|
||||
await persisted_session.start()
|
||||
@@ -200,30 +453,24 @@ class MCPService:
|
||||
try:
|
||||
await persisted_session.refresh()
|
||||
except Exception:
|
||||
# The live connection was stale/dropped: reconnect once
|
||||
# (reusing the live managed process where possible) and
|
||||
# re-probe, instead of reporting a false failure.
|
||||
await persisted_session.start()
|
||||
# Surface the discovered tools so the config page can render them
|
||||
# even for an already-hosted server.
|
||||
ctx.metadata['runtime_info'] = persisted_session.get_runtime_info_dict()
|
||||
|
||||
coroutine = _refresh_and_report()
|
||||
else:
|
||||
runtime_mcp_session = await self.ap.tool_mgr.mcp_tool_loader.load_mcp_server(server_config=server_data)
|
||||
|
||||
# A transient test owns an isolated Box session. Always tear it down
|
||||
# after the test completes (success or failure) so it does not leak.
|
||||
payload = dict(server_data)
|
||||
payload.pop('workspace_uuid', None)
|
||||
payload['workspace_uuid'] = execution_context.workspace_uuid
|
||||
require_stdio_mcp_enabled(self.ap, payload)
|
||||
runtime_mcp_session = await self.ap.tool_mgr.mcp_tool_loader.load_mcp_server(
|
||||
execution_context,
|
||||
payload,
|
||||
)
|
||||
test_session = runtime_mcp_session
|
||||
|
||||
async def _run_and_cleanup() -> None:
|
||||
try:
|
||||
await test_session.start()
|
||||
# Capture the runtime info (status + discovered tools) BEFORE
|
||||
# shutting the transient session down. The create/edit config
|
||||
# page has no persisted server to reload from, so without this
|
||||
# a successful test could only show "no tools found". The
|
||||
# frontend reads ctx.metadata.runtime_info to render the tools.
|
||||
ctx.metadata['runtime_info'] = test_session.get_runtime_info_dict()
|
||||
finally:
|
||||
try:
|
||||
@@ -236,27 +483,41 @@ class MCPService:
|
||||
|
||||
coroutine = _run_and_cleanup()
|
||||
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
coroutine,
|
||||
kind='mcp-operation',
|
||||
name=f'mcp-test-{server_name}',
|
||||
label=f'Testing MCP server {server_name}',
|
||||
context=ctx,
|
||||
)
|
||||
try:
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
coroutine,
|
||||
kind='mcp-operation',
|
||||
name=f'mcp-test-{execution_context.workspace_uuid}-{server_name}',
|
||||
label=f'Testing MCP server {server_name}',
|
||||
context=ctx,
|
||||
instance_uuid=execution_context.instance_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
placement_generation=execution_context.placement_generation,
|
||||
)
|
||||
except taskmgr.TaskCapacityError:
|
||||
if test_session is not None:
|
||||
try:
|
||||
await test_session.shutdown()
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(
|
||||
f'Failed to tear down rejected transient MCP test session '
|
||||
f'{test_session.server_name}: {type(exc).__name__}: {exc}'
|
||||
)
|
||||
raise
|
||||
return wrapper.id
|
||||
|
||||
async def get_mcp_server_logs(self, server_name: str, limit: int = 200, level: str | None = None) -> list[dict]:
|
||||
"""Get recent log lines captured from the MCP server's stderr."""
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
async def get_mcp_server_logs(
|
||||
self,
|
||||
context: TenantContext,
|
||||
server_name: str,
|
||||
limit: int = 200,
|
||||
level: str | None = None,
|
||||
) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
if not session:
|
||||
return []
|
||||
|
||||
# Get logs from the session's buffer
|
||||
logs = list(session._log_buffer)
|
||||
|
||||
# Filter by level if specified
|
||||
if level:
|
||||
logs = [log for log in logs if log.get('level') == level]
|
||||
|
||||
# Return the most recent 'limit' logs
|
||||
return logs[-limit:]
|
||||
|
||||
@@ -9,6 +9,9 @@ from ....core import app
|
||||
from ....entity.persistence import model as persistence_model
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....provider.modelmgr import requester as model_requester
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import mask_secret_value, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
def _parse_provider_api_keys(provider_dict: dict) -> dict:
|
||||
@@ -34,7 +37,29 @@ def _runtime_model_data(model_uuid: str, model_data: dict) -> dict:
|
||||
return {**model_data, 'uuid': model_uuid}
|
||||
|
||||
|
||||
async def _validate_provider_supports(ap: app.Application, provider_uuid: str, model_type: str) -> None:
|
||||
def _redact_model_secrets(model_data: dict) -> dict:
|
||||
"""Return a copy with model args and embedded provider credentials masked."""
|
||||
|
||||
redacted = model_data.copy()
|
||||
if 'extra_args' in redacted:
|
||||
redacted['extra_args'] = redact_secrets(redacted['extra_args'])
|
||||
if isinstance(redacted.get('provider'), dict):
|
||||
provider = redacted['provider'].copy()
|
||||
# ModelProvider never contains another provider. Dropping this key also
|
||||
# makes the serializer robust to a reused/self-referential test double.
|
||||
provider.pop('provider', None)
|
||||
if 'api_keys' in provider:
|
||||
provider['api_keys'] = mask_secret_value(provider['api_keys'])
|
||||
redacted['provider'] = provider
|
||||
return redacted
|
||||
|
||||
|
||||
async def _validate_provider_supports(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
model_type: str,
|
||||
) -> None:
|
||||
"""Validate that the provider's requester declares support for ``model_type``.
|
||||
|
||||
``model_type`` is one of the manifest ``support_type`` values:
|
||||
@@ -47,11 +72,12 @@ async def _validate_provider_supports(ap: app.Application, provider_uuid: str, m
|
||||
if model_mgr is None:
|
||||
return
|
||||
|
||||
provider_dict = getattr(model_mgr, 'provider_dict', None)
|
||||
if not provider_dict:
|
||||
get_provider = getattr(model_mgr, 'get_provider_by_uuid', None)
|
||||
if not callable(get_provider):
|
||||
return
|
||||
runtime_provider = provider_dict.get(provider_uuid)
|
||||
if runtime_provider is None:
|
||||
try:
|
||||
runtime_provider = await get_provider(context, provider_uuid)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
requester_name = getattr(getattr(runtime_provider, 'provider_entity', None), 'requester', None)
|
||||
@@ -74,20 +100,48 @@ async def _validate_provider_supports(ap: app.Application, provider_uuid: str, m
|
||||
raise ValueError(f'Provider requester "{requester_name}" does not support {model_type} models')
|
||||
|
||||
|
||||
async def _require_workspace_provider(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
) -> dict:
|
||||
"""Require the referenced provider to belong to the active Workspace."""
|
||||
|
||||
provider = await ap.provider_service.get_provider(context, provider_uuid)
|
||||
if provider is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
return provider
|
||||
|
||||
|
||||
async def _require_runtime_provider(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
) -> model_requester.RuntimeProvider:
|
||||
try:
|
||||
return await ap.model_mgr.get_provider_by_uuid(context, provider_uuid)
|
||||
except ValueError as exc:
|
||||
raise Exception('provider not found') from exc
|
||||
|
||||
|
||||
class LLMModelsService:
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_llm_models(self, include_secret: bool = True) -> list[dict]:
|
||||
async def get_llm_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all LLM models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.LLMModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_model.LLMModel), persistence_model.LLMModel, context)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
# Get all providers for lookup
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -98,29 +152,50 @@ class LLMModelsService:
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
if not include_secret:
|
||||
provider_dict['api_keys'] = ['***'] * len(provider_dict.get('api_keys', []))
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_llm_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_llm_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get LLM models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.LLMModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.LLMModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_llm_model(
|
||||
self, model_data: dict, preserve_uuid: bool = False, auto_set_to_default_pipeline: bool = True
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_data: dict,
|
||||
preserve_uuid: bool = False,
|
||||
auto_set_to_default_pipeline: bool = True,
|
||||
) -> str:
|
||||
"""Create a new LLM model"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = workspace_uuid
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
# Handle provider creation if needed
|
||||
if 'provider' in model_data:
|
||||
@@ -130,31 +205,35 @@ class LLMModelsService:
|
||||
else:
|
||||
# Create new provider
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'llm')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'llm')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.LLMModel).values(**model_data))
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
context,
|
||||
persistence_model.LLMModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
await self.ap.model_mgr.cache_llm_model(context, runtime_llm_model)
|
||||
|
||||
if auto_set_to_default_pipeline:
|
||||
# set the default pipeline model to this model
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
@@ -167,14 +246,23 @@ class LLMModelsService:
|
||||
'fallbacks': [],
|
||||
}
|
||||
pipeline_data = {'config': pipeline_config}
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline.uuid, pipeline_data)
|
||||
await self.ap.pipeline_service.update_pipeline(context, pipeline.uuid, pipeline_data)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_llm_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_llm_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single LLM model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
@@ -184,21 +272,38 @@ class LLMModelsService:
|
||||
|
||||
# Get provider
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_llm_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_llm_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing LLM model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_llm_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
# Handle provider update if needed
|
||||
if 'provider' in model_data:
|
||||
@@ -207,50 +312,71 @@ class LLMModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'llm')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
await self.ap.model_mgr.remove_llm_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_llm_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
persistence_model.LLMModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
context,
|
||||
persistence_model.LLMModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
await self.ap.model_mgr.cache_llm_model(context, runtime_llm_model)
|
||||
|
||||
async def delete_llm_model(self, model_uuid: str) -> None:
|
||||
async def delete_llm_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete an LLM model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_llm_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_llm_model(context, model_uuid)
|
||||
|
||||
async def test_llm_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def test_llm_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test an LLM model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_llm_model: model_requester.RuntimeLLMModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.llm_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_llm_model = model
|
||||
break
|
||||
if runtime_llm_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_llm_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_llm_model = await self.ap.model_mgr.get_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_llm_model = await self.ap.model_mgr.init_temporary_runtime_llm_model(model_data)
|
||||
runtime_llm_model = await self.ap.model_mgr.init_temporary_runtime_llm_model(context, model_data)
|
||||
|
||||
extra_args = model_data.get('extra_args', {})
|
||||
await runtime_llm_model.provider.invoke_llm(
|
||||
@@ -259,6 +385,7 @@ class LLMModelsService:
|
||||
messages=[provider_message.Message(role='user', content='Hello, world! Please just reply a "Hello".')],
|
||||
funcs=[],
|
||||
extra_args=extra_args,
|
||||
execution_context=runtime_llm_model.execution_context,
|
||||
)
|
||||
|
||||
|
||||
@@ -268,13 +395,19 @@ class EmbeddingModelsService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_embedding_models(self) -> list[dict]:
|
||||
async def get_embedding_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all embedding models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.EmbeddingModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel), persistence_model.EmbeddingModel, context
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -284,25 +417,46 @@ class EmbeddingModelsService:
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_embedding_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_embedding_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get embedding models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_embedding_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
async def create_embedding_model(
|
||||
self, context: TenantContext, model_data: dict, preserve_uuid: bool = False
|
||||
) -> str:
|
||||
"""Create a new embedding model"""
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -310,35 +464,44 @@ class EmbeddingModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'text-embedding')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'text-embedding')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.EmbeddingModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
context,
|
||||
persistence_model.EmbeddingModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
await self.ap.model_mgr.cache_embedding_model(context, runtime_embedding_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_embedding_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_embedding_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single embedding model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
@@ -348,21 +511,38 @@ class EmbeddingModelsService:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_embedding_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_embedding_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing embedding model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_embedding_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -370,57 +550,82 @@ class EmbeddingModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'text-embedding')
|
||||
|
||||
await self.ap.model_mgr.remove_embedding_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
persistence_model.EmbeddingModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
|
||||
async def delete_embedding_model(self, model_uuid: str) -> None:
|
||||
"""Delete an embedding model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_embedding_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
async def test_embedding_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
await self.ap.model_mgr.remove_embedding_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
context,
|
||||
persistence_model.EmbeddingModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
await self.ap.model_mgr.cache_embedding_model(context, runtime_embedding_model)
|
||||
|
||||
async def delete_embedding_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete an embedding model"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_embedding_model(context, model_uuid)
|
||||
|
||||
async def test_embedding_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test an embedding model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_embedding_model: model_requester.RuntimeEmbeddingModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.embedding_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_embedding_model = model
|
||||
break
|
||||
if runtime_embedding_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_embedding_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_embedding_model = await self.ap.model_mgr.get_embedding_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_embedding_model = await self.ap.model_mgr.init_temporary_runtime_embedding_model(model_data)
|
||||
runtime_embedding_model = await self.ap.model_mgr.init_temporary_runtime_embedding_model(
|
||||
context,
|
||||
model_data,
|
||||
)
|
||||
|
||||
await runtime_embedding_model.provider.invoke_embedding(
|
||||
model=runtime_embedding_model,
|
||||
input_text=['Hello, world!'],
|
||||
extra_args={},
|
||||
execution_context=runtime_embedding_model.execution_context,
|
||||
)
|
||||
|
||||
|
||||
@@ -430,13 +635,17 @@ class RerankModelsService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_rerank_models(self) -> list[dict]:
|
||||
async def get_rerank_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all rerank models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.RerankModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_model.RerankModel), persistence_model.RerankModel, context)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -446,25 +655,44 @@ class RerankModelsService:
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_rerank_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_rerank_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get rerank models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_rerank_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
async def create_rerank_model(self, context: TenantContext, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
"""Create a new rerank model"""
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -472,34 +700,45 @@ class RerankModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'rerank')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'rerank')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.RerankModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
context,
|
||||
persistence_model.RerankModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
await self.ap.model_mgr.cache_rerank_model(context, runtime_rerank_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_rerank_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_rerank_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single rerank model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
@@ -508,21 +747,38 @@ class RerankModelsService:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_rerank_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing rerank model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_rerank_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -530,50 +786,76 @@ class RerankModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'rerank')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
persistence_model.RerankModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
context,
|
||||
persistence_model.RerankModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
await self.ap.model_mgr.cache_rerank_model(context, runtime_rerank_model)
|
||||
|
||||
async def delete_rerank_model(self, model_uuid: str) -> None:
|
||||
async def delete_rerank_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete a rerank model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_rerank_model(context, model_uuid)
|
||||
|
||||
async def test_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def test_rerank_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test a rerank model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_rerank_model: model_requester.RuntimeRerankModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.rerank_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_rerank_model = model
|
||||
break
|
||||
if runtime_rerank_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_rerank_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_rerank_model = await self.ap.model_mgr.get_rerank_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(model_data)
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(
|
||||
context,
|
||||
model_data,
|
||||
)
|
||||
|
||||
await runtime_rerank_model.provider.invoke_rerank(
|
||||
model=runtime_rerank_model,
|
||||
@@ -582,4 +864,5 @@ class RerankModelsService:
|
||||
'Artificial intelligence is a branch of computer science.',
|
||||
'The weather is nice today.',
|
||||
],
|
||||
execution_context=runtime_rerank_model.execution_context,
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,9 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
default_stage_order = [
|
||||
@@ -30,7 +33,8 @@ class PipelineService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_pipeline_metadata(self) -> list[dict]:
|
||||
async def get_pipeline_metadata(self, context: TenantContext) -> list[dict]:
|
||||
require_workspace_uuid(context)
|
||||
return [
|
||||
self.ap.pipeline_config_meta_trigger,
|
||||
self.ap.pipeline_config_meta_safety,
|
||||
@@ -38,8 +42,19 @@ class PipelineService:
|
||||
self.ap.pipeline_config_meta_output,
|
||||
]
|
||||
|
||||
async def get_pipelines(self, sort_by: str = 'created_at', sort_order: str = 'DESC') -> list[dict]:
|
||||
query = sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
async def get_pipelines(
|
||||
self,
|
||||
context: TenantContext,
|
||||
sort_by: str = 'created_at',
|
||||
sort_order: str = 'DESC',
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
query = scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
|
||||
if sort_by == 'created_at':
|
||||
if sort_order == 'DESC':
|
||||
@@ -54,15 +69,26 @@ class PipelineService:
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(query)
|
||||
pipelines = result.all()
|
||||
return [
|
||||
serialized = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
for pipeline in pipelines
|
||||
]
|
||||
return serialized if include_secret else [redact_secrets(pipeline) for pipeline in serialized]
|
||||
|
||||
async def get_pipeline(self, pipeline_uuid: str) -> dict | None:
|
||||
async def get_pipeline(
|
||||
self,
|
||||
context: TenantContext,
|
||||
pipeline_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -71,20 +97,24 @@ class PipelineService:
|
||||
if pipeline is None:
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
serialized = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
return serialized if include_secret else redact_secrets(serialized)
|
||||
|
||||
async def create_pipeline(self, pipeline_data: dict, default: bool = False) -> str:
|
||||
async def create_pipeline(self, context: TenantContext, pipeline_data: dict, default: bool = False) -> str:
|
||||
from ....utils import paths as path_utils
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
if max_pipelines >= 0:
|
||||
existing_pipelines = await self.get_pipelines()
|
||||
existing_pipelines = await self.get_pipelines(context)
|
||||
if len(existing_pipelines) >= max_pipelines:
|
||||
raise ValueError(f'Maximum number of pipelines ({max_pipelines}) reached')
|
||||
|
||||
pipeline_data = pipeline_data.copy()
|
||||
pipeline_data['uuid'] = str(uuid.uuid4())
|
||||
pipeline_data['workspace_uuid'] = workspace_uuid
|
||||
pipeline_data['for_version'] = self.ap.ver_mgr.get_current_version()
|
||||
pipeline_data['stages'] = default_stage_order.copy()
|
||||
pipeline_data['is_default'] = default
|
||||
@@ -108,79 +138,122 @@ class PipelineService:
|
||||
sqlalchemy.insert(persistence_pipeline.LegacyPipeline).values(**pipeline_data)
|
||||
)
|
||||
|
||||
pipeline = await self.get_pipeline(pipeline_data['uuid'])
|
||||
pipeline = await self.get_pipeline(context, pipeline_data['uuid'], include_secret=True)
|
||||
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
return pipeline_data['uuid']
|
||||
|
||||
async def update_pipeline(self, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
async def update_pipeline(self, context: TenantContext, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
pipeline_data = pipeline_data.copy()
|
||||
for protected_field in ('uuid', 'for_version', 'stages', 'is_default'):
|
||||
for protected_field in ('uuid', 'workspace_uuid', 'for_version', 'stages', 'is_default'):
|
||||
pipeline_data.pop(protected_field, None)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(**pipeline_data)
|
||||
)
|
||||
if 'config' in pipeline_data:
|
||||
current_config = None
|
||||
if contains_secret_placeholder(pipeline_data['config']):
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
pipeline_data['config'] = restore_secret_placeholders(
|
||||
pipeline_data['config'],
|
||||
current_config if current_config is not None else {},
|
||||
)
|
||||
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(**pipeline_data),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
if 'name' in pipeline_data:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(
|
||||
persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid
|
||||
),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
for bot in bots:
|
||||
bot_data = {'use_pipeline_name': pipeline_data['name']}
|
||||
await self.ap.bot_service.update_bot(bot.uuid, bot_data)
|
||||
await self.ap.bot_service.update_bot(context, bot.uuid, bot_data)
|
||||
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
# update all conversation that use this pipeline
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if session.using_conversation is not None and session.using_conversation.pipeline_uuid == pipeline_uuid:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
and session.using_conversation.pipeline_uuid == pipeline_uuid
|
||||
and getattr(session, 'workspace_uuid', workspace_uuid) == workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
|
||||
async def delete_pipeline(self, pipeline_uuid: str) -> None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
async def delete_pipeline(self, context: TenantContext, pipeline_uuid: str) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
|
||||
async def copy_pipeline(self, pipeline_uuid: str) -> str:
|
||||
async def copy_pipeline(self, context: TenantContext, pipeline_uuid: str) -> str:
|
||||
"""Copy a pipeline with all its configurations"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
if max_pipelines >= 0:
|
||||
existing_pipelines = await self.get_pipelines()
|
||||
existing_pipelines = await self.get_pipelines(context)
|
||||
if len(existing_pipelines) >= max_pipelines:
|
||||
raise ValueError(f'Maximum number of pipelines ({max_pipelines}) reached')
|
||||
|
||||
# Get the original pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
original_pipeline = result.first()
|
||||
if original_pipeline is None:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Create new pipeline data
|
||||
new_uuid = str(uuid.uuid4())
|
||||
new_pipeline_data = {
|
||||
'uuid': new_uuid,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'name': f'{original_pipeline.name} (Copy)',
|
||||
'description': original_pipeline.description,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
@@ -207,13 +280,14 @@ class PipelineService:
|
||||
)
|
||||
|
||||
# Load the new pipeline
|
||||
pipeline = await self.get_pipeline(new_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
pipeline = await self.get_pipeline(context, new_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
return new_uuid
|
||||
|
||||
async def update_pipeline_extensions(
|
||||
self,
|
||||
context: TenantContext,
|
||||
pipeline_uuid: str,
|
||||
bound_plugins: list[dict],
|
||||
bound_mcp_servers: list[str] = None,
|
||||
@@ -225,16 +299,21 @@ class PipelineService:
|
||||
mcp_resource_agent_read_enabled: bool | None = None,
|
||||
) -> None:
|
||||
"""Update the bound plugins and MCP servers for a pipeline"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Get current pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
pipeline = result.first()
|
||||
if pipeline is None:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Update extensions_preferences
|
||||
extensions_preferences = pipeline.extensions_preferences or {}
|
||||
@@ -252,12 +331,16 @@ class PipelineService:
|
||||
extensions_preferences['mcp_resources'] = bound_mcp_resources
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(extensions_preferences=extensions_preferences)
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(extensions_preferences=extensions_preferences),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
# Reload pipeline to apply changes
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
@@ -7,6 +7,9 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import model as persistence_model
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
class ModelProviderService:
|
||||
@@ -35,9 +38,15 @@ class ModelProviderService:
|
||||
|
||||
return normalized_keys
|
||||
|
||||
async def get_providers(self) -> list[dict]:
|
||||
async def get_providers(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all providers"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.ModelProvider))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
providers = result.all()
|
||||
providers_list = []
|
||||
for p in providers:
|
||||
@@ -50,14 +59,25 @@ class ModelProviderService:
|
||||
provider_dict['api_keys'] = json.loads(provider_dict['api_keys'])
|
||||
except Exception:
|
||||
provider_dict['api_keys'] = []
|
||||
if not include_secret:
|
||||
provider_dict = redact_secrets(provider_dict)
|
||||
providers_list.append(provider_dict)
|
||||
return providers_list
|
||||
|
||||
async def get_provider(self, provider_uuid: str) -> dict | None:
|
||||
async def get_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single provider by UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = result.first()
|
||||
@@ -72,103 +92,171 @@ class ModelProviderService:
|
||||
provider_dict['api_keys'] = json.loads(provider_dict['api_keys'])
|
||||
except Exception:
|
||||
provider_dict['api_keys'] = []
|
||||
if not include_secret:
|
||||
provider_dict = redact_secrets(provider_dict)
|
||||
return provider_dict
|
||||
|
||||
async def create_provider(self, provider_data: dict) -> str:
|
||||
async def create_provider(self, context: TenantContext, provider_data: dict) -> str:
|
||||
"""Create a new provider"""
|
||||
provider_data = provider_data.copy()
|
||||
provider_data['uuid'] = str(uuid.uuid4())
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
provider_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
provider_data['api_keys'] = self._normalize_api_keys(
|
||||
restore_secret_placeholders(provider_data.get('api_keys'), sensitive=True)
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
|
||||
)
|
||||
|
||||
# load to runtime
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider_data)
|
||||
self.ap.model_mgr.provider_dict[runtime_provider.provider_entity.uuid] = runtime_provider
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(context, provider_data)
|
||||
await self.ap.model_mgr.cache_provider(context, runtime_provider)
|
||||
return provider_data['uuid']
|
||||
|
||||
async def update_provider(self, provider_uuid: str, provider_data: dict) -> None:
|
||||
async def update_provider(self, context: TenantContext, provider_uuid: str, provider_data: dict) -> None:
|
||||
"""Update an existing provider"""
|
||||
if 'uuid' in provider_data:
|
||||
del provider_data['uuid']
|
||||
provider_data = provider_data.copy()
|
||||
provider_data.pop('uuid', None)
|
||||
provider_data.pop('workspace_uuid', None)
|
||||
if 'api_keys' in provider_data:
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
.values(**provider_data)
|
||||
submitted_keys = provider_data.get('api_keys')
|
||||
if contains_secret_placeholder(submitted_keys, sensitive=True):
|
||||
current_provider = await self.get_provider(context, provider_uuid, include_secret=True)
|
||||
if current_provider is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
submitted_keys = restore_secret_placeholders(
|
||||
submitted_keys,
|
||||
current_provider.get('api_keys', []),
|
||||
sensitive=True,
|
||||
)
|
||||
provider_data['api_keys'] = self._normalize_api_keys(submitted_keys)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
.values(**provider_data),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider(provider_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
await self.ap.model_mgr.reload_provider(context, provider_uuid)
|
||||
|
||||
async def delete_provider(self, provider_uuid: str) -> None:
|
||||
async def delete_provider(self, context: TenantContext, provider_uuid: str) -> None:
|
||||
"""Delete a provider (only if no models reference it)"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check if any models use this provider
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.LLMModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if llm_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: LLM models still reference it')
|
||||
|
||||
embedding_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if embedding_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Embedding models still reference it')
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if rerank_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Rerank models still reference it')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_provider(provider_uuid)
|
||||
await self.ap.model_mgr.remove_provider(context, provider_uuid)
|
||||
|
||||
async def get_provider_model_counts(self, provider_uuid: str) -> dict:
|
||||
async def get_provider_model_counts(self, context: TenantContext, provider_uuid: str) -> dict:
|
||||
"""Get count of models using this provider"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_provider(context, provider_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.provider_uuid == provider_uuid),
|
||||
persistence_model.LLMModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
llm_count = llm_result.scalar() or 0
|
||||
|
||||
embedding_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.provider_uuid == provider_uuid),
|
||||
persistence_model.EmbeddingModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
embedding_count = embedding_result.scalar() or 0
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid),
|
||||
persistence_model.RerankModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
rerank_count = rerank_result.scalar() or 0
|
||||
|
||||
return {'llm_count': llm_count, 'embedding_count': embedding_count, 'rerank_count': rerank_count}
|
||||
|
||||
async def find_or_create_provider(self, requester: str, base_url: str, api_keys: list) -> str:
|
||||
async def find_or_create_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
requester: str,
|
||||
base_url: str,
|
||||
api_keys: list,
|
||||
) -> str:
|
||||
"""Find existing provider or create new one"""
|
||||
api_keys = self._normalize_api_keys(api_keys)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
api_keys = self._normalize_api_keys(restore_secret_placeholders(api_keys, sensitive=True))
|
||||
|
||||
# Try to find existing provider with same config
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.requester == requester,
|
||||
persistence_model.ModelProvider.base_url == base_url,
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.requester == requester,
|
||||
persistence_model.ModelProvider.base_url == base_url,
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
for provider in result.all():
|
||||
@@ -187,29 +275,38 @@ class ModelProviderService:
|
||||
pass
|
||||
|
||||
return await self.create_provider(
|
||||
context,
|
||||
{
|
||||
'name': provider_name,
|
||||
'requester': requester,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
async def update_space_model_provider_api_keys(self, api_key: str) -> None:
|
||||
async def update_space_model_provider_api_keys(self, context: TenantContext, api_key: str) -> None:
|
||||
"""Update Space model provider API keys"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key)),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider('00000000-0000-0000-0000-000000000000')
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
await self.ap.model_mgr.reload_provider(context, '00000000-0000-0000-0000-000000000000')
|
||||
|
||||
async def scan_provider_models(self, provider_uuid: str, model_type: str | None = None) -> dict:
|
||||
provider = await self.get_provider(provider_uuid)
|
||||
async def scan_provider_models(
|
||||
self, context: TenantContext, provider_uuid: str, model_type: str | None = None
|
||||
) -> dict:
|
||||
provider = await self.get_provider(context, provider_uuid, include_secret=True)
|
||||
if provider is None:
|
||||
raise ValueError('provider not found')
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider)
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(context, provider)
|
||||
|
||||
try:
|
||||
scan_result = await runtime_provider.requester.scan_models(
|
||||
@@ -230,11 +327,15 @@ class ModelProviderService:
|
||||
scanned_models = scan_result
|
||||
debug_info = None
|
||||
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(provider_uuid)
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(context, provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
context, provider_uuid
|
||||
)
|
||||
rerank_service = getattr(self.ap, 'rerank_models_service', None)
|
||||
rerank_models = (
|
||||
await rerank_service.get_rerank_models_by_provider(provider_uuid) if rerank_service is not None else []
|
||||
await rerank_service.get_rerank_models_by_provider(context, provider_uuid)
|
||||
if rerank_service is not None
|
||||
else []
|
||||
)
|
||||
existing_llm_names = {model['name'] for model in llm_models}
|
||||
existing_embedding_names = {model['name'] for model in embedding_models}
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
|
||||
SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
|
||||
_SENSITIVE_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'api_keys',
|
||||
'apikey',
|
||||
'apikeys',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'header_value',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
'webhook_url',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_TOKENS = frozenset(
|
||||
{
|
||||
'apikey',
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_URL_QUERY_NAMES = frozenset(
|
||||
{
|
||||
'code',
|
||||
'credential',
|
||||
'credentials',
|
||||
'password',
|
||||
'passwd',
|
||||
'sig',
|
||||
'signature',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def is_sensitive_key(key: object) -> bool:
|
||||
"""Return whether a configuration key conventionally carries a secret."""
|
||||
|
||||
normalized = _normalize_key(key)
|
||||
if normalized in _SENSITIVE_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_TOKENS:
|
||||
return True
|
||||
return bool(tokens & {'key', 'keys'}) and bool(tokens & _KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def is_url_key(key: object) -> bool:
|
||||
"""Return whether a configuration field conventionally carries a URL."""
|
||||
|
||||
normalized = _normalize_key(key)
|
||||
return normalized == 'url' or normalized.endswith('_url')
|
||||
|
||||
|
||||
def _is_sensitive_url_query_key(key: object) -> bool:
|
||||
normalized = _normalize_key(key)
|
||||
return (
|
||||
is_sensitive_key(key) or normalized in _SENSITIVE_URL_QUERY_NAMES or normalized.endswith(('_sig', '_signature'))
|
||||
)
|
||||
|
||||
|
||||
def _redact_url_string(value: str) -> str:
|
||||
if not value:
|
||||
return value
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
netloc = parsed.netloc
|
||||
if '@' in netloc:
|
||||
_, host = netloc.rsplit('@', 1)
|
||||
netloc = f'{SECRET_MASK}@{host}'
|
||||
query = urlencode(
|
||||
[
|
||||
(key, SECRET_MASK if _is_sensitive_url_query_key(key) and item else item)
|
||||
for key, item in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
],
|
||||
doseq=True,
|
||||
safe='*',
|
||||
)
|
||||
return urlunsplit((parsed.scheme, netloc, parsed.path, query, parsed.fragment))
|
||||
except (TypeError, ValueError):
|
||||
# A malformed URL cannot be safely decomposed, so fail closed.
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def redact_url_secrets(value):
|
||||
"""Redact URL userinfo and credential-like query values."""
|
||||
|
||||
if isinstance(value, str):
|
||||
return _redact_url_string(value)
|
||||
if isinstance(value, list):
|
||||
return [redact_url_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_url_secrets(item) for item in value)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def _contains_url_secret_placeholder(value) -> bool:
|
||||
if isinstance(value, str):
|
||||
if value == SECRET_MASK:
|
||||
return True
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
if '@' in parsed.netloc and SECRET_MASK in parsed.netloc.rsplit('@', 1)[0]:
|
||||
return True
|
||||
return any(
|
||||
item == SECRET_MASK and _is_sensitive_url_query_key(key)
|
||||
for key, item in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(_contains_url_secret_placeholder(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def _restore_url_string(value: str, current_value) -> str:
|
||||
if value == SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked URL secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
|
||||
try:
|
||||
submitted = urlsplit(value)
|
||||
except (TypeError, ValueError):
|
||||
return value
|
||||
|
||||
current = None
|
||||
if isinstance(current_value, str):
|
||||
try:
|
||||
current = urlsplit(current_value)
|
||||
except (TypeError, ValueError):
|
||||
current = None
|
||||
|
||||
netloc = submitted.netloc
|
||||
if '@' in netloc:
|
||||
submitted_userinfo, host = netloc.rsplit('@', 1)
|
||||
if SECRET_MASK in submitted_userinfo:
|
||||
if current is None or '@' not in current.netloc:
|
||||
raise ValueError('Masked URL userinfo has no existing value')
|
||||
current_userinfo, _ = current.netloc.rsplit('@', 1)
|
||||
netloc = f'{current_userinfo}@{host}'
|
||||
|
||||
current_query: dict[str, list[str]] = {}
|
||||
if current is not None:
|
||||
for key, item in parse_qsl(current.query, keep_blank_values=True):
|
||||
current_query.setdefault(_normalize_key(key), []).append(item)
|
||||
consumed: dict[str, int] = {}
|
||||
restored_query: list[tuple[str, str]] = []
|
||||
for key, item in parse_qsl(submitted.query, keep_blank_values=True):
|
||||
normalized = _normalize_key(key)
|
||||
if item == SECRET_MASK and _is_sensitive_url_query_key(key):
|
||||
index = consumed.get(normalized, 0)
|
||||
candidates = current_query.get(normalized, [])
|
||||
if index >= len(candidates):
|
||||
raise ValueError('Masked URL query secret has no existing value')
|
||||
item = candidates[index]
|
||||
consumed[normalized] = index + 1
|
||||
restored_query.append((key, item))
|
||||
|
||||
return urlunsplit(
|
||||
(
|
||||
submitted.scheme,
|
||||
netloc,
|
||||
submitted.path,
|
||||
urlencode(restored_query, doseq=True, safe='*'),
|
||||
submitted.fragment,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def restore_url_secret_placeholders(value, current_value=_MISSING_SECRET):
|
||||
"""Restore URL placeholders from the corresponding persisted URL."""
|
||||
|
||||
if isinstance(value, str):
|
||||
return _restore_url_string(value, current_value)
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def mask_secret_value(value):
|
||||
"""Return a shape-preserving copy whose non-empty leaves are masked."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: mask_secret_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [mask_secret_value(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(mask_secret_value(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def redact_secrets(value):
|
||||
"""Return a recursively redacted copy without mutating the source value."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
mask_secret_value(item)
|
||||
if is_sensitive_key(key)
|
||||
else redact_url_secrets(item)
|
||||
if is_url_key(key)
|
||||
else redact_secrets(item)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_secrets(item) for item in value)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def restore_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from existing data before a management write.
|
||||
|
||||
``***`` is a reserved placeholder only inside a sensitive field. A masked
|
||||
leaf without an existing counterpart is rejected so it can never become a
|
||||
persisted credential. Empty values and explicit replacements pass through.
|
||||
"""
|
||||
|
||||
if sensitive and value == SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: (
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
)
|
||||
if not sensitive and not is_sensitive_key(key) and is_url_key(key)
|
||||
else restore_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or is_sensitive_key(key),
|
||||
)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def contains_secret_placeholder(value, *, sensitive: bool = False) -> bool:
|
||||
"""Return whether ``value`` contains a meaningful masked secret leaf."""
|
||||
|
||||
if sensitive and value == SECRET_MASK:
|
||||
return True
|
||||
if isinstance(value, dict):
|
||||
return any(
|
||||
(
|
||||
_contains_url_secret_placeholder(item)
|
||||
if not sensitive and not is_sensitive_key(key) and is_url_key(key)
|
||||
else contains_secret_placeholder(item, sensitive=sensitive or is_sensitive_key(key))
|
||||
)
|
||||
for key, item in value.items()
|
||||
)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(contains_secret_placeholder(item, sensitive=sensitive) for item in value)
|
||||
return False
|
||||
@@ -1,9 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import inspect
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
import zipfile
|
||||
from typing import Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
@@ -12,6 +14,9 @@ import httpx
|
||||
|
||||
from ....core import app
|
||||
from ....skill.utils import parse_frontmatter
|
||||
from ....utils import httpclient
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
_PUBLIC_SKILL_FIELDS = (
|
||||
@@ -32,6 +37,12 @@ _GITHUB_ASSET_HOSTS = {
|
||||
'raw.githubusercontent.com',
|
||||
'codeload.github.com',
|
||||
}
|
||||
_MAX_GITHUB_ARCHIVE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_GITHUB_ARCHIVE_ENTRIES = 4096
|
||||
_MAX_SKILL_ARCHIVE_FILES = 1024
|
||||
_MAX_SKILL_FILE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_SKILL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
|
||||
_MAX_SKILL_COMPRESSION_RATIO = 200
|
||||
|
||||
|
||||
class SkillService:
|
||||
@@ -75,75 +86,112 @@ class SkillService:
|
||||
"""Backwards-compatible alias preserved for clarity at call sites."""
|
||||
self._require_box(action)
|
||||
|
||||
async def _execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip()
|
||||
generation = getattr(context, 'placement_generation', None)
|
||||
if not instance_uuid or isinstance(generation, bool) or not isinstance(generation, int) or generation <= 0:
|
||||
raise ValueError('Skill operations require an explicit fenced execution context')
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=generation,
|
||||
)
|
||||
if binding.instance_uuid != instance_uuid:
|
||||
raise ValueError('Skill execution context belongs to another LangBot instance')
|
||||
return ExecutionContext(
|
||||
instance_uuid=instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
placement_generation=generation,
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _serialize_skill(skill: dict) -> dict:
|
||||
return {field: skill.get(field) for field in _PUBLIC_SKILL_FIELDS if field in skill}
|
||||
|
||||
async def list_skills(self) -> list[dict]:
|
||||
async def list_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
# When Box is unavailable, surface an empty list rather than raising —
|
||||
# the skills page should render cleanly, and the UI separately renders
|
||||
# a "Box disabled / unavailable" banner via useBoxStatus.
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return []
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills()]
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills(execution_context)]
|
||||
|
||||
async def get_skill(self, skill_name: str) -> Optional[dict]:
|
||||
async def get_skill(self, context: TenantContext, skill_name: str) -> Optional[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return None
|
||||
skill = await box_service.get_skill(skill_name)
|
||||
skill = await box_service.get_skill(execution_context, skill_name)
|
||||
return self._serialize_skill(skill) if skill else None
|
||||
|
||||
async def get_skill_by_name(self, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(name)
|
||||
async def get_skill_by_name(self, context: TenantContext, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(context, name)
|
||||
|
||||
async def create_skill(self, data: dict) -> dict:
|
||||
async def create_skill(self, context: TenantContext, data: dict) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Creating a skill')
|
||||
created = await box_service.create_skill(data)
|
||||
await self._reload_skills()
|
||||
created = await box_service.create_skill(execution_context, data)
|
||||
await self._reload_skills(execution_context)
|
||||
return self._serialize_skill(created)
|
||||
|
||||
async def update_skill(self, skill_name: str, data: dict) -> dict:
|
||||
async def update_skill(self, context: TenantContext, skill_name: str, data: dict) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Editing a skill')
|
||||
updated = await box_service.update_skill(skill_name, data)
|
||||
await self._reload_skills()
|
||||
updated = await box_service.update_skill(execution_context, skill_name, data)
|
||||
await self._reload_skills(execution_context)
|
||||
return self._serialize_skill(updated)
|
||||
|
||||
async def delete_skill(self, skill_name: str) -> bool:
|
||||
async def delete_skill(self, context: TenantContext, skill_name: str) -> bool:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Deleting a skill')
|
||||
await box_service.delete_skill(skill_name)
|
||||
await self._reload_skills()
|
||||
await box_service.delete_skill(execution_context, skill_name)
|
||||
await self._reload_skills(execution_context)
|
||||
return True
|
||||
|
||||
async def list_skill_files(
|
||||
self,
|
||||
context: TenantContext,
|
||||
skill_name: str,
|
||||
path: str = '.',
|
||||
include_hidden: bool = False,
|
||||
max_entries: int = 200,
|
||||
) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Browsing skill files')
|
||||
return await box_service.list_skill_files(skill_name, path, include_hidden, max_entries)
|
||||
return await box_service.list_skill_files(execution_context, skill_name, path, include_hidden, max_entries)
|
||||
|
||||
async def read_skill_file(self, skill_name: str, path: str) -> dict:
|
||||
async def read_skill_file(self, context: TenantContext, skill_name: str, path: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Reading a skill file')
|
||||
return await box_service.read_skill_file(skill_name, path)
|
||||
return await box_service.read_skill_file(execution_context, skill_name, path)
|
||||
|
||||
async def write_skill_file(self, skill_name: str, path: str, content: str) -> dict:
|
||||
async def write_skill_file(self, context: TenantContext, skill_name: str, path: str, content: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Editing skill files')
|
||||
result = await box_service.write_skill_file(skill_name, path, content)
|
||||
await self._reload_skills()
|
||||
result = await box_service.write_skill_file(execution_context, skill_name, path, content)
|
||||
await self._reload_skills(execution_context)
|
||||
return result
|
||||
|
||||
async def install_from_github(self, data: dict) -> list[dict]:
|
||||
async def install_from_github(self, context: TenantContext, data: dict) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._install_github_skill_md(raw_asset_url, owner=owner, repo=repo, data=data)
|
||||
return await self._install_github_skill_md(
|
||||
execution_context,
|
||||
raw_asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
data=data,
|
||||
)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
@@ -151,29 +199,37 @@ class SkillService:
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
filename = f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip'
|
||||
installed = await box_service.install_skill_zip(
|
||||
execution_context,
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
source_subdir=source_subdir,
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(execution_context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_github(self, data: dict) -> list[dict]:
|
||||
async def preview_install_from_github(self, context: TenantContext, data: dict) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._preview_github_skill_md(raw_asset_url, owner=owner, repo=repo)
|
||||
return await self._preview_github_skill_md(
|
||||
execution_context,
|
||||
raw_asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
return await box_service.preview_skill_zip(
|
||||
execution_context,
|
||||
zip_bytes,
|
||||
f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip',
|
||||
source_subdir=source_subdir,
|
||||
@@ -181,27 +237,45 @@ class SkillService:
|
||||
|
||||
async def install_from_zip_upload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
source_paths: list[str] | None = None,
|
||||
source_path: str = '',
|
||||
) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Installing a skill from upload')
|
||||
installed = await box_service.install_skill_zip(
|
||||
execution_context,
|
||||
file_bytes,
|
||||
filename,
|
||||
source_paths=source_paths or [],
|
||||
source_path=source_path,
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(execution_context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_zip_upload(self, *, file_bytes: bytes, filename: str) -> list[dict]:
|
||||
async def preview_install_from_zip_upload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Previewing a skill upload')
|
||||
return await box_service.preview_skill_zip(file_bytes, filename)
|
||||
return await box_service.preview_skill_zip(execution_context, file_bytes, filename)
|
||||
|
||||
async def _install_github_skill_md(self, asset_url: str, *, owner: str, repo: str, data: dict) -> list[dict]:
|
||||
async def _install_github_skill_md(
|
||||
self,
|
||||
context: TenantContext,
|
||||
asset_url: str,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
data: dict,
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
zip_bytes, filename, _package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
@@ -210,46 +284,73 @@ class SkillService:
|
||||
)
|
||||
|
||||
installed = await box_service.install_skill_zip(
|
||||
context,
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
target_suffix='',
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def _preview_github_skill_md(self, asset_url: str, *, owner: str, repo: str) -> list[dict]:
|
||||
async def _preview_github_skill_md(
|
||||
self,
|
||||
context: TenantContext,
|
||||
asset_url: str,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
zip_bytes, _filename, package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
return await box_service.preview_skill_zip(zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
return await box_service.preview_skill_zip(context, zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
|
||||
async def reload_skills(self) -> list[dict]:
|
||||
await self._reload_skills()
|
||||
return await self.list_skills()
|
||||
async def reload_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
await self._reload_skills(execution_context)
|
||||
return await self.list_skills(execution_context)
|
||||
|
||||
async def scan_directory_async(self, path: str) -> dict:
|
||||
async def scan_directory_async(self, context: TenantContext, path: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Scanning a skill directory')
|
||||
return await box_service.scan_skill_directory(path)
|
||||
return await box_service.scan_skill_directory(execution_context, path)
|
||||
|
||||
async def _reload_skills(self) -> None:
|
||||
async def _reload_skills(self, context: TenantContext) -> None:
|
||||
skill_mgr = getattr(self.ap, 'skill_mgr', None)
|
||||
reload_skills = getattr(skill_mgr, 'reload_skills', None)
|
||||
if not callable(reload_skills):
|
||||
return
|
||||
result = reload_skills()
|
||||
result = reload_skills(context)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _download_github_asset(self, asset_url: str) -> bytes:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
|
||||
resp = await client.get(asset_url)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=True,
|
||||
timeout=120,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(_MAX_GITHUB_ARCHIVE_BYTES),
|
||||
) as client:
|
||||
async with client.stream('GET', asset_url) as resp:
|
||||
resp.raise_for_status()
|
||||
content_length = resp.headers.get('content-length')
|
||||
if content_length is not None:
|
||||
try:
|
||||
if int(content_length) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
except ValueError as exc:
|
||||
if 'exceeds' in str(exc):
|
||||
raise
|
||||
content = bytearray()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
content.extend(chunk)
|
||||
if len(content) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
return bytes(content)
|
||||
|
||||
async def _download_github_skill_directory_as_zip(
|
||||
self, asset_url: str, *, owner: str, repo: str
|
||||
@@ -257,14 +358,25 @@ class SkillService:
|
||||
info = self._parse_github_skill_md_url(asset_url, owner=owner, repo=repo)
|
||||
archive_url = f'https://codeload.github.com/{owner}/{repo}/zip/{quote(info["ref"], safe="/")}'
|
||||
archive_bytes = await self._download_github_asset(archive_url)
|
||||
return await asyncio.to_thread(self._build_github_skill_directory_zip, archive_bytes, info)
|
||||
|
||||
def _build_github_skill_directory_zip(
|
||||
self,
|
||||
archive_bytes: bytes,
|
||||
info: dict[str, str],
|
||||
) -> tuple[bytes, str, str]:
|
||||
"""Validate and repack a GitHub skill archive outside the event loop."""
|
||||
try:
|
||||
source_archive = zipfile.ZipFile(io.BytesIO(archive_bytes), 'r')
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError('GitHub repository archive must be a valid .zip archive') from exc
|
||||
|
||||
with source_archive as source_zip:
|
||||
if len(source_zip.infolist()) > _MAX_GITHUB_ARCHIVE_ENTRIES:
|
||||
raise ValueError('GitHub repository archive contains too many entries')
|
||||
skill_entry = self._find_github_skill_archive_entry(source_zip, info['file_path'])
|
||||
if skill_entry.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError('GitHub SKILL.md exceeds the file size limit')
|
||||
try:
|
||||
skill_md_content = source_zip.read(skill_entry).decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
@@ -302,6 +414,7 @@ class SkillService:
|
||||
normalized_source_dir = posixpath.normpath(source_skill_dir)
|
||||
source_prefix = f'{normalized_source_dir}/'
|
||||
copied_files = 0
|
||||
copied_bytes = 0
|
||||
|
||||
for member in source_zip.infolist():
|
||||
normalized_member = posixpath.normpath(member.filename)
|
||||
@@ -324,10 +437,33 @@ class SkillService:
|
||||
if member.is_dir():
|
||||
target_zip.writestr(target_info, b'')
|
||||
continue
|
||||
|
||||
target_zip.writestr(target_info, source_zip.read(member))
|
||||
if member.flag_bits & 0x1:
|
||||
raise ValueError('Encrypted GitHub skill archive entries are not supported')
|
||||
unix_mode = member.external_attr >> 16
|
||||
if stat.S_IFMT(unix_mode) == stat.S_IFLNK:
|
||||
raise ValueError(f'GitHub archive contains a symbolic link: {member.filename}')
|
||||
if member.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError(f'GitHub skill file exceeds the size limit: {member.filename}')
|
||||
if member.file_size and member.file_size > max(member.compress_size, 1) * _MAX_SKILL_COMPRESSION_RATIO:
|
||||
raise ValueError(f'GitHub skill file exceeds the compression-ratio limit: {member.filename}')
|
||||
copied_files += 1
|
||||
copied_bytes += member.file_size
|
||||
if copied_files > _MAX_SKILL_ARCHIVE_FILES:
|
||||
raise ValueError('GitHub skill directory contains too many files')
|
||||
if copied_bytes > _MAX_SKILL_UNCOMPRESSED_BYTES:
|
||||
raise ValueError('GitHub skill directory exceeds the uncompressed size limit')
|
||||
|
||||
# Copy in bounded chunks instead of materialising a potentially
|
||||
# large member in Core memory. The Box Runtime independently
|
||||
# revalidates the resulting archive before installation.
|
||||
with source_zip.open(member, 'r') as source_file, target_zip.open(target_info, 'w') as target_file:
|
||||
remaining = member.file_size
|
||||
while remaining:
|
||||
chunk = source_file.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise ValueError(f'GitHub skill file is truncated: {member.filename}')
|
||||
target_file.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if copied_files == 0:
|
||||
raise ValueError('GitHub skill directory is empty')
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from langbot.pkg.utils import httpclient
|
||||
import typing
|
||||
import datetime
|
||||
@@ -11,6 +13,10 @@ from ....entity.persistence import user
|
||||
from ....entity.dto.space_model import SpaceModel
|
||||
|
||||
|
||||
_CREDITS_CACHE_TTL_SECONDS = 60
|
||||
_CREDITS_CACHE_MAX_ENTRIES = 4096
|
||||
|
||||
|
||||
class SpaceService:
|
||||
"""Service for interacting with LangBot Space API"""
|
||||
|
||||
@@ -19,7 +25,24 @@ class SpaceService:
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
self._credits_cache = {}
|
||||
self._credits_cache = OrderedDict()
|
||||
|
||||
def _ordered_credits_cache(
|
||||
self,
|
||||
) -> OrderedDict[str, tuple[int, float]]:
|
||||
if not isinstance(self._credits_cache, OrderedDict):
|
||||
# Preserve compatibility with tests and callers that seed the cache.
|
||||
self._credits_cache = OrderedDict(self._credits_cache)
|
||||
return self._credits_cache
|
||||
|
||||
def _prune_credits_cache(self, now: float) -> None:
|
||||
cache = self._ordered_credits_cache()
|
||||
while cache:
|
||||
email = next(iter(cache))
|
||||
_, cached_at = cache[email]
|
||||
if now - cached_at < _CREDITS_CACHE_TTL_SECONDS:
|
||||
break
|
||||
cache.pop(email, None)
|
||||
|
||||
def _get_space_config(self) -> typing.Dict[str, str]:
|
||||
"""Get Space configuration from config file"""
|
||||
@@ -85,12 +108,14 @@ class SpaceService:
|
||||
|
||||
def get_oauth_authorize_url(self, redirect_uri: str, state: str = '') -> str:
|
||||
"""Get the Space OAuth authorization URL for redirect"""
|
||||
from urllib.parse import urlencode
|
||||
|
||||
space_config = self._get_space_config()
|
||||
authorize_url = space_config['oauth_authorize_url']
|
||||
params = f'redirect_uri={redirect_uri}'
|
||||
params = {'redirect_uri': redirect_uri}
|
||||
if state:
|
||||
params += f'&state={state}'
|
||||
return f'{authorize_url}?{params}'
|
||||
params['state'] = state
|
||||
return f'{authorize_url}?{urlencode(params)}'
|
||||
|
||||
async def exchange_oauth_code(self, code: str) -> typing.Dict:
|
||||
"""Exchange OAuth authorization code for tokens"""
|
||||
@@ -105,8 +130,9 @@ class SpaceService:
|
||||
json={'code': code, 'instance_id': constants.instance_id},
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to exchange OAuth code: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to exchange OAuth code: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to exchange OAuth code: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -121,8 +147,9 @@ class SpaceService:
|
||||
f'{space_url}/api/v1/accounts/token/refresh', json={'refresh_token': refresh_token}
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to refresh token: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to refresh token: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to refresh token: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -137,8 +164,9 @@ class SpaceService:
|
||||
f'{space_url}/api/v1/accounts/me', headers={'Authorization': f'Bearer {access_token}'}
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get user info: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to get user info: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to get user info: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -154,11 +182,13 @@ class SpaceService:
|
||||
|
||||
async def get_credits(self, user_email: str, force_refresh: bool = False) -> int | None:
|
||||
"""Get Space credits for user with caching (60s TTL)"""
|
||||
cache_ttl = 60
|
||||
now = time.time()
|
||||
cached_fallback = self._credits_cache.get(user_email)
|
||||
self._prune_credits_cache(now)
|
||||
|
||||
if not force_refresh and user_email in self._credits_cache:
|
||||
credits, ts = self._credits_cache[user_email]
|
||||
if time.time() - ts < cache_ttl:
|
||||
if now - ts < _CREDITS_CACHE_TTL_SECONDS:
|
||||
return credits
|
||||
|
||||
try:
|
||||
@@ -167,10 +197,14 @@ class SpaceService:
|
||||
return None
|
||||
credits = info.get('credits')
|
||||
if credits is not None:
|
||||
self._credits_cache[user_email] = (credits, time.time())
|
||||
cache = self._ordered_credits_cache()
|
||||
cache.pop(user_email, None)
|
||||
if len(cache) >= _CREDITS_CACHE_MAX_ENTRIES:
|
||||
cache.popitem(last=False)
|
||||
cache[user_email] = (credits, time.time())
|
||||
return credits
|
||||
except Exception:
|
||||
return self._credits_cache.get(user_email, (None, 0))[0]
|
||||
return cached_fallback[0] if cached_fallback is not None else None
|
||||
|
||||
async def get_models(self) -> typing.List[SpaceModel]:
|
||||
"""Get models from Space"""
|
||||
@@ -181,8 +215,9 @@ class SpaceService:
|
||||
session = httpclient.get_session()
|
||||
async with session.get(f'{space_url}/api/v1/models', params={'page_size': 100}) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get models: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to get models: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to get models: {data.get("msg")}')
|
||||
models_data = data.get('data', {}).get('models', [])
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext, RequestContext, WorkspaceContext
|
||||
|
||||
TenantContext: typing.TypeAlias = RequestContext | ExecutionContext | WorkspaceContext | str
|
||||
|
||||
|
||||
def require_workspace_uuid(context: TenantContext | None) -> str:
|
||||
"""Resolve an explicit Workspace UUID without allowing a global fallback."""
|
||||
|
||||
if isinstance(context, str):
|
||||
workspace_uuid = context
|
||||
elif isinstance(context, RequestContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
elif isinstance(context, ExecutionContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
elif isinstance(context, WorkspaceContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
else:
|
||||
raise WorkspaceRequiredError('Workspace context is required')
|
||||
|
||||
normalized = workspace_uuid.strip()
|
||||
if not normalized:
|
||||
raise WorkspaceRequiredError('Workspace context is required')
|
||||
return normalized
|
||||
|
||||
|
||||
def scope_statement(statement: typing.Any, model: typing.Any, context: TenantContext) -> typing.Any:
|
||||
"""Add the mandatory Workspace predicate to a SQLAlchemy statement."""
|
||||
|
||||
return statement.where(model.workspace_uuid == require_workspace_uuid(context))
|
||||
@@ -6,71 +6,391 @@ import jwt
|
||||
import datetime
|
||||
import typing
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import heapq
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import user
|
||||
from ....entity.persistence.workspace import MembershipRole, MembershipStatus, WorkspaceMembership
|
||||
from ....utils import constants
|
||||
from ....entity.errors import account as account_errors
|
||||
from ....workspace.collaboration import normalize_email
|
||||
from ....utils import bounded_executor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
|
||||
_SPACE_OAUTH_STATE_MAX_ENTRIES = 4096
|
||||
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR = 64
|
||||
_SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER = 4
|
||||
|
||||
|
||||
class AccountExistsLoginRequiredError(ValueError):
|
||||
code = 'account_exists_login_required'
|
||||
|
||||
|
||||
class PublicRegistrationClosedError(ValueError):
|
||||
code = 'registration_closed'
|
||||
|
||||
|
||||
class ControlPlaneDirectoryRequiredError(PublicRegistrationClosedError):
|
||||
code = 'control_plane_required'
|
||||
|
||||
|
||||
class AccountDisabledError(ValueError):
|
||||
code = 'account_disabled'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class SpaceOAuthStateConsumption:
|
||||
purpose: typing.Literal['login', 'bind']
|
||||
account: user.User | None
|
||||
launch_workspace_uuid: str | None = None
|
||||
|
||||
|
||||
class UserService:
|
||||
ap: app.Application
|
||||
ap: Application
|
||||
_create_user_lock: asyncio.Lock
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(self, ap: Application) -> None:
|
||||
self.ap = ap
|
||||
self._create_user_lock = asyncio.Lock()
|
||||
self._password_hash_lock = asyncio.Semaphore(1)
|
||||
self._password_hash_lock = asyncio.Lock()
|
||||
self._space_oauth_state_lock = asyncio.Lock()
|
||||
self._space_oauth_states: dict[str, tuple[str, str | None, float, str | None]] = {}
|
||||
self._space_oauth_state_expiry_heap: list[tuple[float, str]] = []
|
||||
|
||||
@staticmethod
|
||||
def _space_oauth_state_digest(state: str) -> str:
|
||||
return hashlib.sha256(state.encode('utf-8')).hexdigest()
|
||||
|
||||
def _prune_space_oauth_states(self, now: float) -> None:
|
||||
while self._space_oauth_state_expiry_heap:
|
||||
expires_at, digest = self._space_oauth_state_expiry_heap[0]
|
||||
entry = self._space_oauth_states.get(digest)
|
||||
if entry is None or entry[2] != expires_at:
|
||||
heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
continue
|
||||
if expires_at > now:
|
||||
break
|
||||
heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
self._space_oauth_states.pop(digest, None)
|
||||
|
||||
max_heap_entries = max(
|
||||
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR,
|
||||
len(self._space_oauth_states) * _SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER,
|
||||
)
|
||||
if len(self._space_oauth_state_expiry_heap) > max_heap_entries:
|
||||
self._space_oauth_state_expiry_heap[:] = [
|
||||
(entry[2], digest) for digest, entry in self._space_oauth_states.items()
|
||||
]
|
||||
heapq.heapify(self._space_oauth_state_expiry_heap)
|
||||
|
||||
def _evict_earliest_space_oauth_state(self) -> None:
|
||||
while self._space_oauth_state_expiry_heap:
|
||||
expires_at, digest = heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
entry = self._space_oauth_states.get(digest)
|
||||
if entry is not None and entry[2] == expires_at:
|
||||
self._space_oauth_states.pop(digest, None)
|
||||
return
|
||||
|
||||
async def issue_space_oauth_state(
|
||||
self,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
*,
|
||||
account_uuid: str | None = None,
|
||||
launch_workspace_uuid: str | None = None,
|
||||
ttl_seconds: int = 600,
|
||||
) -> str:
|
||||
"""Issue an opaque, single-use OAuth state without exposing a JWT."""
|
||||
if purpose == 'bind' and not account_uuid:
|
||||
raise ValueError('An Account is required for Space binding')
|
||||
if purpose == 'login' and account_uuid is not None:
|
||||
raise ValueError('Login state cannot be bound to an Account')
|
||||
if purpose != 'login' and launch_workspace_uuid is not None:
|
||||
raise ValueError('Launch Workspace state is only valid for Space login')
|
||||
if ttl_seconds <= 0:
|
||||
raise ValueError('OAuth state lifetime must be positive')
|
||||
|
||||
raw_state = secrets.token_urlsafe(32)
|
||||
digest = self._space_oauth_state_digest(raw_state)
|
||||
expires_at = time.monotonic() + min(ttl_seconds, 600)
|
||||
async with self._space_oauth_state_lock:
|
||||
now = time.monotonic()
|
||||
self._prune_space_oauth_states(now)
|
||||
if len(self._space_oauth_states) >= _SPACE_OAUTH_STATE_MAX_ENTRIES:
|
||||
self._evict_earliest_space_oauth_state()
|
||||
self._space_oauth_states[digest] = (purpose, account_uuid, expires_at, launch_workspace_uuid)
|
||||
heapq.heappush(
|
||||
self._space_oauth_state_expiry_heap,
|
||||
(expires_at, digest),
|
||||
)
|
||||
return raw_state
|
||||
|
||||
async def consume_space_oauth_state_details(
|
||||
self,
|
||||
raw_state: str,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
) -> SpaceOAuthStateConsumption:
|
||||
"""Atomically consume OAuth state and return any bound launch intent."""
|
||||
if not isinstance(raw_state, str) or not raw_state:
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
digest = self._space_oauth_state_digest(raw_state)
|
||||
async with self._space_oauth_state_lock:
|
||||
entry = self._space_oauth_states.pop(digest, None)
|
||||
if entry is None or entry[0] != purpose or entry[2] <= time.monotonic():
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
if purpose == 'login':
|
||||
return SpaceOAuthStateConsumption(
|
||||
purpose='login',
|
||||
account=None,
|
||||
launch_workspace_uuid=entry[3],
|
||||
)
|
||||
|
||||
account_uuid = entry[1]
|
||||
account = await self.get_user_by_uuid(account_uuid or '')
|
||||
if account is None:
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
self._require_active_account(account)
|
||||
return SpaceOAuthStateConsumption(purpose='bind', account=account)
|
||||
|
||||
async def consume_space_oauth_state(
|
||||
self,
|
||||
raw_state: str,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
) -> user.User | None:
|
||||
"""Atomically consume OAuth state and resolve its active bind Account."""
|
||||
consumed = await self.consume_space_oauth_state_details(raw_state, purpose)
|
||||
return consumed.account
|
||||
|
||||
async def _hash_password(self, password: str) -> str:
|
||||
if self._password_hash_lock.locked():
|
||||
raise bounded_executor.BlockingWorkCapacityError(
|
||||
'Password hashing capacity reached',
|
||||
scope='system:authentication',
|
||||
)
|
||||
async with self._password_hash_lock:
|
||||
return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
|
||||
with bounded_executor.blocking_work_scope('system:authentication'):
|
||||
return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
|
||||
|
||||
def _require_local_directory(self) -> None:
|
||||
if self._uses_control_plane_directory():
|
||||
raise ControlPlaneDirectoryRequiredError(
|
||||
'Cloud Accounts and directory changes are managed by the SaaS control plane'
|
||||
)
|
||||
|
||||
def _uses_control_plane_directory(self) -> bool:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
return bool(workspace_service is not None and workspace_service.policy.multi_workspace_enabled)
|
||||
|
||||
async def _verify_password(self, hashed_password: str, password: str) -> None:
|
||||
if self._password_hash_lock.locked():
|
||||
raise bounded_executor.BlockingWorkCapacityError(
|
||||
'Password hashing capacity reached',
|
||||
scope='system:authentication',
|
||||
)
|
||||
async with self._password_hash_lock:
|
||||
await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
|
||||
with bounded_executor.blocking_work_scope('system:authentication'):
|
||||
await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
|
||||
|
||||
async def _update_space_provider_for_account(self, account: typing.Any, api_key: str) -> None:
|
||||
"""Refresh the OSS Workspace Space provider without guessing a SaaS Workspace.
|
||||
|
||||
Space OAuth credentials belong to an Account, while model-provider secrets
|
||||
belong to a Workspace. Community edition has one unambiguous Workspace, so
|
||||
the historical automatic refresh remains available only to the Workspace owner.
|
||||
In multi-Workspace SaaS mode the OAuth callback has
|
||||
no trusted Workspace selector; the closed control plane or an explicit
|
||||
Workspace settings action must perform that linkage instead.
|
||||
"""
|
||||
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
if workspace_service is None or collaboration_service is None or not isinstance(account_uuid, str):
|
||||
# Never turn a missing tenant kernel into a global secret mutation.
|
||||
return
|
||||
if workspace_service.policy.multi_workspace_enabled:
|
||||
return
|
||||
|
||||
accesses = await collaboration_service.list_account_workspaces(account_uuid)
|
||||
if len(accesses) != 1:
|
||||
return
|
||||
access = accesses[0]
|
||||
if access.membership.role != MembershipRole.OWNER.value:
|
||||
return
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(
|
||||
access.workspace.uuid,
|
||||
api_key,
|
||||
)
|
||||
|
||||
async def is_initialized(self) -> bool:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
account = await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
return account is not None
|
||||
|
||||
result_list = result.all()
|
||||
return result_list is not None and len(result_list) > 0
|
||||
async def get_login_capabilities(self) -> dict[str, bool]:
|
||||
"""Derive enabled public login methods in an explicit discovery scope."""
|
||||
password_count = sqlalchemy.func.count().filter(user.User.password.is_not(None), user.User.password != '')
|
||||
space_count = sqlalchemy.func.count().filter(user.User.space_account_uuid.is_not(None))
|
||||
statement = sqlalchemy.select(password_count, space_count).where(
|
||||
user.User.status == user.AccountStatus.ACTIVE.value
|
||||
)
|
||||
digest = hashlib.sha256(f'login-capabilities:{self._jwt_identity()[1]}'.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
result = await discovery.session.execute(statement)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
password_accounts, space_accounts = result.one()
|
||||
return {
|
||||
'password_login_enabled': bool(password_accounts),
|
||||
'space_login_enabled': bool(space_accounts),
|
||||
}
|
||||
|
||||
async def get_workspace_owner(self, workspace_uuid: str) -> user.User | None:
|
||||
"""Resolve the active owner Account for a Workspace."""
|
||||
statement = (
|
||||
sqlalchemy.select(user.User)
|
||||
.join(WorkspaceMembership, WorkspaceMembership.account_uuid == user.User.uuid)
|
||||
.where(
|
||||
WorkspaceMembership.workspace_uuid == workspace_uuid,
|
||||
WorkspaceMembership.role == MembershipRole.OWNER.value,
|
||||
WorkspaceMembership.status == MembershipStatus.ACTIVE.value,
|
||||
user.User.status == user.AccountStatus.ACTIVE.value,
|
||||
)
|
||||
)
|
||||
current_session = self.ap.persistence_mgr.current_session()
|
||||
if current_session is not None:
|
||||
return await current_session.scalar(statement)
|
||||
return await self._identity_scalar(statement, f'workspace-owner:{workspace_uuid}')
|
||||
|
||||
def _session_factory(self) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(self.ap.persistence_mgr.get_db_engine(), expire_on_commit=False)
|
||||
|
||||
def _jwt_identity(self) -> tuple[str, str]:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
instance_uuid = str(getattr(workspace_service, 'instance_uuid', '') or constants.instance_id).strip()
|
||||
# UserService is constructed only after config/bootstrap in production.
|
||||
# The fallback keeps lightweight isolated unit tests deterministic.
|
||||
if not instance_uuid:
|
||||
instance_uuid = 'uninitialized-test-instance'
|
||||
return 'langbot-core', f'langbot-instance:{instance_uuid}'
|
||||
|
||||
def _legacy_local_tokens_allowed(self) -> bool:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
policy = getattr(workspace_service, 'policy', None)
|
||||
return getattr(policy, 'multi_workspace_enabled', False) is not True
|
||||
|
||||
async def create_user(self, user_email: str, password: str) -> None:
|
||||
"""Create the first local Account and Workspace owner atomically."""
|
||||
|
||||
await self.create_initial_account(user_email, password)
|
||||
|
||||
async def create_initial_account(self, user_email: str, password: str) -> user.User:
|
||||
self._require_local_directory()
|
||||
normalized_email = normalize_email(user_email)
|
||||
hashed_password = await self._hash_password(password)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(user.User).values(user=user_email, password=hashed_password, account_type='local')
|
||||
async with self._create_user_lock:
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
existing_count = int(
|
||||
(await session.scalar(sqlalchemy.select(sqlalchemy.func.count()).select_from(user.User))) or 0
|
||||
)
|
||||
if existing_count:
|
||||
raise PublicRegistrationClosedError('System already initialized')
|
||||
account = self._new_account(normalized_email, hashed_password)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
await self.ap.workspace_service.bootstrap_local_account(account.uuid, session=session)
|
||||
return account
|
||||
|
||||
async def register_invited_account(
|
||||
self,
|
||||
invitation_token: str,
|
||||
user_email: str,
|
||||
password: str,
|
||||
) -> tuple[user.User, typing.Any]:
|
||||
"""Create an invited Account and accept its Membership in one transaction."""
|
||||
|
||||
normalized_email = normalize_email(user_email)
|
||||
if self._uses_control_plane_directory():
|
||||
raise ControlPlaneDirectoryRequiredError(
|
||||
'Cloud invitation registration must use a Space account to preserve control-plane identity'
|
||||
)
|
||||
invitation, _ = await self.ap.workspace_collaboration_service.inspect_invitation(invitation_token)
|
||||
if invitation.normalized_email != normalized_email:
|
||||
from ....workspace.collaboration import InvitationEmailMismatchError
|
||||
|
||||
raise InvitationEmailMismatchError('Invitation email does not match the Account')
|
||||
hashed_password = await self._hash_password(password)
|
||||
|
||||
async with self._create_user_lock:
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
existing = await session.scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email)
|
||||
)
|
||||
if existing is not None:
|
||||
raise AccountExistsLoginRequiredError('An Account already exists for this email')
|
||||
account = self._new_account(normalized_email, hashed_password)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
membership = await self.ap.workspace_collaboration_service.accept_invitation(
|
||||
invitation_token,
|
||||
account.uuid,
|
||||
session=session,
|
||||
)
|
||||
return account, membership
|
||||
|
||||
def _new_account(self, normalized_email: str, hashed_password: str) -> user.User:
|
||||
return user.User(
|
||||
uuid=str(uuid.uuid4()),
|
||||
user=normalized_email,
|
||||
normalized_email=normalized_email,
|
||||
password=hashed_password,
|
||||
account_type='local',
|
||||
status=user.AccountStatus.ACTIVE.value,
|
||||
source=user.AccountSource.LOCAL.value,
|
||||
projection_revision=0,
|
||||
)
|
||||
|
||||
async def get_user_by_email(self, user_email: str) -> user.User | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.user == user_email)
|
||||
normalized_email = user_email.strip().casefold()
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
async def get_user_by_uuid(self, account_uuid: str) -> user.User | None:
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid),
|
||||
f'uuid:{account_uuid}',
|
||||
)
|
||||
|
||||
async def get_user_by_space_account_uuid(self, space_account_uuid: str) -> user.User | None:
|
||||
"""Get user by Space account UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid)
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
|
||||
async def authenticate(self, user_email: str, password: str) -> str | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.user == user_email)
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
|
||||
if result_list is None or len(result_list) == 0:
|
||||
user_obj = await self.get_user_by_email(user_email)
|
||||
if user_obj is None:
|
||||
raise ValueError('用户不存在')
|
||||
|
||||
user_obj = result_list[0]
|
||||
self._require_active_account(user_obj)
|
||||
|
||||
# Check if this user has a local password set
|
||||
if not user_obj.password:
|
||||
@@ -78,30 +398,121 @@ class UserService:
|
||||
|
||||
await self._verify_password(user_obj.password, password)
|
||||
|
||||
return await self.generate_jwt_token(user_email)
|
||||
return await self.generate_jwt_token(user_obj)
|
||||
|
||||
async def generate_jwt_token(self, user_email: str) -> str:
|
||||
async def generate_jwt_token(self, account: user.User | str) -> str:
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
jwt_expire = self.ap.instance_config.data['system']['jwt']['expire']
|
||||
|
||||
account_obj: user.User | None = account if not isinstance(account, str) and hasattr(account, 'user') else None
|
||||
user_email = account_obj.user if account_obj is not None else account
|
||||
if account_obj is None and hasattr(self.ap, 'persistence_mgr'):
|
||||
try:
|
||||
account_obj = await self.get_user_by_email(user_email)
|
||||
except (AttributeError, TypeError):
|
||||
# Lightweight unit-test and bootstrap callers may not have persistence wired.
|
||||
account_obj = None
|
||||
|
||||
payload = {
|
||||
'user': user_email,
|
||||
'iss': 'LangBot-' + constants.edition,
|
||||
'iss': self._jwt_identity()[0],
|
||||
'aud': self._jwt_identity()[1],
|
||||
'exp': datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=jwt_expire),
|
||||
}
|
||||
if account_obj is not None:
|
||||
self._require_active_account(account_obj)
|
||||
payload.update(
|
||||
{
|
||||
'sub': account_obj.uuid,
|
||||
'account_revision': account_obj.projection_revision,
|
||||
}
|
||||
)
|
||||
|
||||
return jwt.encode(payload, jwt_secret, algorithm='HS256')
|
||||
|
||||
async def verify_jwt_token(self, token: str) -> str:
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
account = await self.get_authenticated_account(token, allow_unresolved_legacy=True)
|
||||
if isinstance(account, str):
|
||||
return account
|
||||
return account.user
|
||||
|
||||
return jwt.decode(token, jwt_secret, algorithms=['HS256'])['user']
|
||||
async def get_authenticated_account(
|
||||
self,
|
||||
token: str,
|
||||
*,
|
||||
allow_unresolved_legacy: bool = False,
|
||||
) -> user.User | str:
|
||||
"""Resolve a JWT to an active Account, accepting bounded legacy email tokens."""
|
||||
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
issuer, audience = self._jwt_identity()
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
jwt_secret,
|
||||
algorithms=['HS256'],
|
||||
issuer=issuer,
|
||||
audience=audience,
|
||||
options={'require': ['exp', 'iss', 'aud']},
|
||||
)
|
||||
except jwt.MissingRequiredClaimError:
|
||||
# Preserve one bounded OSS upgrade path for previously issued
|
||||
# community tokens. SaaS/Cloud policy never accepts these tokens,
|
||||
# and a token carrying a new-style or foreign audience cannot fall
|
||||
# back into the legacy decoder.
|
||||
unverified = jwt.decode(token, options={'verify_signature': False})
|
||||
if (
|
||||
not self._legacy_local_tokens_allowed()
|
||||
or 'aud' in unverified
|
||||
or unverified.get('iss') != 'LangBot-community'
|
||||
):
|
||||
raise
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
jwt_secret,
|
||||
algorithms=['HS256'],
|
||||
options={'require': ['exp'], 'verify_aud': False, 'verify_iss': False},
|
||||
)
|
||||
account_obj: user.User | None = None
|
||||
account_uuid = payload.get('sub')
|
||||
if isinstance(account_uuid, str) and account_uuid:
|
||||
try:
|
||||
account_obj = await self.get_user_by_uuid(account_uuid)
|
||||
except AttributeError:
|
||||
account_obj = None
|
||||
if account_obj is None:
|
||||
legacy_email = payload.get('user')
|
||||
if not isinstance(legacy_email, str) or not legacy_email:
|
||||
raise ValueError('JWT Account identity is missing')
|
||||
try:
|
||||
account_obj = await self.get_user_by_email(legacy_email)
|
||||
except AttributeError:
|
||||
account_obj = None
|
||||
if account_obj is None and allow_unresolved_legacy:
|
||||
return legacy_email
|
||||
if account_obj is None:
|
||||
raise ValueError('Account not found')
|
||||
self._require_active_account(account_obj)
|
||||
token_revision = payload.get('account_revision')
|
||||
if token_revision is not None and int(token_revision) != account_obj.projection_revision:
|
||||
raise ValueError('Account token revision is stale')
|
||||
return account_obj
|
||||
|
||||
@staticmethod
|
||||
def _require_active_account(account: user.User) -> None:
|
||||
status = getattr(account, 'status', user.AccountStatus.ACTIVE.value)
|
||||
if isinstance(status, str) and status != user.AccountStatus.ACTIVE.value:
|
||||
raise AccountDisabledError('Account is disabled')
|
||||
|
||||
async def reset_password(self, user_email: str, new_password: str) -> None:
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def change_password(self, user_email: str, current_password: str, new_password: str) -> None:
|
||||
@@ -115,9 +526,13 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Space user management
|
||||
@@ -132,6 +547,16 @@ class UserService:
|
||||
expires_in: int = 0,
|
||||
) -> user.User:
|
||||
"""Create or update a Space user account (only if system not initialized or user exists)"""
|
||||
if self._uses_control_plane_directory():
|
||||
return await self._update_projected_space_user(
|
||||
space_account_uuid=space_account_uuid,
|
||||
email=email,
|
||||
access_token=access_token,
|
||||
refresh_token=refresh_token,
|
||||
api_key=api_key,
|
||||
expires_in=expires_in,
|
||||
)
|
||||
self._require_local_directory()
|
||||
expires_at = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) if expires_in > 0 else None
|
||||
|
||||
async with self._create_user_lock:
|
||||
@@ -140,7 +565,7 @@ class UserService:
|
||||
|
||||
if existing_user:
|
||||
# Update existing user's tokens
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.space_account_uuid == space_account_uuid)
|
||||
.values(
|
||||
@@ -148,19 +573,56 @@ class UserService:
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
await self._update_space_provider_for_account(existing_user, api_key)
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
|
||||
# Check if user with same email exists
|
||||
existing_email_user = await self.get_user_by_email(email)
|
||||
if existing_email_user:
|
||||
# Update existing user to link with Space account
|
||||
# Email is display/contact identity, not an OAuth subject. An
|
||||
# unknown Space subject must never take over an existing local
|
||||
# Account merely by presenting the same email. The Account
|
||||
# owner must first authenticate locally and use the explicit,
|
||||
# account-bound bind flow.
|
||||
raise account_errors.SpaceAccountBindingRequiredError()
|
||||
|
||||
# Check if system is already initialized
|
||||
is_initialized = await self.is_initialized()
|
||||
if is_initialized:
|
||||
raise account_errors.SpaceAccountNotRegisteredError()
|
||||
|
||||
# Create new Space user (first time initialization)
|
||||
if hasattr(self.ap.persistence_mgr, 'get_db_engine') and hasattr(self.ap, 'workspace_service'):
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
account = user.User(
|
||||
uuid=str(uuid.uuid4()),
|
||||
user=normalize_email(email),
|
||||
normalized_email=normalize_email(email),
|
||||
password='',
|
||||
account_type='space',
|
||||
status=user.AccountStatus.ACTIVE.value,
|
||||
source=user.AccountSource.LOCAL.value,
|
||||
projection_revision=0,
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
await self.ap.workspace_service.bootstrap_local_account(account.uuid, session=session)
|
||||
else:
|
||||
# Compatibility path for lightweight service tests without a real engine.
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.user == email)
|
||||
.values(
|
||||
sqlalchemy.insert(user.User).values(
|
||||
user=normalize_email(email),
|
||||
normalized_email=normalize_email(email),
|
||||
password='',
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
@@ -169,30 +631,56 @@ class UserService:
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
return await self.get_user_by_email(email)
|
||||
created_user = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if created_user is not None:
|
||||
await self._update_space_provider_for_account(created_user, api_key)
|
||||
return created_user
|
||||
|
||||
# Check if system is already initialized
|
||||
is_initialized = await self.is_initialized()
|
||||
if is_initialized:
|
||||
raise account_errors.AccountEmailMismatchError()
|
||||
async def _update_projected_space_user(
|
||||
self,
|
||||
*,
|
||||
space_account_uuid: str,
|
||||
email: str,
|
||||
access_token: str,
|
||||
refresh_token: str,
|
||||
api_key: str,
|
||||
expires_in: int,
|
||||
) -> user.User:
|
||||
"""Attach OAuth credentials to an already projected Cloud Account."""
|
||||
|
||||
# Create new Space user (first time initialization)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(user.User).values(
|
||||
user=email,
|
||||
password='', # Space users don't have local password
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
normalized_email = normalize_email(email)
|
||||
expires_at = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) if expires_in > 0 else None
|
||||
async with self._create_user_lock:
|
||||
projected = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if (
|
||||
projected is None
|
||||
or projected.uuid != space_account_uuid
|
||||
or projected.normalized_email != normalized_email
|
||||
or projected.source != user.AccountSource.CLOUD_PROJECTION.value
|
||||
or projected.account_type != 'space'
|
||||
):
|
||||
raise ControlPlaneDirectoryRequiredError('Space Account is not present in the verified Cloud directory')
|
||||
self._require_active_account(projected)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(
|
||||
user.User.uuid == projected.uuid,
|
||||
user.User.space_account_uuid == space_account_uuid,
|
||||
user.User.source == user.AccountSource.CLOUD_PROJECTION.value,
|
||||
)
|
||||
.values(
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
refreshed = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if refreshed is None:
|
||||
raise ControlPlaneDirectoryRequiredError('Space Account disappeared from the verified Cloud directory')
|
||||
self._require_active_account(refreshed)
|
||||
return refreshed
|
||||
|
||||
async def authenticate_space_user(
|
||||
self, access_token: str, refresh_token: str, expires_in: int = 0
|
||||
@@ -221,15 +709,44 @@ class UserService:
|
||||
)
|
||||
|
||||
# Generate JWT token
|
||||
jwt_token = await self.generate_jwt_token(email)
|
||||
jwt_token = await self.generate_jwt_token(user_obj)
|
||||
|
||||
return jwt_token, user_obj
|
||||
|
||||
async def get_first_user(self) -> user.User | None:
|
||||
"""Get the first user (for single-user mode)"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list else None
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
|
||||
async def _identity_scalar(
|
||||
self,
|
||||
statement: typing.Any,
|
||||
identity: str,
|
||||
) -> user.User | None:
|
||||
"""Execute one exact Account lookup in an explicit discovery transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.scalar(statement)
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
rows = result.all()
|
||||
return rows[0] if rows else None
|
||||
|
||||
async def _identity_execute(self, statement: typing.Any, identity: str) -> typing.Any:
|
||||
"""Execute one exact Account mutation in an explicit transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.execute(statement)
|
||||
return await self.ap.persistence_mgr.execute_async(statement)
|
||||
|
||||
async def set_password(self, user_email: str, new_password: str, current_password: str | None = None) -> None:
|
||||
"""Set or change password for a user"""
|
||||
@@ -246,12 +763,19 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def bind_space_account(self, user_email: str, code: str) -> user.User:
|
||||
"""Bind Space account to existing local account"""
|
||||
local_account = await self.get_user_by_email(user_email)
|
||||
if local_account is None:
|
||||
raise ValueError('User not found')
|
||||
# Exchange code for tokens
|
||||
token_data = await self.ap.space_service.exchange_oauth_code(code)
|
||||
access_token = token_data.get('access_token')
|
||||
@@ -273,28 +797,33 @@ class UserService:
|
||||
|
||||
if not space_account_uuid or not space_email:
|
||||
raise ValueError('Invalid Space user info')
|
||||
if normalize_email(space_email) != normalize_email(user_email):
|
||||
raise account_errors.AccountEmailMismatchError()
|
||||
|
||||
# Check if this Space account is already bound to another user
|
||||
existing_space_user = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if existing_space_user and existing_space_user.user != user_email:
|
||||
if existing_space_user and existing_space_user.normalized_email != normalize_email(user_email):
|
||||
raise ValueError('This Space account is already bound to another user')
|
||||
|
||||
# Update local account to Space account
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.user == user_email)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(
|
||||
user=space_email, # Update email to Space email
|
||||
user=normalize_email(space_email), # Update email to Space email
|
||||
normalized_email=normalize_email(space_email),
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Update Space model provider API keys
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
await self._update_space_provider_for_account(local_account, api_key)
|
||||
|
||||
return await self.get_user_by_email(space_email)
|
||||
|
||||
@@ -4,6 +4,12 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import webhook
|
||||
from .secrets import SECRET_MASK, mask_secret_value, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
_DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE = 16
|
||||
_HARD_MAX_WEBHOOKS_PER_WORKSPACE = 64
|
||||
|
||||
|
||||
class WebhookService:
|
||||
@@ -12,31 +18,99 @@ class WebhookService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_webhooks(self) -> list[dict]:
|
||||
def max_per_workspace(self) -> int:
|
||||
"""Return the configured webhook cap within the process hard limit."""
|
||||
|
||||
config = getattr(getattr(self.ap, 'instance_config', None), 'data', {})
|
||||
try:
|
||||
value = int(
|
||||
config.get('webhooks', {}).get(
|
||||
'max_per_workspace',
|
||||
_DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE,
|
||||
)
|
||||
)
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
value = _DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE
|
||||
return min(max(value, 1), _HARD_MAX_WEBHOOKS_PER_WORKSPACE)
|
||||
|
||||
def _serialize_webhook(self, entity, *, include_secret: bool) -> dict:
|
||||
serialized = self.ap.persistence_mgr.serialize_model(webhook.Webhook, entity)
|
||||
if not include_secret:
|
||||
serialized = serialized.copy()
|
||||
serialized['url'] = mask_secret_value(serialized.get('url'))
|
||||
return serialized
|
||||
|
||||
async def get_webhooks(self, context: TenantContext, *, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all webhooks"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(webhook.Webhook))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).order_by(webhook.Webhook.id).limit(_HARD_MAX_WEBHOOKS_PER_WORKSPACE),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
webhooks = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(webhook.Webhook, wh) for wh in webhooks]
|
||||
return [self._serialize_webhook(wh, include_secret=include_secret) for wh in webhooks]
|
||||
|
||||
async def create_webhook(self, name: str, url: str, description: str = '', enabled: bool = True) -> dict:
|
||||
async def create_webhook(
|
||||
self,
|
||||
context: TenantContext,
|
||||
name: str,
|
||||
url: str,
|
||||
description: str = '',
|
||||
enabled: bool = True,
|
||||
) -> dict:
|
||||
"""Create a new webhook"""
|
||||
webhook_data = {'name': name, 'url': url, 'description': description, 'enabled': enabled}
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
max_webhooks = self.max_per_workspace()
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(webhook.Webhook)
|
||||
.where(webhook.Webhook.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
if (count_result.scalar() or 0) >= max_webhooks:
|
||||
raise ValueError(f'Maximum number of webhooks ({max_webhooks}) reached')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(webhook.Webhook).values(**webhook_data))
|
||||
url = restore_secret_placeholders(url, sensitive=True)
|
||||
webhook_data = {
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'name': name,
|
||||
'url': url,
|
||||
'description': description,
|
||||
'enabled': enabled,
|
||||
}
|
||||
|
||||
insert_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(webhook.Webhook).values(**webhook_data)
|
||||
)
|
||||
|
||||
# Retrieve the created webhook
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.url == url).order_by(webhook.Webhook.id.desc())
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == insert_result.inserted_primary_key[0]),
|
||||
webhook.Webhook,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
created_webhook = result.first()
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(webhook.Webhook, created_webhook)
|
||||
|
||||
async def get_webhook(self, webhook_id: int) -> dict | None:
|
||||
async def get_webhook(
|
||||
self,
|
||||
context: TenantContext,
|
||||
webhook_id: int,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a specific webhook by ID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == webhook_id)
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == webhook_id),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
wh = result.first()
|
||||
@@ -44,16 +118,27 @@ class WebhookService:
|
||||
if wh is None:
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(webhook.Webhook, wh)
|
||||
return self._serialize_webhook(wh, include_secret=include_secret)
|
||||
|
||||
async def update_webhook(
|
||||
self, webhook_id: int, name: str = None, url: str = None, description: str = None, enabled: bool = None
|
||||
) -> None:
|
||||
self,
|
||||
context: TenantContext,
|
||||
webhook_id: int,
|
||||
name: str | None = None,
|
||||
url: str | None = None,
|
||||
description: str | None = None,
|
||||
enabled: bool | None = None,
|
||||
) -> bool:
|
||||
"""Update a webhook's metadata"""
|
||||
update_data = {}
|
||||
if name is not None:
|
||||
update_data['name'] = name
|
||||
if url is not None:
|
||||
if url == SECRET_MASK:
|
||||
current = await self.get_webhook(context, webhook_id, include_secret=True)
|
||||
if current is None:
|
||||
return False
|
||||
url = restore_secret_placeholders(url, current.get('url'), sensitive=True)
|
||||
update_data['url'] = url
|
||||
if description is not None:
|
||||
update_data['description'] = description
|
||||
@@ -61,20 +146,37 @@ class WebhookService:
|
||||
update_data['enabled'] = enabled
|
||||
|
||||
if update_data:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(webhook.Webhook).where(webhook.Webhook.id == webhook_id).values(**update_data)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(webhook.Webhook).where(webhook.Webhook.id == webhook_id).values(**update_data),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return (result.rowcount or 0) > 0
|
||||
return await self.get_webhook(context, webhook_id) is not None
|
||||
|
||||
async def delete_webhook(self, webhook_id: int) -> None:
|
||||
async def delete_webhook(self, context: TenantContext, webhook_id: int) -> bool:
|
||||
"""Delete a webhook"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(webhook.Webhook).where(webhook.Webhook.id == webhook_id)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(webhook.Webhook).where(webhook.Webhook.id == webhook_id),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def get_enabled_webhooks(self) -> list[dict]:
|
||||
async def get_enabled_webhooks(self, context: TenantContext) -> list[dict]:
|
||||
"""Get all enabled webhooks"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.enabled == True)
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.enabled == True),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
.order_by(webhook.Webhook.id)
|
||||
.limit(self.max_per_workspace())
|
||||
)
|
||||
|
||||
webhooks = result.all()
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextvars
|
||||
|
||||
from ..http.context import RequestContext
|
||||
|
||||
|
||||
_request_context: contextvars.ContextVar[RequestContext | None] = contextvars.ContextVar(
|
||||
'langbot_mcp_request_context',
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def bind_request_context(context: RequestContext) -> contextvars.Token[RequestContext | None]:
|
||||
"""Bind the authenticated MCP request while its ASGI request is executing."""
|
||||
|
||||
return _request_context.set(context)
|
||||
|
||||
|
||||
def reset_request_context(token: contextvars.Token[RequestContext | None]) -> None:
|
||||
_request_context.reset(token)
|
||||
|
||||
|
||||
def get_request_context() -> RequestContext:
|
||||
"""Return the current trusted MCP context or fail closed."""
|
||||
|
||||
context = _request_context.get()
|
||||
if context is None:
|
||||
raise RuntimeError('MCP Workspace context is unavailable')
|
||||
return context
|
||||
@@ -19,7 +19,10 @@ from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
from ..http.context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from .context import bind_request_context, reset_request_context
|
||||
from .server import LangBotMCPServer
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -28,6 +31,9 @@ if typing.TYPE_CHECKING:
|
||||
|
||||
# JSON-RPC-ish 401 body returned before the MCP app is reached.
|
||||
_UNAUTHORIZED_BODY = b'{"error":"unauthorized","message":"A valid LangBot API key is required for MCP access."}'
|
||||
_ENTITLEMENT_UNAVAILABLE_BODY = (
|
||||
b'{"error":"entitlement_unavailable","message":"Workspace entitlement is unavailable for MCP access."}'
|
||||
)
|
||||
|
||||
|
||||
def _extract_api_key(headers: list[tuple[bytes, bytes]]) -> str:
|
||||
@@ -76,7 +82,7 @@ class MCPMount:
|
||||
def wrap(self, quart_asgi: typing.Callable) -> typing.Callable:
|
||||
"""Return a dispatcher ASGI app fronting ``quart_asgi``."""
|
||||
mcp_asgi = self._mcp_asgi
|
||||
verify_api_key = self.ap.apikey_service.verify_api_key
|
||||
authenticate_api_key = self.ap.apikey_service.authenticate_api_key
|
||||
is_mcp_path = self._is_mcp_path
|
||||
|
||||
async def dispatcher(scope, receive, send): # type: ignore[no-untyped-def]
|
||||
@@ -88,12 +94,12 @@ class MCPMount:
|
||||
|
||||
# Authenticate MCP HTTP requests with a LangBot API key.
|
||||
api_key = _extract_api_key(scope.get('headers', []))
|
||||
authorized = False
|
||||
identity = None
|
||||
if api_key:
|
||||
with contextlib.suppress(Exception):
|
||||
authorized = await verify_api_key(api_key)
|
||||
identity = await authenticate_api_key(api_key)
|
||||
|
||||
if not authorized:
|
||||
if identity is None:
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
@@ -107,6 +113,56 @@ class MCPMount:
|
||||
await send({'type': 'http.response.body', 'body': _UNAUTHORIZED_BODY})
|
||||
return
|
||||
|
||||
await mcp_asgi(scope, receive, send)
|
||||
deployment_admission = getattr(self.ap, 'deployment_admission', None)
|
||||
try:
|
||||
if deployment_admission is not None:
|
||||
deployment_admission.require_active()
|
||||
entitlement_revision = 0
|
||||
deployment = getattr(self.ap, 'deployment', None)
|
||||
if deployment is not None and getattr(deployment, 'multi_workspace_enabled', False):
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None or identity.instance_uuid != resolver.instance_uuid:
|
||||
raise RuntimeError('Workspace entitlement resolver is unavailable')
|
||||
entitlement = await resolver.resolve(identity.workspace_uuid)
|
||||
entitlement_revision = entitlement.entitlement_revision
|
||||
except Exception:
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
'status': 403,
|
||||
'headers': [(b'content-type', b'application/json')],
|
||||
}
|
||||
)
|
||||
await send({'type': 'http.response.body', 'body': _ENTITLEMENT_UNAVAILABLE_BODY})
|
||||
return
|
||||
|
||||
request_context = RequestContext(
|
||||
instance_uuid=identity.instance_uuid,
|
||||
placement_generation=identity.placement_generation,
|
||||
request_id=str(uuid.uuid4()),
|
||||
auth_type='api-key',
|
||||
principal=PrincipalContext(
|
||||
principal_type=PrincipalType.API_KEY,
|
||||
api_key_uuid=identity.api_key_uuid,
|
||||
),
|
||||
workspace=WorkspaceContext(
|
||||
workspace_uuid=identity.workspace_uuid,
|
||||
membership_uuid=None,
|
||||
role=None,
|
||||
permissions=identity.permissions,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('MCP request persistence scope is unavailable')
|
||||
async with tenant_scope(identity.workspace_uuid):
|
||||
token = bind_request_context(request_context)
|
||||
try:
|
||||
await mcp_asgi(scope, receive, send)
|
||||
if deployment_admission is not None:
|
||||
deployment_admission.require_active()
|
||||
finally:
|
||||
reset_request_context(token)
|
||||
|
||||
return dispatcher
|
||||
|
||||
@@ -22,6 +22,9 @@ import typing
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
|
||||
from ..http.authz import Permission, require_permission
|
||||
from .context import get_request_context
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ...core import app as app_module
|
||||
|
||||
@@ -46,6 +49,12 @@ def _dump(value: typing.Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _authorized(permission: Permission):
|
||||
context = get_request_context()
|
||||
require_permission(context, permission)
|
||||
return context
|
||||
|
||||
|
||||
class LangBotMCPServer:
|
||||
"""Builds and owns the FastMCP instance for LangBot."""
|
||||
|
||||
@@ -72,6 +81,7 @@ class LangBotMCPServer:
|
||||
# ----- System (read-only) -------------------------------------- #
|
||||
@mcp.tool(description='Get basic LangBot system/runtime information (version, edition).')
|
||||
async def get_system_info() -> str:
|
||||
_authorized(Permission.WORKSPACE_VIEW)
|
||||
version = None
|
||||
try:
|
||||
version = ap.ver_mgr.get_current_version()
|
||||
@@ -87,11 +97,13 @@ class LangBotMCPServer:
|
||||
# ----- Bots ---------------------------------------------------- #
|
||||
@mcp.tool(description='List all messaging-platform bots. Secrets are redacted.')
|
||||
async def list_bots() -> str:
|
||||
return _dump(await ap.bot_service.get_bots(include_secret=False))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.bot_service.get_bots(context, include_secret=False))
|
||||
|
||||
@mcp.tool(description='Get a single bot by its UUID. Secrets are redacted.')
|
||||
async def get_bot(bot_uuid: str) -> str:
|
||||
return _dump(await ap.bot_service.get_bot(bot_uuid, include_secret=False))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.bot_service.get_bot(context, bot_uuid, include_secret=False))
|
||||
|
||||
@mcp.tool(
|
||||
description=(
|
||||
@@ -101,26 +113,31 @@ class LangBotMCPServer:
|
||||
)
|
||||
)
|
||||
async def create_bot(bot_data: dict) -> str:
|
||||
return _dump({'uuid': await ap.bot_service.create_bot(bot_data)})
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
return _dump({'uuid': await ap.bot_service.create_bot(context, bot_data)})
|
||||
|
||||
@mcp.tool(description='Update a bot by UUID. `bot_data` matches the PUT bot body.')
|
||||
async def update_bot(bot_uuid: str, bot_data: dict) -> str:
|
||||
await ap.bot_service.update_bot(bot_uuid, bot_data)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.bot_service.update_bot(context, bot_uuid, bot_data)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(description='Delete a bot by UUID.')
|
||||
async def delete_bot(bot_uuid: str) -> str:
|
||||
await ap.bot_service.delete_bot(bot_uuid)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.bot_service.delete_bot(context, bot_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
# ----- Pipelines ----------------------------------------------- #
|
||||
@mcp.tool(description='List all pipelines.')
|
||||
async def list_pipelines() -> str:
|
||||
return _dump(await ap.pipeline_service.get_pipelines())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.pipeline_service.get_pipelines(context))
|
||||
|
||||
@mcp.tool(description='Get a single pipeline by UUID.')
|
||||
async def get_pipeline(pipeline_uuid: str) -> str:
|
||||
return _dump(await ap.pipeline_service.get_pipeline(pipeline_uuid))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.pipeline_service.get_pipeline(context, pipeline_uuid))
|
||||
|
||||
@mcp.tool(
|
||||
description=(
|
||||
@@ -129,49 +146,59 @@ class LangBotMCPServer:
|
||||
)
|
||||
)
|
||||
async def create_pipeline(pipeline_data: dict) -> str:
|
||||
return _dump({'uuid': await ap.pipeline_service.create_pipeline(pipeline_data)})
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
return _dump({'uuid': await ap.pipeline_service.create_pipeline(context, pipeline_data)})
|
||||
|
||||
@mcp.tool(description='Update a pipeline by UUID. `pipeline_data` matches the PUT body.')
|
||||
async def update_pipeline(pipeline_uuid: str, pipeline_data: dict) -> str:
|
||||
await ap.pipeline_service.update_pipeline(pipeline_uuid, pipeline_data)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.pipeline_service.update_pipeline(context, pipeline_uuid, pipeline_data)
|
||||
return _dump({'ok': True})
|
||||
|
||||
@mcp.tool(description='Delete a pipeline by UUID.')
|
||||
async def delete_pipeline(pipeline_uuid: str) -> str:
|
||||
await ap.pipeline_service.delete_pipeline(pipeline_uuid)
|
||||
context = _authorized(Permission.RESOURCE_MANAGE)
|
||||
await ap.pipeline_service.delete_pipeline(context, pipeline_uuid)
|
||||
return _dump({'ok': True})
|
||||
|
||||
# ----- Models -------------------------------------------------- #
|
||||
@mcp.tool(description='List all configured LLM models. Secrets are redacted.')
|
||||
async def list_llm_models() -> str:
|
||||
return _dump(await ap.llm_model_service.get_llm_models(include_secret=False))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.llm_model_service.get_llm_models(context, include_secret=False))
|
||||
|
||||
@mcp.tool(description='Get a single LLM model by UUID.')
|
||||
async def get_llm_model(model_uuid: str) -> str:
|
||||
return _dump(await ap.llm_model_service.get_llm_model(model_uuid))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.llm_model_service.get_llm_model(context, model_uuid, include_secret=False))
|
||||
|
||||
@mcp.tool(description='List all configured embedding models.')
|
||||
async def list_embedding_models() -> str:
|
||||
return _dump(await ap.embedding_models_service.get_embedding_models())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.embedding_models_service.get_embedding_models(context, include_secret=False))
|
||||
|
||||
@mcp.tool(description='List all model providers (OpenAI-compatible, Anthropic, etc.).')
|
||||
async def list_model_providers() -> str:
|
||||
return _dump(await ap.provider_service.get_providers())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.provider_service.get_providers(context, include_secret=False))
|
||||
|
||||
# ----- Knowledge bases ----------------------------------------- #
|
||||
@mcp.tool(description='List all knowledge bases (RAG).')
|
||||
async def list_knowledge_bases() -> str:
|
||||
return _dump(await ap.knowledge_service.get_knowledge_bases())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.knowledge_service.get_knowledge_bases(context))
|
||||
|
||||
@mcp.tool(description='Get a single knowledge base by UUID.')
|
||||
async def get_knowledge_base(kb_uuid: str) -> str:
|
||||
return _dump(await ap.knowledge_service.get_knowledge_base(kb_uuid))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.knowledge_service.get_knowledge_base(context, kb_uuid))
|
||||
|
||||
@mcp.tool(
|
||||
description=('Retrieve (semantic search) from a knowledge base. Returns the matched chunks for `query`.')
|
||||
)
|
||||
async def retrieve_knowledge_base(kb_uuid: str, query: str) -> str:
|
||||
return _dump(await ap.knowledge_service.retrieve_knowledge_base(kb_uuid, query))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.knowledge_service.retrieve_knowledge_base(context, kb_uuid, query))
|
||||
|
||||
# ----- MCP servers (LangBot as MCP client) --------------------- #
|
||||
@mcp.tool(
|
||||
@@ -180,16 +207,19 @@ class LangBotMCPServer:
|
||||
)
|
||||
)
|
||||
async def list_mcp_servers() -> str:
|
||||
return _dump(await ap.mcp_service.get_mcp_servers())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.mcp_service.get_mcp_servers(context))
|
||||
|
||||
# ----- Skills -------------------------------------------------- #
|
||||
@mcp.tool(description='List installed skills.')
|
||||
async def list_skills() -> str:
|
||||
return _dump(await ap.skill_service.list_skills())
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.skill_service.list_skills(context))
|
||||
|
||||
@mcp.tool(description='Get a single skill by name.')
|
||||
async def get_skill(skill_name: str) -> str:
|
||||
return _dump(await ap.skill_service.get_skill(skill_name))
|
||||
context = _authorized(Permission.RESOURCE_VIEW)
|
||||
return _dump(await ap.skill_service.get_skill(context, skill_name))
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# ASGI app
|
||||
|
||||
Reference in New Issue
Block a user