mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 12:40:59 +00:00
feat(tenancy): add Workspace multi-tenant foundation (#2353)
* Document multi-tenant workspace architecture * Add OSS and commercial workspace boundaries * docs: redesign multi-tenant workspace architecture * feat(tenancy): implement workspace isolation * docs(tenancy): record verification evidence * docs(tenancy): revise single-instance SaaS topology * docs(tenancy): refine architecture options * docs: finalize cloud v2 multi-tenant decisions * feat(tenancy): establish cloud isolation foundations * feat(tenancy): harden shared cloud runtime boundaries * docs(tenancy): record final isolation verification * fix(tenancy): close isolation and permission gaps * docs(tenancy): record final isolation verification * feat(tenancy): connect cloud workspace control plane * fix(build): install git for pinned SDK * docs(cloud): update control plane verification * chore: update multi-tenant SDK pin * fix(cloud): skip legacy model sync during startup * test(cloud): preserve minimal model manager fixtures * fix(cloud): preserve authenticated account context * fix(cloud): reuse authenticated account for user info * feat(cloud): complete Workspace settings navigation * test(web): cover Workspace dropdown menu * feat(web): place workspace controls in sidebar * refactor(web): streamline workspace controls * style(web): format workspace layout test * fix(cloud): surface runtime and workspace plan status * fix(plugin): keep runtime identity stable across restarts * fix(ui): widen and center workspace switcher * fix(ui): hide roles from workspace switcher * fix(ui): align workspace switcher with sidebar entries * feat(workspace): add in-product collaboration and direct Cloud launch * style: format collaboration changes * fix(workspace): bind collaboration APIs to tenant UoW * fix(cloud): preserve Core-owned collaboration state * test(cloud): require Space identity for invite registration * feat(cloud): complete secure invitation experience * style(web): format invitation flows * fix(cloud): recover box runtime without unscoped skill reload * feat(oss): enforce invitation account and owner billing flows * style: format OSS account service * test(oss): cover invitation logout handoff * fix(oss): resolve workspace owner in scoped session * feat(cloud): harden multi-tenant runtime resources * fix(cloud): bound runtime restart storms * fix(cloud): eliminate periodic runtime CPU spikes * fix(cloud): enforce instance capacity ceilings * fix(cloud): scope public login capability discovery * fix(cloud): bound tenant maintenance and monitoring work * fix(runtime): bound tenant resource amplification * fix(deps): pin green multi-tenant plugin SDK * fix(cloud): handle unavailable skill capability * fix(security): require authentication for image file endpoint (H-2) - Changed /api/v1/files/image from AuthType.NONE to USER_TOKEN_OR_API_KEY - Added Permission.RESOURCE_VIEW requirement - Prevents unauthenticated cross-tenant file access via leaked keys - Fixes HIGH severity finding from multi-tenant security review docs: add comprehensive database migration guide - Complete migration steps for OSS → multi-tenant - Backup, execution, verification procedures - Rollback scenarios and recovery plans - Performance tuning recommendations * test: add comprehensive cross-tenant isolation tests Added 7 critical test scenarios for multi-tenant boundaries: - Cross-tenant bot access prevention - Viewer role read-only enforcement - Removed member immediate access revocation - Model provider credential isolation - WebSocket message isolation - Invitation token workspace scoping - Multi-workspace context validation These tests address P0-2 coverage gaps for: - workspaces.py (membership & invitation flows) - user.py (authentication & authorization) - websocket_chat.py (real-time isolation) - plugins.py (resource access control) docs: finalize database migration guide * fix(security): resolve M-1, M-2, M-3 security findings M-1: WebSocket authorization TOCTOU race (FIXED) - Changed _revalidate_websocket_authorization to return RequestContext - Ensures validated context is used immediately without race window - Prevents removed members from sending messages during revalidation gap M-2: Model Manager cache workspace isolation (VERIFIED) - Confirmed _CacheKey already uses 4-tuple: (instance, workspace, generation, resource) - Cache is properly scoped per workspace, no cross-tenant leakage possible - No code change needed, documented as working correctly M-3: Invitation lock workspace scoping (FIXED) - Changed lock key from token_digest to workspace_uuid:token_digest - Prevents DoS where attacker locks token in Workspace A to block Workspace B - Locks now isolated per workspace All MEDIUM severity findings from security review now resolved. * fix(cloud): unblock tenant CI and enforce knowledge quotas * fix(tenancy): scope rerank model sync --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
@@ -1,97 +1,304 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import datetime
|
||||
import hashlib
|
||||
import secrets
|
||||
import typing
|
||||
import uuid
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import apikey
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..authz import Permission, PermissionDeniedError
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class ApiKeyIdentity:
|
||||
"""Trusted Workspace identity derived from an API-key secret."""
|
||||
|
||||
instance_uuid: str
|
||||
workspace_uuid: str
|
||||
placement_generation: int
|
||||
api_key_uuid: str
|
||||
permissions: frozenset[str]
|
||||
|
||||
|
||||
class ApiKeyService:
|
||||
ap: app.Application
|
||||
"""Manage hashed, Workspace-bound API keys."""
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(self, ap: Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_api_keys(self) -> list[dict]:
|
||||
"""Get all API keys"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(apikey.ApiKey))
|
||||
@staticmethod
|
||||
def _hash_secret(secret: str) -> str:
|
||||
return hashlib.sha256(secret.encode('utf-8')).hexdigest()
|
||||
|
||||
keys = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(apikey.ApiKey, key) for key in keys]
|
||||
@staticmethod
|
||||
def _utcnow() -> datetime.datetime:
|
||||
return datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
|
||||
async def create_api_key(self, name: str, description: str = '') -> dict:
|
||||
"""Create a new API key"""
|
||||
# Generate a secure random API key
|
||||
key = f'lbk_{secrets.token_urlsafe(32)}'
|
||||
@staticmethod
|
||||
def _normalize_scopes(
|
||||
scopes: typing.Iterable[str] | None,
|
||||
*,
|
||||
default: typing.Iterable[str] = (),
|
||||
) -> list[str]:
|
||||
requested = list(default if scopes is None else scopes)
|
||||
valid = {permission.value for permission in Permission}
|
||||
normalized: list[str] = []
|
||||
for scope in requested:
|
||||
if not isinstance(scope, str):
|
||||
raise ValueError('API key scopes must be strings')
|
||||
value = scope.strip()
|
||||
if value not in valid:
|
||||
raise ValueError(f'Unknown API key scope: {value}')
|
||||
if value not in normalized:
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
key_data = {'name': name, 'key': key, 'description': description}
|
||||
def _serialize(self, row: typing.Any) -> dict[str, typing.Any]:
|
||||
value = self.ap.persistence_mgr.serialize_model(apikey.ApiKey, row)
|
||||
value.pop('key_hash', None)
|
||||
# The secret is deliberately unrecoverable after creation.
|
||||
value['secret_available'] = False
|
||||
return value
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(apikey.ApiKey).values(**key_data))
|
||||
|
||||
# Retrieve the created key
|
||||
async def get_api_keys(self, context: TenantContext) -> list[dict[str, typing.Any]]:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).order_by(apikey.ApiKey.created_at, apikey.ApiKey.id),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
created_key = result.first()
|
||||
return [self._serialize(key) for key in result.all()]
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(apikey.ApiKey, created_key)
|
||||
async def create_api_key(
|
||||
self,
|
||||
context: TenantContext,
|
||||
name: str,
|
||||
description: str = '',
|
||||
*,
|
||||
scopes: typing.Iterable[str] | None = None,
|
||||
expires_at: datetime.datetime | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
normalized_name = name.strip()
|
||||
if not normalized_name:
|
||||
raise ValueError('Name is required')
|
||||
if expires_at is not None:
|
||||
if expires_at.tzinfo is not None:
|
||||
expires_at = expires_at.astimezone(datetime.UTC).replace(tzinfo=None)
|
||||
if expires_at <= self._utcnow():
|
||||
raise ValueError('API key expiry must be in the future')
|
||||
|
||||
async def get_api_key(self, key_id: int) -> dict | None:
|
||||
"""Get a specific API key by ID"""
|
||||
default_scopes = getattr(getattr(context, 'workspace', None), 'permissions', frozenset())
|
||||
normalized_scopes = self._normalize_scopes(scopes, default=default_scopes)
|
||||
allowed_scopes = frozenset(default_scopes)
|
||||
unauthorized_scopes = sorted(set(normalized_scopes) - allowed_scopes)
|
||||
if unauthorized_scopes:
|
||||
# API-key management delegates the caller's authority; it must not
|
||||
# become a path for minting a stronger principal.
|
||||
raise PermissionDeniedError(unauthorized_scopes[0])
|
||||
secret = f'lbk_{secrets.token_urlsafe(32)}'
|
||||
key_uuid = str(uuid.uuid4())
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(apikey.ApiKey).values(
|
||||
uuid=key_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
created_by_account_uuid=getattr(context, 'account_uuid', None),
|
||||
name=normalized_name,
|
||||
key_hash=self._hash_secret(secret),
|
||||
scopes=normalized_scopes,
|
||||
status=apikey.ApiKeyStatus.ACTIVE.value,
|
||||
expires_at=expires_at,
|
||||
description=description.strip(),
|
||||
)
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.id == key_id)
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.uuid == key_uuid),
|
||||
apikey.ApiKey,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
created = result.first()
|
||||
if created is None:
|
||||
raise RuntimeError('Created API key could not be loaded')
|
||||
value = self._serialize(created)
|
||||
value['key'] = secret
|
||||
value['secret_available'] = True
|
||||
return value
|
||||
|
||||
async def get_api_key(self, context: TenantContext, key_id: int) -> dict[str, typing.Any] | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.id == key_id),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
key = result.first()
|
||||
return None if key is None else self._serialize(key)
|
||||
|
||||
if key is None:
|
||||
async def authenticate_api_key(self, secret: str) -> ApiKeyIdentity | None:
|
||||
"""Authenticate a secret and derive its Workspace without trusting headers."""
|
||||
|
||||
if not isinstance(secret, str) or not secret.strip():
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(apikey.ApiKey, key)
|
||||
global_secret = self.ap.instance_config.data.get('api', {}).get('global_api_key', '')
|
||||
if global_secret and secrets.compare_digest(secret, global_secret):
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
if workspace_service is None or workspace_service.policy.multi_workspace_enabled:
|
||||
return None
|
||||
binding = await workspace_service.get_local_execution_binding()
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid='global-oss-api-key',
|
||||
permissions=frozenset(permission.value for permission in Permission),
|
||||
)
|
||||
|
||||
async def verify_api_key(self, key: str) -> bool:
|
||||
"""Verify if an API key is valid.
|
||||
if not secret.startswith('lbk_'):
|
||||
return None
|
||||
secret_hash = self._hash_secret(secret)
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
discovery_uow = getattr(self.ap.persistence_mgr, 'api_key_discovery_uow', None)
|
||||
if current_session() is None and callable(discovery_uow):
|
||||
async with discovery_uow(secret_hash) as discovery:
|
||||
key = await discovery.session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
key = result.first()
|
||||
if key is None:
|
||||
return None
|
||||
discovered_workspace_uuid = key.workspace_uuid
|
||||
discovered_key_id = key.id
|
||||
now = self._utcnow()
|
||||
|
||||
A key is accepted if it matches the global API key configured in
|
||||
``config.yaml`` (``api.global_api_key``) — which requires no login
|
||||
session and no database record — or if it matches a key created via
|
||||
the web UI (stored in the database, prefixed with ``lbk_``).
|
||||
"""
|
||||
if not isinstance(key, str) or not key:
|
||||
return False
|
||||
async def bind_and_record_use() -> tuple[typing.Any, typing.Any] | None:
|
||||
# Re-read inside the tenant transaction. A revoke/expiry racing
|
||||
# discovery must not result in an authenticated identity.
|
||||
active_session = current_session()
|
||||
if active_session is not None:
|
||||
scoped_key = await active_session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
else: # compatibility for isolated service tests
|
||||
scoped_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
scoped_key = scoped_result.first()
|
||||
if scoped_key is None or scoped_key.status != apikey.ApiKeyStatus.ACTIVE.value:
|
||||
return None
|
||||
if scoped_key.expires_at is not None and scoped_key.expires_at <= now:
|
||||
return None
|
||||
|
||||
# 1. Global API key from config.yaml (no DB lookup, no login state).
|
||||
# Note: config completion only backfills top-level keys, so existing
|
||||
# installs may not have this key — access it defensively.
|
||||
global_api_key = self.ap.instance_config.data.get('api', {}).get('global_api_key', '')
|
||||
if global_api_key and secrets.compare_digest(key, global_api_key):
|
||||
return True
|
||||
binding = await self.ap.workspace_service.get_execution_binding(discovered_workspace_uuid)
|
||||
updated = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(
|
||||
apikey.ApiKey.id == scoped_key.id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
apikey.ApiKey.status == apikey.ApiKeyStatus.ACTIVE.value,
|
||||
)
|
||||
.values(last_used_at=now)
|
||||
.returning(apikey.ApiKey.id)
|
||||
)
|
||||
# Authentication and revocation race on this atomic predicate. If
|
||||
# revoke won, no active row is returned and the stale object read
|
||||
# above must never become an authenticated identity.
|
||||
if updated.scalar_one_or_none() is None:
|
||||
return None
|
||||
return binding, scoped_key
|
||||
|
||||
# 2. Web-UI-created keys are stored in the database and prefixed lbk_.
|
||||
if not key.startswith('lbk_'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key == key)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if current_session() is None and callable(tenant_uow):
|
||||
async with tenant_uow(discovered_workspace_uuid):
|
||||
bound = await bind_and_record_use()
|
||||
else:
|
||||
bound = await bind_and_record_use()
|
||||
if bound is None:
|
||||
return None
|
||||
binding, scoped_key = bound
|
||||
raw_scopes = list(scoped_key.scopes or [])
|
||||
permissions = (
|
||||
frozenset(permission.value for permission in Permission)
|
||||
if '*' in raw_scopes
|
||||
else frozenset(self._normalize_scopes(raw_scopes))
|
||||
)
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid=scoped_key.uuid,
|
||||
permissions=permissions,
|
||||
)
|
||||
|
||||
key_obj = result.first()
|
||||
return key_obj is not None
|
||||
async def verify_api_key(self, secret: str) -> bool:
|
||||
try:
|
||||
return await self.authenticate_api_key(secret) is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def delete_api_key(self, key_id: int) -> None:
|
||||
"""Delete an API key"""
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.delete(apikey.ApiKey).where(apikey.ApiKey.id == key_id))
|
||||
|
||||
async def update_api_key(self, key_id: int, name: str = None, description: str = None) -> None:
|
||||
"""Update an API key's metadata (name, description)"""
|
||||
update_data = {}
|
||||
if name is not None:
|
||||
update_data['name'] = name
|
||||
if description is not None:
|
||||
update_data['description'] = description
|
||||
|
||||
if update_data:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey).where(apikey.ApiKey.id == key_id).values(**update_data)
|
||||
async def delete_api_key(self, context: TenantContext, key_id: int) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(apikey.ApiKey.id == key_id)
|
||||
.values(status=apikey.ApiKeyStatus.REVOKED.value),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', 0) == 0:
|
||||
raise WorkspaceNotFoundError('API key not found')
|
||||
|
||||
async def update_api_key(
|
||||
self,
|
||||
context: TenantContext,
|
||||
key_id: int,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
) -> None:
|
||||
update_data: dict[str, typing.Any] = {}
|
||||
if name is not None:
|
||||
normalized_name = name.strip()
|
||||
if not normalized_name:
|
||||
raise ValueError('Name is required')
|
||||
update_data['name'] = normalized_name
|
||||
if description is not None:
|
||||
update_data['description'] = description.strip()
|
||||
if not update_data:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(apikey.ApiKey).where(apikey.ApiKey.id == key_id).values(**update_data),
|
||||
apikey.ApiKey,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', 0) == 0:
|
||||
raise WorkspaceNotFoundError('API key not found')
|
||||
|
||||
@@ -2,11 +2,12 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import sqlalchemy
|
||||
import typing
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
class BotService:
|
||||
@@ -17,9 +18,11 @@ class BotService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_bots(self, include_secret: bool = True) -> list[dict]:
|
||||
async def get_bots(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_bot.Bot), persistence_bot.Bot, context)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
@@ -29,10 +32,14 @@ class BotService:
|
||||
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns) for bot in bots]
|
||||
|
||||
async def get_bot(self, bot_uuid: str, include_secret: bool = True) -> dict | None:
|
||||
async def get_bot(self, context: TenantContext, bot_uuid: str, include_secret: bool = False) -> dict | None:
|
||||
"""获取机器人"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
bot = result.first()
|
||||
@@ -46,15 +53,20 @@ class BotService:
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_bot.Bot, bot, masked_columns)
|
||||
|
||||
async def get_runtime_bot_info(self, bot_uuid: str, include_secret: bool = True) -> dict:
|
||||
async def get_runtime_bot_info(
|
||||
self,
|
||||
context: TenantContext,
|
||||
bot_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict:
|
||||
"""获取机器人运行时信息"""
|
||||
persistence_bot = await self.get_bot(bot_uuid, include_secret)
|
||||
persistence_bot = await self.get_bot(context, bot_uuid, include_secret)
|
||||
if persistence_bot is None:
|
||||
raise Exception('Bot not found')
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
|
||||
adapter_runtime_values = {}
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is not None:
|
||||
adapter_runtime_values['bot_account_id'] = runtime_bot.adapter.bot_account_id
|
||||
|
||||
@@ -86,22 +98,29 @@ class BotService:
|
||||
|
||||
return persistence_bot
|
||||
|
||||
async def create_bot(self, bot_data: dict) -> str:
|
||||
async def create_bot(self, context: TenantContext, bot_data: dict) -> str:
|
||||
"""Create bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_bots = limitation.get('max_bots', -1)
|
||||
if max_bots >= 0:
|
||||
existing_bots = await self.get_bots()
|
||||
existing_bots = await self.get_bots(context)
|
||||
if len(existing_bots) >= max_bots:
|
||||
raise ValueError(f'Maximum number of bots ({max_bots}) reached')
|
||||
|
||||
# TODO: 检查配置信息格式
|
||||
bot_data = bot_data.copy()
|
||||
bot_data['uuid'] = str(uuid.uuid4())
|
||||
bot_data['workspace_uuid'] = workspace_uuid
|
||||
|
||||
# bind the most recently updated pipeline if any exist
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
.order_by(persistence_pipeline.LegacyPipeline.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -112,61 +131,84 @@ class BotService:
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(bot_data))
|
||||
|
||||
bot = await self.get_bot(bot_data['uuid'])
|
||||
bot = await self.get_bot(context, bot_data['uuid'], include_secret=True)
|
||||
|
||||
await self.ap.platform_mgr.load_bot(bot)
|
||||
await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
return bot_data['uuid']
|
||||
|
||||
async def update_bot(self, bot_uuid: str, bot_data: dict) -> None:
|
||||
async def update_bot(self, context: TenantContext, bot_uuid: str, bot_data: dict) -> None:
|
||||
"""Update bot"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
update_data = bot_data.copy()
|
||||
|
||||
if 'uuid' in update_data:
|
||||
del update_data['uuid']
|
||||
update_data.pop('uuid', None)
|
||||
update_data.pop('workspace_uuid', None)
|
||||
|
||||
# set use_pipeline_name
|
||||
if 'use_pipeline_uuid' in update_data:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == update_data['use_pipeline_uuid']
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
if pipeline is not None:
|
||||
update_data['use_pipeline_name'] = pipeline.name
|
||||
else:
|
||||
raise Exception('Pipeline not found')
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_bot.Bot).values(update_data).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
# select from db
|
||||
bot = await self.get_bot(bot_uuid)
|
||||
bot = await self.get_bot(context, bot_uuid, include_secret=True)
|
||||
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(bot)
|
||||
runtime_bot = await self.ap.platform_mgr.load_bot(context, bot)
|
||||
|
||||
if runtime_bot.enable:
|
||||
await runtime_bot.run()
|
||||
|
||||
# update all conversation that use this bot
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if session.using_conversation is not None and session.using_conversation.bot_uuid == bot_uuid:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
and session.using_conversation.bot_uuid == bot_uuid
|
||||
and getattr(session, 'workspace_uuid', workspace_uuid) == workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
|
||||
async def delete_bot(self, bot_uuid: str) -> None:
|
||||
async def delete_bot(self, context: TenantContext, bot_uuid: str) -> None:
|
||||
"""Delete bot"""
|
||||
await self.ap.platform_mgr.remove_bot(bot_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_bot.Bot).where(persistence_bot.Bot.uuid == bot_uuid),
|
||||
persistence_bot.Bot,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.platform_mgr.remove_bot(context, bot_uuid)
|
||||
|
||||
async def list_event_logs(
|
||||
self, bot_uuid: str, from_index: int, max_count: int
|
||||
) -> typing.Tuple[list[dict], int, int, int]:
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
self, context: TenantContext, bot_uuid: str, from_index: int, max_count: int
|
||||
) -> tuple[list[dict], int]:
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is None:
|
||||
raise Exception('Bot not found')
|
||||
|
||||
@@ -174,7 +216,14 @@ class BotService:
|
||||
|
||||
return [log.to_json() for log in logs], total_count
|
||||
|
||||
async def send_message(self, bot_uuid: str, target_type: str, target_id: str, message_chain_data: dict) -> None:
|
||||
async def send_message(
|
||||
self,
|
||||
context: TenantContext,
|
||||
bot_uuid: str,
|
||||
target_type: str,
|
||||
target_id: str,
|
||||
message_chain_data: dict,
|
||||
) -> None:
|
||||
"""Send message to a specific target via bot
|
||||
|
||||
Args:
|
||||
@@ -183,11 +232,14 @@ class BotService:
|
||||
target_id: The ID of the target
|
||||
message_chain_data: The message chain data in dict format
|
||||
"""
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
|
||||
# Import here to avoid circular imports
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
|
||||
# Get runtime bot
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid)
|
||||
runtime_bot = await self.ap.platform_mgr.get_bot_by_uuid(context, bot_uuid)
|
||||
if runtime_bot is None:
|
||||
raise Exception(f'Bot not found: {bot_uuid}')
|
||||
|
||||
@@ -202,19 +254,29 @@ class BotService:
|
||||
|
||||
# ============ Bot Admins ============
|
||||
|
||||
async def get_bot_admins(self, bot_uuid: str) -> list[dict]:
|
||||
async def get_bot_admins(self, context: TenantContext, bot_uuid: str) -> list[dict]:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid),
|
||||
persistence_bot.BotAdmin,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return [{'id': r.id, 'launcher_type': r.launcher_type, 'launcher_id': r.launcher_id} for r in result.all()]
|
||||
|
||||
async def add_bot_admin(self, bot_uuid: str, launcher_type: str, launcher_id: str) -> int:
|
||||
async def add_bot_admin(self, context: TenantContext, bot_uuid: str, launcher_type: str, launcher_id: str) -> int:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_bot.BotAdmin).values(
|
||||
workspace_uuid=workspace_uuid,
|
||||
bot_uuid=bot_uuid,
|
||||
launcher_type=launcher_type,
|
||||
launcher_id=launcher_id,
|
||||
@@ -222,12 +284,18 @@ class BotService:
|
||||
)
|
||||
return result.inserted_primary_key[0]
|
||||
|
||||
async def delete_bot_admin(self, bot_uuid: str, admin_id: int) -> None:
|
||||
async def delete_bot_admin(self, context: TenantContext, bot_uuid: str, admin_id: int) -> None:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
if await self.get_bot(context, bot_uuid, include_secret=False) is None:
|
||||
raise WorkspaceNotFoundError('Bot not found')
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_bot.BotAdmin).where(
|
||||
persistence_bot.BotAdmin.bot_uuid == bot_uuid,
|
||||
persistence_bot.BotAdmin.id == admin_id,
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_bot.BotAdmin).where(
|
||||
persistence_bot.BotAdmin.bot_uuid == bot_uuid,
|
||||
persistence_bot.BotAdmin.id == admin_id,
|
||||
),
|
||||
persistence_bot.BotAdmin,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2,8 +2,13 @@ from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ....api.http.authz import WorkspaceRequiredError
|
||||
from ....api.http.context import ExecutionContext, RequestContext
|
||||
from ....core import app
|
||||
from ....entity.persistence import rag as persistence_rag
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
class KnowledgeService:
|
||||
@@ -14,34 +19,69 @@ class KnowledgeService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_knowledge_bases(self) -> list[dict]:
|
||||
@staticmethod
|
||||
def _execution_context(context: RequestContext | ExecutionContext) -> ExecutionContext:
|
||||
if isinstance(context, RequestContext):
|
||||
return ExecutionContext.from_request(context)
|
||||
if isinstance(context, ExecutionContext):
|
||||
return context
|
||||
raise WorkspaceRequiredError('RequestContext or ExecutionContext is required')
|
||||
|
||||
async def get_knowledge_bases(self, context: TenantContext, *, include_secret: bool = False) -> list[dict]:
|
||||
"""获取所有知识库"""
|
||||
return await self.ap.rag_mgr.get_all_knowledge_base_details()
|
||||
require_workspace_uuid(context)
|
||||
knowledge_bases = await self.ap.rag_mgr.get_all_knowledge_base_details(context)
|
||||
return knowledge_bases if include_secret else [redact_secrets(base) for base in knowledge_bases]
|
||||
|
||||
async def get_knowledge_base(self, kb_uuid: str) -> dict | None:
|
||||
async def get_knowledge_base(
|
||||
self,
|
||||
context: TenantContext,
|
||||
kb_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""获取知识库"""
|
||||
return await self.ap.rag_mgr.get_knowledge_base_details(kb_uuid)
|
||||
require_workspace_uuid(context)
|
||||
knowledge_base = await self.ap.rag_mgr.get_knowledge_base_details(context, kb_uuid)
|
||||
if knowledge_base is None or include_secret:
|
||||
return knowledge_base
|
||||
return redact_secrets(knowledge_base)
|
||||
|
||||
async def create_knowledge_base(self, kb_data: dict) -> str:
|
||||
async def create_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_data: dict,
|
||||
) -> str:
|
||||
"""创建知识库"""
|
||||
require_workspace_uuid(context)
|
||||
# In new architecture, we delegate entirely to RAGManager which uses plugins.
|
||||
# Legacy internal KB creation is removed.
|
||||
limitation = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('system', {}).get('limitation', {})
|
||||
)
|
||||
max_knowledge_bases = limitation.get('max_knowledge_bases', -1)
|
||||
if max_knowledge_bases >= 0:
|
||||
knowledge_bases = await self.ap.rag_mgr.get_all_knowledge_base_details(context)
|
||||
if len(knowledge_bases) >= max_knowledge_bases:
|
||||
raise ValueError(f'Maximum number of knowledge bases ({max_knowledge_bases}) reached')
|
||||
|
||||
knowledge_engine_plugin_id = kb_data.get('knowledge_engine_plugin_id')
|
||||
if not knowledge_engine_plugin_id:
|
||||
raise ValueError('knowledge_engine_plugin_id is required')
|
||||
|
||||
creation_settings = kb_data.get('creation_settings', {})
|
||||
creation_settings = restore_secret_placeholders(kb_data.get('creation_settings', {}))
|
||||
retrieval_settings = kb_data.get('retrieval_settings', {})
|
||||
|
||||
# Validate required fields based on plugin's creation_schema and retrieval_schema
|
||||
await self._validate_schema_required_fields(
|
||||
context,
|
||||
knowledge_engine_plugin_id,
|
||||
creation_settings,
|
||||
retrieval_settings,
|
||||
)
|
||||
|
||||
kb = await self.ap.rag_mgr.create_knowledge_base(
|
||||
context,
|
||||
name=kb_data.get('name', 'Untitled'),
|
||||
knowledge_engine_plugin_id=knowledge_engine_plugin_id,
|
||||
creation_settings=creation_settings,
|
||||
@@ -52,6 +92,7 @@ class KnowledgeService:
|
||||
|
||||
async def _validate_schema_required_fields(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
plugin_id: str,
|
||||
creation_settings: dict,
|
||||
retrieval_settings: dict,
|
||||
@@ -69,7 +110,11 @@ class KnowledgeService:
|
||||
Raises:
|
||||
ValueError: If any required field is missing or empty.
|
||||
"""
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return
|
||||
|
||||
# Validate creation_schema
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
creation_schema = await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
self._check_required_fields(creation_schema, creation_settings, 'creation_settings')
|
||||
@@ -79,6 +124,7 @@ class KnowledgeService:
|
||||
self.ap.logger.warning(f'Failed to get creation_schema for validation: {e}')
|
||||
|
||||
# Validate retrieval_schema
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
retrieval_schema = await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
self._check_required_fields(retrieval_schema, retrieval_settings, 'retrieval_settings')
|
||||
@@ -151,8 +197,16 @@ class KnowledgeService:
|
||||
)
|
||||
raise ValueError(f'{field_label} is required ({context}.{field_name})')
|
||||
|
||||
async def update_knowledge_base(self, kb_uuid: str, kb_data: dict) -> None:
|
||||
async def update_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
kb_data: dict,
|
||||
) -> None:
|
||||
"""更新知识库"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
# Filter to only mutable fields
|
||||
filtered_data = {k: v for k, v in kb_data.items() if k in persistence_rag.KnowledgeBase.MUTABLE_FIELDS}
|
||||
|
||||
@@ -162,17 +216,18 @@ class KnowledgeService:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(filtered_data)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(kb_uuid)
|
||||
await self.ap.rag_mgr.remove_knowledge_base_from_runtime(context, kb_uuid)
|
||||
|
||||
kb = await self.get_knowledge_base(kb_uuid)
|
||||
kb = await self.get_knowledge_base(context, kb_uuid, include_secret=True)
|
||||
if kb is None:
|
||||
raise Exception('Knowledge base not found after update')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self.ap.rag_mgr.load_knowledge_base(kb)
|
||||
await self.ap.rag_mgr.load_knowledge_base(context, kb)
|
||||
|
||||
async def _check_doc_capability(self, kb_uuid: str, operation: str) -> None:
|
||||
async def _check_doc_capability(self, context: TenantContext, kb_uuid: str, operation: str) -> None:
|
||||
"""Check if the KB's Knowledge Engine supports document operations.
|
||||
|
||||
Args:
|
||||
@@ -182,104 +237,145 @@ class KnowledgeService:
|
||||
Raises:
|
||||
Exception: If the KB does not support doc_ingestion.
|
||||
"""
|
||||
kb_info = await self.ap.rag_mgr.get_knowledge_base_details(kb_uuid)
|
||||
kb_info = await self.ap.rag_mgr.get_knowledge_base_details(context, kb_uuid)
|
||||
if not kb_info:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
capabilities = kb_info.get('knowledge_engine', {}).get('capabilities', [])
|
||||
if 'doc_ingestion' not in capabilities:
|
||||
raise Exception(f'This knowledge base does not support {operation}')
|
||||
|
||||
async def store_file(self, kb_uuid: str, file_id: str, parser_plugin_id: str | None = None) -> str:
|
||||
async def store_file(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
file_id: str,
|
||||
parser_plugin_id: str | None = None,
|
||||
) -> str:
|
||||
"""存储文件"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self._check_doc_capability(kb_uuid, 'document upload')
|
||||
await self._check_doc_capability(context, kb_uuid, 'document upload')
|
||||
|
||||
result = await runtime_kb.store_file(file_id, parser_plugin_id=parser_plugin_id)
|
||||
result = await runtime_kb.store_file(execution_context, file_id, parser_plugin_id=parser_plugin_id)
|
||||
|
||||
# Update the KB's updated_at timestamp
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(updated_at=sqlalchemy.func.now())
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
async def retrieve_knowledge_base(
|
||||
self, kb_uuid: str, query: str, retrieval_settings: dict | None = None
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
query: str,
|
||||
retrieval_settings: dict | None = None,
|
||||
) -> list[dict]:
|
||||
"""检索知识库"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# Pass retrieval_settings
|
||||
results = await runtime_kb.retrieve(query, settings=retrieval_settings)
|
||||
results = await runtime_kb.retrieve(execution_context, query, settings=retrieval_settings)
|
||||
|
||||
return [result.model_dump() for result in results]
|
||||
|
||||
async def get_files_by_knowledge_base(self, kb_uuid: str) -> list[dict]:
|
||||
async def get_files_by_knowledge_base(self, context: TenantContext, kb_uuid: str) -> list[dict]:
|
||||
"""获取知识库文件"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
files = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_rag.File, file) for file in files]
|
||||
|
||||
async def delete_file(self, kb_uuid: str, file_id: str) -> None:
|
||||
async def delete_file(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
file_id: str,
|
||||
) -> None:
|
||||
"""删除文件"""
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is None:
|
||||
raise Exception('Knowledge base not found')
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
await self._check_doc_capability(kb_uuid, 'document deletion')
|
||||
await self._check_doc_capability(context, kb_uuid, 'document deletion')
|
||||
|
||||
await runtime_kb.delete_file(file_id)
|
||||
await runtime_kb.delete_file(execution_context, file_id)
|
||||
|
||||
# Update the KB's updated_at timestamp
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.values(updated_at=sqlalchemy.func.now())
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
async def delete_knowledge_base(self, kb_uuid: str) -> None:
|
||||
async def delete_knowledge_base(
|
||||
self,
|
||||
context: RequestContext | ExecutionContext,
|
||||
kb_uuid: str,
|
||||
) -> None:
|
||||
"""删除知识库"""
|
||||
# Delete from DB first to commit the deletion, then clean up runtime/plugin (best-effort)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.KnowledgeBase).where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# delete files
|
||||
# NOTE: Chunk cleanup is for legacy (pre-plugin) KBs that stored chunks locally.
|
||||
# For plugin-based Knowledge Engines, the Chunk table is not populated, so this is a no-op.
|
||||
files = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File).where(persistence_rag.File.kb_id == kb_uuid)
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
for file in files:
|
||||
# delete chunks
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.Chunk).where(persistence_rag.Chunk.file_id == file.uuid)
|
||||
sqlalchemy.delete(persistence_rag.Chunk)
|
||||
.where(persistence_rag.Chunk.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.Chunk.file_id == file.uuid)
|
||||
)
|
||||
# delete file
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.File).where(persistence_rag.File.uuid == file.uuid)
|
||||
sqlalchemy.delete(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file.uuid)
|
||||
)
|
||||
|
||||
# Remove from runtime and notify plugin (best-effort, DB is already cleaned up)
|
||||
await self.ap.rag_mgr.delete_knowledge_base(kb_uuid)
|
||||
# Remove from runtime and notify plugin before deleting the owning row.
|
||||
await self.ap.rag_mgr.delete_knowledge_base(context, kb_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_rag.KnowledgeBase)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.KnowledgeBase.uuid == kb_uuid)
|
||||
)
|
||||
|
||||
# ================= Knowledge Engine Discovery =================
|
||||
|
||||
async def list_knowledge_engines(self) -> list[dict]:
|
||||
async def list_knowledge_engines(self, context: TenantContext) -> list[dict]:
|
||||
"""List all available Knowledge Engines from plugins."""
|
||||
require_workspace_uuid(context)
|
||||
engines = []
|
||||
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return engines
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
|
||||
# Get KnowledgeEngine plugins
|
||||
try:
|
||||
@@ -290,10 +386,12 @@ class KnowledgeService:
|
||||
|
||||
return engines
|
||||
|
||||
async def list_parsers(self, mime_type: str | None = None) -> list[dict]:
|
||||
async def list_parsers(self, context: TenantContext, mime_type: str | None = None) -> list[dict]:
|
||||
"""List available parsers, optionally filtered by MIME type."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return []
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
parsers = await self.ap.plugin_connector.list_parsers()
|
||||
if mime_type:
|
||||
@@ -303,16 +401,24 @@ class KnowledgeService:
|
||||
self.ap.logger.warning(f'Failed to list parsers: {e}')
|
||||
return []
|
||||
|
||||
async def get_engine_creation_schema(self, plugin_id: str) -> dict:
|
||||
async def get_engine_creation_schema(self, context: TenantContext, plugin_id: str) -> dict:
|
||||
"""Get creation settings schema for a specific Knowledge Engine."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return {}
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
return await self.ap.plugin_connector.get_rag_creation_schema(plugin_id)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to get creation schema for {plugin_id}: {e}')
|
||||
return {}
|
||||
|
||||
async def get_engine_retrieval_schema(self, plugin_id: str) -> dict:
|
||||
async def get_engine_retrieval_schema(self, context: TenantContext, plugin_id: str) -> dict:
|
||||
"""Get retrieval settings schema for a specific Knowledge Engine."""
|
||||
require_workspace_uuid(context)
|
||||
if not self.ap.plugin_connector.is_enable_plugin:
|
||||
return {}
|
||||
await self.ap.plugin_connector.require_workspace_context(context)
|
||||
try:
|
||||
return await self.ap.plugin_connector.get_rag_retrieval_schema(plugin_id)
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -11,11 +13,36 @@ import sqlalchemy
|
||||
from ....core import app
|
||||
from ....entity.persistence import bstorage as persistence_bstorage
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
LOG_FILE_PATTERN = re.compile(r'^langbot-(\d{4}-\d{2}-\d{2})\.log(?:\.\d+)?$')
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS = 7
|
||||
DEFAULT_LOG_RETENTION_DAYS = 3
|
||||
DEFAULT_MAX_FILES_PER_RUN = 1000
|
||||
HARD_MAX_FILES_PER_RUN = 10000
|
||||
UPLOAD_OWNER_TYPES = ('upload_image', 'upload_document', 'upload')
|
||||
|
||||
|
||||
def _workspace_scope(method):
|
||||
"""Bind maintenance work to a Workspace without spanning external I/O."""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapped(self, context, *args, **kwargs):
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud maintenance requires an explicit tenant scope')
|
||||
async with tenant_scope(workspace_uuid):
|
||||
return await method(self, context, *args, **kwargs)
|
||||
return await method(self, context, *args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class MaintenanceService:
|
||||
@@ -26,7 +53,22 @@ class MaintenanceService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def cleanup_expired_files(self) -> dict[str, int]:
|
||||
def _max_files_per_run(self) -> int:
|
||||
cleanup_cfg = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('storage', {}).get('cleanup', {})
|
||||
)
|
||||
value = self._positive_int(
|
||||
cleanup_cfg.get('max_files_per_run', DEFAULT_MAX_FILES_PER_RUN),
|
||||
DEFAULT_MAX_FILES_PER_RUN,
|
||||
'storage.cleanup.max_files_per_run',
|
||||
)
|
||||
return min(value, HARD_MAX_FILES_PER_RUN)
|
||||
|
||||
@_workspace_scope
|
||||
async def cleanup_expired_files(self, context: ExecutionContext) -> dict[str, int]:
|
||||
if not isinstance(context, ExecutionContext):
|
||||
raise WorkspaceRequiredError('Storage cleanup requires an ExecutionContext')
|
||||
require_workspace_uuid(context)
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
@@ -40,11 +82,17 @@ class MaintenanceService:
|
||||
)
|
||||
|
||||
return {
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(upload_retention_days),
|
||||
'log_files': self._cleanup_expired_log_files(log_retention_days),
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
|
||||
'log_files': await asyncio.to_thread(
|
||||
self._cleanup_expired_log_files,
|
||||
log_retention_days,
|
||||
)
|
||||
if await self._is_oss_singleton(context)
|
||||
else 0,
|
||||
}
|
||||
|
||||
async def get_storage_analysis(self) -> dict[str, Any]:
|
||||
async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:
|
||||
require_workspace_uuid(context)
|
||||
cleanup_cfg = self.ap.instance_config.data.get('storage', {}).get('cleanup', {})
|
||||
upload_retention_days = self._positive_int(
|
||||
cleanup_cfg.get('uploaded_file_retention_days'),
|
||||
@@ -62,32 +110,34 @@ class MaintenanceService:
|
||||
database_path = (
|
||||
Path(database_cfg.get('sqlite', {}).get('path', 'data/langbot.db')) if database_type == 'sqlite' else None
|
||||
)
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
is_oss_singleton = await self._is_oss_singleton(context)
|
||||
if is_oss_singleton:
|
||||
roots: list[tuple[str, Path | None]] = [
|
||||
('database', database_path),
|
||||
('logs', Path('data/logs')),
|
||||
('storage', Path('data/storage')),
|
||||
('vector_store', Path('data/chroma')),
|
||||
('plugins', Path('data/plugins')),
|
||||
('mcp', Path('data/mcp')),
|
||||
('temp', Path('data/temp')),
|
||||
]
|
||||
else:
|
||||
scoped_storage_path = Path('data/storage') / self.ap.storage_mgr.scoped_prefix(context)
|
||||
roots = [('storage', scoped_storage_path)]
|
||||
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
sections = await asyncio.to_thread(self._collect_sections, roots)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts(context)
|
||||
binary_storage = await self._binary_storage_stats(context)
|
||||
upload_candidates = await self._expired_uploaded_candidates(context, upload_retention_days)
|
||||
log_candidates = (
|
||||
await asyncio.to_thread(
|
||||
self._expired_log_candidates,
|
||||
log_retention_days,
|
||||
)
|
||||
|
||||
monitoring_counts = await self._monitoring_counts()
|
||||
binary_storage = await self._binary_storage_stats()
|
||||
upload_candidates = await self._expired_uploaded_candidates(upload_retention_days)
|
||||
log_candidates = self._expired_log_candidates(log_retention_days)
|
||||
if is_oss_singleton
|
||||
else []
|
||||
)
|
||||
|
||||
return {
|
||||
'generated_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
@@ -105,70 +155,156 @@ class MaintenanceService:
|
||||
'uploaded_files': upload_candidates,
|
||||
'log_files': log_candidates,
|
||||
},
|
||||
'tasks': self.ap.task_mgr.get_stats() if self.ap.task_mgr else {},
|
||||
'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
|
||||
}
|
||||
|
||||
async def _cleanup_expired_uploaded_files(self, retention_days: int) -> int:
|
||||
def _collect_sections(
|
||||
self,
|
||||
roots: list[tuple[str, Path | None]],
|
||||
) -> list[dict[str, Any]]:
|
||||
sections = []
|
||||
for key, path in roots:
|
||||
sections.append(
|
||||
{
|
||||
'key': key,
|
||||
'path': str(path) if path else '',
|
||||
'exists': path.exists() if path else False,
|
||||
'size_bytes': self._path_size(path) if path else 0,
|
||||
'file_count': self._file_count(path) if path else 0,
|
||||
}
|
||||
)
|
||||
return sections
|
||||
|
||||
async def _is_oss_singleton(self, context: TenantContext) -> bool:
|
||||
try:
|
||||
await self.ap.workspace_service.get_local_execution_binding(
|
||||
require_workspace_uuid(context),
|
||||
expected_generation=getattr(context, 'placement_generation', None),
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _cleanup_expired_uploaded_files(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
provider_name = provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
candidates = self._expired_local_upload_candidates(retention_days, include_paths=True)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
candidates = await asyncio.to_thread(
|
||||
self._expired_local_upload_candidates,
|
||||
context,
|
||||
retention_days,
|
||||
True,
|
||||
)
|
||||
return await asyncio.to_thread(
|
||||
self._delete_local_candidates,
|
||||
candidates,
|
||||
)
|
||||
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._cleanup_expired_s3_uploaded_files(retention_days)
|
||||
return await self._cleanup_expired_s3_uploaded_files(context, retention_days)
|
||||
|
||||
return 0
|
||||
|
||||
async def _expired_uploaded_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
async def _expired_uploaded_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider_name = self.ap.storage_mgr.storage_provider.__class__.__name__
|
||||
if provider_name == 'LocalStorageProvider':
|
||||
return self._expired_local_upload_candidates(retention_days)
|
||||
return await asyncio.to_thread(
|
||||
self._expired_local_upload_candidates,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
if provider_name == 'S3StorageProvider':
|
||||
return await self._expired_s3_upload_candidates(retention_days)
|
||||
return await self._expired_s3_upload_candidates(context, retention_days)
|
||||
return []
|
||||
|
||||
async def _cleanup_expired_s3_uploaded_files(self, retention_days: int) -> int:
|
||||
async def _cleanup_expired_s3_uploaded_files(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
candidates = await self._expired_s3_upload_candidates(retention_days)
|
||||
candidates = await self._expired_s3_upload_candidates(context, retention_days)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
await provider.delete(item['key'])
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
async def _expired_s3_upload_candidates(self, retention_days: int) -> list[dict[str, Any]]:
|
||||
async def _expired_s3_upload_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
run_io = getattr(provider, '_run_io', None)
|
||||
if callable(run_io):
|
||||
return await run_io(
|
||||
self._expired_s3_upload_candidates_sync,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
return await asyncio.to_thread(
|
||||
self._expired_s3_upload_candidates_sync,
|
||||
context,
|
||||
retention_days,
|
||||
)
|
||||
|
||||
def _expired_s3_upload_candidates_sync(
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
candidates = []
|
||||
max_candidates = self._max_files_per_run()
|
||||
paginator = provider.s3_client.get_paginator('list_objects_v2')
|
||||
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
seen_prefixes: set[str] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
prefix = self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
if prefix in seen_prefixes:
|
||||
continue
|
||||
seen_prefixes.add(prefix)
|
||||
for page in paginator.paginate(Bucket=provider.bucket_name, Prefix=prefix):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj.get('Key', '')
|
||||
last_modified = obj.get('LastModified')
|
||||
if not self._is_uploaded_file_key(context, key):
|
||||
continue
|
||||
if last_modified and last_modified < cutoff:
|
||||
candidates.append(
|
||||
{
|
||||
'key': key,
|
||||
'size_bytes': obj.get('Size', 0),
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
|
||||
return candidates
|
||||
|
||||
def _delete_local_candidates(self, candidates: list[dict[str, Any]]) -> int:
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
try:
|
||||
os.remove(item['path'])
|
||||
deleted += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
|
||||
return deleted
|
||||
|
||||
def _cleanup_expired_log_files(self, retention_days: int) -> int:
|
||||
deleted = 0
|
||||
for item in self._expired_log_candidates(retention_days, include_paths=True):
|
||||
@@ -182,28 +318,42 @@ class MaintenanceService:
|
||||
return deleted
|
||||
|
||||
def _expired_local_upload_candidates(
|
||||
self, retention_days: int, include_paths: bool = False
|
||||
self,
|
||||
context: TenantContext,
|
||||
retention_days: int,
|
||||
include_paths: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
storage_root = Path('data/storage')
|
||||
if not storage_root.exists():
|
||||
return []
|
||||
|
||||
cutoff = datetime.datetime.now().timestamp() - retention_days * 86400
|
||||
candidates = []
|
||||
for entry in storage_root.iterdir():
|
||||
if not entry.is_file() or not self._is_uploaded_file_key(entry.name):
|
||||
max_candidates = self._max_files_per_run()
|
||||
seen_roots: set[Path] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
scoped_root = storage_root / self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
if scoped_root in seen_roots:
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
seen_roots.add(scoped_root)
|
||||
if not scoped_root.exists():
|
||||
continue
|
||||
item = {
|
||||
'key': entry.name,
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(stat.st_mtime, datetime.timezone.utc).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
for entry in scoped_root.rglob('*'):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
stat = entry.stat()
|
||||
if stat.st_mtime >= cutoff:
|
||||
continue
|
||||
item = {
|
||||
'key': entry.relative_to(storage_root).as_posix(),
|
||||
'size_bytes': stat.st_size,
|
||||
'modified_at': datetime.datetime.fromtimestamp(
|
||||
stat.st_mtime,
|
||||
datetime.timezone.utc,
|
||||
).isoformat(),
|
||||
}
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
return candidates
|
||||
|
||||
def _expired_log_candidates(self, retention_days: int, include_paths: bool = False) -> list[dict[str, Any]]:
|
||||
@@ -236,33 +386,51 @@ class MaintenanceService:
|
||||
candidates.append(item)
|
||||
return candidates
|
||||
|
||||
def _is_uploaded_file_key(self, key: str) -> bool:
|
||||
return '/' not in key and not key.startswith('plugin_config_')
|
||||
def _is_uploaded_file_key(self, context: TenantContext, key: str) -> bool:
|
||||
return any(
|
||||
key.startswith(self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type))
|
||||
and self.ap.storage_mgr.is_scoped_object_key(key, expected_owner_type=owner_type)
|
||||
for owner_type in UPLOAD_OWNER_TYPES
|
||||
)
|
||||
|
||||
async def _monitoring_counts(self) -> dict[str, int]:
|
||||
async def _monitoring_counts(self, context: TenantContext) -> dict[str, int]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
tables = {
|
||||
'messages': persistence_monitoring.MonitoringMessage.id,
|
||||
'llm_calls': persistence_monitoring.MonitoringLLMCall.id,
|
||||
'tool_calls': persistence_monitoring.MonitoringToolCall.id,
|
||||
'embedding_calls': persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
'errors': persistence_monitoring.MonitoringError.id,
|
||||
'sessions': persistence_monitoring.MonitoringSession.session_id,
|
||||
'feedback': persistence_monitoring.MonitoringFeedback.id,
|
||||
'messages': (persistence_monitoring.MonitoringMessage, persistence_monitoring.MonitoringMessage.id),
|
||||
'llm_calls': (persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.id),
|
||||
'tool_calls': (persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.id),
|
||||
'embedding_calls': (
|
||||
persistence_monitoring.MonitoringEmbeddingCall,
|
||||
persistence_monitoring.MonitoringEmbeddingCall.id,
|
||||
),
|
||||
'errors': (persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.id),
|
||||
'sessions': (
|
||||
persistence_monitoring.MonitoringSession,
|
||||
persistence_monitoring.MonitoringSession.session_id,
|
||||
),
|
||||
'feedback': (persistence_monitoring.MonitoringFeedback, persistence_monitoring.MonitoringFeedback.id),
|
||||
}
|
||||
counts: dict[str, int] = {}
|
||||
for key, column in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.func.count(column)))
|
||||
for key, (model, column) in tables.items():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(column)).where(model.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
counts[key] = result.scalar() or 0
|
||||
return counts
|
||||
|
||||
async def _binary_storage_stats(self) -> dict[str, Any]:
|
||||
async def _binary_storage_stats(self, context: TenantContext) -> dict[str, Any]:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key))
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_bstorage.BinaryStorage.unique_key)).where(
|
||||
persistence_bstorage.BinaryStorage.workspace_uuid == workspace_uuid
|
||||
)
|
||||
)
|
||||
size_bytes = None
|
||||
try:
|
||||
size_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value)))
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.sum(sqlalchemy.func.length(persistence_bstorage.BinaryStorage.value))
|
||||
).where(persistence_bstorage.BinaryStorage.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
size_bytes = size_result.scalar() or 0
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,198 +1,451 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
import copy
|
||||
import re
|
||||
import uuid
|
||||
import asyncio
|
||||
|
||||
from ....core import app
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app, taskmgr
|
||||
from ....core.task_boundary import create_detached_task
|
||||
from ....entity.persistence import mcp as persistence_mcp
|
||||
from ....core import taskmgr
|
||||
from ....provider.tools.loaders.mcp import RuntimeMCPSession, MCPSessionStatus
|
||||
from ....entity.persistence import plugin as persistence_plugin
|
||||
from ....provider.tools.loaders.mcp import MCPSessionStatus, RuntimeMCPSession
|
||||
from ....provider.tools.loaders.mcp_policy import require_stdio_mcp_enabled
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from ..context import ExecutionContext
|
||||
from .secrets import is_url_key, redact_url_secrets, restore_url_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
_SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
_SENSITIVE_CONFIG_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'apikey',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_CONFIG_TOKENS = frozenset(
|
||||
{
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_config_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def _is_sensitive_config_key(key: object) -> bool:
|
||||
normalized = _normalize_config_key(key)
|
||||
if normalized in _SENSITIVE_CONFIG_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_CONFIG_TOKENS:
|
||||
return True
|
||||
return 'key' in tokens and bool(tokens & _SENSITIVE_KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def _mask_secret_structure(value):
|
||||
if isinstance(value, dict):
|
||||
return {key: _mask_secret_structure(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_mask_secret_structure(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_mask_secret_structure(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return _SECRET_MASK
|
||||
|
||||
|
||||
def redact_mcp_secrets(value):
|
||||
"""Return a recursively redacted copy of MCP configuration data."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
_mask_secret_structure(item)
|
||||
if _is_sensitive_config_key(key)
|
||||
else redact_url_secrets(item)
|
||||
if is_url_key(key)
|
||||
else redact_mcp_secrets(item)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_mcp_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_mcp_secrets(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def restore_mcp_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from the current MCP config before a write."""
|
||||
|
||||
if sensitive and value == _SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked MCP secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: (
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
)
|
||||
if not sensitive and not _is_sensitive_config_key(key) and is_url_key(key)
|
||||
else restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or _is_sensitive_config_key(key),
|
||||
)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_mcp_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class MCPService:
|
||||
"""Workspace-scoped MCP configuration and runtime facade."""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_runtime_info(self, server_name: str) -> dict | None:
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
if session:
|
||||
return session.get_runtime_info_dict()
|
||||
return None
|
||||
async def _execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip()
|
||||
generation = getattr(context, 'placement_generation', None)
|
||||
if not instance_uuid or not isinstance(generation, int) or isinstance(generation, bool) or generation <= 0:
|
||||
raise ValueError('MCP operations require an explicit fenced execution context')
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=generation,
|
||||
)
|
||||
if binding.instance_uuid != instance_uuid:
|
||||
raise ValueError('MCP execution context belongs to another LangBot instance')
|
||||
return ExecutionContext(
|
||||
instance_uuid=instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
placement_generation=generation,
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
)
|
||||
|
||||
async def get_mcp_servers(self, contain_runtime_info: bool = False) -> list[dict]:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
|
||||
async def get_runtime_info(self, context: TenantContext, server_name: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
return session.get_runtime_info_dict() if session else None
|
||||
|
||||
servers = result.all()
|
||||
async def get_mcp_servers(self, context: TenantContext, contain_runtime_info: bool = False) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_mcp.MCPServer), persistence_mcp.MCPServer, context)
|
||||
)
|
||||
serialized_servers = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server) for server in servers
|
||||
redact_mcp_secrets(self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server))
|
||||
for server in result.all()
|
||||
]
|
||||
if contain_runtime_info:
|
||||
for server in serialized_servers:
|
||||
runtime_info = await self.get_runtime_info(server['name'])
|
||||
|
||||
server['runtime_info'] = runtime_info if runtime_info else None
|
||||
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server['name'])
|
||||
server['runtime_info'] = session.get_runtime_info_dict() if session else None
|
||||
return serialized_servers
|
||||
|
||||
async def create_mcp_server(self, server_data: dict) -> str:
|
||||
# Check limitation (extensions = MCP servers + plugins)
|
||||
async def create_mcp_server(self, context: TenantContext, server_data: dict) -> str:
|
||||
execution_context = await self._execution_context(context)
|
||||
workspace_uuid = execution_context.workspace_uuid
|
||||
|
||||
# This gate is independent of Box availability. Cloud v2 disables
|
||||
# stdio MCP even though Box Runtime itself remains available.
|
||||
require_stdio_mcp_enabled(self.ap, server_data)
|
||||
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_extensions = limitation.get('max_extensions', -1)
|
||||
if max_extensions >= 0:
|
||||
existing_mcp_servers = await self.get_mcp_servers()
|
||||
plugins = await self.ap.plugin_connector.list_plugins()
|
||||
total_extensions = len(existing_mcp_servers) + len(plugins)
|
||||
if total_extensions >= max_extensions:
|
||||
mcp_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count(persistence_mcp.MCPServer.uuid)).where(
|
||||
persistence_mcp.MCPServer.workspace_uuid == workspace_uuid
|
||||
)
|
||||
)
|
||||
plugin_count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
if (mcp_count_result.scalar() or 0) + (plugin_count_result.scalar() or 0) >= max_extensions:
|
||||
raise ValueError(f'Maximum number of extensions ({max_extensions}) reached')
|
||||
|
||||
server_name = str(server_data.get('name') or '').strip()
|
||||
payload = dict(server_data)
|
||||
payload.pop('workspace_uuid', None)
|
||||
server_name = str(payload.get('name') or '').strip()
|
||||
if not server_name:
|
||||
raise ValueError('MCP server name is required')
|
||||
server_data['name'] = server_name
|
||||
payload['name'] = server_name
|
||||
payload['workspace_uuid'] = workspace_uuid
|
||||
payload['uuid'] = str(uuid.uuid4())
|
||||
|
||||
existing_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name)
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(
|
||||
persistence_mcp.MCPServer.workspace_uuid == workspace_uuid,
|
||||
persistence_mcp.MCPServer.name == server_name,
|
||||
)
|
||||
)
|
||||
if existing_result.first() is not None:
|
||||
raise ValueError(f'MCP server already exists: {server_name}')
|
||||
|
||||
server_data['uuid'] = str(uuid.uuid4())
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(server_data))
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_data['uuid'])
|
||||
)
|
||||
server_entity = result.first()
|
||||
if server_entity:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server_entity)
|
||||
if self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(payload))
|
||||
created = await self._get_mcp_server_by_uuid_raw(execution_context, payload['uuid'])
|
||||
if created and self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = create_detached_task(
|
||||
self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
)
|
||||
tracker = getattr(
|
||||
self.ap.tool_mgr.mcp_tool_loader,
|
||||
'track_hosted_task',
|
||||
None,
|
||||
)
|
||||
if callable(tracker):
|
||||
tracker(task, execution_context)
|
||||
else:
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
return payload['uuid']
|
||||
|
||||
return server_data['uuid']
|
||||
async def get_mcp_server_by_uuid(self, context: TenantContext, server_uuid: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server_data = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
return redact_mcp_secrets(server_data) if server_data is not None else None
|
||||
|
||||
async def get_mcp_server_by_name(self, server_name: str) -> dict | None:
|
||||
async def _get_mcp_server_by_uuid_raw(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
server_uuid: str,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server) if server else None
|
||||
|
||||
async def get_mcp_server_by_name(self, context: TenantContext, server_name: str) -> dict | None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server_data = await self._get_mcp_server_by_name_raw(execution_context, server_name)
|
||||
if server_data is None:
|
||||
return None
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
response_data = {
|
||||
**server_data,
|
||||
'runtime_info': session.get_runtime_info_dict() if session else None,
|
||||
}
|
||||
return redact_mcp_secrets(response_data)
|
||||
|
||||
async def _get_mcp_server_by_name_raw(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
server_name: str,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.name == server_name),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
if server is None:
|
||||
return None
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
|
||||
runtime_info = await self.get_runtime_info(server.name)
|
||||
server_data = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
server_data['runtime_info'] = runtime_info if runtime_info else None
|
||||
return server_data
|
||||
async def update_mcp_server(self, context: TenantContext, server_uuid: str, server_data: dict) -> None:
|
||||
execution_context = await self._execution_context(context)
|
||||
old_server = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if old_server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
|
||||
payload = dict(server_data)
|
||||
payload.pop('uuid', None)
|
||||
payload.pop('workspace_uuid', None)
|
||||
payload = restore_mcp_secret_placeholders(payload, old_server)
|
||||
if 'name' in payload:
|
||||
payload['name'] = str(payload['name'] or '').strip()
|
||||
if not payload['name']:
|
||||
raise ValueError('MCP server name is required')
|
||||
duplicate = await self._get_mcp_server_by_name_raw(execution_context, payload['name'])
|
||||
if duplicate is not None and duplicate['uuid'] != server_uuid:
|
||||
raise ValueError(f'MCP server already exists: {payload["name"]}')
|
||||
|
||||
effective_server = {**old_server, **payload}
|
||||
# Existing disabled rows remain readable/deletable. Switching away
|
||||
# from stdio or explicitly disabling one is also allowed, but an
|
||||
# update may never leave a disabled stdio server enabled.
|
||||
if bool(effective_server.get('enable', True)):
|
||||
require_stdio_mcp_enabled(self.ap, effective_server)
|
||||
|
||||
async def update_mcp_server(self, server_uuid: str, server_data: dict) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
.values(payload),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
old_server = result.first()
|
||||
old_server_name = old_server.name if old_server else None
|
||||
old_enable = old_server.enable if old_server else False
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
.values(server_data)
|
||||
)
|
||||
loader = self.ap.tool_mgr.mcp_tool_loader
|
||||
if loader is None:
|
||||
return
|
||||
old_name = old_server['name']
|
||||
old_enable = bool(old_server['enable'])
|
||||
updated = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if updated is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
new_enable = bool(updated['enable'])
|
||||
if old_enable and loader.has_session(execution_context, old_name):
|
||||
await loader.remove_mcp_server(execution_context, old_name)
|
||||
if new_enable:
|
||||
task = create_detached_task(
|
||||
loader.host_mcp_server(execution_context, updated),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
)
|
||||
tracker = getattr(loader, 'track_hosted_task', None)
|
||||
if callable(tracker):
|
||||
tracker(task, execution_context)
|
||||
else:
|
||||
loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
if self.ap.tool_mgr.mcp_tool_loader:
|
||||
new_enable = server_data.get('enable', False)
|
||||
|
||||
need_remove = old_server_name and old_server_name in self.ap.tool_mgr.mcp_tool_loader.sessions
|
||||
|
||||
if old_enable and not new_enable:
|
||||
if need_remove:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(old_server_name)
|
||||
|
||||
elif not old_enable and new_enable:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
updated_server = result.first()
|
||||
if updated_server:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, updated_server)
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
elif old_enable and new_enable:
|
||||
if need_remove:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(old_server_name)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
updated_server = result.first()
|
||||
if updated_server:
|
||||
server_config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, updated_server)
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(server_config))
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
async def delete_mcp_server(self, server_uuid: str) -> None:
|
||||
async def delete_mcp_server(self, context: TenantContext, server_uuid: str) -> None:
|
||||
execution_context = await self._execution_context(context)
|
||||
server = await self._get_mcp_server_by_uuid_raw(execution_context, server_uuid)
|
||||
if server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid),
|
||||
persistence_mcp.MCPServer,
|
||||
execution_context,
|
||||
)
|
||||
)
|
||||
server = result.first()
|
||||
server_name = server.name if server else None
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
loader = self.ap.tool_mgr.mcp_tool_loader
|
||||
if loader and loader.has_session(execution_context, server['name']):
|
||||
await loader.remove_mcp_server(execution_context, server['name'])
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_mcp.MCPServer).where(persistence_mcp.MCPServer.uuid == server_uuid)
|
||||
)
|
||||
async def _require_server(self, context: TenantContext, server_name: str) -> tuple[ExecutionContext, dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
server = await self._get_mcp_server_by_name_raw(execution_context, server_name)
|
||||
if server is None:
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
return execution_context, server
|
||||
|
||||
if server_name and self.ap.tool_mgr.mcp_tool_loader:
|
||||
if server_name in self.ap.tool_mgr.mcp_tool_loader.sessions:
|
||||
await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(server_name)
|
||||
async def get_mcp_server_resources(self, context: TenantContext, server_name: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resources(execution_context, server_name)
|
||||
|
||||
async def get_mcp_server_resources(self, server_name: str) -> list[dict]:
|
||||
"""Get resources from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resources(server_name)
|
||||
|
||||
async def get_mcp_server_resource_templates(self, server_name: str) -> list[dict]:
|
||||
"""Get resource templates from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(server_name)
|
||||
async def get_mcp_server_resource_templates(self, context: TenantContext, server_name: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(execution_context, server_name)
|
||||
|
||||
async def read_mcp_server_resource_envelope(
|
||||
self,
|
||||
context: TenantContext,
|
||||
server_name: str,
|
||||
uri: str,
|
||||
*,
|
||||
max_bytes: int | None = None,
|
||||
include_blob: bool = False,
|
||||
) -> dict:
|
||||
"""Read a resource from a specific MCP server with metadata."""
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
kwargs = {'include_blob': include_blob, 'source': 'ui_preview'}
|
||||
if max_bytes is not None:
|
||||
kwargs['max_bytes'] = max_bytes
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(server_name, uri, **kwargs)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(
|
||||
execution_context,
|
||||
server_name,
|
||||
uri,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def read_mcp_server_resource(self, server_name: str, uri: str) -> list[dict]:
|
||||
"""Read a resource from a specific MCP server."""
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource(server_name, uri)
|
||||
|
||||
async def test_mcp_server(self, server_name: str, server_data: dict) -> int:
|
||||
"""测试 MCP 服务器连接并返回任务 ID"""
|
||||
async def read_mcp_server_resource(self, context: TenantContext, server_name: str, uri: str) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
return await self.ap.tool_mgr.mcp_tool_loader.read_resource(execution_context, server_name, uri)
|
||||
|
||||
async def test_mcp_server(self, context: TenantContext, server_name: str, server_data: dict) -> int:
|
||||
execution_context = await self._execution_context(context)
|
||||
runtime_mcp_session: RuntimeMCPSession | None = None
|
||||
|
||||
test_session: RuntimeMCPSession | None = None
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
|
||||
if server_name != '_':
|
||||
runtime_mcp_session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
_, persisted_server = await self._require_server(execution_context, server_name)
|
||||
require_stdio_mcp_enabled(self.ap, persisted_server)
|
||||
runtime_mcp_session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
if runtime_mcp_session is None:
|
||||
raise ValueError(f'Server not found: {server_name}')
|
||||
|
||||
raise WorkspaceNotFoundError('MCP server not found')
|
||||
persisted_session = runtime_mcp_session
|
||||
|
||||
async def _refresh_and_report() -> None:
|
||||
# Testing a persisted server should REUSE its live shared-session
|
||||
# process, not rebuild it. Try a lightweight refresh (a real
|
||||
# list_tools probe over the existing connection) first; only fall
|
||||
# back to a full start() when the session has no live connection
|
||||
# to probe (never connected, or the process is actually gone).
|
||||
needs_start = persisted_session.status == MCPSessionStatus.ERROR or persisted_session.session is None
|
||||
if needs_start:
|
||||
await persisted_session.start()
|
||||
@@ -200,30 +453,24 @@ class MCPService:
|
||||
try:
|
||||
await persisted_session.refresh()
|
||||
except Exception:
|
||||
# The live connection was stale/dropped: reconnect once
|
||||
# (reusing the live managed process where possible) and
|
||||
# re-probe, instead of reporting a false failure.
|
||||
await persisted_session.start()
|
||||
# Surface the discovered tools so the config page can render them
|
||||
# even for an already-hosted server.
|
||||
ctx.metadata['runtime_info'] = persisted_session.get_runtime_info_dict()
|
||||
|
||||
coroutine = _refresh_and_report()
|
||||
else:
|
||||
runtime_mcp_session = await self.ap.tool_mgr.mcp_tool_loader.load_mcp_server(server_config=server_data)
|
||||
|
||||
# A transient test owns an isolated Box session. Always tear it down
|
||||
# after the test completes (success or failure) so it does not leak.
|
||||
payload = dict(server_data)
|
||||
payload.pop('workspace_uuid', None)
|
||||
payload['workspace_uuid'] = execution_context.workspace_uuid
|
||||
require_stdio_mcp_enabled(self.ap, payload)
|
||||
runtime_mcp_session = await self.ap.tool_mgr.mcp_tool_loader.load_mcp_server(
|
||||
execution_context,
|
||||
payload,
|
||||
)
|
||||
test_session = runtime_mcp_session
|
||||
|
||||
async def _run_and_cleanup() -> None:
|
||||
try:
|
||||
await test_session.start()
|
||||
# Capture the runtime info (status + discovered tools) BEFORE
|
||||
# shutting the transient session down. The create/edit config
|
||||
# page has no persisted server to reload from, so without this
|
||||
# a successful test could only show "no tools found". The
|
||||
# frontend reads ctx.metadata.runtime_info to render the tools.
|
||||
ctx.metadata['runtime_info'] = test_session.get_runtime_info_dict()
|
||||
finally:
|
||||
try:
|
||||
@@ -236,27 +483,41 @@ class MCPService:
|
||||
|
||||
coroutine = _run_and_cleanup()
|
||||
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
coroutine,
|
||||
kind='mcp-operation',
|
||||
name=f'mcp-test-{server_name}',
|
||||
label=f'Testing MCP server {server_name}',
|
||||
context=ctx,
|
||||
)
|
||||
try:
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
coroutine,
|
||||
kind='mcp-operation',
|
||||
name=f'mcp-test-{execution_context.workspace_uuid}-{server_name}',
|
||||
label=f'Testing MCP server {server_name}',
|
||||
context=ctx,
|
||||
instance_uuid=execution_context.instance_uuid,
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
placement_generation=execution_context.placement_generation,
|
||||
)
|
||||
except taskmgr.TaskCapacityError:
|
||||
if test_session is not None:
|
||||
try:
|
||||
await test_session.shutdown()
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(
|
||||
f'Failed to tear down rejected transient MCP test session '
|
||||
f'{test_session.server_name}: {type(exc).__name__}: {exc}'
|
||||
)
|
||||
raise
|
||||
return wrapper.id
|
||||
|
||||
async def get_mcp_server_logs(self, server_name: str, limit: int = 200, level: str | None = None) -> list[dict]:
|
||||
"""Get recent log lines captured from the MCP server's stderr."""
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(server_name)
|
||||
async def get_mcp_server_logs(
|
||||
self,
|
||||
context: TenantContext,
|
||||
server_name: str,
|
||||
limit: int = 200,
|
||||
level: str | None = None,
|
||||
) -> list[dict]:
|
||||
execution_context, _ = await self._require_server(context, server_name)
|
||||
session = self.ap.tool_mgr.mcp_tool_loader.get_session(execution_context, server_name)
|
||||
if not session:
|
||||
return []
|
||||
|
||||
# Get logs from the session's buffer
|
||||
logs = list(session._log_buffer)
|
||||
|
||||
# Filter by level if specified
|
||||
if level:
|
||||
logs = [log for log in logs if log.get('level') == level]
|
||||
|
||||
# Return the most recent 'limit' logs
|
||||
return logs[-limit:]
|
||||
|
||||
@@ -9,6 +9,9 @@ from ....core import app
|
||||
from ....entity.persistence import model as persistence_model
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....provider.modelmgr import requester as model_requester
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import mask_secret_value, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
def _parse_provider_api_keys(provider_dict: dict) -> dict:
|
||||
@@ -34,7 +37,29 @@ def _runtime_model_data(model_uuid: str, model_data: dict) -> dict:
|
||||
return {**model_data, 'uuid': model_uuid}
|
||||
|
||||
|
||||
async def _validate_provider_supports(ap: app.Application, provider_uuid: str, model_type: str) -> None:
|
||||
def _redact_model_secrets(model_data: dict) -> dict:
|
||||
"""Return a copy with model args and embedded provider credentials masked."""
|
||||
|
||||
redacted = model_data.copy()
|
||||
if 'extra_args' in redacted:
|
||||
redacted['extra_args'] = redact_secrets(redacted['extra_args'])
|
||||
if isinstance(redacted.get('provider'), dict):
|
||||
provider = redacted['provider'].copy()
|
||||
# ModelProvider never contains another provider. Dropping this key also
|
||||
# makes the serializer robust to a reused/self-referential test double.
|
||||
provider.pop('provider', None)
|
||||
if 'api_keys' in provider:
|
||||
provider['api_keys'] = mask_secret_value(provider['api_keys'])
|
||||
redacted['provider'] = provider
|
||||
return redacted
|
||||
|
||||
|
||||
async def _validate_provider_supports(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
model_type: str,
|
||||
) -> None:
|
||||
"""Validate that the provider's requester declares support for ``model_type``.
|
||||
|
||||
``model_type`` is one of the manifest ``support_type`` values:
|
||||
@@ -47,11 +72,12 @@ async def _validate_provider_supports(ap: app.Application, provider_uuid: str, m
|
||||
if model_mgr is None:
|
||||
return
|
||||
|
||||
provider_dict = getattr(model_mgr, 'provider_dict', None)
|
||||
if not provider_dict:
|
||||
get_provider = getattr(model_mgr, 'get_provider_by_uuid', None)
|
||||
if not callable(get_provider):
|
||||
return
|
||||
runtime_provider = provider_dict.get(provider_uuid)
|
||||
if runtime_provider is None:
|
||||
try:
|
||||
runtime_provider = await get_provider(context, provider_uuid)
|
||||
except ValueError:
|
||||
return
|
||||
|
||||
requester_name = getattr(getattr(runtime_provider, 'provider_entity', None), 'requester', None)
|
||||
@@ -74,20 +100,48 @@ async def _validate_provider_supports(ap: app.Application, provider_uuid: str, m
|
||||
raise ValueError(f'Provider requester "{requester_name}" does not support {model_type} models')
|
||||
|
||||
|
||||
async def _require_workspace_provider(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
) -> dict:
|
||||
"""Require the referenced provider to belong to the active Workspace."""
|
||||
|
||||
provider = await ap.provider_service.get_provider(context, provider_uuid)
|
||||
if provider is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
return provider
|
||||
|
||||
|
||||
async def _require_runtime_provider(
|
||||
ap: app.Application,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
) -> model_requester.RuntimeProvider:
|
||||
try:
|
||||
return await ap.model_mgr.get_provider_by_uuid(context, provider_uuid)
|
||||
except ValueError as exc:
|
||||
raise Exception('provider not found') from exc
|
||||
|
||||
|
||||
class LLMModelsService:
|
||||
ap: app.Application
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_llm_models(self, include_secret: bool = True) -> list[dict]:
|
||||
async def get_llm_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all LLM models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.LLMModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_model.LLMModel), persistence_model.LLMModel, context)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
# Get all providers for lookup
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -98,29 +152,50 @@ class LLMModelsService:
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
if not include_secret:
|
||||
provider_dict['api_keys'] = ['***'] * len(provider_dict.get('api_keys', []))
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_llm_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_llm_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get LLM models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.LLMModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.LLMModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_llm_model(
|
||||
self, model_data: dict, preserve_uuid: bool = False, auto_set_to_default_pipeline: bool = True
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_data: dict,
|
||||
preserve_uuid: bool = False,
|
||||
auto_set_to_default_pipeline: bool = True,
|
||||
) -> str:
|
||||
"""Create a new LLM model"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = workspace_uuid
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
# Handle provider creation if needed
|
||||
if 'provider' in model_data:
|
||||
@@ -130,31 +205,35 @@ class LLMModelsService:
|
||||
else:
|
||||
# Create new provider
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'llm')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'llm')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.LLMModel).values(**model_data))
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
context,
|
||||
persistence_model.LLMModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
await self.ap.model_mgr.cache_llm_model(context, runtime_llm_model)
|
||||
|
||||
if auto_set_to_default_pipeline:
|
||||
# set the default pipeline model to this model
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
pipeline = result.first()
|
||||
@@ -167,14 +246,23 @@ class LLMModelsService:
|
||||
'fallbacks': [],
|
||||
}
|
||||
pipeline_data = {'config': pipeline_config}
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline.uuid, pipeline_data)
|
||||
await self.ap.pipeline_service.update_pipeline(context, pipeline.uuid, pipeline_data)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_llm_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_llm_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single LLM model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
@@ -184,21 +272,38 @@ class LLMModelsService:
|
||||
|
||||
# Get provider
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_llm_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_llm_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing LLM model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_llm_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
# Handle provider update if needed
|
||||
if 'provider' in model_data:
|
||||
@@ -207,50 +312,71 @@ class LLMModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'llm')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
await self.ap.model_mgr.remove_llm_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_llm_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_llm_model = await self.ap.model_mgr.load_llm_model_with_provider(
|
||||
persistence_model.LLMModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
context,
|
||||
persistence_model.LLMModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.llm_models.append(runtime_llm_model)
|
||||
await self.ap.model_mgr.cache_llm_model(context, runtime_llm_model)
|
||||
|
||||
async def delete_llm_model(self, model_uuid: str) -> None:
|
||||
async def delete_llm_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete an LLM model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.LLMModel).where(persistence_model.LLMModel.uuid == model_uuid),
|
||||
persistence_model.LLMModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_llm_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_llm_model(context, model_uuid)
|
||||
|
||||
async def test_llm_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def test_llm_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test an LLM model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_llm_model: model_requester.RuntimeLLMModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.llm_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_llm_model = model
|
||||
break
|
||||
if runtime_llm_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_llm_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_llm_model = await self.ap.model_mgr.get_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_llm_model = await self.ap.model_mgr.init_temporary_runtime_llm_model(model_data)
|
||||
runtime_llm_model = await self.ap.model_mgr.init_temporary_runtime_llm_model(context, model_data)
|
||||
|
||||
extra_args = model_data.get('extra_args', {})
|
||||
await runtime_llm_model.provider.invoke_llm(
|
||||
@@ -259,6 +385,7 @@ class LLMModelsService:
|
||||
messages=[provider_message.Message(role='user', content='Hello, world! Please just reply a "Hello".')],
|
||||
funcs=[],
|
||||
extra_args=extra_args,
|
||||
execution_context=runtime_llm_model.execution_context,
|
||||
)
|
||||
|
||||
|
||||
@@ -268,13 +395,19 @@ class EmbeddingModelsService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_embedding_models(self) -> list[dict]:
|
||||
async def get_embedding_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all embedding models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.EmbeddingModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel), persistence_model.EmbeddingModel, context
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -284,25 +417,46 @@ class EmbeddingModelsService:
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_embedding_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_embedding_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get embedding models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_embedding_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
async def create_embedding_model(
|
||||
self, context: TenantContext, model_data: dict, preserve_uuid: bool = False
|
||||
) -> str:
|
||||
"""Create a new embedding model"""
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -310,35 +464,44 @@ class EmbeddingModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'text-embedding')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'text-embedding')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.EmbeddingModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
context,
|
||||
persistence_model.EmbeddingModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
await self.ap.model_mgr.cache_embedding_model(context, runtime_embedding_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_embedding_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_embedding_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single embedding model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
@@ -348,21 +511,38 @@ class EmbeddingModelsService:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.EmbeddingModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_embedding_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_embedding_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing embedding model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_embedding_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -370,57 +550,82 @@ class EmbeddingModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'text-embedding')
|
||||
|
||||
await self.ap.model_mgr.remove_embedding_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
persistence_model.EmbeddingModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.embedding_models.append(runtime_embedding_model)
|
||||
|
||||
async def delete_embedding_model(self, model_uuid: str) -> None:
|
||||
"""Delete an embedding model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_embedding_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
async def test_embedding_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
await self.ap.model_mgr.remove_embedding_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_embedding_model = await self.ap.model_mgr.load_embedding_model_with_provider(
|
||||
context,
|
||||
persistence_model.EmbeddingModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
await self.ap.model_mgr.cache_embedding_model(context, runtime_embedding_model)
|
||||
|
||||
async def delete_embedding_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete an embedding model"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_embedding_model(context, model_uuid)
|
||||
|
||||
async def test_embedding_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test an embedding model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_embedding_model: model_requester.RuntimeEmbeddingModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.embedding_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_embedding_model = model
|
||||
break
|
||||
if runtime_embedding_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_embedding_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_embedding_model = await self.ap.model_mgr.get_embedding_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_embedding_model = await self.ap.model_mgr.init_temporary_runtime_embedding_model(model_data)
|
||||
runtime_embedding_model = await self.ap.model_mgr.init_temporary_runtime_embedding_model(
|
||||
context,
|
||||
model_data,
|
||||
)
|
||||
|
||||
await runtime_embedding_model.provider.invoke_embedding(
|
||||
model=runtime_embedding_model,
|
||||
input_text=['Hello, world!'],
|
||||
extra_args={},
|
||||
execution_context=runtime_embedding_model.execution_context,
|
||||
)
|
||||
|
||||
|
||||
@@ -430,13 +635,17 @@ class RerankModelsService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_rerank_models(self) -> list[dict]:
|
||||
async def get_rerank_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all rerank models with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.RerankModel))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(sqlalchemy.select(persistence_model.RerankModel), persistence_model.RerankModel, context)
|
||||
)
|
||||
models = result.all()
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context
|
||||
)
|
||||
)
|
||||
providers = {p.uuid: p for p in providers_result.all()}
|
||||
|
||||
@@ -446,25 +655,44 @@ class RerankModelsService:
|
||||
provider = providers.get(model.provider_uuid)
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
models_list.append(model_dict)
|
||||
|
||||
return models_list
|
||||
|
||||
async def get_rerank_models_by_provider(self, provider_uuid: str) -> list[dict]:
|
||||
async def get_rerank_models_by_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
"""Get rerank models by provider UUID"""
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
models = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, m) for m in models]
|
||||
return serialized if include_secret else [_redact_model_secrets(model) for model in serialized]
|
||||
|
||||
async def create_rerank_model(self, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
async def create_rerank_model(self, context: TenantContext, model_data: dict, preserve_uuid: bool = False) -> str:
|
||||
"""Create a new rerank model"""
|
||||
model_data = model_data.copy()
|
||||
if not preserve_uuid:
|
||||
model_data['uuid'] = str(uuid.uuid4())
|
||||
model_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args'])
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -472,34 +700,45 @@ class RerankModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await _validate_provider_supports(self.ap, model_data['provider_uuid'], 'rerank')
|
||||
await _require_workspace_provider(self.ap, context, model_data['provider_uuid'])
|
||||
await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'rerank')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.RerankModel).values(**model_data)
|
||||
)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid'])
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
context,
|
||||
persistence_model.RerankModel(**model_data),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
await self.ap.model_mgr.cache_rerank_model(context, runtime_rerank_model)
|
||||
|
||||
return model_data['uuid']
|
||||
|
||||
async def get_rerank_model(self, model_uuid: str) -> dict | None:
|
||||
async def get_rerank_model(
|
||||
self,
|
||||
context: TenantContext,
|
||||
model_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single rerank model with provider info"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
model = result.first()
|
||||
if model is None:
|
||||
@@ -508,21 +747,38 @@ class RerankModelsService:
|
||||
model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.RerankModel, model)
|
||||
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == model.provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = provider_result.first()
|
||||
if provider:
|
||||
provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider)
|
||||
model_dict['provider'] = _parse_provider_api_keys(provider_dict)
|
||||
provider_dict = _parse_provider_api_keys(provider_dict)
|
||||
model_dict['provider'] = provider_dict
|
||||
|
||||
if not include_secret:
|
||||
model_dict = _redact_model_secrets(model_dict)
|
||||
|
||||
return model_dict
|
||||
|
||||
async def update_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def update_rerank_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Update an existing rerank model"""
|
||||
if 'uuid' in model_data:
|
||||
del model_data['uuid']
|
||||
existing_model = await self.get_rerank_model(context, model_uuid, include_secret=True)
|
||||
if existing_model is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
model_data = model_data.copy()
|
||||
model_data.pop('uuid', None)
|
||||
model_data.pop('workspace_uuid', None)
|
||||
if 'extra_args' in model_data:
|
||||
model_data['extra_args'] = restore_secret_placeholders(
|
||||
model_data['extra_args'],
|
||||
existing_model.get('extra_args', {}),
|
||||
)
|
||||
|
||||
if 'provider' in model_data:
|
||||
provider_data = model_data.pop('provider')
|
||||
@@ -530,50 +786,76 @@ class RerankModelsService:
|
||||
model_data['provider_uuid'] = provider_data['uuid']
|
||||
else:
|
||||
provider_uuid = await self.ap.provider_service.find_or_create_provider(
|
||||
context,
|
||||
requester=provider_data.get('requester', ''),
|
||||
base_url=provider_data.get('base_url', ''),
|
||||
api_keys=provider_data.get('api_keys', []),
|
||||
)
|
||||
model_data['provider_uuid'] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data)
|
||||
provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid'])
|
||||
await _require_workspace_provider(self.ap, context, provider_uuid)
|
||||
await _validate_provider_supports(self.ap, context, provider_uuid, 'rerank')
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
.values(**model_data),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
|
||||
runtime_provider = self.ap.model_mgr.provider_dict.get(model_data['provider_uuid'])
|
||||
if runtime_provider is None:
|
||||
raise Exception('provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_rerank_model(context, model_uuid)
|
||||
runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid)
|
||||
runtime_rerank_model = await self.ap.model_mgr.load_rerank_model_with_provider(
|
||||
persistence_model.RerankModel(**_runtime_model_data(model_uuid, model_data)),
|
||||
context,
|
||||
persistence_model.RerankModel(
|
||||
**_runtime_model_data(
|
||||
model_uuid,
|
||||
{
|
||||
key: value
|
||||
for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items()
|
||||
if key not in {'provider', 'created_at', 'updated_at'}
|
||||
},
|
||||
)
|
||||
),
|
||||
runtime_provider,
|
||||
)
|
||||
self.ap.model_mgr.rerank_models.append(runtime_rerank_model)
|
||||
await self.ap.model_mgr.cache_rerank_model(context, runtime_rerank_model)
|
||||
|
||||
async def delete_rerank_model(self, model_uuid: str) -> None:
|
||||
async def delete_rerank_model(self, context: TenantContext, model_uuid: str) -> None:
|
||||
"""Delete a rerank model"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(persistence_model.RerankModel.uuid == model_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.uuid == model_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.remove_rerank_model(model_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
await self.ap.model_mgr.remove_rerank_model(context, model_uuid)
|
||||
|
||||
async def test_rerank_model(self, model_uuid: str, model_data: dict) -> None:
|
||||
async def test_rerank_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None:
|
||||
"""Test a rerank model"""
|
||||
require_workspace_uuid(context)
|
||||
runtime_rerank_model: model_requester.RuntimeRerankModel | None = None
|
||||
|
||||
if model_uuid != '_':
|
||||
for model in self.ap.model_mgr.rerank_models:
|
||||
if model.model_entity.uuid == model_uuid:
|
||||
runtime_rerank_model = model
|
||||
break
|
||||
if runtime_rerank_model is None:
|
||||
raise Exception('model not found')
|
||||
if await self.get_rerank_model(context, model_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Model not found')
|
||||
runtime_rerank_model = await self.ap.model_mgr.get_rerank_model_by_uuid(context, model_uuid)
|
||||
else:
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(model_data)
|
||||
runtime_rerank_model = await self.ap.model_mgr.init_temporary_runtime_rerank_model(
|
||||
context,
|
||||
model_data,
|
||||
)
|
||||
|
||||
await runtime_rerank_model.provider.invoke_rerank(
|
||||
model=runtime_rerank_model,
|
||||
@@ -582,4 +864,5 @@ class RerankModelsService:
|
||||
'Artificial intelligence is a branch of computer science.',
|
||||
'The weather is nice today.',
|
||||
],
|
||||
execution_context=runtime_rerank_model.execution_context,
|
||||
)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,6 +6,9 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
default_stage_order = [
|
||||
@@ -30,7 +33,8 @@ class PipelineService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_pipeline_metadata(self) -> list[dict]:
|
||||
async def get_pipeline_metadata(self, context: TenantContext) -> list[dict]:
|
||||
require_workspace_uuid(context)
|
||||
return [
|
||||
self.ap.pipeline_config_meta_trigger,
|
||||
self.ap.pipeline_config_meta_safety,
|
||||
@@ -38,8 +42,19 @@ class PipelineService:
|
||||
self.ap.pipeline_config_meta_output,
|
||||
]
|
||||
|
||||
async def get_pipelines(self, sort_by: str = 'created_at', sort_order: str = 'DESC') -> list[dict]:
|
||||
query = sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
async def get_pipelines(
|
||||
self,
|
||||
context: TenantContext,
|
||||
sort_by: str = 'created_at',
|
||||
sort_order: str = 'DESC',
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> list[dict]:
|
||||
query = scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
|
||||
if sort_by == 'created_at':
|
||||
if sort_order == 'DESC':
|
||||
@@ -54,15 +69,26 @@ class PipelineService:
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(query)
|
||||
pipelines = result.all()
|
||||
return [
|
||||
serialized = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
for pipeline in pipelines
|
||||
]
|
||||
return serialized if include_secret else [redact_secrets(pipeline) for pipeline in serialized]
|
||||
|
||||
async def get_pipeline(self, pipeline_uuid: str) -> dict | None:
|
||||
async def get_pipeline(
|
||||
self,
|
||||
context: TenantContext,
|
||||
pipeline_uuid: str,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -71,20 +97,24 @@ class PipelineService:
|
||||
if pipeline is None:
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
serialized = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
return serialized if include_secret else redact_secrets(serialized)
|
||||
|
||||
async def create_pipeline(self, pipeline_data: dict, default: bool = False) -> str:
|
||||
async def create_pipeline(self, context: TenantContext, pipeline_data: dict, default: bool = False) -> str:
|
||||
from ....utils import paths as path_utils
|
||||
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
if max_pipelines >= 0:
|
||||
existing_pipelines = await self.get_pipelines()
|
||||
existing_pipelines = await self.get_pipelines(context)
|
||||
if len(existing_pipelines) >= max_pipelines:
|
||||
raise ValueError(f'Maximum number of pipelines ({max_pipelines}) reached')
|
||||
|
||||
pipeline_data = pipeline_data.copy()
|
||||
pipeline_data['uuid'] = str(uuid.uuid4())
|
||||
pipeline_data['workspace_uuid'] = workspace_uuid
|
||||
pipeline_data['for_version'] = self.ap.ver_mgr.get_current_version()
|
||||
pipeline_data['stages'] = default_stage_order.copy()
|
||||
pipeline_data['is_default'] = default
|
||||
@@ -108,79 +138,122 @@ class PipelineService:
|
||||
sqlalchemy.insert(persistence_pipeline.LegacyPipeline).values(**pipeline_data)
|
||||
)
|
||||
|
||||
pipeline = await self.get_pipeline(pipeline_data['uuid'])
|
||||
pipeline = await self.get_pipeline(context, pipeline_data['uuid'], include_secret=True)
|
||||
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
return pipeline_data['uuid']
|
||||
|
||||
async def update_pipeline(self, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
async def update_pipeline(self, context: TenantContext, pipeline_uuid: str, pipeline_data: dict) -> None:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
pipeline_data = pipeline_data.copy()
|
||||
for protected_field in ('uuid', 'for_version', 'stages', 'is_default'):
|
||||
for protected_field in ('uuid', 'workspace_uuid', 'for_version', 'stages', 'is_default'):
|
||||
pipeline_data.pop(protected_field, None)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(**pipeline_data)
|
||||
)
|
||||
if 'config' in pipeline_data:
|
||||
current_config = None
|
||||
if contains_secret_placeholder(pipeline_data['config']):
|
||||
current_pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if current_pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
current_config = current_pipeline.get('config', {})
|
||||
pipeline_data['config'] = restore_secret_placeholders(
|
||||
pipeline_data['config'],
|
||||
current_config if current_config is not None else {},
|
||||
)
|
||||
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(**pipeline_data),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
if pipeline is None:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
|
||||
if 'name' in pipeline_data:
|
||||
from ....entity.persistence import bot as persistence_bot
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_bot.Bot).where(
|
||||
persistence_bot.Bot.use_pipeline_uuid == pipeline_uuid
|
||||
),
|
||||
persistence_bot.Bot,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
bots = result.all()
|
||||
|
||||
for bot in bots:
|
||||
bot_data = {'use_pipeline_name': pipeline_data['name']}
|
||||
await self.ap.bot_service.update_bot(bot.uuid, bot_data)
|
||||
await self.ap.bot_service.update_bot(context, bot.uuid, bot_data)
|
||||
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
# update all conversation that use this pipeline
|
||||
for session in self.ap.sess_mgr.session_list:
|
||||
if session.using_conversation is not None and session.using_conversation.pipeline_uuid == pipeline_uuid:
|
||||
if (
|
||||
session.using_conversation is not None
|
||||
and session.using_conversation.pipeline_uuid == pipeline_uuid
|
||||
and getattr(session, 'workspace_uuid', workspace_uuid) == workspace_uuid
|
||||
):
|
||||
session.using_conversation = None
|
||||
|
||||
async def delete_pipeline(self, pipeline_uuid: str) -> None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
async def delete_pipeline(self, context: TenantContext, pipeline_uuid: str) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Pipeline not found')
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
|
||||
async def copy_pipeline(self, pipeline_uuid: str) -> str:
|
||||
async def copy_pipeline(self, context: TenantContext, pipeline_uuid: str) -> str:
|
||||
"""Copy a pipeline with all its configurations"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
if max_pipelines >= 0:
|
||||
existing_pipelines = await self.get_pipelines()
|
||||
existing_pipelines = await self.get_pipelines(context)
|
||||
if len(existing_pipelines) >= max_pipelines:
|
||||
raise ValueError(f'Maximum number of pipelines ({max_pipelines}) reached')
|
||||
|
||||
# Get the original pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
original_pipeline = result.first()
|
||||
if original_pipeline is None:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Create new pipeline data
|
||||
new_uuid = str(uuid.uuid4())
|
||||
new_pipeline_data = {
|
||||
'uuid': new_uuid,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'name': f'{original_pipeline.name} (Copy)',
|
||||
'description': original_pipeline.description,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
@@ -207,13 +280,14 @@ class PipelineService:
|
||||
)
|
||||
|
||||
# Load the new pipeline
|
||||
pipeline = await self.get_pipeline(new_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
pipeline = await self.get_pipeline(context, new_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
return new_uuid
|
||||
|
||||
async def update_pipeline_extensions(
|
||||
self,
|
||||
context: TenantContext,
|
||||
pipeline_uuid: str,
|
||||
bound_plugins: list[dict],
|
||||
bound_mcp_servers: list[str] = None,
|
||||
@@ -225,16 +299,21 @@ class PipelineService:
|
||||
mcp_resource_agent_read_enabled: bool | None = None,
|
||||
) -> None:
|
||||
"""Update the bound plugins and MCP servers for a pipeline"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Get current pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid
|
||||
),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
pipeline = result.first()
|
||||
if pipeline is None:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
raise WorkspaceNotFoundError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Update extensions_preferences
|
||||
extensions_preferences = pipeline.extensions_preferences or {}
|
||||
@@ -252,12 +331,16 @@ class PipelineService:
|
||||
extensions_preferences['mcp_resources'] = bound_mcp_resources
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(extensions_preferences=extensions_preferences)
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == pipeline_uuid)
|
||||
.values(extensions_preferences=extensions_preferences),
|
||||
persistence_pipeline.LegacyPipeline,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
|
||||
# Reload pipeline to apply changes
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
await self.ap.pipeline_mgr.remove_pipeline(context, pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(context, pipeline_uuid, include_secret=True)
|
||||
await self.ap.pipeline_mgr.load_pipeline(context, pipeline)
|
||||
|
||||
@@ -7,6 +7,9 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import model as persistence_model
|
||||
from ....workspace.errors import WorkspaceNotFoundError
|
||||
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
class ModelProviderService:
|
||||
@@ -35,9 +38,15 @@ class ModelProviderService:
|
||||
|
||||
return normalized_keys
|
||||
|
||||
async def get_providers(self) -> list[dict]:
|
||||
async def get_providers(self, context: TenantContext, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all providers"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_model.ModelProvider))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
providers = result.all()
|
||||
providers_list = []
|
||||
for p in providers:
|
||||
@@ -50,14 +59,25 @@ class ModelProviderService:
|
||||
provider_dict['api_keys'] = json.loads(provider_dict['api_keys'])
|
||||
except Exception:
|
||||
provider_dict['api_keys'] = []
|
||||
if not include_secret:
|
||||
provider_dict = redact_secrets(provider_dict)
|
||||
providers_list.append(provider_dict)
|
||||
return providers_list
|
||||
|
||||
async def get_provider(self, provider_uuid: str) -> dict | None:
|
||||
async def get_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
provider_uuid: str,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a single provider by UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
provider = result.first()
|
||||
@@ -72,103 +92,171 @@ class ModelProviderService:
|
||||
provider_dict['api_keys'] = json.loads(provider_dict['api_keys'])
|
||||
except Exception:
|
||||
provider_dict['api_keys'] = []
|
||||
if not include_secret:
|
||||
provider_dict = redact_secrets(provider_dict)
|
||||
return provider_dict
|
||||
|
||||
async def create_provider(self, provider_data: dict) -> str:
|
||||
async def create_provider(self, context: TenantContext, provider_data: dict) -> str:
|
||||
"""Create a new provider"""
|
||||
provider_data = provider_data.copy()
|
||||
provider_data['uuid'] = str(uuid.uuid4())
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
provider_data['workspace_uuid'] = require_workspace_uuid(context)
|
||||
provider_data['api_keys'] = self._normalize_api_keys(
|
||||
restore_secret_placeholders(provider_data.get('api_keys'), sensitive=True)
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
|
||||
)
|
||||
|
||||
# load to runtime
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider_data)
|
||||
self.ap.model_mgr.provider_dict[runtime_provider.provider_entity.uuid] = runtime_provider
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(context, provider_data)
|
||||
await self.ap.model_mgr.cache_provider(context, runtime_provider)
|
||||
return provider_data['uuid']
|
||||
|
||||
async def update_provider(self, provider_uuid: str, provider_data: dict) -> None:
|
||||
async def update_provider(self, context: TenantContext, provider_uuid: str, provider_data: dict) -> None:
|
||||
"""Update an existing provider"""
|
||||
if 'uuid' in provider_data:
|
||||
del provider_data['uuid']
|
||||
provider_data = provider_data.copy()
|
||||
provider_data.pop('uuid', None)
|
||||
provider_data.pop('workspace_uuid', None)
|
||||
if 'api_keys' in provider_data:
|
||||
provider_data['api_keys'] = self._normalize_api_keys(provider_data.get('api_keys'))
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
.values(**provider_data)
|
||||
submitted_keys = provider_data.get('api_keys')
|
||||
if contains_secret_placeholder(submitted_keys, sensitive=True):
|
||||
current_provider = await self.get_provider(context, provider_uuid, include_secret=True)
|
||||
if current_provider is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
submitted_keys = restore_secret_placeholders(
|
||||
submitted_keys,
|
||||
current_provider.get('api_keys', []),
|
||||
sensitive=True,
|
||||
)
|
||||
provider_data['api_keys'] = self._normalize_api_keys(submitted_keys)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == provider_uuid)
|
||||
.values(**provider_data),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider(provider_uuid)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
await self.ap.model_mgr.reload_provider(context, provider_uuid)
|
||||
|
||||
async def delete_provider(self, provider_uuid: str) -> None:
|
||||
async def delete_provider(self, context: TenantContext, provider_uuid: str) -> None:
|
||||
"""Delete a provider (only if no models reference it)"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# Check if any models use this provider
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.LLMModel).where(
|
||||
persistence_model.LLMModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.LLMModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if llm_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: LLM models still reference it')
|
||||
|
||||
embedding_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.EmbeddingModel).where(
|
||||
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.EmbeddingModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if embedding_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Embedding models still reference it')
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.RerankModel).where(
|
||||
persistence_model.RerankModel.provider_uuid == provider_uuid
|
||||
),
|
||||
persistence_model.RerankModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if rerank_result.first() is not None:
|
||||
raise ValueError('Cannot delete provider: Rerank models still reference it')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.uuid == provider_uuid
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
|
||||
await self.ap.model_mgr.remove_provider(provider_uuid)
|
||||
await self.ap.model_mgr.remove_provider(context, provider_uuid)
|
||||
|
||||
async def get_provider_model_counts(self, provider_uuid: str) -> dict:
|
||||
async def get_provider_model_counts(self, context: TenantContext, provider_uuid: str) -> dict:
|
||||
"""Get count of models using this provider"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_provider(context, provider_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.LLMModel)
|
||||
.where(persistence_model.LLMModel.provider_uuid == provider_uuid),
|
||||
persistence_model.LLMModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
llm_count = llm_result.scalar() or 0
|
||||
|
||||
embedding_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.EmbeddingModel)
|
||||
.where(persistence_model.EmbeddingModel.provider_uuid == provider_uuid),
|
||||
persistence_model.EmbeddingModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
embedding_count = embedding_result.scalar() or 0
|
||||
|
||||
rerank_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid)
|
||||
scope_statement(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(persistence_model.RerankModel)
|
||||
.where(persistence_model.RerankModel.provider_uuid == provider_uuid),
|
||||
persistence_model.RerankModel,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
rerank_count = rerank_result.scalar() or 0
|
||||
|
||||
return {'llm_count': llm_count, 'embedding_count': embedding_count, 'rerank_count': rerank_count}
|
||||
|
||||
async def find_or_create_provider(self, requester: str, base_url: str, api_keys: list) -> str:
|
||||
async def find_or_create_provider(
|
||||
self,
|
||||
context: TenantContext,
|
||||
requester: str,
|
||||
base_url: str,
|
||||
api_keys: list,
|
||||
) -> str:
|
||||
"""Find existing provider or create new one"""
|
||||
api_keys = self._normalize_api_keys(api_keys)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
api_keys = self._normalize_api_keys(restore_secret_placeholders(api_keys, sensitive=True))
|
||||
|
||||
# Try to find existing provider with same config
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.requester == requester,
|
||||
persistence_model.ModelProvider.base_url == base_url,
|
||||
scope_statement(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.requester == requester,
|
||||
persistence_model.ModelProvider.base_url == base_url,
|
||||
),
|
||||
persistence_model.ModelProvider,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
for provider in result.all():
|
||||
@@ -187,29 +275,38 @@ class ModelProviderService:
|
||||
pass
|
||||
|
||||
return await self.create_provider(
|
||||
context,
|
||||
{
|
||||
'name': provider_name,
|
||||
'requester': requester,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
async def update_space_model_provider_api_keys(self, api_key: str) -> None:
|
||||
async def update_space_model_provider_api_keys(self, context: TenantContext, api_key: str) -> None:
|
||||
"""Update Space model provider API keys"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(persistence_model.ModelProvider)
|
||||
.where(persistence_model.ModelProvider.uuid == '00000000-0000-0000-0000-000000000000')
|
||||
.values(api_keys=self._normalize_api_keys(api_key)),
|
||||
persistence_model.ModelProvider,
|
||||
context,
|
||||
)
|
||||
)
|
||||
await self.ap.model_mgr.reload_provider('00000000-0000-0000-0000-000000000000')
|
||||
if getattr(result, 'rowcount', None) == 0:
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
await self.ap.model_mgr.reload_provider(context, '00000000-0000-0000-0000-000000000000')
|
||||
|
||||
async def scan_provider_models(self, provider_uuid: str, model_type: str | None = None) -> dict:
|
||||
provider = await self.get_provider(provider_uuid)
|
||||
async def scan_provider_models(
|
||||
self, context: TenantContext, provider_uuid: str, model_type: str | None = None
|
||||
) -> dict:
|
||||
provider = await self.get_provider(context, provider_uuid, include_secret=True)
|
||||
if provider is None:
|
||||
raise ValueError('provider not found')
|
||||
raise WorkspaceNotFoundError('Provider not found')
|
||||
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(provider)
|
||||
runtime_provider = await self.ap.model_mgr.load_provider(context, provider)
|
||||
|
||||
try:
|
||||
scan_result = await runtime_provider.requester.scan_models(
|
||||
@@ -230,11 +327,15 @@ class ModelProviderService:
|
||||
scanned_models = scan_result
|
||||
debug_info = None
|
||||
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(provider_uuid)
|
||||
llm_models = await self.ap.llm_model_service.get_llm_models_by_provider(context, provider_uuid)
|
||||
embedding_models = await self.ap.embedding_models_service.get_embedding_models_by_provider(
|
||||
context, provider_uuid
|
||||
)
|
||||
rerank_service = getattr(self.ap, 'rerank_models_service', None)
|
||||
rerank_models = (
|
||||
await rerank_service.get_rerank_models_by_provider(provider_uuid) if rerank_service is not None else []
|
||||
await rerank_service.get_rerank_models_by_provider(context, provider_uuid)
|
||||
if rerank_service is not None
|
||||
else []
|
||||
)
|
||||
existing_llm_names = {model['name'] for model in llm_models}
|
||||
existing_embedding_names = {model['name'] for model in embedding_models}
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
|
||||
SECRET_MASK = '***'
|
||||
_MISSING_SECRET = object()
|
||||
|
||||
_SENSITIVE_NAMES = frozenset(
|
||||
{
|
||||
'api_key',
|
||||
'api_keys',
|
||||
'apikey',
|
||||
'apikeys',
|
||||
'auth',
|
||||
'authorization',
|
||||
'cookie',
|
||||
'credentials',
|
||||
'database_url',
|
||||
'dsn',
|
||||
'header_value',
|
||||
'key',
|
||||
'proxy_authorization',
|
||||
'set_cookie',
|
||||
'webhook_url',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_TOKENS = frozenset(
|
||||
{
|
||||
'apikey',
|
||||
'credential',
|
||||
'credentials',
|
||||
'passwd',
|
||||
'password',
|
||||
'secret',
|
||||
'token',
|
||||
}
|
||||
)
|
||||
_KEY_QUALIFIERS = frozenset(
|
||||
{
|
||||
'access',
|
||||
'api',
|
||||
'auth',
|
||||
'bearer',
|
||||
'client',
|
||||
'debug',
|
||||
'encryption',
|
||||
'private',
|
||||
'signing',
|
||||
}
|
||||
)
|
||||
_SENSITIVE_URL_QUERY_NAMES = frozenset(
|
||||
{
|
||||
'code',
|
||||
'credential',
|
||||
'credentials',
|
||||
'password',
|
||||
'passwd',
|
||||
'sig',
|
||||
'signature',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _normalize_key(key: object) -> str:
|
||||
value = re.sub(r'([a-z0-9])([A-Z])', r'\1_\2', str(key or ''))
|
||||
return re.sub(r'[^a-zA-Z0-9]+', '_', value).strip('_').lower()
|
||||
|
||||
|
||||
def is_sensitive_key(key: object) -> bool:
|
||||
"""Return whether a configuration key conventionally carries a secret."""
|
||||
|
||||
normalized = _normalize_key(key)
|
||||
if normalized in _SENSITIVE_NAMES:
|
||||
return True
|
||||
tokens = frozenset(token for token in normalized.split('_') if token)
|
||||
if tokens & _SENSITIVE_TOKENS:
|
||||
return True
|
||||
return bool(tokens & {'key', 'keys'}) and bool(tokens & _KEY_QUALIFIERS)
|
||||
|
||||
|
||||
def is_url_key(key: object) -> bool:
|
||||
"""Return whether a configuration field conventionally carries a URL."""
|
||||
|
||||
normalized = _normalize_key(key)
|
||||
return normalized == 'url' or normalized.endswith('_url')
|
||||
|
||||
|
||||
def _is_sensitive_url_query_key(key: object) -> bool:
|
||||
normalized = _normalize_key(key)
|
||||
return (
|
||||
is_sensitive_key(key) or normalized in _SENSITIVE_URL_QUERY_NAMES or normalized.endswith(('_sig', '_signature'))
|
||||
)
|
||||
|
||||
|
||||
def _redact_url_string(value: str) -> str:
|
||||
if not value:
|
||||
return value
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
netloc = parsed.netloc
|
||||
if '@' in netloc:
|
||||
_, host = netloc.rsplit('@', 1)
|
||||
netloc = f'{SECRET_MASK}@{host}'
|
||||
query = urlencode(
|
||||
[
|
||||
(key, SECRET_MASK if _is_sensitive_url_query_key(key) and item else item)
|
||||
for key, item in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
],
|
||||
doseq=True,
|
||||
safe='*',
|
||||
)
|
||||
return urlunsplit((parsed.scheme, netloc, parsed.path, query, parsed.fragment))
|
||||
except (TypeError, ValueError):
|
||||
# A malformed URL cannot be safely decomposed, so fail closed.
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def redact_url_secrets(value):
|
||||
"""Redact URL userinfo and credential-like query values."""
|
||||
|
||||
if isinstance(value, str):
|
||||
return _redact_url_string(value)
|
||||
if isinstance(value, list):
|
||||
return [redact_url_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_url_secrets(item) for item in value)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def _contains_url_secret_placeholder(value) -> bool:
|
||||
if isinstance(value, str):
|
||||
if value == SECRET_MASK:
|
||||
return True
|
||||
try:
|
||||
parsed = urlsplit(value)
|
||||
if '@' in parsed.netloc and SECRET_MASK in parsed.netloc.rsplit('@', 1)[0]:
|
||||
return True
|
||||
return any(
|
||||
item == SECRET_MASK and _is_sensitive_url_query_key(key)
|
||||
for key, item in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(_contains_url_secret_placeholder(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def _restore_url_string(value: str, current_value) -> str:
|
||||
if value == SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked URL secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
|
||||
try:
|
||||
submitted = urlsplit(value)
|
||||
except (TypeError, ValueError):
|
||||
return value
|
||||
|
||||
current = None
|
||||
if isinstance(current_value, str):
|
||||
try:
|
||||
current = urlsplit(current_value)
|
||||
except (TypeError, ValueError):
|
||||
current = None
|
||||
|
||||
netloc = submitted.netloc
|
||||
if '@' in netloc:
|
||||
submitted_userinfo, host = netloc.rsplit('@', 1)
|
||||
if SECRET_MASK in submitted_userinfo:
|
||||
if current is None or '@' not in current.netloc:
|
||||
raise ValueError('Masked URL userinfo has no existing value')
|
||||
current_userinfo, _ = current.netloc.rsplit('@', 1)
|
||||
netloc = f'{current_userinfo}@{host}'
|
||||
|
||||
current_query: dict[str, list[str]] = {}
|
||||
if current is not None:
|
||||
for key, item in parse_qsl(current.query, keep_blank_values=True):
|
||||
current_query.setdefault(_normalize_key(key), []).append(item)
|
||||
consumed: dict[str, int] = {}
|
||||
restored_query: list[tuple[str, str]] = []
|
||||
for key, item in parse_qsl(submitted.query, keep_blank_values=True):
|
||||
normalized = _normalize_key(key)
|
||||
if item == SECRET_MASK and _is_sensitive_url_query_key(key):
|
||||
index = consumed.get(normalized, 0)
|
||||
candidates = current_query.get(normalized, [])
|
||||
if index >= len(candidates):
|
||||
raise ValueError('Masked URL query secret has no existing value')
|
||||
item = candidates[index]
|
||||
consumed[normalized] = index + 1
|
||||
restored_query.append((key, item))
|
||||
|
||||
return urlunsplit(
|
||||
(
|
||||
submitted.scheme,
|
||||
netloc,
|
||||
submitted.path,
|
||||
urlencode(restored_query, doseq=True, safe='*'),
|
||||
submitted.fragment,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def restore_url_secret_placeholders(value, current_value=_MISSING_SECRET):
|
||||
"""Restore URL placeholders from the corresponding persisted URL."""
|
||||
|
||||
if isinstance(value, str):
|
||||
return _restore_url_string(value, current_value)
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def mask_secret_value(value):
|
||||
"""Return a shape-preserving copy whose non-empty leaves are masked."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {key: mask_secret_value(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [mask_secret_value(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(mask_secret_value(item) for item in value)
|
||||
if value is None or value == '':
|
||||
return value
|
||||
return SECRET_MASK
|
||||
|
||||
|
||||
def redact_secrets(value):
|
||||
"""Return a recursively redacted copy without mutating the source value."""
|
||||
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
key: (
|
||||
mask_secret_value(item)
|
||||
if is_sensitive_key(key)
|
||||
else redact_url_secrets(item)
|
||||
if is_url_key(key)
|
||||
else redact_secrets(item)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_secrets(item) for item in value]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(redact_secrets(item) for item in value)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def restore_secret_placeholders(value, current_value=_MISSING_SECRET, *, sensitive: bool = False):
|
||||
"""Restore masked leaves from existing data before a management write.
|
||||
|
||||
``***`` is a reserved placeholder only inside a sensitive field. A masked
|
||||
leaf without an existing counterpart is rejected so it can never become a
|
||||
persisted credential. Empty values and explicit replacements pass through.
|
||||
"""
|
||||
|
||||
if sensitive and value == SECRET_MASK:
|
||||
if current_value is _MISSING_SECRET:
|
||||
raise ValueError('Masked secret has no existing value')
|
||||
return copy.deepcopy(current_value)
|
||||
if isinstance(value, dict):
|
||||
current_mapping = current_value if isinstance(current_value, dict) else {}
|
||||
return {
|
||||
key: (
|
||||
restore_url_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
)
|
||||
if not sensitive and not is_sensitive_key(key) and is_url_key(key)
|
||||
else restore_secret_placeholders(
|
||||
item,
|
||||
current_mapping.get(key, _MISSING_SECRET),
|
||||
sensitive=sensitive or is_sensitive_key(key),
|
||||
)
|
||||
)
|
||||
for key, item in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return [
|
||||
restore_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
current_items = current_value if isinstance(current_value, (list, tuple)) else ()
|
||||
return tuple(
|
||||
restore_secret_placeholders(
|
||||
item,
|
||||
current_items[index] if index < len(current_items) else _MISSING_SECRET,
|
||||
sensitive=sensitive,
|
||||
)
|
||||
for index, item in enumerate(value)
|
||||
)
|
||||
return copy.deepcopy(value)
|
||||
|
||||
|
||||
def contains_secret_placeholder(value, *, sensitive: bool = False) -> bool:
|
||||
"""Return whether ``value`` contains a meaningful masked secret leaf."""
|
||||
|
||||
if sensitive and value == SECRET_MASK:
|
||||
return True
|
||||
if isinstance(value, dict):
|
||||
return any(
|
||||
(
|
||||
_contains_url_secret_placeholder(item)
|
||||
if not sensitive and not is_sensitive_key(key) and is_url_key(key)
|
||||
else contains_secret_placeholder(item, sensitive=sensitive or is_sensitive_key(key))
|
||||
)
|
||||
for key, item in value.items()
|
||||
)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(contains_secret_placeholder(item, sensitive=sensitive) for item in value)
|
||||
return False
|
||||
@@ -1,9 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import inspect
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
import zipfile
|
||||
from typing import Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
@@ -12,6 +14,9 @@ import httpx
|
||||
|
||||
from ....core import app
|
||||
from ....skill.utils import parse_frontmatter
|
||||
from ....utils import httpclient
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
_PUBLIC_SKILL_FIELDS = (
|
||||
@@ -32,6 +37,12 @@ _GITHUB_ASSET_HOSTS = {
|
||||
'raw.githubusercontent.com',
|
||||
'codeload.github.com',
|
||||
}
|
||||
_MAX_GITHUB_ARCHIVE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_GITHUB_ARCHIVE_ENTRIES = 4096
|
||||
_MAX_SKILL_ARCHIVE_FILES = 1024
|
||||
_MAX_SKILL_FILE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_SKILL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
|
||||
_MAX_SKILL_COMPRESSION_RATIO = 200
|
||||
|
||||
|
||||
class SkillService:
|
||||
@@ -75,75 +86,112 @@ class SkillService:
|
||||
"""Backwards-compatible alias preserved for clarity at call sites."""
|
||||
self._require_box(action)
|
||||
|
||||
async def _execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip()
|
||||
generation = getattr(context, 'placement_generation', None)
|
||||
if not instance_uuid or isinstance(generation, bool) or not isinstance(generation, int) or generation <= 0:
|
||||
raise ValueError('Skill operations require an explicit fenced execution context')
|
||||
binding = await self.ap.workspace_service.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=generation,
|
||||
)
|
||||
if binding.instance_uuid != instance_uuid:
|
||||
raise ValueError('Skill execution context belongs to another LangBot instance')
|
||||
return ExecutionContext(
|
||||
instance_uuid=instance_uuid,
|
||||
workspace_uuid=workspace_uuid,
|
||||
placement_generation=generation,
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _serialize_skill(skill: dict) -> dict:
|
||||
return {field: skill.get(field) for field in _PUBLIC_SKILL_FIELDS if field in skill}
|
||||
|
||||
async def list_skills(self) -> list[dict]:
|
||||
async def list_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
# When Box is unavailable, surface an empty list rather than raising —
|
||||
# the skills page should render cleanly, and the UI separately renders
|
||||
# a "Box disabled / unavailable" banner via useBoxStatus.
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return []
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills()]
|
||||
return [self._serialize_skill(skill) for skill in await box_service.list_skills(execution_context)]
|
||||
|
||||
async def get_skill(self, skill_name: str) -> Optional[dict]:
|
||||
async def get_skill(self, context: TenantContext, skill_name: str) -> Optional[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._box_service()
|
||||
if box_service is None:
|
||||
return None
|
||||
skill = await box_service.get_skill(skill_name)
|
||||
skill = await box_service.get_skill(execution_context, skill_name)
|
||||
return self._serialize_skill(skill) if skill else None
|
||||
|
||||
async def get_skill_by_name(self, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(name)
|
||||
async def get_skill_by_name(self, context: TenantContext, name: str) -> Optional[dict]:
|
||||
return await self.get_skill(context, name)
|
||||
|
||||
async def create_skill(self, data: dict) -> dict:
|
||||
async def create_skill(self, context: TenantContext, data: dict) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Creating a skill')
|
||||
created = await box_service.create_skill(data)
|
||||
await self._reload_skills()
|
||||
created = await box_service.create_skill(execution_context, data)
|
||||
await self._reload_skills(execution_context)
|
||||
return self._serialize_skill(created)
|
||||
|
||||
async def update_skill(self, skill_name: str, data: dict) -> dict:
|
||||
async def update_skill(self, context: TenantContext, skill_name: str, data: dict) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Editing a skill')
|
||||
updated = await box_service.update_skill(skill_name, data)
|
||||
await self._reload_skills()
|
||||
updated = await box_service.update_skill(execution_context, skill_name, data)
|
||||
await self._reload_skills(execution_context)
|
||||
return self._serialize_skill(updated)
|
||||
|
||||
async def delete_skill(self, skill_name: str) -> bool:
|
||||
async def delete_skill(self, context: TenantContext, skill_name: str) -> bool:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Deleting a skill')
|
||||
await box_service.delete_skill(skill_name)
|
||||
await self._reload_skills()
|
||||
await box_service.delete_skill(execution_context, skill_name)
|
||||
await self._reload_skills(execution_context)
|
||||
return True
|
||||
|
||||
async def list_skill_files(
|
||||
self,
|
||||
context: TenantContext,
|
||||
skill_name: str,
|
||||
path: str = '.',
|
||||
include_hidden: bool = False,
|
||||
max_entries: int = 200,
|
||||
) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Browsing skill files')
|
||||
return await box_service.list_skill_files(skill_name, path, include_hidden, max_entries)
|
||||
return await box_service.list_skill_files(execution_context, skill_name, path, include_hidden, max_entries)
|
||||
|
||||
async def read_skill_file(self, skill_name: str, path: str) -> dict:
|
||||
async def read_skill_file(self, context: TenantContext, skill_name: str, path: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Reading a skill file')
|
||||
return await box_service.read_skill_file(skill_name, path)
|
||||
return await box_service.read_skill_file(execution_context, skill_name, path)
|
||||
|
||||
async def write_skill_file(self, skill_name: str, path: str, content: str) -> dict:
|
||||
async def write_skill_file(self, context: TenantContext, skill_name: str, path: str, content: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Editing skill files')
|
||||
result = await box_service.write_skill_file(skill_name, path, content)
|
||||
await self._reload_skills()
|
||||
result = await box_service.write_skill_file(execution_context, skill_name, path, content)
|
||||
await self._reload_skills(execution_context)
|
||||
return result
|
||||
|
||||
async def install_from_github(self, data: dict) -> list[dict]:
|
||||
async def install_from_github(self, context: TenantContext, data: dict) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._install_github_skill_md(raw_asset_url, owner=owner, repo=repo, data=data)
|
||||
return await self._install_github_skill_md(
|
||||
execution_context,
|
||||
raw_asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
data=data,
|
||||
)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
@@ -151,29 +199,37 @@ class SkillService:
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
filename = f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip'
|
||||
installed = await box_service.install_skill_zip(
|
||||
execution_context,
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
source_subdir=source_subdir,
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(execution_context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_github(self, data: dict) -> list[dict]:
|
||||
async def preview_install_from_github(self, context: TenantContext, data: dict) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
owner = str(data['owner']).strip()
|
||||
repo = str(data['repo']).strip()
|
||||
release_tag = str(data.get('release_tag', '')).strip()
|
||||
raw_asset_url = str(data['asset_url']).strip()
|
||||
if self._is_github_skill_md_url(raw_asset_url):
|
||||
return await self._preview_github_skill_md(raw_asset_url, owner=owner, repo=repo)
|
||||
return await self._preview_github_skill_md(
|
||||
execution_context,
|
||||
raw_asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
|
||||
asset_url = self._validate_github_asset_url(raw_asset_url, owner=owner, repo=repo, release_tag=release_tag)
|
||||
source_subdir = str(data.get('source_subdir', '') or '').strip()
|
||||
|
||||
zip_bytes = await self._download_github_asset(asset_url)
|
||||
return await box_service.preview_skill_zip(
|
||||
execution_context,
|
||||
zip_bytes,
|
||||
f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip',
|
||||
source_subdir=source_subdir,
|
||||
@@ -181,27 +237,45 @@ class SkillService:
|
||||
|
||||
async def install_from_zip_upload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
source_paths: list[str] | None = None,
|
||||
source_path: str = '',
|
||||
) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Installing a skill from upload')
|
||||
installed = await box_service.install_skill_zip(
|
||||
execution_context,
|
||||
file_bytes,
|
||||
filename,
|
||||
source_paths=source_paths or [],
|
||||
source_path=source_path,
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(execution_context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def preview_install_from_zip_upload(self, *, file_bytes: bytes, filename: str) -> list[dict]:
|
||||
async def preview_install_from_zip_upload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
*,
|
||||
file_bytes: bytes,
|
||||
filename: str,
|
||||
) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Previewing a skill upload')
|
||||
return await box_service.preview_skill_zip(file_bytes, filename)
|
||||
return await box_service.preview_skill_zip(execution_context, file_bytes, filename)
|
||||
|
||||
async def _install_github_skill_md(self, asset_url: str, *, owner: str, repo: str, data: dict) -> list[dict]:
|
||||
async def _install_github_skill_md(
|
||||
self,
|
||||
context: TenantContext,
|
||||
asset_url: str,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
data: dict,
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Installing a skill from GitHub')
|
||||
zip_bytes, filename, _package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
@@ -210,46 +284,73 @@ class SkillService:
|
||||
)
|
||||
|
||||
installed = await box_service.install_skill_zip(
|
||||
context,
|
||||
zip_bytes,
|
||||
filename,
|
||||
source_paths=data.get('source_paths') or [],
|
||||
source_path=str(data.get('source_path', '') or ''),
|
||||
target_suffix='',
|
||||
)
|
||||
await self._reload_skills()
|
||||
await self._reload_skills(context)
|
||||
return [self._serialize_skill(skill) for skill in installed]
|
||||
|
||||
async def _preview_github_skill_md(self, asset_url: str, *, owner: str, repo: str) -> list[dict]:
|
||||
async def _preview_github_skill_md(
|
||||
self,
|
||||
context: TenantContext,
|
||||
asset_url: str,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
) -> list[dict]:
|
||||
box_service = self._require_box('Previewing a skill from GitHub')
|
||||
zip_bytes, _filename, package_name = await self._download_github_skill_directory_as_zip(
|
||||
asset_url,
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
)
|
||||
return await box_service.preview_skill_zip(zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
return await box_service.preview_skill_zip(context, zip_bytes, f'{package_name}.zip', target_suffix='')
|
||||
|
||||
async def reload_skills(self) -> list[dict]:
|
||||
await self._reload_skills()
|
||||
return await self.list_skills()
|
||||
async def reload_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._execution_context(context)
|
||||
await self._reload_skills(execution_context)
|
||||
return await self.list_skills(execution_context)
|
||||
|
||||
async def scan_directory_async(self, path: str) -> dict:
|
||||
async def scan_directory_async(self, context: TenantContext, path: str) -> dict:
|
||||
execution_context = await self._execution_context(context)
|
||||
box_service = self._require_box('Scanning a skill directory')
|
||||
return await box_service.scan_skill_directory(path)
|
||||
return await box_service.scan_skill_directory(execution_context, path)
|
||||
|
||||
async def _reload_skills(self) -> None:
|
||||
async def _reload_skills(self, context: TenantContext) -> None:
|
||||
skill_mgr = getattr(self.ap, 'skill_mgr', None)
|
||||
reload_skills = getattr(skill_mgr, 'reload_skills', None)
|
||||
if not callable(reload_skills):
|
||||
return
|
||||
result = reload_skills()
|
||||
result = reload_skills(context)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _download_github_asset(self, asset_url: str) -> bytes:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
|
||||
resp = await client.get(asset_url)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
async with httpx.AsyncClient(
|
||||
follow_redirects=True,
|
||||
timeout=120,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(_MAX_GITHUB_ARCHIVE_BYTES),
|
||||
) as client:
|
||||
async with client.stream('GET', asset_url) as resp:
|
||||
resp.raise_for_status()
|
||||
content_length = resp.headers.get('content-length')
|
||||
if content_length is not None:
|
||||
try:
|
||||
if int(content_length) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
except ValueError as exc:
|
||||
if 'exceeds' in str(exc):
|
||||
raise
|
||||
content = bytearray()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
content.extend(chunk)
|
||||
if len(content) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
return bytes(content)
|
||||
|
||||
async def _download_github_skill_directory_as_zip(
|
||||
self, asset_url: str, *, owner: str, repo: str
|
||||
@@ -257,14 +358,25 @@ class SkillService:
|
||||
info = self._parse_github_skill_md_url(asset_url, owner=owner, repo=repo)
|
||||
archive_url = f'https://codeload.github.com/{owner}/{repo}/zip/{quote(info["ref"], safe="/")}'
|
||||
archive_bytes = await self._download_github_asset(archive_url)
|
||||
return await asyncio.to_thread(self._build_github_skill_directory_zip, archive_bytes, info)
|
||||
|
||||
def _build_github_skill_directory_zip(
|
||||
self,
|
||||
archive_bytes: bytes,
|
||||
info: dict[str, str],
|
||||
) -> tuple[bytes, str, str]:
|
||||
"""Validate and repack a GitHub skill archive outside the event loop."""
|
||||
try:
|
||||
source_archive = zipfile.ZipFile(io.BytesIO(archive_bytes), 'r')
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ValueError('GitHub repository archive must be a valid .zip archive') from exc
|
||||
|
||||
with source_archive as source_zip:
|
||||
if len(source_zip.infolist()) > _MAX_GITHUB_ARCHIVE_ENTRIES:
|
||||
raise ValueError('GitHub repository archive contains too many entries')
|
||||
skill_entry = self._find_github_skill_archive_entry(source_zip, info['file_path'])
|
||||
if skill_entry.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError('GitHub SKILL.md exceeds the file size limit')
|
||||
try:
|
||||
skill_md_content = source_zip.read(skill_entry).decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
@@ -302,6 +414,7 @@ class SkillService:
|
||||
normalized_source_dir = posixpath.normpath(source_skill_dir)
|
||||
source_prefix = f'{normalized_source_dir}/'
|
||||
copied_files = 0
|
||||
copied_bytes = 0
|
||||
|
||||
for member in source_zip.infolist():
|
||||
normalized_member = posixpath.normpath(member.filename)
|
||||
@@ -324,10 +437,33 @@ class SkillService:
|
||||
if member.is_dir():
|
||||
target_zip.writestr(target_info, b'')
|
||||
continue
|
||||
|
||||
target_zip.writestr(target_info, source_zip.read(member))
|
||||
if member.flag_bits & 0x1:
|
||||
raise ValueError('Encrypted GitHub skill archive entries are not supported')
|
||||
unix_mode = member.external_attr >> 16
|
||||
if stat.S_IFMT(unix_mode) == stat.S_IFLNK:
|
||||
raise ValueError(f'GitHub archive contains a symbolic link: {member.filename}')
|
||||
if member.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError(f'GitHub skill file exceeds the size limit: {member.filename}')
|
||||
if member.file_size and member.file_size > max(member.compress_size, 1) * _MAX_SKILL_COMPRESSION_RATIO:
|
||||
raise ValueError(f'GitHub skill file exceeds the compression-ratio limit: {member.filename}')
|
||||
copied_files += 1
|
||||
copied_bytes += member.file_size
|
||||
if copied_files > _MAX_SKILL_ARCHIVE_FILES:
|
||||
raise ValueError('GitHub skill directory contains too many files')
|
||||
if copied_bytes > _MAX_SKILL_UNCOMPRESSED_BYTES:
|
||||
raise ValueError('GitHub skill directory exceeds the uncompressed size limit')
|
||||
|
||||
# Copy in bounded chunks instead of materialising a potentially
|
||||
# large member in Core memory. The Box Runtime independently
|
||||
# revalidates the resulting archive before installation.
|
||||
with source_zip.open(member, 'r') as source_file, target_zip.open(target_info, 'w') as target_file:
|
||||
remaining = member.file_size
|
||||
while remaining:
|
||||
chunk = source_file.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise ValueError(f'GitHub skill file is truncated: {member.filename}')
|
||||
target_file.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if copied_files == 0:
|
||||
raise ValueError('GitHub skill directory is empty')
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from langbot.pkg.utils import httpclient
|
||||
import typing
|
||||
import datetime
|
||||
@@ -11,6 +13,10 @@ from ....entity.persistence import user
|
||||
from ....entity.dto.space_model import SpaceModel
|
||||
|
||||
|
||||
_CREDITS_CACHE_TTL_SECONDS = 60
|
||||
_CREDITS_CACHE_MAX_ENTRIES = 4096
|
||||
|
||||
|
||||
class SpaceService:
|
||||
"""Service for interacting with LangBot Space API"""
|
||||
|
||||
@@ -19,7 +25,24 @@ class SpaceService:
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
self._credits_cache = {}
|
||||
self._credits_cache = OrderedDict()
|
||||
|
||||
def _ordered_credits_cache(
|
||||
self,
|
||||
) -> OrderedDict[str, tuple[int, float]]:
|
||||
if not isinstance(self._credits_cache, OrderedDict):
|
||||
# Preserve compatibility with tests and callers that seed the cache.
|
||||
self._credits_cache = OrderedDict(self._credits_cache)
|
||||
return self._credits_cache
|
||||
|
||||
def _prune_credits_cache(self, now: float) -> None:
|
||||
cache = self._ordered_credits_cache()
|
||||
while cache:
|
||||
email = next(iter(cache))
|
||||
_, cached_at = cache[email]
|
||||
if now - cached_at < _CREDITS_CACHE_TTL_SECONDS:
|
||||
break
|
||||
cache.pop(email, None)
|
||||
|
||||
def _get_space_config(self) -> typing.Dict[str, str]:
|
||||
"""Get Space configuration from config file"""
|
||||
@@ -85,12 +108,14 @@ class SpaceService:
|
||||
|
||||
def get_oauth_authorize_url(self, redirect_uri: str, state: str = '') -> str:
|
||||
"""Get the Space OAuth authorization URL for redirect"""
|
||||
from urllib.parse import urlencode
|
||||
|
||||
space_config = self._get_space_config()
|
||||
authorize_url = space_config['oauth_authorize_url']
|
||||
params = f'redirect_uri={redirect_uri}'
|
||||
params = {'redirect_uri': redirect_uri}
|
||||
if state:
|
||||
params += f'&state={state}'
|
||||
return f'{authorize_url}?{params}'
|
||||
params['state'] = state
|
||||
return f'{authorize_url}?{urlencode(params)}'
|
||||
|
||||
async def exchange_oauth_code(self, code: str) -> typing.Dict:
|
||||
"""Exchange OAuth authorization code for tokens"""
|
||||
@@ -105,8 +130,9 @@ class SpaceService:
|
||||
json={'code': code, 'instance_id': constants.instance_id},
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to exchange OAuth code: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to exchange OAuth code: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to exchange OAuth code: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -121,8 +147,9 @@ class SpaceService:
|
||||
f'{space_url}/api/v1/accounts/token/refresh', json={'refresh_token': refresh_token}
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to refresh token: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to refresh token: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to refresh token: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -137,8 +164,9 @@ class SpaceService:
|
||||
f'{space_url}/api/v1/accounts/me', headers={'Authorization': f'Bearer {access_token}'}
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get user info: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to get user info: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to get user info: {data.get("msg")}')
|
||||
return data.get('data', {})
|
||||
@@ -154,11 +182,13 @@ class SpaceService:
|
||||
|
||||
async def get_credits(self, user_email: str, force_refresh: bool = False) -> int | None:
|
||||
"""Get Space credits for user with caching (60s TTL)"""
|
||||
cache_ttl = 60
|
||||
now = time.time()
|
||||
cached_fallback = self._credits_cache.get(user_email)
|
||||
self._prune_credits_cache(now)
|
||||
|
||||
if not force_refresh and user_email in self._credits_cache:
|
||||
credits, ts = self._credits_cache[user_email]
|
||||
if time.time() - ts < cache_ttl:
|
||||
if now - ts < _CREDITS_CACHE_TTL_SECONDS:
|
||||
return credits
|
||||
|
||||
try:
|
||||
@@ -167,10 +197,14 @@ class SpaceService:
|
||||
return None
|
||||
credits = info.get('credits')
|
||||
if credits is not None:
|
||||
self._credits_cache[user_email] = (credits, time.time())
|
||||
cache = self._ordered_credits_cache()
|
||||
cache.pop(user_email, None)
|
||||
if len(cache) >= _CREDITS_CACHE_MAX_ENTRIES:
|
||||
cache.popitem(last=False)
|
||||
cache[user_email] = (credits, time.time())
|
||||
return credits
|
||||
except Exception:
|
||||
return self._credits_cache.get(user_email, (None, 0))[0]
|
||||
return cached_fallback[0] if cached_fallback is not None else None
|
||||
|
||||
async def get_models(self) -> typing.List[SpaceModel]:
|
||||
"""Get models from Space"""
|
||||
@@ -181,8 +215,9 @@ class SpaceService:
|
||||
session = httpclient.get_session()
|
||||
async with session.get(f'{space_url}/api/v1/models', params={'page_size': 100}) as response:
|
||||
if response.status != 200:
|
||||
raise ValueError(f'Failed to get models: {await response.text()}')
|
||||
data = await response.json()
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise ValueError(f'Failed to get models: {error}')
|
||||
data = await httpclient.read_json_limited(response)
|
||||
if data.get('code') != 0:
|
||||
raise ValueError(f'Failed to get models: {data.get("msg")}')
|
||||
models_data = data.get('data', {}).get('models', [])
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext, RequestContext, WorkspaceContext
|
||||
|
||||
TenantContext: typing.TypeAlias = RequestContext | ExecutionContext | WorkspaceContext | str
|
||||
|
||||
|
||||
def require_workspace_uuid(context: TenantContext | None) -> str:
|
||||
"""Resolve an explicit Workspace UUID without allowing a global fallback."""
|
||||
|
||||
if isinstance(context, str):
|
||||
workspace_uuid = context
|
||||
elif isinstance(context, RequestContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
elif isinstance(context, ExecutionContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
elif isinstance(context, WorkspaceContext):
|
||||
workspace_uuid = context.workspace_uuid
|
||||
else:
|
||||
raise WorkspaceRequiredError('Workspace context is required')
|
||||
|
||||
normalized = workspace_uuid.strip()
|
||||
if not normalized:
|
||||
raise WorkspaceRequiredError('Workspace context is required')
|
||||
return normalized
|
||||
|
||||
|
||||
def scope_statement(statement: typing.Any, model: typing.Any, context: TenantContext) -> typing.Any:
|
||||
"""Add the mandatory Workspace predicate to a SQLAlchemy statement."""
|
||||
|
||||
return statement.where(model.workspace_uuid == require_workspace_uuid(context))
|
||||
@@ -6,71 +6,391 @@ import jwt
|
||||
import datetime
|
||||
import typing
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import heapq
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import user
|
||||
from ....entity.persistence.workspace import MembershipRole, MembershipStatus, WorkspaceMembership
|
||||
from ....utils import constants
|
||||
from ....entity.errors import account as account_errors
|
||||
from ....workspace.collaboration import normalize_email
|
||||
from ....utils import bounded_executor
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from ....core.app import Application
|
||||
|
||||
|
||||
_SPACE_OAUTH_STATE_MAX_ENTRIES = 4096
|
||||
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR = 64
|
||||
_SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER = 4
|
||||
|
||||
|
||||
class AccountExistsLoginRequiredError(ValueError):
|
||||
code = 'account_exists_login_required'
|
||||
|
||||
|
||||
class PublicRegistrationClosedError(ValueError):
|
||||
code = 'registration_closed'
|
||||
|
||||
|
||||
class ControlPlaneDirectoryRequiredError(PublicRegistrationClosedError):
|
||||
code = 'control_plane_required'
|
||||
|
||||
|
||||
class AccountDisabledError(ValueError):
|
||||
code = 'account_disabled'
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class SpaceOAuthStateConsumption:
|
||||
purpose: typing.Literal['login', 'bind']
|
||||
account: user.User | None
|
||||
launch_workspace_uuid: str | None = None
|
||||
|
||||
|
||||
class UserService:
|
||||
ap: app.Application
|
||||
ap: Application
|
||||
_create_user_lock: asyncio.Lock
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(self, ap: Application) -> None:
|
||||
self.ap = ap
|
||||
self._create_user_lock = asyncio.Lock()
|
||||
self._password_hash_lock = asyncio.Semaphore(1)
|
||||
self._password_hash_lock = asyncio.Lock()
|
||||
self._space_oauth_state_lock = asyncio.Lock()
|
||||
self._space_oauth_states: dict[str, tuple[str, str | None, float, str | None]] = {}
|
||||
self._space_oauth_state_expiry_heap: list[tuple[float, str]] = []
|
||||
|
||||
@staticmethod
|
||||
def _space_oauth_state_digest(state: str) -> str:
|
||||
return hashlib.sha256(state.encode('utf-8')).hexdigest()
|
||||
|
||||
def _prune_space_oauth_states(self, now: float) -> None:
|
||||
while self._space_oauth_state_expiry_heap:
|
||||
expires_at, digest = self._space_oauth_state_expiry_heap[0]
|
||||
entry = self._space_oauth_states.get(digest)
|
||||
if entry is None or entry[2] != expires_at:
|
||||
heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
continue
|
||||
if expires_at > now:
|
||||
break
|
||||
heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
self._space_oauth_states.pop(digest, None)
|
||||
|
||||
max_heap_entries = max(
|
||||
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR,
|
||||
len(self._space_oauth_states) * _SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER,
|
||||
)
|
||||
if len(self._space_oauth_state_expiry_heap) > max_heap_entries:
|
||||
self._space_oauth_state_expiry_heap[:] = [
|
||||
(entry[2], digest) for digest, entry in self._space_oauth_states.items()
|
||||
]
|
||||
heapq.heapify(self._space_oauth_state_expiry_heap)
|
||||
|
||||
def _evict_earliest_space_oauth_state(self) -> None:
|
||||
while self._space_oauth_state_expiry_heap:
|
||||
expires_at, digest = heapq.heappop(self._space_oauth_state_expiry_heap)
|
||||
entry = self._space_oauth_states.get(digest)
|
||||
if entry is not None and entry[2] == expires_at:
|
||||
self._space_oauth_states.pop(digest, None)
|
||||
return
|
||||
|
||||
async def issue_space_oauth_state(
|
||||
self,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
*,
|
||||
account_uuid: str | None = None,
|
||||
launch_workspace_uuid: str | None = None,
|
||||
ttl_seconds: int = 600,
|
||||
) -> str:
|
||||
"""Issue an opaque, single-use OAuth state without exposing a JWT."""
|
||||
if purpose == 'bind' and not account_uuid:
|
||||
raise ValueError('An Account is required for Space binding')
|
||||
if purpose == 'login' and account_uuid is not None:
|
||||
raise ValueError('Login state cannot be bound to an Account')
|
||||
if purpose != 'login' and launch_workspace_uuid is not None:
|
||||
raise ValueError('Launch Workspace state is only valid for Space login')
|
||||
if ttl_seconds <= 0:
|
||||
raise ValueError('OAuth state lifetime must be positive')
|
||||
|
||||
raw_state = secrets.token_urlsafe(32)
|
||||
digest = self._space_oauth_state_digest(raw_state)
|
||||
expires_at = time.monotonic() + min(ttl_seconds, 600)
|
||||
async with self._space_oauth_state_lock:
|
||||
now = time.monotonic()
|
||||
self._prune_space_oauth_states(now)
|
||||
if len(self._space_oauth_states) >= _SPACE_OAUTH_STATE_MAX_ENTRIES:
|
||||
self._evict_earliest_space_oauth_state()
|
||||
self._space_oauth_states[digest] = (purpose, account_uuid, expires_at, launch_workspace_uuid)
|
||||
heapq.heappush(
|
||||
self._space_oauth_state_expiry_heap,
|
||||
(expires_at, digest),
|
||||
)
|
||||
return raw_state
|
||||
|
||||
async def consume_space_oauth_state_details(
|
||||
self,
|
||||
raw_state: str,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
) -> SpaceOAuthStateConsumption:
|
||||
"""Atomically consume OAuth state and return any bound launch intent."""
|
||||
if not isinstance(raw_state, str) or not raw_state:
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
digest = self._space_oauth_state_digest(raw_state)
|
||||
async with self._space_oauth_state_lock:
|
||||
entry = self._space_oauth_states.pop(digest, None)
|
||||
if entry is None or entry[0] != purpose or entry[2] <= time.monotonic():
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
if purpose == 'login':
|
||||
return SpaceOAuthStateConsumption(
|
||||
purpose='login',
|
||||
account=None,
|
||||
launch_workspace_uuid=entry[3],
|
||||
)
|
||||
|
||||
account_uuid = entry[1]
|
||||
account = await self.get_user_by_uuid(account_uuid or '')
|
||||
if account is None:
|
||||
raise ValueError('Invalid or expired OAuth state')
|
||||
self._require_active_account(account)
|
||||
return SpaceOAuthStateConsumption(purpose='bind', account=account)
|
||||
|
||||
async def consume_space_oauth_state(
|
||||
self,
|
||||
raw_state: str,
|
||||
purpose: typing.Literal['login', 'bind'],
|
||||
) -> user.User | None:
|
||||
"""Atomically consume OAuth state and resolve its active bind Account."""
|
||||
consumed = await self.consume_space_oauth_state_details(raw_state, purpose)
|
||||
return consumed.account
|
||||
|
||||
async def _hash_password(self, password: str) -> str:
|
||||
if self._password_hash_lock.locked():
|
||||
raise bounded_executor.BlockingWorkCapacityError(
|
||||
'Password hashing capacity reached',
|
||||
scope='system:authentication',
|
||||
)
|
||||
async with self._password_hash_lock:
|
||||
return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
|
||||
with bounded_executor.blocking_work_scope('system:authentication'):
|
||||
return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
|
||||
|
||||
def _require_local_directory(self) -> None:
|
||||
if self._uses_control_plane_directory():
|
||||
raise ControlPlaneDirectoryRequiredError(
|
||||
'Cloud Accounts and directory changes are managed by the SaaS control plane'
|
||||
)
|
||||
|
||||
def _uses_control_plane_directory(self) -> bool:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
return bool(workspace_service is not None and workspace_service.policy.multi_workspace_enabled)
|
||||
|
||||
async def _verify_password(self, hashed_password: str, password: str) -> None:
|
||||
if self._password_hash_lock.locked():
|
||||
raise bounded_executor.BlockingWorkCapacityError(
|
||||
'Password hashing capacity reached',
|
||||
scope='system:authentication',
|
||||
)
|
||||
async with self._password_hash_lock:
|
||||
await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
|
||||
with bounded_executor.blocking_work_scope('system:authentication'):
|
||||
await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
|
||||
|
||||
async def _update_space_provider_for_account(self, account: typing.Any, api_key: str) -> None:
|
||||
"""Refresh the OSS Workspace Space provider without guessing a SaaS Workspace.
|
||||
|
||||
Space OAuth credentials belong to an Account, while model-provider secrets
|
||||
belong to a Workspace. Community edition has one unambiguous Workspace, so
|
||||
the historical automatic refresh remains available only to the Workspace owner.
|
||||
In multi-Workspace SaaS mode the OAuth callback has
|
||||
no trusted Workspace selector; the closed control plane or an explicit
|
||||
Workspace settings action must perform that linkage instead.
|
||||
"""
|
||||
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
collaboration_service = getattr(self.ap, 'workspace_collaboration_service', None)
|
||||
account_uuid = getattr(account, 'uuid', None)
|
||||
if workspace_service is None or collaboration_service is None or not isinstance(account_uuid, str):
|
||||
# Never turn a missing tenant kernel into a global secret mutation.
|
||||
return
|
||||
if workspace_service.policy.multi_workspace_enabled:
|
||||
return
|
||||
|
||||
accesses = await collaboration_service.list_account_workspaces(account_uuid)
|
||||
if len(accesses) != 1:
|
||||
return
|
||||
access = accesses[0]
|
||||
if access.membership.role != MembershipRole.OWNER.value:
|
||||
return
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(
|
||||
access.workspace.uuid,
|
||||
api_key,
|
||||
)
|
||||
|
||||
async def is_initialized(self) -> bool:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
account = await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
return account is not None
|
||||
|
||||
result_list = result.all()
|
||||
return result_list is not None and len(result_list) > 0
|
||||
async def get_login_capabilities(self) -> dict[str, bool]:
|
||||
"""Derive enabled public login methods in an explicit discovery scope."""
|
||||
password_count = sqlalchemy.func.count().filter(user.User.password.is_not(None), user.User.password != '')
|
||||
space_count = sqlalchemy.func.count().filter(user.User.space_account_uuid.is_not(None))
|
||||
statement = sqlalchemy.select(password_count, space_count).where(
|
||||
user.User.status == user.AccountStatus.ACTIVE.value
|
||||
)
|
||||
digest = hashlib.sha256(f'login-capabilities:{self._jwt_identity()[1]}'.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
result = await discovery.session.execute(statement)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
password_accounts, space_accounts = result.one()
|
||||
return {
|
||||
'password_login_enabled': bool(password_accounts),
|
||||
'space_login_enabled': bool(space_accounts),
|
||||
}
|
||||
|
||||
async def get_workspace_owner(self, workspace_uuid: str) -> user.User | None:
|
||||
"""Resolve the active owner Account for a Workspace."""
|
||||
statement = (
|
||||
sqlalchemy.select(user.User)
|
||||
.join(WorkspaceMembership, WorkspaceMembership.account_uuid == user.User.uuid)
|
||||
.where(
|
||||
WorkspaceMembership.workspace_uuid == workspace_uuid,
|
||||
WorkspaceMembership.role == MembershipRole.OWNER.value,
|
||||
WorkspaceMembership.status == MembershipStatus.ACTIVE.value,
|
||||
user.User.status == user.AccountStatus.ACTIVE.value,
|
||||
)
|
||||
)
|
||||
current_session = self.ap.persistence_mgr.current_session()
|
||||
if current_session is not None:
|
||||
return await current_session.scalar(statement)
|
||||
return await self._identity_scalar(statement, f'workspace-owner:{workspace_uuid}')
|
||||
|
||||
def _session_factory(self) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(self.ap.persistence_mgr.get_db_engine(), expire_on_commit=False)
|
||||
|
||||
def _jwt_identity(self) -> tuple[str, str]:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
instance_uuid = str(getattr(workspace_service, 'instance_uuid', '') or constants.instance_id).strip()
|
||||
# UserService is constructed only after config/bootstrap in production.
|
||||
# The fallback keeps lightweight isolated unit tests deterministic.
|
||||
if not instance_uuid:
|
||||
instance_uuid = 'uninitialized-test-instance'
|
||||
return 'langbot-core', f'langbot-instance:{instance_uuid}'
|
||||
|
||||
def _legacy_local_tokens_allowed(self) -> bool:
|
||||
workspace_service = getattr(self.ap, 'workspace_service', None)
|
||||
policy = getattr(workspace_service, 'policy', None)
|
||||
return getattr(policy, 'multi_workspace_enabled', False) is not True
|
||||
|
||||
async def create_user(self, user_email: str, password: str) -> None:
|
||||
"""Create the first local Account and Workspace owner atomically."""
|
||||
|
||||
await self.create_initial_account(user_email, password)
|
||||
|
||||
async def create_initial_account(self, user_email: str, password: str) -> user.User:
|
||||
self._require_local_directory()
|
||||
normalized_email = normalize_email(user_email)
|
||||
hashed_password = await self._hash_password(password)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(user.User).values(user=user_email, password=hashed_password, account_type='local')
|
||||
async with self._create_user_lock:
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
existing_count = int(
|
||||
(await session.scalar(sqlalchemy.select(sqlalchemy.func.count()).select_from(user.User))) or 0
|
||||
)
|
||||
if existing_count:
|
||||
raise PublicRegistrationClosedError('System already initialized')
|
||||
account = self._new_account(normalized_email, hashed_password)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
await self.ap.workspace_service.bootstrap_local_account(account.uuid, session=session)
|
||||
return account
|
||||
|
||||
async def register_invited_account(
|
||||
self,
|
||||
invitation_token: str,
|
||||
user_email: str,
|
||||
password: str,
|
||||
) -> tuple[user.User, typing.Any]:
|
||||
"""Create an invited Account and accept its Membership in one transaction."""
|
||||
|
||||
normalized_email = normalize_email(user_email)
|
||||
if self._uses_control_plane_directory():
|
||||
raise ControlPlaneDirectoryRequiredError(
|
||||
'Cloud invitation registration must use a Space account to preserve control-plane identity'
|
||||
)
|
||||
invitation, _ = await self.ap.workspace_collaboration_service.inspect_invitation(invitation_token)
|
||||
if invitation.normalized_email != normalized_email:
|
||||
from ....workspace.collaboration import InvitationEmailMismatchError
|
||||
|
||||
raise InvitationEmailMismatchError('Invitation email does not match the Account')
|
||||
hashed_password = await self._hash_password(password)
|
||||
|
||||
async with self._create_user_lock:
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
existing = await session.scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email)
|
||||
)
|
||||
if existing is not None:
|
||||
raise AccountExistsLoginRequiredError('An Account already exists for this email')
|
||||
account = self._new_account(normalized_email, hashed_password)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
membership = await self.ap.workspace_collaboration_service.accept_invitation(
|
||||
invitation_token,
|
||||
account.uuid,
|
||||
session=session,
|
||||
)
|
||||
return account, membership
|
||||
|
||||
def _new_account(self, normalized_email: str, hashed_password: str) -> user.User:
|
||||
return user.User(
|
||||
uuid=str(uuid.uuid4()),
|
||||
user=normalized_email,
|
||||
normalized_email=normalized_email,
|
||||
password=hashed_password,
|
||||
account_type='local',
|
||||
status=user.AccountStatus.ACTIVE.value,
|
||||
source=user.AccountSource.LOCAL.value,
|
||||
projection_revision=0,
|
||||
)
|
||||
|
||||
async def get_user_by_email(self, user_email: str) -> user.User | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.user == user_email)
|
||||
normalized_email = user_email.strip().casefold()
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
async def get_user_by_uuid(self, account_uuid: str) -> user.User | None:
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid),
|
||||
f'uuid:{account_uuid}',
|
||||
)
|
||||
|
||||
async def get_user_by_space_account_uuid(self, space_account_uuid: str) -> user.User | None:
|
||||
"""Get user by Space account UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid)
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
|
||||
async def authenticate(self, user_email: str, password: str) -> str | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.user == user_email)
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
|
||||
if result_list is None or len(result_list) == 0:
|
||||
user_obj = await self.get_user_by_email(user_email)
|
||||
if user_obj is None:
|
||||
raise ValueError('用户不存在')
|
||||
|
||||
user_obj = result_list[0]
|
||||
self._require_active_account(user_obj)
|
||||
|
||||
# Check if this user has a local password set
|
||||
if not user_obj.password:
|
||||
@@ -78,30 +398,121 @@ class UserService:
|
||||
|
||||
await self._verify_password(user_obj.password, password)
|
||||
|
||||
return await self.generate_jwt_token(user_email)
|
||||
return await self.generate_jwt_token(user_obj)
|
||||
|
||||
async def generate_jwt_token(self, user_email: str) -> str:
|
||||
async def generate_jwt_token(self, account: user.User | str) -> str:
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
jwt_expire = self.ap.instance_config.data['system']['jwt']['expire']
|
||||
|
||||
account_obj: user.User | None = account if not isinstance(account, str) and hasattr(account, 'user') else None
|
||||
user_email = account_obj.user if account_obj is not None else account
|
||||
if account_obj is None and hasattr(self.ap, 'persistence_mgr'):
|
||||
try:
|
||||
account_obj = await self.get_user_by_email(user_email)
|
||||
except (AttributeError, TypeError):
|
||||
# Lightweight unit-test and bootstrap callers may not have persistence wired.
|
||||
account_obj = None
|
||||
|
||||
payload = {
|
||||
'user': user_email,
|
||||
'iss': 'LangBot-' + constants.edition,
|
||||
'iss': self._jwt_identity()[0],
|
||||
'aud': self._jwt_identity()[1],
|
||||
'exp': datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=jwt_expire),
|
||||
}
|
||||
if account_obj is not None:
|
||||
self._require_active_account(account_obj)
|
||||
payload.update(
|
||||
{
|
||||
'sub': account_obj.uuid,
|
||||
'account_revision': account_obj.projection_revision,
|
||||
}
|
||||
)
|
||||
|
||||
return jwt.encode(payload, jwt_secret, algorithm='HS256')
|
||||
|
||||
async def verify_jwt_token(self, token: str) -> str:
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
account = await self.get_authenticated_account(token, allow_unresolved_legacy=True)
|
||||
if isinstance(account, str):
|
||||
return account
|
||||
return account.user
|
||||
|
||||
return jwt.decode(token, jwt_secret, algorithms=['HS256'])['user']
|
||||
async def get_authenticated_account(
|
||||
self,
|
||||
token: str,
|
||||
*,
|
||||
allow_unresolved_legacy: bool = False,
|
||||
) -> user.User | str:
|
||||
"""Resolve a JWT to an active Account, accepting bounded legacy email tokens."""
|
||||
|
||||
jwt_secret = self.ap.instance_config.data['system']['jwt']['secret']
|
||||
issuer, audience = self._jwt_identity()
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
jwt_secret,
|
||||
algorithms=['HS256'],
|
||||
issuer=issuer,
|
||||
audience=audience,
|
||||
options={'require': ['exp', 'iss', 'aud']},
|
||||
)
|
||||
except jwt.MissingRequiredClaimError:
|
||||
# Preserve one bounded OSS upgrade path for previously issued
|
||||
# community tokens. SaaS/Cloud policy never accepts these tokens,
|
||||
# and a token carrying a new-style or foreign audience cannot fall
|
||||
# back into the legacy decoder.
|
||||
unverified = jwt.decode(token, options={'verify_signature': False})
|
||||
if (
|
||||
not self._legacy_local_tokens_allowed()
|
||||
or 'aud' in unverified
|
||||
or unverified.get('iss') != 'LangBot-community'
|
||||
):
|
||||
raise
|
||||
payload = jwt.decode(
|
||||
token,
|
||||
jwt_secret,
|
||||
algorithms=['HS256'],
|
||||
options={'require': ['exp'], 'verify_aud': False, 'verify_iss': False},
|
||||
)
|
||||
account_obj: user.User | None = None
|
||||
account_uuid = payload.get('sub')
|
||||
if isinstance(account_uuid, str) and account_uuid:
|
||||
try:
|
||||
account_obj = await self.get_user_by_uuid(account_uuid)
|
||||
except AttributeError:
|
||||
account_obj = None
|
||||
if account_obj is None:
|
||||
legacy_email = payload.get('user')
|
||||
if not isinstance(legacy_email, str) or not legacy_email:
|
||||
raise ValueError('JWT Account identity is missing')
|
||||
try:
|
||||
account_obj = await self.get_user_by_email(legacy_email)
|
||||
except AttributeError:
|
||||
account_obj = None
|
||||
if account_obj is None and allow_unresolved_legacy:
|
||||
return legacy_email
|
||||
if account_obj is None:
|
||||
raise ValueError('Account not found')
|
||||
self._require_active_account(account_obj)
|
||||
token_revision = payload.get('account_revision')
|
||||
if token_revision is not None and int(token_revision) != account_obj.projection_revision:
|
||||
raise ValueError('Account token revision is stale')
|
||||
return account_obj
|
||||
|
||||
@staticmethod
|
||||
def _require_active_account(account: user.User) -> None:
|
||||
status = getattr(account, 'status', user.AccountStatus.ACTIVE.value)
|
||||
if isinstance(status, str) and status != user.AccountStatus.ACTIVE.value:
|
||||
raise AccountDisabledError('Account is disabled')
|
||||
|
||||
async def reset_password(self, user_email: str, new_password: str) -> None:
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def change_password(self, user_email: str, current_password: str, new_password: str) -> None:
|
||||
@@ -115,9 +526,13 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Space user management
|
||||
@@ -132,6 +547,16 @@ class UserService:
|
||||
expires_in: int = 0,
|
||||
) -> user.User:
|
||||
"""Create or update a Space user account (only if system not initialized or user exists)"""
|
||||
if self._uses_control_plane_directory():
|
||||
return await self._update_projected_space_user(
|
||||
space_account_uuid=space_account_uuid,
|
||||
email=email,
|
||||
access_token=access_token,
|
||||
refresh_token=refresh_token,
|
||||
api_key=api_key,
|
||||
expires_in=expires_in,
|
||||
)
|
||||
self._require_local_directory()
|
||||
expires_at = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) if expires_in > 0 else None
|
||||
|
||||
async with self._create_user_lock:
|
||||
@@ -140,7 +565,7 @@ class UserService:
|
||||
|
||||
if existing_user:
|
||||
# Update existing user's tokens
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.space_account_uuid == space_account_uuid)
|
||||
.values(
|
||||
@@ -148,19 +573,56 @@ class UserService:
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
await self._update_space_provider_for_account(existing_user, api_key)
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
|
||||
# Check if user with same email exists
|
||||
existing_email_user = await self.get_user_by_email(email)
|
||||
if existing_email_user:
|
||||
# Update existing user to link with Space account
|
||||
# Email is display/contact identity, not an OAuth subject. An
|
||||
# unknown Space subject must never take over an existing local
|
||||
# Account merely by presenting the same email. The Account
|
||||
# owner must first authenticate locally and use the explicit,
|
||||
# account-bound bind flow.
|
||||
raise account_errors.SpaceAccountBindingRequiredError()
|
||||
|
||||
# Check if system is already initialized
|
||||
is_initialized = await self.is_initialized()
|
||||
if is_initialized:
|
||||
raise account_errors.SpaceAccountNotRegisteredError()
|
||||
|
||||
# Create new Space user (first time initialization)
|
||||
if hasattr(self.ap.persistence_mgr, 'get_db_engine') and hasattr(self.ap, 'workspace_service'):
|
||||
async with self._session_factory()() as session:
|
||||
async with session.begin():
|
||||
account = user.User(
|
||||
uuid=str(uuid.uuid4()),
|
||||
user=normalize_email(email),
|
||||
normalized_email=normalize_email(email),
|
||||
password='',
|
||||
account_type='space',
|
||||
status=user.AccountStatus.ACTIVE.value,
|
||||
source=user.AccountSource.LOCAL.value,
|
||||
projection_revision=0,
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
session.add(account)
|
||||
await session.flush()
|
||||
await self.ap.workspace_service.bootstrap_local_account(account.uuid, session=session)
|
||||
else:
|
||||
# Compatibility path for lightweight service tests without a real engine.
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.user == email)
|
||||
.values(
|
||||
sqlalchemy.insert(user.User).values(
|
||||
user=normalize_email(email),
|
||||
normalized_email=normalize_email(email),
|
||||
password='',
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
@@ -169,30 +631,56 @@ class UserService:
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
return await self.get_user_by_email(email)
|
||||
created_user = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if created_user is not None:
|
||||
await self._update_space_provider_for_account(created_user, api_key)
|
||||
return created_user
|
||||
|
||||
# Check if system is already initialized
|
||||
is_initialized = await self.is_initialized()
|
||||
if is_initialized:
|
||||
raise account_errors.AccountEmailMismatchError()
|
||||
async def _update_projected_space_user(
|
||||
self,
|
||||
*,
|
||||
space_account_uuid: str,
|
||||
email: str,
|
||||
access_token: str,
|
||||
refresh_token: str,
|
||||
api_key: str,
|
||||
expires_in: int,
|
||||
) -> user.User:
|
||||
"""Attach OAuth credentials to an already projected Cloud Account."""
|
||||
|
||||
# Create new Space user (first time initialization)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(user.User).values(
|
||||
user=email,
|
||||
password='', # Space users don't have local password
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
normalized_email = normalize_email(email)
|
||||
expires_at = datetime.datetime.now() + datetime.timedelta(seconds=expires_in) if expires_in > 0 else None
|
||||
async with self._create_user_lock:
|
||||
projected = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if (
|
||||
projected is None
|
||||
or projected.uuid != space_account_uuid
|
||||
or projected.normalized_email != normalized_email
|
||||
or projected.source != user.AccountSource.CLOUD_PROJECTION.value
|
||||
or projected.account_type != 'space'
|
||||
):
|
||||
raise ControlPlaneDirectoryRequiredError('Space Account is not present in the verified Cloud directory')
|
||||
self._require_active_account(projected)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(
|
||||
user.User.uuid == projected.uuid,
|
||||
user.User.space_account_uuid == space_account_uuid,
|
||||
user.User.source == user.AccountSource.CLOUD_PROJECTION.value,
|
||||
)
|
||||
.values(
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
refreshed = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if refreshed is None:
|
||||
raise ControlPlaneDirectoryRequiredError('Space Account disappeared from the verified Cloud directory')
|
||||
self._require_active_account(refreshed)
|
||||
return refreshed
|
||||
|
||||
async def authenticate_space_user(
|
||||
self, access_token: str, refresh_token: str, expires_in: int = 0
|
||||
@@ -221,15 +709,44 @@ class UserService:
|
||||
)
|
||||
|
||||
# Generate JWT token
|
||||
jwt_token = await self.generate_jwt_token(email)
|
||||
jwt_token = await self.generate_jwt_token(user_obj)
|
||||
|
||||
return jwt_token, user_obj
|
||||
|
||||
async def get_first_user(self) -> user.User | None:
|
||||
"""Get the first user (for single-user mode)"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list else None
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
|
||||
async def _identity_scalar(
|
||||
self,
|
||||
statement: typing.Any,
|
||||
identity: str,
|
||||
) -> user.User | None:
|
||||
"""Execute one exact Account lookup in an explicit discovery transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.scalar(statement)
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
rows = result.all()
|
||||
return rows[0] if rows else None
|
||||
|
||||
async def _identity_execute(self, statement: typing.Any, identity: str) -> typing.Any:
|
||||
"""Execute one exact Account mutation in an explicit transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.execute(statement)
|
||||
return await self.ap.persistence_mgr.execute_async(statement)
|
||||
|
||||
async def set_password(self, user_email: str, new_password: str, current_password: str | None = None) -> None:
|
||||
"""Set or change password for a user"""
|
||||
@@ -246,12 +763,19 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def bind_space_account(self, user_email: str, code: str) -> user.User:
|
||||
"""Bind Space account to existing local account"""
|
||||
local_account = await self.get_user_by_email(user_email)
|
||||
if local_account is None:
|
||||
raise ValueError('User not found')
|
||||
# Exchange code for tokens
|
||||
token_data = await self.ap.space_service.exchange_oauth_code(code)
|
||||
access_token = token_data.get('access_token')
|
||||
@@ -273,28 +797,33 @@ class UserService:
|
||||
|
||||
if not space_account_uuid or not space_email:
|
||||
raise ValueError('Invalid Space user info')
|
||||
if normalize_email(space_email) != normalize_email(user_email):
|
||||
raise account_errors.AccountEmailMismatchError()
|
||||
|
||||
# Check if this Space account is already bound to another user
|
||||
existing_space_user = await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
if existing_space_user and existing_space_user.user != user_email:
|
||||
if existing_space_user and existing_space_user.normalized_email != normalize_email(user_email):
|
||||
raise ValueError('This Space account is already bound to another user')
|
||||
|
||||
# Update local account to Space account
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.user == user_email)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(
|
||||
user=space_email, # Update email to Space email
|
||||
user=normalize_email(space_email), # Update email to Space email
|
||||
normalized_email=normalize_email(space_email),
|
||||
account_type='space',
|
||||
space_account_uuid=space_account_uuid,
|
||||
space_access_token=access_token,
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Update Space model provider API keys
|
||||
await self.ap.provider_service.update_space_model_provider_api_keys(api_key)
|
||||
await self._update_space_provider_for_account(local_account, api_key)
|
||||
|
||||
return await self.get_user_by_email(space_email)
|
||||
|
||||
@@ -4,6 +4,12 @@ import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import webhook
|
||||
from .secrets import SECRET_MASK, mask_secret_value, restore_secret_placeholders
|
||||
from .tenant import TenantContext, require_workspace_uuid, scope_statement
|
||||
|
||||
|
||||
_DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE = 16
|
||||
_HARD_MAX_WEBHOOKS_PER_WORKSPACE = 64
|
||||
|
||||
|
||||
class WebhookService:
|
||||
@@ -12,31 +18,99 @@ class WebhookService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
async def get_webhooks(self) -> list[dict]:
|
||||
def max_per_workspace(self) -> int:
|
||||
"""Return the configured webhook cap within the process hard limit."""
|
||||
|
||||
config = getattr(getattr(self.ap, 'instance_config', None), 'data', {})
|
||||
try:
|
||||
value = int(
|
||||
config.get('webhooks', {}).get(
|
||||
'max_per_workspace',
|
||||
_DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE,
|
||||
)
|
||||
)
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
value = _DEFAULT_MAX_WEBHOOKS_PER_WORKSPACE
|
||||
return min(max(value, 1), _HARD_MAX_WEBHOOKS_PER_WORKSPACE)
|
||||
|
||||
def _serialize_webhook(self, entity, *, include_secret: bool) -> dict:
|
||||
serialized = self.ap.persistence_mgr.serialize_model(webhook.Webhook, entity)
|
||||
if not include_secret:
|
||||
serialized = serialized.copy()
|
||||
serialized['url'] = mask_secret_value(serialized.get('url'))
|
||||
return serialized
|
||||
|
||||
async def get_webhooks(self, context: TenantContext, *, include_secret: bool = False) -> list[dict]:
|
||||
"""Get all webhooks"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(webhook.Webhook))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).order_by(webhook.Webhook.id).limit(_HARD_MAX_WEBHOOKS_PER_WORKSPACE),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
webhooks = result.all()
|
||||
return [self.ap.persistence_mgr.serialize_model(webhook.Webhook, wh) for wh in webhooks]
|
||||
return [self._serialize_webhook(wh, include_secret=include_secret) for wh in webhooks]
|
||||
|
||||
async def create_webhook(self, name: str, url: str, description: str = '', enabled: bool = True) -> dict:
|
||||
async def create_webhook(
|
||||
self,
|
||||
context: TenantContext,
|
||||
name: str,
|
||||
url: str,
|
||||
description: str = '',
|
||||
enabled: bool = True,
|
||||
) -> dict:
|
||||
"""Create a new webhook"""
|
||||
webhook_data = {'name': name, 'url': url, 'description': description, 'enabled': enabled}
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
max_webhooks = self.max_per_workspace()
|
||||
count_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count())
|
||||
.select_from(webhook.Webhook)
|
||||
.where(webhook.Webhook.workspace_uuid == workspace_uuid)
|
||||
)
|
||||
if (count_result.scalar() or 0) >= max_webhooks:
|
||||
raise ValueError(f'Maximum number of webhooks ({max_webhooks}) reached')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(webhook.Webhook).values(**webhook_data))
|
||||
url = restore_secret_placeholders(url, sensitive=True)
|
||||
webhook_data = {
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'name': name,
|
||||
'url': url,
|
||||
'description': description,
|
||||
'enabled': enabled,
|
||||
}
|
||||
|
||||
insert_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(webhook.Webhook).values(**webhook_data)
|
||||
)
|
||||
|
||||
# Retrieve the created webhook
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.url == url).order_by(webhook.Webhook.id.desc())
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == insert_result.inserted_primary_key[0]),
|
||||
webhook.Webhook,
|
||||
workspace_uuid,
|
||||
)
|
||||
)
|
||||
created_webhook = result.first()
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(webhook.Webhook, created_webhook)
|
||||
|
||||
async def get_webhook(self, webhook_id: int) -> dict | None:
|
||||
async def get_webhook(
|
||||
self,
|
||||
context: TenantContext,
|
||||
webhook_id: int,
|
||||
*,
|
||||
include_secret: bool = False,
|
||||
) -> dict | None:
|
||||
"""Get a specific webhook by ID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == webhook_id)
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.id == webhook_id),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
|
||||
wh = result.first()
|
||||
@@ -44,16 +118,27 @@ class WebhookService:
|
||||
if wh is None:
|
||||
return None
|
||||
|
||||
return self.ap.persistence_mgr.serialize_model(webhook.Webhook, wh)
|
||||
return self._serialize_webhook(wh, include_secret=include_secret)
|
||||
|
||||
async def update_webhook(
|
||||
self, webhook_id: int, name: str = None, url: str = None, description: str = None, enabled: bool = None
|
||||
) -> None:
|
||||
self,
|
||||
context: TenantContext,
|
||||
webhook_id: int,
|
||||
name: str | None = None,
|
||||
url: str | None = None,
|
||||
description: str | None = None,
|
||||
enabled: bool | None = None,
|
||||
) -> bool:
|
||||
"""Update a webhook's metadata"""
|
||||
update_data = {}
|
||||
if name is not None:
|
||||
update_data['name'] = name
|
||||
if url is not None:
|
||||
if url == SECRET_MASK:
|
||||
current = await self.get_webhook(context, webhook_id, include_secret=True)
|
||||
if current is None:
|
||||
return False
|
||||
url = restore_secret_placeholders(url, current.get('url'), sensitive=True)
|
||||
update_data['url'] = url
|
||||
if description is not None:
|
||||
update_data['description'] = description
|
||||
@@ -61,20 +146,37 @@ class WebhookService:
|
||||
update_data['enabled'] = enabled
|
||||
|
||||
if update_data:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(webhook.Webhook).where(webhook.Webhook.id == webhook_id).values(**update_data)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.update(webhook.Webhook).where(webhook.Webhook.id == webhook_id).values(**update_data),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return (result.rowcount or 0) > 0
|
||||
return await self.get_webhook(context, webhook_id) is not None
|
||||
|
||||
async def delete_webhook(self, webhook_id: int) -> None:
|
||||
async def delete_webhook(self, context: TenantContext, webhook_id: int) -> bool:
|
||||
"""Delete a webhook"""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(webhook.Webhook).where(webhook.Webhook.id == webhook_id)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
scope_statement(
|
||||
sqlalchemy.delete(webhook.Webhook).where(webhook.Webhook.id == webhook_id),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
)
|
||||
return (result.rowcount or 0) > 0
|
||||
|
||||
async def get_enabled_webhooks(self) -> list[dict]:
|
||||
async def get_enabled_webhooks(self, context: TenantContext) -> list[dict]:
|
||||
"""Get all enabled webhooks"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.enabled == True)
|
||||
scope_statement(
|
||||
sqlalchemy.select(webhook.Webhook).where(webhook.Webhook.enabled == True),
|
||||
webhook.Webhook,
|
||||
context,
|
||||
)
|
||||
.order_by(webhook.Webhook.id)
|
||||
.limit(self.max_per_workspace())
|
||||
)
|
||||
|
||||
webhooks = result.all()
|
||||
|
||||
Reference in New Issue
Block a user