feat(cloud): harden multi-tenant runtime resources

src/langbot/pkg/api/http/service/maintenance.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import datetime
 import functools
 import os
@@ -69,7 +70,10 @@ class MaintenanceService:
 
         return {
             'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
-            'log_files': self._cleanup_expired_log_files(log_retention_days)
+            'log_files': await asyncio.to_thread(
+                self._cleanup_expired_log_files,
+                log_retention_days,
+            )
             if await self._is_oss_singleton(context)
             else 0,
         }
@@ -108,22 +112,19 @@ class MaintenanceService:
             scoped_storage_path = Path('data/storage') / self.ap.storage_mgr.scoped_prefix(context)
             roots = [('storage', scoped_storage_path)]
 
-        sections = []
-        for key, path in roots:
-            sections.append(
-                {
-                    'key': key,
-                    'path': str(path) if path else '',
-                    'exists': path.exists() if path else False,
-                    'size_bytes': self._path_size(path) if path else 0,
-                    'file_count': self._file_count(path) if path else 0,
-                }
-            )
+        sections = await asyncio.to_thread(self._collect_sections, roots)
 
         monitoring_counts = await self._monitoring_counts(context)
         binary_storage = await self._binary_storage_stats(context)
         upload_candidates = await self._expired_uploaded_candidates(context, upload_retention_days)
-        log_candidates = self._expired_log_candidates(log_retention_days) if is_oss_singleton else []
+        log_candidates = (
+            await asyncio.to_thread(
+                self._expired_log_candidates,
+                log_retention_days,
+            )
+            if is_oss_singleton
+            else []
+        )
 
         return {
             'generated_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
@@ -144,6 +145,23 @@ class MaintenanceService:
             'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
         }
 
+    def _collect_sections(
+        self,
+        roots: list[tuple[str, Path | None]],
+    ) -> list[dict[str, Any]]:
+        sections = []
+        for key, path in roots:
+            sections.append(
+                {
+                    'key': key,
+                    'path': str(path) if path else '',
+                    'exists': path.exists() if path else False,
+                    'size_bytes': self._path_size(path) if path else 0,
+                    'file_count': self._file_count(path) if path else 0,
+                }
+            )
+        return sections
+
     async def _is_oss_singleton(self, context: TenantContext) -> bool:
         try:
             await self.ap.workspace_service.get_local_execution_binding(
@@ -162,21 +180,16 @@ class MaintenanceService:
         provider = self.ap.storage_mgr.storage_provider
         provider_name = provider.__class__.__name__
         if provider_name == 'LocalStorageProvider':
-            candidates = self._expired_local_upload_candidates(
+            candidates = await asyncio.to_thread(
+                self._expired_local_upload_candidates,
                 context,
                 retention_days,
-                include_paths=True,
+                True,
+            )
+            return await asyncio.to_thread(
+                self._delete_local_candidates,
+                candidates,
             )
-            deleted = 0
-            for item in candidates:
-                try:
-                    os.remove(item['path'])
-                    deleted += 1
-                except FileNotFoundError:
-                    pass
-                except Exception as e:
-                    self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
-            return deleted
 
         if provider_name == 'S3StorageProvider':
             return await self._cleanup_expired_s3_uploaded_files(context, retention_days)
@@ -190,7 +203,11 @@ class MaintenanceService:
     ) -> list[dict[str, Any]]:
         provider_name = self.ap.storage_mgr.storage_provider.__class__.__name__
         if provider_name == 'LocalStorageProvider':
-            return self._expired_local_upload_candidates(context, retention_days)
+            return await asyncio.to_thread(
+                self._expired_local_upload_candidates,
+                context,
+                retention_days,
+            )
         if provider_name == 'S3StorageProvider':
             return await self._expired_s3_upload_candidates(context, retention_days)
         return []
@@ -212,6 +229,25 @@ class MaintenanceService:
         self,
         context: TenantContext,
         retention_days: int,
+    ) -> list[dict[str, Any]]:
+        provider = self.ap.storage_mgr.storage_provider
+        run_io = getattr(provider, '_run_io', None)
+        if callable(run_io):
+            return await run_io(
+                self._expired_s3_upload_candidates_sync,
+                context,
+                retention_days,
+            )
+        return await asyncio.to_thread(
+            self._expired_s3_upload_candidates_sync,
+            context,
+            retention_days,
+        )
+
+    def _expired_s3_upload_candidates_sync(
+        self,
+        context: TenantContext,
+        retention_days: int,
     ) -> list[dict[str, Any]]:
         provider = self.ap.storage_mgr.storage_provider
         cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
@@ -241,6 +277,18 @@ class MaintenanceService:
 
         return candidates
 
+    def _delete_local_candidates(self, candidates: list[dict[str, Any]]) -> int:
+        deleted = 0
+        for item in candidates:
+            try:
+                os.remove(item['path'])
+                deleted += 1
+            except FileNotFoundError:
+                pass
+            except Exception as e:
+                self.ap.logger.warning(f'Failed to delete expired uploaded file {item["key"]}: {e}')
+        return deleted
+
     def _cleanup_expired_log_files(self, retention_days: int) -> int:
         deleted = 0
         for item in self._expired_log_candidates(retention_days, include_paths=True):

src/langbot/pkg/api/http/service/mcp.py

@@ -252,8 +252,17 @@ class MCPService:
             task = create_detached_task(
                 self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created),
                 after_commit_manager=self.ap.persistence_mgr,
+                workspace_uuid=execution_context.workspace_uuid,
             )
-            self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
+            tracker = getattr(
+                self.ap.tool_mgr.mcp_tool_loader,
+                'track_hosted_task',
+                None,
+            )
+            if callable(tracker):
+                tracker(task, execution_context)
+            else:
+                self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
         return payload['uuid']
 
     async def get_mcp_server_by_uuid(self, context: TenantContext, server_uuid: str) -> dict | None:
@@ -357,8 +366,13 @@ class MCPService:
             task = create_detached_task(
                 loader.host_mcp_server(execution_context, updated),
                 after_commit_manager=self.ap.persistence_mgr,
+                workspace_uuid=execution_context.workspace_uuid,
             )
-            loader._hosted_mcp_tasks.append(task)
+            tracker = getattr(loader, 'track_hosted_task', None)
+            if callable(tracker):
+                tracker(task, execution_context)
+            else:
+                loader._hosted_mcp_tasks.append(task)
 
     async def delete_mcp_server(self, context: TenantContext, server_uuid: str) -> None:
         execution_context = await self._execution_context(context)
@@ -420,6 +434,7 @@ class MCPService:
     async def test_mcp_server(self, context: TenantContext, server_name: str, server_data: dict) -> int:
         execution_context = await self._execution_context(context)
         runtime_mcp_session: RuntimeMCPSession | None = None
+        test_session: RuntimeMCPSession | None = None
         ctx = taskmgr.TaskContext.new()
 
         if server_name != '_':
@@ -468,16 +483,27 @@ class MCPService:
 
             coroutine = _run_and_cleanup()
 
-        wrapper = self.ap.task_mgr.create_user_task(
-            coroutine,
-            kind='mcp-operation',
-            name=f'mcp-test-{execution_context.workspace_uuid}-{server_name}',
-            label=f'Testing MCP server {server_name}',
-            context=ctx,
-            instance_uuid=execution_context.instance_uuid,
-            workspace_uuid=execution_context.workspace_uuid,
-            placement_generation=execution_context.placement_generation,
-        )
+        try:
+            wrapper = self.ap.task_mgr.create_user_task(
+                coroutine,
+                kind='mcp-operation',
+                name=f'mcp-test-{execution_context.workspace_uuid}-{server_name}',
+                label=f'Testing MCP server {server_name}',
+                context=ctx,
+                instance_uuid=execution_context.instance_uuid,
+                workspace_uuid=execution_context.workspace_uuid,
+                placement_generation=execution_context.placement_generation,
+            )
+        except taskmgr.TaskCapacityError:
+            if test_session is not None:
+                try:
+                    await test_session.shutdown()
+                except Exception as exc:
+                    self.ap.logger.warning(
+                        f'Failed to tear down rejected transient MCP test session '
+                        f'{test_session.server_name}: {type(exc).__name__}: {exc}'
+                    )
+            raise
         return wrapper.id
 
     async def get_mcp_server_logs(

src/langbot/pkg/api/http/service/skill.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import io
 import inspect
 import os
@@ -13,6 +14,7 @@ import httpx
 
 from ....core import app
 from ....skill.utils import parse_frontmatter
+from ....utils import httpclient
 from ..context import ExecutionContext
 from .tenant import TenantContext, require_workspace_uuid
 
@@ -328,7 +330,11 @@ class SkillService:
             await result
 
     async def _download_github_asset(self, asset_url: str) -> bytes:
-        async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
+        async with httpx.AsyncClient(
+            follow_redirects=True,
+            timeout=120,
+            event_hooks=httpclient.httpx_response_limit_hooks(_MAX_GITHUB_ARCHIVE_BYTES),
+        ) as client:
             async with client.stream('GET', asset_url) as resp:
                 resp.raise_for_status()
                 content_length = resp.headers.get('content-length')
@@ -352,7 +358,14 @@ class SkillService:
         info = self._parse_github_skill_md_url(asset_url, owner=owner, repo=repo)
         archive_url = f'https://codeload.github.com/{owner}/{repo}/zip/{quote(info["ref"], safe="/")}'
         archive_bytes = await self._download_github_asset(archive_url)
+        return await asyncio.to_thread(self._build_github_skill_directory_zip, archive_bytes, info)
 
+    def _build_github_skill_directory_zip(
+        self,
+        archive_bytes: bytes,
+        info: dict[str, str],
+    ) -> tuple[bytes, str, str]:
+        """Validate and repack a GitHub skill archive outside the event loop."""
         try:
             source_archive = zipfile.ZipFile(io.BytesIO(archive_bytes), 'r')
         except zipfile.BadZipFile as exc:

src/langbot/pkg/api/http/service/space.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from collections import OrderedDict
+
 from langbot.pkg.utils import httpclient
 import typing
 import datetime
@@ -11,6 +13,10 @@ from ....entity.persistence import user
 from ....entity.dto.space_model import SpaceModel
 
 
+_CREDITS_CACHE_TTL_SECONDS = 60
+_CREDITS_CACHE_MAX_ENTRIES = 4096
+
+
 class SpaceService:
     """Service for interacting with LangBot Space API"""
 
@@ -19,7 +25,24 @@ class SpaceService:
 
     def __init__(self, ap: app.Application) -> None:
         self.ap = ap
-        self._credits_cache = {}
+        self._credits_cache = OrderedDict()
+
+    def _ordered_credits_cache(
+        self,
+    ) -> OrderedDict[str, tuple[int, float]]:
+        if not isinstance(self._credits_cache, OrderedDict):
+            # Preserve compatibility with tests and callers that seed the cache.
+            self._credits_cache = OrderedDict(self._credits_cache)
+        return self._credits_cache
+
+    def _prune_credits_cache(self, now: float) -> None:
+        cache = self._ordered_credits_cache()
+        while cache:
+            email = next(iter(cache))
+            _, cached_at = cache[email]
+            if now - cached_at < _CREDITS_CACHE_TTL_SECONDS:
+                break
+            cache.pop(email, None)
 
     def _get_space_config(self) -> typing.Dict[str, str]:
         """Get Space configuration from config file"""
@@ -107,8 +130,9 @@ class SpaceService:
             json={'code': code, 'instance_id': constants.instance_id},
         ) as response:
             if response.status != 200:
-                raise ValueError(f'Failed to exchange OAuth code: {await response.text()}')
-            data = await response.json()
+                error = await httpclient.read_text_limited(response)
+                raise ValueError(f'Failed to exchange OAuth code: {error}')
+            data = await httpclient.read_json_limited(response)
             if data.get('code') != 0:
                 raise ValueError(f'Failed to exchange OAuth code: {data.get("msg")}')
             return data.get('data', {})
@@ -123,8 +147,9 @@ class SpaceService:
             f'{space_url}/api/v1/accounts/token/refresh', json={'refresh_token': refresh_token}
         ) as response:
             if response.status != 200:
-                raise ValueError(f'Failed to refresh token: {await response.text()}')
-            data = await response.json()
+                error = await httpclient.read_text_limited(response)
+                raise ValueError(f'Failed to refresh token: {error}')
+            data = await httpclient.read_json_limited(response)
             if data.get('code') != 0:
                 raise ValueError(f'Failed to refresh token: {data.get("msg")}')
             return data.get('data', {})
@@ -139,8 +164,9 @@ class SpaceService:
             f'{space_url}/api/v1/accounts/me', headers={'Authorization': f'Bearer {access_token}'}
         ) as response:
             if response.status != 200:
-                raise ValueError(f'Failed to get user info: {await response.text()}')
-            data = await response.json()
+                error = await httpclient.read_text_limited(response)
+                raise ValueError(f'Failed to get user info: {error}')
+            data = await httpclient.read_json_limited(response)
             if data.get('code') != 0:
                 raise ValueError(f'Failed to get user info: {data.get("msg")}')
             return data.get('data', {})
@@ -156,11 +182,13 @@ class SpaceService:
 
     async def get_credits(self, user_email: str, force_refresh: bool = False) -> int | None:
         """Get Space credits for user with caching (60s TTL)"""
-        cache_ttl = 60
+        now = time.time()
+        cached_fallback = self._credits_cache.get(user_email)
+        self._prune_credits_cache(now)
 
         if not force_refresh and user_email in self._credits_cache:
             credits, ts = self._credits_cache[user_email]
-            if time.time() - ts < cache_ttl:
+            if now - ts < _CREDITS_CACHE_TTL_SECONDS:
                 return credits
 
         try:
@@ -169,10 +197,14 @@ class SpaceService:
                 return None
             credits = info.get('credits')
             if credits is not None:
-                self._credits_cache[user_email] = (credits, time.time())
+                cache = self._ordered_credits_cache()
+                cache.pop(user_email, None)
+                if len(cache) >= _CREDITS_CACHE_MAX_ENTRIES:
+                    cache.popitem(last=False)
+                cache[user_email] = (credits, time.time())
             return credits
         except Exception:
-            return self._credits_cache.get(user_email, (None, 0))[0]
+            return cached_fallback[0] if cached_fallback is not None else None
 
     async def get_models(self) -> typing.List[SpaceModel]:
         """Get models from Space"""
@@ -183,8 +215,9 @@ class SpaceService:
         session = httpclient.get_session()
         async with session.get(f'{space_url}/api/v1/models', params={'page_size': 100}) as response:
             if response.status != 200:
-                raise ValueError(f'Failed to get models: {await response.text()}')
-            data = await response.json()
+                error = await httpclient.read_text_limited(response)
+                raise ValueError(f'Failed to get models: {error}')
+            data = await httpclient.read_json_limited(response)
             if data.get('code') != 0:
                 raise ValueError(f'Failed to get models: {data.get("msg")}')
             models_data = data.get('data', {}).get('models', [])

src/langbot/pkg/api/http/service/user.py

@@ -7,6 +7,7 @@ import datetime
 import typing
 import asyncio
 import dataclasses
+import heapq
 import hashlib
 import secrets
 import time
@@ -19,11 +20,17 @@ from ....entity.persistence.workspace import MembershipRole, MembershipStatus, W
 from ....utils import constants
 from ....entity.errors import account as account_errors
 from ....workspace.collaboration import normalize_email
+from ....utils import bounded_executor
 
 if typing.TYPE_CHECKING:
     from ....core.app import Application
 
 
+_SPACE_OAUTH_STATE_MAX_ENTRIES = 4096
+_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR = 64
+_SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER = 4
+
+
 class AccountExistsLoginRequiredError(ValueError):
     code = 'account_exists_login_required'
 
@@ -54,14 +61,45 @@ class UserService:
     def __init__(self, ap: Application) -> None:
         self.ap = ap
         self._create_user_lock = asyncio.Lock()
-        self._password_hash_lock = asyncio.Semaphore(1)
+        self._password_hash_lock = asyncio.Lock()
         self._space_oauth_state_lock = asyncio.Lock()
         self._space_oauth_states: dict[str, tuple[str, str | None, float, str | None]] = {}
+        self._space_oauth_state_expiry_heap: list[tuple[float, str]] = []
 
     @staticmethod
     def _space_oauth_state_digest(state: str) -> str:
         return hashlib.sha256(state.encode('utf-8')).hexdigest()
 
+    def _prune_space_oauth_states(self, now: float) -> None:
+        while self._space_oauth_state_expiry_heap:
+            expires_at, digest = self._space_oauth_state_expiry_heap[0]
+            entry = self._space_oauth_states.get(digest)
+            if entry is None or entry[2] != expires_at:
+                heapq.heappop(self._space_oauth_state_expiry_heap)
+                continue
+            if expires_at > now:
+                break
+            heapq.heappop(self._space_oauth_state_expiry_heap)
+            self._space_oauth_states.pop(digest, None)
+
+        max_heap_entries = max(
+            _SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR,
+            len(self._space_oauth_states) * _SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER,
+        )
+        if len(self._space_oauth_state_expiry_heap) > max_heap_entries:
+            self._space_oauth_state_expiry_heap[:] = [
+                (entry[2], digest) for digest, entry in self._space_oauth_states.items()
+            ]
+            heapq.heapify(self._space_oauth_state_expiry_heap)
+
+    def _evict_earliest_space_oauth_state(self) -> None:
+        while self._space_oauth_state_expiry_heap:
+            expires_at, digest = heapq.heappop(self._space_oauth_state_expiry_heap)
+            entry = self._space_oauth_states.get(digest)
+            if entry is not None and entry[2] == expires_at:
+                self._space_oauth_states.pop(digest, None)
+                return
+
     async def issue_space_oauth_state(
         self,
         purpose: typing.Literal['login', 'bind'],
@@ -85,11 +123,14 @@ class UserService:
         expires_at = time.monotonic() + min(ttl_seconds, 600)
         async with self._space_oauth_state_lock:
             now = time.monotonic()
-            self._space_oauth_states = {key: value for key, value in self._space_oauth_states.items() if value[2] > now}
-            if len(self._space_oauth_states) >= 4096:
-                oldest = min(self._space_oauth_states, key=lambda key: self._space_oauth_states[key][2])
-                self._space_oauth_states.pop(oldest, None)
+            self._prune_space_oauth_states(now)
+            if len(self._space_oauth_states) >= _SPACE_OAUTH_STATE_MAX_ENTRIES:
+                self._evict_earliest_space_oauth_state()
             self._space_oauth_states[digest] = (purpose, account_uuid, expires_at, launch_workspace_uuid)
+            heapq.heappush(
+                self._space_oauth_state_expiry_heap,
+                (expires_at, digest),
+            )
         return raw_state
 
     async def consume_space_oauth_state_details(
@@ -129,8 +170,14 @@ class UserService:
         return consumed.account
 
     async def _hash_password(self, password: str) -> str:
+        if self._password_hash_lock.locked():
+            raise bounded_executor.BlockingWorkCapacityError(
+                'Password hashing capacity reached',
+                scope='system:authentication',
+            )
         async with self._password_hash_lock:
-            return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
+            with bounded_executor.blocking_work_scope('system:authentication'):
+                return await asyncio.to_thread(argon2.PasswordHasher().hash, password)
 
     def _require_local_directory(self) -> None:
         if self._uses_control_plane_directory():
@@ -143,8 +190,14 @@ class UserService:
         return bool(workspace_service is not None and workspace_service.policy.multi_workspace_enabled)
 
     async def _verify_password(self, hashed_password: str, password: str) -> None:
+        if self._password_hash_lock.locked():
+            raise bounded_executor.BlockingWorkCapacityError(
+                'Password hashing capacity reached',
+                scope='system:authentication',
+            )
         async with self._password_hash_lock:
-            await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
+            with bounded_executor.blocking_work_scope('system:authentication'):
+                await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password)
 
     async def _update_space_provider_for_account(self, account: typing.Any, api_key: str) -> None:
         """Refresh the OSS Workspace Space provider without guessing a SaaS Workspace.