mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-29 21:57:14 +00:00
feat(tenancy): add Workspace multi-tenant foundation (#2353)
* Document multi-tenant workspace architecture * Add OSS and commercial workspace boundaries * docs: redesign multi-tenant workspace architecture * feat(tenancy): implement workspace isolation * docs(tenancy): record verification evidence * docs(tenancy): revise single-instance SaaS topology * docs(tenancy): refine architecture options * docs: finalize cloud v2 multi-tenant decisions * feat(tenancy): establish cloud isolation foundations * feat(tenancy): harden shared cloud runtime boundaries * docs(tenancy): record final isolation verification * fix(tenancy): close isolation and permission gaps * docs(tenancy): record final isolation verification * feat(tenancy): connect cloud workspace control plane * fix(build): install git for pinned SDK * docs(cloud): update control plane verification * chore: update multi-tenant SDK pin * fix(cloud): skip legacy model sync during startup * test(cloud): preserve minimal model manager fixtures * fix(cloud): preserve authenticated account context * fix(cloud): reuse authenticated account for user info * feat(cloud): complete Workspace settings navigation * test(web): cover Workspace dropdown menu * feat(web): place workspace controls in sidebar * refactor(web): streamline workspace controls * style(web): format workspace layout test * fix(cloud): surface runtime and workspace plan status * fix(plugin): keep runtime identity stable across restarts * fix(ui): widen and center workspace switcher * fix(ui): hide roles from workspace switcher * fix(ui): align workspace switcher with sidebar entries * feat(workspace): add in-product collaboration and direct Cloud launch * style: format collaboration changes * fix(workspace): bind collaboration APIs to tenant UoW * fix(cloud): preserve Core-owned collaboration state * test(cloud): require Space identity for invite registration * feat(cloud): complete secure invitation experience * style(web): format invitation flows * fix(cloud): recover box runtime without unscoped skill reload * feat(oss): enforce invitation account and owner billing flows * style: format OSS account service * test(oss): cover invitation logout handoff * fix(oss): resolve workspace owner in scoped session * feat(cloud): harden multi-tenant runtime resources * fix(cloud): bound runtime restart storms * fix(cloud): eliminate periodic runtime CPU spikes * fix(cloud): enforce instance capacity ceilings * fix(cloud): scope public login capability discovery * fix(cloud): bound tenant maintenance and monitoring work * fix(runtime): bound tenant resource amplification * fix(deps): pin green multi-tenant plugin SDK * fix(cloud): handle unavailable skill capability * fix(security): require authentication for image file endpoint (H-2) - Changed /api/v1/files/image from AuthType.NONE to USER_TOKEN_OR_API_KEY - Added Permission.RESOURCE_VIEW requirement - Prevents unauthenticated cross-tenant file access via leaked keys - Fixes HIGH severity finding from multi-tenant security review docs: add comprehensive database migration guide - Complete migration steps for OSS → multi-tenant - Backup, execution, verification procedures - Rollback scenarios and recovery plans - Performance tuning recommendations * test: add comprehensive cross-tenant isolation tests Added 7 critical test scenarios for multi-tenant boundaries: - Cross-tenant bot access prevention - Viewer role read-only enforcement - Removed member immediate access revocation - Model provider credential isolation - WebSocket message isolation - Invitation token workspace scoping - Multi-workspace context validation These tests address P0-2 coverage gaps for: - workspaces.py (membership & invitation flows) - user.py (authentication & authorization) - websocket_chat.py (real-time isolation) - plugins.py (resource access control) docs: finalize database migration guide * fix(security): resolve M-1, M-2, M-3 security findings M-1: WebSocket authorization TOCTOU race (FIXED) - Changed _revalidate_websocket_authorization to return RequestContext - Ensures validated context is used immediately without race window - Prevents removed members from sending messages during revalidation gap M-2: Model Manager cache workspace isolation (VERIFIED) - Confirmed _CacheKey already uses 4-tuple: (instance, workspace, generation, resource) - Cache is properly scoped per workspace, no cross-tenant leakage possible - No code change needed, documented as working correctly M-3: Invitation lock workspace scoping (FIXED) - Changed lock key from token_digest to workspace_uuid:token_digest - Prevents DoS where attacker locks token in Workspace A to block Workspace B - Locks now isolated per workspace All MEDIUM severity findings from security review now resolved. * fix(cloud): unblock tenant CI and enforce knowledge quotas * fix(tenancy): scope rerank model sync --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
RockChinQ
@@ -0,0 +1,312 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import contextvars
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_MAX_WORKERS = 8
|
||||
DEFAULT_MAX_PENDING = 128
|
||||
DEFAULT_MAX_INFLIGHT_PER_SCOPE = 4
|
||||
HARD_MAX_WORKERS = 64
|
||||
HARD_MAX_PENDING = 4096
|
||||
BLOCKING_CLEANUP_SCOPE = 'system:cleanup'
|
||||
_CLEANUP_RETRY_INITIAL_SECONDS = 0.01
|
||||
_CLEANUP_RETRY_MAX_SECONDS = 0.25
|
||||
|
||||
_blocking_work_scope: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
'langbot_blocking_work_scope',
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
class BlockingWorkCapacityError(RuntimeError):
|
||||
"""Raised before unbounded blocking work can enter the executor queue."""
|
||||
|
||||
def __init__(self, message: str, *, scope: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.scope = scope
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def blocking_work_scope(scope: str | None):
|
||||
"""Attribute blocking submissions to one trusted tenant scope."""
|
||||
|
||||
normalized = str(scope).strip() if scope is not None else None
|
||||
if not normalized:
|
||||
yield
|
||||
return
|
||||
token = _blocking_work_scope.set(normalized)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_blocking_work_scope.reset(token)
|
||||
|
||||
|
||||
def current_blocking_work_scope() -> str | None:
|
||||
"""Return the active trusted blocking-work scope, if any."""
|
||||
|
||||
return _blocking_work_scope.get()
|
||||
|
||||
|
||||
async def run_blocking_atomic(
|
||||
fn: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Let an admitted filesystem operation finish before propagating cancel."""
|
||||
|
||||
task = asyncio.create_task(asyncio.to_thread(fn, *args, **kwargs))
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
raise
|
||||
|
||||
|
||||
async def run_blocking_cleanup(
|
||||
fn: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Wait for bounded executor capacity and complete cleanup atomically."""
|
||||
|
||||
retry_delay = _CLEANUP_RETRY_INITIAL_SECONDS
|
||||
while True:
|
||||
try:
|
||||
with blocking_work_scope(BLOCKING_CLEANUP_SCOPE):
|
||||
return await run_blocking_atomic(fn, *args, **kwargs)
|
||||
except BlockingWorkCapacityError as exc:
|
||||
if exc.scope != BLOCKING_CLEANUP_SCOPE:
|
||||
raise
|
||||
await asyncio.sleep(retry_delay)
|
||||
retry_delay = min(
|
||||
retry_delay * 2,
|
||||
_CLEANUP_RETRY_MAX_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
async def run_in_blocking_work_scope(
|
||||
coro,
|
||||
scope: str | None,
|
||||
):
|
||||
"""Run a coroutine with blocking-work fairness attribution."""
|
||||
|
||||
with blocking_work_scope(scope):
|
||||
return await coro
|
||||
|
||||
|
||||
def _bounded_integer(
|
||||
value: Any,
|
||||
*,
|
||||
name: str,
|
||||
minimum: int,
|
||||
maximum: int,
|
||||
) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f'{name} must be an integer')
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f'{name} must be an integer') from exc
|
||||
if parsed < minimum or parsed > maximum:
|
||||
raise ValueError(f'{name} must be between {minimum} and {maximum}')
|
||||
return parsed
|
||||
|
||||
|
||||
def _validated_limits(
|
||||
max_workers: Any,
|
||||
max_pending: Any,
|
||||
max_inflight_per_scope: Any | None,
|
||||
) -> tuple[int, int, int]:
|
||||
workers = _bounded_integer(
|
||||
max_workers,
|
||||
name='blocking_executor.max_workers',
|
||||
minimum=1,
|
||||
maximum=HARD_MAX_WORKERS,
|
||||
)
|
||||
pending = _bounded_integer(
|
||||
max_pending,
|
||||
name='blocking_executor.max_pending',
|
||||
minimum=0,
|
||||
maximum=HARD_MAX_PENDING,
|
||||
)
|
||||
fair_share = max(1, workers // 2)
|
||||
scope_limit = (
|
||||
min(DEFAULT_MAX_INFLIGHT_PER_SCOPE, fair_share)
|
||||
if max_inflight_per_scope is None
|
||||
else _bounded_integer(
|
||||
max_inflight_per_scope,
|
||||
name='blocking_executor.max_inflight_per_scope',
|
||||
minimum=1,
|
||||
maximum=HARD_MAX_PENDING,
|
||||
)
|
||||
)
|
||||
if scope_limit > fair_share:
|
||||
raise ValueError(f'blocking_executor.max_inflight_per_scope must not exceed half of max_workers ({fair_share})')
|
||||
return workers, pending, scope_limit
|
||||
|
||||
|
||||
class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
|
||||
"""Thread pool with a hard cap on running plus queued submissions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_workers: int = DEFAULT_MAX_WORKERS,
|
||||
max_pending: int = DEFAULT_MAX_PENDING,
|
||||
max_inflight_per_scope: int | None = None,
|
||||
thread_name_prefix: str = 'langbot-blocking',
|
||||
) -> None:
|
||||
max_workers, max_pending, max_inflight_per_scope = _validated_limits(
|
||||
max_workers,
|
||||
max_pending,
|
||||
max_inflight_per_scope,
|
||||
)
|
||||
super().__init__(
|
||||
max_workers=max_workers,
|
||||
thread_name_prefix=thread_name_prefix,
|
||||
)
|
||||
self.max_workers = max_workers
|
||||
self.max_pending = max_pending
|
||||
self.max_inflight_per_scope = max_inflight_per_scope
|
||||
self._capacity = threading.BoundedSemaphore(max_workers + max_pending)
|
||||
self._stats_lock = threading.Lock()
|
||||
self._inflight_by_scope: dict[str, int] = {}
|
||||
self._inflight = 0
|
||||
self._running = 0
|
||||
self._submitted_total = 0
|
||||
self._completed_total = 0
|
||||
self._rejected_total = 0
|
||||
self._global_rejected_total = 0
|
||||
self._scope_rejected_total = 0
|
||||
|
||||
def submit(
|
||||
self,
|
||||
fn: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> concurrent.futures.Future:
|
||||
scope = current_blocking_work_scope()
|
||||
if not self._capacity.acquire(blocking=False):
|
||||
with self._stats_lock:
|
||||
self._rejected_total += 1
|
||||
self._global_rejected_total += 1
|
||||
raise BlockingWorkCapacityError(
|
||||
'Blocking executor capacity reached',
|
||||
scope=scope,
|
||||
)
|
||||
|
||||
with self._stats_lock:
|
||||
if scope is not None and self._inflight_by_scope.get(scope, 0) >= self.max_inflight_per_scope:
|
||||
self._rejected_total += 1
|
||||
self._scope_rejected_total += 1
|
||||
self._capacity.release()
|
||||
raise BlockingWorkCapacityError(
|
||||
'Workspace blocking executor capacity reached',
|
||||
scope=scope,
|
||||
)
|
||||
self._inflight += 1
|
||||
self._submitted_total += 1
|
||||
if scope is not None:
|
||||
self._inflight_by_scope[scope] = self._inflight_by_scope.get(scope, 0) + 1
|
||||
|
||||
def run() -> Any:
|
||||
with self._stats_lock:
|
||||
self._running += 1
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
with self._stats_lock:
|
||||
self._running -= 1
|
||||
|
||||
try:
|
||||
future = super().submit(run)
|
||||
except BaseException:
|
||||
with self._stats_lock:
|
||||
self._inflight -= 1
|
||||
self._release_scope_locked(scope)
|
||||
self._capacity.release()
|
||||
raise
|
||||
|
||||
def complete(_future: concurrent.futures.Future) -> None:
|
||||
with self._stats_lock:
|
||||
self._inflight -= 1
|
||||
self._completed_total += 1
|
||||
self._release_scope_locked(scope)
|
||||
self._capacity.release()
|
||||
|
||||
future.add_done_callback(complete)
|
||||
return future
|
||||
|
||||
def _release_scope_locked(self, scope: str | None) -> None:
|
||||
if scope is None:
|
||||
return
|
||||
remaining = self._inflight_by_scope.get(scope, 0) - 1
|
||||
if remaining > 0:
|
||||
self._inflight_by_scope[scope] = remaining
|
||||
else:
|
||||
self._inflight_by_scope.pop(scope, None)
|
||||
|
||||
def snapshot(self) -> dict[str, int]:
|
||||
with self._stats_lock:
|
||||
inflight = self._inflight
|
||||
running = self._running
|
||||
return {
|
||||
'max_workers': self.max_workers,
|
||||
'max_pending': self.max_pending,
|
||||
'max_inflight_per_scope': self.max_inflight_per_scope,
|
||||
'inflight': inflight,
|
||||
'running': running,
|
||||
'pending': max(inflight - running, 0),
|
||||
'active_scopes': len(self._inflight_by_scope),
|
||||
'submitted_total': self._submitted_total,
|
||||
'completed_total': self._completed_total,
|
||||
'rejected_total': self._rejected_total,
|
||||
'global_rejected_total': self._global_rejected_total,
|
||||
'scope_rejected_total': self._scope_rejected_total,
|
||||
}
|
||||
|
||||
|
||||
def configure_bounded_default_executor(
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
*,
|
||||
max_workers: int = DEFAULT_MAX_WORKERS,
|
||||
max_pending: int = DEFAULT_MAX_PENDING,
|
||||
max_inflight_per_scope: int | None = None,
|
||||
thread_name_prefix: str = 'langbot-blocking',
|
||||
) -> BoundedThreadPoolExecutor:
|
||||
"""Install one bounded owner for every ``asyncio.to_thread`` call."""
|
||||
|
||||
max_workers, max_pending, max_inflight_per_scope = _validated_limits(
|
||||
max_workers,
|
||||
max_pending,
|
||||
max_inflight_per_scope,
|
||||
)
|
||||
existing = getattr(loop, '_default_executor', None)
|
||||
if isinstance(existing, BoundedThreadPoolExecutor):
|
||||
if (
|
||||
existing.max_workers != max_workers
|
||||
or existing.max_pending != max_pending
|
||||
or existing.max_inflight_per_scope != max_inflight_per_scope
|
||||
):
|
||||
raise RuntimeError('The blocking executor is already configured with different limits')
|
||||
return existing
|
||||
if existing is not None:
|
||||
raise RuntimeError('The event loop default executor was initialized before LangBot resource limits')
|
||||
|
||||
executor = BoundedThreadPoolExecutor(
|
||||
max_workers=max_workers,
|
||||
max_pending=max_pending,
|
||||
max_inflight_per_scope=max_inflight_per_scope,
|
||||
thread_name_prefix=thread_name_prefix,
|
||||
)
|
||||
loop.set_default_executor(executor)
|
||||
return executor
|
||||
Reference in New Issue
Block a user