mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-28 13:17:14 +00:00
feat(tenancy): add Workspace multi-tenant foundation (#2353)
* Document multi-tenant workspace architecture * Add OSS and commercial workspace boundaries * docs: redesign multi-tenant workspace architecture * feat(tenancy): implement workspace isolation * docs(tenancy): record verification evidence * docs(tenancy): revise single-instance SaaS topology * docs(tenancy): refine architecture options * docs: finalize cloud v2 multi-tenant decisions * feat(tenancy): establish cloud isolation foundations * feat(tenancy): harden shared cloud runtime boundaries * docs(tenancy): record final isolation verification * fix(tenancy): close isolation and permission gaps * docs(tenancy): record final isolation verification * feat(tenancy): connect cloud workspace control plane * fix(build): install git for pinned SDK * docs(cloud): update control plane verification * chore: update multi-tenant SDK pin * fix(cloud): skip legacy model sync during startup * test(cloud): preserve minimal model manager fixtures * fix(cloud): preserve authenticated account context * fix(cloud): reuse authenticated account for user info * feat(cloud): complete Workspace settings navigation * test(web): cover Workspace dropdown menu * feat(web): place workspace controls in sidebar * refactor(web): streamline workspace controls * style(web): format workspace layout test * fix(cloud): surface runtime and workspace plan status * fix(plugin): keep runtime identity stable across restarts * fix(ui): widen and center workspace switcher * fix(ui): hide roles from workspace switcher * fix(ui): align workspace switcher with sidebar entries * feat(workspace): add in-product collaboration and direct Cloud launch * style: format collaboration changes * fix(workspace): bind collaboration APIs to tenant UoW * fix(cloud): preserve Core-owned collaboration state * test(cloud): require Space identity for invite registration * feat(cloud): complete secure invitation experience * style(web): format invitation flows * fix(cloud): recover box runtime without unscoped skill reload * feat(oss): enforce invitation account and owner billing flows * style: format OSS account service * test(oss): cover invitation logout handoff * fix(oss): resolve workspace owner in scoped session * feat(cloud): harden multi-tenant runtime resources * fix(cloud): bound runtime restart storms * fix(cloud): eliminate periodic runtime CPU spikes * fix(cloud): enforce instance capacity ceilings * fix(cloud): scope public login capability discovery * fix(cloud): bound tenant maintenance and monitoring work * fix(runtime): bound tenant resource amplification * fix(deps): pin green multi-tenant plugin SDK * fix(cloud): handle unavailable skill capability * fix(security): require authentication for image file endpoint (H-2) - Changed /api/v1/files/image from AuthType.NONE to USER_TOKEN_OR_API_KEY - Added Permission.RESOURCE_VIEW requirement - Prevents unauthenticated cross-tenant file access via leaked keys - Fixes HIGH severity finding from multi-tenant security review docs: add comprehensive database migration guide - Complete migration steps for OSS → multi-tenant - Backup, execution, verification procedures - Rollback scenarios and recovery plans - Performance tuning recommendations * test: add comprehensive cross-tenant isolation tests Added 7 critical test scenarios for multi-tenant boundaries: - Cross-tenant bot access prevention - Viewer role read-only enforcement - Removed member immediate access revocation - Model provider credential isolation - WebSocket message isolation - Invitation token workspace scoping - Multi-workspace context validation These tests address P0-2 coverage gaps for: - workspaces.py (membership & invitation flows) - user.py (authentication & authorization) - websocket_chat.py (real-time isolation) - plugins.py (resource access control) docs: finalize database migration guide * fix(security): resolve M-1, M-2, M-3 security findings M-1: WebSocket authorization TOCTOU race (FIXED) - Changed _revalidate_websocket_authorization to return RequestContext - Ensures validated context is used immediately without race window - Prevents removed members from sending messages during revalidation gap M-2: Model Manager cache workspace isolation (VERIFIED) - Confirmed _CacheKey already uses 4-tuple: (instance, workspace, generation, resource) - Cache is properly scoped per workspace, no cross-tenant leakage possible - No code change needed, documented as working correctly M-3: Invitation lock workspace scoping (FIXED) - Changed lock key from token_digest to workspace_uuid:token_digest - Prevents DoS where attacker locks token in Workspace A to block Workspace B - Locks now isolated per workspace All MEDIUM severity findings from security review now resolved. * fix(cloud): unblock tenant CI and enforce knowledge quotas * fix(tenancy): scope rerank model sync --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
RockChinQ
@@ -0,0 +1,312 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import contextvars
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_MAX_WORKERS = 8
|
||||
DEFAULT_MAX_PENDING = 128
|
||||
DEFAULT_MAX_INFLIGHT_PER_SCOPE = 4
|
||||
HARD_MAX_WORKERS = 64
|
||||
HARD_MAX_PENDING = 4096
|
||||
BLOCKING_CLEANUP_SCOPE = 'system:cleanup'
|
||||
_CLEANUP_RETRY_INITIAL_SECONDS = 0.01
|
||||
_CLEANUP_RETRY_MAX_SECONDS = 0.25
|
||||
|
||||
_blocking_work_scope: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
'langbot_blocking_work_scope',
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
class BlockingWorkCapacityError(RuntimeError):
|
||||
"""Raised before unbounded blocking work can enter the executor queue."""
|
||||
|
||||
def __init__(self, message: str, *, scope: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.scope = scope
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def blocking_work_scope(scope: str | None):
|
||||
"""Attribute blocking submissions to one trusted tenant scope."""
|
||||
|
||||
normalized = str(scope).strip() if scope is not None else None
|
||||
if not normalized:
|
||||
yield
|
||||
return
|
||||
token = _blocking_work_scope.set(normalized)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_blocking_work_scope.reset(token)
|
||||
|
||||
|
||||
def current_blocking_work_scope() -> str | None:
|
||||
"""Return the active trusted blocking-work scope, if any."""
|
||||
|
||||
return _blocking_work_scope.get()
|
||||
|
||||
|
||||
async def run_blocking_atomic(
|
||||
fn: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Let an admitted filesystem operation finish before propagating cancel."""
|
||||
|
||||
task = asyncio.create_task(asyncio.to_thread(fn, *args, **kwargs))
|
||||
try:
|
||||
return await asyncio.shield(task)
|
||||
except asyncio.CancelledError:
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
raise
|
||||
|
||||
|
||||
async def run_blocking_cleanup(
|
||||
fn: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Wait for bounded executor capacity and complete cleanup atomically."""
|
||||
|
||||
retry_delay = _CLEANUP_RETRY_INITIAL_SECONDS
|
||||
while True:
|
||||
try:
|
||||
with blocking_work_scope(BLOCKING_CLEANUP_SCOPE):
|
||||
return await run_blocking_atomic(fn, *args, **kwargs)
|
||||
except BlockingWorkCapacityError as exc:
|
||||
if exc.scope != BLOCKING_CLEANUP_SCOPE:
|
||||
raise
|
||||
await asyncio.sleep(retry_delay)
|
||||
retry_delay = min(
|
||||
retry_delay * 2,
|
||||
_CLEANUP_RETRY_MAX_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
async def run_in_blocking_work_scope(
|
||||
coro,
|
||||
scope: str | None,
|
||||
):
|
||||
"""Run a coroutine with blocking-work fairness attribution."""
|
||||
|
||||
with blocking_work_scope(scope):
|
||||
return await coro
|
||||
|
||||
|
||||
def _bounded_integer(
|
||||
value: Any,
|
||||
*,
|
||||
name: str,
|
||||
minimum: int,
|
||||
maximum: int,
|
||||
) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f'{name} must be an integer')
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f'{name} must be an integer') from exc
|
||||
if parsed < minimum or parsed > maximum:
|
||||
raise ValueError(f'{name} must be between {minimum} and {maximum}')
|
||||
return parsed
|
||||
|
||||
|
||||
def _validated_limits(
|
||||
max_workers: Any,
|
||||
max_pending: Any,
|
||||
max_inflight_per_scope: Any | None,
|
||||
) -> tuple[int, int, int]:
|
||||
workers = _bounded_integer(
|
||||
max_workers,
|
||||
name='blocking_executor.max_workers',
|
||||
minimum=1,
|
||||
maximum=HARD_MAX_WORKERS,
|
||||
)
|
||||
pending = _bounded_integer(
|
||||
max_pending,
|
||||
name='blocking_executor.max_pending',
|
||||
minimum=0,
|
||||
maximum=HARD_MAX_PENDING,
|
||||
)
|
||||
fair_share = max(1, workers // 2)
|
||||
scope_limit = (
|
||||
min(DEFAULT_MAX_INFLIGHT_PER_SCOPE, fair_share)
|
||||
if max_inflight_per_scope is None
|
||||
else _bounded_integer(
|
||||
max_inflight_per_scope,
|
||||
name='blocking_executor.max_inflight_per_scope',
|
||||
minimum=1,
|
||||
maximum=HARD_MAX_PENDING,
|
||||
)
|
||||
)
|
||||
if scope_limit > fair_share:
|
||||
raise ValueError(f'blocking_executor.max_inflight_per_scope must not exceed half of max_workers ({fair_share})')
|
||||
return workers, pending, scope_limit
|
||||
|
||||
|
||||
class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
|
||||
"""Thread pool with a hard cap on running plus queued submissions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
max_workers: int = DEFAULT_MAX_WORKERS,
|
||||
max_pending: int = DEFAULT_MAX_PENDING,
|
||||
max_inflight_per_scope: int | None = None,
|
||||
thread_name_prefix: str = 'langbot-blocking',
|
||||
) -> None:
|
||||
max_workers, max_pending, max_inflight_per_scope = _validated_limits(
|
||||
max_workers,
|
||||
max_pending,
|
||||
max_inflight_per_scope,
|
||||
)
|
||||
super().__init__(
|
||||
max_workers=max_workers,
|
||||
thread_name_prefix=thread_name_prefix,
|
||||
)
|
||||
self.max_workers = max_workers
|
||||
self.max_pending = max_pending
|
||||
self.max_inflight_per_scope = max_inflight_per_scope
|
||||
self._capacity = threading.BoundedSemaphore(max_workers + max_pending)
|
||||
self._stats_lock = threading.Lock()
|
||||
self._inflight_by_scope: dict[str, int] = {}
|
||||
self._inflight = 0
|
||||
self._running = 0
|
||||
self._submitted_total = 0
|
||||
self._completed_total = 0
|
||||
self._rejected_total = 0
|
||||
self._global_rejected_total = 0
|
||||
self._scope_rejected_total = 0
|
||||
|
||||
def submit(
|
||||
self,
|
||||
fn: Callable[..., Any],
|
||||
/,
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> concurrent.futures.Future:
|
||||
scope = current_blocking_work_scope()
|
||||
if not self._capacity.acquire(blocking=False):
|
||||
with self._stats_lock:
|
||||
self._rejected_total += 1
|
||||
self._global_rejected_total += 1
|
||||
raise BlockingWorkCapacityError(
|
||||
'Blocking executor capacity reached',
|
||||
scope=scope,
|
||||
)
|
||||
|
||||
with self._stats_lock:
|
||||
if scope is not None and self._inflight_by_scope.get(scope, 0) >= self.max_inflight_per_scope:
|
||||
self._rejected_total += 1
|
||||
self._scope_rejected_total += 1
|
||||
self._capacity.release()
|
||||
raise BlockingWorkCapacityError(
|
||||
'Workspace blocking executor capacity reached',
|
||||
scope=scope,
|
||||
)
|
||||
self._inflight += 1
|
||||
self._submitted_total += 1
|
||||
if scope is not None:
|
||||
self._inflight_by_scope[scope] = self._inflight_by_scope.get(scope, 0) + 1
|
||||
|
||||
def run() -> Any:
|
||||
with self._stats_lock:
|
||||
self._running += 1
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
with self._stats_lock:
|
||||
self._running -= 1
|
||||
|
||||
try:
|
||||
future = super().submit(run)
|
||||
except BaseException:
|
||||
with self._stats_lock:
|
||||
self._inflight -= 1
|
||||
self._release_scope_locked(scope)
|
||||
self._capacity.release()
|
||||
raise
|
||||
|
||||
def complete(_future: concurrent.futures.Future) -> None:
|
||||
with self._stats_lock:
|
||||
self._inflight -= 1
|
||||
self._completed_total += 1
|
||||
self._release_scope_locked(scope)
|
||||
self._capacity.release()
|
||||
|
||||
future.add_done_callback(complete)
|
||||
return future
|
||||
|
||||
def _release_scope_locked(self, scope: str | None) -> None:
|
||||
if scope is None:
|
||||
return
|
||||
remaining = self._inflight_by_scope.get(scope, 0) - 1
|
||||
if remaining > 0:
|
||||
self._inflight_by_scope[scope] = remaining
|
||||
else:
|
||||
self._inflight_by_scope.pop(scope, None)
|
||||
|
||||
def snapshot(self) -> dict[str, int]:
|
||||
with self._stats_lock:
|
||||
inflight = self._inflight
|
||||
running = self._running
|
||||
return {
|
||||
'max_workers': self.max_workers,
|
||||
'max_pending': self.max_pending,
|
||||
'max_inflight_per_scope': self.max_inflight_per_scope,
|
||||
'inflight': inflight,
|
||||
'running': running,
|
||||
'pending': max(inflight - running, 0),
|
||||
'active_scopes': len(self._inflight_by_scope),
|
||||
'submitted_total': self._submitted_total,
|
||||
'completed_total': self._completed_total,
|
||||
'rejected_total': self._rejected_total,
|
||||
'global_rejected_total': self._global_rejected_total,
|
||||
'scope_rejected_total': self._scope_rejected_total,
|
||||
}
|
||||
|
||||
|
||||
def configure_bounded_default_executor(
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
*,
|
||||
max_workers: int = DEFAULT_MAX_WORKERS,
|
||||
max_pending: int = DEFAULT_MAX_PENDING,
|
||||
max_inflight_per_scope: int | None = None,
|
||||
thread_name_prefix: str = 'langbot-blocking',
|
||||
) -> BoundedThreadPoolExecutor:
|
||||
"""Install one bounded owner for every ``asyncio.to_thread`` call."""
|
||||
|
||||
max_workers, max_pending, max_inflight_per_scope = _validated_limits(
|
||||
max_workers,
|
||||
max_pending,
|
||||
max_inflight_per_scope,
|
||||
)
|
||||
existing = getattr(loop, '_default_executor', None)
|
||||
if isinstance(existing, BoundedThreadPoolExecutor):
|
||||
if (
|
||||
existing.max_workers != max_workers
|
||||
or existing.max_pending != max_pending
|
||||
or existing.max_inflight_per_scope != max_inflight_per_scope
|
||||
):
|
||||
raise RuntimeError('The blocking executor is already configured with different limits')
|
||||
return existing
|
||||
if existing is not None:
|
||||
raise RuntimeError('The event loop default executor was initialized before LangBot resource limits')
|
||||
|
||||
executor = BoundedThreadPoolExecutor(
|
||||
max_workers=max_workers,
|
||||
max_pending=max_pending,
|
||||
max_inflight_per_scope=max_inflight_per_scope,
|
||||
thread_name_prefix=thread_name_prefix,
|
||||
)
|
||||
loop.set_default_executor(executor)
|
||||
return executor
|
||||
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import math
|
||||
from collections import deque
|
||||
|
||||
|
||||
DEFAULT_SAMPLE_INTERVAL_SECONDS = 1.0
|
||||
DEFAULT_RECENT_SAMPLE_COUNT = 120
|
||||
|
||||
|
||||
class EventLoopLagMonitor:
|
||||
"""Measure event-loop scheduling delay with fixed, bounded state."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
sample_interval_seconds: float = DEFAULT_SAMPLE_INTERVAL_SECONDS,
|
||||
recent_sample_count: int = DEFAULT_RECENT_SAMPLE_COUNT,
|
||||
) -> None:
|
||||
interval = float(sample_interval_seconds)
|
||||
if not math.isfinite(interval) or interval <= 0:
|
||||
raise ValueError('sample_interval_seconds must be greater than zero')
|
||||
sample_count = int(recent_sample_count)
|
||||
if sample_count < 2 or sample_count > 3600:
|
||||
raise ValueError('recent_sample_count must be between 2 and 3600')
|
||||
self.sample_interval_seconds = interval
|
||||
self.recent_sample_count = sample_count
|
||||
self._recent_lag_ms: deque[float] = deque(maxlen=sample_count)
|
||||
self._samples_total = 0
|
||||
self._max_lag_ms = 0.0
|
||||
self._last_lag_ms = 0.0
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
|
||||
@property
|
||||
def running(self) -> bool:
|
||||
return self._task is not None and not self._task.done()
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start sampling on the current event loop; repeated calls are safe."""
|
||||
|
||||
if self.running:
|
||||
return
|
||||
self._task = asyncio.create_task(
|
||||
self._run(),
|
||||
name='event-loop-lag-monitor',
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Cancel and await the owned sampler task."""
|
||||
|
||||
task = self._task
|
||||
self._task = None
|
||||
if task is None:
|
||||
return
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
async def _run(self) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
expected_at = loop.time() + self.sample_interval_seconds
|
||||
while True:
|
||||
await asyncio.sleep(max(expected_at - loop.time(), 0.0))
|
||||
observed_at = loop.time()
|
||||
self._record_lag_seconds(max(observed_at - expected_at, 0.0))
|
||||
# One observation captures a long stall; do not replay every
|
||||
# missed interval in a tight loop after the scheduler recovers.
|
||||
expected_at = observed_at + self.sample_interval_seconds
|
||||
|
||||
def _record_lag_seconds(self, lag_seconds: float) -> None:
|
||||
lag_ms = max(float(lag_seconds), 0.0) * 1000
|
||||
self._last_lag_ms = lag_ms
|
||||
self._max_lag_ms = max(self._max_lag_ms, lag_ms)
|
||||
self._recent_lag_ms.append(lag_ms)
|
||||
self._samples_total += 1
|
||||
|
||||
def snapshot(self) -> dict[str, int | float | bool]:
|
||||
"""Return aggregate metrics without exposing task or tenant state."""
|
||||
|
||||
recent = sorted(self._recent_lag_ms)
|
||||
if recent:
|
||||
p95_index = max(math.ceil(len(recent) * 0.95) - 1, 0)
|
||||
recent_p95_ms = recent[p95_index]
|
||||
recent_max_ms = recent[-1]
|
||||
else:
|
||||
recent_p95_ms = 0.0
|
||||
recent_max_ms = 0.0
|
||||
return {
|
||||
'running': self.running,
|
||||
'samples_total': self._samples_total,
|
||||
'last_lag_ms': self._last_lag_ms,
|
||||
'recent_p95_lag_ms': recent_p95_ms,
|
||||
'recent_max_lag_ms': recent_max_ms,
|
||||
'max_lag_ms': self._max_lag_ms,
|
||||
}
|
||||
@@ -11,9 +11,76 @@ reuses the same underlying SSL context and connection pool.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import typing
|
||||
|
||||
import aiohttp
|
||||
import httpx
|
||||
|
||||
_sessions: dict[str, aiohttp.ClientSession] = {}
|
||||
DEFAULT_REMOTE_BODY_LIMIT = 10 * 1024 * 1024
|
||||
|
||||
|
||||
class RemoteResponseTooLargeError(ValueError):
|
||||
"""Raised before an untrusted remote response can exhaust process memory."""
|
||||
|
||||
|
||||
class _LimitedHTTPXAsyncByteStream(httpx.AsyncByteStream):
|
||||
def __init__(self, inner: httpx.AsyncByteStream, max_bytes: int) -> None:
|
||||
self._inner = inner
|
||||
self._max_bytes = max_bytes
|
||||
self._read_bytes = 0
|
||||
|
||||
async def __aiter__(self):
|
||||
try:
|
||||
async for chunk in self._inner:
|
||||
self._read_bytes += len(chunk)
|
||||
if self._read_bytes > self._max_bytes:
|
||||
raise RemoteResponseTooLargeError(f'Remote response exceeds the {self._max_bytes}-byte limit')
|
||||
yield chunk
|
||||
except BaseException:
|
||||
# HTTPX only closes a response after normal stream exhaustion. If
|
||||
# this limiter raises (or its consumer is cancelled), explicitly
|
||||
# release the underlying connection before propagating the original
|
||||
# failure so persistent clients cannot accumulate stranded streams.
|
||||
try:
|
||||
await self._inner.aclose()
|
||||
except BaseException:
|
||||
pass
|
||||
raise
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._inner.aclose()
|
||||
|
||||
|
||||
def httpx_response_limit_hooks(
|
||||
max_bytes: int = DEFAULT_REMOTE_BODY_LIMIT,
|
||||
) -> dict[str, list]:
|
||||
"""Return hooks that cap HTTPX bodies before its automatic buffering."""
|
||||
|
||||
max_bytes = max(int(max_bytes), 1)
|
||||
|
||||
async def limit_response(response: httpx.Response) -> None:
|
||||
content_length = response.headers.get('Content-Length')
|
||||
if content_length is not None:
|
||||
try:
|
||||
declared_size = int(content_length)
|
||||
except (TypeError, ValueError):
|
||||
declared_size = None
|
||||
if declared_size is not None and declared_size > max_bytes:
|
||||
await response.aclose()
|
||||
raise RemoteResponseTooLargeError(f'Remote response exceeds the {max_bytes}-byte limit')
|
||||
|
||||
if response.is_stream_consumed:
|
||||
if len(response.content) > max_bytes:
|
||||
await response.aclose()
|
||||
raise RemoteResponseTooLargeError(f'Remote response exceeds the {max_bytes}-byte limit')
|
||||
return
|
||||
response.stream = _LimitedHTTPXAsyncByteStream(response.stream, max_bytes)
|
||||
|
||||
return {'response': [limit_response]}
|
||||
|
||||
|
||||
def get_session(*, trust_env: bool = False) -> aiohttp.ClientSession:
|
||||
@@ -29,7 +96,13 @@ def get_session(*, trust_env: bool = False) -> aiohttp.ClientSession:
|
||||
|
||||
session = _sessions.get(key)
|
||||
if session is None or session.closed:
|
||||
session = aiohttp.ClientSession(trust_env=trust_env)
|
||||
# Shared transport pools must never share upstream cookie state across
|
||||
# Workspace-scoped requests. Callers that need a stateful cookie jar
|
||||
# must own a dedicated session instead of using this global pool.
|
||||
session = aiohttp.ClientSession(
|
||||
trust_env=trust_env,
|
||||
cookie_jar=aiohttp.DummyCookieJar(),
|
||||
)
|
||||
_sessions[key] = session
|
||||
|
||||
return session
|
||||
@@ -41,3 +114,69 @@ async def close_all():
|
||||
if not session.closed:
|
||||
await session.close()
|
||||
_sessions.clear()
|
||||
|
||||
|
||||
async def read_limited(
|
||||
response: aiohttp.ClientResponse,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_REMOTE_BODY_LIMIT,
|
||||
) -> bytes:
|
||||
"""Read an HTTP response incrementally with a strict byte limit."""
|
||||
|
||||
max_bytes = max(int(max_bytes), 1)
|
||||
content_length = response.headers.get('Content-Length')
|
||||
if content_length is not None:
|
||||
try:
|
||||
declared_size = int(content_length)
|
||||
except (TypeError, ValueError):
|
||||
declared_size = None
|
||||
if declared_size is not None and declared_size > max_bytes:
|
||||
raise RemoteResponseTooLargeError(f'Remote response exceeds the {max_bytes}-byte limit')
|
||||
|
||||
body = bytearray()
|
||||
async for chunk in response.content.iter_chunked(64 * 1024):
|
||||
body.extend(chunk)
|
||||
if len(body) > max_bytes:
|
||||
raise RemoteResponseTooLargeError(f'Remote response exceeds the {max_bytes}-byte limit')
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def read_text_limited(
|
||||
response: aiohttp.ClientResponse,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_REMOTE_BODY_LIMIT,
|
||||
) -> str:
|
||||
body = await read_limited(response, max_bytes=max_bytes)
|
||||
return body.decode(response.charset or 'utf-8', errors='replace')
|
||||
|
||||
|
||||
async def read_json_limited(
|
||||
response: aiohttp.ClientResponse,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_REMOTE_BODY_LIMIT,
|
||||
) -> typing.Any:
|
||||
body = await read_limited(response, max_bytes=max_bytes)
|
||||
return await asyncio.to_thread(json.loads, body)
|
||||
|
||||
|
||||
async def parse_json_response(response: typing.Any) -> typing.Any:
|
||||
"""Parse an already bounded HTTP response without blocking the event loop."""
|
||||
|
||||
parsed = await asyncio.to_thread(response.json)
|
||||
if inspect.isawaitable(parsed):
|
||||
parsed = await parsed
|
||||
return parsed
|
||||
|
||||
|
||||
async def response_text(
|
||||
response: typing.Any,
|
||||
*,
|
||||
max_chars: int = 4096,
|
||||
) -> str:
|
||||
"""Decode an already bounded response body off-loop and cap diagnostics."""
|
||||
|
||||
text = await asyncio.to_thread(lambda: str(response.text))
|
||||
max_chars = max(int(max_chars), 1)
|
||||
if len(text) <= max_chars:
|
||||
return text
|
||||
return f'{text[:max_chars]}... [truncated]'
|
||||
|
||||
@@ -8,10 +8,46 @@ import aiohttp
|
||||
|
||||
from langbot.pkg.utils import httpclient
|
||||
import PIL.Image
|
||||
import httpx
|
||||
|
||||
import asyncio
|
||||
|
||||
_INSECURE_SSL_CONTEXT = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
_INSECURE_SSL_CONTEXT.check_hostname = False
|
||||
_INSECURE_SSL_CONTEXT.verify_mode = ssl.CERT_NONE
|
||||
DEFAULT_BASE64_MEDIA_LIMIT = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def _detect_image_format(file_bytes: bytes) -> str:
|
||||
with PIL.Image.open(io.BytesIO(file_bytes)) as image:
|
||||
return str(image.format or 'jpeg').lower()
|
||||
|
||||
|
||||
def _decode_base64_limited(value: str, max_bytes: int) -> bytes:
|
||||
max_bytes = max(int(max_bytes), 1)
|
||||
max_encoded_chars = 4 * ((max_bytes + 2) // 3) + 4
|
||||
if len(value) > max_encoded_chars:
|
||||
raise ValueError(f'Base64 media exceeds the {max_bytes}-byte limit')
|
||||
decoded = base64.b64decode(value)
|
||||
if len(decoded) > max_bytes:
|
||||
raise ValueError(f'Base64 media exceeds the {max_bytes}-byte limit')
|
||||
return decoded
|
||||
|
||||
|
||||
async def decode_base64_limited(
|
||||
value: str,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_BASE64_MEDIA_LIMIT,
|
||||
) -> bytes:
|
||||
"""Decode bounded media outside the event loop."""
|
||||
|
||||
return await asyncio.to_thread(_decode_base64_limited, value, max_bytes)
|
||||
|
||||
|
||||
async def encode_base64(data: bytes) -> str:
|
||||
"""Encode a bounded byte payload outside the event loop."""
|
||||
|
||||
return (await asyncio.to_thread(base64.b64encode, data)).decode('utf-8')
|
||||
|
||||
|
||||
async def get_gewechat_image_base64(
|
||||
gewechat_url: str,
|
||||
@@ -59,10 +95,10 @@ async def get_gewechat_image_base64(
|
||||
timeout=timeout,
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
# print(response)
|
||||
raise Exception(f'获取gewechat图片下载失败: {await response.text()}')
|
||||
error = await httpclient.read_text_limited(response)
|
||||
raise Exception(f'获取gewechat图片下载失败: {error}')
|
||||
|
||||
resp_data = await response.json()
|
||||
resp_data = await httpclient.read_json_limited(response)
|
||||
if resp_data.get('ret') != 200:
|
||||
raise Exception(f'获取gewechat图片下载链接失败: {resp_data}')
|
||||
|
||||
@@ -80,9 +116,10 @@ async def get_gewechat_image_base64(
|
||||
try:
|
||||
async with session.get(download_url) as img_response:
|
||||
if img_response.status != 200:
|
||||
raise Exception(f'下载图片失败: {await img_response.text()}, URL: {download_url}')
|
||||
error = await httpclient.read_text_limited(img_response)
|
||||
raise Exception(f'下载图片失败: {error}, URL: {download_url}')
|
||||
|
||||
image_data = await img_response.read()
|
||||
image_data = await httpclient.read_limited(img_response)
|
||||
|
||||
content_type = img_response.headers.get('Content-Type', '')
|
||||
if content_type:
|
||||
@@ -90,7 +127,7 @@ async def get_gewechat_image_base64(
|
||||
else:
|
||||
image_format = file_url.split('.')[-1]
|
||||
|
||||
base64_str = base64.b64encode(image_data).decode('utf-8')
|
||||
base64_str = await encode_base64(image_data)
|
||||
|
||||
return base64_str, image_format
|
||||
except asyncio.TimeoutError:
|
||||
@@ -113,16 +150,13 @@ async def get_wecom_image_base64(pic_url: str) -> tuple[str, str]:
|
||||
raise Exception(f'Failed to download image: {response.status}')
|
||||
|
||||
# 读取图片数据
|
||||
image_data = await response.read()
|
||||
image_data = await httpclient.read_limited(response)
|
||||
|
||||
# 获取图片格式
|
||||
content_type = response.headers.get('Content-Type', '')
|
||||
image_format = content_type.split('/')[-1] # 例如 'image/jpeg' -> 'jpeg'
|
||||
|
||||
# 转换为 base64
|
||||
import base64
|
||||
|
||||
image_base64 = base64.b64encode(image_data).decode('utf-8')
|
||||
image_base64 = await encode_base64(image_data)
|
||||
|
||||
return image_base64, image_format
|
||||
|
||||
@@ -132,11 +166,11 @@ async def get_qq_official_image_base64(pic_url: str, content_type: str) -> tuple
|
||||
下载QQ官方图片，
|
||||
并且转换为base64格式
|
||||
"""
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.get(pic_url)
|
||||
response.raise_for_status() # 确保请求成功
|
||||
image_data = response.content
|
||||
base64_data = base64.b64encode(image_data).decode('utf-8')
|
||||
session = httpclient.get_session()
|
||||
async with session.get(pic_url) as response:
|
||||
response.raise_for_status()
|
||||
image_data = await httpclient.read_limited(response)
|
||||
base64_data = await encode_base64(image_data)
|
||||
|
||||
return f'data:{content_type};base64,{base64_data}'
|
||||
|
||||
@@ -153,19 +187,20 @@ async def get_qq_image_bytes(image_url: str, query: dict = {}) -> tuple[bytes, s
|
||||
"""[弃用]获取QQ图片的bytes"""
|
||||
image_url, query_in_url = get_qq_image_downloadable_url(image_url)
|
||||
query = {**query, **query_in_url}
|
||||
ssl_context = ssl.create_default_context()
|
||||
ssl_context.check_hostname = False
|
||||
ssl_context.verify_mode = ssl.CERT_NONE
|
||||
session = httpclient.get_session()
|
||||
async with session.get(image_url, params=query, ssl=ssl_context, timeout=aiohttp.ClientTimeout(total=30.0)) as resp:
|
||||
async with session.get(
|
||||
image_url,
|
||||
params=query,
|
||||
ssl=_INSECURE_SSL_CONTEXT,
|
||||
timeout=aiohttp.ClientTimeout(total=30.0),
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
file_bytes = await resp.read()
|
||||
file_bytes = await httpclient.read_limited(resp)
|
||||
content_type = resp.headers.get('Content-Type')
|
||||
if not content_type:
|
||||
image_format = 'jpeg'
|
||||
elif not content_type.startswith('image/'):
|
||||
pil_img = PIL.Image.open(io.BytesIO(file_bytes))
|
||||
image_format = pil_img.format.lower()
|
||||
image_format = await asyncio.to_thread(_detect_image_format, file_bytes)
|
||||
else:
|
||||
image_format = content_type.split('/')[-1]
|
||||
return file_bytes, image_format
|
||||
@@ -187,7 +222,7 @@ async def qq_image_url_to_base64(image_url: str) -> typing.Tuple[str, str]:
|
||||
|
||||
file_bytes, image_format = await get_qq_image_bytes(image_url, query)
|
||||
|
||||
base64_str = base64.b64encode(file_bytes).decode()
|
||||
base64_str = await encode_base64(file_bytes)
|
||||
|
||||
return base64_str, image_format
|
||||
|
||||
@@ -209,8 +244,8 @@ async def get_slack_image_to_base64(pic_url: str, bot_token: str):
|
||||
session = httpclient.get_session()
|
||||
async with session.get(pic_url, headers=headers) as resp:
|
||||
mime_type = resp.headers.get('Content-Type', 'application/octet-stream')
|
||||
file_bytes = await resp.read()
|
||||
base64_str = base64.b64encode(file_bytes).decode('utf-8')
|
||||
file_bytes = await httpclient.read_limited(resp)
|
||||
base64_str = await encode_base64(file_bytes)
|
||||
return f'data:{mime_type};base64,{base64_str}'
|
||||
except Exception as e:
|
||||
raise (e)
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
|
||||
LOG_PAGE_SIZE = 20
|
||||
MAX_CACHED_PAGES = 10
|
||||
MAX_LOG_LINE_CHARS = 20000
|
||||
|
||||
|
||||
class LogPage:
|
||||
@@ -40,6 +41,10 @@ class LogCache:
|
||||
|
||||
def add_log(self, log: str):
|
||||
"""添加日志"""
|
||||
log = str(log)
|
||||
if len(log) > MAX_LOG_LINE_CHARS:
|
||||
marker = '\n[log truncated]'
|
||||
log = log[: MAX_LOG_LINE_CHARS - len(marker)] + marker
|
||||
if self.log_pages[-1].add_log(log):
|
||||
self.log_pages.append(LogPage(number=self.log_pages[-1].number + 1))
|
||||
|
||||
|
||||
@@ -31,7 +31,11 @@ class ManagedRuntimeConnector:
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
self._closing = False
|
||||
|
||||
async def _start_runtime_subprocess(self, *args: str) -> None:
|
||||
async def _start_runtime_subprocess(
|
||||
self,
|
||||
*args: str,
|
||||
env_overrides: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Launch a local runtime as a subprocess of the current Python interpreter.
|
||||
|
||||
If a subprocess is already running (no *returncode* yet), this is a no-op.
|
||||
@@ -41,6 +45,8 @@ class ManagedRuntimeConnector:
|
||||
|
||||
python_path = sys.executable
|
||||
env = os.environ.copy()
|
||||
if env_overrides:
|
||||
env.update(env_overrides)
|
||||
self.runtime_subprocess = await asyncio.create_subprocess_exec(
|
||||
python_path,
|
||||
*args,
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
|
||||
import regex
|
||||
|
||||
|
||||
MAX_PATTERN_COUNT = 64
|
||||
MAX_PATTERN_CHARS = 1024
|
||||
MAX_INPUT_CHARS = 1024 * 1024
|
||||
MAX_REPLACEMENT_CHARS = 64
|
||||
MAX_MASKED_OUTPUT_CHARS = 2 * 1024 * 1024
|
||||
DEFAULT_OPERATION_TIMEOUT_SECONDS = 0.05
|
||||
|
||||
|
||||
class SafeRegexError(ValueError):
|
||||
"""Base class for rejected, invalid, or timed-out tenant regex work."""
|
||||
|
||||
|
||||
class SafeRegexLimitError(SafeRegexError):
|
||||
"""Raised when a regex operation exceeds a deterministic resource limit."""
|
||||
|
||||
|
||||
class SafeRegexTimeoutError(SafeRegexError):
|
||||
"""Raised when the regex engine exhausts the operation CPU budget."""
|
||||
|
||||
|
||||
def _validate_patterns(patterns: Sequence[str]) -> tuple[str, ...]:
|
||||
normalized = tuple(patterns)
|
||||
if len(normalized) > MAX_PATTERN_COUNT:
|
||||
raise SafeRegexLimitError(f'At most {MAX_PATTERN_COUNT} regex patterns are allowed')
|
||||
for pattern in normalized:
|
||||
if not isinstance(pattern, str):
|
||||
raise SafeRegexError('Regex patterns must be strings')
|
||||
if len(pattern) > MAX_PATTERN_CHARS:
|
||||
raise SafeRegexLimitError(f'Regex patterns may contain at most {MAX_PATTERN_CHARS} characters')
|
||||
return normalized
|
||||
|
||||
|
||||
def _validate_input(value: str) -> None:
|
||||
if not isinstance(value, str):
|
||||
raise SafeRegexError('Regex input must be a string')
|
||||
if len(value) > MAX_INPUT_CHARS:
|
||||
raise SafeRegexLimitError(f'Regex input may contain at most {MAX_INPUT_CHARS} characters')
|
||||
|
||||
|
||||
def _remaining_seconds(deadline: float) -> float:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise SafeRegexTimeoutError('Regex operation timed out')
|
||||
return remaining
|
||||
|
||||
|
||||
def _compile(pattern: str):
|
||||
try:
|
||||
return regex.compile(pattern)
|
||||
except regex.error as exc:
|
||||
raise SafeRegexError(f'Invalid regex: {exc}') from exc
|
||||
|
||||
|
||||
def _matches_any_sync(
|
||||
patterns: Sequence[str],
|
||||
value: str,
|
||||
*,
|
||||
mode: str,
|
||||
timeout_seconds: float,
|
||||
) -> bool:
|
||||
normalized_patterns = _validate_patterns(patterns)
|
||||
_validate_input(value)
|
||||
if mode not in {'match', 'search'}:
|
||||
raise ValueError(f'Unsupported safe regex mode: {mode}')
|
||||
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
try:
|
||||
for pattern in normalized_patterns:
|
||||
compiled = _compile(pattern)
|
||||
matcher = compiled.match if mode == 'match' else compiled.search
|
||||
if matcher(
|
||||
value,
|
||||
timeout=_remaining_seconds(deadline),
|
||||
concurrent=True,
|
||||
):
|
||||
return True
|
||||
except TimeoutError as exc:
|
||||
raise SafeRegexTimeoutError('Regex operation timed out') from exc
|
||||
return False
|
||||
|
||||
|
||||
async def matches_any(
|
||||
patterns: Sequence[str],
|
||||
value: str,
|
||||
*,
|
||||
mode: str = 'search',
|
||||
timeout_seconds: float = DEFAULT_OPERATION_TIMEOUT_SECONDS,
|
||||
) -> bool:
|
||||
"""Match untrusted patterns without blocking the shared event loop."""
|
||||
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError('timeout_seconds must be positive')
|
||||
return await asyncio.to_thread(
|
||||
_matches_any_sync,
|
||||
patterns,
|
||||
value,
|
||||
mode=mode,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
|
||||
|
||||
def _mask_patterns_sync(
|
||||
patterns: Sequence[str],
|
||||
value: str,
|
||||
*,
|
||||
mask: str,
|
||||
mask_word: str,
|
||||
timeout_seconds: float,
|
||||
) -> tuple[bool, str]:
|
||||
normalized_patterns = _validate_patterns(patterns)
|
||||
_validate_input(value)
|
||||
if len(mask) > MAX_REPLACEMENT_CHARS or len(mask_word) > MAX_REPLACEMENT_CHARS:
|
||||
raise SafeRegexLimitError(f'Regex replacements may contain at most {MAX_REPLACEMENT_CHARS} characters')
|
||||
|
||||
# Reject amplification before invoking a replacement callback. This is
|
||||
# deliberately conservative: a hostile replacement must not allocate tens
|
||||
# of megabytes before the post-operation output check can run.
|
||||
replacement_width = len(mask_word) if mask_word else len(mask)
|
||||
if replacement_width * max(1, len(value)) > MAX_MASKED_OUTPUT_CHARS:
|
||||
raise SafeRegexLimitError('Regex replacement could exceed the masked output limit')
|
||||
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
found = False
|
||||
current = value
|
||||
|
||||
def replace(match) -> str:
|
||||
nonlocal found
|
||||
found = True
|
||||
if mask_word:
|
||||
return mask_word
|
||||
return mask * len(match.group(0))
|
||||
|
||||
try:
|
||||
for pattern in normalized_patterns:
|
||||
compiled = _compile(pattern)
|
||||
current = compiled.sub(
|
||||
replace,
|
||||
current,
|
||||
timeout=_remaining_seconds(deadline),
|
||||
concurrent=True,
|
||||
)
|
||||
if len(current) > MAX_MASKED_OUTPUT_CHARS:
|
||||
raise SafeRegexLimitError('Regex replacement exceeded the masked output limit')
|
||||
except TimeoutError as exc:
|
||||
raise SafeRegexTimeoutError('Regex operation timed out') from exc
|
||||
return found, current
|
||||
|
||||
|
||||
async def mask_patterns(
|
||||
patterns: Sequence[str],
|
||||
value: str,
|
||||
*,
|
||||
mask: str,
|
||||
mask_word: str,
|
||||
timeout_seconds: float = DEFAULT_OPERATION_TIMEOUT_SECONDS,
|
||||
) -> tuple[bool, str]:
|
||||
"""Apply untrusted masking patterns with bounded CPU and output growth."""
|
||||
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError('timeout_seconds must be positive')
|
||||
return await asyncio.to_thread(
|
||||
_mask_patterns_sync,
|
||||
patterns,
|
||||
value,
|
||||
mask=mask,
|
||||
mask_word=mask_word,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
@@ -1,12 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import typing
|
||||
import logging
|
||||
|
||||
import requests
|
||||
|
||||
from ..core import app
|
||||
from . import constants
|
||||
from . import constants, httpclient
|
||||
|
||||
|
||||
class VersionManager:
|
||||
@@ -26,13 +27,14 @@ class VersionManager:
|
||||
async def get_release_list(self) -> list:
|
||||
"""Fetch release list from Space API (cached GitHub releases)."""
|
||||
try:
|
||||
rls_list_resp = requests.get(
|
||||
url='https://space.langbot.app/api/v1/dist/info/releases',
|
||||
rls_list_resp = await asyncio.to_thread(
|
||||
requests.get,
|
||||
'https://space.langbot.app/api/v1/dist/info/releases',
|
||||
proxies=self.ap.proxy_mgr.get_forward_proxies(),
|
||||
timeout=10,
|
||||
)
|
||||
rls_list_resp.raise_for_status()
|
||||
resp_json = rls_list_resp.json()
|
||||
resp_json = await httpclient.parse_json_response(rls_list_resp)
|
||||
if resp_json.get('code') == 0 and isinstance(resp_json.get('data'), list):
|
||||
return resp_json['data']
|
||||
self.ap.logger.warning(f'Failed to fetch release list: unexpected response: {resp_json.get("msg", "")}')
|
||||
|
||||
Reference in New Issue
Block a user