LangBot/src/langbot/pkg/utils/bounded_executor.py

from __future__ import annotations

import asyncio
import concurrent.futures
import contextlib
import contextvars
import threading
from collections.abc import Callable
from typing import Any


DEFAULT_MAX_WORKERS = 8
DEFAULT_MAX_PENDING = 128
DEFAULT_MAX_INFLIGHT_PER_SCOPE = 4
HARD_MAX_WORKERS = 64
HARD_MAX_PENDING = 4096
BLOCKING_CLEANUP_SCOPE = 'system:cleanup'
_CLEANUP_RETRY_INITIAL_SECONDS = 0.01
_CLEANUP_RETRY_MAX_SECONDS = 0.25

_blocking_work_scope: contextvars.ContextVar[str | None] = contextvars.ContextVar(
    'langbot_blocking_work_scope',
    default=None,
)


class BlockingWorkCapacityError(RuntimeError):
    """Raised before unbounded blocking work can enter the executor queue."""

    def __init__(self, message: str, *, scope: str | None = None) -> None:
        super().__init__(message)
        self.scope = scope


@contextlib.contextmanager
def blocking_work_scope(scope: str | None):
    """Attribute blocking submissions to one trusted tenant scope."""

    normalized = str(scope).strip() if scope is not None else None
    if not normalized:
        yield
        return
    token = _blocking_work_scope.set(normalized)
    try:
        yield
    finally:
        _blocking_work_scope.reset(token)


def current_blocking_work_scope() -> str | None:
    """Return the active trusted blocking-work scope, if any."""

    return _blocking_work_scope.get()


async def run_blocking_atomic(
    fn: Callable[..., Any],
    /,
    *args: Any,
    **kwargs: Any,
) -> Any:
    """Let an admitted filesystem operation finish before propagating cancel."""

    task = asyncio.create_task(asyncio.to_thread(fn, *args, **kwargs))
    try:
        return await asyncio.shield(task)
    except asyncio.CancelledError:
        await asyncio.gather(task, return_exceptions=True)
        raise


async def run_blocking_cleanup(
    fn: Callable[..., Any],
    /,
    *args: Any,
    **kwargs: Any,
) -> Any:
    """Wait for bounded executor capacity and complete cleanup atomically."""

    retry_delay = _CLEANUP_RETRY_INITIAL_SECONDS
    while True:
        try:
            with blocking_work_scope(BLOCKING_CLEANUP_SCOPE):
                return await run_blocking_atomic(fn, *args, **kwargs)
        except BlockingWorkCapacityError as exc:
            if exc.scope != BLOCKING_CLEANUP_SCOPE:
                raise
            await asyncio.sleep(retry_delay)
            retry_delay = min(
                retry_delay * 2,
                _CLEANUP_RETRY_MAX_SECONDS,
            )


async def run_in_blocking_work_scope(
    coro,
    scope: str | None,
):
    """Run a coroutine with blocking-work fairness attribution."""

    with blocking_work_scope(scope):
        return await coro


def _bounded_integer(
    value: Any,
    *,
    name: str,
    minimum: int,
    maximum: int,
) -> int:
    if isinstance(value, bool):
        raise ValueError(f'{name} must be an integer')
    try:
        parsed = int(value)
    except (TypeError, ValueError) as exc:
        raise ValueError(f'{name} must be an integer') from exc
    if parsed < minimum or parsed > maximum:
        raise ValueError(f'{name} must be between {minimum} and {maximum}')
    return parsed


def _validated_limits(
    max_workers: Any,
    max_pending: Any,
    max_inflight_per_scope: Any | None,
) -> tuple[int, int, int]:
    workers = _bounded_integer(
        max_workers,
        name='blocking_executor.max_workers',
        minimum=1,
        maximum=HARD_MAX_WORKERS,
    )
    pending = _bounded_integer(
        max_pending,
        name='blocking_executor.max_pending',
        minimum=0,
        maximum=HARD_MAX_PENDING,
    )
    fair_share = max(1, workers // 2)
    scope_limit = (
        min(DEFAULT_MAX_INFLIGHT_PER_SCOPE, fair_share)
        if max_inflight_per_scope is None
        else _bounded_integer(
            max_inflight_per_scope,
            name='blocking_executor.max_inflight_per_scope',
            minimum=1,
            maximum=HARD_MAX_PENDING,
        )
    )
    if scope_limit > fair_share:
        raise ValueError(f'blocking_executor.max_inflight_per_scope must not exceed half of max_workers ({fair_share})')
    return workers, pending, scope_limit


class BoundedThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
    """Thread pool with a hard cap on running plus queued submissions."""

    def __init__(
        self,
        *,
        max_workers: int = DEFAULT_MAX_WORKERS,
        max_pending: int = DEFAULT_MAX_PENDING,
        max_inflight_per_scope: int | None = None,
        thread_name_prefix: str = 'langbot-blocking',
    ) -> None:
        max_workers, max_pending, max_inflight_per_scope = _validated_limits(
            max_workers,
            max_pending,
            max_inflight_per_scope,
        )
        super().__init__(
            max_workers=max_workers,
            thread_name_prefix=thread_name_prefix,
        )
        self.max_workers = max_workers
        self.max_pending = max_pending
        self.max_inflight_per_scope = max_inflight_per_scope
        self._capacity = threading.BoundedSemaphore(max_workers + max_pending)
        self._stats_lock = threading.Lock()
        self._inflight_by_scope: dict[str, int] = {}
        self._inflight = 0
        self._running = 0
        self._submitted_total = 0
        self._completed_total = 0
        self._rejected_total = 0
        self._global_rejected_total = 0
        self._scope_rejected_total = 0

    def submit(
        self,
        fn: Callable[..., Any],
        /,
        *args: Any,
        **kwargs: Any,
    ) -> concurrent.futures.Future:
        scope = current_blocking_work_scope()
        if not self._capacity.acquire(blocking=False):
            with self._stats_lock:
                self._rejected_total += 1
                self._global_rejected_total += 1
            raise BlockingWorkCapacityError(
                'Blocking executor capacity reached',
                scope=scope,
            )

        with self._stats_lock:
            if scope is not None and self._inflight_by_scope.get(scope, 0) >= self.max_inflight_per_scope:
                self._rejected_total += 1
                self._scope_rejected_total += 1
                self._capacity.release()
                raise BlockingWorkCapacityError(
                    'Workspace blocking executor capacity reached',
                    scope=scope,
                )
            self._inflight += 1
            self._submitted_total += 1
            if scope is not None:
                self._inflight_by_scope[scope] = self._inflight_by_scope.get(scope, 0) + 1

        def run() -> Any:
            with self._stats_lock:
                self._running += 1
            try:
                return fn(*args, **kwargs)
            finally:
                with self._stats_lock:
                    self._running -= 1

        try:
            future = super().submit(run)
        except BaseException:
            with self._stats_lock:
                self._inflight -= 1
                self._release_scope_locked(scope)
            self._capacity.release()
            raise

        def complete(_future: concurrent.futures.Future) -> None:
            with self._stats_lock:
                self._inflight -= 1
                self._completed_total += 1
                self._release_scope_locked(scope)
            self._capacity.release()

        future.add_done_callback(complete)
        return future

    def _release_scope_locked(self, scope: str | None) -> None:
        if scope is None:
            return
        remaining = self._inflight_by_scope.get(scope, 0) - 1
        if remaining > 0:
            self._inflight_by_scope[scope] = remaining
        else:
            self._inflight_by_scope.pop(scope, None)

    def snapshot(self) -> dict[str, int]:
        with self._stats_lock:
            inflight = self._inflight
            running = self._running
            return {
                'max_workers': self.max_workers,
                'max_pending': self.max_pending,
                'max_inflight_per_scope': self.max_inflight_per_scope,
                'inflight': inflight,
                'running': running,
                'pending': max(inflight - running, 0),
                'active_scopes': len(self._inflight_by_scope),
                'submitted_total': self._submitted_total,
                'completed_total': self._completed_total,
                'rejected_total': self._rejected_total,
                'global_rejected_total': self._global_rejected_total,
                'scope_rejected_total': self._scope_rejected_total,
            }


def configure_bounded_default_executor(
    loop: asyncio.AbstractEventLoop,
    *,
    max_workers: int = DEFAULT_MAX_WORKERS,
    max_pending: int = DEFAULT_MAX_PENDING,
    max_inflight_per_scope: int | None = None,
    thread_name_prefix: str = 'langbot-blocking',
) -> BoundedThreadPoolExecutor:
    """Install one bounded owner for every ``asyncio.to_thread`` call."""

    max_workers, max_pending, max_inflight_per_scope = _validated_limits(
        max_workers,
        max_pending,
        max_inflight_per_scope,
    )
    existing = getattr(loop, '_default_executor', None)
    if isinstance(existing, BoundedThreadPoolExecutor):
        if (
            existing.max_workers != max_workers
            or existing.max_pending != max_pending
            or existing.max_inflight_per_scope != max_inflight_per_scope
        ):
            raise RuntimeError('The blocking executor is already configured with different limits')
        return existing
    if existing is not None:
        raise RuntimeError('The event loop default executor was initialized before LangBot resource limits')

    executor = BoundedThreadPoolExecutor(
        max_workers=max_workers,
        max_pending=max_pending,
        max_inflight_per_scope=max_inflight_per_scope,
        thread_name_prefix=thread_name_prefix,
    )
    loop.set_default_executor(executor)
    return executor