mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 04:40:57 +00:00
feat(tenancy): add Workspace multi-tenant foundation (#2353)
* Document multi-tenant workspace architecture * Add OSS and commercial workspace boundaries * docs: redesign multi-tenant workspace architecture * feat(tenancy): implement workspace isolation * docs(tenancy): record verification evidence * docs(tenancy): revise single-instance SaaS topology * docs(tenancy): refine architecture options * docs: finalize cloud v2 multi-tenant decisions * feat(tenancy): establish cloud isolation foundations * feat(tenancy): harden shared cloud runtime boundaries * docs(tenancy): record final isolation verification * fix(tenancy): close isolation and permission gaps * docs(tenancy): record final isolation verification * feat(tenancy): connect cloud workspace control plane * fix(build): install git for pinned SDK * docs(cloud): update control plane verification * chore: update multi-tenant SDK pin * fix(cloud): skip legacy model sync during startup * test(cloud): preserve minimal model manager fixtures * fix(cloud): preserve authenticated account context * fix(cloud): reuse authenticated account for user info * feat(cloud): complete Workspace settings navigation * test(web): cover Workspace dropdown menu * feat(web): place workspace controls in sidebar * refactor(web): streamline workspace controls * style(web): format workspace layout test * fix(cloud): surface runtime and workspace plan status * fix(plugin): keep runtime identity stable across restarts * fix(ui): widen and center workspace switcher * fix(ui): hide roles from workspace switcher * fix(ui): align workspace switcher with sidebar entries * feat(workspace): add in-product collaboration and direct Cloud launch * style: format collaboration changes * fix(workspace): bind collaboration APIs to tenant UoW * fix(cloud): preserve Core-owned collaboration state * test(cloud): require Space identity for invite registration * feat(cloud): complete secure invitation experience * style(web): format invitation flows * fix(cloud): recover box runtime without unscoped skill reload * feat(oss): enforce invitation account and owner billing flows * style: format OSS account service * test(oss): cover invitation logout handoff * fix(oss): resolve workspace owner in scoped session * feat(cloud): harden multi-tenant runtime resources * fix(cloud): bound runtime restart storms * fix(cloud): eliminate periodic runtime CPU spikes * fix(cloud): enforce instance capacity ceilings * fix(cloud): scope public login capability discovery * fix(cloud): bound tenant maintenance and monitoring work * fix(runtime): bound tenant resource amplification * fix(deps): pin green multi-tenant plugin SDK * fix(cloud): handle unavailable skill capability * fix(security): require authentication for image file endpoint (H-2) - Changed /api/v1/files/image from AuthType.NONE to USER_TOKEN_OR_API_KEY - Added Permission.RESOURCE_VIEW requirement - Prevents unauthenticated cross-tenant file access via leaked keys - Fixes HIGH severity finding from multi-tenant security review docs: add comprehensive database migration guide - Complete migration steps for OSS → multi-tenant - Backup, execution, verification procedures - Rollback scenarios and recovery plans - Performance tuning recommendations * test: add comprehensive cross-tenant isolation tests Added 7 critical test scenarios for multi-tenant boundaries: - Cross-tenant bot access prevention - Viewer role read-only enforcement - Removed member immediate access revocation - Model provider credential isolation - WebSocket message isolation - Invitation token workspace scoping - Multi-workspace context validation These tests address P0-2 coverage gaps for: - workspaces.py (membership & invitation flows) - user.py (authentication & authorization) - websocket_chat.py (real-time isolation) - plugins.py (resource access control) docs: finalize database migration guide * fix(security): resolve M-1, M-2, M-3 security findings M-1: WebSocket authorization TOCTOU race (FIXED) - Changed _revalidate_websocket_authorization to return RequestContext - Ensures validated context is used immediately without race window - Prevents removed members from sending messages during revalidation gap M-2: Model Manager cache workspace isolation (VERIFIED) - Confirmed _CacheKey already uses 4-tuple: (instance, workspace, generation, resource) - Cache is properly scoped per workspace, no cross-tenant leakage possible - No code change needed, documented as working correctly M-3: Invitation lock workspace scoping (FIXED) - Changed lock key from token_digest to workspace_uuid:token_digest - Prevents DoS where attacker locks token in Workspace A to block Workspace B - Locks now isolated per workspace All MEDIUM severity findings from security review now resolved. * fix(cloud): unblock tenant CI and enforce knowledge quotas * fix(tenancy): scope rerank model sync --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
RockChinQ
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from langbot.pkg.cloud.bootstrap import (
|
||||
CloudBootstrapError,
|
||||
CloudManifestRefreshService,
|
||||
CloudRuntimeUnavailableError,
|
||||
DeploymentAdmissionGuard,
|
||||
OpenSourceDeployment,
|
||||
VerifiedCloudDeployment,
|
||||
resolve_deployment,
|
||||
)
|
||||
from langbot.pkg.cloud.entitlements import EntitlementSnapshot
|
||||
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
class _Entitlements:
|
||||
async def get_workspace_entitlement(self, workspace_uuid: str) -> EntitlementSnapshot:
|
||||
return EntitlementSnapshot(
|
||||
instance_uuid='instance-a',
|
||||
workspace_uuid=workspace_uuid,
|
||||
entitlement_revision=1,
|
||||
status='active',
|
||||
not_before=1,
|
||||
expires_at=4_000_000_000,
|
||||
features={'managed_sandbox': True},
|
||||
limits={'managed_sandbox_sessions': 1},
|
||||
)
|
||||
|
||||
|
||||
class _Directory:
|
||||
async def fetch_snapshot(self, instance_uuid: str):
|
||||
del instance_uuid
|
||||
raise AssertionError('not used by bootstrap contract tests')
|
||||
|
||||
async def fetch_events(self, instance_uuid: str, after_cursor: int, limit: int):
|
||||
del instance_uuid, after_cursor, limit
|
||||
raise AssertionError('not used by bootstrap contract tests')
|
||||
|
||||
async def fetch_workspaces(self, instance_uuid: str, workspace_uuids: tuple[str, ...]):
|
||||
del instance_uuid, workspace_uuids
|
||||
raise AssertionError('not used by bootstrap contract tests')
|
||||
|
||||
|
||||
class _Manifest:
|
||||
def __init__(self):
|
||||
self.candidate = None
|
||||
self.closed = False
|
||||
|
||||
async def refresh_manifest(self):
|
||||
if self.candidate is None:
|
||||
raise AssertionError('no refreshed Manifest was configured')
|
||||
return self.candidate
|
||||
|
||||
async def aclose(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class _Provider:
|
||||
def __init__(self):
|
||||
self.manifest_provider = _Manifest()
|
||||
|
||||
def bootstrap(self, *, instance_uuid: str, instance_config: dict):
|
||||
del instance_config
|
||||
return VerifiedCloudDeployment(
|
||||
instance_uuid=instance_uuid,
|
||||
manifest_jti='manifest-a',
|
||||
manifest_generation=3,
|
||||
expires_at=4_000_000_000,
|
||||
release='cloud-v2',
|
||||
capabilities=frozenset({'multi_workspace_v2'}),
|
||||
tenant_isolation_version=2,
|
||||
entitlement_provider=_Entitlements(),
|
||||
directory_provider=_Directory(),
|
||||
manifest_provider=self.manifest_provider,
|
||||
verification_key_id='root-2026',
|
||||
)
|
||||
|
||||
|
||||
class _EntryPoint:
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def load(self):
|
||||
return self.value
|
||||
|
||||
|
||||
class _EntryPoints(list):
|
||||
def select(self, *, group: str):
|
||||
return self if group == 'langbot.cloud_bootstrap' else []
|
||||
|
||||
|
||||
def _cloud_config() -> dict:
|
||||
return {
|
||||
'database': {'use': 'postgresql'},
|
||||
'vdb': {
|
||||
'use': 'pgvector',
|
||||
'pgvector': {
|
||||
'use_business_database': True,
|
||||
'allowed_dimensions': [384, 768, 1536],
|
||||
},
|
||||
},
|
||||
'mcp': {'stdio': {'enabled': False}},
|
||||
'plugin': {'worker': {'require_hard_limits': True}},
|
||||
'box': {
|
||||
'enabled': True,
|
||||
'backend': 'nsjail',
|
||||
'runtime': {'endpoint': 'ws://langbot-box:5410'},
|
||||
'admission': {
|
||||
'required': True,
|
||||
'logical_session_id': 'global',
|
||||
'required_backend': 'nsjail',
|
||||
'max_sessions': 1,
|
||||
'max_managed_processes': 0,
|
||||
'max_grant_ttl_sec': 300,
|
||||
'workspace_quota_mb': 32,
|
||||
},
|
||||
'local': {
|
||||
'host_root': '/var/lib/langbot/box',
|
||||
'default_workspace': '/var/lib/langbot/box/workspaces',
|
||||
'allowed_mount_roots': ['/var/lib/langbot/box'],
|
||||
},
|
||||
},
|
||||
# Proves mutable product metadata does not participate in selection.
|
||||
'system': {'edition': 'community'},
|
||||
}
|
||||
|
||||
|
||||
async def test_no_closed_entry_point_selects_oss_singleton_even_if_edition_says_cloud():
|
||||
deployment = await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config={'system': {'edition': 'cloud'}},
|
||||
entry_points=lambda: _EntryPoints(),
|
||||
)
|
||||
|
||||
assert isinstance(deployment, OpenSourceDeployment)
|
||||
assert deployment.multi_workspace_enabled is False
|
||||
|
||||
|
||||
async def test_verified_closed_entry_point_activates_cloud_policy():
|
||||
deployment = await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=_cloud_config(),
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(_Provider)]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
assert isinstance(deployment, VerifiedCloudDeployment)
|
||||
assert deployment.multi_workspace_enabled is True
|
||||
assert deployment.persistence_mode == 'cloud_runtime'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('field', 'value', 'message'),
|
||||
[
|
||||
('database', {'use': 'sqlite'}, 'database.use=postgresql'),
|
||||
('vdb', {'use': 'chroma'}, 'vdb.use=pgvector'),
|
||||
('mcp', {'stdio': {'enabled': True}}, 'mcp.stdio.enabled=false'),
|
||||
('plugin', {'worker': {'require_hard_limits': False}}, 'plugin.worker.require_hard_limits=true'),
|
||||
],
|
||||
)
|
||||
async def test_cloud_runtime_config_is_fail_closed(field, value, message):
|
||||
config = _cloud_config()
|
||||
config[field] = value
|
||||
|
||||
with pytest.raises(CloudBootstrapError, match=message):
|
||||
await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=config,
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(_Provider())]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('directory_config', 'message'),
|
||||
[
|
||||
({'max_active_workspaces': 0}, 'greater than or equal to 1'),
|
||||
({'max_active_workspaces': True}, 'must be an integer'),
|
||||
(
|
||||
{
|
||||
'max_active_workspaces': 10,
|
||||
'max_snapshot_workspaces': 9,
|
||||
},
|
||||
'max_snapshot_workspaces',
|
||||
),
|
||||
({'max_response_bytes': 64 * 1024 * 1024 + 1}, 'less than or equal to'),
|
||||
],
|
||||
)
|
||||
async def test_cloud_directory_capacity_contract_is_fail_closed(directory_config, message):
|
||||
config = _cloud_config()
|
||||
config['cloud'] = {'directory': directory_config}
|
||||
|
||||
with pytest.raises(CloudBootstrapError, match=message):
|
||||
await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=config,
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(_Provider())]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('pgvector_config', 'message'),
|
||||
[
|
||||
({'use_business_database': False, 'allowed_dimensions': [1536]}, 'use_business_database=true'),
|
||||
({'use_business_database': True, 'allowed_dimensions': []}, 'allowed_dimensions'),
|
||||
({'use_business_database': True, 'allowed_dimensions': [3072]}, 'allowed_dimensions'),
|
||||
({'use_business_database': True, 'allowed_dimensions': [True]}, 'allowed_dimensions'),
|
||||
],
|
||||
)
|
||||
async def test_cloud_pgvector_contract_is_fail_closed(pgvector_config, message):
|
||||
config = _cloud_config()
|
||||
config['vdb']['pgvector'] = pgvector_config
|
||||
|
||||
with pytest.raises(CloudBootstrapError, match=message):
|
||||
await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=config,
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(_Provider())]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('mutate', 'message'),
|
||||
[
|
||||
(lambda config: config['box'].update(enabled=False), 'box.enabled=true'),
|
||||
(lambda config: config['box'].update(backend='docker'), 'box.backend=nsjail'),
|
||||
(lambda config: config['box']['runtime'].update(endpoint=''), 'box.runtime.endpoint'),
|
||||
(
|
||||
lambda config: config['box']['admission'].update(max_sessions=2),
|
||||
'grant-enforced Box admission',
|
||||
),
|
||||
(
|
||||
lambda config: config['box']['admission'].update(max_managed_processes=1),
|
||||
'zero managed processes',
|
||||
),
|
||||
(
|
||||
lambda config: config['box']['admission'].update(max_grant_ttl_sec=301),
|
||||
'max_grant_ttl_sec',
|
||||
),
|
||||
(
|
||||
lambda config: config['box']['admission'].update(workspace_quota_mb=0),
|
||||
'workspace_quota_mb must be a positive integer',
|
||||
),
|
||||
(
|
||||
lambda config: config['box']['admission'].update(workspace_quota_mb=True),
|
||||
'workspace_quota_mb must be a positive integer',
|
||||
),
|
||||
(
|
||||
lambda config: config['box']['local'].update(default_workspace='relative/workspaces'),
|
||||
'default_workspace must be an absolute',
|
||||
),
|
||||
(
|
||||
lambda config: config['box']['local'].update(
|
||||
default_workspace='/other/workspaces',
|
||||
),
|
||||
'under allowed_mount_roots',
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_cloud_box_contract_is_fail_closed(mutate, message):
|
||||
config = _cloud_config()
|
||||
mutate(config)
|
||||
|
||||
with pytest.raises(CloudBootstrapError, match=message):
|
||||
await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=config,
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(_Provider())]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
|
||||
async def test_invalid_provider_never_falls_back_to_oss():
|
||||
provider = SimpleNamespace(bootstrap=lambda **_: object())
|
||||
|
||||
with pytest.raises(CloudBootstrapError, match='must return VerifiedCloudDeployment'):
|
||||
await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=_cloud_config(),
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(provider)]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
|
||||
async def test_duplicate_closed_providers_fail_closed():
|
||||
with pytest.raises(CloudBootstrapError, match='Exactly one'):
|
||||
await resolve_deployment(
|
||||
instance_uuid='instance-a',
|
||||
instance_config=_cloud_config(),
|
||||
entry_points=lambda: _EntryPoints([_EntryPoint(_Provider()), _EntryPoint(_Provider())]),
|
||||
now=1_000,
|
||||
)
|
||||
|
||||
|
||||
async def test_deployment_admission_expires_even_after_wall_clock_rollback():
|
||||
wall = [1_000.0]
|
||||
monotonic = [50.0]
|
||||
deployment = dataclasses.replace(
|
||||
_Provider().bootstrap(instance_uuid='instance-a', instance_config={}),
|
||||
expires_at=1_010,
|
||||
)
|
||||
guard = DeploymentAdmissionGuard(
|
||||
'instance-a',
|
||||
deployment,
|
||||
wall_time=lambda: wall[0],
|
||||
monotonic_time=lambda: monotonic[0],
|
||||
)
|
||||
|
||||
assert guard.require_active() is deployment
|
||||
wall[0] = 900.0
|
||||
monotonic[0] = 60.0
|
||||
with pytest.raises(CloudRuntimeUnavailableError, match='expired'):
|
||||
guard.require_active()
|
||||
|
||||
|
||||
async def test_deployment_admission_accepts_only_monotonic_non_conflicting_renewal():
|
||||
wall = [1_000.0]
|
||||
monotonic = [50.0]
|
||||
current = dataclasses.replace(
|
||||
_Provider().bootstrap(instance_uuid='instance-a', instance_config={}),
|
||||
expires_at=1_010,
|
||||
)
|
||||
guard = DeploymentAdmissionGuard(
|
||||
'instance-a',
|
||||
current,
|
||||
wall_time=lambda: wall[0],
|
||||
monotonic_time=lambda: monotonic[0],
|
||||
)
|
||||
renewed = dataclasses.replace(
|
||||
current,
|
||||
manifest_jti='manifest-b',
|
||||
manifest_generation=4,
|
||||
expires_at=2_000,
|
||||
)
|
||||
guard.replace(renewed)
|
||||
assert guard.require_active() is renewed
|
||||
|
||||
rollback = dataclasses.replace(current, manifest_generation=2)
|
||||
with pytest.raises(CloudRuntimeUnavailableError, match='rolled back'):
|
||||
guard.replace(rollback)
|
||||
|
||||
conflicting = dataclasses.replace(renewed, manifest_jti='different')
|
||||
with pytest.raises(CloudRuntimeUnavailableError, match='conflicting'):
|
||||
guard.replace(conflicting)
|
||||
|
||||
|
||||
async def test_manifest_refresh_replaces_receipt_before_short_ttl_expires():
|
||||
wall = [1_000.0]
|
||||
provider = _Provider()
|
||||
current = dataclasses.replace(
|
||||
provider.bootstrap(instance_uuid='instance-a', instance_config={}),
|
||||
expires_at=1_300,
|
||||
)
|
||||
guard = DeploymentAdmissionGuard('instance-a', current, wall_time=lambda: wall[0])
|
||||
renewed = dataclasses.replace(
|
||||
current,
|
||||
manifest_jti='manifest-renewed',
|
||||
manifest_generation=current.manifest_generation + 1,
|
||||
expires_at=2_000,
|
||||
)
|
||||
provider.manifest_provider.candidate = renewed
|
||||
service = CloudManifestRefreshService(
|
||||
guard,
|
||||
provider.manifest_provider,
|
||||
SimpleNamespace(exception=lambda *_: None),
|
||||
wall_time=lambda: wall[0],
|
||||
)
|
||||
|
||||
assert service.next_refresh_delay() == 120
|
||||
assert await service.refresh_once() is renewed
|
||||
assert guard.deployment is renewed
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,161 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from langbot.pkg.cloud.entitlements import EntitlementResolver, EntitlementSnapshot, EntitlementUnavailableError
|
||||
|
||||
|
||||
def _snapshot(**overrides) -> EntitlementSnapshot:
|
||||
values = {
|
||||
'instance_uuid': 'instance-a',
|
||||
'workspace_uuid': 'workspace-a',
|
||||
'entitlement_revision': 7,
|
||||
'status': 'active',
|
||||
'not_before': 100,
|
||||
'expires_at': 200,
|
||||
'features': {'managed_sandbox': True, 'mcp_stdio': False},
|
||||
'limits': {'managed_sandbox_sessions': 1},
|
||||
}
|
||||
values.update(overrides)
|
||||
return EntitlementSnapshot(**values)
|
||||
|
||||
|
||||
def test_active_snapshot_exposes_only_generic_features_and_limits():
|
||||
snapshot = _snapshot().require_active(
|
||||
instance_uuid='instance-a',
|
||||
workspace_uuid='workspace-a',
|
||||
now=150,
|
||||
)
|
||||
|
||||
snapshot.require_feature('managed_sandbox')
|
||||
assert snapshot.limit('managed_sandbox_sessions') == 1
|
||||
assert 'plan' not in snapshot.model_fields
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'snapshot,now',
|
||||
[
|
||||
(_snapshot(status='suspended'), 150),
|
||||
(_snapshot(), 99),
|
||||
(_snapshot(), 200),
|
||||
],
|
||||
)
|
||||
def test_inactive_or_expired_snapshot_fails_closed(snapshot, now):
|
||||
with pytest.raises(EntitlementUnavailableError):
|
||||
snapshot.require_active(
|
||||
instance_uuid='instance-a',
|
||||
workspace_uuid='workspace-a',
|
||||
now=now,
|
||||
)
|
||||
|
||||
|
||||
def test_scope_mismatch_fails_closed():
|
||||
with pytest.raises(EntitlementUnavailableError, match='scope'):
|
||||
_snapshot().require_active(
|
||||
instance_uuid='instance-a',
|
||||
workspace_uuid='workspace-b',
|
||||
now=150,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolver_rejects_revision_rollback():
|
||||
provider = AsyncMock()
|
||||
provider.get_workspace_entitlement = AsyncMock(side_effect=[_snapshot(), _snapshot(entitlement_revision=6)])
|
||||
resolver = EntitlementResolver('instance-a', provider)
|
||||
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
with pytest.raises(EntitlementUnavailableError, match='rolled back'):
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolver_rejects_same_revision_with_different_contents():
|
||||
provider = AsyncMock()
|
||||
provider.get_workspace_entitlement = AsyncMock(
|
||||
side_effect=[
|
||||
_snapshot(),
|
||||
_snapshot(features={'managed_sandbox': False}),
|
||||
]
|
||||
)
|
||||
resolver = EntitlementResolver('instance-a', provider)
|
||||
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
with pytest.raises(EntitlementUnavailableError, match='conflicting contents'):
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolver_checks_deployment_admission_before_and_after_provider_call():
|
||||
checks = 0
|
||||
|
||||
def require_admission() -> None:
|
||||
nonlocal checks
|
||||
checks += 1
|
||||
if checks == 2:
|
||||
raise RuntimeError('manifest expired during provider call')
|
||||
|
||||
provider = AsyncMock()
|
||||
provider.get_workspace_entitlement = AsyncMock(return_value=_snapshot())
|
||||
resolver = EntitlementResolver(
|
||||
'instance-a',
|
||||
provider,
|
||||
deployment_admission=require_admission,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match='expired during provider call'):
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
assert checks == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_directory_activity_reconciliation_drops_historical_snapshots():
|
||||
provider = AsyncMock()
|
||||
provider.get_workspace_entitlement = AsyncMock(return_value=_snapshot())
|
||||
resolver = EntitlementResolver('instance-a', provider)
|
||||
await resolver.reconcile_active_workspaces({'workspace-a', 'workspace-b'})
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
|
||||
await resolver.reconcile_active_workspaces({'workspace-b'})
|
||||
|
||||
assert resolver.snapshot_counts() == {
|
||||
'active_workspaces': 1,
|
||||
'cached_snapshots': 0,
|
||||
}
|
||||
with pytest.raises(EntitlementUnavailableError, match='directory projection'):
|
||||
await resolver.resolve('workspace-a', now=150)
|
||||
provider.get_workspace_entitlement.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_directory_fence_wins_race_with_inflight_entitlement_fetch():
|
||||
provider_started = asyncio.Event()
|
||||
release_provider = asyncio.Event()
|
||||
|
||||
async def fetch(_workspace_uuid: str) -> EntitlementSnapshot:
|
||||
provider_started.set()
|
||||
await release_provider.wait()
|
||||
return _snapshot()
|
||||
|
||||
provider = AsyncMock()
|
||||
provider.get_workspace_entitlement = AsyncMock(side_effect=fetch)
|
||||
resolver = EntitlementResolver('instance-a', provider)
|
||||
await resolver.reconcile_active_workspaces({'workspace-a'})
|
||||
resolve_task = asyncio.create_task(resolver.resolve('workspace-a', now=150))
|
||||
await provider_started.wait()
|
||||
|
||||
await resolver.update_workspace_activity(
|
||||
active_workspace_uuids=set(),
|
||||
inactive_workspace_uuids={'workspace-a'},
|
||||
)
|
||||
release_provider.set()
|
||||
|
||||
with pytest.raises(EntitlementUnavailableError, match='directory projection'):
|
||||
await resolve_task
|
||||
assert resolver.snapshot_counts() == {
|
||||
'active_workspaces': 0,
|
||||
'cached_snapshots': 0,
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
from langbot.pkg.cloud.launch import SpaceLaunchError, SpaceLaunchService
|
||||
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
INSTANCE_UUID = 'instance-test'
|
||||
ACCOUNT_UUID = '11111111-1111-4111-8111-111111111111'
|
||||
WORKSPACE_UUID = '22222222-2222-4222-8222-222222222222'
|
||||
KEY_ID = 'space-key-1'
|
||||
|
||||
|
||||
def _base64url(raw: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b'=').decode('ascii')
|
||||
|
||||
|
||||
def _sign(private_key: Ed25519PrivateKey, claims: dict, *, key_id: str = KEY_ID) -> str:
|
||||
header = {'alg': 'EdDSA', 'kid': key_id, 'typ': 'langbot-control-plane+jwt'}
|
||||
encoded_header = _base64url(json.dumps(header, separators=(',', ':')).encode('utf-8'))
|
||||
encoded_claims = _base64url(json.dumps(claims, separators=(',', ':')).encode('utf-8'))
|
||||
signing_input = f'{encoded_header}.{encoded_claims}'
|
||||
return f'{signing_input}.{_base64url(private_key.sign(signing_input.encode("ascii")))}'
|
||||
|
||||
|
||||
def _claims(*, now: int, jti: str | None = None, workspace_uuid: str = WORKSPACE_UUID) -> dict:
|
||||
return {
|
||||
'iss': 'langbot-space',
|
||||
'aud': 'langbot-cloud-runtime',
|
||||
'sub': f'langbot-instance:{INSTANCE_UUID}',
|
||||
'jti': jti or str(uuid.uuid4()),
|
||||
'iat': now,
|
||||
'nbf': now - 5,
|
||||
'exp': now + 90,
|
||||
'instance_uuid': INSTANCE_UUID,
|
||||
'kind': 'workspace.launch',
|
||||
'payload': {
|
||||
'account_uuid': ACCOUNT_UUID,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _service(private_key: Ed25519PrivateKey, *, now: int) -> SpaceLaunchService:
|
||||
public_key = private_key.public_key().public_bytes(
|
||||
encoding=serialization.Encoding.Raw,
|
||||
format=serialization.PublicFormat.Raw,
|
||||
)
|
||||
app = SimpleNamespace(
|
||||
deployment=SimpleNamespace(multi_workspace_enabled=True, verification_key_id=KEY_ID),
|
||||
workspace_service=SimpleNamespace(instance_uuid=INSTANCE_UUID),
|
||||
instance_config=SimpleNamespace(
|
||||
data={
|
||||
'space': {
|
||||
'launch': {
|
||||
'control_plane_public_key': _base64url(public_key),
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
)
|
||||
return SpaceLaunchService(app, wall_time=lambda: now)
|
||||
|
||||
|
||||
async def test_consumes_valid_workspace_launch_assertion_once():
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
now = int(time.time())
|
||||
service = _service(private_key, now=now)
|
||||
token = _sign(private_key, _claims(now=now))
|
||||
|
||||
launch = await service.consume_assertion(token, expected_workspace_uuid=WORKSPACE_UUID)
|
||||
|
||||
assert launch == {'account_uuid': ACCOUNT_UUID, 'workspace_uuid': WORKSPACE_UUID}
|
||||
with pytest.raises(SpaceLaunchError, match='already been consumed'):
|
||||
await service.consume_assertion(token, expected_workspace_uuid=WORKSPACE_UUID)
|
||||
|
||||
|
||||
async def test_replay_cache_does_not_scan_all_live_assertions(monkeypatch):
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
now = int(time.time())
|
||||
service = _service(private_key, now=now)
|
||||
for index in range(512):
|
||||
await service._consume_jti(f'jti-{index}', now + 90)
|
||||
|
||||
class NoGlobalIterationDict(dict):
|
||||
def __iter__(self):
|
||||
raise AssertionError('replay admission scanned all live assertions')
|
||||
|
||||
def keys(self):
|
||||
raise AssertionError('replay admission scanned all live assertions')
|
||||
|
||||
def items(self):
|
||||
raise AssertionError('replay admission scanned all live assertions')
|
||||
|
||||
def values(self):
|
||||
raise AssertionError('replay admission scanned all live assertions')
|
||||
|
||||
guarded_jtis = NoGlobalIterationDict(service._consumed_jtis)
|
||||
monkeypatch.setattr(service, '_consumed_jtis', guarded_jtis)
|
||||
|
||||
await service._consume_jti('jti-new', now + 90)
|
||||
|
||||
assert len(guarded_jtis) == 513
|
||||
|
||||
|
||||
async def test_replay_cache_fails_closed_at_capacity(monkeypatch):
|
||||
from langbot.pkg.cloud import launch
|
||||
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
now = int(time.time())
|
||||
service = _service(private_key, now=now)
|
||||
monkeypatch.setattr(launch, '_CONSUMED_JTI_MAX_ENTRIES', 2)
|
||||
await service._consume_jti('jti-1', now + 90)
|
||||
await service._consume_jti('jti-2', now + 90)
|
||||
|
||||
with pytest.raises(SpaceLaunchError, match='replay cache capacity'):
|
||||
await service._consume_jti('jti-3', now + 90)
|
||||
with pytest.raises(SpaceLaunchError, match='already been consumed'):
|
||||
await service._consume_jti('jti-1', now + 90)
|
||||
|
||||
|
||||
async def test_rejects_expired_wrong_workspace_and_wrong_instance_assertions():
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
now = int(time.time())
|
||||
service = _service(private_key, now=now)
|
||||
|
||||
expired = _claims(now=now)
|
||||
expired['exp'] = now - 60
|
||||
with pytest.raises(SpaceLaunchError, match='expired'):
|
||||
await service.consume_assertion(_sign(private_key, expired), expected_workspace_uuid=WORKSPACE_UUID)
|
||||
|
||||
wrong_workspace = _sign(private_key, _claims(now=now, workspace_uuid='33333333-3333-4333-8333-333333333333'))
|
||||
with pytest.raises(SpaceLaunchError, match='another Workspace'):
|
||||
await service.consume_assertion(wrong_workspace, expected_workspace_uuid=WORKSPACE_UUID)
|
||||
|
||||
wrong_instance = _claims(now=now)
|
||||
wrong_instance['instance_uuid'] = 'other-instance'
|
||||
with pytest.raises(SpaceLaunchError, match='instance UUID'):
|
||||
await service.consume_assertion(_sign(private_key, wrong_instance), expected_workspace_uuid=WORKSPACE_UUID)
|
||||
|
||||
|
||||
async def test_rejects_invalid_signature_and_non_cloud_mode():
|
||||
private_key = Ed25519PrivateKey.generate()
|
||||
now = int(time.time())
|
||||
token = _sign(private_key, _claims(now=now))
|
||||
service = _service(Ed25519PrivateKey.generate(), now=now)
|
||||
with pytest.raises(SpaceLaunchError, match='signature'):
|
||||
await service.consume_assertion(token, expected_workspace_uuid=WORKSPACE_UUID)
|
||||
|
||||
oss_service = _service(private_key, now=now)
|
||||
oss_service.ap.deployment.multi_workspace_enabled = False
|
||||
with pytest.raises(SpaceLaunchError, match='verified Cloud mode'):
|
||||
await oss_service.consume_assertion(token, expected_workspace_uuid=WORKSPACE_UUID)
|
||||
Reference in New Issue
Block a user