From 90a977488212d3f3f50fb92d94e3fcae60f27eab Mon Sep 17 00:00:00 2001 From: Junyan Qin Date: Mon, 20 Jul 2026 04:02:24 +0800 Subject: [PATCH] fix(tenancy): close isolation and permission gaps --- docs/multi-tenant/implementation-checklist.md | 36 +- docs/multi-tenant/implementation-decisions.md | 68 +- .../pending-architecture-decisions.md | 24 +- .../workspace-multi-user-architecture.md | 2240 ++++++----------- pyproject.toml | 2 +- .../api/http/controller/groups/extensions.py | 16 +- .../controller/groups/knowledge/migration.py | 105 +- .../api/http/controller/groups/monitoring.py | 26 +- .../http/controller/groups/platform/bots.py | 2 +- .../api/http/controller/groups/workspaces.py | 41 +- src/langbot/pkg/persistence/mgr.py | 7 +- src/langbot/pkg/persistence/tenant_uow.py | 939 ++++++- tests/integration/api/test_bots.py | 8 +- tests/integration/api/test_monitoring.py | 28 + tests/integration/api/test_workspaces.py | 26 + .../persistence/test_migrations_postgres.py | 146 +- .../persistence/test_pgvector_postgres.py | 61 +- .../api/test_extensions_runtime_fence.py | 48 + .../test_knowledge_migration_runtime_fence.py | 224 ++ .../unit_tests/persistence/test_tenant_uow.py | 677 +++++ uv.lock | 4 +- web/src/app/home/bots/BotDetailContent.tsx | 12 +- web/src/app/home/monitoring/page.tsx | 6 +- .../home/pipelines/PipelineDetailContent.tsx | 8 +- web/src/app/infra/http/BackendClient.ts | 2 +- web/src/app/invitations/accept/page.tsx | 69 +- web/src/app/login/page.tsx | 43 +- web/tests/e2e/crud-smoke.spec.ts | 60 +- web/tests/e2e/fixtures/langbot-api.ts | 3 +- web/tests/e2e/invitations.spec.ts | 127 + 30 files changed, 3294 insertions(+), 1764 deletions(-) diff --git a/docs/multi-tenant/implementation-checklist.md b/docs/multi-tenant/implementation-checklist.md index b36f47747..6f86ed91b 100644 --- a/docs/multi-tenant/implementation-checklist.md +++ b/docs/multi-tenant/implementation-checklist.md @@ -21,16 +21,18 @@ closed SaaS product or a production Cloud v2 deployment. Checked implementation items later in this document do not supersede these gates. - [ ] The closed Control Plane owns the global Account, Workspace, Membership, and Invitation directory. -- [ ] The closed placement service issues monotonic generations and leases for projected Workspaces. +- [ ] The closed Control Plane execution-ownership module issues monotonic generations and owner leases for projected Workspaces. - [ ] Core verifies a signed `InstanceManifest` before the closed bootstrap can inject `CloudWorkspacePolicy`. -- [ ] Tenant database writes hold a generation-aware shared transaction fence through commit, while placement cutovers take the exclusive fence. +- [ ] Tenant database writes hold a generation-aware shared transaction fence through commit, while execution-owner cutovers take the exclusive fence. - [ ] Business writes and non-transactional side effects use a generation-stamped outbox or equivalent publish fence. -- [ ] Durable object references survive a placement-generation change through stable published keys or an explicitly atomic key/reference migration. +- [ ] Durable object references survive an execution-generation change through stable published keys or an explicitly atomic key/reference migration. - [ ] The SaaS runtime pools enforce tenant-safe egress and SSRF controls for Webhooks, providers, MCP servers, and every tenant-configurable outbound URL. - [ ] Entitlement checks, usage aggregation, subscription lifecycle, and billing are implemented in the closed Control Plane. - [ ] OAuth state and directory projection use an atomic shared store suitable for horizontally scaled SaaS services. - [ ] A greenfield Cloud v2 deployment is designed and validated independently of the legacy Space deployment scheme. - [ ] The Plugin Runtime deployment provides delegated cgroup v2 and tenant-safe egress; the shared profile refuses to run without hard CPU, memory, and PID limits. +- [ ] The Plugin Runtime Supervisor automatically restores an unexpectedly exited enabled worker with bounded backoff and cannot create a cross-tenant restart storm. +- [ ] Plugin installation data has an operator-owned hard disk quota provider that atomically rejects writes over the limit; directory scans are not accepted as enforcement. - [ ] The Box deployment provides an operator-owned quota provider that proves hard byte and inode limits for Workspace, Skill, root, tmp, and home storage. - [ ] Core and Box Runtime mount the same durable volume and pass the authenticated marker challenge during startup and reconnect. - [ ] Production provisions distinct migrator/runtime credentials and runs the implemented same-host/port/database release command as a one-shot Job, with tested orchestration retry, backup, and rollback procedures. @@ -71,6 +73,16 @@ items later in this document do not supersede these gates. - [x] Legacy pgvector migration succeeds as a non-superuser, non-`BYPASSRLS` source-table owner and restores mixed source-table RLS/FORCE states exactly. - [ ] Legacy pgvector migration still needs explicit failure-and-retry integration coverage before SaaS activation. +### Runtime transaction enforcement + +- [x] Each tenant UoW owns one task, root transaction, database bind, and transaction-local scope. +- [x] A scoped Session and every captured bound method become permanently unusable when the owning UoW exits. +- [x] Public transaction/session control, raw/textual SQL, connection/bind escape, nested transactions, execution/loader options, foreign binds, live results, unapproved functions/operators/casts/types, `INSERT FROM SELECT`, hidden `ON CONFLICT` and batch-value expressions, forced-unquoted identifiers, and custom AST/compiler nodes fail closed and make the UoW rollback-only. +- [x] ORM `SessionEvents` fail before a registered callback can receive the synchronous Session or transaction connection; rollback cleanup cannot execute the rejected listener. +- [x] ORM flush, implicit autoflush, and commit reject SQL expressions assigned to mapped attributes before compilation. +- [x] Tenant relationship loading uses eager loading or explicit async `refresh`; synchronous object-session access and `AsyncAttrs.awaitable_attrs` are not supported tenant APIs. +- [x] The UoW guard is documented as a trusted-Core misuse boundary rather than an in-process Python sandbox; mapped metadata/compiler registration is trusted, plugins remain out of process, and SQLAlchemy upgrades must rerun the private-container regression suite. + ## 2. Authentication and authorization ### Identity @@ -145,8 +157,8 @@ Each row type must have a non-null Workspace UUID, scoped indexes, scoped unique ### Core runtime -- [x] RuntimeBot carries Workspace UUID and placement generation. -- [x] RuntimePipeline carries Workspace UUID and placement generation. +- [x] RuntimeBot carries Workspace UUID and execution generation (currently stored in the compatibility field `placement_generation`). +- [x] RuntimePipeline carries Workspace UUID and execution generation (currently stored in the compatibility field `placement_generation`). - [x] Query and Event carry Workspace UUID without making it an authorization source. - [x] Session key includes Workspace UUID, Bot UUID, launcher type, and launcher ID. - [x] QueryPool and manager indexes cannot collide across Workspaces. @@ -157,7 +169,7 @@ Each row type must have a non-null Workspace UUID, scoped indexes, scoped unique ### Plugin - [x] Plugin installation and configuration are Workspace scoped. -- [x] Runtime control actions carry trusted Workspace binding and placement generation. +- [x] Runtime control actions carry trusted Workspace binding and execution generation (wire-compatible as `placement_generation`). - [x] The Plugin Runtime supervisor is instance-scoped and intentionally serves multiple Workspaces. - [x] Every plugin process is bound to exactly one Workspace, installation, generation, revision, and verified artifact digest. - [x] Same-digest plugin code may be cached once, while worker processes and writable data remain isolated. @@ -170,22 +182,22 @@ Each row type must have a non-null Workspace UUID, scoped indexes, scoped unique ### MCP, RAG, and Box -- [x] MCP runtime key contains instance UUID, Workspace UUID, placement generation, and server UUID. +- [x] MCP runtime key contains instance UUID, Workspace UUID, execution generation, and server UUID. - [x] Same-named MCP servers in two Workspaces do not share sessions. - [x] Pipeline cannot reference another Workspace's MCP resource. - [x] RAG collection names and handles are server-derived and Workspace scoped. - [x] Legacy global vector migration is available only to the local OSS singleton Workspace. -- [x] Object storage paths include instance, Workspace, and placement generation for the fixed-generation OSS runtime. +- [x] Object storage paths include instance, Workspace, and execution generation for the fixed-generation OSS runtime. - [x] Object storage revalidates generation before touching a provider or resolving an opaque key. - [ ] Cloud cutover uses generation-scoped staging plus stable published object references, rather than making the staging generation the durable identity. - [x] Box persistent and ephemeral namespaces include the required instance, Workspace, and generation scope. -- [x] Same-named Box sessions and processes cannot collide across Workspaces or placement generations. +- [x] Same-named Box sessions and processes cannot collide across Workspaces or execution generations. - [x] Box relay and process I/O reject or retire stale generations. - [x] External paths and privileged mounts cannot be supplied by an untrusted plugin. - [x] Cloud attachment host I/O uses query UUIDs and link-free dirfd operations with bounded inode traversal. - [x] Cloud Skill package paths are Runtime-owned, Workspace-scoped, read-only mounts; Python env/cache stays tenant-writable. - [x] Skill ZIP preview/install rejects path escape, links, non-regular files, duplicate entries, excessive compression ratio, entry count, per-file size, and total size. -- [x] Cloud Box startup proves Core and Runtime see the same durable volume. +- [x] Cloud Box code paths and automated tests require the authenticated marker challenge before startup or reconnect can proceed. - [x] Cloud Box readiness fails until hard Workspace, Skill, ephemeral-storage, and inode quota capabilities are available. ## 6. SDK and protocol @@ -233,7 +245,7 @@ Each row type must have a non-null Workspace UUID, scoped indexes, scoped unique - [x] API Key cannot cross Workspace. - [x] Plugin cannot enumerate or invoke another Workspace's resources. - [x] Sessions, caches, locks, MCP, RAG, Box, storage, and monitoring do not collide. -- [x] Background jobs cannot execute without an explicit Workspace and placement generation. +- [x] Background jobs cannot execute without an explicit Workspace and execution generation. ### Security and revocation @@ -241,7 +253,7 @@ Each row type must have a non-null Workspace UUID, scoped indexes, scoped unique - [x] OAuth redirects trust only server-configured WebUI or webhook origins, never request `Host` or `Origin` headers. - [x] Dashboard WebSockets revalidate authentication, Membership, resource, permission, and generation per message. - [x] Public embed WebSockets re-resolve Bot availability and execution binding per message. -- [x] Runtime, storage, Plugin Runtime, MCP, RAG, and Box reject a stale placement generation. +- [x] Runtime, storage, Plugin Runtime, MCP, RAG, and Box reject a stale execution generation. - [x] Unhandled API and webhook failures return a generic error plus request ID without exception text. - [x] URL user information and sensitive query parameters are redacted before configuration is serialized or logged. diff --git a/docs/multi-tenant/implementation-decisions.md b/docs/multi-tenant/implementation-decisions.md index 5dd3d9afa..ff98eac12 100644 --- a/docs/multi-tenant/implementation-decisions.md +++ b/docs/multi-tenant/implementation-decisions.md @@ -2,8 +2,9 @@ This log records implementation choices made while delivering the Workspace architecture. It is intended to make trade-offs auditable without interrupting implementation for routine decisions. -> Open challenges that may supersede parts of this log are tracked in -> [pending-architecture-decisions.md](./pending-architecture-decisions.md). They remain proposals until a decision is recorded here. +> Architecture decisions, activation gates, and still-open follow-ups are tracked in +> [pending-architecture-decisions.md](./pending-architecture-decisions.md). Sections marked as decided there are authoritative; +> this file records the concrete implementation choices and compatibility names used to realize them. ## 2026-07-18 @@ -11,8 +12,8 @@ This log records implementation choices made while delivering the Workspace arch - Decision: Community builds create exactly one Workspace per LangBot instance and allow multiple Accounts through invitations. - Reason: This preserves a simple self-hosted deployment while making authorization and ownership explicit. Creating a second Workspace is an edition error, not a hidden fallback. -- SaaS boundary: Multi-Workspace directory, placement, entitlement, and billing are the responsibility of a separate closed SaaS Control Plane. Core consumes a validated projection and remains the final isolation and authorization enforcement point; it does not become the SaaS system of record or billing engine. -- Deployment boundary: Cloud v2 is a greenfield deployment design. The previous Space instance/pod deployment scheme is not migrated, preserved for compatibility, or extended by this implementation. Existing Space OAuth, marketplace, and payment concepts may be reused only through explicit adapters where they still fit; `langbot-space` itself is not changed. +- SaaS boundary: Multi-Workspace directory, execution ownership, entitlement, and billing are the responsibility of a separate closed SaaS Control Plane. Core consumes a validated projection and remains the final isolation and authorization enforcement point; it does not become the SaaS system of record or billing engine. +- Deployment boundary: Cloud v2 is a greenfield deployment design. The previous per-account instance/pod scheme is not migrated, preserved for compatibility, or extended by this implementation. Existing OAuth, marketplace, and payment concepts may be reused only through explicit adapters where they still fit; `langbot-space` itself is not changed. ### Workspace selection is trusted only after authentication @@ -37,10 +38,10 @@ This log records implementation choices made while delivering the Workspace arch ### Plugin Runtime is shared; every plugin process is single-Workspace -- Decision: One instance-scoped Plugin Runtime control plane serves all Workspaces in the logical LangBot instance. Each plugin installation still launches as its own nsjail worker with an immutable binding containing `instance_uuid`, `workspace_uuid`, `placement_generation`, `installation_uuid`, `runtime_revision`, and verified artifact digest. A worker never routes actions for another Workspace or installation, and plugin-supplied scope fields are stripped. +- Decision: One instance-scoped Plugin Runtime control plane serves all Workspaces in the logical LangBot instance. Each running plugin installation has its own nsjail worker with an immutable binding containing `instance_uuid`, `workspace_uuid`, `execution_generation` (stored as the compatibility field `placement_generation` until the schema rename), `installation_uuid`, `runtime_revision`, and verified artifact digest; enabled-resident is the desired semantic. A worker never routes actions for another Workspace or installation, and plugin-supplied scope fields are stripped. - Isolation: Plugin code is mounted read-only. Home, tmp, and data paths are installation-scoped; process, file-descriptor, file-size, CPU, memory, and PID limits come only from `data/config.yaml` (including native environment overrides), never from a plugin manifest. Cloud requires nsjail and delegated cgroup v2 hard limits or fails closed. - Cost boundary: Identical verified package bytes share one digest-addressed code cache. A dependency environment is keyed by the artifact and requirements digests, Python ABI, Runtime version, and installer schema, then atomically published read-only for reuse. Installations and processes are not merged, even for the same plugin and version. Registration creates database desired state only; a worker is launched only for an enabled installation. -- Recovery: PostgreSQL installation desired state and durable binary storage are authoritative. Runtime reconnect performs an instance-wide full reconciliation, removes stale workers, and can replay a verified package after Runtime-local cache loss. Dependency preparation failure is recorded per installation with `dependency_prepare_failed`; it prevents that worker launch without blocking recovery of other desired installations, and the same revision can be retried. +- Recovery: PostgreSQL installation desired state and durable binary storage are authoritative. Runtime reconnect performs an instance-wide full reconciliation, removes stale workers, and can replay a verified package after Runtime-local cache loss. Dependency preparation failure is recorded per installation with `dependency_prepare_failed`; it prevents that worker launch without blocking recovery of other desired installations, and the same revision can be retried. Enabled-resident is the desired semantic, but the current Supervisor restores an unexpectedly exited worker only on the next Core apply/reconcile; a completion callback, bounded backoff, and cross-tenant restart-storm isolation remain Cloud activation gates. - Compatibility: Older SDK payloads and legacy `data/plugins` remain an OSS-only bridge. Shared mode requires complete bindings and rejects incomplete context. - Reason: Sharing the supervisor and immutable code cache removes per-Workspace service cost without turning an untrusted plugin process into a cross-tenant router. @@ -68,7 +69,7 @@ This log records implementation choices made while delivering the Workspace arch ### SaaS execution state is a validated Core projection - Decision: Core can resolve both local and `cloud_projection` Workspaces, but only from an explicit Workspace UUID and an active, unfenced `WorkspaceExecutionState` for the current instance and matching source. OSS-only bootstrap paths additionally require `source=local`. -- Reason: The closed control plane owns placement decisions, while Core remains the enforcement point for instance binding, generation, and write fences. +- Reason: The closed control plane owns execution ownership and generation decisions, while Core remains the enforcement point for instance binding, generation, and write fences. ### API-key secrets are one-time and Workspace-bound @@ -88,7 +89,7 @@ This log records implementation choices made while delivering the Workspace arch ### Cloud directory writes stay outside Core - Decision: The open-source Core startup always installs `SingleWorkspacePolicy`, creates or repairs one local Workspace, and permits local membership/invitation workflows. Changing mutable configuration such as `system.edition` cannot activate multi-Workspace routing. The future closed Cloud bootstrap will install `CloudWorkspacePolicy` only after verifying a signed `InstanceManifest`; that policy requires an explicit projected Workspace selector, does not create Workspaces, and rejects invitation or membership mutations with `control_plane_required`; member reads use the versioned local projection. -- Ownership split: The closed Control Plane owns the global Account/Workspace/Membership/Invitation directory, placement and lease generations, entitlements, subscription state, usage aggregation, and billing decisions. Core owns request authorization, resource scoping, execution-generation validation, and fail-closed enforcement. Provisioning and invoice computation do not belong in open-source Core. +- Ownership split: The closed Control Plane owns the global Account/Workspace/Membership/Invitation directory, execution ownership and generation, entitlements, subscription state, usage aggregation, and billing decisions. Core owns request authorization, resource scoping, execution-generation validation, and fail-closed enforcement. Provisioning and invoice computation do not belong in open-source Core. - Reason: The closed control plane is authoritative for SaaS Account, Workspace, Membership, and Invitation state. Allowing Core to mutate the same directory would create split-brain ownership and would make an ownerless compatibility Workspace a dangerous fallback. - Release gate: Multi-Workspace activation is deliberately unavailable in the open-source bootstrap. Production Cloud v2 must implement the signed `InstanceManifest` verifier and closed bootstrap described in the architecture document before it can inject `CloudWorkspacePolicy`; `edition=cloud`, an environment variable, or any unsigned local configuration is never a valid activation credential. @@ -110,7 +111,7 @@ This log records implementation choices made while delivering the Workspace arch ### Dashboard WebSocket sessions are tenant runtime objects -- Decision: A dashboard WebSocket sends an authentication frame immediately after upgrade. The server validates Account, Membership, permission, Pipeline ownership, instance, Workspace, and placement generation before registering the connection. Connection indexes, sessions, broadcasts, attachments, and resets include the complete execution scope. +- Decision: A dashboard WebSocket sends an authentication frame immediately after upgrade. The server validates Account, Membership, permission, Pipeline ownership, instance, Workspace, and execution generation before registering the connection. Connection indexes, sessions, broadcasts, attachments, and resets include the complete execution scope. - Reason: Browser WebSocket APIs cannot attach the normal authorization headers, and a process-global `pipeline_uuid` or `session_type` index can collide across Workspaces. ### Read permissions never imply secret permissions @@ -120,7 +121,7 @@ This log records implementation choices made while delivering the Workspace arch ### Temporary credential exchanges are bound to their initiator -- Decision: Lark, Weixin, DingTalk, WeComBot, and QQOfficial one-click registration sessions require `resource.manage` and store the initiating instance, Workspace, placement generation, and principal. Status and cancellation by any other scope return the same 404 as an unknown session. +- Decision: Lark, Weixin, DingTalk, WeComBot, and QQOfficial one-click registration sessions require `resource.manage` and store the initiating instance, Workspace, execution generation, and principal. Status and cancellation by any other scope return the same 404 as an unknown session. - Reason: Random session IDs reduce guessing probability but do not authorize access to credentials returned by a completed exchange. ### Uploaded images and documents use different storage capabilities @@ -153,18 +154,18 @@ This log records implementation choices made while delivering the Workspace arch - Compatibility: In-memory SQLite cannot provide this recovery guarantee and is rejected for destructive production migration boundaries; it remains usable in tests that create the final schema directly. - Reason: SQLite batch table rebuilds can leave an installation between schemas if a process or migration fails. A verified pre-boundary image makes retry behavior recoverable instead of merely idempotent in the happy path. -### Placement generation is an execution revocation capability +### Execution generation is an execution revocation capability -- Decision: RuntimeBot, RuntimePipeline, background tasks, object storage, Plugin Runtime, MCP, RAG, and Box operations carry the complete instance, Workspace, and placement-generation scope. They revalidate the active execution binding before accessing a provider or transport; long-running calls validate again before accepting results. A stale generation is fenced before it can read, write, or reuse a cached object. +- Decision: RuntimeBot, RuntimePipeline, background tasks, object storage, Plugin Runtime, MCP, RAG, and Box operations carry the complete instance, Workspace, and execution-generation scope. The current schema and wire compatibility field remains `placement_generation` until a coordinated rename. They revalidate the active execution binding before accessing a provider or transport; long-running calls validate again before accepting results. A stale generation is fenced before it can read, write, or reuse a cached object. - Plugin boundary: Each locally launched plugin receives a short-lived, one-use registration capability bound to the expected manifest identity and execution scope. The production child environment does not inherit the reusable debug credential, and Host APIs derive scope from the trusted connection and action context. -- Box boundary: Persistent skill content remains Workspace-scoped, while session/process state and relay requests also include placement generation. A generation change retires matching live sessions and closes a stale relay before further stdin, stdout, or file operations. -- Transaction boundary: Request admission and runtime side effects are fenced in this branch, but ordinary tenant database mutations do not yet hold a generation-aware lock through commit. The closed Cloud bootstrap must remain disabled until Core provides the shared-write/exclusive-cutover transaction primitive and a generation-stamped outbox (or an equivalent atomic publish fence). The OSS singleton policy has a fixed local generation and cannot trigger a placement cutover. +- Box boundary: Persistent skill content remains Workspace-scoped, while session/process state and relay requests also include execution generation. A generation change retires matching live sessions and closes a stale relay before further stdin, stdout, or file operations. +- Transaction boundary: Request admission and runtime side effects are fenced in this branch, but ordinary tenant database mutations do not yet hold a generation-aware lock through commit. The closed Cloud bootstrap must remain disabled until Core provides the shared-write/exclusive-cutover transaction primitive and a generation-stamped outbox (or an equivalent atomic publish fence). The OSS singleton policy has a fixed local generation and cannot trigger an execution-owner cutover. - Durable-object boundary: Current opaque storage keys include generation and therefore fail closed after a generation change. That is safe for OSS's fixed generation, but a Cloud cutover must not strand durable KB files, images, or plugin references. Cloud v2 must publish stable final object identities from generation-scoped staging, or perform an atomic object-and-reference migration before activating the new generation. -- Reason: Workspace UUID prevents cross-tenant collisions, but it cannot revoke work after a Workspace is moved or fenced. Placement generation is the monotonic lease that makes old runtimes unusable. +- Reason: Workspace UUID prevents cross-tenant collisions, but it cannot revoke work after execution ownership changes or is fenced. Execution generation is the monotonic revocation value that makes old runtimes unusable; it does not express membership in a product-level deployment entity. ### Long-lived WebSockets continuously revalidate authority -- Decision: Dashboard WebSockets re-authenticate the Account, Membership, permission, resource ownership, instance, Workspace, and placement generation for every inbound message, not only during the initial frame. A changed role, removed Membership, or fenced placement takes effect without waiting for reconnect. +- Decision: Dashboard WebSockets re-authenticate the Account, Membership, permission, resource ownership, instance, Workspace, and execution generation for every inbound message, not only during the initial frame. A changed role, removed Membership, or fenced execution binding takes effect without waiting for reconnect. - Public embed boundary: The embed connection re-resolves its Bot before every message and rejects a Bot that was disabled, deleted, moved, or rebound. The public connection may identify a Bot, but it cannot make the initial Bot object an indefinite authorization capability. - Reason: Authorization and resource state can change while a socket remains open. Connection-time validation alone leaves a revocation gap. @@ -181,7 +182,7 @@ This log records implementation choices made while delivering the Workspace arch ### Box is one shared control plane with one admitted sandbox per Workspace -- Decision: The logical instance has one shared Box Runtime. A closed entitlement adapter projects generic `managed_sandbox` capability and `managed_sandbox_sessions` limit; Core and Runtime never branch on a plan name. An eligible Workspace receives at most one persistent logical `global` session, while each ordinary command remains a one-shot nsjail process. Managed processes and network are disabled in the first Cloud release. +- Decision: The logical instance has one shared Box Runtime control plane, implemented by one Runtime replica in M0. A closed entitlement adapter projects generic `managed_sandbox` capability and `managed_sandbox_sessions` limit; Core and Runtime never branch on a plan name. An eligible Workspace receives at most one persistent logical `global` session, while each ordinary command remains a one-shot nsjail process. Managed processes and network are disabled in the first Cloud release. - Storage boundary: Core and Runtime prove they see the same durable volume with an authenticated random-marker challenge. Attachments use opaque query UUID directories and link-free dirfd operations. Skill packages remain in the Runtime-owned Workspace store and enter a sandbox only as a read-only logical-name mount; Python environments and caches live in the tenant's writable Workspace. - Resource boundary: Cloud readiness requires cgroup v2 plus hard byte and inode limits for Workspace files, Skill storage, root, tmp, and home. The existing directory scan is only a compatibility soft check. Plain nsjail reports these storage capabilities as unavailable, so Cloud Box intentionally cannot start until the greenfield deployment supplies and verifies a real quota provider. - Archive boundary: Skill ZIP processing is bounded by compressed input, entry count, per-entry size, total uncompressed size, and compression ratio, and rejects links, non-regular entries, duplicates, and path escape before streaming extraction. @@ -208,3 +209,36 @@ This log records implementation choices made while delivering the Workspace arch - Decision: `mcp.stdio.enabled` is independent of Box availability and entitlement. OSS defaults it on for compatibility; Cloud requires it off at bootstrap and enforces the same gate on create, update, test, startup loading, and final runtime execution. - Reason: Treating Box availability as stdio permission would silently create another persistent `mcp-shared` sandbox for each Workspace and bypass the one-sandbox subscription and cost boundary. + +## 2026-07-20 + +### The tenant UoW owns its task, root transaction, bind, and scope + +- Decision: A tenant UoW creates one task-owned `TenantScopedAsyncSession` and one root transaction. Public commit, rollback, close, connection, bind, nested-transaction, synchronous-Session, live-streaming, raw SQL, public execution options, and public `set_config` paths fail closed and mark the transaction rollback-only. ORM objects cannot expose a usable synchronous Session, captured methods cannot run in child tasks, an explicit foreign bind is rejected, and a captured Session is permanently retired when its UoW exits rather than being reset for reuse. Tenant scope is installed only through a private UoW capability; pgvector index-plan `SET LOCAL`/`EXPLAIN` diagnostics use a test/operator connection rather than the business Session API. +- SQL boundary: Public UoW calls accept only structured SQLAlchemy query and DML trees. `TextClause`, literal SQL columns, textual labels, prefixes/suffixes/hints, statement execution options, `VALUES` roots, `INSERT FROM SELECT`, `EXTRACT`, literal-execute parameters, unknown/custom AST nodes, forced-unquoted identifiers, named `ON CONFLICT` constraints, unknown dialect post-values clauses, and untrusted casts/types fail closed. PostgreSQL/SQLite `ON CONFLICT DO UPDATE` and batch-insert containers are traversed explicitly because SQLAlchemy's standard visitor omits their executable values. Function classes are exactly allowlisted as `count`, `coalesce`, `sum`, `now`, `length`, and `nullif`; the only custom operator/cast admitted is the validated pgvector cosine operator and `Vector` cast. +- Legacy migration boundary: The local-only RAG backup restore uses explicit table and column objects, never raw SQL, but deliberately leaves legacy values untyped. This preserves SQLite's string-valued `DATETIME` rows and both the historical PostgreSQL `TEXT` and fresh-schema `JSON` settings columns while keeping every value bound rather than interpolated. +- ORM boundary: SQLAlchemy `SessionEvents` are unsupported on a tenant-scoped Session. If a listener is registered before or during a UoW, the operation fails before the callback executes and cleanup proceeds against an empty dispatch surface. Public `get`, `get_one`, `refresh`, and `merge` reject caller-supplied loader, bind, lock, shard, and execution options. Flush, implicit autoflush, and commit reject a SQL expression assigned to a mapped attribute before it can reach the compiler. Tenant code uses the async Session directly; relationships use eager loading or explicit `await session.refresh(entity, [attribute])`. LangBot's persistence base does not expose `AsyncAttrs.awaitable_attrs` as a supported tenant API. +- Compiler trust boundary: This guard prevents accidental scope/transaction escape by trusted LangBot Core code; it is not an in-process Python sandbox. Registered SQLAlchemy compilers and mapped schema metadata are trusted boot-time code. The fail-closed traversal of dialect containers necessarily covers SQLAlchemy private fields, so dependency upgrades require the regression suite and remain pinned until verified. Untrusted plugins cannot import or call this Session because they remain isolated in Plugin Runtime child processes. +- Result boundary: A caught database or boundary failure rolls back the root transaction and cancels after-commit work. Buffered results contain only already-authorized rows and no live connection; live database results cannot escape the UoW operation. +- Reason: `SET LOCAL` plus RLS protects a tenant only while every statement stays on the same owned connection and callers cannot end the transaction, replace the GUC, recover the synchronous proxy, or route a statement through another bind. + +### Parallel request work re-enters tenant scope explicitly + +- Decision: Child coroutines created by request-level `asyncio.gather` open their own explicit transaction-free tenant scope before calling persistence-backed Plugin, MCP, or Skill operations. They never inherit the parent's active database Session. +- Reason: Python copies ContextVars into child tasks, but SQLAlchemy Sessions are not task-safe and task identity is part of the tenant UoW boundary. + +### Ordinary monitoring is readable; audit and export remain privileged + +- Decision: Workspace monitoring dashboards, Bot logs, sessions, messages, calls, errors, and feedback require `resource.view`. Monitoring export requires `data.export`; system/runtime audit logs keep `audit.view`. Frontend tabs and controls use the same split. +- Reason: A Viewer needs useful read-only product observability, while bulk data extraction and privileged runtime/system logs are separate capabilities. + +### Invitation failures survive login without contradictory success state + +- Decision: Invitation terminal codes map to stable browser states. A login-mediated email mismatch preserves the fragment-captured secret only in session storage, returns to the acceptance page with the stable error code, and suppresses the generic login-success toast. A transient acceptance failure retains the authenticated session and offers the same one-time invitation for retry. Invalid Bearer tokens on acceptance map to the normal authentication error instead of an internal failure. +- Reason: Invitation acceptance crosses signed-out and authenticated states; losing or masking the domain error makes recovery ambiguous and can present contradictory UI feedback. + +### Shared Plugin Runtime starts only from verified desired state + +- Decision: SDK shared mode waits for immutable runtime configuration before inspecting plugin state, never scans or launches legacy `data/plugins`, and rejects legacy install/restart/delete/upgrade control actions. Worker RPC files use installation-private directories with aggregate size enforcement, and resident nsjail workers explicitly disable the default 600-second wall-time limit. +- Remaining gate: Unexpected worker exit is still recovered only by the next Core apply/reconcile. Completion callbacks, bounded backoff, hard installation disk quota, and production Linux/cgroup/egress evidence remain Cloud activation requirements. +- Reason: A shared supervisor reduces per-Workspace services only if legacy global paths, writable transfer state, and lifecycle defaults cannot bypass installation isolation. diff --git a/docs/multi-tenant/pending-architecture-decisions.md b/docs/multi-tenant/pending-architecture-decisions.md index 651d3400d..b47b46e9e 100644 --- a/docs/multi-tenant/pending-architecture-decisions.md +++ b/docs/multi-tenant/pending-architecture-decisions.md @@ -28,7 +28,7 @@ | 编号 | 结论 | 首期状态 | | ----- | --------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | -| D-001 | 一个共享 Plugin Runtime 控制面；每个运行中的 plugin installation 独占一个 nsjail 子进程；只有 digest 相同且已验证的代码 artifact 可以只读共享 | `IMPLEMENTED — real Linux/egress deployment gate pending` | +| D-001 | 一个共享 Plugin Runtime 控制面；每个运行中的 plugin installation 独占一个 nsjail 子进程；只有 digest 相同且已验证的代码 artifact 可以只读共享 | `FOUNDATION IMPLEMENTED — Linux/egress, crash recovery and disk-quota gates pending` | | D-002 | 一个共享 Box Runtime；Cloud 固定使用 nsjail；符合套餐的 Workspace 最多一个持久 `global` 逻辑 sandbox，普通执行按需启动 nsjail 进程 | `IMPLEMENTED FAIL-CLOSED — hard filesystem quota provider pending` | | D-003 | SaaS 业务数据使用 PostgreSQL shared schema、应用层作用域和 RLS 双重隔离；pgvector 使用同一 PostgreSQL，作为 SaaS 默认向量后端 | `PARTIALLY IMPLEMENTED — transaction/outbox/deployment gates remain` | | D-004 | stdio MCP 与 Box availability 解耦；Cloud v2 首期强制关闭 stdio MCP，避免为每个 Workspace 创建额外的 `mcp-shared` persistent sandbox | `IMPLEMENTED` | @@ -82,12 +82,12 @@ Core 不能继承 Plugin Runtime 所需的 nsjail/cgroup 权限。Box Runtime ## 3. D-001：Plugin Runtime 多租户控制面 -状态：`IMPLEMENTED — Cloud deployment verification pending` +状态：`FOUNDATION IMPLEMENTED — Cloud deployment and worker-recovery gates pending` ### 3.1 已实现的基础 -- Plugin Runtime 控制连接只绑定稳定实例身份；一个 Supervisor 通过完整 installation binding 管理多个 Workspace。 -- 每个 enabled installation 使用独立 nsjail worker；代码只读，home/tmp/data 私有，shared profile 不读取 artifact `.env`。 +- Plugin Runtime 控制连接只绑定稳定实例身份；一个逻辑共享控制面通过完整 installation binding 管理多个 Workspace，M0 由一个 Supervisor replica 承担。 +- 每个运行中的 installation 使用独立 nsjail worker；enabled-resident 是 desired semantics。代码只读，home/tmp/data 私有，shared profile 不读取 artifact `.env`。 - 实例级 `PluginWorkerPolicy` 由 Core 的 `data/config.yaml` 下发，支持原生环境变量覆写；manifest 不能覆盖。 - `installation_uuid`、`artifact_digest` 和 `runtime_revision` 已持久化并进入 desired-state、注册、Host API 和 generation/revision fence。 - 已验证 `.lbpkg` 先进入 Workspace-scoped durable binary storage；Runtime 本地缓存丢失后可由 Core replay。 @@ -110,13 +110,14 @@ Core 不能继承 Plugin Runtime 所需的 nsjail/cgroup 权限。Box Runtime ### 3.3 首期执行模型 -- 整个 SaaS 实例共享一个可信 Plugin Runtime Supervisor；新 Workspace 不创建专属 Runtime、连接、卷或进程。 +- 整个 SaaS 实例共享一个可信 Plugin Runtime 逻辑控制面，M0 运行一个 Supervisor replica；新 Workspace 不创建专属 Runtime、连接、卷或进程。 - Supervisor 的控制连接只绑定稳定 `instance_uuid` 和短期 Runtime identity，不绑定某个 Workspace。 每条 installation desired-state 命令都携带并验证完整的 installation binding；每个 worker action context 在注册后永久绑定该 tuple。 - 安装并启用插件后，Supervisor 在自己的 Runtime 容器内直接启动一个 nsjail 子进程； 不再为每个插件创建 nested container、Pod、sidecar 或租户级 Runtime service。 -- 为兼容现有事件监听和定时任务语义，首期继续采用“enabled installation 保持 resident”，不做 idle eviction； - 停用、删除、revision/generation 变化或 entitlement 撤销时停止并按需重建。 +- desired semantics 要求 enabled installation 保持 resident，不做 idle eviction；停用、删除、revision/generation 变化或 entitlement 撤销时停止并按需重建。 + 当前实现只会在 Runtime 重连或 Core apply/reconcile 时恢复意外退出的 worker，尚缺 completion callback、有界 backoff 和跨租户重启风暴抑制； + 这属于 Cloud 激活门禁，不能把 desired semantics 描述成已经具备即时自愈。 - 子进程使用一次性 registration capability 向 Supervisor 注册；capability 由可信 desired state 派生并绑定完整 installation tuple， 不是插件直接建立 Core Host connection，也不能只绑定 author/name/path。Supervisor/Core 据此注入 tenant context， 丢弃插件 payload 中自带的 scope 字段。 @@ -193,8 +194,9 @@ installation data 的总空间硬配额需要 filesystem project quota 或独立 | MVP 不引入 | Runtime 专用数据库、Redis、Kafka 或独立 scheduler | 当前无容量证据，会增加组件和运维面 | | 后续演进 | 多 Supervisor replica、owner lease、dedicated pool | 保留接口，达到容量或可用性阈值后再决定具体存储与调度方式 | -后续仍需决定的只有：Core/Supervisor 是否共置、artifact/venv cache 的签名/来源/撤销/GC 规范、v1 connection 的兼容期限， -以及进入多 replica 后的 lease TTL、fencing token 和 owner 转移顺序。这些不改变“每 installation 一个隔离进程”的首期边界。 +架构扩展项包括：Core/Supervisor 是否共置、artifact/venv cache 的签名/来源/撤销/GC 规范、installation data hard-quota provider、 +v1 connection 的兼容期限，以及进入多 replica 后的 lease TTL、fencing token 和 owner 转移顺序。 +这些不改变“每个运行中的 installation 一个隔离进程”的首期边界。 ### 3.7 验收条件 @@ -203,8 +205,10 @@ installation data 的总空间硬配额需要 filesystem project quota 或独立 - 插件不能读取其他 installation 文件、枚举或 signal 其他进程，也不能修改共享代码/依赖目录。 - CPU、内存、PID、open files 和单文件上限在真实 nsjail/cgroup 环境中生效；超额只终止或拒绝对应 installation。 - 修改 manifest 不能改变任何资源上限。 +- installation data 的总空间硬配额在写入边界原子拒绝超额，并证明目录扫描不是生产 enforcement。 - 旧 generation/revision 的回调、消息、副作用和存储访问全部失败关闭。 - Runtime 重启能从业务 desired state 恢复，不依赖本地进程表作为权威真相。 +- 意外退出的 enabled worker 由 completion callback 触发带有界 backoff 的自动恢复；连续失败只影响对应 installation，不能形成跨租户重启风暴。 - requirements 中存在 Runtime 基础镜像未预装的包时，Supervisor 仍能先完成共享依赖环境准备再启动 worker； 安装失败不会留下持续重启的半启动进程，也不会影响同 digest 已就绪环境的其他 installation。 @@ -253,7 +257,7 @@ installation data 的总空间硬配额需要 filesystem project quota 或独立 ### 4.3 Cloud nsjail 执行模型 -- 整个逻辑 SaaS 实例共享一个 Box Runtime 控制面；不创建每 Workspace Box service、worker pool、PVC、bucket、scheduler、Redis 或 Box 数据库。 +- 整个逻辑 SaaS 实例共享一个 Box Runtime 逻辑控制面，M0 运行一个 Runtime replica；不创建每 Workspace Box service、worker pool、PVC、bucket、scheduler、Redis 或 Box 数据库。 - Cloud 显式固定 `box.backend: nsjail`。sandbox 直接作为 Box Runtime 容器内的 nsjail 子进程运行， 不创建 nested Docker container、独立 Pod、microVM 或 warm pool，也不挂宿主机 `docker.sock`。 - 符合 entitlement 的 Workspace 首次使用时懒创建一个逻辑 session，内部固定 ID 为 `global`，并强制 `persistent=True`； diff --git a/docs/multi-tenant/workspace-multi-user-architecture.md b/docs/multi-tenant/workspace-multi-user-architecture.md index 1eecb8574..ff50def2e 100644 --- a/docs/multi-tenant/workspace-multi-user-architecture.md +++ b/docs/multi-tenant/workspace-multi-user-architecture.md @@ -1,111 +1,176 @@ # LangBot Workspace 多用户与 SaaS 多租户架构 -> **架构修订中：** 已确认的 SaaS 形态是一个逻辑 LangBot 实例承载全部 Workspace；当前不做分布式， -> 但协议和状态必须支持未来在同一逻辑实例内增加 replica、worker 和数据库 shard。 -> 本文后续 Cell、CloudInstance、Workspace Placement 和按 CloudInstance 分配数据库/Runtime 的章节属于旧提案， -> 在 [待决策项](./pending-architecture-decisions.md) 完成讨论前不得作为最终 Cloud v2 拓扑实施。 +状态：`ARCHITECTURE BASELINE — isolation kernel implemented; SaaS activation gates remain` -## 1. 决策摘要 +本文描述 Cloud v2 的目标架构和安全边界。详细的 Runtime、Box、PostgreSQL、pgvector 与 stdio MCP 决策以 +[pending-architecture-decisions.md](./pending-architecture-decisions.md) 为权威来源；已经落地的实现选择记录在 +[implementation-decisions.md](./implementation-decisions.md)。 -本方案采用以下架构： +“隔离内核已实现”仅表示开源 Core/SDK 已具备多租户数据和运行时隔离所需的基础能力， +不表示闭源控制面、计费、生产部署或 Cloud v2 已经可以上线。 -> 开源 LangBot Core 提供完整的租户隔离内核和单 Workspace 多用户能力；SaaS 使用独立闭源 Cloud Control Plane 管理全局账户、多 Workspace、计费、权益和 SaaS 生命周期。 +## 1. 架构决策摘要 -这是本次设计的核心边界： +Cloud v2 采用以下模型： -- Workspace 是 LangBot 实例内的逻辑租户边界，不是一个 Pod，也不是一个 Kubernetes namespace。 -- OSS 每个 LangBot 实例只能存在一个 Workspace，但该 Workspace 可以有多个用户、邀请和固定角色权限。 -- SaaS 一个账户可以拥有或加入多个 Workspace；全部 Workspace 由同一个逻辑 LangBot 实例承载。 -- 当前实例可以单副本运行；未来 Core、Runtime、Box 或数据库可以在不引入多个产品级 LangBot 实例的前提下横向扩展。 -- SaaS 普通注册会自动创建个人 Workspace。通过邀请注册的新用户也会创建个人 Workspace，并同时加入受邀 Workspace。 -- OSS 首位用户创建实例唯一 Workspace；后续用户通过邀请加入该 Workspace，不再创建第二个 Workspace。 -- 数据隔离、权限校验、运行时隔离必须留在开源 Core 中，不能依赖闭源服务在线代理每次请求。 -- SaaS 全局账户、Workspace 目录、Membership、Invitation、Subscription、Billing 和 Entitlement 由闭源控制面维护；Core 只保存执行所需的 Account、Workspace 和 Membership 版本化投影，不投影 pending Invitation secret。 -- 现有 Space 的 Pod-per-account、namespace-per-account 和 Pod subscription 模型不进入新架构。Cloud 部署按绿地方案重新设计。 -- 现有 Space 只复用账户交互、OAuth、支付适配器、邮件、财务后台和生命周期可靠性经验，不复用旧 Cloud 领域模型。 +> SaaS 对外只有一个逻辑 LangBot 实例，全部 Workspace 都是该实例内的租户； +> 开源 Core 提供完整隔离内核，闭源 Cloud Control Plane 管理 SaaS 目录、订阅、权益和计费。 -推荐将 Cloud Control Plane 做成独立进程、独立发布、独立数据库和独立信任域的闭源服务。第一阶段可以和 Space 放在同一个私有代码仓库中，但不能作为 LangBot Core 的 Python 闭源补丁，也不应继续把新 Cloud 逻辑堆进现有 Space 单体。 +核心决策如下： + +1. `Workspace` 是数据、成员、权限、用量和不可信执行的租户边界，不是一个 Pod、namespace、数据库或独立 LangBot 部署。 +2. SaaS 注册 Account 时自动创建个人 Workspace；这只新增目录与业务记录，不创建租户专属服务、数据库、队列或 Runtime。 +3. OSS 每个 LangBot 实例只能存在一个 Workspace，但该 Workspace 可以有多个 Account、邀请和固定角色。 +4. SaaS 才允许一个 Account 拥有或加入多个 Workspace，并在 WebUI 中切换当前 Workspace。 +5. MVP 可以各运行一个 Core、Plugin Runtime 和 Box Runtime 进程；未来增加副本或 PostgreSQL shard 仍属于同一个逻辑实例的内部扩展，不改变产品模型和外部 API。 +6. 一个共享 Plugin Runtime 控制面管理所有 Workspace，但每个运行中的 plugin installation 独占一个 nsjail 进程；enabled-resident 是 desired semantics，只读代码和依赖可按已验证摘要共享。 +7. 一个共享 Box Runtime 管理所有 Workspace；首期符合 entitlement 的 Workspace 最多拥有一个持久 `global` 逻辑 sandbox，实际命令继续以 nsjail 子进程执行。 +8. SaaS 业务数据使用 PostgreSQL shared schema、应用层 scope 与 RLS 双重隔离；pgvector 位于同一个业务数据库并作为 SaaS 默认向量后端。 +9. stdio MCP 有独立实例开关，Cloud v2 首期强制关闭，不能由 Box availability 或套餐能力隐式开启。 +10. 闭源 Control Plane 可以作为模块化单体复用现有账户、支付和运营能力，但历史 Cloud 的租户专属部署模型不进入新架构。 +11. Workspace 创建、释放、export、单 Workspace restore 和在线迁移的具体流程仍待后续决策。 + +本轮重构的最高目标是： + +> 共享可信控制面和基础设施池，隔离不可信执行单元；减少独立部署和常驻组件，使新增 Account 或空 Workspace 的静态成本接近零。 + +减少组件数量不意味着合并安全边界。插件进程、sandbox、secret、可写文件和租户数据仍必须严格隔离。 ## 2. 范围与非目标 ### 2.1 本方案覆盖 - OSS 单 Workspace 多用户、邀请和固定 RBAC。 -- SaaS 多 Workspace 账户与成员模型。 -- Workspace 行级数据隔离。 -- HTTP、API Key、Bot、Webhook 和内部服务鉴权。 -- Query、Session、Plugin、MCP、RAG、Box、Storage 和 Monitoring 运行时隔离。 -- SaaS Control Plane 与 LangBot Data Plane 的协议边界。 -- Workspace 级订阅、权益、用量和计费模型。 -- 单逻辑 LangBot SaaS 实例，以及未来内部 replica、worker 和 database shard 的兼容边界。 -- 分阶段实施和验收策略。 +- SaaS 多 Workspace 账户、成员和 Workspace 切换模型。 +- HTTP、WebSocket、API Key、Bot、Webhook、后台任务和内部调用的可信 Workspace 上下文。 +- Bot、Pipeline、Provider、Knowledge、Plugin、MCP、RAG、Session、Storage 和 Monitoring 的租户隔离。 +- Plugin Runtime 与 Box Runtime 的共享控制面和进程级隔离。 +- SaaS PostgreSQL shared schema、RLS 与 pgvector 边界。 +- 开源 Core 与闭源 Control Plane 的职责、协议和故障边界。 +- 当前单副本运行和未来同一逻辑实例内横向扩展的兼容约束。 +- 分阶段实施、激活门禁和验收策略。 ### 2.2 本方案不覆盖 -- 兼容旧 Space 的 Pod、账户 namespace 或旧 CloudPlan 部署方式。 -- 在旧 Pod 模型上原地升级为多租户。 -- 旧 Cloud 客户数据和基础设施的具体迁移执行方案。 +- 兼容或原地升级历史 Cloud 的租户专属部署方案。 +- 为每个 Workspace 创建独立服务、数据库、schema、role、bucket、PVC、队列或 Runtime。 +- 当前阶段实现多副本调度、跨地域 active-active 或 PostgreSQL 在线分片迁移。 - 第一版自定义角色、SAML、SCIM 或企业离线授权。 -- 第一版跨 Cell active-active。 -- 第一版 database-per-workspace 或 schema-per-workspace。 +- 第一版 Workspace 级 BYOK E2B WebUI 配置。 +- Cloud v2 首期 stdio MCP。 +- Workspace export、释放、单租户恢复和在线迁移的具体产品流程。 -如果未来需要迁移旧 Cloud 的账户、财务记录或客户数据，应单独立项；旧部署模型不作为新架构的设计约束。 +历史客户数据、账户和财务记录如需迁移，应单独立项；旧部署拓扑不作为本架构的设计约束。 ## 3. 术语与不变量 ### 3.1 术语 -| 术语 | 定义 | -| ------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| Account | 登录主体。OSS 中是实例本地账户；SaaS 中是全局账户 | -| Workspace | LangBot 实例内的逻辑租户；active 后属于一个 LangBot Instance，是资源、成员、权限、运行时和计费的主要边界 | -| Membership | Account 与 Workspace 的关系，包含角色、状态和权限版本 | -| Invitation | 邀请一个用户或邮箱加入 Workspace 的一次性凭证；SaaS 要求 verified email | -| LangBot Core | 开源数据面，运行 Bot、Pipeline、Model、Plugin、MCP、RAG 等业务 | -| Cloud Control Plane | 闭源 SaaS 控制面，管理全局身份、Workspace 目录、计费、权益、放置和运维 | -| CloudInstance | 一个可承载多个 Workspace 的 LangBot 逻辑部署单元，拥有独立 Manifest、Core deployment、数据 namespace、release 和 runtime ownership domain | -| Cell | 一个区域或故障域中的基础设施与运维单元，共享计算和存储集群但承载多个逻辑隔离的 CloudInstance | -| Placement | Workspace 到 Cell 和 CloudInstance 的版本化放置关系 | -| BillingAccount | 付款主体，可以为一个或多个 Workspace 付费 | -| Entitlement | 控制面签发、Core 本地执行的功能和额度快照 | +| 术语 | 定义 | +| --- | --- | +| Account | 登录主体。OSS 中是实例本地账户；SaaS 中是全局账户 | +| Workspace | 逻辑 LangBot 实例内的租户，是资源、成员、权限、用量和不可信执行的首要边界 | +| Membership | Account 与 Workspace 的关系，包含固定角色、状态和权限版本 | +| Invitation | 邀请一个 Account 或邮箱加入 Workspace 的一次性凭证 | +| Logical Instance | 对外唯一的 LangBot 服务与安全域，拥有稳定 `instance_uuid`，不等同于某个进程或 Pod | +| Replica | Core、Plugin Runtime 或 Box Runtime 的短期内部运行副本，不是产品实体 | +| Execution Generation | Workspace 执行所有权和撤销的单调代数，用于隔离旧任务、旧连接和故障转移 | +| Billing Account | SaaS 付款主体，可以为一个或多个 Workspace 付费 | +| Entitlement | Control Plane 签发、Core 与 Runtime 本地执行的功能和数值额度快照 | +| Cloud Control Plane | 闭源 SaaS 控制面，管理全局身份、Workspace 目录、订阅、权益、计费和生命周期 | +| LangBot Core | 开源数据面，执行 Bot、Pipeline、Plugin、MCP、RAG 等业务并实施最终授权与隔离 | + +当前代码中的 `placement_generation` 字段在迁移完成前保留兼容；其架构语义和目标命名均为 +`execution_generation`，不表达 Workspace 属于某个产品级部署单元。 ### 3.2 必须始终成立的不变量 -1. SaaS 中一个 active 或 routable Workspace 在任一时刻必须且只能属于一个有效 CloudInstance placement；provisioning、dormant 或 archived Workspace 可以暂时没有 Placement。OSS Workspace 直接属于本地 LangBot Instance，不使用 Cloud Placement。 -2. 一个 CloudInstance 可以承载多个 Workspace。 -3. OSS 一个实例最多只有一个 Workspace。 -4. Workspace 可以有多个 Account；Account 在 SaaS 可以加入多个 Workspace。 -5. 所有租户业务资源都有非空 workspace_uuid。 -6. 所有资源读取和写入都使用 workspace_uuid 与 resource_id 共同定位。 -7. Workspace 选择器不是授权凭证；服务端必须重新验证 Membership。 -8. Core 是资源访问和运行时授权的最后一道边界。 -9. Control Plane 不参与每条消息或每个普通资源请求的同步鉴权。 -10. SaaS 缺少有效 Workspace 上下文时必须失败，不能回退到第一个或最近 Workspace。 -11. 后台任务和运行时任务必须携带显式 Workspace 或受约束的 SystemContext。 -12. Workspace、Deployment 和 BillingAccount 是三个不同概念，不能再次合并为 Pod。 +1. SaaS 只有一个稳定 `instance_uuid`；所有副本共享该身份。 +2. `replica_id`、`worker_id`、Pod 名称、进程地址和数据库连接地址都是短期运行信息，不能进入业务资源的永久主键或外部 URL。 +3. `workspace_uuid` 是租户数据、任务、缓存、文件、日志、用量和运行时隔离的稳定键，也是未来内部路由与分片的候选键。 +4. OSS 一个实例最多一个 Workspace；SaaS 才能激活多个 Workspace。 +5. 一个 Workspace 可以有多个 Account；一个 SaaS Account 可以加入多个 Workspace。 +6. 所有租户业务资源都具有非空 `workspace_uuid`，并使用 `(workspace_uuid, resource_uuid)` 定位。 +7. Workspace 选择器只是路由输入，不是授权凭证；服务端必须重新验证 Account、Membership、资源所有权和权限。 +8. API Key、Bot、Webhook、后台任务、Plugin 与 Box 调用从可信所有权或绑定派生 Workspace，不能信任调用方自报 scope。 +9. SaaS 缺少有效 Workspace 上下文时必须失败关闭，不能回退到第一个、最近或 OSS 默认 Workspace。 +10. Core 是资源访问、运行时授权和 entitlement 执行的最后一道边界；Control Plane 不同步代理每条消息或普通资源请求。 +11. 一个不可信插件进程只能属于一个 installation；一个 sandbox/session 只能属于一个 Workspace。 +12. execution generation 失效后，旧任务、连接、回调和副作用必须被拒绝。 +13. 本地进程表、缓存和临时目录都可重建，不能成为 desired state、撤销状态或业务数据的唯一真相。 +14. 创建空 Workspace 不启动插件 worker、sandbox 或租户专属常驻组件。 +15. 未来横向扩展不能改变 Workspace UUID、外部 API、权限模型或隔离语义。 -Control Plane 可以先创建 status = provisioning 或 dormant 的 Workspace directory record，但它只是全局索引，不承载 LangBot 业务资源。SaaS Workspace 只有获得唯一 active Placement 后才成为可使用的实例内租户；所有 Bot、Pipeline、Plugin 等资源始终落在该 CloudInstance 内。OSS 初始化时直接在本地实例创建 active Workspace。 +## 4. 产品与部署模型 -## 4. 产品行为 +### 4.1 SaaS 逻辑拓扑 -### 4.1 OSS 与 SaaS 能力矩阵 +```mermaid +flowchart LR + User["Browser / API / Bot traffic"] --> Edge["SaaS Edge"] + User --> CP["Closed Cloud Control Plane
directory + subscription + billing"] + Edge --> Core["One logical LangBot instance
Core replica pool; MVP = 1"] + CP -->|"signed manifest, directory projection,
entitlement and desired state"| Core + Core -->|"usage outbox and observed state"| CP + Core --> PG["Shared PostgreSQL business database
RLS + pgvector"] + Core --> PluginRT["Shared Plugin Runtime
trusted supervisor"] + Core --> BoxRT["Shared Box Runtime
trusted supervisor"] + PluginRT --> PluginA["Workspace A installation
isolated nsjail process"] + PluginRT --> PluginB["Workspace B installation
isolated nsjail process"] + BoxRT --> SandboxA["Workspace A
persistent global logical sandbox"] + BoxRT --> SandboxB["Workspace B
persistent global logical sandbox"] + Core --> ObjectStore["Shared durable object storage
Workspace-scoped keys"] +``` -| 能力 | OSS | SaaS | -| ------------------------------------- | ---------------------------- | --------------------------------------- | -| 每个 Account 可拥有的 Workspace | 不适用；本地实例只有一个 | 由 ProductPolicy 决定 | -| 每个 CloudInstance 可承载的 Workspace | 固定 1 | 由 RuntimeClass 和 capacity policy 决定 | -| Workspace 成员数 | 多用户 | 按 Workspace 权益 | -| 邀请成员 | 支持 | 支持 | -| 固定 RBAC | 支持 | 支持 | -| 自定义角色 | 不支持 | 后续商业能力 | -| Workspace 创建 | 仅初始化时创建唯一 Workspace | 注册自动创建，也可按权益继续创建 | -| Workspace 切换 | 无需展示 | 支持 | -| 全局账户 | 无 | 支持 | -| 订阅与计费 | 无远端依赖 | 控制面管理 | -| SSO、SCIM | 不支持 | 后续商业能力 | -| 租户隔离 | 必须完整实现 | 必须完整实现 | +这里的“一个逻辑实例”是一个服务、安全域和稳定身份，不是“永远只有一个 OS 进程”。 +MVP 不实现分布式，但从第一天保留内部扩展所需的身份、幂等、generation 和 owner 抽象。 -OSS 的 edition policy 应表达为： +### 4.2 容量演进 + +| 阶段 | 内部部署形态 | 新 Workspace 静态成本 | 启用条件 | +| --- | --- | --- | --- | +| M0 单副本 MVP | 一个 Core、一个共享 Plugin Runtime、一个共享 Box Runtime、一个 PostgreSQL business database | 只新增目录和业务行 | 当前目标 | +| M1 同逻辑实例横向扩展 | 按容量增加 Core/Runtime 副本；使用 owner lease、fencing 和 generation；PostgreSQL 可增加 shared shard | 不创建 Workspace 专属部署 | 出现容量或可用性证据后 | +| M2 Dedicated 资源等级 | 特定 workload 使用独享 worker pool、sandbox class 或 database shard，但沿用相同身份、协议和 schema | 仅购买该等级的客户承担 | 合规、驻留或超大负载需求 | + +M1 是 M0 的透明扩容，M2 是相同架构下的资源等级。外部 API 只认识稳定的 +`instance_uuid` 和 `workspace_uuid`，不认识 replica、worker、pool 或 shard。 + +### 4.3 当前不做分布式时必须预留的能力 + +1. 运行时协议携带稳定 `instance_uuid`、`workspace_uuid` 和 `execution_generation`，不依赖进程地址表达身份。 +2. Plugin installation 和 Box session 使用稳定 owner 抽象；启用第二个副本前再实现带 expiry、CAS 和 fencing token 的 lease。 +3. 创建、重试、回调、worker 注册和 outbox 使用稳定 idempotency key，重复投递不能产生第二个 owner 或副作用。 +4. Repository/UoW 不允许无边界跨 Workspace 事务；`workspace_uuid` 可直接作为未来 shard key。 +5. schema migration、后台任务扫描、监控聚合和运维接口不能假设永远只有一个 Core 进程。 +6. Runtime 重启通过 durable desired state reconciliation 恢复，不依赖原进程或本地 cache。 +7. 只有出现容量、可用性、地域或合规证据后才增加副本、lease store 或 shard router；预留协议不等于提前部署组件。 + +### 4.4 组件边界 + +- Core、Plugin Runtime 和 Box Runtime 可以由发布与故障域决定是否共用 Pod，但必须保持独立进程身份、容器和 security context。 +- Core 不能继承 nsjail、cgroup 或 mount namespace 所需的高权限。 +- Plugin Runtime 与 Box Runtime 不合并为一个高权限进程。 +- MVP 不新增 Runtime 专用数据库、Box 专用数据库、Kafka、Redis、租户级 scheduler 或 artifact service。 +- 可信 supervisor、数据库连接池、只读 artifact cache 和基础容量可以多租户共享。 + +## 5. OSS 与 SaaS 产品行为 + +### 5.1 能力矩阵 + +| 能力 | OSS | SaaS | +| --- | --- | --- | +| Workspace 数量 | 实例固定一个 | Account 可拥有或加入多个，受 ProductPolicy 约束 | +| Workspace 成员 | 多用户 | 多用户，受 entitlement 约束 | +| 邀请成员 | 支持 | 支持 | +| 固定 RBAC | 支持 | 支持 | +| 自定义角色 | 不支持 | 后续商业能力 | +| Workspace 创建 | 首次初始化创建唯一 Workspace | 注册自动创建个人 Workspace；后续创建受 ProductPolicy 约束 | +| Workspace 切换 | 无需展示 | 支持 | +| 订阅与计费 | 无远端依赖 | 闭源 Control Plane 管理 | +| 租户隔离 | 完整实现 | 完整实现 | + +OSS edition policy 应表达为： ```text workspace_limit = 1 @@ -115,11 +180,11 @@ fixed_rbac_enabled = true multi_workspace_enabled = false ``` -不能再使用 member_limit = 1，也不能通过关闭邀请或关闭 RBAC 来限制 OSS。 +不能用 `member_limit = 1`、关闭邀请或移除 RBAC 来实现单租户限制。 -### 4.2 OSS 注册与邀请 +### 5.2 OSS 初始化和邀请 -首次初始化必须在一个数据库事务内完成： +首次初始化在一个事务中完成： 1. 创建本地 Account。 2. 创建实例唯一 Workspace。 @@ -127,427 +192,177 @@ multi_workspace_enabled = false 4. 创建默认 Pipeline、metadata 等 Workspace 初始资源。 5. 标记实例初始化完成。 -初始化完成后默认关闭公开注册。后续用户流程： +初始化后默认关闭公开注册。后续用户由 owner/admin 创建一次性 Invitation，注册或登录后接受邀请并加入唯一 Workspace。 +OSS 后续注册不创建第二个 Workspace。未配置 SMTP 时，系统返回只展示一次的邀请链接供管理员通过可信渠道发送。 -1. owner 或 admin 创建 Invitation。 -2. 被邀请用户打开一次性邀请链接并完成注册。 -3. 接受 Invitation。 -4. 原子创建或激活 Membership。 -5. 用户进入实例唯一 Workspace。 +### 5.3 SaaS 注册和邀请 -OSS 后续注册不创建新 Workspace。这是“注册自动创建 Workspace”规则在单 Workspace edition 下唯一必要的例外。 +普通注册由 Control Plane 通过幂等工作流完成： -OSS 不能强制依赖 SMTP。未配置邮件服务时，管理员可以生成只展示一次的一次性邀请链接并通过可信渠道发送；配置邮件后再由系统发送并校验邮箱。可以提供显式配置允许管理员直接创建本地用户或开放注册，但新用户仍只能加入唯一 Workspace，默认角色不得高于 viewer。 +1. 创建或确认全局 Account 与 AuthIdentity。 +2. 创建 personal Workspace 和 owner Membership。 +3. 创建初始 Subscription/Entitlement 投影。 +4. 完成 verified email、速率限制和基础风控。 +5. 将 Account、Workspace 和 Membership 投影到 Core。 +6. Core 达到要求的目录 revision 后返回可访问 route。 -### 4.3 SaaS 普通注册 +注册只创建逻辑记录，不启动 Runtime 或租户专属基础设施。 -普通注册必须由 Control Plane 通过幂等工作流完成： +通过邀请注册的新用户也创建自己的 personal Workspace，同时加入受邀 Workspace；已注册用户接受邀请时只新增目标 Membership。 +个人 Workspace 与团队 Workspace 的付费关系必须由 ProductPolicy 明确，不允许代码根据名称或创建路径隐式推断。 -1. 创建全局 Account 和 AuthIdentity。 -2. 创建 personal Workspace。 -3. 创建 owner Membership。 -4. 创建个人免费 Subscription 或 Entitlement。 -5. 完成 verified email、速率限制和基础风控。 -6. 首次访问需要运行资源时再为 Workspace 分配 Placement。 -7. 将 Workspace、Account 和 Membership 投影到目标 Core。 -8. 目标 Core 就绪后返回可访问 route。 - -注册请求必须带 idempotency key，任何步骤重试都不能创建重复 Account、Workspace、Subscription 或 Placement。 - -逻辑 personal Workspace directory record 可以在注册事务中立即创建，但在 Placement active 前不能创建 LangBot 业务资源，也不应为每个未活跃账户启动完整 Runtime。Control Plane 需要对未使用的免费 Workspace 懒分配、休眠和回收，防止邀请或注册风暴放大基础设施成本。 - -### 4.4 SaaS 邀请注册 - -邀请一个未注册用户时： - -1. 用户通过一次性 Invitation token 完成注册。 -2. Control Plane 创建全局 Account。 -3. 按产品要求创建该用户的 personal Workspace 和 owner Membership。 -4. 同时接受目标 Workspace 的 Invitation。 -5. 登录后默认进入受邀 Workspace。 - -如果用户已注册，接受邀请只创建目标 Membership，不再创建新的 personal Workspace。 - -个人 Workspace 建议使用独立的免费 entitlement，不自动成为团队 Workspace 的付费席位或额外付费对象。是否计入账户可拥有 Workspace 上限，应在 ProductPolicy 中显式定义，不能由代码隐式推断。 - -### 4.5 Invitation 安全规则 +### 5.4 Invitation 安全规则 - token 使用至少 256-bit 加密安全随机数，数据库只保存 hash。 -- token 有 expires_at、accepted_at、revoked_at，并且只能使用一次。 -- 重发邀请必须撤销旧 token。 -- 同一 workspace_uuid 与 normalized_email 同时只能有一个有效邀请。 -- 接受邀请必须锁定记录，Membership 创建和 token 消费在同一事务内提交。 +- token 具有 `expires_at`、`accepted_at`、`revoked_at`，只能使用一次。 +- Membership 创建与 token 消费在同一事务中提交。 - Invitation 不能授予 owner；owner 转移使用独立流程。 -- SaaS 或启用邮件验证的 OSS 中，OAuth 邮箱相同不能自动接受邀请，必须验证 token、显式确认并要求 verified email。 -- 未配置 SMTP 的 OSS 使用邀请链接本身作为高熵、单次使用的授权证明，并记录创建者、接受者、来源 IP 和审计事件。 -- Workspace 必须始终至少有一个 active owner。 -- admin 不能移除或降级 owner。 +- SaaS 接受邀请时必须验证目标邮箱；OAuth 邮箱相同不能跳过 token 和显式确认。 +- Workspace 必须始终至少有一个 active owner；admin 不能移除或降级 owner。 +- 浏览器邀请链接把 secret 放在 URL fragment 中，页面读取后立即清除 fragment，并只短期保存在 `sessionStorage`。 -## 5. 当前代码现状 +### 5.5 固定 RBAC -### 5.1 LangBot Core 仍是单用户模型 +Core 权威定义 `owner`、`admin`、`developer`、`operator` 和 `viewer` 固定角色。 +权限按能力划分，例如资源查看、资源管理、运行操作、成员管理、provider secret 管理、审计查看和数据导出。 -当前代码已有 users 表和 JWT，但仍围绕单管理员初始化： +规则： -- src/langbot/pkg/entity/persistence/user.py 的 User 没有稳定 account_uuid、角色或 Workspace 关系。 -- src/langbot/pkg/api/http/service/user.py 的 JWT 仍主要使用 email 标识用户。 -- 同一服务通过 initialized guard 拒绝第二个不匹配的 Space 用户。 -- src/langbot/pkg/api/http/controller/group.py 只向 handler 注入 user_email，没有统一 RequestContext。 -- API Key 没有 Workspace 所属和 scope。 -- Web 客户端只保存 token，没有可信的当前 Workspace 上下文。 +- 普通资源可见性不自动授予 secret 可见性。 +- 跨 Workspace 猜测资源 UUID 返回 404，不泄露存在性。 +- 同 Workspace 资源存在但缺少权限时返回 403。 +- 最后一个 owner 不能被删除或降级。 +- 前端隐藏或禁用无权限入口只改善体验；后端仍必须执行所有授权检查。 -### 5.2 业务资源没有租户字段 +## 6. 开源与闭源职责边界 -以下资源目前以实例全局方式存取： - -- bots -- legacy_pipelines -- pipeline_run_records -- model_providers -- llm_models -- embedding_models -- rerank_models -- plugin_settings -- mcp_servers -- knowledge_bases、files、chunks -- monitoring 相关表 -- api_keys -- webhooks -- binary_storages -- metadata - -对应 Service 中仍存在直接查询全表的路径。因此只增加 Workspace API 不会形成租户隔离，必须逐表、逐服务和逐运行时完成 scope 改造。 - -### 5.3 运行时是实例级单例 - -当前 Application 中的 platform、pipeline、model、tool、plugin、session、RAG 和 vector manager 主要是进程内单例。第一阶段不能简单把现有进程扩成多个活跃副本，也不能假设加一个 workspace_uuid 字段后所有缓存和连接自然隔离。 - -### 5.4 Plugin SDK 不认识可信 Workspace - -当前 Query、Session 和 Host API 协议没有完整的 Workspace 上下文。SDK 的 workspace storage 仍使用 default owner。插件可以通过 Host API 枚举或调用全局资源，因此隔离必须进入连接、协议和 Host handler，而不只是 Web API。 - -## 6. 架构选择 - -### 6.1 推荐方案 - -采用以下三部分： - -1. LangBot Core OSS tenancy kernel。 -2. 独立闭源 Cloud Control Plane。 -3. 每个 Cell 内的 Cell Agent 和 LangBot Data Plane。 - -```mermaid -flowchart LR - User["User / API Client"] --> Edge["Global Edge & Workspace Router"] - Edge --> CP["Closed Cloud Control Plane"] - Edge --> Cell["Cloud Cell"] - CP -->|"Directory events / Entitlements / Desired state"| Cell - Cell --> Core["LangBot Core Data Plane"] - Core -->|"Usage events / Observed state"| CP - Core --> DB["Workspace-scoped Data"] - Core --> Runtime["Bot / Pipeline / Plugin / MCP / RAG Runtime"] -``` - -### 6.2 为什么使用独立闭源服务 - -独立 Control Plane 的优点： - -- 商业规则、支付密钥、集群凭证和平台管理权限不进入数据面。 -- Account 可跨多个 Cell 和 CloudInstance 加入 Workspace。 -- 订阅、权益、用量、路由和部署可以独立扩容和高可用。 -- Core 与控制面通过版本化协议耦合，不依赖 Python ABI 或同进程插件。 -- Control Plane 故障时 Core 可以使用本地投影和权益快照继续处理已有工作负载。 - -第一阶段不需要拆成大量微服务。可以先做模块化单体，但必须拥有独立部署单元和数据库。 - -### 6.3 为什么不使用 Core 内闭源模块 - -不推荐把 SaaS 多租户和计费做成 Core 内闭源模块： - -- 会把支付、全局身份和集群密钥带进数据面。 -- 与 Core 发布版本强耦合，难以独立扩容和回滚。 -- 多 Cell 路由和全局账户无法由单实例模块可靠管理。 -- 数据面被入侵时会扩大到计费和云运维信任域。 - -Core 内闭源模块可以保留为未来企业离线授权的交付形态，但不作为官方 SaaS 主架构。 - -### 6.4 为什么不能把隔离能力全部闭源 - -以下能力必须在开源 Core 中完整存在： - -- Account、Workspace、Membership、Invitation 基础模型。 -- OSS 单 Workspace edition policy。 -- 固定角色和权限映射。 -- RequestContext 和 WorkspaceScoped Repository。 -- 所有资源的 Workspace 行级隔离。 -- API Key、Webhook、Bot route 的 Workspace 绑定。 -- Plugin、MCP、RAG、Box、Session、Storage 和 Monitoring 隔离。 -- Entitlement 验签、额度执行钩子和 Usage Outbox。 -- 跨租户测试。 - -否则 OSS 多用户本身不安全，SaaS 也会把最关键的隔离边界变成网络依赖。 - -### 6.5 商业边界的现实限制 - -Core 为了安全地支持 OSS 多用户，必须包含 Workspace schema、scope 和固定 RBAC；测试环境也必须能构造多个 Workspace 验证隔离。因此技术上无法阻止第三方 fork 后移除 OSS 的 Workspace 数量限制。 - -官方 SaaS 的商业边界应建立在闭源 Control Plane、计费、全局目录、Cloud 运维、官方服务和商标授权上，而不是故意削弱 Core 的隔离能力。不要把一个本地环境变量或可修改的前端 feature flag 当作商业保护。 - -## 7. 开源与闭源职责边界 - -### 7.1 LangBot Core OSS +### 6.1 LangBot Core OSS Core 负责： -- 实例本地 Account 和 AuthIdentity。 -- Workspace、Membership，以及 OSS 本地 Invitation。 -- 固定 RBAC。 +- 本地 Account、Workspace、Membership 和 OSS Invitation。 +- 固定 RBAC 与单 Workspace edition policy。 - 业务资源及其 Workspace scope。 -- 运行时资源和连接隔离。 -- 本地身份、SaaS 身份投影和 Membership 投影；SaaS pending Invitation 不进入 Core。 -- 本地 edition policy。 -- Cloud InstanceManifest 和 EntitlementSnapshot 验签。 -- 本地 quota enforcement。 -- UsageEvent durable outbox。 -- 基础安全审计事件。 +- HTTP、WebSocket、后台任务和运行时请求上下文。 +- Plugin、MCP、RAG、Box、Session、Storage 和 Monitoring 隔离。 +- SaaS Account/Workspace/Membership 的版本化执行投影。 +- InstanceManifest、EntitlementSnapshot 和 Runtime 控制通道验证。 +- 通用 capability 与数值 quota enforcement。 +- UsageEvent/business outbox 和基础安全审计。 -Core 是 Bot、Pipeline、Model、Knowledge、Plugin installation、MCP configuration 和 Monitoring 数据的权威来源。 +Core 是 Bot、Pipeline、Model、Knowledge、Plugin installation、MCP configuration 和 Monitoring 数据的权威来源， +也是每个业务和运行时请求的最终授权边界。 -### 7.2 Closed Cloud Control Plane +### 6.2 Closed Cloud Control Plane Control Plane 负责： - SaaS 全局 Account、AuthIdentity、Session、OIDC 和后续 SSO。 -- SaaS Workspace directory。 -- SaaS Membership 和 Invitation 权威状态。 -- Workspace 创建、归档、暂停和删除工作流。 -- BillingAccount、Product、PlanVersion、Price 和 Subscription。 -- Order、PaymentAttempt、ProviderEvent、Invoice 和 Refund。 -- Entitlement 计算和签名。 +- SaaS Workspace、Membership 和 Invitation 的权威目录。 +- Workspace 创建、暂停、归档和删除工作流。 +- BillingAccount、Product、PlanVersion、Price、Subscription、Invoice、Refund 和 provider event。 +- Entitlement 计算、签名与版本。 - Usage ledger、聚合、额度和欠费策略。 -- Cell、CloudInstance、Placement、Route 和 Domain。 -- RuntimeClass、Release 和 DesiredState。 +- 实例 manifest、release、capacity、内部 desired state 和 observed state。 - SaaS 运营后台、平台角色和高级审计。 -Control Plane 不保存 Bot、Pipeline、Model 或 Knowledge 等业务资源，也不代理普通 Bot 消息执行。 +首期不把这些职责拆成多个租户、计费和调度微服务。推荐以一个独立于 Core 的闭源模块化单体承载， +并通过模块边界复用已有账户、OAuth、支付、邮件和运营能力。历史 Cloud 的租户专属部署代码不复用。 -### 7.3 SaaS Adapter +Control Plane 不保存 Bot、Pipeline、Model 或 Knowledge 等业务内容，也不代理普通消息执行。 -Core 内只保留薄的协议适配层： +### 6.3 SaaS Adapter + +Core 中只保留薄的协议适配层： - 验证 InstanceManifest、Account token 和 JWKS。 - 消费 DirectoryEvent 并写入本地投影。 -- 缓存 EntitlementSnapshot。 -- 将 UsageEvent 写入 outbox。 -- 上报 ObservedState。 +- 缓存并验证 EntitlementSnapshot。 +- 将 UsageEvent 写入 durable outbox。 +- 接收 execution desired state 并上报 observed state。 -适配层不得 monkey patch ORM，不得绕过 Core 的权限检查，也不得在一次普通资源请求中同步调用 Control Plane。 +适配层不得 monkey patch ORM、绕过 Core 权限检查或在普通资源请求中同步调用 Control Plane。 -### 7.4 Provider 接口 +### 6.4 Source of Truth -建议在 Core 定义稳定接口： +| 数据 | OSS | SaaS | +| --- | --- | --- | +| Account、Workspace、Membership | Core 本地数据库 | Control Plane 权威，Core 保存版本化投影 | +| Invitation | Core 本地数据库 | Control Plane 权威，不向 Core 投影 pending secret | +| Bot、Pipeline、Model、KB、Plugin、MCP | Core | Core | +| Subscription、Payment、Invoice、Usage ledger | 无远端依赖 | Control Plane | +| Feature 和 quota | 本地 edition policy | Control Plane 签发，Core/Runtime 验证执行 | +| Execution generation | OSS 固定本地值 | Control Plane desired state，Core 执行 | +| 运行时授权 | Core | Core 根据本地投影和 entitlement 执行 | -```python -class IdentityProvider: - async def authenticate(self, credential) -> PrincipalContext: ... +SaaS 不维护两套可写目录。Control Plane 是目录权威写模型；Core 只保存带 revision 的执行投影。 -class WorkspaceDirectoryProvider: - async def resolve_membership( - self, - account_uuid: str, - workspace_uuid: str, - ) -> WorkspaceMembership: ... +## 7. 控制面协议 -class EditionPolicyProvider: - async def can_create_workspace(self, account_uuid: str) -> bool: ... +### 7.1 InstanceManifest -class EntitlementProvider: - async def get_snapshot(self, workspace_uuid: str) -> EntitlementSnapshot: ... +仅设置 `system.edition=cloud`、环境变量或前端 feature flag 不得启用 SaaS 多 Workspace。 +Cloud bootstrap 必须验证由预置根信任签名的 InstanceManifest，并据此安装闭源 Workspace policy。 -class UsageSink: - async def append(self, event: UsageEvent) -> None: ... -``` - -OSS 使用 LocalProvider；SaaS 使用本地 ProjectionProvider 和签名快照，不使用每请求 RemoteProvider。 - -## 8. Source of Truth 与同步协议 - -### 8.1 Source of Truth - -| 数据 | OSS | SaaS | -| -------------------------------------------- | ------------------- | --------------------------------------- | -| Account、Workspace、Membership | Core 本地数据库 | Control Plane 权威，Core 保存版本化投影 | -| Invitation | Core 本地数据库 | Control Plane 权威，不向 Core 投影 | -| Bot、Pipeline、Model、KB、Plugin、MCP | Core | Core | -| Subscription、Payment、Invoice、Usage ledger | 无远端依赖 | Control Plane | -| Feature 和 Quota | 本地 edition policy | Control Plane 签发，Core 验签执行 | -| Placement 和 Route | 不适用 | Control Plane | -| Execution generation | Core 本地固定 epoch | Control Plane DesiredState，Core 执行 | -| 运行时授权 | Core | Core 根据本地投影和 entitlement 执行 | - -SaaS 中不是两套并行真相。Control Plane 的 directory 是权威写模型；Core 的 Account、Workspace 和 Membership 是带 revision 的本地读模型和执行模型。 - -### 8.2 DirectoryEvent - -Control Plane 通过 transactional outbox 发布： - -- account.upserted -- workspace.upserted -- workspace.status_changed -- membership.upserted -- membership.removed - -SaaS Invitation 创建、撤销和接受只在 Control Plane directory 内流转；接受成功后发布最终 membership.upserted。Core 不接收 pending Invitation、email 或 token hash。 - -每个事件至少包含： +Manifest 至少绑定： ```text -event_id -event_type -aggregate_id -aggregate_revision -instance_sequence +iss, aud, sub, jti, iat, nbf, exp instance_uuid -workspace_uuid -occurred_at -payload -schema_version -``` - -目录事件只表达 Account、Workspace 和 Membership。Placement、release 和 runtime capacity 只进入 DesiredState 流，不能由两个 revision 体系同时修改。 - -传输与恢复要求： - -- Control Plane outbox 向受认证的 broker 发布，或通过 mTLS HTTPS 发送签名 event batch。 -- Broker 使用 producer ACL；event batch 使用由 InstanceManifest 授权的 JWS key 签名并绑定 audience。 -- 以 workspace_uuid 为主要 partition key；Account 事件按 account_uuid 分区。 -- Control Plane 为每个目标 instance_uuid 维护 gap-free 的逻辑 directory log offset，作为 instance_sequence；不能直接假设 PostgreSQL sequence 或预分配自增 ID 无空洞。 -- 顺序只保证到 aggregate，Core 以 aggregate_revision 判断新旧，不依赖全局顺序。 -- Core 使用 inbox 按 event_id 去重，只接受比本地 revision 更新的事件。 -- Core 同时追踪 instance_sequence 缺口，只把无缺口的最大连续序号记为 contiguous_applied_sequence。 -- 因 aggregate_revision 过旧而业务 no-op 的已验签事件，仍要在 inbox 事务提交后把该 instance_sequence 标记为 processed。 -- 删除使用 tombstone，不依赖物理删除顺序。 -- 新实例先导入带 high_watermark 的全量 snapshot，再从该 cursor 消费增量。 -- 断流或 cursor 超出保留期时重新做 snapshot reconciliation。 -- schema_version 采用向后兼容演进；移除字段必须经过双读和版本退役窗口。 -- Core 持久化 projection readiness 和 applied watermark，并向 Control Plane ACK。 -- 注册、邀请接受和 Placement 切换只有在目标 Core 的 contiguous_applied_sequence 达到要求水位后才返回 ready route。 - -Control Plane 周期性签发两种短期 lease： - -- InteractiveDirectoryLease：约束交互账户和 Membership 授权，TTL 较短。 -- WorkspaceStatusLease：约束 Bot、Webhook、Workspace API Key 等自动化工作负载能否继续运行，TTL 可以稍长但必须有硬上限。 - -两种 lease 的 JWS protected header 都包含 alg、kid 和 typ，payload 至少包含： - -```text -iss -aud = langbot-instance:{instance_uuid} -sub = {instance_uuid} -jti -iat -nbf -exp -instance_uuid -min_required_sequence -schema_version -keyset_revision -``` - -Core 只有在 projection_ready = true 且 contiguous_applied_sequence 大于等于 min_required_sequence 时才能接受 lease。投影落后时先等待同步或 reconciliation，超过请求时限后 fail closed。Lease 只证明投影新鲜度，不替代具体 Membership 或 Workspace status 检查。 - -instance_sequence 只在 committed outbox logical log 中形成。如果底层实现发生保留号、回滚或废弃，必须写入可验签的 no-op 或 skip record；snapshot reconciliation 也可以原子重置到一个明确 high_watermark，不能留下永久空洞。 - -Lease signer 只能把已经提交且可从 directory log 连续重放的 committed watermark 写入 min_required_sequence，不能使用数值最大的已分配 ID，也不能为尚未可重放的状态签发新鲜度证明。 - -重复、乱序、延迟、断流和全量重放都必须安全。 - -### 8.3 InstanceManifest - -仅配置 system.edition = cloud 不足以启用 SaaS 多 Workspace。Core 镜像必须内置官方 root public key bundle，或在部署时通过独立安全通道注入 pinned root key 和 issuer。该信任根不能来自待验的 Manifest。 - -InstanceManifest 由 root key 签名，再授权 Gateway assertion、DirectoryEvent、DirectoryLease 和 Entitlement 使用的下级 issuer 与 JWKS。公网 Account token 的信任配置属于 Edge 全局身份配置，不由 Data Plane Manifest 授权。JWS protected header 至少包含 alg、kid 和 typ；kid 不放在业务 payload 中。 - -```text -iss -aud = langbot-instance:{instance_uuid} -sub = {instance_uuid} -jti -iat -nbf -exp -instance_uuid -cell_uuid -gateway_assertion_issuer -gateway_jwks_uri -directory_event_issuer -directory_event_audience -directory_jwks_uri -entitlement_issuer -entitlement_jwks_uri -keyset_revision release capabilities tenant_isolation_version -generation +execution_generation +delegated issuers and keyset revision ``` -Manifest 校验要求： +签名错误、audience 不匹配、过期、generation 回滚或信任链缺失时必须失败关闭，不能降级为 OSS 默认 Workspace。 -- 使用 out-of-band root bundle 验证签名。 -- aud 与目标 instance_uuid 精确匹配。 -- iss、sub、nbf、exp、jti 和 generation 有效。 -- 默认只允许很小的可配置时钟偏差。 -- root 和 delegated key 轮换使用重叠有效期与递增 keyset_revision。 -- 紧急撤销通过 root-signed key revocation manifest，并限制本地缓存最长期限。 -- Manifest 续签应在有效期过半前完成；其 exp 必须覆盖计划中的最大 Entitlement grace 和安全余量。 +### 7.2 DirectoryEvent 与目录新鲜度 -Manifest 缺失、签名错误、audience 不匹配或 generation 回滚时，SaaS 实例必须 fail closed，不能自动降级成 OSS default Workspace。 +Control Plane 通过 transactional outbox 发布 Account、Workspace 和 Membership 的版本化事件。 +Core 使用 inbox 按 `event_id` 去重，以 aggregate revision 拒绝旧写，并追踪连续应用水位。 -### 8.4 EntitlementSnapshot +要求： -建议使用非对称签名 JWS。Protected header 至少包含 alg、kid 和 typ；payload 至少包含标准 claims 和业务字段： +- 事件和 batch 经过实例绑定的强认证与签名。 +- 重复、乱序、延迟、断流和全量 replay 都安全。 +- 删除使用 tombstone。 +- 新实例先导入带 high watermark 的 snapshot，再消费增量。 +- projection 未就绪或落后于授权 lease 要求时，交互与自动化请求按策略失败关闭。 +- SaaS pending Invitation、email 和 token hash 不进入 Core 投影。 -```json -{ - "iss": "cloud_control_plane", - "aud": "langbot-instance:instance_uuid", - "sub": "workspace_uuid", - "jti": "snapshot_uuid", - "iat": 0, - "nbf": 0, - "exp": 0, - "instance_uuid": "instance_uuid", - "workspace_uuid": "workspace_uuid", - "subject_type": "workspace", - "plan_revision": 12, - "entitlement_revision": 42, - "status": "active", - "features": { - "box": true, - "custom_roles": false - }, - "limits": { - "members": 20, - "bots": 50, - "pipelines": 100, - "storage_bytes": 107374182400 - }, - "grace_until": 0 -} +MVP 可采用一个共享、原子且可恢复的 Control Plane store；未来多副本不能继续使用进程内状态承担一次性 token 或目录水位。 + +### 7.3 EntitlementSnapshot + +Entitlement 使用版本化签名快照，至少绑定： + +```text +instance_uuid +workspace_uuid +plan_revision +entitlement_revision +status +features +limits +nbf, exp, grace_until ``` -Core 只信任当前有效 Manifest 授权的 entitlement issuer 和 key，校验 issuer、audience、subject、instance_uuid、revision、nbf、exp 和 signature。旧 revision 不得覆盖新快照。now 超过 exp 后不能继续视为 active，只能按 grace_until 进入受限的已有工作负载宽限模式。Entitlement 的 grace 不延长 Account token 或 Membership authorization lease。 +Core 校验 issuer、audience、subject、instance、revision、时间和签名；旧 revision 不覆盖新快照。 +套餐名称和价格规则只存在于闭源 Control Plane，Core 与 Runtime 只理解通用 capability 和数值限额。 -### 8.5 UsageEvent +Control Plane 故障时，已缓存且仍有效的快照可继续执行；过期后只能进入明确、有限的 grace 模式或失败关闭。 -Usage 采用 append-only、至少一次投递： +### 7.4 UsageEvent 与 outbox + +用量事件 append-only、至少一次投递，Control Plane 按 `event_id` 去重。事件至少包含： ```text event_id -workspace_uuid instance_uuid -placement_generation +workspace_uuid +execution_generation meter quantity_integer unit @@ -557,1200 +372,563 @@ entitlement_revision schema_version ``` -Control Plane 按 event_id 去重，并记录 received_at。Data Plane 不提交 billing period、price 或最终金额；中央 ledger 根据 occurred_at、不可变 Subscription/Price version 和迟到事件策略推导账期与价格。entitlement_revision 只用于审计事件发生时执行的权益版本。 +Core 不计算账单金额，也不在普通请求中同步扣费。业务写入与相应 business outbox 必须在同一事务中提交； +generation-aware write fence 与 outbox 原子性尚是 SaaS 激活门禁。 -Core 不在普通请求中同步扣费。未来若需要本地实时额度，可以设计签名 BudgetLease，但不属于 Cloud v2 MVP；模型 token 如果经过中央模型网关，可由网关作为权威计量来源。 +### 7.5 Desired state 与 observed state -### 8.6 DesiredState 与 ObservedState +闭源控制面发布版本化的 release、capacity 和 execution desired state，Core/Runtime 幂等 reconcile 并上报 observed state。 +desired state 只描述同一逻辑实例内部的执行所有权和容量，不产生新的产品级实例或租户实体。 -Cell Agent 拉取或接收版本化 DesiredState，并幂等 reconcile： +Workspace 安全状态由 directory revision 决定，订阅状态由 entitlement revision 决定，执行撤销由 +execution generation 决定。三者取最严格有效状态，但任何通道都不能修改另一个通道的权威字段。 -- CloudInstance release。 -- RuntimeClass。 -- capacity。 -- route。 -- placement generation。 +## 8. 身份、鉴权与请求上下文 -Cell Agent 上报 ObservedState。Space Web API 或 Control Plane Web API 不直接持有并使用集群管理员凭证执行任意 K8s CRUD。 +### 8.1 上下文模型 -Workspace 的安全或管理员状态只由 Directory aggregate 的 revision 决定；订阅状态只由 Entitlement revision 决定；Placement 只由 DesiredState generation 决定。Core 计算三者的最严格有效状态，但任何通道都不能修改另一个通道的字段。 - -## 9. 身份、鉴权与请求上下文 - -### 9.1 上下文分层 - -建议分为四层： +租户业务入口统一解析不可变的 `RequestContext`： ```python -class PrincipalContext: - account_uuid: str | None - api_key_uuid: str | None - principal_type: str - -class WorkspaceContext: - workspace_uuid: str - membership_uuid: str | None - role: str | None - permissions: frozenset[str] - membership_revision: int - +@dataclass(frozen=True) class RequestContext: - instance_uuid: str - placement_generation: int - request_id: str - auth_type: str - principal: PrincipalContext - workspace: WorkspaceContext - entitlement_revision: int - -class ExecutionContext: instance_uuid: str workspace_uuid: str - placement_generation: int - bot_uuid: str | None - pipeline_uuid: str | None - query_uuid: str | None - trigger_principal: PrincipalContext | None + execution_generation: int + principal_type: str + principal_uuid: str + permissions: frozenset[str] + auth_method: str + entitlement_revision: int | None + request_id: str ``` -服务方法不得通过全局变量或最近 Workspace 获取上下文。Gateway 的签名内部 assertion、直接进入 Core 的本地请求、后台任务、Plugin Host 调用和 runtime lease 都必须生成或继承当前 placement_generation。 +不同入口的 Workspace 来源： -### 9.2 Account token +| 入口 | Workspace 来源 | +| --- | --- | +| Browser Account token | `X-Workspace-Id` 只作候选；服务端校验 Membership | +| API Key | key 记录绑定的 Workspace，忽略 caller selector | +| Public Bot / Webhook | Bot 或 webhook route 的可信所有权 | +| Background job | durable payload 中的完整 scope，执行前重新验证 generation | +| Plugin Host API | 认证控制连接和 immutable action context | +| Box operation | 已验证 entitlement、admission grant 和 Runtime namespace | +| System operation | 显式、最小能力的 SystemContext，禁止隐式全局上下文 | -SaaS Account access token 使用非对称签名和短 TTL，至少包含： +禁止从模块全局变量、进程默认 Workspace、请求 payload 或“第一个 Workspace”推断 scope。 + +### 8.2 Account token 与 Workspace discovery + +- 新 JWT 使用稳定 Account UUID 作为 `sub`，并绑定 issuer、当前 `instance_uuid` audience 和 expiry。 +- 账户级 Workspace discovery 是一个窄 bootstrap capability，只列出该 Account 的 active Membership，不能执行租户业务。 +- multi-Workspace 模式下，tenant route 缺少 selector 必须拒绝；OSS singleton 模式可由 policy 选择唯一 Workspace。 +- Account token 不直接证明任一 Workspace 权限；Membership 必须在服务端解析并验证状态与 revision。 + +### 8.3 API Key、WebSocket 与长任务 + +- API Key 只持久化 hash，raw secret 仅返回一次；记录绑定 Workspace、固定 scopes、状态、expiry 和 creator。 +- Dashboard WebSocket 在升级后认证，并在每条入站消息前重新验证 Account、Membership、权限、资源所有权和 generation。 +- 长时间 LLM、MCP、Plugin 或 Box 调用在产生副作用或接受结果前再次校验 execution generation。 +- 临时凭证交换绑定发起者、Workspace、instance 和 generation；其他 scope 查询返回与不存在相同的 404。 + +### 8.4 错误语义 + +| 场景 | 语义 | +| --- | --- | +| 未认证或 token 无效 | 401 | +| 同 Workspace 资源存在但权限不足 | 403 | +| 资源不存在或属于其他 Workspace | 404 | +| edition / entitlement / quota 禁止 | 稳定领域错误码，不伪装为 500 | +| execution generation 过期 | fail closed，并停止旧运行态 | +| 未处理异常 | 稳定 `internal_error` + request ID；细节只进入服务端日志 | + +## 9. Core 数据模型 + +### 9.1 Account、Workspace 与 Membership + +核心实体至少包含： + +```text +Account + uuid + email_normalized + display_name + status + auth bindings + +Workspace + uuid + name + status + source: local | cloud_projection + directory_revision + +WorkspaceExecutionState + workspace_uuid + instance_uuid + execution_generation + status + write_fenced_at + revision + +WorkspaceMembership + workspace_uuid + account_uuid + role + status + directory_revision +``` + +约束： + +- Membership 对 `(workspace_uuid, account_uuid)` 唯一。 +- Workspace 的 source 不允许通过可变本地配置从 local 升级成 cloud projection。 +- Cloud projection 只有在 manifest、instance binding、目录 revision 和 execution state 均有效时才可路由。 +- OSS bootstrap 只创建或修复 local singleton Workspace。 + +### 9.2 Invitation + +OSS Invitation 存在 Core 本地数据库；SaaS Invitation 只存在于闭源目录。 + +```text +WorkspaceInvitation + uuid + workspace_uuid + email_normalized + role + token_hash + expires_at + accepted_at + revoked_at + created_by +``` + +数据库约束必须保证同一 Workspace 与邮箱只有一个有效邀请，并保证 token hash 全局唯一。 + +### 9.3 业务资源 + +所有租户资源显式包含 `workspace_uuid`，包括但不限于： + +- Bot、Pipeline、Provider、Model、Knowledge Base 和 vector record。 +- Plugin installation、MCP configuration、API Key 和 webhook binding。 +- Query、Message、Session、Monitoring、Usage 和 AuditEvent。 +- Upload、ObjectRef、Skill、Runtime desired state 和 temporary credential session。 + +唯一键、索引、缓存 key、object key、日志维度和幂等键都必须包含 Workspace scope。 +服务层不得暴露可绕过 Workspace 条件的普通 `get(id)`、`list()` 或 `delete(id)`。 + +### 9.4 防御性约束 + +- tenant table 的 `workspace_uuid` 非空并有外键。 +- SaaS PostgreSQL 关键表启用并强制 RLS。 +- 需要全局唯一的 opaque token 使用 hash 唯一索引，不依赖 Workspace 内唯一。 +- owner 保底、Membership revision、invitation one-shot 等规则同时由 service 和数据库事务保护。 +- 任何跨 Workspace 运维操作必须走显式受审计的 system capability，不得复用普通 repository。 + +## 10. PostgreSQL、pgvector 与存储 + +### 10.1 数据库边界 + +- OSS 继续默认 SQLite，并可显式选择自托管 PostgreSQL。 +- SaaS 使用一个 PostgreSQL business database、一个 `public` shared schema 和共享连接池。 +- 创建 Workspace 不创建 database、schema、role 或专属连接池。 +- 每个 tenant transaction 使用 `SET LOCAL` 建立 scope，并由统一 TenantUnitOfWork 保证 context 与 SQL 使用同一事务和连接。 +- 应用层 Workspace scope 是第一道边界，`ENABLE` + `FORCE ROW LEVEL SECURITY` 是第二道边界。 +- runtime role 必须是非 owner、最小权限、无 superuser、无 `BYPASSRLS`、无 role membership 和跨 schema 权限。 +- schema、extension、policy 和 ACL 只由独立 release migrator 创建与验证；Cloud runtime 不执行 DDL。 +- PostgreSQL 仅承载业务数据和 pgvector，不成为 Plugin/Box 通用协调数据库、进程目录或新的控制面数据库。 + +首期 migrator 和 runtime URL 必须连接同一个 host、port、database，但使用不同 role。 +生产部署还必须证明 runtime credential 无法连接 PostgreSQL 集群中的其他 database；专用 endpoint 或经验证的 HBA/proxy 隔离仍是激活门禁。 + +### 10.2 Transaction 与后台任务 + +- 一个 TenantUnitOfWork 只绑定一个 Workspace、一个 execution generation 和一个事务所有者任务。 +- 子任务不能继承并提交、回滚或关闭父任务的 tenant session。 +- 长时间 LLM 或网络等待不持有数据库连接；每次数据库 helper 打开短事务。 +- detached task 只在父事务提交后启动，并自行建立新 scope；父事务回滚时取消待启动任务。 +- generation-aware write fence 必须保持到 commit，并与 business outbox 原子提交；该能力完成前不得激活 SaaS 写流量。 + +### 10.3 pgvector + +- SaaS 默认使用同一业务 PostgreSQL 中的 pgvector，不静默回退到 Chroma。 +- 向量身份至少为 `(workspace_uuid, knowledge_base_uuid, vector_id)`。 +- 向量操作使用相同 tenant context 与 RLS 契约。 +- embedding 维度显式存储和校验；不匹配时失败关闭，不截断、补齐或改用无界扫描。 +- extension、表、constraint 和 ANN index 由 release migration 创建。 +- OSS 默认仍可使用 SQLite + Chroma；选择 pgvector 时遵守相同 scope。 + +### 10.4 Object storage + +- 大对象、plugin artifact、upload、knowledge 文件和 sandbox 文件不作为 PostgreSQL blob 存储。 +- durable object key 和 metadata 都包含 Workspace scope；临时 staging 可包含 generation，但稳定业务引用不能因未来 generation 切换而永久失效。 +- 现有 generation-scoped opaque key 在固定 generation 的 OSS 中安全，但 Cloud cutover 前必须实现稳定 final identity 或原子引用迁移。 +- public image 与 private document 使用不同 capability；不能把通用 upload key 当作公开读取凭证。 + +## 11. Plugin Runtime + +### 11.1 共享 supervisor、独立 worker + +整个逻辑实例共享一个可信 Plugin Runtime 逻辑控制面；M0 由一个 supervisor replica 承担。新 Workspace 不创建专属 Runtime、连接、卷或进程。 + +每个运行中的 plugin installation 独占一个 nsjail worker process tree；enabled-resident 是 desired semantics。worker 运行期间永久绑定： + +```text +instance_uuid +workspace_uuid +execution_generation +installation_uuid +runtime_revision +artifact_digest +``` + +插件不能通过 payload、Host API 参数、环境变量或重连改变该绑定。Supervisor 不在自身解释器中加载第三方插件代码。 +停用、删除、revision/generation 变化或 entitlement 撤销时，旧 worker 必须停止并失去 Host API 权限。 + +### 11.2 文件和进程边界 + +```text +data/plugin-runtime/ +├── artifacts/sha256//code/ # 已验证、只读共享 +├── environments/sha256// # 原子发布、只读共享 +└── installations// + ├── home/ # 私有可写 + ├── tmp/ # 私有可写 + └── data/ # 私有持久数据 +``` + +- 同插件同版本只有在 package digest 完全相同且完整性已验证时才共享只读代码。 +- dependency environment key 包含 artifact/requirements digest、Python ABI、Runtime version 和 installer schema。 +- installation 进程、配置、secret、home、tmp、data 和日志永不合并。 +- namespace、private `/proc`、mount、PID、IPC、UTS、cgroup 与 rlimit 阻止读取其他文件、枚举或 signal 其他进程。 +- Cloud 不从 artifact 自动加载 `.env`；secret 只由可信控制面按 installation 注入。 +- 插件 egress 必须阻止访问 Core loopback、Box Runtime、数据库和平台 metadata endpoint。 + +### 11.3 统一资源上限 + +资源限制只来自实例级 `data/config.yaml`，并支持现有环境变量覆写；plugin manifest 不能声明、放宽或覆盖。 + +```yaml +plugin: + worker: + max_cpus: 1.0 + max_memory_mb: 512 + max_pids: 128 + max_open_files: 256 + max_file_size_mb: 512 + require_hard_limits: true +``` + +CPU、内存和 PID 使用 cgroup 硬限制，open files 和单文件大小使用 rlimit。 +Cloud deployment profile 强制 nsjail；硬限制不可用时 readiness 失败，不能降级为普通子进程。 +installation 总磁盘配额需要可原子拒绝写入的 quota provider，不能以目录扫描冒充硬限制。 + +### 11.4 Desired state 与恢复 + +- PostgreSQL 中的 installation desired state 与 durable binary storage 是权威状态。 +- Runtime 本地进程表、nsjail 目录、artifact/venv cache 都可重建。 +- Runtime 重连执行实例范围 full reconciliation，清理 stale worker 并恢复 enabled installation。 +- dependency preparation 失败记录在对应 installation，不启动半就绪 worker，也不阻塞其他 installation。 +- desired semantics 要求 enabled installation 常驻，不做 idle eviction；是否按负载回收以后再决定。 +- 当前 Supervisor 可在 Runtime 重连或 Core apply/reconcile 时恢复 desired state，但尚未为意外退出的 worker + 实现带有界 backoff 的 completion callback；这项可靠性缺口在 Cloud 激活前必须补齐并验证。 + +真实 Linux/nsjail/cgroup 与受控 egress 的 Cloud 部署验证尚未完成，是生产激活门禁。 + +## 12. Box Runtime 与 stdio MCP + +### 12.1 共享 Box 控制面 + +整个逻辑实例共享一个可信 Box Runtime 逻辑控制面；M0 由一个 Runtime replica 承担。Core 与 Runtime 控制通道绑定稳定 instance identity， +每个 operation 绑定 `workspace_uuid`、`execution_generation`、session revision 和短期 admission grant。 + +首期 entitlement 模型： ```json { - "sub": "account_uuid", - "iss": "cloud_control_plane", - "aud": "langbot_cloud_edge", - "iat": 0, - "exp": 0, - "jti": "token_uuid", - "token_version": 3 + "features": { + "managed_sandbox": true, + "external_sandbox": false + }, + "limits": { + "managed_sandbox_sessions": 1 + } } ``` -JWT 表示全局账户身份，不直接授予任何 Workspace 权限，也不需要因切换 Workspace 重新签发。公网 Account token 只发给 Edge 或 Cloud BFF，Data Plane 不直接接受该 audience。 +闭源订阅模块把套餐映射为该通用 capability；Core 与 Runtime 不判断 `plan == pro`。 +预期 Pro 得到 `managed_sandbox_sessions = 1`，其他套餐为 `0`。 -OSS token 同样应从 email 主体迁移到 account_uuid，并保留 issuer、audience、expiry 和 token version。 +### 12.2 Sandbox 模型 -### 9.3 Workspace 选择 +- 合资格 Workspace 首次使用时懒创建一个持久 `global` 逻辑 session。 +- `global` 表示 Workspace 内默认逻辑 sandbox，不表示跨 Workspace 共享。 +- session TTL 不自动回收；Runtime 重启后进程和临时目录失效，但 `/workspace` 持久数据保留。 +- 每次普通命令在 Box Runtime 容器内启动一个 one-shot nsjail 子进程。 +- 首期禁止 managed background process 和 network，避免 session 被当成常驻共享主机。 +- Core 与 Runtime 通过认证 random-marker challenge 证明看到同一 durable volume，不能只比较路径字符串。 +- 文件同步、attachment 和 skill mount 沿用现有 nsjail 机制，但所有 host path 解析必须由可信 Workspace context 派生并防止 symlink/path escape。 -浏览器和 REST 客户端可发送： +Cloud readiness 必须证明 cgroup、namespace、mount、Workspace/Skill/ephemeral byte quota 和 inode quota 均为硬限制。 +当前普通 nsjail backend 不具备全部硬磁盘能力，因此 Cloud Box 应失败关闭，直到绿地部署提供并验证真实 quota provider； +不能把软目录扫描写成“生产已就绪”。 -```http -X-Workspace-Id: workspace_uuid +### 12.3 外部 E2B + +非 Pro 用户后续可在 WebUI 配置 Workspace 自有的远程 E2B sandbox。该功能尚未实现，首期不纳入。 +未来 credential 必须属于 Workspace、加密存储且读取受 secret 权限保护，不消耗 Cloud managed sandbox 配额。 + +### 12.4 stdio MCP 独立开关 + +```yaml +mcp: + stdio: + enabled: true ``` -规则： +- OSS 默认 `true` 保持兼容。 +- Cloud v2 通过 `MCP__STDIO__ENABLED=false` 强制关闭。 +- 该 gate 独立于 `box.enabled`、managed sandbox entitlement 和 session quota。 +- gate 同时覆盖 create、update、test、bootstrap load 和最终 Runtime execution。 +- 已有 stdio 配置在 gate 关闭时保留但不启动，并返回明确的 feature-disabled 错误。 +- HTTP/SSE 等远程 MCP transport 不受影响。 -- 该 header 只是选择器，不是授权证明。 -- SaaS Workspace 资源接口缺少选择器时返回 workspace_required。 -- SaaS 不得自动选择最近或第一个 Workspace。 -- OSS 可以因为实例只有一个 Workspace 而省略 header，后端解析到唯一 Workspace。 -- API endpoint 路径中的 workspace_uuid 仍必须与 RequestContext 一致。 +## 13. HTTP API 与 WebUI -公网 Edge 可以使用外部 header 或稳定 route 选择 Workspace，但必须移除客户端提供的内部上下文 header。Edge 只验证 Account token、route 和 Placement generation，不同步调用 Control Plane 验证 Membership；Membership 的最终判定只由 Core 本地投影完成。 +### 13.1 Core API -Edge 通过 mTLS 注入短期、instance-scoped 的签名内部 assertion，至少包含： - -```text -iss = gateway_assertion_issuer -aud = langbot-instance:{instance_uuid} -sub = account_uuid -jti -iat -exp -auth_type -workspace_uuid -instance_uuid -placement_generation -directory_min_sequence -source_token_jti -``` - -Edge 的 route projection 由 Control Plane 异步发布，并使用与 Core 相同的 directory sequence 坐标；directory_min_sequence 只要求 Core 已追平到该水位，不代表 Membership 已通过。Core 验证 assertion issuer、audience、签名、短 TTL、Workspace、Placement generation 和投影水位，然后再加载 Membership。 - -如果未来 Edge 为了提前拒绝无权限请求而缓存 Membership，该缓存只能作为优化，不能成为放行依据，并且必须遵守相同的 watermark 与 lease 规则。 - -### 9.4 API Key、Bot 和 Webhook - -API Key： - -- 绑定 workspace_uuid。 -- 绑定 scopes、status、expires_at。 -- 数据库只保存 key hash。 -- 可选绑定 service_account_uuid。 -- 不能通过 header 切换到其他 Workspace。 - -Bot/Webhook： - -- 通过不可猜的 route token、bot_uuid 或发布记录反查 Workspace。 -- 不信任公网传入的 workspace_uuid。 -- webhook route token 可轮换和撤销。 -- Workspace slug 只用于展示，不能作为授权依据。 - -### 9.5 错误语义 - -- 资源不存在或属于其他 Workspace：返回 404。 -- 资源属于当前 Workspace但权限不足：返回 403。 -- SaaS 缺少 Workspace：返回 400 workspace_required。 -- Membership inactive 或 Workspace suspended：返回 403。 -- Entitlement 不允许新建或超限：返回明确的 edition_limit 或 quota_exceeded。 - -## 10. Core 数据模型 - -### 10.1 Account - -升级现有 users 语义，建议字段： - -- uuid -- email_normalized -- display_name -- avatar_url -- status -- source: local 或 cloud_projection -- projection_revision -- created_at -- updated_at - -OSS 本地凭证单独放在 local_credentials： - -- account_uuid -- password_hash -- password_updated_at -- token_version - -OAuth 身份不要只按 email 绑定，本地模式新增 auth_identities： - -- provider -- provider_subject -- account_uuid -- email_at_link_time -- created_at - -唯一键为 provider 与 provider_subject。 - -SaaS 投影直接使用 Control Plane 的全局 Account UUID，不再维护另一套 external_account_uuid。投影只包含显示身份和授权所需状态，绝不包含 password_hash、OAuth refresh token、浏览器 session secret 或支付凭证。 - -### 10.2 Workspace - -新增 workspaces： - -- uuid -- instance_uuid -- name -- slug -- type: personal 或 team -- status: provisioning、active、suspended、archived、deleted -- created_by_account_uuid -- source: local 或 cloud_projection -- projection_revision -- created_at -- updated_at - -OSS 数据库必须用约束和 edition policy 保证一个实例只有一个 Workspace。SaaS Workspace 必须与签名 InstanceManifest 的 instance_uuid 一致。 - -dormant 是 Control Plane directory 的未放置状态，不写入 Core workspaces。开始 Placement 后，目标 Core 才收到 provisioning Workspace 投影；Placement active 后再切换为 active。 - -### 10.3 WorkspaceExecutionState - -Core 本地新增 workspace_execution_states，统一表达当前实例可接受的写入 generation： - -- workspace_uuid -- instance_uuid -- active_generation -- state: provisioning、active、migrating、draining、inactive -- write_fenced -- source: local 或 cloud -- desired_state_revision -- updated_at - -OSS 使用 LocalExecutionStateProvider，在初始化事务中创建 generation = 1、state = active、write_fenced = false 的固定本地状态，不消费 Cloud DesiredState。SaaS 使用 CloudExecutionStateProvider，由 DesiredState consumer 更新，不由用户 API 修改。 - -每个 Unit of Work 或 Repository 写入口都必须验证： - -- RequestContext 或 ExecutionContext 的 placement_generation 等于 active_generation。 -- write_fenced = false。 -- state 允许当前操作。 - -校验必须与提交形成同一个事务协议，不能只在事务开始前 check： - -- 普通写事务对 WorkspaceExecutionState 行持有 shared transaction lock，验证 generation 后一直持有到 commit。 -- fence 操作获取 exclusive lock，等待已有写事务提交或回滚，再设置 write_fenced 和新 generation。 -- PostgreSQL 可以使用行锁或 transaction advisory lock；SQLite 使用单写事务和显式 maintenance fence 实现同一语义。 -- DB outbox 与业务写入同事务保存 operation_id 和 generation。 -- Object Store、平台消息、Plugin Host 等不可回滚副作用由 outbox dispatcher 执行，并在实际副作用前再次校验 generation。 -- Object Store 大文件先写 generation-scoped staging key，校验后再发布最终引用；旧 generation 的 staging data 可回收。 -- 无法让外部平台原生校验 fencing token 时，dispatcher 必须保证单 owner，并在切换前 drain。 - -旧 generation 即使仍能连接数据库也不能提交写入或发出新副作用。读请求是否允许在 draining 阶段继续，由迁移策略显式决定。 - -### 10.4 WorkspaceMembership - -新增 workspace_memberships： - -- uuid -- workspace_uuid -- account_uuid -- role -- status: active、disabled、removed -- invited_by_account_uuid -- joined_at -- projection_revision -- created_at -- updated_at - -唯一键： - -- workspace_uuid 与 account_uuid。 - -角色先固定为： - -| Role | 说明 | -| --------- | ------------------------------------------------- | -| owner | Workspace 所有者，可转让 owner | -| admin | 管理成员和绝大多数资源，不能处理 owner | -| developer | 管理 Bot、Pipeline、Model、Knowledge、Plugin、MCP | -| operator | 运行和观察工作负载，不能读取或修改敏感密钥 | -| viewer | 只读非敏感资源和监控 | - -固定权限矩阵： - -| Permission | owner | admin | developer | operator | viewer | -| ---------------------- | ----- | ------------------ | --------- | -------- | ------ | -| workspace.view | 是 | 是 | 是 | 是 | 是 | -| workspace.update | 是 | 是 | 否 | 否 | 否 | -| workspace.delete | 是 | 否 | 否 | 否 | 否 | -| owner.transfer | 是 | 否 | 否 | 否 | 否 | -| member.view | 是 | 是 | 是 | 是 | 是 | -| member.invite | 是 | 是 | 否 | 否 | 否 | -| member.update_role | 是 | 是，不能操作 owner | 否 | 否 | 否 | -| member.remove | 是 | 是，不能操作 owner | 否 | 否 | 否 | -| resource.view | 是 | 是 | 是 | 是 | 是 | -| resource.manage | 是 | 是 | 是 | 否 | 否 | -| runtime.operate | 是 | 是 | 是 | 是 | 否 | -| provider_secret.manage | 是 | 是 | 是 | 否 | 否 | -| api_key.manage | 是 | 是 | 否 | 否 | 否 | -| audit.view | 是 | 是 | 否 | 否 | 否 | -| data.export | 是 | 是 | 否 | 否 | 否 | -| billing_link.manage | 是 | 否 | 否 | 否 | 否 | - -resource 代表 Bot、Pipeline、Model、Knowledge、Plugin、MCP 和 Webhook；具体 Service 可以再拆细粒度 permission，但默认映射不得放宽。Monitoring 普通查看属于 resource.view，安全审计属于 audit.view。 - -权限映射在 Core 中有单一代码定义。自定义角色后续由控制面下发角色策略，但 Core 仍负责执行。Workspace role 不授予 BillingAccount 或平台管理权限。 - -### 10.5 WorkspaceInvitation - -新增 workspace_invitations： - -- uuid -- workspace_uuid -- normalized_email -- role -- token_hash -- status -- expires_at -- accepted_at -- revoked_at -- created_by_account_uuid -- created_at -- updated_at - -该表只用于 OSS 本地邀请，是本地权威表。SaaS Invitation 只存在于 Control Plane；Data Plane 不保存 pending email、token 或 token hash，接受邀请后只接收 Membership 投影。 - -### 10.6 业务资源 - -以下表新增非空 workspace_uuid： - -- bots -- legacy_pipelines -- pipeline_run_records -- model_providers -- llm_models -- embedding_models -- rerank_models -- plugin_settings 或 plugin_installations -- mcp_servers -- knowledge_bases -- knowledge_base_files -- knowledge_base_chunks -- monitoring 相关表 -- api_keys -- webhooks -- binary_storages -- Workspace metadata - -建议： - -- 每个租户表都有 workspace_uuid 与 created_at 或 updated_at 的索引。 -- 父子表使用包含 workspace_uuid 的复合外键。 -- 关联关系使用 workspace_uuid、parent_uuid、child_uuid 共同约束。 -- 全局唯一 uuid 仍保留，但所有 Repository 查询必须同时带 workspace_uuid。 -- plugin_settings 唯一键改为 workspace_uuid、plugin_author、plugin_name。 -- mcp_servers 的名称只在 Workspace 内唯一。 -- binary storage key 必须包含 instance_uuid、workspace_uuid、owner_type、owner 和 key。 - -metadata 拆为： - -- system_metadata：实例版本、迁移和全局运行信息。 -- workspace_metadata：wizard、页面设置和 Workspace 配置。 - -### 10.7 API Key - -新增或升级字段： - -- uuid -- workspace_uuid -- created_by_account_uuid -- key_hash -- scopes -- status -- expires_at -- last_used_at -- created_at - -当前全局 API Key 在 SaaS 模式必须禁用。 - -### 10.8 防御性数据库约束 - -- 所有租户表 workspace_uuid 为 NOT NULL。 -- 复合外键阻止跨 Workspace 关联。 -- owner 最后一个成员保护使用事务锁和服务层约束。 -- SaaS 投影 revision 单调增加。 -- PostgreSQL RLS 可以在后续作为 defense-in-depth，但不能替代 Repository 和 Service 权限检查。 - -## 11. Control Plane 数据模型 - -建议按领域拆分。 - -### 11.1 Identity 与 Directory - -```text -accounts -auth_identities -sessions -workspaces -workspace_memberships -workspace_invitations -workspace_routes -``` - -平台管理员角色和 Workspace 角色必须彻底分开。 - -### 11.2 Billing - -```text -billing_accounts -billing_account_memberships -billing_account_workspaces -products -plan_versions -prices -plan_features -subscriptions -subscription_items -orders -order_items -payment_attempts -provider_events -invoices -refunds -credit_ledger -``` - -Payment provider webhook 先按唯一 provider_event_id 持久化，再通过 outbox 异步履约。支付事务中不能直接创建 Deployment。 - -### 11.3 Entitlement 与 Usage - -```text -entitlement_grants -entitlement_snapshots -usage_events -usage_aggregates -budget_leases -``` - -### 11.4 Cloud - -```text -cells -cloud_instances -workspace_placements -workspace_routes -runtime_classes -releases -deployment_operations -domains -desired_states -observed_states -outbox_events -inbox_events -audit_logs -``` - -Space 已有 Instance 模型表达 OAuth 连接记录，新模型应明确命名为 CloudInstance 或 RuntimeShard，不能复用旧名字造成语义冲突。 - -## 12. Billing 与权益设计 - -### 12.1 计费主体 - -推荐： - -- Subscription 的主要 subject 是 Workspace。 -- BillingAccount 是付款主体。 -- 一个 BillingAccount 可以为多个 Workspace 付款。 -- Account 级权益控制可拥有或新建的 Workspace 数量。 -- Workspace 级权益控制成员、Bot、Pipeline、Plugin、MCP、Token、Storage 等限额。 - -这样团队 Workspace 可以独立升级、暂停或迁移，同时企业客户可以统一付款。 - -BillingAccount 使用独立 Membership 和角色： - -| Billing role | 能力 | -| -------------- | ------------------------------------------------------ | -| owner | 管理付款方式、成员、Workspace 绑定、订阅、发票和所有权 | -| billing_admin | 管理付款方式、订阅和发票，不能转移所有权 | -| billing_viewer | 只读账单、发票和用量 | - -将 Workspace 绑定到 BillingAccount 必须同时满足 Workspace 侧 billing_link.manage 授权和 BillingAccount 侧 owner 或 billing_admin 授权。Workspace admin 不会因为角色而自动看到同一 BillingAccount 下其他 Workspace 的发票。平台管理员角色同样独立，并且所有 support 操作需要审计。 - -### 12.2 Personal Workspace - -SaaS 注册自动创建的 personal Workspace 建议默认绑定免费 entitlement： - -- 不因接受团队邀请而产生额外账单。 -- 可以限制 Bot、模型额度和成员数。 -- 是否可邀请其他用户由产品计划决定。 -- 升级 personal Workspace 时创建或绑定 BillingAccount。 - -### 12.3 Subscription 状态 - -建议状态机： - -```text -trialing -> active -trialing -> expired -active -> past_due -> grace -> suspended -past_due -> active -grace -> active -suspended -> active -active -> canceled_at_period_end -> canceled -canceled_at_period_end -> active -``` - -past_due、grace 或 suspended 只有在支付恢复、ProviderEvent 已幂等入账并重新签发 Entitlement 后才回到 active。canceled_at_period_end 可以在当前 period 结束前撤销；已经 canceled 的订阅重新购买时创建新 Subscription 或新 versioned lifecycle，不原地篡改历史。 - -Subscription 是权益来源，PaymentOrder 只是一次交易。Product、PlanVersion 和 Price 一旦用于账单就不可变，只能新建版本。不得继续用订单成功或 Pod 到期日期直接代表订阅真相。 - -### 12.4 产品权益与部署规格分离 - -必须拆开： - -- Product、PlanVersion、Price、Entitlement：用户购买什么。 -- RuntimeClass、CapacityProfile、Release：平台如何部署。 -- Placement isolation_tier：shared 或 dedicated。 - -同一 Product 可以随容量策略变化而放在不同 RuntimeClass；独享实例只是 placement policy，不是新的租户模型。 - -## 13. 现有 Space 的复用边界 - -对最新 Space 架构的结论是：外围能力可复用，旧 Cloud 核心模型不可复用。 - -本次审查基于 langbot-space origin/main 4c1de05778f3，关键事实锚点： - -- internal/entities/persistence/account.go：AccountResources 仍以 MaxPods 等账户级资源建模。 -- internal/entities/persistence/payment.go：PaymentOrder 直接包含 PlanType、BillingCycle、PodUUID 和 PodSubdomain。 -- internal/entities/persistence/cloud_plan.go：CloudPlan 同时保存价格、产品限制、镜像、CPU、内存和 PVC。 -- internal/entities/persistence/pod.go：Pod 同时保存账户、部署和订阅状态。 -- internal/service/cloud.go：namespace 仍由 Account UUID 派生，Pod 删除路径会操作账户 namespace。 -- internal/controller/tasks/lifecycle_consumer.go：retry、heartbeat、generation、fencing 和 reconcile 模式值得复用。 - -### 13.1 复用与重构矩阵 - -| 模块 | 结论 | 可复用内容 | 新架构要求 | -| ---------------- | --------------- | ------------------------------------------------ | ------------------------------------------------------------------------ | -| Account 与 OAuth | 部分复用 | GitHub、Google、邮箱验证、Device Flow 和交互 | 采用 Account、AuthIdentity、Session；provider subject 绑定，不能只按邮箱 | -| Payment | 适配器级复用 | Stripe、EPay 签名、回调解析、金额单位转换 | 新建 ProviderEvent、PaymentAttempt、Order、Invoice、Refund | -| Plan | 规则思路复用 | 本地化名称、价格展示、feature limit、试用和审计 | Product entitlement 与 RuntimeClass 完全分离 | -| Credit 与 Usage | 账本思路复用 | before、after balance 和交易记录 | 改为 BillingAccount 或 Workspace，增加 reference 和 idempotency | -| Email | 高度复用 | SMTP、Resend、i18n template、HTML layout、幂等头 | 事务邮件进入持久队列，重写 Pod 相关模板 | -| Admin | UI 和查询可复用 | 用户、订单、收入、支出列表 | 显式平台角色、审计、限时 support impersonation | -| Lifecycle | 设计经验复用 | generation、fencing、retry、heartbeat、reconcile | 由 Cell Agent 和 DeploymentOperation 实现 | -| Marketplace | 独立保留 | 插件市场和公共站点 | 不承担数据面租户授权 | -| Pod 与 K8s | 废弃 | 无领域模型复用 | 使用 Cell、CloudInstance、Placement 和 DesiredState | - -### 13.2 必须废弃的旧模型 - -- AccountResources.MaxPods。 -- AccountUUID 到 namespace 的映射。 -- Pod 同时承载部署、租户、套餐和订阅状态。 -- 删除 Pod 时删除整个账户 namespace。 -- PaymentOrder 中的 PodUUID 和 PodSubdomain。 -- CloudPlan 同时包含售价、feature、image、CPU、memory 和 PVC。 -- Pod 到期、续费和购买任务作为 Subscription 真相。 -- Space Web API 直接操作 Kubernetes 资源。 - -这些模型与 Workspace 内嵌于 LangBot 实例的方向根本不同，不做兼容层。 - -### 13.3 推荐代码组织 - -- 保留 Space 作为 Marketplace、公共网站和 Cloud Portal/BFF。 -- 新建独立部署的 closed Cloud Control Plane。 -- 从 Space 提取 OAuth、Payment、Email 和 Admin 共享内部 Go package。 -- Control Plane 从第一天使用版本化 SQL migration，不依赖 AutoMigrate 作为生产 schema 变更机制。 -- Cell Agent 独立部署，只持有当前 Cell 所需的最小凭证。 - -## 14. Cloud v2 部署架构 - -### 14.1 拓扑 - -```mermaid -flowchart TD - Edge["Global Edge / Workspace Router"] --> CP["Cloud Control Plane"] - Edge --> CellA["Cell A"] - Edge --> CellB["Cell B"] - CP --> AgentA["Cell Agent A"] - CP --> AgentB["Cell Agent B"] - AgentA --> InstanceA["CloudInstance A"] - AgentA --> InstanceB["CloudInstance B"] - AgentB --> InstanceC["CloudInstance C"] - InstanceA --> WA["Workspace A / B / C"] - InstanceB --> WB["Workspace D"] - InstanceC --> WC["Workspace E / F"] -``` - -一个 Cell 提供共享基础设施： - -- Cell Agent 和 Cell ingress。 -- 计算资源池。 -- PostgreSQL cluster。 -- Valkey 或 Redis cluster。 -- Object Store 和 Secret Store 接入。 -- Box sandbox worker pool。 -- 日志、指标和审计出口。 - -每个 CloudInstance 在 Cell 内拥有独立的： - -- InstanceManifest 和 release。 -- Core deployment 与 service route。 -- PostgreSQL database 或 schema namespace。 -- cache、lock 和 secret prefix。 -- runtime ownership domain。 -- capacity budget。 - -物理 PostgreSQL、Valkey 和 Object Store 可以由同一个 Cell 共享，但 CloudInstance 的逻辑 namespace 不能共享。Router 的实际目标是 CloudInstance service，而不是一个无法区分实例的 Cell-wide Core API。 - -所有权矩阵： - -| 层级 | 拥有内容 | 不是它的职责 | -| ------------- | ---------------------------------------------------------------------- | ---------------------------------- | -| Cell | Agent、ingress、compute pool、存储集群、故障域和区域 | Workspace Membership、Subscription | -| CloudInstance | Core deployment、Manifest、DB namespace、release、runtime owner 和容量 | 全局账户、支付 | -| Workspace | 成员关系投影、业务资源、Plugin worker、Storage prefix 和 Entitlement | 物理集群、付款身份 | - -由于一个 CloudInstance 使用独立数据 namespace，Core 普通 Repository 在 namespace 内以 workspace_uuid 过滤；跨 namespace 的 Router、Storage、Cache 和运维协议必须同时使用 instance_uuid 与 workspace_uuid。 - -### 14.2 Placement - -workspace_placements 至少包含： - -```text -workspace_uuid -cell_uuid -cloud_instance_uuid -isolation_tier -region -residency -state -generation -desired_release -created_at -updated_at -``` - -Router 缓存 route_key 到 cell_uuid、cloud_instance_uuid、generation。路由切换必须 CAS generation，旧 generation 的内部请求和 runtime owner 必须被拒绝。 - -### 14.3 第一阶段运行约束 - -当前 Core manager 和平台连接是进程内单例。Cloud v2 MVP 应： - -- 每个 CloudInstance 的完整 Core 只运行一个 active replica。 -- 一个 runtime owner 可以服务多个 Workspace，但所有索引、连接和任务必须 Workspace-aware。 -- 只有 Edge、Portal 和 Control Plane 可以在此阶段独立无状态扩展。 -- 为每个 CloudInstance 设置 Workspace 和负载上限。 -- dedicated 套餐仍使用同一 Workspace 模型，只分配独享 CloudInstance 或 Cell。 - -当前 Core 尚未拆分 API 和 runtime，因此不能宣称 API replica 可独立扩展，也不能直接把现有整体 Deployment replicas 从 1 改为 2；否则平台长连接、scheduler、插件和消息发送可能重复执行。 - -### 14.4 后续 HA - -演进方向： - -- 将 API 与 runtime ownership 拆分。 -- runtime worker 按 Workspace 或 Bot 获取 lease。 -- lease 带 generation 和 fencing token。 -- scheduler、platform adapter 和 plugin worker 都受 lease 约束。 -- 旧 worker 恢复后不能继续发送消息或写入新 generation。 -- PostgreSQL 使用多 AZ 和 PITR。 -- Object Store 启用版本化。 -- Valkey 只做缓存、锁和临时状态，不作为权威数据库。 - -第一阶段不做单 Workspace 跨 Cell active-active。优先实现 Cell 内多 AZ 和跨 Cell 备份恢复。 - -### 14.5 Workspace 迁移 - -未来 Workspace 在 CloudInstance 间迁移的通用流程： - -1. Placement 进入 migrating。 -2. 目标实例建立 Manifest、Entitlement、directory projection 和数据副本。 -3. 通过 snapshot 与 delta 或 CDC 追平并记录 source watermark。 -4. 在源端开启 write fence，停止新写入并排空 API、job 和 outbox。 -5. 完成 final delta，校验 target watermark、资源计数和 checksum。 -6. 提升 placement generation；源数据库拒绝旧 generation 的后续写入。 -7. 冻结旧 runtime owner并停止旧平台连接。 -8. CAS 切换 route。 -9. 目标实例确认 projection、entitlement 和 runtime ready 后启动新 owner。 -10. 保留源端只读快照和回滚 generation 窗口。 - -该流程不依赖旧 Space Pod，也不要求 Workspace 等于 Deployment。 - -## 15. Core 服务层改造 - -### 15.1 WorkspaceScoped Repository - -所有业务 Service 显式接收 RequestContext： - -```python -async def get_bot(self, ctx: RequestContext, bot_uuid: str) -> Bot: - require(ctx, "bot.view") - stmt = ( - select(Bot) - .where(Bot.workspace_uuid == ctx.workspace.workspace_uuid) - .where(Bot.uuid == bot_uuid) - ) - bot = await self.session.scalar(stmt) - if bot is None: - raise NotFoundError() - return bot -``` - -禁止： - -- Service 内 select 全表后在 Controller 过滤。 -- 通过可选 workspace_uuid = None 表示系统权限。 -- 使用最近 Workspace 或默认 Workspace 执行后台任务。 -- 仅检查资源 uuid 而不检查 workspace_uuid。 - -需要逐一改造： - -- Account、Workspace、Membership、Invitation。 -- API Key。 -- Bot 和 Platform。 -- Pipeline 和 RunRecord。 -- ModelProvider 和所有 Model。 -- Plugin installation、configuration 和 page API。 -- MCP。 -- Knowledge 和 RAG。 -- Monitoring。 -- Webhook。 -- Binary Storage。 - -### 15.2 后台任务 - -后台任务 payload 必须包含： - -- instance_uuid。 -- workspace_uuid。 -- actor 或受约束的 SystemContext。 -- resource_uuid。 -- operation_id。 -- generation。 - -跨所有 Workspace 的平台任务只能使用专门的 SystemContext，并由明确的 allowlist repository 执行，不能复用普通 Service 的 context=None。 - -## 16. 运行时与 SDK 隔离 - -### 16.1 Query、Event 和 Session - -Query 和 Event 增加 workspace_uuid 用于传递和观测，但它不是授权来源。Host 必须从可信 Bot、Pipeline、Plugin installation 或连接记录推导 Workspace，并校验传入字段一致。 - -Session key 至少为： - -```text -workspace_uuid -bot_uuid -launcher_type -launcher_id -``` - -Query、Session、RuntimeBot、RuntimePipeline、cache key、lock key 和 manager index 都必须包含 Workspace。 - -### 16.2 Plugin - -SaaS 不允许多个 Workspace 共享同一个第三方插件进程或可信 Host 连接。原因是插件可以保留跨请求内存、启动后台任务并主动调用 Host API。 - -安全不变量： - -- 一个不可信 Plugin 进程、容器或 supervisor 不能服务多个 Workspace。 -- Plugin code artifact 和只读包缓存可以共享。 -- Plugin config、secret、storage 和 connection 不能跨 Workspace 共享。 -- Host connection 在服务端至少绑定 workspace_uuid；每次调用还绑定不可伪造的 installation capability。 -- Plugin 不能提交任意 workspace_uuid。 -- Host API 从连接上下文推导 Workspace。 -- Plugin page API 同时校验 Account Membership 和 Plugin installation。 - -隔离等级： - -| Level | 进程边界 | 用途 | -| -------------------- | ---------------------------------------------------------------------- | ------------------------------------- | -| workspace_worker | 一个 Workspace 一个 supervisor，可承载该 Workspace 内多个 installation | SaaS 默认基线 | -| installation_sandbox | 每个 installation 独立进程或容器 | 高风险、未知来源或申请高权限的 Plugin | -| dedicated_worker | Workspace 独享节点或更强 sandbox | 企业与高隔离套餐 | - -同一 Workspace 内多个 Plugin 共享 worker 意味着它们共享租户内信任域；一旦需要防止 Plugin 之间读取内存或 secret，必须提升为 installation_sandbox。具体分级由签名、来源、权限声明和产品策略决定，MVP 不必无条件为每个 installation 启动容器。 - -OSS 因为实例只有一个 Workspace，可以继续使用单一插件 runtime，但协议和存储仍必须 Workspace-aware。 - -### 16.3 Plugin Host API - -以下 API 必须限定当前 Workspace： - -- get_bots 和 get_bot_info。 -- send_message。 -- get_llm_models 和 invoke_llm。 -- list_plugins_manifest。 -- list_commands 和 list_tools。 -- call_tool。 -- invoke_embedding 和 vector 相关接口。 -- list_knowledge_bases 和 retrieve_knowledge。 -- workspace storage。 - -跨 Workspace 管理能力不能通过普通 Plugin Host API 暴露。 - -### 16.4 MCP - -- mcp_servers 带 workspace_uuid。 -- MCP runtime key 使用 workspace_uuid 与 server_uuid。 -- 同名 server 或 tool 不能共享 session。 -- Pipeline 只能引用同 Workspace 的 MCP server。 -- 当前全局 mcp-shared Box session 必须改成每 Workspace 隔离。 -- MCP Resource 的 list、read、附件和二进制读取必须在 Workspace 内校验。 - -### 16.5 Box - -Box namespace 至少是 instance_uuid 与 workspace_uuid： - -- Session、Process、目录、端口和 WebSocket attach token 都在该 namespace 内。 -- SaaS 插件不能传任意 host_path。 -- 不允许额外特权挂载。 -- 所谓 global sandbox 只能表示 Workspace 内 global，不能表示整个 CloudInstance global。 -- 配额从 EntitlementSnapshot 获取并在 Core 或 Box gateway 执行。 - -### 16.6 RAG、Vector 和 Object Storage - -- collection 的物理名称由服务端使用 instance_uuid 和 workspace_uuid 派生。 -- Plugin 只使用不透明 collection handle。 -- 所有 embedding 和 retrieval 记录带 workspace_uuid。 -- 对象路径固定在 instances、instance_uuid、workspaces、workspace_uuid 前缀下。 -- 外部 API 不接受任意 storage path。 -- Secret Store 和 Redis key 必须带 Workspace。 -- 日志和 trace 保留完整 instance_uuid 与 workspace_uuid。 -- Metrics 使用 Cell、plan、runtime class 等受控低基数标签；除受控小规模内部指标外，不把完整 Workspace UUID 作为时序标签，避免高基数爆炸。 - -## 17. HTTP API 与前端 - -### 17.1 OSS Core Workspace API - -OSS Core 提供本地 Workspace 和成员协作 API： +OSS 与 SaaS 执行面共用通用 Workspace API： ```text GET /api/v1/workspaces -POST /api/v1/workspaces -GET /api/v1/workspaces/current GET /api/v1/workspaces/{workspace_uuid} GET /api/v1/workspaces/{workspace_uuid}/members POST /api/v1/workspaces/{workspace_uuid}/invitations -POST /api/v1/invitations/accept PATCH /api/v1/workspaces/{workspace_uuid}/members/{account_uuid} DELETE /api/v1/workspaces/{workspace_uuid}/members/{account_uuid} ``` -GET 返回实例唯一 Workspace；POST /workspaces 为保持客户端契约可以存在，但固定返回 edition_limit。Invitation token 放在 POST body 中并全链路日志脱敏，不能放在 URL path、query、Referer 或分析事件中。Invitation 接受和 Membership 修改在 OSS 本地事务中完成。 +Cloud policy 下，目录 mutation 由闭源 Control Plane 负责；Core 对本地创建、邀请和成员修改返回稳定的 +`control_plane_required`，只提供执行投影的安全读取。 -### 17.2 SaaS Cloud API +所有 tenant resource route 必须经过统一 decorator/middleware： -SaaS 的全局 Workspace lifecycle、Membership 和 Invitation 权威写入由 Control Plane 或 Cloud BFF 提供： +1. 认证 principal。 +2. 解析可信 Workspace。 +3. 校验 Workspace/ExecutionState。 +4. 校验 Membership 或资源绑定。 +5. 校验 permission 和 entitlement。 +6. 创建 RequestContext 与 TenantUnitOfWork。 + +### 13.2 SaaS Control Plane API + +SaaS 产品 API 包含： ```text -GET /cloud/v1/workspaces -POST /cloud/v1/workspaces -GET /cloud/v1/workspaces/{workspace_uuid} -GET /cloud/v1/workspaces/{workspace_uuid}/members -POST /cloud/v1/workspaces/{workspace_uuid}/invitations -POST /cloud/v1/invitations/accept -PATCH /cloud/v1/workspaces/{workspace_uuid}/members/{account_uuid} -DELETE /cloud/v1/workspaces/{workspace_uuid}/members/{account_uuid} -GET /cloud/v1/workspaces/{workspace_uuid}/billing +POST /cloud/workspaces +GET /cloud/workspaces +POST /cloud/workspaces/{workspace_uuid}/invitations +POST /cloud/invitations/{token}/accept +GET /cloud/workspaces/{workspace_uuid}/subscription +POST /cloud/workspaces/{workspace_uuid}/checkout +GET /cloud/workspaces/{workspace_uuid}/usage ``` -OSS endpoint 由 LocalDirectoryProvider 执行；Cloud endpoint 由 Control Plane directory service 执行。Cloud API 完成事务后发布 DirectoryEvent，并在需要立即进入 Workspace 的流程中等待目标 Core 的 contiguous_applied_sequence 达到 required instance_sequence。SaaS 不应把 Invitation 或 Membership 写入请求直接发送给 Data Plane。 +这些 API 管理目录、产品和计费，不直接操作 Bot/Pipeline 等 Core 业务资源。 -SaaS Core 对外只提供当前 placement 内的 Workspace 投影读取和业务资源 API。对用户发起的 directory mutation 固定返回 cloud_directory_read_only。控制面投影写入使用独立的 mTLS internal endpoint 或 event consumer，不复用用户管理 API。 +### 13.3 WebUI -现有 Bot、Pipeline、Provider、Plugin、MCP、Knowledge 和 Monitoring API 路径可保留，但必须从 RequestContext 限定 Workspace。 +OSS： -### 17.3 OSS 前端 +- 首次注册进入唯一 Workspace。 +- owner/admin 可邀请成员并管理固定角色。 +- 不展示 Workspace 切换器和创建第二 Workspace 的入口。 -- 不显示 Workspace switcher 和 Create Workspace。 -- 可以显示唯一 Workspace 名称。 -- 提供 Members 和 Invitations 页面。 -- 提供固定角色选择。 -- 初始化页面创建首位 owner 和唯一 Workspace。 +SaaS： -### 17.4 SaaS 前端 +- 登录先获取 Account 级 Workspace 列表，再显式选择当前 Workspace。 +- 当前 Workspace UUID 保存在受控客户端状态中；所有 tenant request 自动附带 selector。 +- 切换 Account 或 Workspace 时清理缓存、WebSocket、上传、表单、错误和 optimistic state，不能显示前一租户数据。 +- 页面 refresh、新 tab 和邀请跳转恢复同一个经过授权的 Workspace；失效 Membership 不回退到其他 Workspace。 +- UI 权限变化必须响应式更新，但 API 仍是最终授权边界。 -- Cloud shell 从 Control Plane 获取全局 Workspace directory。 -- 明确选择 Workspace 后进入稳定 Workspace route。 -- 显示 Workspace switcher、Create Workspace、Members、Billing 和 Usage。 -- 切换 Workspace 后清理前一个 Workspace 的客户端 query cache。 -- 所有 Data Plane 请求显式带当前 Workspace 选择器。 -- 不能仅把 currentWorkspaceId 写入 localStorage 就认为完成安全隔离。 +## 14. 故障、安全与降级 -### 17.5 登录后的 SaaS 路径 +### 14.1 Fail-closed 场景 -1. Control Plane 完成全局登录。 -2. 前端读取 Workspace directory。 -3. 用户选择 Workspace，或注册工作流明确返回刚创建的 Workspace。 -4. Router 解析 Placement。 -5. Edge 验证 Account token并签发 instance-scoped assertion；Data Plane 验证 assertion、Workspace、Placement generation 和本地 Membership 投影。 -6. 前端进入 Workspace 内页面。 +以下情况必须拒绝新的租户业务和副作用： -普通资源请求缺少 Workspace 时不做隐式跳转，由前端处理 workspace_required。 +- Cloud manifest 缺失、签名失败、audience 错误或回滚。 +- Account token、Membership、Workspace status 或 execution generation 无效。 +- 目录投影未就绪或落后于有效 lease 要求。 +- Entitlement 缺失、过期且不在明确 grace 范围内。 +- Runtime 控制通道认证失败或实例绑定不一致。 +- Plugin nsjail/cgroup hard limit 在 Cloud profile 下不可用。 +- Box 的任一硬存储或 namespace capability 无法证明。 +- PostgreSQL RLS、runtime role、schema、catalog 或 endpoint 隔离校验失败。 +- stdio MCP 在 Cloud profile 下被尝试启用。 -## 18. 故障与降级策略 +不能把上述错误静默降级为 OSS singleton、普通子进程、Chroma、软 quota 或 caller-supplied Workspace。 -### 18.1 不同租约不能混用 +### 14.2 撤销语义 -Manifest、Account token、Directory freshness 和 Entitlement 使用不同的有效期： +- Membership 删除或降权必须影响下一次 HTTP 请求，并使长连接在下一条消息前重新授权。 +- Workspace 暂停禁止新交互、自动化工作负载和新副作用；恢复只允许当前 generation。 +- entitlement 到期按 capability 明确停止新创建或新执行，不隐式删除已有数据。 +- generation 变化使旧 worker、session、callback、cached runtime object 和 outbox publisher 失效。 +- 控制面暂时不可达时，只能在有效签名快照和本地投影允许的范围内继续；过期后失败关闭。 -| 状态 | 建议有效期 | 控制面故障后的行为 | -| ------------------------- | ---------------------------- | ---------------------------------------------------------------- | -| InstanceManifest | 长于最大 entitlement grace | 运行到 exp；过期后整个 SaaS instance fail closed | -| Account access token | 分钟级短 TTL | 已签发 token 只运行到 exp；不能登录或刷新 | -| InteractiveDirectoryLease | 分钟级，独立可配置 | 交互用户访问只运行到 lease 到期，之后 fail closed | -| WorkspaceStatusLease | 可长于交互 lease，但有硬上限 | Bot、Webhook、API Key 只运行到 lease 到期，之后安全暂停 | -| Entitlement grace | 小时到天 | 只维持已有自动化工作负载和已购能力，不延长 Manifest 或目录 lease | -| Router cache | 短 TTL，带 generation | 可路由到最后已知健康目标；不能忽略 generation | +### 14.3 安全清单 -因此“24 到 72 小时宽限”只适用于 Entitlement 和已有 Bot 等自动化工作负载，不代表用户可以在身份服务或 Directory 断开后继续登录 24 到 72 小时。 +- 所有 identifier 使用不可猜 UUID，但不把随机性当成授权。 +- 所有 token/secret 只存 hash 或加密值，raw secret 一次展示。 +- 日志、trace、metric、cache 和 object key 都包含 Workspace 维度并过滤 secret。 +- Provider、Bot、Plugin、MCP 配置的 read response 递归遮蔽 credential。 +- Runtime control、debug、registration 和 attachment capability 分离，不能复用万能 secret。 +- untrusted code 不访问 Core loopback、数据库、其他 Runtime、宿主文件系统或 metadata endpoint。 +- bulk operation、后台扫描和 monitoring 聚合使用显式 tenant/system capability。 +- 所有跨 Workspace 运维操作记录 principal、reason、scope、request ID 和结果。 -### 18.2 故障矩阵 +## 15. 实现状态与 SaaS 激活门禁 -| 故障 | 交互用户 | 已有 Bot、Webhook、Workspace API Key | 控制类操作 | -| ------------------------------------ | ---------------------------------------------- | ------------------------------------------------------------- | ------------------------------------ | -| Manifest signer 或续签不可用 | 有效 Manifest 与 token 内可用 | 只运行到 Manifest exp | 新实例绑定禁止 | -| Identity 不可用 | 已有 token 到期前可用，不能登录或刷新 | 不受 Account token 影响，但仍检查 WorkspaceStatusLease | 禁止 | -| Directory 不可用 | 有效 token 与 InteractiveDirectoryLease 内可用 | 只运行到 WorkspaceStatusLease exp | 邀请、角色、Workspace lifecycle 禁止 | -| Billing 或 Entitlement signer 不可用 | 按最后快照读取已有资源 | 同时满足有效 Manifest、WorkspaceStatusLease 和 grace 才能继续 | 购买、提额、套餐变更禁止 | -| Router control 不可用 | 使用未过期 route cache | 已建立的数据面连接仍检查 generation 和所有 lease | 新 Placement 和迁移禁止 | -| Data Plane 不可用 | 当前 Workspace 不可用 | 当前 Workspace 停止 | Control Plane 只显示故障，不伪造成功 | +### 15.1 已实现的隔离内核 -UsageEvent 在控制面故障时写入 durable outbox。 +当前分支已经实现或具备基础的部分包括： -### 18.3 必须 fail closed 的情况 +- OSS singleton Workspace、多 Account、Invitation 和固定 RBAC。 +- trusted RequestContext、Workspace-scoped repository 和资源所有权检查。 +- tenant-aware Plugin SDK protocol 与 Runtime installation binding。 +- shared Plugin Runtime / Box Runtime 控制协议和 execution generation fence。 +- stdio MCP 独立 gate。 +- PostgreSQL shared schema、transaction-local scope、FORCE RLS 与 pgvector adapter。 +- Cloud bootstrap 默认不可由普通配置激活，并对缺失安全能力失败关闭。 -- SaaS 首次启动没有有效 InstanceManifest。 -- Manifest 或 Entitlement 签名无效。 -- audience 或 instance_uuid 不匹配。 -- revision 或 placement generation 回滚。 -- 已知 Workspace suspended。 -- 没有本地 Membership 投影。 -- InteractiveDirectoryLease 已过期的交互请求。 -- WorkspaceStatusLease 已过期的 Bot、Webhook、API Key、job 或 runtime 请求。 -- SaaS 请求缺少 Workspace。 +这些是代码能力边界，不等于完成闭源 SaaS 产品或生产部署验收。 -不得回退到 OSS mode、default Workspace 或全局资源访问。 +### 15.2 尚未完成的激活门禁 -### 18.4 Membership 撤销与 Workspace 暂停 +以下事项完成并取得真实环境证据前，不得宣称 Cloud v2 production-ready： -- Account token 使用短 TTL。 -- Membership 变更立即发布高优先级 DirectoryEvent。 -- 本地 projection 保存 membership_revision。 -- Edge 或 Cloud BFF 的签名内部 assertion 携带 directory_min_sequence；Account access token 本身不携带 Membership revision。 -- Core 的 contiguous_applied_sequence 低于 directory_min_sequence 时拒绝请求并等待同步，不能按旧 Membership 放行。 -- InteractiveDirectoryLease 设置用户撤权最坏生效上限；WorkspaceStatusLease 设置安全暂停对自动化工作负载的最坏生效上限。 -- Cell 无法续租时，交互用户、Bot、Webhook、API Key、job 和 runtime 分别在对应 lease 到期后 fail closed。 -- 安全暂停来自 Directory revision，欠费暂停来自 Entitlement revision；两者任一个禁止时都不能运行受影响操作。 -- Billing entitlement grace 不能延长或覆盖安全暂停和 WorkspaceStatusLease。 -- 紧急暂停的最坏生效时间由 WorkspaceStatusLease TTL 明确写入安全 SLA。 +1. 闭源 Control Plane 的全局目录、注册、邀请、订阅、计费、entitlement 签发和签名 manifest bootstrap；横向扩展前 OAuth exchange 与目录投影还必须使用原子共享存储。 +2. 普通业务写入贯穿 commit 的 generation-aware fence，以及与外部副作用同事务的 business outbox。 +3. generation cutover 后稳定的 durable object identity 或原子对象引用迁移。 +4. 所有 tenant-configurable outbound URL 的 SSRF 防护与 tenant-safe egress；Plugin Runtime 还需在真实 Linux/nsjail/cgroup v2 环境验证 namespace、资源限制和文件隔离。 +5. Plugin Runtime 对意外退出的 enabled worker 实现 completion callback、有界 backoff 和自动恢复，并验证不会形成跨租户重启风暴。 +6. Plugin installation data 的 production hard disk quota provider，能够在写入边界原子拒绝超额，不能以目录扫描代替。 +7. Box Runtime 的 production hard quota provider，包括 Workspace、Skill、root/tmp/home 的 byte 与 inode quota；真实部署还必须在启动和重连时通过共享卷 marker challenge。 +8. PostgreSQL runtime credential 的专用 endpoint 或 HBA/proxy 跨 database 隔离证明、生产 migration/rollback 流程，以及 legacy pgvector migration 失败后精确恢复 RLS/FORCE 并可安全重试的集成证据。 +9. 闭源目录事件、lease、snapshot、entitlement 和 usage/outbox 的重放、断流与灾难恢复验证。 +10. 真实浏览器多 Account/RBAC/邀请/刷新场景已完成；仍需生产 Runtime 重启、worker crash、断流、异常回滚和闭源 Control Plane 的 fault-injection 验收。 -### 18.5 Entitlement 到期 +### 15.3 有意暂缓的产品决策 -宽限期结束后： +- Workspace 创建后的休眠、释放、删除和保留策略。 +- Workspace export 与单 Workspace restore。 +- 非 Pro Workspace 的 BYOK E2B WebUI。 +- 多副本 owner lease 的 store、TTL、fencing token 和转移顺序。 +- PostgreSQL shard resolver、在线迁移和 dedicated shard 产品规则。 +- artifact/cache 的签名来源、撤销、GC 和磁盘配额机制。 +- custom roles、SSO、SCIM 和企业合规能力。 -- 阻止付费功能的新建和扩大资源。 -- 是否停止已有 Bot 应作为独立产品策略，不能由通用 quota 代码隐式决定。 -- 欠费暂停必须可审计并支持恢复。 +暂缓项不得被实现代码用隐式默认值提前固化。 -## 19. 安全清单 - -必须防止： - -- Account A 猜测 Account B Workspace 的资源 uuid。 -- API Key A 通过 header 选择 Workspace B。 -- Plugin 枚举其他 Workspace 的 Bot、Model 或 Knowledge。 -- Plugin 在内存中保留 Workspace A 数据后服务 Workspace B。 -- MCP 同名 server 复用 session。 -- Box 同名 session、process 或 attach token 串租户。 -- RAG collection name 冲突。 -- Monitoring session_id 冲突。 -- 后台任务缺少 Workspace 后访问全表。 -- OAuth 只按 email 错误合并不同 provider identity。 -- 邀请并发消费或移除最后一个 owner。 -- 伪造 X-Workspace-Id、内部 assertion 或旧 placement generation。 - -安全原则： - -- 最小权限。 -- 服务端推导 Workspace。 -- 复合外键。 -- 密钥只保存 hash。 -- 内部通信使用 mTLS 和短期签名 service token。 -- 支付 webhook 先持久化再履约。 -- 所有控制面事件和运行操作幂等。 -- 跨租户访问默认返回 404。 - -## 20. 实施阶段 +## 16. 实施顺序 ### Phase 0：契约和基线 -- 冻结术语、产品矩阵和 Source of Truth。 -- 定义 RequestContext、ExecutionContext 和 Provider 接口。 -- 定义 DirectoryEvent、DirectoryLease、InstanceManifest、EntitlementSnapshot、UsageEvent 和 Placement generation schema。 -- 建立跨租户测试辅助工具。 -- 列出所有需要 workspace_uuid 的表、cache、lock、storage 和 runtime key。 +- 固定术语、RequestContext、角色矩阵、edition policy 和错误语义。 +- 建立升级备份、回滚和跨租户负向测试基线。 ### Phase 1：OSS tenancy kernel -- Account 引入稳定 uuid。 -- 新增 Workspace、Membership、Invitation。 -- 首次初始化创建唯一 Workspace 和 owner。 -- 后续邀请注册加入唯一 Workspace。 -- 固定 RBAC。 -- OSS edition policy 强制 workspace_limit = 1。 -- JWT 从 email 主体迁移到 account_uuid。 +- Account、Workspace、Membership、Invitation。 +- singleton bootstrap、多用户邀请、RBAC 和前端权限。 -### Phase 2：资源行级隔离 +### Phase 2：数据与入口隔离 -- 按 expand、backfill、validate、contract 迁移所有现有数据到默认 Workspace。 -- 先增加可回填字段，再校验所有行和关联，最后收紧非空与复合外键。 -- Repository 和 Service 强制 scope。 -- API Key、Webhook 和 Monitoring 完成隔离。 -- SQLite 和 PostgreSQL 迁移均可中断重试并支持 forward recovery。 +- 为所有资源补充 Workspace scope。 +- HTTP、API Key、Bot、Webhook、WebSocket、后台任务和 storage 统一上下文。 +- SQLite migration recovery 与 PostgreSQL RLS 集成测试。 -SQLite 增加复合外键或非空字段可能需要重建表。升级前必须备份并验证恢复，不承诺任意阶段无损 downgrade。 +### Phase 3：Runtime 与 SDK 隔离 -### Phase 3：Runtime 和 SDK 隔离 +- Plugin installation binding、nsjail、资源上限和 artifact replay。 +- Box admission、session namespace、skill/attachment 文件边界。 +- MCP gate、RAG/vector 与 long-running generation revalidation。 -- Query、Event、Session 和 ExecutionContext 增加 Workspace。 -- Bot、Pipeline、Model、RAG manager 的 key 和 cache 改造。 -- Plugin connection、process、config 和 storage 隔离。 -- MCP 和 Box 隔离。 -- SDK Host API 不允许任意 Workspace 参数。 +### Phase 4：闭源 SaaS 控制面 -### Phase 4：Cloud 协议 +- signed manifest bootstrap。 +- 全局目录、注册、邀请、Subscription、Entitlement 和 Usage ledger。 +- projection、lease、outbox、reconciliation 和运维后台。 -- InstanceManifest 和 JWKS。 -- SaaS Account、Workspace、Membership projection。 -- Directory inbox/outbox。 -- instance_sequence、snapshot reconciliation、InteractiveDirectoryLease 和 WorkspaceStatusLease。 -- Entitlement cache 和 quota enforcement。 -- Usage outbox。 +### Phase 5：生产部署激活 -### Phase 5：Closed Control Plane 与 Billing +- 真实 Linux Plugin/Box hard isolation。 +- PostgreSQL credential、migration、backup 和 rollback 验证。 +- 完整浏览器/API/Runtime E2E 和故障注入。 +- 所有激活门禁通过后才开启多 Workspace Cloud policy。 -- 全局 Account 和 Workspace directory。 -- BillingAccount、Product、Plan、Price 和 Subscription。 -- Payment provider event 和财务模型。 -- Entitlement signer 和 Usage ledger。 -- Cloud Portal 和运营后台。 +### Phase 6：同逻辑实例内部扩展 -### Phase 6：Cloud Cell +- 有容量证据后增加副本、owner lease 和 fencing。 +- 有地域、合规或规模证据后增加 shared/dedicated shard。 +- 保持外部身份、API 和 Workspace URL 不变。 -- Cell Agent。 -- CloudInstance、Placement 和 Router。 -- WorkspaceExecutionState、签名 gateway assertion 和 Repository write fence。 -- Workspace provisioning workflow。 -- runtime ownership 和 generation fencing。 -- shared 与 dedicated placement。 -- Cloud v2 新用户完整链路。 +## 17. 测试与验收 -### Phase 7：HA 与规模化 +### 17.1 数据隔离 -- API 与 runtime worker 拆分。 -- Workspace 或 Bot lease。 -- 同 Cell 多 AZ。 -- Workspace migration。 -- 跨 Cell DR。 +- 两个 Workspace 使用相同 resource UUID、name、vector ID 和 cache key，不发生冲突或越权。 +- 故意遗漏应用层 Workspace filter 时，PostgreSQL RLS 仍阻止跨租户读写。 +- 连接池复用、异常回滚、子任务、后台任务和 transaction pooling 不残留 tenant context。 +- 跨 Workspace 猜测返回 404；同租户缺权限返回 403。 -### Multi-workspace 发布门禁 +### 17.2 产品行为 -- Phase 1 和 Phase 2 只允许发布 OSS 单 Workspace 多用户。 -- 数据库出现 workspace_uuid 不代表可以启用 SaaS 多 Workspace。 -- 在 Phase 3 完成前，Plugin、Session、MCP、Box 和 Runtime 仍可能有全局状态，任何环境都不得创建第二个生产 Workspace。 -- SaaS multi_workspace capability 只有在资源隔离、Runtime、SDK、Directory projection、两类 DirectoryLease、Manifest、Entitlement、Placement write fence 和跨租户测试全部通过后才能由签名 Manifest 开启。 -- Manifest 应携带 tenant_isolation_version；Core 只接受自身声明支持且通过 release gate 的版本。 -- 未通过门禁的 release 固定执行 workspace_limit = 1，即使 Control Plane 错误下发更高额度也不放开。 -- 第一批 schema PR 不暴露 SaaS Workspace 创建、切换或 Cloud directory 写入。 +- OSS 首个 Account 创建唯一 Workspace；第二个 Account 只能通过邀请加入；创建第二 Workspace 返回 edition error。 +- 邀请覆盖有效、已使用、撤销、过期、邮箱不匹配和并发接受。 +- owner/admin/developer/operator/viewer 的 API 和 WebUI 权限一致。 +- SaaS 普通注册和邀请注册都创建个人 Workspace，但不创建专属部署或 Runtime。 -## 21. 第一批代码改动建议 +### 17.3 Runtime -第一批 PR 只做可独立验证的基础骨架，不同时改完所有资源： +- 两个 Workspace 安装同一已验证 artifact 时只共享只读 code/env，进程、secret、home/tmp/data、日志和 Host API 完全隔离。 +- cgroup、rlimit、namespace、egress 和 generation fence 在真实 Linux 环境生效。 +- Runtime restart/cache loss 通过 durable desired state 与 binary storage 恢复。 +- 两个 Workspace 的 Box session、files、process、skill、attachment 和 quota 完全隔离。 +- stdio MCP gate 对 UI、API、bootstrap 和最终 execution 同时生效。 -1. 新增 Account uuid、Workspace、Membership、Invitation 表和 Alembic migration。 -2. 迁移旧实例到唯一 Default Workspace。 -3. 首个用户成为 owner。 -4. 引入 RequestContext 和固定权限定义。 -5. JWT sub 改为 account_uuid，并保留兼容读取旧 token 的短期迁移策略。 -6. OSS 创建第二个 Workspace 返回 edition_limit。 -7. 增加 OSS 多用户邀请的 Service 测试。 -8. 增加两个 Workspace 的 Repository 隔离测试基础设施。 +### 17.4 Control Plane 与故障 -第一批 PR 不开启 SaaS multi_workspace capability。第二批再从 Bot 开始逐资源强制 Workspace scope，不能长期停留在“表已有 workspace_uuid、查询仍访问全表”的中间状态。 +- DirectoryEvent 重复、乱序、缺口、snapshot + replay 和过期 lease 均安全。 +- Entitlement 旧 revision、签名错误、过期和撤销均失败关闭。 +- UsageEvent 重放不重复计费；业务事务回滚不发送副作用。 +- Runtime、Core 或 Control Plane 重启不创建重复 Workspace、worker 或 sandbox。 +- manifest、数据库安全校验或 hard quota 缺失时实例保持不可激活，而不是静默降级。 -## 22. 测试与验收 +### 17.5 浏览器端到端 -### 22.1 Migration +真实浏览器至少覆盖： -- SQLite 旧实例升级。 -- PostgreSQL 旧实例升级。 -- 已有 Account 成为默认 Workspace owner。 -- 所有旧资源完成 workspace_uuid backfill。 -- migration 中断后可安全重试。 -- SQLite 重建表前生成并验证备份，失败时可 forward recover。 -- system metadata 与 workspace metadata 正确拆分。 +1. clean database 首位 owner 注册与 singleton Workspace bootstrap。 +2. owner 创建邀请，第二个用户注册/登录并接受。 +3. 角色在 viewer/operator/developer/admin 间变化时，导航、控制项和 API 结果同步变化。 +4. Account/Workspace 切换清空前一 scope 状态，refresh 和新 tab 恢复正确 Workspace。 +5. 第二 Workspace edition limit，以及 invitation used/revoked/expired/email mismatch 的可见错误。 +6. 直接 API 越权、伪造 selector 和跨租户 UUID 猜测不能绕过 UI。 -### 22.2 OSS 产品行为 +## 18. 最终结论 -- 首次注册原子创建 Account、唯一 Workspace 和 owner。 -- 第二个 Workspace 无法创建。 -- 邀请用户可以注册并加入唯一 Workspace。 -- 未配置 SMTP 时一次性邀请链接仍可完成注册。 -- viewer、operator、developer、admin 和 owner 权限符合矩阵。 -- 最后一个 owner 不能被移除或降级。 -- OSS 不显示 Workspace switcher，但显示成员管理。 +Cloud v2 的产品模型只有一个逻辑 LangBot 实例和实例内多个 Workspace。 +当前选择单副本 MVP 是为了减少组件和新增租户成本，不是把单进程假设写进业务身份或协议。 +未来需要容量或高可用时，在同一逻辑实例内部增加 Core/Runtime 副本和 PostgreSQL shard， +Workspace 的 UUID、权限、数据边界和外部 API 均保持不变。 -### 22.3 SaaS 注册与目录 +开源 Core 必须完整实现安全的 Workspace 隔离和 OSS 单 Workspace 多用户；闭源 Control Plane +管理 SaaS 的全局目录、订阅、权益、计费和生命周期。共享可信控制面、连接池、只读 artifact 和数据库组件， +同时让每个不可信插件进程、sandbox、secret、可写文件和 tenant transaction 保持独占边界， +才能在不增加每租户部署的前提下最大化降低新增用户成本。 -- 普通注册只创建一个 personal Workspace。 -- 邀请注册创建 personal Workspace 并加入目标 Workspace。 -- 已有用户接受邀请不创建额外 personal Workspace。 -- 未 active Placement 的 provisioning 或 dormant Workspace 不能创建业务资源，首次使用完成 Placement 后才可进入。 -- 重试注册工作流不产生重复数据。 -- DirectoryEvent 重复、乱序和延迟时投影正确。 -- snapshot 与增量 cursor 能恢复断流，instance_sequence 缺口不会被错误标记 ready。 -- contiguous_applied_sequence 未达到 lease.min_required_sequence 时请求 fail closed。 -- SaaS pending Invitation、email 和 token hash 不进入 Data Plane。 -- 缺少 Workspace 的 Data Plane 请求返回 workspace_required。 - -### 22.4 资源隔离 - -即使 OSS UI 只允许一个 Workspace，CI 也必须使用测试 policy 创建至少两个 Workspace，覆盖每类资源的： - -- list。 -- get。 -- create。 -- update。 -- delete。 -- export。 -- 关联与复制。 - -还要覆盖 Pipeline 引用其他 Workspace 的 Model、Knowledge、MCP 或 Plugin 时被拒绝。 - -### 22.5 Runtime - -- 两个 Workspace 使用相同 launcher_id 不共享 Session。 -- 相同 MCP server name 不共享连接或 tool。 -- Plugin 猜测 Query ID 或 resource uuid 无法越权。 -- Plugin 不能伪造 workspace_uuid。 -- 同一 Plugin 在两个 Workspace 使用独立 process、connection、config 和 storage。 -- Box 同名 Session、Process 和 WebSocket attach 不串租户。 -- RAG collection 和对象路径不串租户。 -- Runtime lease 过期或 generation 变化后旧 owner 无法继续发送消息。 -- 普通 API、Plugin Host、job 和 runtime 携带旧 placement_generation 时都不能写入。 -- 并发写事务与 fence 使用 shared/exclusive lock，不会发生校验后 fence、旧事务再提交的竞态。 -- 旧 generation outbox event、Object Store staging write 和平台副作用在 dispatcher 再校验时被拒绝。 - -### 22.6 Billing 与控制面 - -- Payment webhook 重复投递只履约一次。 -- 订单成功不会直接绕过 Subscription 状态机。 -- Entitlement 签名、过期、回滚和 audience 校验。 -- Manifest root trust、delegated JWKS、续签、过期和 key revocation。 -- InteractiveDirectoryLease 到期后交互用户停止，WorkspaceStatusLease 到期后自动化工作负载停止。 -- UsageEvent 重复和乱序不会重复计费。 -- 迟到 UsageEvent 由中央 ledger 按 occurred_at 和不可变 Subscription/Price version 归入正确账期。 -- past_due、grace、suspended 支付恢复和 canceled_at_period_end 撤销会生成正确的新 Entitlement revision。 -- Control Plane 断开时按宽限策略运行。 -- 没有有效 Manifest 时 SaaS fail closed。 -- Placement 切换后旧 generation 请求被拒绝。 - -### 22.7 端到端 - -- OSS 初始化、邀请、注册、登录、成员管理和权限拒绝。 -- SaaS 注册、personal Workspace、创建 Workspace、切换、邀请和计费。 -- Workspace A 与 B 的 Bot、Pipeline、Plugin、MCP、Knowledge 和 Monitoring 完整隔离。 -- Cloud Router 到正确 Cell 和 CloudInstance。 -- 欠费、暂停、恢复和 entitlement 更新。 - -端到端验收必须使用真实浏览器操作，不以组件测试或接口调用替代用户路径。 - -## 23. 已确认决策与待定项 - -### 23.1 已确认 - -- Workspace 是 LangBot 实例级逻辑租户。 -- OSS 单 Workspace、多用户。 -- SaaS 多 Workspace。 -- SaaS 注册自动创建 personal Workspace。 -- Cloud 部署绿地重做，不兼容旧 Pod 架构。 -- Core 内置完整 Workspace 隔离。 -- 使用独立闭源 Control Plane 管理 SaaS 商业和 Cloud 能力。 -- Workspace 与 Deployment 分离。 -- Product entitlement 与 runtime capacity 分离。 - -### 23.2 推荐默认值 - -- OSS 初始化后默认仅邀请注册。 -- OSS 使用固定角色。 -- SaaS Membership 权威在 Control Plane，Core 使用本地版本化投影。 -- Subscription 以 Workspace 为 subject，BillingAccount 作为付款主体。 -- personal Workspace 使用免费 entitlement。 -- SaaS Plugin installation 按 Workspace 隔离进程。 -- entitlement grace 为 24 到 72 小时。 -- Cloud v2 MVP 每个 CloudInstance 一个 active runtime owner。 - -### 23.3 后续需要产品确认 - -- personal Workspace 是否计入可拥有 Workspace 上限。 -- personal Workspace 是否允许邀请成员。 -- 订阅按 Workspace 独立购买，还是企业 BillingAccount 统一承诺消费。 -- past_due 宽限期后是否停止已有 Bot，还是只禁止新建和扩容。 -- 各计划的成员、Bot、Token、Storage 和 Plugin 限额。 -- 企业私有化是否需要离线 Commercial Module。 -- 自定义角色、SSO、SCIM 和高级审计的套餐边界。 - -## 24. 最终结论 - -LangBot 多租户不能等同于把旧 Space Pod 变成共享 Pod，也不能只把 Workspace UI 放进闭源模块。 - -正确的边界是： - -- Core 开源并负责安全：单 Workspace 多用户、固定 RBAC、数据和运行时隔离。 -- Control Plane 闭源并负责 SaaS：全局 Account、多 Workspace 编排、Subscription、Billing、Entitlement、Placement 和 Cloud 运维。 -- Space 复用外围能力：OAuth、支付适配器、邮件、后台和可靠性经验。 -- Cloud v2 重新设计：一个 CloudInstance 承载多个 Workspace，shared 或 dedicated 只是 placement 策略。 - -这样既能让 OSS 成为完整、安全的多人协作产品，也能把真正的 SaaS 商业能力和全局控制能力留在独立闭源服务中。 +在闭源控制面、事务 fence/outbox、真实 Runtime hard isolation、Box hard quota 和 PostgreSQL 生产隔离等门禁完成之前， +本架构仍处于隔离内核阶段，不应被描述为可上线的 SaaS 多租户部署。 diff --git a/pyproject.toml b/pyproject.toml index c52f6a941..30ee73b2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,7 @@ dependencies = [ "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", "pyseekdb==1.1.0.post3", - "langbot-plugin @ git+https://github.com/langbot-app/langbot-plugin-sdk.git@044536f3720a2a8424d92f78934b9623da9f7f1d", + "langbot-plugin @ git+https://github.com/langbot-app/langbot-plugin-sdk.git@95a1805af2038c745de2c018a00db1305089a32e", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/src/langbot/pkg/api/http/controller/groups/extensions.py b/src/langbot/pkg/api/http/controller/groups/extensions.py index 690e6f0a1..040c04379 100644 --- a/src/langbot/pkg/api/http/controller/groups/extensions.py +++ b/src/langbot/pkg/api/http/controller/groups/extensions.py @@ -23,10 +23,20 @@ class ExtensionsRouterGroup(group.RouterGroup): async def _(request_context: RequestContext) -> quart.Response: if self.ap.plugin_connector.is_enable_plugin: await self.ap.plugin_connector.require_workspace_context(request_context) + + async def read_in_task_scope(operation): + tenant_scope = getattr(getattr(self.ap, 'persistence_mgr', None), 'tenant_scope', None) + if callable(tenant_scope): + async with tenant_scope(request_context.workspace_uuid): + return await operation() + return await operation() + plugins, mcp_servers, skills = await asyncio.gather( - self.ap.plugin_connector.list_plugins(), - self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True), - self.ap.skill_service.list_skills(request_context), + read_in_task_scope(self.ap.plugin_connector.list_plugins), + read_in_task_scope( + lambda: self.ap.mcp_service.get_mcp_servers(request_context, contain_runtime_info=True) + ), + read_in_task_scope(lambda: self.ap.skill_service.list_skills(request_context)), return_exceptions=True, ) diff --git a/src/langbot/pkg/api/http/controller/groups/knowledge/migration.py b/src/langbot/pkg/api/http/controller/groups/knowledge/migration.py index f93c759d6..85e949603 100644 --- a/src/langbot/pkg/api/http/controller/groups/knowledge/migration.py +++ b/src/langbot/pkg/api/http/controller/groups/knowledge/migration.py @@ -34,6 +34,54 @@ EXTERNAL_PLUGIN_CREATION_FIELDS: dict[str, set[str] | None] = { 'langbot-team/FastGPTConnector': None, # all fields -> creation_settings } +_INFORMATION_SCHEMA_TABLES = sqlalchemy.table( + 'tables', + sqlalchemy.column('table_schema'), + sqlalchemy.column('table_name'), + schema='information_schema', +) +_SQLITE_MASTER = sqlalchemy.table( + 'sqlite_master', + sqlalchemy.column('type'), + sqlalchemy.column('name'), +) +_LEGACY_KNOWLEDGE_BASE_BACKUP = sqlalchemy.table( + 'knowledge_bases_backup', + sqlalchemy.column('uuid'), + sqlalchemy.column('name'), + sqlalchemy.column('description'), + sqlalchemy.column('emoji'), + sqlalchemy.column('embedding_model_uuid'), + sqlalchemy.column('top_k'), + sqlalchemy.column('created_at'), + sqlalchemy.column('updated_at'), +) +_LEGACY_EXTERNAL_KNOWLEDGE_BASE = sqlalchemy.table( + 'external_knowledge_bases', + sqlalchemy.column('uuid'), + sqlalchemy.column('name'), + sqlalchemy.column('description'), + sqlalchemy.column('emoji'), + sqlalchemy.column('plugin_author'), + sqlalchemy.column('plugin_name'), + sqlalchemy.column('retriever_config'), + sqlalchemy.column('created_at'), +) +_CURRENT_KNOWLEDGE_BASE = sqlalchemy.table( + 'knowledge_bases', + sqlalchemy.column('uuid'), + sqlalchemy.column('workspace_uuid'), + sqlalchemy.column('name'), + sqlalchemy.column('description'), + sqlalchemy.column('emoji'), + sqlalchemy.column('created_at'), + sqlalchemy.column('updated_at'), + sqlalchemy.column('knowledge_engine_plugin_id'), + sqlalchemy.column('collection_id'), + sqlalchemy.column('creation_settings'), + sqlalchemy.column('retrieval_settings'), +) + @group.group_class('knowledge/migration', '/api/v1/knowledge/migration') class KnowledgeMigrationRouterGroup(group.RouterGroup): @@ -87,16 +135,18 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): """Check if a table exists.""" if self.ap.persistence_mgr.db.name == 'postgresql': result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text( - 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);' - ).bindparams(table_name=table_name) + sqlalchemy.select(_INFORMATION_SCHEMA_TABLES.c.table_name) + .where(_INFORMATION_SCHEMA_TABLES.c.table_schema == 'public') + .where(_INFORMATION_SCHEMA_TABLES.c.table_name == table_name) + .limit(1) ) - return result.scalar() + return result.first() is not None else: result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams( - table_name=table_name - ) + sqlalchemy.select(_SQLITE_MASTER.c.name) + .where(_SQLITE_MASTER.c.type == 'table') + .where(_SQLITE_MASTER.c.name == table_name) + .limit(1) ) return result.first() is not None @@ -151,7 +201,10 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): has_external = await self._table_exists('external_knowledge_bases') if has_external: result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text('SELECT DISTINCT plugin_author, plugin_name FROM external_knowledge_bases;') + sqlalchemy.select( + _LEGACY_EXTERNAL_KNOWLEDGE_BASE.c.plugin_author, + _LEGACY_EXTERNAL_KNOWLEDGE_BASE.c.plugin_name, + ).distinct() ) for row in result.fetchall(): plugin_author = row[0] or '' @@ -222,9 +275,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): # Step 3: Restore internal knowledge bases from backup task_context.trace('Restoring internal knowledge bases...', action='restore-internal') if await self._table_exists('knowledge_bases_backup'): - result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text('SELECT * FROM knowledge_bases_backup;') - ) + result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(_LEGACY_KNOWLEDGE_BASE_BACKUP)) rows = result.fetchall() columns = result.keys() @@ -239,17 +290,15 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): created_at = row_dict.get('created_at') updated_at = row_dict.get('updated_at') + # DB migration 20 created these columns as TEXT, while a fresh + # schema uses SQLAlchemy JSON. Keep the statement structured, + # but retain untyped bound values so both physical schemas and + # SQLite's string-valued legacy DATETIME rows remain valid. creation_settings = json.dumps({'embedding_model_uuid': embedding_model_uuid}) retrieval_settings = json.dumps({'top_k': top_k}) await self.ap.persistence_mgr.execute_async( - sqlalchemy.text( - 'INSERT INTO knowledge_bases ' - '(uuid, workspace_uuid, name, description, emoji, created_at, updated_at, ' - 'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) ' - 'VALUES (:uuid, :workspace_uuid, :name, :description, :emoji, :created_at, :updated_at, ' - ':plugin_id, :collection_id, :creation_settings, :retrieval_settings);' - ).bindparams( + sqlalchemy.insert(_CURRENT_KNOWLEDGE_BASE).values( uuid=kb_uuid, workspace_uuid=execution_context.workspace_uuid, name=name, @@ -257,7 +306,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): emoji=emoji, created_at=created_at, updated_at=updated_at, - plugin_id=LANGRAG_PLUGIN_ID, + knowledge_engine_plugin_id=LANGRAG_PLUGIN_ID, collection_id=kb_uuid, creation_settings=creation_settings, retrieval_settings=retrieval_settings, @@ -279,9 +328,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): # Step 4: Restore external knowledge bases task_context.trace('Restoring external knowledge bases...', action='restore-external') if has_external: - result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text('SELECT * FROM external_knowledge_bases;') - ) + result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(_LEGACY_EXTERNAL_KNOWLEDGE_BASE)) rows = result.fetchall() columns = result.keys() @@ -324,13 +371,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): retrieval_settings_dict = {k: v for k, v in retriever_config.items() if k not in creation_fields} await self.ap.persistence_mgr.execute_async( - sqlalchemy.text( - 'INSERT INTO knowledge_bases ' - '(uuid, workspace_uuid, name, description, emoji, created_at, updated_at, ' - 'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) ' - 'VALUES (:uuid, :workspace_uuid, :name, :description, :emoji, :created_at, :updated_at, ' - ':plugin_id, :collection_id, :creation_settings, :retrieval_settings);' - ).bindparams( + sqlalchemy.insert(_CURRENT_KNOWLEDGE_BASE).values( uuid=kb_uuid, workspace_uuid=execution_context.workspace_uuid, name=name, @@ -338,7 +379,7 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): emoji=emoji, created_at=created_at, updated_at=created_at, - plugin_id=external_plugin_id, + knowledge_engine_plugin_id=external_plugin_id, collection_id=kb_uuid, creation_settings=json.dumps(creation_settings_dict), retrieval_settings=json.dumps(retrieval_settings_dict), @@ -391,13 +432,13 @@ class KnowledgeMigrationRouterGroup(group.RouterGroup): if needed: if await self._table_exists('knowledge_bases_backup'): result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases_backup;') + sqlalchemy.select(sqlalchemy.func.count()).select_from(_LEGACY_KNOWLEDGE_BASE_BACKUP) ) internal_kb_count = result.scalar() or 0 if await self._table_exists('external_knowledge_bases'): result = await self.ap.persistence_mgr.execute_async( - sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;') + sqlalchemy.select(sqlalchemy.func.count()).select_from(_LEGACY_EXTERNAL_KNOWLEDGE_BASE) ) external_kb_count = result.scalar() or 0 diff --git a/src/langbot/pkg/api/http/controller/groups/monitoring.py b/src/langbot/pkg/api/http/controller/groups/monitoring.py index 60f79ad01..d3aa03c2e 100644 --- a/src/langbot/pkg/api/http/controller/groups/monitoring.py +++ b/src/langbot/pkg/api/http/controller/groups/monitoring.py @@ -26,7 +26,7 @@ def parse_iso_datetime(datetime_str: str | None) -> datetime.datetime | None: @group.group_class('monitoring', '/api/v1/monitoring') class MonitoringRouterGroup(group.RouterGroup): async def initialize(self) -> None: - @self.route('/overview', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/overview', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_overview(request_context: RequestContext) -> str: """Get overview metrics""" # Parse query parameters @@ -49,7 +49,7 @@ class MonitoringRouterGroup(group.RouterGroup): return self.success(data=metrics) - @self.route('/token-statistics', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_token_statistics(request_context: RequestContext) -> str: """Get detailed token usage statistics (summary, per-model, timeseries).""" bot_ids = quart.request.args.getlist('botId') @@ -74,7 +74,7 @@ class MonitoringRouterGroup(group.RouterGroup): return self.success(data=stats) - @self.route('/messages', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/messages', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_messages(request_context: RequestContext) -> str: """Get message logs""" # Parse query parameters @@ -110,7 +110,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/llm-calls', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/llm-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_llm_calls(request_context: RequestContext) -> str: """Get LLM call records""" # Parse query parameters @@ -144,7 +144,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/tool-calls', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/tool-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_tool_calls(request_context: RequestContext) -> str: """Get tool call records""" bot_ids = quart.request.args.getlist('botId') @@ -178,7 +178,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/embedding-calls', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/embedding-calls', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_embedding_calls(request_context: RequestContext) -> str: """Get embedding call records""" # Parse query parameters @@ -210,7 +210,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/sessions', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/sessions', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_sessions(request_context: RequestContext) -> str: """Get session information""" # Parse query parameters @@ -251,7 +251,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/errors', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/errors', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_errors(request_context: RequestContext) -> str: """Get error logs""" # Parse query parameters @@ -285,7 +285,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/data', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/data', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_all_data(request_context: RequestContext) -> str: """Get all monitoring data in a single request""" # Parse query parameters @@ -393,7 +393,7 @@ class MonitoringRouterGroup(group.RouterGroup): } ) - @self.route('/sessions//analysis', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/sessions//analysis', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_session_analysis(session_id: str, request_context: RequestContext) -> str: """Get detailed analysis for a specific session""" analysis = await self.ap.monitoring_service.get_session_analysis(request_context, session_id) @@ -402,7 +402,7 @@ class MonitoringRouterGroup(group.RouterGroup): # The frontend will handle the 'found: false' case return self.success(data=analysis) - @self.route('/messages//details', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/messages//details', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_message_details(message_id: str, request_context: RequestContext) -> str: """Get detailed information for a specific message""" details = await self.ap.monitoring_service.get_message_details(request_context, message_id) @@ -604,7 +604,7 @@ class MonitoringRouterGroup(group.RouterGroup): return response, 200 - @self.route('/feedback/stats', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/feedback/stats', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_feedback_stats(request_context: RequestContext) -> str: """Get feedback statistics""" # Parse query parameters @@ -627,7 +627,7 @@ class MonitoringRouterGroup(group.RouterGroup): return self.success(data=stats) - @self.route('/feedback', methods=['GET'], permission=Permission.AUDIT_VIEW) + @self.route('/feedback', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_feedback(request_context: RequestContext) -> str: """Get feedback list""" # Parse query parameters diff --git a/src/langbot/pkg/api/http/controller/groups/platform/bots.py b/src/langbot/pkg/api/http/controller/groups/platform/bots.py index 373088024..55867f189 100644 --- a/src/langbot/pkg/api/http/controller/groups/platform/bots.py +++ b/src/langbot/pkg/api/http/controller/groups/platform/bots.py @@ -72,7 +72,7 @@ class BotsRouterGroup(group.RouterGroup): '//logs', methods=['POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, - permission=Permission.AUDIT_VIEW, + permission=Permission.RESOURCE_VIEW, ) async def _(bot_uuid: str, request_context: RequestContext) -> str: json_data = await quart.request.json diff --git a/src/langbot/pkg/api/http/controller/groups/workspaces.py b/src/langbot/pkg/api/http/controller/groups/workspaces.py index 273cadf05..fa19b099d 100644 --- a/src/langbot/pkg/api/http/controller/groups/workspaces.py +++ b/src/langbot/pkg/api/http/controller/groups/workspaces.py @@ -87,22 +87,24 @@ class WorkspacesRouterGroup(group.RouterGroup): } ) - @self.route('', methods=['GET', 'POST'], permission=Permission.WORKSPACE_VIEW) - async def _(request_context: RequestContext) -> typing.Any: - if quart.request.method == 'POST': - if self.ap.workspace_service.policy.multi_workspace_enabled: - return self.http_status( - 409, - 'control_plane_required', - 'Cloud Workspaces are created by the SaaS control plane', - ) - return self.http_status(403, 'edition_limit', 'This edition supports one Workspace per instance') - - accesses = await self.ap.workspace_collaboration_service.list_account_workspaces( - request_context.account_uuid - ) + @self.route('', methods=['GET'], auth_type=group.AuthType.ACCOUNT_TOKEN) + async def _(user_email: str) -> typing.Any: + account = await self.ap.user_service.get_user_by_email(user_email) + if account is None: + return self.http_status(401, 'invalid_authentication', 'Account not found') + accesses = await self.ap.workspace_collaboration_service.list_account_workspaces(account.uuid) return self.success(data={'workspaces': [_workspace_payload(access.workspace) for access in accesses]}) + @self.route('', methods=['POST'], permission=Permission.WORKSPACE_VIEW) + async def _(request_context: RequestContext) -> typing.Any: + if self.ap.workspace_service.policy.multi_workspace_enabled: + return self.http_status( + 409, + 'control_plane_required', + 'Cloud Workspaces are created by the SaaS control plane', + ) + return self.http_status(403, 'edition_limit', 'This edition supports one Workspace per instance') + @self.route('/current', methods=['GET'], permission=Permission.WORKSPACE_VIEW) async def _(request_context: RequestContext) -> typing.Any: membership = quart.g.workspace_membership @@ -270,9 +272,14 @@ class InvitationsRouterGroup(group.RouterGroup): authorization = quart.request.headers.get('Authorization', '') if authorization.startswith('Bearer '): - account = await self.ap.user_service.get_authenticated_account(authorization.removeprefix('Bearer ')) - if isinstance(account, str): - account = await self.ap.user_service.get_user_by_email(account) + try: + account = await self.ap.user_service.get_authenticated_account( + authorization.removeprefix('Bearer ') + ) + if isinstance(account, str): + account = await self.ap.user_service.get_user_by_email(account) + except Exception as exc: + return self._auth_error_response(exc) if account is None: return self.http_status(401, 'invalid_authentication', 'Account not found') membership = await self.ap.workspace_collaboration_service.accept_invitation( diff --git a/src/langbot/pkg/persistence/mgr.py b/src/langbot/pkg/persistence/mgr.py index 126f413df..e075e9cc8 100644 --- a/src/langbot/pkg/persistence/mgr.py +++ b/src/langbot/pkg/persistence/mgr.py @@ -35,6 +35,7 @@ from .tenant_uow import ( PersistenceScopeBoundary, PersistenceScopeKind, TenantScopeRequiredError, + TenantScopedAsyncSession, TenantUnitOfWork, ) @@ -1700,9 +1701,9 @@ class PersistenceManager: calls retain normal ORM result semantics. """ - await session.flush() - connection = await session.connection() - return await connection.execute(*args, **kwargs) + if not isinstance(session, TenantScopedAsyncSession): + raise TypeError('Scoped Core execution requires a TenantScopedAsyncSession') + return await session.execute_on_transaction_connection(*args, **kwargs) def tenant_uow(self, workspace_uuid: str) -> TenantUnitOfWork: return self._scoped_uow(PersistenceScope.workspace(workspace_uuid)) diff --git a/src/langbot/pkg/persistence/tenant_uow.py b/src/langbot/pkg/persistence/tenant_uow.py index 0fae2c2d3..428186d8b 100644 --- a/src/langbot/pkg/persistence/tenant_uow.py +++ b/src/langbot/pkg/persistence/tenant_uow.py @@ -1,14 +1,21 @@ from __future__ import annotations import asyncio +import collections.abc +import contextlib import contextvars import dataclasses import enum +import functools import types import typing import sqlalchemy import sqlalchemy.ext.asyncio as sqlalchemy_asyncio +import sqlalchemy.orm as sqlalchemy_orm +from pgvector.sqlalchemy import Vector +from sqlalchemy.dialects.postgresql.dml import OnConflictDoUpdate as PostgreSQLOnConflictDoUpdate +from sqlalchemy.dialects.sqlite.dml import OnConflictDoUpdate as SQLiteOnConflictDoUpdate TENANT_SETTING = 'langbot.workspace_uuid' @@ -129,6 +136,10 @@ class TransactionRollbackOnlyError(RuntimeError): """Raised when a caught failure made the outer transaction unsafe to commit.""" +class ScopedSessionTransactionError(RuntimeError): + """Raised when callers try to escape a UoW-owned transaction boundary.""" + + @dataclasses.dataclass(slots=True) class ActiveScopedTransaction: scope: PersistenceScope @@ -148,6 +159,883 @@ class ActiveScopedTransaction: self.rollback_only_cause = cause +_UOW_SESSION_CONTROL_CAPABILITY = object() + + +@dataclasses.dataclass(slots=True) +class _ScopedSessionGuardState: + owner_task: asyncio.Task[typing.Any] | None + transaction_state: ActiveScopedTransaction | None = None + active: bool = True + session_events_blocked: bool = False + + +_SYNC_PROXY_CAPABILITY: contextvars.ContextVar[_ScopedSessionGuardState | None] = contextvars.ContextVar( + 'langbot_sync_session_proxy_capability', + default=None, +) + +_ALLOWED_SCOPED_BUILTIN_FUNCTION_TYPES = { + 'coalesce': sqlalchemy.sql.functions.coalesce, + 'count': sqlalchemy.sql.functions.count, + 'now': sqlalchemy.sql.functions.now, + 'sum': sqlalchemy.sql.functions.sum, +} +_ALLOWED_SCOPED_GENERIC_FUNCTIONS = frozenset({'length', 'nullif'}) +_ALLOWED_SCOPED_CUSTOM_OPERATORS = frozenset({'<=>'}) +_ALLOWED_SCOPED_STATEMENT_TYPES = ( + sqlalchemy.sql.dml.UpdateBase, + sqlalchemy.sql.selectable.SelectBase, +) +_ALLOWED_SCOPED_POST_VALUES_TYPES = ( + PostgreSQLOnConflictDoUpdate, + SQLiteOnConflictDoUpdate, +) + + +def _collect_embedded_clause_elements(value: typing.Any) -> list[sqlalchemy.sql.elements.ClauseElement]: + """Find executable expressions in SQLAlchemy containers omitted by visitors.""" + + if isinstance(value, sqlalchemy.sql.elements.ClauseElement): + return [value] + if isinstance(value, collections.abc.Mapping): + elements: list[sqlalchemy.sql.elements.ClauseElement] = [] + for key, item in value.items(): + elements.extend(_collect_embedded_clause_elements(key)) + elements.extend(_collect_embedded_clause_elements(item)) + return elements + if isinstance(value, (list, tuple)): + elements = [] + for item in value: + elements.extend(_collect_embedded_clause_elements(item)) + return elements + clause_factory = getattr(value, '__clause_element__', None) + if callable(clause_factory): + clause_element = clause_factory() + if isinstance(clause_element, sqlalchemy.sql.elements.ClauseElement): + return [clause_element] + return [] + + +def _validate_opaque_identifier( + value: typing.Any, + *, + label: str, +) -> list[sqlalchemy.sql.elements.ClauseElement]: + """Validate identifiers stored outside SQLAlchemy's normal AST traversal.""" + + if isinstance(value, sqlalchemy.sql.elements.ClauseElement): + return [value] + if not isinstance(value, str): + raise ScopedSessionTransactionError(f'TenantUnitOfWork does not allow an unknown {label}') + if isinstance(value, sqlalchemy.sql.elements.quoted_name) and value.quote is False: + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow forced-unquoted SQL identifiers') + if not value or value[0].isdigit() or any(not (character.isalnum() or character == '_') for character in value): + raise ScopedSessionTransactionError(f'TenantUnitOfWork does not allow an unsafe {label}') + return [] + + +def _validate_scoped_sql_type( + sql_type: typing.Any, + *, + seen: set[int] | None = None, +) -> None: + """Reject caller-defined SQL compilers hidden behind otherwise safe nodes.""" + + if not isinstance(sql_type, sqlalchemy.types.TypeEngine): + return + if seen is None: + seen = set() + identity = id(sql_type) + if identity in seen: + return + seen.add(identity) + + if type(sql_type) is Vector: + return + if not type(sql_type).__module__.startswith('sqlalchemy.'): + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow custom SQL types in public statements') + + for identifier_attribute in ('name', 'schema', 'collation'): + identifier = getattr(sql_type, identifier_attribute, None) + if isinstance(identifier, sqlalchemy.sql.elements.quoted_name) and identifier.quote is False: + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow forced-unquoted SQL type identifiers') + + nested_types: list[typing.Any] = [ + getattr(sql_type, 'item_type', None), + getattr(sql_type, 'impl', None), + ] + nested_types.extend(getattr(sql_type, 'types', ()) or ()) + for nested_type in nested_types: + if nested_type is not sql_type: + _validate_scoped_sql_type(nested_type, seen=seen) + + +def _collect_multi_value_elements( + multi_values: typing.Any, +) -> list[sqlalchemy.sql.elements.ClauseElement]: + """Validate batch INSERT rows which SQLAlchemy omits from AST visitors.""" + + children: list[sqlalchemy.sql.elements.ClauseElement] = [] + for value_group in multi_values or (): + if not isinstance(value_group, (list, tuple)): + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow an unknown batch INSERT shape') + for row in value_group: + if isinstance(row, collections.abc.Mapping): + for key, value in row.items(): + children.extend(_validate_opaque_identifier(key, label='batch INSERT column')) + children.extend(_collect_embedded_clause_elements(value)) + elif isinstance(row, (list, tuple)): + children.extend(_collect_embedded_clause_elements(row)) + else: + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow an unknown batch INSERT row') + return children + + +def _iter_scoped_statement_elements( + statement: sqlalchemy.sql.elements.ClauseElement, +) -> typing.Iterator[sqlalchemy.sql.elements.ClauseElement]: + """Walk the SQL AST, including dialect clauses SQLAlchemy omits from visitors. + + PostgreSQL and SQLite ``ON CONFLICT`` clauses currently expose no children + through ``get_children()``. Their conflict targets and update expressions + still form executable SQL, so this walker adds those containers explicitly + and rejects any future, unknown post-values clause by default. + """ + + pending: list[sqlalchemy.sql.elements.ClauseElement] = [statement] + seen: set[int] = set() + while pending: + element = pending.pop() + identity = id(element) + if identity in seen: + continue + seen.add(identity) + yield element + + children = [ + child for child in element.get_children() if isinstance(child, sqlalchemy.sql.elements.ClauseElement) + ] + post_values_clause = getattr(element, '_post_values_clause', None) + if post_values_clause is not None: + if not isinstance(post_values_clause, _ALLOWED_SCOPED_POST_VALUES_TYPES): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow an unknown dialect post-values SQL clause' + ) + children.append(post_values_clause) + + children.extend(_collect_multi_value_elements(getattr(element, '_multi_values', ()))) + + if isinstance(element, _ALLOWED_SCOPED_POST_VALUES_TYPES): + if getattr(element, 'constraint_target', None) is not None: + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow named ON CONFLICT constraints') + for target in getattr(element, 'inferred_target_elements', ()) or (): + children.extend(_validate_opaque_identifier(target, label='ON CONFLICT target')) + children.extend(_collect_embedded_clause_elements(getattr(element, 'inferred_target_whereclause', None))) + children.extend(_collect_embedded_clause_elements(getattr(element, 'update_whereclause', None))) + for update_pair in getattr(element, 'update_values_to_set', ()): + if not isinstance(update_pair, (list, tuple)) or len(update_pair) != 2: + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow an unknown ON CONFLICT update shape' + ) + key, value = update_pair + children.extend(_validate_opaque_identifier(key, label='ON CONFLICT update column')) + children.extend(_collect_embedded_clause_elements(value)) + + pending.extend(children) + + +def _validate_scoped_statement_call(args: tuple[typing.Any, ...], kwargs: dict[str, typing.Any]) -> None: + """Admit only structured SQLAlchemy statements with a small safe vocabulary. + + A textual denylist cannot be complete on PostgreSQL: ordinary built-in + functions can execute SQL supplied in string arguments, while statement + prefixes, suffixes, hints, and custom operators can inject syntax without + appearing as a top-level ``TextClause``. Tenant code therefore uses + SQLAlchemy's structured query/DML AST exclusively. Raw fragments and + unknown functions/operators fail closed before compilation. + """ + + statement = args[0] if args else kwargs.get('statement') + if statement is None: + return + if kwargs.get('execution_options'): + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow public SQL execution options') + if kwargs.get('bind_arguments'): + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow foreign database bind arguments') + if not isinstance(statement, _ALLOWED_SCOPED_STATEMENT_TYPES): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork public SQL must use a structured SQLAlchemy query, values, or DML statement' + ) + + for element in _iter_scoped_statement_elements(statement): + if not type(element).__module__.startswith('sqlalchemy.'): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow custom SQL AST nodes in public statements' + ) + + _validate_scoped_sql_type(getattr(element, 'type', None)) + + if isinstance(element, sqlalchemy.sql.selectable.Values): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow SQL VALUES constructs in public statements' + ) + + if isinstance(element, sqlalchemy.sql.dml.Insert) and getattr(element, 'select', None) is not None: + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow INSERT FROM SELECT in public statements' + ) + + for identifier_attribute in ('name', 'schema', 'collation'): + identifier = getattr(element, identifier_attribute, None) + if isinstance(identifier, sqlalchemy.sql.elements.quoted_name) and identifier.quote is False: + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow forced-unquoted SQL identifiers') + + if any( + getattr(element, attribute_name, None) + for attribute_name in ( + '_prefixes', + '_suffixes', + '_hints', + '_statement_hints', + '_execution_options', + '_with_options', + ) + ): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow textual statement modifiers or execution options' + ) + + if ( + isinstance(element, sqlalchemy.sql.elements.TextClause) + or getattr( + element, + '__visit_name__', + None, + ) + == 'textual_label_reference' + ): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow raw or textual SQL fragments in public statements' + ) + + if isinstance(element, sqlalchemy.sql.elements.ColumnClause) and element.is_literal and element.name != '*': + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow literal SQL columns in public statements' + ) + + if isinstance(element, sqlalchemy.sql.elements.Extract): + raise ScopedSessionTransactionError( + 'TenantUnitOfWork does not allow SQL EXTRACT fields in public statements' + ) + + if isinstance(element, sqlalchemy.sql.elements.BindParameter) and element.literal_execute: + raise ScopedSessionTransactionError('TenantUnitOfWork does not allow literal-execute SQL parameters') + + if isinstance(element, sqlalchemy.sql.elements.Cast) and type(element.type) is not Vector: + raise ScopedSessionTransactionError( + 'TenantUnitOfWork only allows the trusted pgvector cast used by tenant vector search' + ) + + if isinstance(element, sqlalchemy.sql.functions.FunctionElement): + function_name = str(getattr(element, 'name', '')).casefold() + package_names = tuple(getattr(element, 'packagenames', ())) + generic_function_allowed = ( + type(element) is sqlalchemy.sql.functions.Function + and function_name in _ALLOWED_SCOPED_GENERIC_FUNCTIONS + ) + builtin_function_allowed = _ALLOWED_SCOPED_BUILTIN_FUNCTION_TYPES.get(function_name) is type(element) + if package_names or not (generic_function_allowed or builtin_function_allowed): + raise ScopedSessionTransactionError( + f'TenantUnitOfWork does not allow SQL function {function_name or ""!r}' + ) + + for operator_attribute in ('operator', 'modifier'): + operator = getattr(element, operator_attribute, None) + if not isinstance(operator, sqlalchemy.sql.operators.custom_op): + continue + operator_name = str(operator.opstring) + if not ( + isinstance(element, sqlalchemy.sql.elements.BinaryExpression) + and operator_attribute == 'operator' + and operator_name in _ALLOWED_SCOPED_CUSTOM_OPERATORS + ): + raise ScopedSessionTransactionError( + f'TenantUnitOfWork does not allow custom SQL operator {operator_name!r}' + ) + + +class _NoopSessionEventCollection: + """Empty SQLAlchemy SessionEvents collection used after fail-closed detection.""" + + def __bool__(self) -> bool: + return False + + def __iter__(self) -> typing.Iterator[typing.Any]: + return iter(()) + + def __call__(self, *args: typing.Any, **kwargs: typing.Any) -> None: + del args, kwargs + + +class _NoopSessionEventsDispatch: + """Prevent a rejected SessionEvents listener from firing during rollback cleanup.""" + + _event_names: tuple[str, ...] = () + + def __getattr__(self, name: str) -> _NoopSessionEventCollection: + del name + return _NOOP_SESSION_EVENT_COLLECTION + + +_NOOP_SESSION_EVENT_COLLECTION = _NoopSessionEventCollection() +_NOOP_SESSION_EVENTS_DISPATCH = _NoopSessionEventsDispatch() + + +class TenantScopedSyncSession(sqlalchemy_orm.Session): + """Synchronous proxy usable only from its owning AsyncSession operation.""" + + def __init__( + self, + bind: sqlalchemy.engine.Engine | sqlalchemy.engine.Connection | None = None, + *, + binds: dict[typing.Any, typing.Any] | None = None, + langbot_guard_state: _ScopedSessionGuardState, + **kwargs: typing.Any, + ) -> None: + if binds: + raise ScopedSessionTransactionError('Tenant-scoped Sessions cannot use multiple database binds') + object.__setattr__(self, '_langbot_guard_state', langbot_guard_state) + object.__setattr__(self, '_langbot_expected_bind', bind) + object.__setattr__(self, '_langbot_initializing', True) + try: + super().__init__(bind=bind, binds=None, **kwargs) + finally: + object.__setattr__(self, '_langbot_initializing', False) + + def __getattribute__(self, name: str) -> typing.Any: + if name.startswith('_'): + return super().__getattribute__(name) + self._require_proxy_capability() + guard = object.__getattribute__(self, '_langbot_guard_state') + if name == 'dispatch' and guard.session_events_blocked: + return _NOOP_SESSION_EVENTS_DISPATCH + attribute = super().__getattribute__(name) + if name == 'dispatch' and not object.__getattribute__(self, '_langbot_initializing'): + event_name = next( + (candidate for candidate in attribute._event_names if bool(getattr(attribute, candidate))), + None, + ) + if event_name is not None: + guard.session_events_blocked = True + self._reject_proxy_escape(f'ORM Session event listener {event_name}') + if isinstance(attribute, types.MethodType): + return self._guard_method_call(attribute) + return attribute + + def __setattr__(self, name: str, value: typing.Any) -> None: + if name.startswith('_') or object.__getattribute__(self, '_langbot_initializing'): + super().__setattr__(name, value) + return + self._require_proxy_capability() + super().__setattr__(name, value) + + def __delattr__(self, name: str) -> None: + if name.startswith('_') or object.__getattribute__(self, '_langbot_initializing'): + super().__delattr__(name) + return + self._require_proxy_capability() + super().__delattr__(name) + + def get_bind( + self, + mapper: typing.Any = None, + *, + clause: typing.Any = None, + bind: typing.Any = None, + **kwargs: typing.Any, + ) -> sqlalchemy.engine.Engine | sqlalchemy.engine.Connection: + self._require_proxy_capability() + expected_bind = object.__getattribute__(self, '_langbot_expected_bind') + if bind is not None and bind is not expected_bind: + self._reject_proxy_escape('foreign database bind') + resolved = super().get_bind(mapper, clause=clause, bind=bind, **kwargs) + if resolved is not expected_bind: + self._reject_proxy_escape('foreign database bind') + return resolved + + def flush(self, objects: typing.Sequence[typing.Any] | None = None) -> None: + """Reject caller-supplied SQL expressions hidden in ORM state.""" + + self._require_proxy_capability() + for instance in self.new.union(self.dirty): + state = sqlalchemy.inspect(instance) + for column_property in state.mapper.column_attrs: + for value in state.attrs[column_property.key].history.added: + if isinstance(value, sqlalchemy.sql.elements.ClauseElement): + self._reject_proxy_escape('ORM SQL expression attribute value') + clause_factory = getattr(value, '__clause_element__', None) + if callable(clause_factory) and isinstance( + clause_factory(), + sqlalchemy.sql.elements.ClauseElement, + ): + self._reject_proxy_escape('ORM SQL expression attribute value') + super().flush(objects) + + def _guard_method_call(self, method: types.MethodType) -> typing.Callable[..., typing.Any]: + @functools.wraps(method) + def guarded(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._require_proxy_capability() + return method(*args, **kwargs) + + return guarded + + def _require_proxy_capability(self) -> None: + if object.__getattribute__(self, '_langbot_initializing'): + return + guard = object.__getattribute__(self, '_langbot_guard_state') + if not guard.active: + raise ScopedSessionTransactionError('TenantUnitOfWork scoped Session is no longer active') + try: + current_task = asyncio.current_task() + except RuntimeError: + current_task = None + if guard.owner_task is not current_task: + raise CrossScopeTransactionError( + 'Scoped database sessions cannot be inherited by child tasks; open an explicit task scope' + ) + if _SYNC_PROXY_CAPABILITY.get() is not guard: + self._reject_proxy_escape('synchronous Session access') + + def _reject_proxy_escape(self, operation: str) -> typing.NoReturn: + guard = object.__getattribute__(self, '_langbot_guard_state') + if guard.transaction_state is not None: + error = ScopedSessionTransactionError( + f'TenantUnitOfWork owns AsyncSession lifecycle; direct {operation} is not allowed' + ) + guard.transaction_state.mark_rollback_only(error) + raise error + raise ScopedSessionTransactionError( + f'TenantUnitOfWork owns AsyncSession lifecycle; direct {operation} is not allowed' + ) + + +class TenantScopedAsyncSession(sqlalchemy_asyncio.AsyncSession): + """An AsyncSession whose task and transaction lifecycle are UoW-owned. + + Normal ORM operations remain available to the task which opened the UoW. + Transaction-control and connection-escape APIs are deliberately withheld: + committing, rolling back, or closing the raw Session would otherwise let a + nested caller defeat the outer unit of work. A blanket public-attribute + owner check also covers a Session captured in the parent task and later + passed directly to a child task. + """ + + def __init__( + self, + bind: sqlalchemy_asyncio.AsyncEngine, + *, + owner_task: asyncio.Task[typing.Any] | None, + **kwargs: typing.Any, + ) -> None: + guard = _ScopedSessionGuardState(owner_task=owner_task) + object.__setattr__(self, '_langbot_guard_state', guard) + object.__setattr__(self, '_langbot_bind', None) + object.__setattr__(self, '_langbot_internal_access_depth', 0) + object.__setattr__(self, '_langbot_sync_capability_tokens', []) + object.__setattr__(self, '_langbot_initializing', True) + capability_token = _SYNC_PROXY_CAPABILITY.set(guard) + try: + super().__init__( + bind, + sync_session_class=TenantScopedSyncSession, + langbot_guard_state=guard, + **kwargs, + ) + finally: + _SYNC_PROXY_CAPABILITY.reset(capability_token) + object.__setattr__(self, '_langbot_initializing', False) + + def __getattribute__(self, name: str) -> typing.Any: + # ContextVars are copied into asyncio child tasks, and callers can also + # copy the Session object itself. Guard the whole supported public + # surface rather than trying to enumerate every current/future ORM I/O + # method. Private attributes remain an internal implementation detail. + if name.startswith('_'): + return super().__getattribute__(name) + + self._require_owner_task() + if ( + name in {'sync_session', 'binds', 'object_session'} + and object.__getattribute__(self, '_langbot_internal_access_depth') == 0 + ): + self._reject_transaction_escape(f'{name} access') + self._enter_internal_access() + try: + attribute = super().__getattribute__(name) + finally: + self._exit_internal_access() + if isinstance(attribute, types.MethodType): + return self._guard_method_call(attribute) + return attribute + + def __setattr__(self, name: str, value: typing.Any) -> None: + if name.startswith('_'): + super().__setattr__(name, value) + return + self._require_owner_task() + if not object.__getattribute__(self, '_langbot_initializing') and name in {'bind', 'binds', 'sync_session'}: + self._reject_transaction_escape(f'{name} replacement') + super().__setattr__(name, value) + + def __delattr__(self, name: str) -> None: + if name.startswith('_'): + super().__delattr__(name) + return + self._require_owner_task() + if name in {'bind', 'binds', 'sync_session'}: + self._reject_transaction_escape(f'{name} deletion') + super().__delattr__(name) + + @property + def bind(self) -> typing.NoReturn: + self._reject_transaction_escape('bind access') + + @bind.setter + def bind(self, value: sqlalchemy_asyncio.AsyncEngine) -> None: + existing = object.__getattribute__(self, '_langbot_bind') + if existing is not None: + self._reject_transaction_escape('bind replacement') + object.__setattr__(self, '_langbot_bind', value) + + @property + def no_autoflush(self) -> typing.ContextManager[None]: + """Preserve SQLAlchemy's helper without exposing the synchronous Session.""" + + @contextlib.contextmanager + def boundary() -> typing.Iterator[None]: + self._require_owner_task() + sync_session = object.__getattribute__(self, '_proxied') + previous = object.__getattribute__(sync_session, 'autoflush') + object.__setattr__(sync_session, 'autoflush', False) + try: + yield + finally: + self._require_owner_task() + object.__setattr__(sync_session, 'autoflush', previous) + + return boundary() + + def _bind_transaction_state(self, capability: object, state: ActiveScopedTransaction) -> None: + self._require_control_capability(capability, 'transaction-state binding') + self._require_owner_task() + object.__getattribute__(self, '_langbot_guard_state').transaction_state = state + + def begin(self) -> typing.NoReturn: + self._reject_transaction_escape('begin') + + def begin_nested(self) -> typing.NoReturn: + self._reject_transaction_escape('begin_nested') + + async def commit(self) -> typing.NoReturn: + self._reject_transaction_escape('commit') + + async def rollback(self) -> typing.NoReturn: + self._reject_transaction_escape('rollback') + + async def close(self) -> typing.NoReturn: + self._reject_transaction_escape('close') + + async def aclose(self) -> typing.NoReturn: + self._reject_transaction_escape('aclose') + + async def reset(self) -> typing.NoReturn: + self._reject_transaction_escape('reset') + + async def invalidate(self) -> typing.NoReturn: + self._reject_transaction_escape('invalidate') + + async def close_all(self) -> typing.NoReturn: + self._reject_transaction_escape('close_all') + + async def connection(self, *args: typing.Any, **kwargs: typing.Any) -> typing.NoReturn: + del args, kwargs + self._reject_transaction_escape('connection') + + def get_bind(self, *args: typing.Any, **kwargs: typing.Any) -> typing.NoReturn: + del args, kwargs + self._reject_transaction_escape('get_bind') + + def get_transaction(self) -> typing.NoReturn: + self._reject_transaction_escape('get_transaction') + + def get_nested_transaction(self) -> typing.NoReturn: + self._reject_transaction_escape('get_nested_transaction') + + async def run_sync(self, *args: typing.Any, **kwargs: typing.Any) -> typing.NoReturn: + del args, kwargs + self._reject_transaction_escape('run_sync') + + async def __aenter__(self) -> typing.NoReturn: + self._reject_transaction_escape('context-manager entry') + + async def __aexit__(self, *args: typing.Any, **kwargs: typing.Any) -> typing.NoReturn: + del args, kwargs + self._reject_transaction_escape('context-manager exit') + + async def _start_owned_transaction(self, capability: object) -> sqlalchemy_asyncio.AsyncSessionTransaction: + """Start the one root transaction; callable only by TenantUnitOfWork.""" + + self._require_control_capability(capability, 'owned transaction start') + self._require_owner_task() + self._enter_internal_access() + try: + transaction = super().begin() + await transaction + return transaction + finally: + self._exit_internal_access() + + async def _close_owned_session(self, capability: object) -> None: + """Close the Session after its root transaction has been finalized.""" + + self._require_control_capability(capability, 'owned Session close') + self._require_owner_task() + self._enter_internal_access() + try: + await super().close() + finally: + try: + self._exit_internal_access() + finally: + guard = object.__getattribute__(self, '_langbot_guard_state') + guard.active = False + guard.transaction_state = None + guard.owner_task = None + + async def _commit_owned_transaction( + self, + capability: object, + transaction: sqlalchemy_asyncio.AsyncSessionTransaction, + ) -> None: + self._require_control_capability(capability, 'owned transaction commit') + self._require_owner_task() + self._enter_internal_access() + try: + await transaction.commit() + finally: + self._exit_internal_access() + + async def _rollback_owned_transaction( + self, + capability: object, + transaction: sqlalchemy_asyncio.AsyncSessionTransaction, + ) -> None: + self._require_control_capability(capability, 'owned transaction rollback') + self._require_owner_task() + self._enter_internal_access() + try: + await transaction.rollback() + finally: + self._exit_internal_access() + + async def _apply_owned_scope_setting( + self, + capability: object, + setting_name: str, + setting_value: str, + ) -> None: + self._require_control_capability(capability, 'tenant scope configuration') + if setting_name not in { + TENANT_SETTING, + ACCOUNT_SETTING, + API_KEY_HASH_SETTING, + INVITATION_HASH_SETTING, + INSTANCE_SETTING, + IDENTITY_DIGEST_SETTING, + }: + self._reject_transaction_escape('unknown tenant scope configuration') + self._require_owner_task() + self._enter_internal_access() + try: + await super().execute( + sqlalchemy.text('SELECT set_config(:setting_name, :setting_value, true)'), + {'setting_name': setting_name, 'setting_value': setting_value}, + ) + finally: + self._exit_internal_access() + + async def execute_on_transaction_connection( + self, + *args: typing.Any, + **kwargs: typing.Any, + ) -> sqlalchemy.engine.cursor.CursorResult[typing.Any]: + """Execute Core SQL without exposing the transaction-bound Connection.""" + + self._require_owner_task() + self._validate_statement_or_reject(args, kwargs) + self._enter_internal_access() + try: + await super().flush() + connection = await super().connection() + return await connection.execute(*args, **kwargs) + finally: + self._exit_internal_access() + + async def execute(self, *args: typing.Any, **kwargs: typing.Any) -> sqlalchemy.engine.Result[typing.Any]: + self._validate_statement_or_reject(args, kwargs) + return await super().execute(*args, **kwargs) + + async def get(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._reject_nonempty_orm_options( + 'get', + kwargs, + {'options', 'with_for_update', 'identity_token', 'execution_options', 'bind_arguments'}, + ) + return await super().get(*args, **kwargs) + + async def get_one(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._reject_nonempty_orm_options( + 'get_one', + kwargs, + {'options', 'with_for_update', 'identity_token', 'execution_options', 'bind_arguments'}, + ) + return await super().get_one(*args, **kwargs) + + async def refresh(self, *args: typing.Any, **kwargs: typing.Any) -> None: + self._reject_nonempty_orm_options('refresh', kwargs, {'with_for_update'}) + await super().refresh(*args, **kwargs) + + async def merge(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._reject_nonempty_orm_options('merge', kwargs, {'options'}) + return await super().merge(*args, **kwargs) + + async def scalar(self, *args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._validate_statement_or_reject(args, kwargs) + return await super().scalar(*args, **kwargs) + + async def scalars(self, *args: typing.Any, **kwargs: typing.Any) -> sqlalchemy.engine.ScalarResult[typing.Any]: + self._validate_statement_or_reject(args, kwargs) + return await super().scalars(*args, **kwargs) + + async def stream(self, *args: typing.Any, **kwargs: typing.Any) -> sqlalchemy_asyncio.AsyncResult[typing.Any]: + del args, kwargs + self._reject_transaction_escape('stream; live database results cannot outlive the task-owned UoW operation') + + async def stream_scalars( + self, + *args: typing.Any, + **kwargs: typing.Any, + ) -> sqlalchemy_asyncio.AsyncScalarResult[typing.Any]: + del args, kwargs + self._reject_transaction_escape( + 'stream_scalars; live database results cannot outlive the task-owned UoW operation' + ) + + def _validate_statement_or_reject( + self, + args: tuple[typing.Any, ...], + kwargs: dict[str, typing.Any], + ) -> None: + try: + _validate_scoped_statement_call(args, kwargs) + except ScopedSessionTransactionError as exc: + guard = object.__getattribute__(self, '_langbot_guard_state') + if guard.transaction_state is not None: + guard.transaction_state.mark_rollback_only(exc) + raise + + def _reject_nonempty_orm_options( + self, + operation: str, + kwargs: dict[str, typing.Any], + option_names: set[str], + ) -> None: + for option_name in option_names: + value = kwargs.get(option_name) + if option_name == 'with_for_update': + if value is None or value is False: + continue + self._reject_transaction_escape(f'{operation} option {option_name}') + if option_name == 'identity_token': + if value is None: + continue + self._reject_transaction_escape(f'{operation} option {option_name}') + if value is None or value is False: + continue + if isinstance(value, collections.abc.Sized) and len(value) == 0: + continue + self._reject_transaction_escape(f'{operation} option {option_name}') + + def _guard_method_call(self, method: types.MethodType) -> typing.Callable[..., typing.Any]: + if asyncio.iscoroutinefunction(method): + + @functools.wraps(method) + async def guarded_async(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._require_owner_task() + self._enter_internal_access() + try: + return await method(*args, **kwargs) + finally: + self._exit_internal_access() + + return guarded_async + + @functools.wraps(method) + def guarded_sync(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + self._require_owner_task() + self._enter_internal_access() + try: + return method(*args, **kwargs) + finally: + self._exit_internal_access() + + return guarded_sync + + def _enter_internal_access(self) -> None: + depth = object.__getattribute__(self, '_langbot_internal_access_depth') + guard = object.__getattribute__(self, '_langbot_guard_state') + tokens = object.__getattribute__(self, '_langbot_sync_capability_tokens') + tokens.append(_SYNC_PROXY_CAPABILITY.set(guard)) + object.__setattr__(self, '_langbot_internal_access_depth', depth + 1) + + def _exit_internal_access(self) -> None: + depth = object.__getattribute__(self, '_langbot_internal_access_depth') + if depth <= 0: + raise RuntimeError('Scoped Session internal access stack underflow') + tokens = object.__getattribute__(self, '_langbot_sync_capability_tokens') + _SYNC_PROXY_CAPABILITY.reset(tokens.pop()) + object.__setattr__(self, '_langbot_internal_access_depth', depth - 1) + + def _require_control_capability(self, capability: object, operation: str) -> None: + if capability is not _UOW_SESSION_CONTROL_CAPABILITY: + self._reject_transaction_escape(operation) + + def _require_owner_task(self) -> None: + guard = object.__getattribute__(self, '_langbot_guard_state') + if not guard.active: + raise ScopedSessionTransactionError('TenantUnitOfWork scoped Session is no longer active') + owner_task = guard.owner_task + try: + current_task = asyncio.current_task() + except RuntimeError: + current_task = None + if owner_task is not current_task: + raise CrossScopeTransactionError( + 'Scoped database sessions cannot be inherited by child tasks; open an explicit task scope' + ) + + def _reject_transaction_escape(self, operation: str) -> typing.NoReturn: + self._require_owner_task() + error = ScopedSessionTransactionError( + f'TenantUnitOfWork owns AsyncSession transaction lifecycle; direct {operation} is not allowed' + ) + state = object.__getattribute__(self, '_langbot_guard_state').transaction_state + if state is not None: + state.mark_rollback_only(error) + raise error + + ActiveTransactionVar = contextvars.ContextVar[ActiveScopedTransaction | None] # ``AsyncSession`` delegates database I/O to a greenlet-backed synchronous @@ -312,6 +1200,7 @@ class TenantUnitOfWork: self._active_transaction = active_transaction self._active_scope = active_scope self._session: sqlalchemy_asyncio.AsyncSession | None = None + self._transaction: sqlalchemy_asyncio.AsyncSessionTransaction | None = None self._active_state: ActiveScopedTransaction | None = None self._context_token: contextvars.Token[ActiveScopedTransaction | None] | None = None self._database_operation_token: contextvars.Token[ActiveScopedTransaction | None] | None = None @@ -323,6 +1212,8 @@ class TenantUnitOfWork: def session(self) -> sqlalchemy_asyncio.AsyncSession: if self._session is None: raise RuntimeError('TenantUnitOfWork is not active') + if self._active_state is not None: + self._require_owner_task(self._active_state) return self._session async def __aenter__(self) -> TenantUnitOfWork: @@ -363,22 +1254,32 @@ class TenantUnitOfWork: # explicit UoW is allowed to replace the inherited pointer with an # independent transaction owned by the child task. - session = sqlalchemy_asyncio.AsyncSession(self._engine, expire_on_commit=False) + owner_task = asyncio.current_task() + session = TenantScopedAsyncSession( + self._engine, + owner_task=owner_task, + expire_on_commit=False, + close_resets_only=False, + ) self._session = session + transaction: sqlalchemy_asyncio.AsyncSessionTransaction | None = None try: - await session.begin() + transaction = await session._start_owned_transaction(_UOW_SESSION_CONTROL_CAPABILITY) + self._transaction = transaction if self._engine.dialect.name == 'postgresql': for setting_name, setting_value in self.scope.settings: - await session.execute( - sqlalchemy.text('SELECT set_config(:setting_name, :setting_value, true)'), - {'setting_name': setting_name, 'setting_value': setting_value}, + await session._apply_owned_scope_setting( + _UOW_SESSION_CONTROL_CAPABILITY, + setting_name, + setting_value, ) state = ActiveScopedTransaction( scope=self.scope, session=session, engine=self._engine, - owner_task=asyncio.current_task(), + owner_task=owner_task, ) + session._bind_transaction_state(_UOW_SESSION_CONTROL_CAPABILITY, state) self._active_state = state if self._active_transaction is not None: self._context_token = self._active_transaction.set(state) @@ -391,9 +1292,11 @@ class TenantUnitOfWork: if self._active_transaction is not None and self._context_token is not None: self._active_transaction.reset(self._context_token) self._context_token = None - await session.rollback() - await session.close() + if transaction is not None and transaction.sync_transaction is not None: + await session._rollback_owned_transaction(_UOW_SESSION_CONTROL_CAPABILITY, transaction) + await session._close_owned_session(_UOW_SESSION_CONTROL_CAPABILITY) self._session = None + self._transaction = None raise return self @@ -417,26 +1320,38 @@ class TenantUnitOfWork: return False session = self.session + transaction = self._transaction + if transaction is None: + raise RuntimeError('TenantUnitOfWork has no active root transaction') if exc_type is not None: state.mark_rollback_only(exc_value) rollback_only = state.rollback_only committed = False try: if exc_type is None and not rollback_only: - await session.commit() + await typing.cast(TenantScopedAsyncSession, session)._commit_owned_transaction( + _UOW_SESSION_CONTROL_CAPABILITY, + transaction, + ) committed = True else: - await session.rollback() + await typing.cast(TenantScopedAsyncSession, session)._rollback_owned_transaction( + _UOW_SESSION_CONTROL_CAPABILITY, + transaction, + ) finally: try: if self._active_transaction is not None and self._context_token is not None: self._active_transaction.reset(self._context_token) - await session.close() + await typing.cast(TenantScopedAsyncSession, session)._close_owned_session( + _UOW_SESSION_CONTROL_CAPABILITY + ) finally: if self._database_operation_token is not None: _DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token) - self._database_operation_token = None + self._database_operation_token = None self._session = None + self._transaction = None self._active_state = None state.depth = 0 self._complete_after_commit_waiters(state, committed=committed) diff --git a/tests/integration/api/test_bots.py b/tests/integration/api/test_bots.py index fe40b6291..bb8f423e9 100644 --- a/tests/integration/api/test_bots.py +++ b/tests/integration/api/test_bots.py @@ -231,7 +231,7 @@ class TestBotLogsEndpoint: assert 'total_count' in data['data'] @pytest.mark.asyncio - async def test_viewer_cannot_read_message_logs(self, quart_test_client, fake_bot_app): + async def test_viewer_can_read_ordinary_bot_logs(self, quart_test_client, fake_bot_app): access = fake_bot_app.workspace_collaboration_service.resolve_account_workspace.return_value original_role = access.membership.role access.membership.role = 'viewer' @@ -245,9 +245,9 @@ class TestBotLogsEndpoint: finally: access.membership.role = original_role - assert response.status_code == 403 - assert (await response.get_json())['code'] == 'permission_denied' - fake_bot_app.bot_service.list_event_logs.assert_not_awaited() + assert response.status_code == 200 + assert (await response.get_json())['code'] == 0 + fake_bot_app.bot_service.list_event_logs.assert_awaited_once() @pytest.mark.usefixtures('mock_circular_import_chain') diff --git a/tests/integration/api/test_monitoring.py b/tests/integration/api/test_monitoring.py index 9edc1e4ba..9a10ea61a 100644 --- a/tests/integration/api/test_monitoring.py +++ b/tests/integration/api/test_monitoring.py @@ -155,6 +155,34 @@ class TestMonitoringOverviewEndpoint: data = await response.get_json() assert data['code'] == 0 + @pytest.mark.asyncio + async def test_viewer_can_read_monitoring_but_cannot_export( + self, + quart_test_client, + fake_monitoring_app, + ): + """Ordinary monitoring is resource.view; export remains data.export.""" + membership = ( + fake_monitoring_app.workspace_collaboration_service.resolve_account_workspace.return_value.membership + ) + original_role = membership.role + membership.role = 'viewer' + try: + response = await quart_test_client.get( + '/api/v1/monitoring/overview', + headers={'Authorization': 'Bearer test_token'}, + ) + assert response.status_code == 200 + + export_response = await quart_test_client.get( + '/api/v1/monitoring/export?type=messages', + headers={'Authorization': 'Bearer test_token'}, + ) + assert export_response.status_code == 403 + assert (await export_response.get_json())['code'] == 'permission_denied' + finally: + membership.role = original_role + @pytest.mark.usefixtures('mock_circular_import_chain') class TestMonitoringMessagesEndpoint: diff --git a/tests/integration/api/test_workspaces.py b/tests/integration/api/test_workspaces.py index b8d0374e2..56127b0f7 100644 --- a/tests/integration/api/test_workspaces.py +++ b/tests/integration/api/test_workspaces.py @@ -233,6 +233,22 @@ async def test_owner_invites_second_account_and_secret_is_not_persisted(workspac assert (await forbidden_invite.get_json())['code'] == 'permission_denied' +async def test_invitation_accept_rejects_invalid_bearer_as_authentication_failure(workspace_api): + _, client, _, _ = workspace_api + + response = await client.post( + '/api/v1/invitations/accept', + headers={'Authorization': 'Bearer definitely-not-a-jwt'}, + json={'token': 'lbi_not-a-real-invitation'}, + ) + + assert response.status_code == 401 + assert await response.get_json() == { + 'code': 'invalid_authentication', + 'msg': 'Invalid authentication credentials', + } + + async def test_workspace_selector_and_path_cannot_escape_membership(workspace_api): _, client, _, owner_token = workspace_api @@ -412,6 +428,16 @@ async def test_cloud_projection_is_selected_explicitly_and_directory_writes_use_ assert by_uuid[singleton_uuid]['permissions'] assert by_uuid[cloud_workspace_uuid]['placement_generation'] == 12 + list_response = await client.get( + '/api/v1/workspaces', + headers=_auth(owner_token, singleton_uuid), + ) + assert list_response.status_code == 200 + assert {workspace['uuid'] for workspace in (await list_response.get_json())['data']['workspaces']} == { + singleton_uuid, + cloud_workspace_uuid, + } + current_response = await client.get( '/api/v1/workspaces/current', headers=_auth(owner_token, cloud_workspace_uuid), diff --git a/tests/integration/persistence/test_migrations_postgres.py b/tests/integration/persistence/test_migrations_postgres.py index b5755e58b..c45086ace 100644 --- a/tests/integration/persistence/test_migrations_postgres.py +++ b/tests/integration/persistence/test_migrations_postgres.py @@ -37,6 +37,7 @@ from langbot.pkg.persistence.mgr import PersistenceManager, PersistenceMode from langbot.pkg.persistence.tenant_uow import ( TENANT_POLICY_NAME, TENANT_TABLE_COLUMNS, + ScopedSessionTransactionError, TenantUnitOfWork, TransactionRollbackOnlyError, ) @@ -55,6 +56,7 @@ from langbot.pkg.workspace.policy import CloudWorkspacePolicy from langbot.pkg.workspace.service import WorkspaceService from langbot.pkg.api.http.authz import Permission from langbot.pkg.api.http.controller import group as http_group +from langbot.pkg.api.http.controller.groups.knowledge.migration import _CURRENT_KNOWLEDGE_BASE from langbot.pkg.api.http.controller.groups.system import SystemRouterGroup from langbot.pkg.api.http.controller.groups.webhooks import WebhookRouterGroup from langbot.pkg.api.http.context import ExecutionContext, RequestContext @@ -341,6 +343,57 @@ class TestPostgreSQLMigrationUpgrade: rev2 = await get_alembic_current(postgres_engine) assert rev2 == rev1, f'Expected {rev1}, got {rev2}' + @pytest.mark.asyncio + @pytest.mark.parametrize('settings_type', ['TEXT', 'JSON']) + async def test_legacy_knowledge_restore_statement_accepts_historical_and_fresh_settings_types( + self, + postgres_engine, + clean_tables, + clean_alembic_version, + settings_type, + ): + timestamp = datetime.datetime(2026, 7, 20, 3, 0, 0, 123456) + async with postgres_engine.begin() as conn: + await conn.execute( + text(f""" + CREATE TABLE knowledge_bases ( + uuid TEXT PRIMARY KEY, + workspace_uuid TEXT NOT NULL, + name TEXT, + description TEXT, + emoji TEXT, + created_at TIMESTAMP, + updated_at TIMESTAMP, + knowledge_engine_plugin_id TEXT, + collection_id TEXT, + creation_settings {settings_type}, + retrieval_settings {settings_type} + ) + """) + ) + await conn.execute( + _CURRENT_KNOWLEDGE_BASE.insert().values( + uuid=f'kb-{settings_type.lower()}', + workspace_uuid='workspace-a', + name='Legacy KB', + description='Compatibility probe', + emoji='📚', + created_at=timestamp, + updated_at=timestamp, + knowledge_engine_plugin_id='langbot-team/LangRAG', + collection_id=f'kb-{settings_type.lower()}', + creation_settings='{"embedding_model_uuid": "embedding-model"}', + retrieval_settings='{"top_k": 5}', + ) + ) + row = ( + await conn.execute( + text('SELECT created_at, creation_settings::text AS creation_settings FROM knowledge_bases') + ) + ).one() + assert row.created_at == timestamp + assert 'embedding_model_uuid' in row.creation_settings + class TestPostgreSQLMigrationGetCurrent: """Tests for get_alembic_current behavior on PostgreSQL.""" @@ -680,27 +733,23 @@ class TestPostgreSQLTenantRuntime: for workspace_uuid, slug in ((workspace_a, 'workspace-a'), (workspace_b, 'workspace-b')): async with TenantUnitOfWork(release_engine, workspace_uuid) as uow: await uow.execute( - text( - """ - INSERT INTO workspaces - (uuid, instance_uuid, name, slug, type, status, source, projection_revision) - VALUES - (:uuid, :instance_uuid, :name, :slug, 'team', 'active', 'cloud_projection', 0) - """ - ), - { - 'uuid': workspace_uuid, - 'instance_uuid': instance_uuid, - 'name': slug, - 'slug': slug, - }, + sa.insert(Workspace).values( + uuid=workspace_uuid, + instance_uuid=instance_uuid, + name=slug, + slug=slug, + type='team', + status='active', + source='cloud_projection', + projection_revision=0, + ) ) await uow.execute( - text( - 'INSERT INTO workspace_metadata (workspace_uuid, key, value) ' - "VALUES (:workspace_uuid, 'seed', :value)" - ), - {'workspace_uuid': workspace_uuid, 'value': slug}, + sa.insert(WorkspaceMetadata).values( + workspace_uuid=workspace_uuid, + key='seed', + value=slug, + ) ) await create_role(runtime_role) @@ -721,32 +770,39 @@ class TestPostgreSQLTenantRuntime: ) async with TenantUnitOfWork(runtime_engine, workspace_a) as uow: - assert (await uow.execute(text('SELECT uuid FROM workspaces'))).scalars().all() == [workspace_a] - assert ( - await uow.execute( - text('SELECT uuid FROM workspaces WHERE uuid = :workspace_uuid'), - {'workspace_uuid': workspace_b}, - ) - ).all() == [] + assert (await uow.execute(sa.select(Workspace.uuid))).scalars().all() == [workspace_a] + assert (await uow.execute(sa.select(Workspace.uuid).where(Workspace.uuid == workspace_b))).all() == [] with pytest.raises(sa.exc.DBAPIError): async with TenantUnitOfWork(runtime_engine, workspace_a) as uow: await uow.execute( - text( - 'INSERT INTO workspace_metadata (workspace_uuid, key, value) ' - "VALUES (:workspace_uuid, 'cross-scope', 'rejected')" - ), - {'workspace_uuid': workspace_b}, + sa.insert(WorkspaceMetadata).values( + workspace_uuid=workspace_b, + key='cross-scope', + value='rejected', + ) ) async with TenantUnitOfWork(runtime_engine, workspace_a) as uow: - assert (await uow.execute(text('SELECT value FROM workspace_metadata'))).scalars().all() == [ - 'workspace-a' - ] + assert (await uow.execute(sa.select(WorkspaceMetadata.value))).scalars().all() == ['workspace-a'] async with TenantUnitOfWork(runtime_engine, workspace_b) as uow: - assert (await uow.execute(text('SELECT value FROM workspace_metadata'))).scalars().all() == [ - 'workspace-b' - ] + assert (await uow.execute(sa.select(WorkspaceMetadata.value))).scalars().all() == ['workspace-b'] + + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with TenantUnitOfWork(runtime_engine, workspace_a) as uow: + with pytest.raises(ScopedSessionTransactionError, match='SQL function'): + await uow.execute( + sa.select( + sa.func.query_to_xml( + sa.literal("SELECT set_config('langbot.workspace_uuid', 'workspace-b', true)"), + sa.literal(True), + sa.literal(False), + sa.literal(''), + ) + ) + ) + assert (await uow.execute(sa.select(Workspace.uuid))).scalars().all() == [workspace_a] + async with runtime_engine.connect() as conn: setting = await conn.scalar(text("SELECT current_setting('langbot.workspace_uuid', true)")) assert setting in (None, '') @@ -755,16 +811,20 @@ class TestPostgreSQLTenantRuntime: with pytest.raises(RuntimeError, match='force rollback'): async with TenantUnitOfWork(runtime_engine, workspace_a) as uow: await uow.execute( - text( - 'INSERT INTO workspace_metadata (workspace_uuid, key, value) ' - "VALUES (:workspace_uuid, 'rolled-back', 'no')" - ), - {'workspace_uuid': workspace_a}, + sa.insert(WorkspaceMetadata).values( + workspace_uuid=workspace_a, + key='rolled-back', + value='no', + ) ) raise RuntimeError('force rollback') async with TenantUnitOfWork(runtime_engine, workspace_a) as uow: assert ( - await uow.session.scalar(text("SELECT COUNT(*) FROM workspace_metadata WHERE key = 'rolled-back'")) + await uow.session.scalar( + sa.select(sa.func.count()) + .select_from(WorkspaceMetadata) + .where(WorkspaceMetadata.key == 'rolled-back') + ) == 0 ) async with runtime_engine.connect() as conn: diff --git a/tests/integration/persistence/test_pgvector_postgres.py b/tests/integration/persistence/test_pgvector_postgres.py index a55dae40d..7764fb7e0 100644 --- a/tests/integration/persistence/test_pgvector_postgres.py +++ b/tests/integration/persistence/test_pgvector_postgres.py @@ -13,10 +13,12 @@ from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from langbot.pkg.entity.persistence.base import Base +from langbot.pkg.entity.persistence.rag import KnowledgeBase +from langbot.pkg.entity.persistence.workspace import Workspace from langbot.pkg.persistence.alembic_runner import get_alembic_current, run_alembic_stamp, run_alembic_upgrade from langbot.pkg.persistence.mgr import PersistenceManager, PersistenceMode from langbot.pkg.utils import constants -from langbot.pkg.vector.vdbs.pgvector_db import PgVectorDatabase, PgVectorScope +from langbot.pkg.vector.vdbs.pgvector_db import PgVectorDatabase, PgVectorEntry, PgVectorScope pytestmark = [pytest.mark.integration, pytest.mark.slow, pytest.mark.asyncio] @@ -393,30 +395,24 @@ async def test_pgvector_shared_database_is_scoped_indexed_and_ddl_free_at_runtim ): async with release_manager.tenant_uow(workspace_uuid) as uow: await uow.execute( - text( - """ - INSERT INTO workspaces - (uuid, instance_uuid, name, slug, type, status, source, projection_revision) - VALUES - (:uuid, :instance_uuid, :name, :slug, 'team', 'active', 'cloud_projection', 0) - """ - ), - { - 'uuid': workspace_uuid, - 'instance_uuid': instance_uuid, - 'name': slug, - 'slug': slug, - }, + sa.insert(Workspace).values( + uuid=workspace_uuid, + instance_uuid=instance_uuid, + name=slug, + slug=slug, + type='team', + status='active', + source='cloud_projection', + projection_revision=0, + ) ) await uow.execute( - text( - """ - INSERT INTO knowledge_bases - (uuid, workspace_uuid, name, embedding_dimension) - VALUES (:uuid, :workspace_uuid, :name, 384) - """ - ), - {'uuid': kb_uuid, 'workspace_uuid': workspace_uuid, 'name': slug}, + sa.insert(KnowledgeBase).values( + uuid=kb_uuid, + workspace_uuid=workspace_uuid, + name=slug, + embedding_dimension=384, + ) ) async with postgres_engine.connect() as conn: @@ -495,14 +491,26 @@ async def test_pgvector_shared_database_is_scoped_indexed_and_ddl_free_at_runtim async with runtime_manager.tenant_uow(workspace_a) as uow: rows = ( await uow.execute( - text("SELECT workspace_uuid, vector_id FROM langbot_vectors WHERE vector_id = 'same-vector'") + sa.select(PgVectorEntry.workspace_uuid, PgVectorEntry.vector_id).where( + PgVectorEntry.vector_id == 'same-vector' + ) ) ).all() assert rows == [(workspace_a, 'same-vector')] - await uow.execute(text('SET LOCAL enable_seqscan = off')) + + # Index-plan inspection is deployment diagnostics, not a public tenant + # Session capability. Establish the same transaction-local RLS scope on + # a test-only connection and keep raw EXPLAIN outside TenantUnitOfWork. + runtime_engine = runtime_manager.get_db_engine() + async with runtime_engine.begin() as conn: + await conn.execute( + text('SELECT set_config(:setting_name, :setting_value, true)'), + {'setting_name': 'langbot.workspace_uuid', 'setting_value': workspace_a}, + ) + await conn.execute(text('SET LOCAL enable_seqscan = off')) plan = '\n'.join( ( - await uow.execute( + await conn.execute( text( """ EXPLAIN SELECT vector_id @@ -524,7 +532,6 @@ async def test_pgvector_shared_database_is_scoped_indexed_and_ddl_free_at_runtim ) assert 'ix_langbot_vectors_hnsw_cosine_384' in plan - runtime_engine = runtime_manager.get_db_engine() async with runtime_engine.connect() as conn: assert await conn.scalar(text('SELECT COUNT(*) FROM langbot_vectors')) == 0 assert await conn.scalar(text("SELECT current_setting('langbot.workspace_uuid', true)")) in (None, '') diff --git a/tests/unit_tests/api/test_extensions_runtime_fence.py b/tests/unit_tests/api/test_extensions_runtime_fence.py index 0ca854ce3..12f36917e 100644 --- a/tests/unit_tests/api/test_extensions_runtime_fence.py +++ b/tests/unit_tests/api/test_extensions_runtime_fence.py @@ -5,8 +5,11 @@ from unittest.mock import AsyncMock import pytest import quart +import sqlalchemy +from sqlalchemy.ext.asyncio import create_async_engine from langbot.pkg.api.http.controller.groups.extensions import ExtensionsRouterGroup +from langbot.pkg.persistence.mgr import PersistenceManager from langbot.pkg.workspace.errors import WorkspaceNotFoundError @@ -92,3 +95,48 @@ async def test_extensions_route_redacts_plugin_secrets_without_mutating_runtime_ assert plugin['plugin_config']['nested']['token'] == '***' assert plugin['debug']['plugin_debug_key'] == '***' assert raw_plugin['plugin_config']['apiKey'] == 'plugin-secret' + + +@pytest.mark.asyncio +async def test_extensions_parallel_reads_open_explicit_child_task_scopes(): + engine = create_async_engine('sqlite+aiosqlite:///:memory:') + account = SimpleNamespace(uuid='account-a', user='owner@example.com') + ap = SimpleNamespace() + ap.persistence_mgr = PersistenceManager(ap) + ap.persistence_mgr.db = SimpleNamespace(get_engine=lambda: engine) + ap.user_service = SimpleNamespace(get_authenticated_account=AsyncMock(return_value=account)) + ap.workspace_collaboration_service = SimpleNamespace( + resolve_account_workspace=AsyncMock( + return_value=SimpleNamespace( + workspace=SimpleNamespace(uuid='workspace-a'), + membership=SimpleNamespace(uuid='membership-a', role='owner', projection_revision=0), + execution=SimpleNamespace(instance_uuid='instance-a', placement_generation=2), + ) + ) + ) + ap.plugin_connector = SimpleNamespace( + is_enable_plugin=False, + list_plugins=AsyncMock(return_value=[]), + ) + + async def list_mcp_servers(_context, *, contain_runtime_info): + await ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.literal(1))) + assert contain_runtime_info is True + return [{'name': 'Scoped MCP'}] + + ap.mcp_service = SimpleNamespace(get_mcp_servers=AsyncMock(side_effect=list_mcp_servers)) + ap.skill_service = SimpleNamespace(list_skills=AsyncMock(return_value=[])) + quart_app = quart.Quart(__name__) + router = ExtensionsRouterGroup(ap, quart_app) + await router.initialize() + + try: + response = await quart_app.test_client().get( + '/api/v1/extensions', + headers={'Authorization': 'Bearer token', 'X-Workspace-Id': 'workspace-a'}, + ) + finally: + await engine.dispose() + + assert response.status_code == 200 + assert (await response.get_json())['data']['extensions'] == [{'type': 'mcp', 'server': {'name': 'Scoped MCP'}}] diff --git a/tests/unit_tests/api/test_knowledge_migration_runtime_fence.py b/tests/unit_tests/api/test_knowledge_migration_runtime_fence.py index 5750b7374..17bc67001 100644 --- a/tests/unit_tests/api/test_knowledge_migration_runtime_fence.py +++ b/tests/unit_tests/api/test_knowledge_migration_runtime_fence.py @@ -1,12 +1,16 @@ from __future__ import annotations +from datetime import datetime +import json from types import SimpleNamespace from unittest.mock import AsyncMock, Mock import pytest +import sqlalchemy from langbot.pkg.api.http.context import ExecutionContext from langbot.pkg.api.http.controller.groups.knowledge.migration import KnowledgeMigrationRouterGroup +from langbot.pkg.persistence.tenant_uow import _validate_scoped_statement_call from langbot.pkg.workspace.errors import WorkspaceInvariantError, WorkspaceNotFoundError @@ -67,3 +71,223 @@ async def test_cloud_migration_is_rejected_before_legacy_table_access(): router._table_exists.assert_not_awaited() router.ap.plugin_connector.require_workspace_context.assert_not_awaited() router._set_migration_flag.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('database_name, exists', [('postgresql', True), ('sqlite', False)]) +async def test_legacy_table_discovery_uses_scoped_structured_queries(database_name: str, exists: bool): + result = Mock() + result.first.return_value = ('knowledge_bases_backup',) if exists else None + execute_async = AsyncMock(return_value=result) + router = object.__new__(KnowledgeMigrationRouterGroup) + router.ap = SimpleNamespace( + persistence_mgr=SimpleNamespace( + db=SimpleNamespace(name=database_name), + execute_async=execute_async, + ) + ) + + assert await router._table_exists('knowledge_bases_backup') is exists + + statement = execute_async.await_args.args[0] + assert isinstance(statement, sqlalchemy.sql.selectable.SelectBase) + _validate_scoped_statement_call((statement,), {}) + + +@pytest.mark.asyncio +async def test_legacy_restore_emits_only_scoped_structured_statements(): + missing_table_result = Mock() + missing_table_result.first.return_value = None + existing_table_result = Mock() + existing_table_result.first.return_value = ('knowledge_bases_backup',) + backup_result = Mock() + backup_result.keys.return_value = [ + 'uuid', + 'name', + 'description', + 'emoji', + 'embedding_model_uuid', + 'top_k', + 'created_at', + 'updated_at', + ] + now = datetime.now() + backup_result.fetchall.return_value = [ + ('kb-legacy', 'Legacy KB', 'Description', 'U0001f4da', 'embedding-model', 7, now, now) + ] + execute_async = AsyncMock( + side_effect=[ + missing_table_result, + existing_table_result, + backup_result, + Mock(), + Mock(), + ] + ) + connector = SimpleNamespace( + require_workspace_context=AsyncMock(return_value=CONTEXT), + list_knowledge_engines=AsyncMock(return_value=[]), + rag_on_kb_create=AsyncMock(), + ) + router = object.__new__(KnowledgeMigrationRouterGroup) + router.ap = SimpleNamespace( + workspace_service=SimpleNamespace( + get_local_execution_binding=AsyncMock(return_value=CONTEXT), + ), + plugin_connector=connector, + persistence_mgr=SimpleNamespace( + db=SimpleNamespace(name='sqlite'), + execute_async=execute_async, + ), + rag_mgr=SimpleNamespace(load_knowledge_bases_from_db=AsyncMock()), + logger=Mock(), + ) + task_context = SimpleNamespace(trace=Mock()) + + await router._execute_rag_migration(CONTEXT, task_context, install_plugin=False) + + statements = [call.args[0] for call in execute_async.await_args_list] + assert any(isinstance(statement, sqlalchemy.sql.dml.Insert) for statement in statements) + assert any(isinstance(statement, sqlalchemy.sql.dml.Update) for statement in statements) + for statement in statements: + assert not isinstance(statement, sqlalchemy.sql.elements.TextClause) + _validate_scoped_statement_call((statement,), {}) + connector.rag_on_kb_create.assert_awaited_once_with( + 'langbot-team/LangRAG', + 'kb-legacy', + {'embedding_model_uuid': 'embedding-model'}, + ) + + +@pytest.mark.asyncio +async def test_legacy_restore_accepts_sqlite_string_dates_and_text_json_columns(): + engine = sqlalchemy.ext.asyncio.create_async_engine('sqlite+aiosqlite:///:memory:') + try: + async with engine.begin() as connection: + await connection.exec_driver_sql( + """ + CREATE TABLE knowledge_bases_backup ( + uuid TEXT PRIMARY KEY, + name TEXT, + description TEXT, + emoji TEXT, + embedding_model_uuid TEXT, + top_k INTEGER, + created_at DATETIME, + updated_at DATETIME + ) + """ + ) + await connection.exec_driver_sql( + """ + CREATE TABLE external_knowledge_bases ( + uuid TEXT PRIMARY KEY, + name TEXT, + description TEXT, + emoji TEXT, + plugin_author TEXT, + plugin_name TEXT, + retriever_config TEXT, + created_at DATETIME + ) + """ + ) + await connection.exec_driver_sql( + """ + CREATE TABLE knowledge_bases ( + uuid TEXT PRIMARY KEY, + workspace_uuid TEXT NOT NULL, + name TEXT, + description TEXT, + emoji TEXT, + created_at DATETIME, + updated_at DATETIME, + knowledge_engine_plugin_id TEXT, + collection_id TEXT, + creation_settings TEXT, + retrieval_settings TEXT + ) + """ + ) + await connection.exec_driver_sql( + 'CREATE TABLE workspace_metadata (workspace_uuid TEXT, key TEXT, value TEXT)' + ) + legacy_timestamp = '2026-07-20 03:00:00.123456' + await connection.exec_driver_sql( + """ + INSERT INTO knowledge_bases_backup + (uuid, name, description, emoji, embedding_model_uuid, top_k, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 'kb-internal', + 'Internal', + 'Internal legacy KB', + 'U0001f4da', + 'embedding-model', + 5, + legacy_timestamp, + legacy_timestamp, + ), + ) + await connection.exec_driver_sql( + """ + INSERT INTO external_knowledge_bases + (uuid, name, description, emoji, plugin_author, plugin_name, retriever_config, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + 'kb-external', + 'External', + 'External legacy KB', + 'U0001f517', + 'langbot-team', + 'DifyDatasetsRetriever', + json.dumps({'api_base_url': 'https://example.invalid', 'top_k': 8}), + legacy_timestamp, + ), + ) + + connector = SimpleNamespace( + require_workspace_context=AsyncMock(return_value=CONTEXT), + list_knowledge_engines=AsyncMock(return_value=[]), + rag_on_kb_create=AsyncMock(), + ) + router = object.__new__(KnowledgeMigrationRouterGroup) + router.ap = SimpleNamespace( + workspace_service=SimpleNamespace( + get_local_execution_binding=AsyncMock(return_value=CONTEXT), + ), + plugin_connector=connector, + persistence_mgr=SimpleNamespace( + db=SimpleNamespace(name='sqlite'), + execute_async=connection.execute, + ), + rag_mgr=SimpleNamespace(load_knowledge_bases_from_db=AsyncMock()), + logger=Mock(), + ) + + await router._execute_rag_migration( + CONTEXT, + SimpleNamespace(trace=Mock()), + install_plugin=False, + ) + + restored = ( + await connection.exec_driver_sql( + """ + SELECT uuid, created_at, updated_at, creation_settings, retrieval_settings + FROM knowledge_bases + ORDER BY uuid + """ + ) + ).all() + assert [row.uuid for row in restored] == ['kb-external', 'kb-internal'] + assert all(row.created_at == legacy_timestamp for row in restored) + assert all(row.updated_at == legacy_timestamp for row in restored) + assert json.loads(restored[0].creation_settings)['api_base_url'] == 'https://example.invalid' + assert json.loads(restored[0].retrieval_settings) == {'top_k': 8} + assert json.loads(restored[1].creation_settings) == {'embedding_model_uuid': 'embedding-model'} + assert json.loads(restored[1].retrieval_settings) == {'top_k': 5} + finally: + await engine.dispose() diff --git a/tests/unit_tests/persistence/test_tenant_uow.py b/tests/unit_tests/persistence/test_tenant_uow.py index 9c7ce11ee..a4dd8c76a 100644 --- a/tests/unit_tests/persistence/test_tenant_uow.py +++ b/tests/unit_tests/persistence/test_tenant_uow.py @@ -6,7 +6,13 @@ from types import SimpleNamespace import pytest import sqlalchemy as sa +from pgvector.sqlalchemy import Vector +from sqlalchemy.dialects.postgresql import insert as postgresql_insert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.ext.asyncio import async_object_session +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, registry, relationship, with_loader_criteria +from sqlalchemy.sql import quoted_name from langbot.pkg.core.task_boundary import create_detached_task from langbot.pkg.entity.persistence.user import User @@ -14,15 +20,60 @@ from langbot.pkg.persistence.mgr import PersistenceManager, PersistenceMode from langbot.pkg.persistence.tenant_uow import ( CrossScopeTransactionError, PersistenceScopeKind, + ScopedSessionTransactionError, + TenantScopedSyncSession, TenantScopeRequiredError, TenantUnitOfWork, TransactionRollbackOnlyError, + _validate_scoped_statement_call, ) pytestmark = pytest.mark.asyncio +def _on_conflict_statement(*, update_value, update_key='value', index_element=None): + table = sa.table('conflict_rows', sa.column('id'), sa.column('value')) + if index_element is None: + index_element = table.c.id + return ( + sqlite_insert(table) + .values(id=1, value=1) + .on_conflict_do_update( + index_elements=[index_element], + set_={update_key: update_value}, + ) + ) + + +def _on_conflict_constraint_statement(*, constraint): + table = sa.table('conflict_rows', sa.column('id'), sa.column('value')) + return ( + postgresql_insert(table) + .values(id=1, value=1) + .on_conflict_do_update( + constraint=constraint, + set_={'value': 2}, + ) + ) + + +def _multi_value_statement(*, value, value_key='value'): + table = sa.table('multi_value_rows', sa.column('id'), sa.column('value')) + return sa.insert(table).values([{'id': 1, value_key: value}]) + + +class _SpoofedCount(sa.sql.functions.FunctionElement): + name = 'count' + inherit_cache = True + + +class _UntrustedCastType(sa.types.UserDefinedType): + def get_col_spec(self, **kwargs) -> str: + del kwargs + return 'INTEGER' + + async def test_sqlite_tenant_uow_commits_and_rolls_back() -> None: engine = create_async_engine('sqlite+aiosqlite:///:memory:') table = sa.Table( @@ -319,6 +370,632 @@ async def test_child_task_must_open_its_own_explicit_uow(tmp_path) -> None: await engine.dispose() +async def test_captured_session_rejects_child_task_database_access(tmp_path) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "captured-session-child.db"}') + table = sa.Table('captured_session_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + mapper_registry = registry() + + class CapturedSessionRow: + pass + + mapper_registry.map_imperatively(CapturedSessionRow, table) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + async with manager.tenant_uow('workspace-a') as uow: + captured_session = uow.session + captured_execute = captured_session.execute + captured_add = captured_session.add + await captured_session.execute(sa.insert(table).values(id=1)) + + async def inherited_session_write() -> None: + await captured_execute(sa.insert(table).values(id=2)) + + with pytest.raises(CrossScopeTransactionError, match='cannot be inherited by child tasks'): + await asyncio.create_task(inherited_session_write()) + + async def inherited_session_mutation() -> None: + captured_add(CapturedSessionRow(id=2)) + + with pytest.raises(CrossScopeTransactionError, match='cannot be inherited by child tasks'): + await asyncio.create_task(inherited_session_mutation()) + + # The rejected child never touched the connection and therefore + # must not poison valid work still owned by the parent task. + await captured_session.execute(sa.insert(table).values(id=3)) + + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table.c.id).order_by(table.c.id))).scalars().all() == [1, 3] + finally: + await engine.dispose() + + +async def test_captured_session_is_permanently_inactive_after_uow_exit(tmp_path) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "captured-session-exit.db"}') + table = sa.Table('captured_session_exit_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + async with manager.tenant_uow('workspace-a') as uow: + captured_session = uow.session + captured_execute = captured_session.execute + await captured_execute(sa.insert(table).values(id=1)) + + with pytest.raises(ScopedSessionTransactionError, match='no longer active'): + await captured_session.execute(sa.select(table)) + with pytest.raises(ScopedSessionTransactionError, match='no longer active'): + await captured_execute(sa.select(table)) + with pytest.raises(ScopedSessionTransactionError, match='no longer active'): + captured_session.in_transaction() + + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table.c.id))).scalars().all() == [1] + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + 'operation', + [ + 'begin', + 'commit', + 'rollback', + 'close', + 'close_all', + 'connection', + 'get_bind', + 'bind', + 'sync_session', + 'stream', + 'stream_scalars', + ], +) +async def test_scoped_session_cannot_escape_uow_transaction_lifecycle(tmp_path, operation: str) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / f"session-escape-{operation}.db"}') + table = sa.Table('session_escape_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with manager.tenant_uow('workspace-a') as uow: + await uow.execute(sa.insert(table).values(id=1)) + with pytest.raises(ScopedSessionTransactionError, match=f'direct {operation}'): + if operation == 'begin': + uow.session.begin() + elif operation == 'commit': + await uow.session.commit() + elif operation == 'rollback': + await uow.session.rollback() + elif operation == 'close': + await uow.session.close() + elif operation == 'close_all': + await uow.session.close_all() + elif operation == 'connection': + await uow.session.connection() + elif operation == 'get_bind': + uow.session.get_bind().connect() + elif operation == 'bind': + _ = uow.session.bind + elif operation == 'sync_session': + _ = uow.session.sync_session + elif operation == 'stream': + await uow.session.stream(sa.select(table)) + else: + await uow.session.stream_scalars(sa.select(table.c.id)) + + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table))).all() == [] + finally: + await engine.dispose() + + +async def test_scoped_session_no_autoflush_keeps_the_sync_proxy_private(tmp_path) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "no-autoflush.db"}') + table = sa.Table('no_autoflush_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + mapper_registry = registry() + + class NoAutoflushRow: + pass + + mapper_registry.map_imperatively(NoAutoflushRow, table) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + async with manager.tenant_uow('workspace-a') as uow: + assert uow.session.autoflush is True + with uow.session.no_autoflush: + assert uow.session.autoflush is False + uow.session.add(NoAutoflushRow(id=1)) + assert uow.session.autoflush is True + + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table.c.id))).scalars().all() == [1] + finally: + await engine.dispose() + + +@pytest.mark.parametrize('access_kind', ['async_instance', 'global_sync']) +async def test_orm_object_cannot_expose_the_uow_sync_session(tmp_path, access_kind: str) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / f"object-session-{access_kind}.db"}') + table = sa.Table('object_session_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + mapper_registry = registry() + + class ObjectSessionRow: + pass + + mapper_registry.map_imperatively(ObjectSessionRow, table) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with manager.tenant_uow('workspace-a') as uow: + row = ObjectSessionRow(id=1) + uow.session.add(row) + await uow.session.flush() + assert async_object_session(row) is uow.session + gate = manager.create_after_commit_gate() + assert gate is not None + + if access_kind == 'async_instance': + with pytest.raises(ScopedSessionTransactionError, match='object_session access'): + uow.session.object_session(row) + else: + sync_session = sa.orm.object_session(row) + assert sync_session is not None + with pytest.raises(ScopedSessionTransactionError, match='synchronous Session access'): + sync_session.rollback() + + assert gate.cancelled() + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table))).all() == [] + finally: + await engine.dispose() + + +@pytest.mark.parametrize('event_name', ['do_orm_execute', 'before_flush']) +async def test_orm_session_events_fail_closed_before_callbacks_run(tmp_path, event_name: str) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / f"orm-event-{event_name}.db"}') + table = sa.Table('orm_event_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + mapper_registry = registry() + + class EventRow: + pass + + mapper_registry.map_imperatively(EventRow, table) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + callback_called = False + listener_registered = False + + def do_orm_execute_escape(state) -> None: + nonlocal callback_called + callback_called = True + state.session.connection().exec_driver_sql('COMMIT') + + def before_flush_escape(session, flush_context, instances) -> None: + nonlocal callback_called + del flush_context, instances + callback_called = True + session.bind.connect() + + listener = do_orm_execute_escape if event_name == 'do_orm_execute' else before_flush_escape + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with manager.tenant_uow('workspace-a') as uow: + await uow.execute(sa.insert(table).values(id=1)) + sa.event.listen(TenantScopedSyncSession, event_name, listener) + listener_registered = True + with pytest.raises(ScopedSessionTransactionError, match=f'event listener {event_name}'): + if event_name == 'do_orm_execute': + await uow.session.execute(sa.select(table)) + else: + uow.session.add(EventRow(id=2)) + await uow.session.flush() + + assert not callback_called + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table))).all() == [] + finally: + if listener_registered: + sa.event.remove(TenantScopedSyncSession, event_name, listener) + await engine.dispose() + + +async def test_pre_registered_orm_session_event_prevents_uow_start() -> None: + engine = create_async_engine('sqlite+aiosqlite:///:memory:') + callback_called = False + + def after_transaction_create_escape(session, transaction) -> None: + nonlocal callback_called + del session, transaction + callback_called = True + + sa.event.listen(TenantScopedSyncSession, 'after_transaction_create', after_transaction_create_escape) + try: + with pytest.raises(ScopedSessionTransactionError, match='event listener after_transaction_create'): + async with TenantUnitOfWork(engine, 'workspace-a'): + pass + assert not callback_called + finally: + sa.event.remove(TenantScopedSyncSession, 'after_transaction_create', after_transaction_create_escape) + await engine.dispose() + + +async def test_explicit_async_refresh_loads_relationship_without_exposing_sync_session(tmp_path) -> None: + class Base(DeclarativeBase): + pass + + class Parent(Base): + __tablename__ = 'async_attr_parents' + + id: Mapped[int] = mapped_column(primary_key=True) + children: Mapped[list[Child]] = relationship(back_populates='parent') + + class Child(Base): + __tablename__ = 'async_attr_children' + + id: Mapped[int] = mapped_column(primary_key=True) + parent_id: Mapped[int] = mapped_column(sa.ForeignKey('async_attr_parents.id')) + parent: Mapped[Parent] = relationship(back_populates='children') + + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "async-attrs.db"}') + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + async with manager.tenant_uow('workspace-a') as uow: + uow.session.add(Parent(id=1, children=[Child(id=1)])) + + async with manager.tenant_uow('workspace-a') as uow: + parent = await uow.session.get(Parent, 1) + assert parent is not None + assert 'children' not in parent.__dict__ + await uow.session.refresh(parent, ['children']) + assert [child.id for child in parent.children] == [1] + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + ('statement', 'params', 'keyword_call'), + [ + (sa.text('ROLLBACK'), None, False), + (sa.text('/* caller comment */ COMMIT'), None, False), + (sa.text('SET LOCAL langbot.workspace_uuid = :value'), {'value': 'workspace-b'}, False), + (sa.text("SET LOCAL langbot . workspace_uuid = 'workspace-b'"), None, False), + (sa.text("SET/**/ LOCAL /**/ langbot . workspace_uuid = 'workspace-b'"), None, False), + ( + sa.text('SELECT set_config(:setting_name, :setting_value, true)'), + {'setting_name': 'langbot.workspace_uuid', 'setting_value': 'workspace-b'}, + False, + ), + ( + sa.text('SELECT set_config/**/(:setting_name, :setting_value, true)'), + {'setting_name': 'langbot.workspace_uuid', 'setting_value': 'workspace-b'}, + False, + ), + ( + sa.text('SELECT "set_config"(:setting_name, :setting_value, true)'), + {'setting_name': 'langbot.workspace_uuid', 'setting_value': 'workspace-b'}, + False, + ), + (sa.text("DO $$ BEGIN PERFORM set_config('langbot.workspace_uuid', 'workspace-b', true); END $$"), None, False), + (sa.text('CALL tenant_scope_escape()'), None, False), + (sa.text('CREATE FUNCTION tenant_scope_escape() RETURNS void LANGUAGE SQL AS $$ SELECT 1 $$'), None, False), + (sa.text('SELECT * INTO TEMP leaked_rows FROM sql_escape_rows'), None, False), + (sa.text('SELECT * INTO pg_temp.leaked_rows FROM sql_escape_rows'), None, False), + (sa.text('DECLARE leaked_rows CURSOR WITH HOLD FOR SELECT * FROM sql_escape_rows'), None, False), + (sa.text('FETCH ALL FROM leaked_rows'), None, False), + (sa.text('PREPARE scope_escape AS SELECT 1'), None, False), + (sa.text('EXECUTE scope_escape'), None, False), + (sa.text('EXPLAIN (ANALYZE true) EXECUTE scope_escape'), None, False), + (sa.text('LISTEN tenant_scope_escape'), None, False), + (sa.text("NOTIFY tenant_scope_escape, 'payload'"), None, False), + (sa.text('LOCK TABLE sql_escape_rows'), None, False), + (sa.text('SELECT pg_advisory_lock(123)'), None, False), + (sa.text('VALUES (pg_try_advisory_lock(123))'), None, False), + (sa.text('EXPLAIN (ANALYZE TRUE) SELECT pg_advisory_lock(123)'), None, False), + (sa.text("SELECT pg_notify('tenant_scope_escape', 'payload')"), None, False), + (sa.text("SELECT lo_from_bytea(0, decode('AA==', 'base64'))"), None, False), + (sa.text('SELECT lo_get(123)'), None, False), + (sa.text("SELECT currval('shared_sequence')"), None, False), + (sa.text('SELECT lastval()'), None, False), + ( + sa.select( + sa.func.query_to_xml( + sa.literal('SELECT 1'), + sa.literal(True), + sa.literal(False), + sa.literal(''), + ) + ), + None, + False, + ), + (sa.select(sa.func.ts_stat(sa.literal('SELECT 1'))), None, False), + (sa.select(sa.func.pg_catalog.count()), None, False), + (sa.select(_SpoofedCount()), None, False), + (sa.select(sa.literal_column('1')), None, False), + (sa.select(sa.text('1')), None, False), + (sa.select(sa.bindparam('value', 1, literal_execute=True)), None, False), + (sa.select(sa.bindparam('value', 1, type_=_UntrustedCastType())), None, False), + (sa.select(sa.column('value').op('@@')(sa.literal('query'))), None, False), + ( + sa.select( + sa.sql.expression.UnaryExpression( + sa.literal(True), + operator=sa.sql.operators.custom_op('unsafe_prefix'), + ) + ), + None, + False, + ), + ( + sa.select( + sa.sql.expression.UnaryExpression( + sa.literal(True), + modifier=sa.sql.operators.custom_op('unsafe_suffix'), + ) + ), + None, + False, + ), + (sa.select(sa.extract('year', sa.literal('2026-01-01'))), None, False), + (sa.select(sa.cast(sa.literal(1), _UntrustedCastType())), None, False), + ( + sa.select(User).options(with_loader_criteria(User, sa.literal_column('1 = 1'))), + None, + False, + ), + (sa.select(sa.table(quoted_name('forced unquoted table', quote=False))), None, False), + (sa.select(sa.literal(1).label(quoted_name('forced unquoted label', quote=False))), None, False), + ( + sa.select(sa.collate(sa.column('value'), quoted_name('forced unquoted collation', quote=False))), + None, + False, + ), + (sa.select(sa.literal(1)).prefix_with('/* caller prefix */'), None, False), + (sa.select(sa.literal(1)).suffix_with('FOR UPDATE'), None, False), + (sa.select(sa.literal(1)).with_statement_hint('caller hint'), None, False), + ( + _on_conflict_statement( + update_value=sa.func.query_to_xml( + sa.literal('SELECT 1'), + sa.literal(True), + sa.literal(False), + sa.literal(''), + ) + ), + None, + False, + ), + (_on_conflict_statement(update_value=sa.text('set_config(:name, :value, true)')), None, False), + ( + _on_conflict_statement( + update_value=1, + update_key=quoted_name('forced unquoted update', quote=False), + ), + None, + False, + ), + ( + _on_conflict_statement( + update_value=1, + index_element=quoted_name('forced unquoted target', quote=False), + ), + None, + False, + ), + ( + _on_conflict_constraint_statement(constraint=quoted_name('forced unquoted constraint', quote=False)), + None, + False, + ), + (_multi_value_statement(value=sa.func.ts_stat(sa.literal('SELECT 1'))), None, False), + (_multi_value_statement(value=sa.text('set_config(:name, :value, true)')), None, False), + ( + _multi_value_statement( + value=1, + value_key=quoted_name('forced unquoted batch key', quote=False), + ), + None, + False, + ), + (sa.values(sa.column('value')).data([(sa.func.ts_stat(sa.literal('SELECT 1')),)]), None, False), + ( + sa.insert(sa.table('rows', sa.column('value'))).from_select( + ['value'], + sa.select(sa.literal(1)), + ), + None, + False, + ), + (sa.text('ROLLBACK'), None, True), + ( + sa.text('SELECT set_config(:setting_name, :setting_value, true)'), + {'setting_name': 'langbot.workspace_uuid', 'setting_value': 'workspace-b'}, + True, + ), + ], +) +async def test_scoped_session_rejects_raw_or_unapproved_sql( + tmp_path, + statement, + params, + keyword_call: bool, +) -> None: + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "sql-transaction-escape.db"}') + table = sa.Table('sql_escape_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: engine) + try: + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + with pytest.raises(TransactionRollbackOnlyError, match='after-commit work was cancelled'): + async with manager.tenant_uow('workspace-a') as uow: + await uow.execute(sa.insert(table).values(id=1)) + gate = manager.create_after_commit_gate() + assert gate is not None + with pytest.raises(ScopedSessionTransactionError): + if keyword_call: + await uow.session.execute(statement=statement, params=params) + elif params is None: + await uow.session.execute(statement) + else: + await uow.session.execute(statement, params) + + assert gate.cancelled() + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table))).all() == [] + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + 'statement', + [ + sa.select(sa.literal('set_config(')), + sa.select(sa.func.count()), + sa.select(sa.func.coalesce(sa.func.sum(sa.literal(1)), sa.literal(0))), + sa.select( + sa.func.now(), + sa.func.length(sa.literal('value')), + sa.func.nullif(sa.literal('value'), sa.literal('')), + ), + sa.select(sa.column('embedding').op('<=>')(sa.literal([0.1]))), + sa.select(sa.cast(sa.column('embedding'), Vector(384))), + sa.insert(sa.table('rows', sa.column('id'))).values(id=1), + _multi_value_statement(value=1), + _on_conflict_statement(update_value=sa.func.coalesce(sa.literal(1), sa.literal(0))), + ], +) +async def test_scoped_sql_structure_allows_only_the_production_vocabulary(statement) -> None: + _validate_scoped_statement_call((statement,), {}) + + +async def test_scoped_sql_rejects_public_execution_options() -> None: + statement = sa.select(sa.literal(1)) + with pytest.raises(ScopedSessionTransactionError, match='execution options'): + _validate_scoped_statement_call( + (statement,), + {'execution_options': {'schema_translate_map': {None: 'other_schema'}}}, + ) + + +@pytest.mark.parametrize('operation', ['get', 'get_one', 'refresh', 'merge']) +async def test_scoped_orm_loaders_reject_public_query_options(operation: str) -> None: + engine = create_async_engine('sqlite+aiosqlite:///:memory:') + try: + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with TenantUnitOfWork(engine, 'workspace-a') as uow: + with pytest.raises(ScopedSessionTransactionError, match=operation): + if operation == 'get': + await uow.session.get( + User, + 1, + execution_options={'schema_translate_map': {None: 'other_schema'}}, + ) + elif operation == 'get_one': + await uow.session.get_one(User, 1, options=[object()]) + elif operation == 'refresh': + await uow.session.refresh(object(), with_for_update=True) + else: + await uow.session.merge(User(), options=[object()]) + finally: + await engine.dispose() + + +async def test_scoped_get_rejects_empty_for_update_mapping() -> None: + engine = create_async_engine('sqlite+aiosqlite:///:memory:') + try: + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with TenantUnitOfWork(engine, 'workspace-a') as uow: + with pytest.raises(ScopedSessionTransactionError, match='get option with_for_update'): + await uow.session.get(User, 1, with_for_update={}) + finally: + await engine.dispose() + + +@pytest.mark.parametrize('flush_kind', ['explicit', 'autoflush']) +async def test_scoped_orm_writes_reject_attribute_sql_expressions(tmp_path, flush_kind: str) -> None: + class Base(DeclarativeBase): + pass + + class Row(Base): + __tablename__ = f'orm_expression_{flush_kind}' + + id: Mapped[int] = mapped_column(primary_key=True) + value: Mapped[int] = mapped_column() + + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / f"orm-expression-{flush_kind}.db"}') + try: + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with TenantUnitOfWork(engine, 'workspace-a') as uow: + uow.session.add(Row(id=1, value=sa.literal_column('40 + 2'))) + with pytest.raises(ScopedSessionTransactionError, match='ORM SQL expression'): + if flush_kind == 'explicit': + await uow.session.flush() + else: + await uow.session.execute(sa.select(Row.id)) + + async with engine.connect() as conn: + assert (await conn.execute(sa.select(Row))).all() == [] + finally: + await engine.dispose() + + +async def test_scoped_session_rejects_an_explicit_foreign_bind(tmp_path) -> None: + primary = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "primary-bind.db"}') + foreign = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "foreign-bind.db"}') + table = sa.Table('bind_escape_rows', sa.MetaData(), sa.Column('id', sa.Integer, primary_key=True)) + manager = PersistenceManager(object(), mode=PersistenceMode.CLOUD_RUNTIME) + manager.db = SimpleNamespace(get_engine=lambda: primary) + try: + for engine in (primary, foreign): + async with engine.begin() as conn: + await conn.run_sync(table.metadata.create_all) + + with pytest.raises(TransactionRollbackOnlyError, match='transaction was rolled back'): + async with manager.tenant_uow('workspace-a') as uow: + await uow.execute(sa.insert(table).values(id=1)) + with pytest.raises(ScopedSessionTransactionError, match='foreign database bind'): + await uow.session.execute( + sa.insert(table).values(id=2), + bind_arguments={'bind': foreign.sync_engine}, + ) + + for engine in (primary, foreign): + async with engine.connect() as conn: + assert (await conn.execute(sa.select(table))).all() == [] + finally: + await primary.dispose() + await foreign.dispose() + + async def test_detached_task_starts_without_parent_scope_and_rolls_back_its_uow(tmp_path) -> None: engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "detached-task.db"}') table = sa.Table( diff --git a/uv.lock b/uv.lock index f8a132c46..f98708414 100644 --- a/uv.lock +++ b/uv.lock @@ -2115,7 +2115,7 @@ requires-dist = [ { name = "ebooklib", specifier = ">=0.18" }, { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, - { name = "langbot-plugin", git = "https://github.com/langbot-app/langbot-plugin-sdk.git?rev=044536f3720a2a8424d92f78934b9623da9f7f1d" }, + { name = "langbot-plugin", git = "https://github.com/langbot-app/langbot-plugin-sdk.git?rev=95a1805af2038c745de2c018a00db1305089a32e" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2181,7 +2181,7 @@ dev = [ [[package]] name = "langbot-plugin" version = "0.4.15" -source = { git = "https://github.com/langbot-app/langbot-plugin-sdk.git?rev=044536f3720a2a8424d92f78934b9623da9f7f1d#044536f3720a2a8424d92f78934b9623da9f7f1d" } +source = { git = "https://github.com/langbot-app/langbot-plugin-sdk.git?rev=95a1805af2038c745de2c018a00db1305089a32e#95a1805af2038c745de2c018a00db1305089a32e" } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, diff --git a/web/src/app/home/bots/BotDetailContent.tsx b/web/src/app/home/bots/BotDetailContent.tsx index 02ede4081..52556515d 100644 --- a/web/src/app/home/bots/BotDetailContent.tsx +++ b/web/src/app/home/bots/BotDetailContent.tsx @@ -38,8 +38,8 @@ export default function BotDetailContent({ id }: { id: string }) { const currentWorkspace = useCurrentWorkspace(); const canManage = currentWorkspace?.permissions.includes('resource.manage') ?? false; - const canViewAudit = - currentWorkspace?.permissions.includes('audit.view') ?? false; + const canViewMonitoring = + currentWorkspace?.permissions.includes('resource.view') ?? false; const { refreshBots, bots, setDetailEntityName } = useSidebarData(); // Set breadcrumb entity name @@ -210,13 +210,13 @@ export default function BotDetailContent({ id }: { id: string }) { {t('bots.configuration')} - {canViewAudit && ( + {canViewMonitoring && ( {t('bots.logs')} )} - {canViewAudit && ( + {canViewMonitoring && ( {t('bots.sessionMonitor.title')} @@ -303,7 +303,7 @@ export default function BotDetailContent({ id }: { id: string }) { {/* Tab: Logs */} - {canViewAudit && ( + {canViewMonitoring && ( diff --git a/web/src/app/home/monitoring/page.tsx b/web/src/app/home/monitoring/page.tsx index 1ae5dd945..0d05a4c1a 100644 --- a/web/src/app/home/monitoring/page.tsx +++ b/web/src/app/home/monitoring/page.tsx @@ -23,9 +23,13 @@ import { FeedbackStatsCards } from './components/FeedbackCard'; import { FeedbackList } from './components/FeedbackList'; import { buildConversationTurns } from './utils/conversationTurns'; import { LoadingSpinner, LoadingPage } from '@/components/ui/loading-spinner'; +import { useCurrentWorkspace } from '@/app/infra/http'; function MonitoringPageContent() { const { t } = useTranslation(); + const currentWorkspace = useCurrentWorkspace(); + const canExport = + currentWorkspace?.permissions.includes('data.export') ?? false; const { filterState, setSelectedBots, setSelectedPipelines, setTimeRange } = useMonitoringFilters(); const { data, loading, refetch } = useMonitoringData(filterState); @@ -154,7 +158,7 @@ function MonitoringPageContent() { onTimeRangeChange={setTimeRange} />
- + {canExport && }