mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-22 00:15:20 +08:00
Merge remote-tracking branch 'origin/dev/4.11.x' into dev/4.11.x
# Conflicts: # uv.lock # web/src/i18n/locales/en-US.ts # web/src/i18n/locales/ja-JP.ts # web/src/i18n/locales/zh-Hans.ts # web/src/i18n/locales/zh-Hant.ts
This commit is contained in:
fdc310
@@ -0,0 +1,48 @@
|
||||
"""Add isolated server-only Codex credentials and tenant RLS.
|
||||
|
||||
Revision ID: 0022_codex_credentials
|
||||
Revises: 0021_merge_reasoning_config
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = '0022_codex_credentials'
|
||||
down_revision = '0021_merge_reasoning_config'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
# Fresh startup creates ORM metadata before running Alembic.
|
||||
if 'codex_credentials' not in sa.inspect(conn).get_table_names():
|
||||
op.create_table(
|
||||
'codex_credentials',
|
||||
sa.Column('provider_uuid', sa.String(255), primary_key=True),
|
||||
sa.Column('workspace_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('payload', sa.JSON(), nullable=False),
|
||||
sa.Column('version', sa.Integer(), nullable=False),
|
||||
sa.Column('lease_owner', sa.String(64), nullable=True),
|
||||
sa.Column('lease_until', sa.Float(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'provider_uuid'],
|
||||
['model_providers.workspace_uuid', 'model_providers.uuid'],
|
||||
name='fk_codex_credentials_workspace_provider',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
)
|
||||
op.create_index('ix_codex_credentials_workspace', 'codex_credentials', ['workspace_uuid'])
|
||||
if conn.dialect.name == 'postgresql':
|
||||
op.execute('ALTER TABLE codex_credentials ENABLE ROW LEVEL SECURITY')
|
||||
op.execute('ALTER TABLE codex_credentials FORCE ROW LEVEL SECURITY')
|
||||
op.execute('DROP POLICY IF EXISTS langbot_workspace_isolation ON codex_credentials')
|
||||
expression = "workspace_uuid::text = NULLIF(current_setting('langbot.workspace_uuid', true), '')"
|
||||
op.execute(
|
||||
f'CREATE POLICY langbot_workspace_isolation ON codex_credentials '
|
||||
f'FOR ALL USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table('codex_credentials')
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Scope monitoring sessions by bot without changing runtime session IDs.
|
||||
|
||||
Revision ID: 0023_bot_scoped_sessions
|
||||
Revises: 0022_codex_credentials
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql, sqlite
|
||||
|
||||
revision = '0023_bot_scoped_sessions'
|
||||
down_revision = '0022_codex_credentials'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = 'monitoring_sessions'
|
||||
_KEY = ['workspace_uuid', 'bot_id', 'session_id']
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
inspector = sa.inspect(conn)
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return
|
||||
pk = inspector.get_pk_constraint(_TABLE)
|
||||
if pk['constrained_columns'] == _KEY:
|
||||
return
|
||||
# PostgreSQL alters in place, retaining indexes, grants, policies and RLS.
|
||||
# SQLite batch reflection retains all existing indexes and foreign keys.
|
||||
with op.batch_alter_table(_TABLE, naming_convention={'pk': 'pk_%(table_name)s'}) as batch:
|
||||
batch.drop_constraint(pk['name'] or f'pk_{_TABLE}', type_='primary')
|
||||
batch.create_primary_key(f'pk_{_TABLE}', _KEY)
|
||||
|
||||
metadata = sa.MetaData()
|
||||
sessions = sa.Table(_TABLE, metadata, autoload_with=conn)
|
||||
messages = sa.Table('monitoring_messages', metadata, autoload_with=conn)
|
||||
m = messages.c
|
||||
collisions = (
|
||||
sa.select(m.workspace_uuid, m.session_id)
|
||||
.group_by(m.workspace_uuid, m.session_id)
|
||||
.having(sa.func.count(sa.distinct(m.bot_id)) > 1)
|
||||
.subquery()
|
||||
)
|
||||
partition = [m.workspace_uuid, m.bot_id, m.session_id]
|
||||
# Repair only demonstrable collisions. Retention may have removed earlier
|
||||
# evidence; these summaries describe surviving messages, never invented text.
|
||||
ranked = (
|
||||
sa.select(
|
||||
*[m[name] for name in _KEY],
|
||||
m.bot_name,
|
||||
m.pipeline_id,
|
||||
m.pipeline_name,
|
||||
m.platform,
|
||||
m.user_id,
|
||||
m.user_name,
|
||||
sa.func.sum(sa.case((sa.or_(m.role == 'user', m.role.is_(None)), 1), else_=0))
|
||||
.over(partition_by=partition)
|
||||
.label('message_count'),
|
||||
sa.func.min(m.timestamp).over(partition_by=partition).label('start_time'),
|
||||
sa.func.max(m.timestamp).over(partition_by=partition).label('last_activity'),
|
||||
sa.func.row_number().over(partition_by=partition, order_by=[m.timestamp.desc(), m.id.desc()]).label('rank'),
|
||||
)
|
||||
.join(
|
||||
collisions,
|
||||
sa.and_(m.workspace_uuid == collisions.c.workspace_uuid, m.session_id == collisions.c.session_id),
|
||||
)
|
||||
.subquery()
|
||||
)
|
||||
columns = _KEY + [
|
||||
'bot_name',
|
||||
'pipeline_id',
|
||||
'pipeline_name',
|
||||
'platform',
|
||||
'user_id',
|
||||
'user_name',
|
||||
'message_count',
|
||||
'start_time',
|
||||
'last_activity',
|
||||
'is_active',
|
||||
]
|
||||
select = sa.select(*[ranked.c[name] for name in columns[:-1]], sa.literal(True)).where(ranked.c.rank == 1)
|
||||
insert = postgresql.insert if conn.dialect.name == 'postgresql' else sqlite.insert
|
||||
statement = insert(sessions).from_select(columns, select)
|
||||
conn.execute(
|
||||
statement.on_conflict_do_update(
|
||||
index_elements=_KEY,
|
||||
set_={name: statement.excluded[name] for name in columns if name not in _KEY and name != 'is_active'},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if _TABLE not in sa.inspect(conn).get_table_names():
|
||||
return
|
||||
collisions = conn.execute(
|
||||
sa.text('SELECT 1 FROM monitoring_sessions GROUP BY workspace_uuid, session_id HAVING COUNT(*) > 1 LIMIT 1')
|
||||
).first()
|
||||
if collisions:
|
||||
raise RuntimeError('Cannot downgrade bot-scoped sessions without losing colliding bot records')
|
||||
pk = sa.inspect(conn).get_pk_constraint(_TABLE)
|
||||
with op.batch_alter_table(_TABLE, naming_convention={'pk': 'pk_%(table_name)s'}) as batch:
|
||||
batch.drop_constraint(pk['name'] or f'pk_{_TABLE}', type_='primary')
|
||||
batch.create_primary_key(f'pk_{_TABLE}', ['workspace_uuid', 'session_id'])
|
||||
@@ -0,0 +1,54 @@
|
||||
"""add passkey credentials table
|
||||
|
||||
Revision ID: 0024_passkey_credentials
|
||||
Revises: 0023_bot_scoped_sessions
|
||||
Create Date: 2026-09-12
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0024_passkey_credentials'
|
||||
down_revision = '0023_bot_scoped_sessions'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE_NAME = 'passkey_credentials'
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
if _TABLE_NAME not in existing_tables:
|
||||
op.create_table(
|
||||
_TABLE_NAME,
|
||||
sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column('uuid', sa.String(36), nullable=False),
|
||||
sa.Column(
|
||||
'account_uuid',
|
||||
sa.String(36),
|
||||
sa.ForeignKey('users.uuid', ondelete='CASCADE'),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column('name', sa.String(255), nullable=False),
|
||||
sa.Column('credential_id', sa.String(255), nullable=False),
|
||||
sa.Column('public_key', sa.Text(), nullable=False),
|
||||
sa.Column('sign_count', sa.Integer(), nullable=False, server_default='0'),
|
||||
sa.Column('aaguid', sa.String(64), nullable=True),
|
||||
sa.Column('transports', sa.String(255), nullable=True),
|
||||
sa.Column('backed_up', sa.Boolean(), nullable=False, server_default='0'),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.Column('last_used_at', sa.DateTime(), nullable=True),
|
||||
)
|
||||
op.create_index('uq_passkey_credentials_uuid', _TABLE_NAME, ['uuid'], unique=True)
|
||||
op.create_index('uq_passkey_credentials_cred_id', _TABLE_NAME, ['credential_id'], unique=True)
|
||||
op.create_index('ix_passkey_credentials_account', _TABLE_NAME, ['account_uuid'], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('ix_passkey_credentials_account', table_name=_TABLE_NAME)
|
||||
op.drop_index('uq_passkey_credentials_cred_id', table_name=_TABLE_NAME)
|
||||
op.drop_index('uq_passkey_credentials_uuid', table_name=_TABLE_NAME)
|
||||
op.drop_table(_TABLE_NAME)
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Join the published master credentials and 4.11 bot processor branches.
|
||||
|
||||
Revision ID: 0026_merge_master_beta
|
||||
Revises: 0024_passkey_credentials, 0025_bot_plugin_processors
|
||||
|
||||
Both parent chains are already published. Keep their identities and schema/data
|
||||
operations intact; Alembic applies the missing branch before this no-op merge.
|
||||
"""
|
||||
|
||||
revision = '0026_merge_master_beta'
|
||||
down_revision = ('0024_passkey_credentials', '0025_bot_plugin_processors')
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Only unmerge the revision graph; neither branch's schema/data is removed.
|
||||
pass
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Create the manual migration journal only; never convert pipeline JSON."""
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = '0027_pipeline_migration'
|
||||
down_revision = '0026_merge_master_beta'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
table = 'pipeline_migration_snapshots'
|
||||
if not sa.inspect(bind).has_table(table):
|
||||
op.create_table(
|
||||
table,
|
||||
sa.Column('uuid', sa.String(36), primary_key=True),
|
||||
sa.Column(
|
||||
'workspace_uuid', sa.String(36), sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), nullable=False
|
||||
),
|
||||
sa.Column('pipeline_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('source_fingerprint', sa.String(64), nullable=False),
|
||||
sa.Column('target_fingerprint', sa.String(64), nullable=False),
|
||||
sa.Column('planner_version', sa.String(64), nullable=False),
|
||||
sa.Column('source_snapshot', sa.JSON(), nullable=False),
|
||||
sa.Column('state', sa.String(32), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'pipeline_uuid'],
|
||||
['legacy_pipelines.workspace_uuid', 'legacy_pipelines.uuid'],
|
||||
name='fk_pipeline_migration_workspace_pipeline',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
'workspace_uuid', 'pipeline_uuid', 'source_fingerprint', name='uq_pipeline_migration_source'
|
||||
),
|
||||
sa.CheckConstraint("state IN ('activation_pending', 'active')", name='ck_pipeline_migration_state'),
|
||||
)
|
||||
op.create_index('ix_pipeline_migration_workspace_pipeline', table, ['workspace_uuid', 'pipeline_uuid'])
|
||||
if bind.dialect.name == 'postgresql':
|
||||
op.execute(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY')
|
||||
op.execute(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY')
|
||||
op.execute(f'DROP POLICY IF EXISTS langbot_workspace_isolation ON {table}')
|
||||
expression = "workspace_uuid = NULLIF(current_setting('langbot.workspace_uuid', true), '')"
|
||||
op.execute(
|
||||
f'CREATE POLICY langbot_workspace_isolation ON {table} FOR ALL TO PUBLIC '
|
||||
f'USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_table('pipeline_migration_snapshots')
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Join pipeline migration and knowledge base draft branches.
|
||||
|
||||
Revision ID: 0028_merge_knowledge_drafts
|
||||
Revises: 0027_pipeline_migration, 0026_knowledge_base_drafts
|
||||
"""
|
||||
|
||||
revision = '0028_merge_knowledge_drafts'
|
||||
down_revision = ('0027_pipeline_migration', '0026_knowledge_base_drafts')
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Only unmerge the revision graph; neither branch's schema is removed.
|
||||
pass
|
||||
@@ -64,11 +64,14 @@ _ALEMBIC_TENANT_TABLES = {
|
||||
'binary_storages',
|
||||
'mcp_servers',
|
||||
'model_providers',
|
||||
'codex_credentials',
|
||||
'passkey_credentials',
|
||||
'llm_models',
|
||||
'embedding_models',
|
||||
'rerank_models',
|
||||
'legacy_pipelines',
|
||||
'pipeline_run_records',
|
||||
'pipeline_migration_snapshots',
|
||||
'plugin_settings',
|
||||
'knowledge_bases',
|
||||
'knowledge_base_files',
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Transaction-held, tenant/pipeline-scoped admission for form producers.
|
||||
|
||||
Every pending producer must hold this row until its INSERT commits. Migration
|
||||
holds the same row for both CAS and activation. No table/process/advisory lock.
|
||||
The no-op UPDATE also acquires SQLite's write reservation before any reads.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from ..entity.persistence.pipeline import LegacyPipeline
|
||||
|
||||
|
||||
async def lock_pipeline_admission(session, workspace_uuid: str, pipeline_uuid: str):
|
||||
if not workspace_uuid or not pipeline_uuid:
|
||||
raise ValueError('Pipeline admission requires exact ownership')
|
||||
t = LegacyPipeline.__table__
|
||||
result = await session.execute(
|
||||
sa.update(t)
|
||||
.where(t.c.workspace_uuid == workspace_uuid, t.c.uuid == pipeline_uuid)
|
||||
.values(updated_at=t.c.updated_at)
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
raise ValueError('Pipeline admission authority unavailable')
|
||||
return (
|
||||
await session.execute(
|
||||
sa.select(t.c.config).where(t.c.workspace_uuid == workspace_uuid, t.c.uuid == pipeline_uuid)
|
||||
)
|
||||
).scalar_one()
|
||||
@@ -52,11 +52,13 @@ TENANT_TABLE_COLUMNS: dict[str, str] = {
|
||||
'binary_storages': 'workspace_uuid',
|
||||
'mcp_servers': 'workspace_uuid',
|
||||
'model_providers': 'workspace_uuid',
|
||||
'codex_credentials': 'workspace_uuid',
|
||||
'llm_models': 'workspace_uuid',
|
||||
'embedding_models': 'workspace_uuid',
|
||||
'rerank_models': 'workspace_uuid',
|
||||
'legacy_pipelines': 'workspace_uuid',
|
||||
'pipeline_run_records': 'workspace_uuid',
|
||||
'pipeline_migration_snapshots': 'workspace_uuid',
|
||||
'plugin_settings': 'workspace_uuid',
|
||||
'knowledge_bases': 'workspace_uuid',
|
||||
'knowledge_base_files': 'workspace_uuid',
|
||||
@@ -207,6 +209,8 @@ _SYNC_PROXY_CAPABILITY: contextvars.ContextVar[_ScopedSessionGuardState | None]
|
||||
_ALLOWED_SCOPED_BUILTIN_FUNCTION_TYPES = {
|
||||
'coalesce': sqlalchemy.sql.functions.coalesce,
|
||||
'count': sqlalchemy.sql.functions.count,
|
||||
'min': sqlalchemy.sql.functions.min,
|
||||
'max': sqlalchemy.sql.functions.max,
|
||||
'now': sqlalchemy.sql.functions.now,
|
||||
'sum': sqlalchemy.sql.functions.sum,
|
||||
}
|
||||
@@ -464,9 +468,21 @@ def _validate_scoped_statement_call(args: tuple[typing.Any, ...], kwargs: dict[s
|
||||
raise ScopedSessionTransactionError('TenantUnitOfWork does not allow literal-execute SQL parameters')
|
||||
|
||||
if isinstance(element, sqlalchemy.sql.elements.Cast) and type(element.type) not in {Vector, HALFVEC}:
|
||||
raise ScopedSessionTransactionError(
|
||||
'TenantUnitOfWork only allows the trusted pgvector cast used by tenant vector search'
|
||||
# Manual Pipeline migration compares DB-native JSON text, because
|
||||
# PostgreSQL JSON has no equality operator. Permit only these fixed
|
||||
# mapped columns and a built-in Text target, never arbitrary casts.
|
||||
source = element.clause
|
||||
pipeline_json_cas = (
|
||||
type(element.type) is sqlalchemy.Text
|
||||
and isinstance(source, sqlalchemy.Column)
|
||||
and type(source.type) is sqlalchemy.JSON
|
||||
and getattr(getattr(source, 'table', None), 'name', None) == 'legacy_pipelines'
|
||||
and source.name in {'config', 'stages', 'extensions_preferences'}
|
||||
)
|
||||
if not pipeline_json_cas:
|
||||
raise ScopedSessionTransactionError(
|
||||
'TenantUnitOfWork only allows trusted pgvector and Pipeline JSON CAS casts'
|
||||
)
|
||||
|
||||
if isinstance(element, sqlalchemy.sql.functions.FunctionElement):
|
||||
function_name = str(getattr(element, 'name', '')).casefold()
|
||||
@@ -853,7 +869,30 @@ class TenantScopedAsyncSession(sqlalchemy_asyncio.AsyncSession):
|
||||
self._require_owner_task()
|
||||
self._enter_internal_access()
|
||||
try:
|
||||
await transaction.commit()
|
||||
# Retain the actual connection before COMMIT: after a failed SQLite
|
||||
# COMMIT the logical transaction is inactive, but the DBAPI writer
|
||||
# can still hold PENDING/RESERVED locks. Session.close()/rollback()
|
||||
# alone can then return that poisoned connection to the pool.
|
||||
connection = await super().connection()
|
||||
try:
|
||||
await transaction.commit()
|
||||
except BaseException as exc:
|
||||
cleanup = asyncio.create_task(connection.invalidate())
|
||||
# Invalidation does not access the task-owned Session. Shield
|
||||
# physical cleanup, including against repeated cancellation,
|
||||
# before the owner closes the Session and releases its scope.
|
||||
while not cleanup.done():
|
||||
try:
|
||||
await asyncio.shield(cleanup)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
except BaseException:
|
||||
break
|
||||
try:
|
||||
cleanup.result()
|
||||
except BaseException as cleanup_error:
|
||||
exc.add_note(f'Failed to invalidate transaction connection: {cleanup_error!r}')
|
||||
raise
|
||||
finally:
|
||||
self._exit_internal_access()
|
||||
|
||||
@@ -1370,6 +1409,7 @@ class TenantUnitOfWork:
|
||||
state.mark_rollback_only(exc_value)
|
||||
rollback_only = state.rollback_only
|
||||
committed = False
|
||||
transaction_error: BaseException | None = None
|
||||
try:
|
||||
if exc_type is None and not rollback_only:
|
||||
await typing.cast(TenantScopedAsyncSession, session)._commit_owned_transaction(
|
||||
@@ -1382,6 +1422,9 @@ class TenantUnitOfWork:
|
||||
_UOW_SESSION_CONTROL_CAPABILITY,
|
||||
transaction,
|
||||
)
|
||||
except BaseException as exc:
|
||||
transaction_error = exc
|
||||
raise
|
||||
finally:
|
||||
try:
|
||||
if self._active_transaction is not None and self._context_token is not None:
|
||||
@@ -1389,6 +1432,10 @@ class TenantUnitOfWork:
|
||||
await typing.cast(TenantScopedAsyncSession, session)._close_owned_session(
|
||||
_UOW_SESSION_CONTROL_CAPABILITY
|
||||
)
|
||||
except BaseException as cleanup_error:
|
||||
if transaction_error is None:
|
||||
raise
|
||||
transaction_error.add_note(f'Failed to close transaction Session: {cleanup_error!r}')
|
||||
finally:
|
||||
if self._database_operation_token is not None:
|
||||
_DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token)
|
||||
|
||||
Reference in New Issue
Block a user