merge: integrate master certification and document identity fixes into 4.11

This commit is contained in:
RockChinQ
2026-09-21 10:55:40 +00:00
51 changed files with 2518 additions and 247 deletions
@@ -3,11 +3,13 @@
from __future__ import annotations
import copy
import io
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, call
import pytest
import quart
from quart.datastructures import FileStorage
pytestmark = pytest.mark.integration
@@ -287,3 +289,34 @@ async def test_github_install_rejects_internal_asset_url_before_task_creation(
assert response.status_code == 400
assert 'HTTPS GitHub release asset URL' in (await response.get_json())['msg']
application.task_mgr.create_user_task.assert_not_called()
@pytest.mark.asyncio
async def test_local_install_forwards_explicit_administrator_force(plugin_security_api):
application, client, _ = plugin_security_api
execution_context = SimpleNamespace(
instance_uuid='instance-test',
workspace_uuid=WORKSPACE_UUID,
placement_generation=1,
)
application.persistence_mgr.tenant_scope = None
application.plugin_connector.require_workspace_context = AsyncMock(return_value=execution_context)
application.plugin_connector.install_plugin = AsyncMock()
application.task_mgr.create_user_task = Mock(return_value=SimpleNamespace(id='task-certification'))
response = await client.post(
'/api/v1/plugins/install/local',
headers=_headers('manager-token'),
files={
'file': FileStorage(stream=io.BytesIO(b'archive'), filename='plugin.lbpkg'),
},
form={'administrator_force': 'true'},
)
assert response.status_code == 200
operation = application.task_mgr.create_user_task.call_args.args[0]
await operation
assert application.plugin_connector.install_plugin.await_args.args[1] == {
'plugin_file': b'archive',
'administrator_force': True,
}
@@ -71,7 +71,13 @@ def test_migration_graph_has_one_head_containing_both_released_branches():
heads = scripts.get_heads()
assert len(heads) == 1, f'Release migrations must converge, found {heads}'
ancestors = {revision.revision for revision in scripts.walk_revisions()}
assert {'0024_passkey_credentials', '0025_bot_plugin_processors'} <= ancestors
assert {
'0024_passkey_credentials',
'0025_bot_plugin_processors',
'0025_rag_document_identity',
'0027_pipeline_migration',
'0028_merge_knowledge_drafts',
} <= ancestors
assert all(len(revision) <= 32 for revision in ancestors)
@@ -0,0 +1,421 @@
"""Real database regressions for Host/engine identity; no live plugin or customer data."""
import asyncio
import os
import uuid
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
import pytest_asyncio
import sqlalchemy as sa
from sqlalchemy.ext.asyncio import create_async_engine
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.entity.persistence.base import Base
from langbot.pkg.entity.persistence.rag import File, KnowledgeBase
from langbot.pkg.entity.persistence.user import User
from langbot.pkg.entity.persistence.workspace import Workspace
from langbot.pkg.persistence.alembic_runner import (
get_alembic_current,
run_alembic_downgrade,
run_alembic_stamp,
run_alembic_upgrade,
)
from langbot.pkg.persistence.mgr import PersistenceManager
from langbot.pkg.rag.knowledge.kbmgr import RuntimeKnowledgeBase
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
OLD_HEAD = '0024_passkey_credentials'
def current_head():
from alembic.config import Config
from alembic.script import ScriptDirectory
from langbot.pkg.persistence.alembic_runner import _ALEMBIC_DIR
config = Config()
config.set_main_option('script_location', _ALEMBIC_DIR)
return ScriptDirectory.from_config(config).get_current_head()
DOCUMENT_IDENTITY_REVISION = '0025_rag_document_identity'
CONTEXT = ExecutionContext(instance_uuid='instance-a', workspace_uuid='workspace-a', placement_generation=5)
@pytest_asyncio.fixture(params=['sqlite', 'postgres'])
async def database(request, tmp_path):
admin = None
schema = 'ke_identity_' + uuid.uuid4().hex
if request.param == 'postgres':
url = os.environ.get('TEST_POSTGRES_URL')
if not url:
pytest.skip('TEST_POSTGRES_URL is required for disposable PostgreSQL tests')
admin = create_async_engine(url)
async with admin.begin() as conn:
await conn.execute(sa.text(f'CREATE SCHEMA {schema}'))
engine = create_async_engine(url, connect_args={'server_settings': {'search_path': schema}})
else:
engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "identity.db"}')
@sa.event.listens_for(engine.sync_engine, 'connect')
def enable_foreign_keys(connection, _):
connection.execute('PRAGMA foreign_keys=ON')
try:
yield engine
finally:
await engine.dispose()
if admin is not None:
async with admin.begin() as conn:
await conn.execute(sa.text(f'DROP SCHEMA {schema} CASCADE'))
await admin.dispose()
async def create_schema(engine, *, legacy=False):
# Only the fixture-owned dependency closure; never all imported application tables.
async with engine.begin() as conn:
await conn.run_sync(
lambda sync: Base.metadata.create_all(
sync, tables=[User.__table__, Workspace.__table__, KnowledgeBase.__table__]
)
)
if legacy:
# Exact pre-0025 File shape, NOT current metadata stamped with an old revision.
await conn.execute(
sa.text("""CREATE TABLE knowledge_base_files (
uuid VARCHAR(255) PRIMARY KEY UNIQUE,
workspace_uuid VARCHAR(36) NOT NULL REFERENCES workspaces(uuid) ON DELETE CASCADE,
kb_id VARCHAR(255), file_name VARCHAR, extension VARCHAR, created_at TIMESTAMP, status VARCHAR,
CONSTRAINT uq_knowledge_base_files_workspace_uuid UNIQUE (workspace_uuid, uuid),
CONSTRAINT fk_knowledge_base_files_workspace_kb FOREIGN KEY (workspace_uuid, kb_id)
REFERENCES knowledge_bases(workspace_uuid, uuid) ON DELETE CASCADE
)""")
)
await conn.execute(
sa.text(
'CREATE INDEX ix_knowledge_base_files_workspace_kb ON knowledge_base_files (workspace_uuid, kb_id)'
)
)
else:
await conn.run_sync(lambda sync: File.__table__.create(sync))
for workspace in ('workspace-a', 'workspace-b'):
await conn.execute(
sa.insert(Workspace).values(
uuid=workspace,
instance_uuid='instance-a',
name=workspace,
slug=workspace,
source='cloud_projection',
)
)
for kb, workspace in [('kb-a', 'workspace-a'), ('kb-other', 'workspace-a'), ('kb-b', 'workspace-b')]:
await conn.execute(
sa.insert(KnowledgeBase).values(
uuid=kb,
workspace_uuid=workspace,
name=kb,
knowledge_engine_plugin_id='author/engine',
collection_id=kb,
)
)
@pytest_asyncio.fixture
async def runtime(database):
await create_schema(database)
ap = SimpleNamespace(
logger=Mock(),
workspace_service=SimpleNamespace(
get_execution_binding=AsyncMock(
return_value=SimpleNamespace(
instance_uuid='instance-a',
placement_generation=5,
)
)
),
storage_mgr=SimpleNamespace(
require_scoped_object_key=Mock(),
size_scoped_object_key=AsyncMock(return_value=12),
delete_scoped_object_key=AsyncMock(),
),
plugin_connector=SimpleNamespace(
require_workspace_context=AsyncMock(side_effect=lambda context: context),
call_rag_ingest=AsyncMock(return_value={'document_id': 'upstream-id', 'status': 'processing'}),
call_rag_delete_document=AsyncMock(return_value=True),
),
)
ap.persistence_mgr = PersistenceManager(ap)
ap.persistence_mgr.db = SimpleNamespace(get_engine=lambda: database)
kb = KnowledgeBase(uuid='kb-a', workspace_uuid='workspace-a', name='kb', knowledge_engine_plugin_id='author/engine')
return RuntimeKnowledgeBase(ap, kb, CONTEXT)
async def seed(runtime, *, status='pending', file_id='host-id', workspace='workspace-a', kb='kb-a'):
values = dict(
uuid=file_id, workspace_uuid=workspace, kb_id=kb, file_name='upload.txt', extension='txt', status=status
)
await runtime.ap.persistence_mgr.execute_async(sa.insert(File).values(**values))
return File(**values)
async def read_file_row(runtime, file_id='host-id'):
# A fresh connection proves committed state rather than an identity-map/mock result.
async with runtime.ap.persistence_mgr.get_db_engine().connect() as conn:
row = (await conn.execute(sa.select(File).where(File.uuid == file_id))).first()
return None if row is None else dict(row._mapping)
@pytest.mark.asyncio
@pytest.mark.parametrize(
'engine_id', ['dify-upstream', 'fastgpt-collection', 'ragflow-upstream', 'host-id', ' opaque ID ']
)
async def test_ingest_persists_exact_engine_identity_and_restart_delete(runtime, engine_id):
file = await seed(runtime)
runtime.ap.plugin_connector.call_rag_ingest.return_value['document_id'] = engine_id
await runtime._store_file_task(CONTEXT, file, Mock())
row = await read_file_row(runtime)
assert row['uuid'] == 'host-id'
assert row.get('engine_document_id') == engine_id
assert row['status'] == 'completed'
restarted = RuntimeKnowledgeBase(runtime.ap, runtime.knowledge_base_entity, CONTEXT)
await restarted.delete_file(CONTEXT, 'host-id')
runtime.ap.plugin_connector.call_rag_delete_document.assert_awaited_once_with('author/engine', engine_id, 'kb-a')
assert await read_file_row(runtime) is None
@pytest.mark.asyncio
@pytest.mark.parametrize('response', [False, None, 0, 1, 'true', {}])
async def test_delete_without_explicit_confirmation_preserves_row(runtime, response):
await seed(runtime, status='completed')
runtime.ap.plugin_connector.call_rag_delete_document.return_value = response
with pytest.raises(RuntimeError, match='delet'):
await runtime.delete_file(CONTEXT, 'host-id')
assert await read_file_row(runtime) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize('failure', ['exception', 'missing_plugin'])
async def test_delete_unavailable_retains_row(runtime, failure):
await seed(runtime, status='completed')
if failure == 'exception':
runtime.ap.plugin_connector.call_rag_delete_document.side_effect = RuntimeError('upstream offline')
else:
runtime.knowledge_base_entity.knowledge_engine_plugin_id = None
with pytest.raises(RuntimeError, match='delet'):
await runtime.delete_file(CONTEXT, 'host-id')
assert await read_file_row(runtime) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize('status', ['pending', 'processing'])
async def test_delete_rejects_inflight_ingestion(runtime, status):
await seed(runtime, status=status)
with pytest.raises(RuntimeError, match='ingest'):
await runtime.delete_file(CONTEXT, 'host-id')
runtime.ap.plugin_connector.call_rag_delete_document.assert_not_awaited()
assert await read_file_row(runtime) is not None
@pytest.mark.asyncio
@pytest.mark.parametrize('document_id', [None, '', ' ', 123, [], {}])
async def test_invalid_response_identity_cannot_complete(runtime, document_id):
file = await seed(runtime)
runtime.ap.plugin_connector.call_rag_ingest.return_value = {'status': 'completed', 'document_id': document_id}
with pytest.raises(ValueError, match='document_id'):
await runtime._store_file_task(CONTEXT, file, Mock())
assert (await read_file_row(runtime))['status'] == 'interrupted'
@pytest.mark.asyncio
async def test_missing_response_identity_cannot_complete(runtime):
file = await seed(runtime)
runtime.ap.plugin_connector.call_rag_ingest.return_value = {'status': 'completed'}
with pytest.raises(ValueError, match='document_id'):
await runtime._store_file_task(CONTEXT, file, Mock())
assert (await read_file_row(runtime))['status'] == 'interrupted'
@pytest.mark.asyncio
async def test_failed_ingestion_retains_returned_identity_for_cleanup(runtime):
file = await seed(runtime)
runtime.ap.plugin_connector.call_rag_ingest.return_value = {
'document_id': 'uploaded-before-parsing-failed',
'status': 'failed',
'error_message': 'parsing failed',
}
with pytest.raises(Exception, match='parsing failed'):
await runtime._store_file_task(CONTEXT, file, Mock())
row = await read_file_row(runtime)
assert row['status'] == 'failed'
assert row.get('engine_document_id') == 'uploaded-before-parsing-failed'
await runtime.delete_file(CONTEXT, 'host-id')
runtime.ap.plugin_connector.call_rag_delete_document.assert_awaited_once_with(
'author/engine', 'uploaded-before-parsing-failed', 'kb-a'
)
@pytest.mark.asyncio
@pytest.mark.parametrize('status', ['completed', 'failed'])
async def test_legacy_rows_use_host_id_only_with_confirmed_delete(runtime, status):
await seed(runtime, status=status)
await runtime.delete_file(CONTEXT, 'host-id')
runtime.ap.plugin_connector.call_rag_delete_document.assert_awaited_once_with('author/engine', 'host-id', 'kb-a')
assert await read_file_row(runtime) is None
@pytest.mark.asyncio
@pytest.mark.parametrize('workspace,kb', [('workspace-a', 'kb-other'), ('workspace-b', 'kb-b')])
async def test_status_update_and_delete_do_not_touch_other_scope(runtime, workspace, kb):
await seed(runtime, workspace=workspace, kb=kb)
assert not await runtime._set_file_status(CONTEXT, 'host-id', 'processing')
with pytest.raises(WorkspaceNotFoundError):
await runtime.delete_file(CONTEXT, 'host-id')
assert (await read_file_row(runtime))['status'] == 'pending'
runtime.ap.plugin_connector.call_rag_delete_document.assert_not_awaited()
@pytest.mark.asyncio
async def test_delete_rechecks_generation_after_plugin_response(runtime):
await seed(runtime, status='completed')
async def delete(*_):
runtime.ap.workspace_service.get_execution_binding.side_effect = WorkspaceNotFoundError('stale placement')
return True
runtime.ap.plugin_connector.call_rag_delete_document.side_effect = delete
with pytest.raises(WorkspaceNotFoundError):
await runtime.delete_file(CONTEXT, 'host-id')
assert await read_file_row(runtime) is not None
@pytest.mark.asyncio
async def test_ingest_rechecks_generation_before_mapping_write(runtime):
file = await seed(runtime)
async def ingest(*_):
runtime.ap.workspace_service.get_execution_binding.side_effect = WorkspaceNotFoundError('stale placement')
return {'document_id': 'upstream-id', 'status': 'completed'}
runtime.ap.plugin_connector.call_rag_ingest.side_effect = ingest
with pytest.raises(WorkspaceNotFoundError):
await runtime._store_file_task(CONTEXT, file, Mock())
row = await read_file_row(runtime)
assert row['status'] == 'processing'
assert row.get('engine_document_id') is None
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
@pytest.mark.asyncio
async def test_concurrent_delete_cannot_remove_ingestion_tracking(runtime):
file = await seed(runtime)
entered, release = asyncio.Event(), asyncio.Event()
async def ingest(*_):
entered.set()
await release.wait()
return {'document_id': 'upstream-id', 'status': 'completed'}
runtime.ap.plugin_connector.call_rag_ingest.side_effect = ingest
task = asyncio.create_task(runtime._store_file_task(CONTEXT, file, Mock()))
try:
await asyncio.wait_for(entered.wait(), 5)
with pytest.raises(RuntimeError, match='ingest'):
await runtime.delete_file(CONTEXT, 'host-id')
finally:
release.set()
await task
assert (await read_file_row(runtime)).get('engine_document_id') == 'upstream-id'
@pytest.mark.asyncio
async def test_identity_and_completion_are_one_atomic_write(runtime):
file = await seed(runtime)
engine = runtime.ap.persistence_mgr.get_db_engine()
attempts = []
def reject_completion(_conn, _cursor, statement, parameters, _context, _many):
if statement.startswith('UPDATE knowledge_base_files') and 'completed' in parameters:
attempts.append(statement)
raise RuntimeError('fixture mapping write failure')
sa.event.listen(engine.sync_engine, 'before_cursor_execute', reject_completion)
try:
with pytest.raises(RuntimeError, match='mapping write failure'):
await runtime._store_file_task(CONTEXT, file, Mock())
finally:
sa.event.remove(engine.sync_engine, 'before_cursor_execute', reject_completion)
assert len(attempts) == 1
assert 'engine_document_id=' in attempts[0]
row = await read_file_row(runtime)
assert row['status'] != 'completed'
assert row.get('engine_document_id') == 'upstream-id'
@pytest.mark.asyncio
async def test_populated_legacy_migration_roundtrip(database):
await create_schema(database, legacy=True)
async with database.begin() as conn:
await conn.execute(
sa.text("""INSERT INTO knowledge_base_files
(uuid, workspace_uuid, kb_id, file_name, extension, status)
VALUES ('legacy', 'workspace-a', 'kb-a', 'original.txt', 'txt', 'completed')""")
)
assert 'engine_document_id' not in await conn.run_sync(
lambda sync: {col['name'] for col in sa.inspect(sync).get_columns('knowledge_base_files')}
)
await run_alembic_stamp(database, OLD_HEAD)
await run_alembic_upgrade(database, DOCUMENT_IDENTITY_REVISION)
async with database.connect() as conn:
columns = await conn.run_sync(lambda sync: sa.inspect(sync).get_columns('knowledge_base_files'))
assert 'engine_document_id' in {col['name'] for col in columns}
assert next(col for col in columns if col['name'] == 'engine_document_id')['nullable']
row = (await conn.execute(sa.text('SELECT * FROM knowledge_base_files'))).mappings().one()
assert row['uuid'] == 'legacy' and row['status'] == 'completed'
assert row['engine_document_id'] is None
assert await get_alembic_current(database) == DOCUMENT_IDENTITY_REVISION
await run_alembic_upgrade(database, DOCUMENT_IDENTITY_REVISION)
await run_alembic_stamp(database, OLD_HEAD)
await run_alembic_upgrade(database, DOCUMENT_IDENTITY_REVISION)
await run_alembic_downgrade(database, OLD_HEAD)
async with database.connect() as conn:
assert 'engine_document_id' not in await conn.run_sync(
lambda sync: {col['name'] for col in sa.inspect(sync).get_columns('knowledge_base_files')}
)
assert (await conn.execute(sa.text('SELECT uuid FROM knowledge_base_files'))).scalar_one() == 'legacy'
await run_alembic_upgrade(database)
@pytest.mark.asyncio
async def test_published_document_identity_branch_converges_to_release_head(database):
await create_schema(database, legacy=True)
await run_alembic_stamp(database, OLD_HEAD)
await run_alembic_upgrade(database, DOCUMENT_IDENTITY_REVISION)
async with database.begin() as conn:
await conn.execute(
sa.text("""INSERT INTO knowledge_base_files
(uuid, workspace_uuid, kb_id, file_name, extension, status, engine_document_id)
VALUES ('host-stable', 'workspace-a', 'kb-a', 'original.txt', 'txt', 'completed', 'opaque-engine-id')""")
)
await run_alembic_upgrade(database)
await run_alembic_upgrade(database)
assert await get_alembic_current(database) == current_head()
async with database.connect() as conn:
row = (await conn.execute(sa.text('SELECT * FROM knowledge_base_files'))).mappings().one()
assert row['uuid'] == 'host-stable'
assert row['engine_document_id'] == 'opaque-engine-id'
assert row['status'] == 'completed'
tables = await conn.run_sync(lambda sync: sa.inspect(sync).get_table_names())
assert 'pipeline_migration_snapshots' in tables
@pytest.mark.asyncio
async def test_fresh_metadata_then_migration_is_idempotent(database):
await create_schema(database)
await run_alembic_stamp(database, OLD_HEAD)
await run_alembic_upgrade(database)
assert await get_alembic_current(database) == current_head()
async with database.connect() as conn:
assert 'engine_document_id' in await conn.run_sync(
lambda sync: {col['name'] for col in sa.inspect(sync).get_columns('knowledge_base_files')}
)
@@ -0,0 +1,406 @@
"""B1 lifecycle regressions; isolated databases, no external engines or customer data."""
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from tests.integration.persistence import test_rag_document_identity as identity
from tests.integration.persistence.test_rag_document_identity import CONTEXT, read_file_row, seed
from langbot.pkg.core.entities import LifecycleControlScope
from langbot.pkg.core.taskmgr import AsyncTaskManager, TaskContext
from langbot.pkg.rag.knowledge.kbmgr import RuntimeKnowledgeBase
# Reuse the real isolated-database fixtures without duplicating their lifecycle.
database = identity.database
runtime = identity.runtime
def attach_task_manager(runtime):
runtime.ap.event_loop = asyncio.get_running_loop()
runtime.ap.instance_config = SimpleNamespace(data={})
runtime.ap.task_mgr = AsyncTaskManager(runtime.ap)
return runtime.ap.task_mgr
@pytest.mark.asyncio
async def test_cancelled_ingestion_retains_recovery_resources(runtime):
"""Local cancellation is interrupted observation, not remote quiescence."""
file = await seed(runtime)
entered, terminal = asyncio.Event(), asyncio.Event()
async def ingest(*_):
entered.set()
try:
await asyncio.Event().wait()
finally:
terminal.set()
runtime.ap.plugin_connector.call_rag_ingest.side_effect = ingest
manager = attach_task_manager(runtime)
wrapper = manager.create_user_task(
runtime._store_file_task(CONTEXT, file, TaskContext.new()),
kind='knowledge-operation',
name=f'knowledge-store-file-{file.file_name}',
instance_uuid=CONTEXT.instance_uuid,
workspace_uuid=CONTEXT.workspace_uuid,
placement_generation=CONTEXT.placement_generation,
)
try:
await asyncio.wait_for(entered.wait(), 5)
manager.cancel_by_scope(LifecycleControlScope.APPLICATION)
with pytest.raises(asyncio.CancelledError):
await wrapper.task
assert terminal.is_set()
assert wrapper.task.cancelled()
assert (await read_file_row(runtime))['status'] == 'interrupted'
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
with pytest.raises(RuntimeError, match='interrupted'):
await runtime.delete_file(CONTEXT, file.uuid)
runtime.ap.plugin_connector.call_rag_delete_document.assert_not_awaited()
assert await read_file_row(runtime) is not None
finally:
if not wrapper.task.done():
wrapper.task.cancel()
await asyncio.gather(wrapper.task, return_exceptions=True)
@pytest.mark.asyncio
@pytest.mark.parametrize('status', ['pending', 'processing'])
async def test_recreated_host_marks_abandoned_rows_interrupted(runtime, status):
"""Fresh Host state recovers observation without authorizing remote deletion."""
await seed(runtime, status=status)
new_ap = SimpleNamespace(**vars(runtime.ap))
recreated = RuntimeKnowledgeBase(new_ap, runtime.knowledge_base_entity, CONTEXT)
manager = attach_task_manager(recreated)
assert manager.get_all_tasks() == []
await recreated.initialize()
assert (await read_file_row(recreated))['status'] == 'interrupted'
with pytest.raises(RuntimeError, match='interrupted'):
await recreated.delete_file(CONTEXT, 'host-id')
recreated.ap.plugin_connector.call_rag_delete_document.assert_not_awaited()
recreated.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
@pytest.mark.asyncio
async def test_b1_live_task_still_blocks_delete_after_runtime_object_recreation(runtime):
file = await seed(runtime)
entered, release = asyncio.Event(), asyncio.Event()
async def ingest(*_):
entered.set()
await release.wait()
return {'document_id': file.uuid, 'status': 'completed'}
runtime.ap.plugin_connector.call_rag_ingest.side_effect = ingest
manager = attach_task_manager(runtime)
wrapper = manager.create_user_task(
runtime._store_file_task(CONTEXT, file, Mock()),
kind='knowledge-operation',
name=f'knowledge-store-file-{file.file_name}',
instance_uuid=CONTEXT.instance_uuid,
workspace_uuid=CONTEXT.workspace_uuid,
placement_generation=CONTEXT.placement_generation,
)
try:
await asyncio.wait_for(entered.wait(), 5)
recreated = RuntimeKnowledgeBase(runtime.ap, runtime.knowledge_base_entity, CONTEXT)
await recreated.initialize()
assert not wrapper.task.done()
with pytest.raises(RuntimeError, match='ingest'):
await recreated.delete_file(CONTEXT, file.uuid)
recreated.ap.plugin_connector.call_rag_delete_document.assert_not_awaited()
assert (await read_file_row(recreated))['status'] == 'processing'
finally:
release.set()
await asyncio.wait_for(wrapper.task, 5)
await recreated.delete_file(CONTEXT, file.uuid)
assert await read_file_row(recreated) is None
@pytest.mark.asyncio
@pytest.mark.parametrize('failure', [TimeoutError('timeout'), ConnectionError('disconnect')])
async def test_transport_failure_is_interrupted_not_failed(runtime, failure):
file = await seed(runtime)
runtime.ap.plugin_connector.call_rag_ingest.side_effect = failure
with pytest.raises(type(failure)):
await runtime._store_file_task(CONTEXT, file, Mock())
assert (await read_file_row(runtime))['status'] == 'interrupted'
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
@pytest.mark.asyncio
async def test_recovered_row_cannot_be_replayed_by_delayed_old_task(runtime):
file = await seed(runtime)
await runtime.initialize()
with pytest.raises(Exception):
await runtime._store_file_task(CONTEXT, file, Mock())
assert (await read_file_row(runtime))['status'] == 'interrupted'
runtime.ap.plugin_connector.call_rag_ingest.assert_not_awaited()
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
@pytest.mark.asyncio
async def test_reconciliation_preserves_other_scopes_and_terminal_states(runtime):
await seed(runtime, status='processing')
await seed(runtime, file_id='other-kb', kb='kb-other', status='processing')
await seed(runtime, file_id='other-workspace', workspace='workspace-b', kb='kb-b', status='processing')
for status in ('completed', 'failed', 'interrupted'):
await seed(runtime, file_id=status, status=status)
await runtime.initialize()
assert (await read_file_row(runtime))['status'] == 'interrupted'
for file_id in ('other-kb', 'other-workspace'):
assert (await read_file_row(runtime, file_id))['status'] == 'processing'
for status in ('completed', 'failed', 'interrupted'):
assert (await read_file_row(runtime, status))['status'] == status
@pytest.mark.asyncio
async def test_cancellation_after_result_retains_identity(runtime):
file = await seed(runtime)
original = runtime._set_file_status
async def cancel_completion(context, uuid, status, **kwargs):
if status == 'completed':
raise asyncio.CancelledError()
return await original(context, uuid, status, **kwargs)
runtime._set_file_status = cancel_completion
with pytest.raises(asyncio.CancelledError):
await runtime._store_file_task(CONTEXT, file, Mock())
row = await read_file_row(runtime)
assert row['status'] == 'interrupted'
assert row['engine_document_id'] == 'upstream-id'
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
@pytest.mark.asyncio
async def test_cancellation_after_completion_commit_does_not_downgrade(runtime):
file = await seed(runtime)
original = runtime._set_file_status
async def cancel_after_commit(context, uuid, status, **kwargs):
result = await original(context, uuid, status, **kwargs)
if status == 'completed':
raise asyncio.CancelledError()
return result
runtime._set_file_status = cancel_after_commit
with pytest.raises(asyncio.CancelledError):
await runtime._store_file_task(CONTEXT, file, Mock())
row = await read_file_row(runtime)
assert row['status'] == 'completed'
assert row['engine_document_id'] == 'upstream-id'
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
def make_service(runtime):
from langbot.pkg.api.http.service.knowledge import KnowledgeService
runtime.ap.rag_mgr = SimpleNamespace(
get_knowledge_base_details=AsyncMock(return_value={'uuid': 'kb-a'}),
get_knowledge_base_by_uuid=AsyncMock(return_value=runtime),
delete_knowledge_base=AsyncMock(),
)
return KnowledgeService(runtime.ap)
@pytest.mark.asyncio
async def test_file_list_recovers_abandoned_task_without_restart(runtime):
await seed(runtime, status='processing')
files = await make_service(runtime).get_files_by_knowledge_base(CONTEXT, 'kb-a')
assert files[0]['status'] == 'interrupted'
assert (await read_file_row(runtime))['status'] == 'interrupted'
@pytest.mark.asyncio
@pytest.mark.parametrize('status', ['pending', 'processing', 'interrupted'])
async def test_bulk_kb_delete_cannot_discard_unsettled_ingestion(runtime, status):
from langbot.pkg.entity.persistence.rag import Chunk
async with runtime.ap.persistence_mgr.get_db_engine().begin() as conn:
await conn.run_sync(lambda sync: Chunk.__table__.create(sync))
await seed(runtime, status=status)
service = make_service(runtime)
with pytest.raises(RuntimeError, match='ingestion'):
await service.delete_knowledge_base(CONTEXT, 'kb-a')
assert await read_file_row(runtime) is not None
runtime.ap.rag_mgr.delete_knowledge_base.assert_not_awaited()
@pytest.mark.asyncio
async def test_queued_task_cancelled_before_start_recovers_on_file_list(runtime):
runtime.ap.storage_mgr.exists_scoped_object_key = AsyncMock(return_value=True)
manager = attach_task_manager(runtime)
await runtime.store_file(CONTEXT, 'upload.txt')
wrapper = manager.get_all_tasks()[0]
wrapper.task.cancel()
await asyncio.gather(wrapper.task, return_exceptions=True)
assert wrapper.task.cancelled()
files = await make_service(runtime).get_files_by_knowledge_base(CONTEXT, 'kb-a')
assert files[0]['status'] == 'interrupted'
runtime.ap.plugin_connector.call_rag_ingest.assert_not_awaited()
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize('disconnect', [False, True], ids=['cancel-waiter', 'close-host-connection'])
async def test_actual_sdk_late_write_preserves_interrupted_state(runtime, tmp_path, monkeypatch, disconnect):
from langbot.pkg.api.http.context import ExecutionContext
from langbot_plugin.entities.io.actions.enums import RuntimeToPluginAction
from langbot_plugin.runtime.io.handler import ActionResponse
from tests.integration.plugin.test_rag_file_transfer_protocol import BINDING, protocol_stack
context = ExecutionContext(instance_uuid='instance-a', workspace_uuid='workspace-a', placement_generation=7)
runtime.execution_context = context
file = await seed(runtime)
async with protocol_stack(tmp_path, monkeypatch, 'shared', BINDING) as stack:
entered, release, finished = asyncio.Event(), asyncio.Event(), asyncio.Event()
selected = SimpleNamespace(_runtime_plugin_handler=stack.bridge)
monkeypatch.setattr(stack.runtime.plugin_mgr, '_get_connected_rag_plugin', lambda *_: (selected, 'engine'))
vectors = set()
@stack.plugin.action(RuntimeToPluginAction.INGEST_DOCUMENT)
async def ingest(data):
entered.set()
await release.wait()
vectors.add('opaque-upstream-id')
finished.set()
return ActionResponse.success({'document_id': 'opaque-upstream-id', 'status': 'completed'})
async def send_ingest(_plugin_id, data):
with stack.core.installation_scope(BINDING):
return await stack.core.rag_ingest_document('tester', 'engine', data)
runtime.ap.plugin_connector.call_rag_ingest = send_ingest
task = asyncio.create_task(runtime._store_file_task(context, file, Mock()))
try:
await asyncio.wait_for(entered.wait(), 5)
if disconnect:
await stack.core.close()
else:
task.cancel()
outcome = await asyncio.gather(task, return_exceptions=True)
assert isinstance(outcome[0], BaseException)
assert (await read_file_row(runtime))['status'] == 'interrupted'
assert not finished.is_set()
with pytest.raises(RuntimeError, match='interrupted'):
await runtime.delete_file(context, file.uuid)
runtime.ap.plugin_connector.call_rag_delete_document.assert_not_awaited()
release.set()
await asyncio.wait_for(finished.wait(), 5)
assert vectors == {'opaque-upstream-id'}
assert (await read_file_row(runtime))['status'] == 'interrupted'
assert (await read_file_row(runtime))['engine_document_id'] is None
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
finally:
release.set()
if not task.done():
task.cancel()
await asyncio.gather(task, return_exceptions=True)
@pytest.mark.asyncio
async def test_late_observed_identity_survives_recovery_without_false_completion(runtime):
file = await seed(runtime)
entered, release = asyncio.Event(), asyncio.Event()
async def ingest(*_):
entered.set()
await release.wait()
return {'document_id': 'late-id', 'status': 'completed'}
runtime.ap.plugin_connector.call_rag_ingest.side_effect = ingest
task = asyncio.create_task(runtime._store_file_task(CONTEXT, file, Mock()))
try:
await asyncio.wait_for(entered.wait(), 5)
# Independent Host observation, not shared process state.
new_ap = SimpleNamespace(**{k: v for k, v in vars(runtime.ap).items() if not k.startswith('_knowledge_')})
recreated = RuntimeKnowledgeBase(new_ap, runtime.knowledge_base_entity, CONTEXT)
await recreated.initialize()
release.set()
await asyncio.gather(task, return_exceptions=True)
row = await read_file_row(runtime)
assert row['status'] == 'interrupted'
assert row['engine_document_id'] == 'late-id'
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
finally:
release.set()
await asyncio.gather(task, return_exceptions=True)
@pytest.mark.asyncio
async def test_parser_transport_failure_retains_upload(runtime):
file = await seed(runtime)
runtime.ap.storage_mgr.load_scoped_object_key = AsyncMock(return_value=b'file')
runtime.ap.plugin_connector.call_parser = AsyncMock(side_effect=TimeoutError())
with pytest.raises(TimeoutError):
await runtime._store_file_task(CONTEXT, file, Mock(), parser_plugin_id='author/parser')
assert (await read_file_row(runtime))['status'] == 'interrupted'
runtime.ap.storage_mgr.delete_scoped_object_key.assert_not_awaited()
runtime.ap.plugin_connector.call_rag_ingest.assert_not_awaited()
@pytest.mark.asyncio
async def test_kb_delete_serializes_with_new_upload_admission(runtime):
from langbot.pkg.entity.persistence.rag import Chunk
async with runtime.ap.persistence_mgr.get_db_engine().begin() as conn:
await conn.run_sync(lambda sync: Chunk.__table__.create(sync))
runtime.ap.storage_mgr.exists_scoped_object_key = AsyncMock(return_value=True)
attach_task_manager(runtime)
service = make_service(runtime)
entered, release = asyncio.Event(), asyncio.Event()
async def pause_delete(*_):
entered.set()
await release.wait()
runtime.ap.rag_mgr.delete_knowledge_base.side_effect = pause_delete
deletion = asyncio.create_task(service.delete_knowledge_base(CONTEXT, 'kb-a'))
upload = None
try:
await asyncio.wait_for(entered.wait(), 5)
upload = asyncio.create_task(runtime.store_file(CONTEXT, 'upload.txt'))
await asyncio.sleep(0.1)
assert not upload.done(), 'upload admission must wait for the in-progress KB deletion'
runtime.ap.plugin_connector.call_rag_ingest.assert_not_awaited()
release.set()
await deletion
with pytest.raises(Exception):
await upload # FK rejects admission after the KB was deleted.
runtime.ap.plugin_connector.call_rag_ingest.assert_not_awaited()
finally:
release.set()
await asyncio.gather(deletion, *([upload] if upload else []), return_exceptions=True)
await asyncio.gather(*(wrapper.task for wrapper in runtime.ap.task_mgr.get_all_tasks()), return_exceptions=True)
@pytest.mark.asyncio
@pytest.mark.parametrize('provider_name', ['LocalStorageProvider', 'S3StorageProvider'])
async def test_retention_cleanup_preserves_ingestion_recovery_upload(runtime, tmp_path, provider_name):
from langbot.pkg.api.http.service.maintenance import MaintenanceService
await seed(runtime, status='interrupted')
retained = tmp_path / 'retained.txt'
expired = tmp_path / 'expired.txt'
retained.write_text('recovery source')
expired.write_text('unreferenced upload')
provider = type(provider_name, (), {})()
provider.delete = AsyncMock()
runtime.ap.storage_mgr.storage_provider = provider
candidates = [
{'key': 'upload.txt', 'path': str(retained)},
{'key': 'unreferenced.txt', 'path': str(expired)},
]
service = MaintenanceService(runtime.ap)
service._expired_local_upload_candidates = Mock(return_value=candidates)
service._expired_s3_upload_candidates = AsyncMock(return_value=candidates)
count = await service._cleanup_expired_uploaded_files(CONTEXT, 7)
assert count == 1
if provider_name == 'LocalStorageProvider':
assert retained.read_text() == 'recovery source'
assert not expired.exists()
else:
provider.delete.assert_awaited_once_with('unreferenced.txt')
@@ -0,0 +1,223 @@
"""Certified archive admission through the public Core installation API."""
from __future__ import annotations
import base64
import hashlib
import io
import zipfile
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
import yaml
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.plugin.connector import PluginRuntimeConnector
from langbot_plugin.entities.io.context import InstallationBinding
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
pytestmark = pytest.mark.integration
@pytest.mark.asyncio
@pytest.mark.parametrize(
('deployment', 'archive_kind', 'administrator_force', 'expected_profile'),
[
('cloud', 'signed_shared', False, 'shared-runtime-v1'),
('oss', 'signed_shared', False, 'shared-runtime-v1'),
('oss', 'legacy', False, 'dedicated'),
('oss', 'invalid_shared', True, 'dedicated'),
],
)
async def test_install_plugin_admits_archive_before_persistence_and_applies_selected_profile(
deployment: str,
archive_kind: str,
administrator_force: bool,
expected_profile: str,
) -> None:
package, trusted_public_keys = _archive(archive_kind)
connector, execution_context, binding = _connector(deployment, trusted_public_keys)
await connector.install_plugin(
PluginInstallSource.LOCAL,
{
'plugin_file': package,
'administrator_force': administrator_force,
},
)
connector._store_artifact_package.assert_awaited_once_with(
execution_context,
hashlib.sha256(package).hexdigest(),
package,
)
persisted_info = connector._persist_installation_package.await_args.kwargs['install_info']
assert persisted_info['_certification']['runtime_profile'] == expected_profile
assert persisted_info['_certification']['normalized_digest'] == _normalized_digest(package)
connector.handler.apply_plugin_installation.assert_awaited_once_with(
binding,
artifact_package=package,
enabled=True,
)
@pytest.mark.asyncio
@pytest.mark.parametrize('archive_kind', ['legacy', 'invalid_shared'])
async def test_cloud_rejects_untrusted_archive_before_storage_persistence_or_runtime_apply(archive_kind: str) -> None:
package, trusted_public_keys = _archive(archive_kind)
connector, _execution_context, _binding = _connector('cloud', trusted_public_keys)
with pytest.raises(ValueError, match='CERTIFIED_PLUGIN_CLOUD_CERTIFICATE_'):
await connector.install_plugin(PluginInstallSource.LOCAL, {'plugin_file': package})
connector._store_artifact_package.assert_not_awaited()
connector._persist_installation_package.assert_not_awaited()
connector.handler.apply_plugin_installation.assert_not_awaited()
@pytest.mark.asyncio
async def test_oss_requires_explicit_administrator_force_for_declared_invalid_archive() -> None:
package, trusted_public_keys = _archive('invalid_shared')
connector, _execution_context, _binding = _connector('oss', trusted_public_keys)
with pytest.raises(ValueError, match='CERTIFIED_PLUGIN_OSS_FORCE_REQUIRED'):
await connector.install_plugin(PluginInstallSource.LOCAL, {'plugin_file': package})
connector._store_artifact_package.assert_not_awaited()
connector._persist_installation_package.assert_not_awaited()
connector.handler.apply_plugin_installation.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize('requested_version', [None, '1.0.0'])
@pytest.mark.parametrize('archive_kind', ['signed_shared', 'legacy'])
async def test_marketplace_version_selection_keeps_certificate_gate_and_single_apply(
monkeypatch, requested_version, archive_kind
):
import json
import langbot.pkg.plugin.connector as connector_module
from langbot.pkg.core.taskmgr import TaskContext
package, trusted_public_keys = _archive(archive_kind)
connector, _execution_context, binding = _connector('cloud', trusted_public_keys)
connector._refresh_runner_registry = AsyncMock()
requests = []
async def marketplace_get(_client, url, **kwargs):
requests.append(url)
if '/plugins/download/' in url:
assert url.endswith('/certified/example/1.0.0')
return 200, package
if url.endswith('/versions'):
return 200, json.dumps({'data': {'versions': [{'version': '1.0.0'}]}}).encode()
return 404, b'{}'
monkeypatch.setattr(connector_module, '_marketplace_get', marketplace_get)
info = {'plugin_author': 'certified', 'plugin_name': 'example'}
if requested_version is not None:
info['plugin_version'] = requested_version
task_context = TaskContext.new()
if archive_kind == 'legacy':
with pytest.raises(ValueError, match='CERTIFIED_PLUGIN_CLOUD_CERTIFICATE_REQUIRED'):
await connector.install_plugin(PluginInstallSource.MARKETPLACE, info, task_context)
connector._store_artifact_package.assert_not_awaited()
connector._persist_installation_package.assert_not_awaited()
connector.handler.apply_plugin_installation.assert_not_awaited()
connector._refresh_runner_registry.assert_not_awaited()
else:
await connector.install_plugin(PluginInstallSource.MARKETPLACE, info, task_context)
persisted_info = connector._persist_installation_package.await_args.kwargs['install_info']
assert persisted_info['plugin_version'] == '1.0.0'
assert persisted_info['_certification']['runtime_profile'] == 'shared-runtime-v1'
connector.handler.apply_plugin_installation.assert_awaited_once_with(
binding, artifact_package=package, enabled=True
)
connector._refresh_runner_registry.assert_awaited_once()
assert task_context.metadata['progress_percent'] == 100
if requested_version is not None:
# Confirmed migrations must fetch the reviewed release, never latest/MCP/skill.
assert len(requests) == 1
else:
assert any(url.endswith('/versions') for url in requests)
def _connector(deployment: str, trusted_public_keys: dict[str, str]):
package_digest = 'a' * 64
execution_context = ExecutionContext(
instance_uuid='instance-a',
workspace_uuid='workspace-a',
placement_generation=1,
)
binding = InstallationBinding(
instance_uuid='instance-a',
workspace_uuid='workspace-a',
placement_generation=1,
installation_uuid='00000000-0000-4000-8000-000000000001',
runtime_revision=1,
artifact_digest=package_digest,
)
app = SimpleNamespace(
instance_config=SimpleNamespace(
data={
'plugin': {
'enable': True,
'certification': {'trusted_public_keys': trusted_public_keys},
}
}
),
deployment=SimpleNamespace(mode=deployment),
logger=Mock(),
)
connector = PluginRuntimeConnector(app, AsyncMock())
connector.handler = SimpleNamespace(
register_installation_binding=Mock(),
apply_plugin_installation=AsyncMock(return_value={'state': 'running'}),
)
connector._current_execution_context = AsyncMock(return_value=execution_context)
connector._store_artifact_package = AsyncMock()
connector._persist_installation_package = AsyncMock(return_value=(binding, None, False))
connector._wait_for_installed_plugin_ready = AsyncMock()
return connector, execution_context, binding
def _archive(kind: str) -> tuple[bytes, dict[str, str]]:
manifest = {
'metadata': {'author': 'certified', 'name': 'example', 'version': '1.0.0'},
'execution': {'sharedRuntime': 'shared-runtime-v1'},
}
if kind == 'legacy':
manifest.pop('execution')
archive = io.BytesIO()
with zipfile.ZipFile(archive, 'w') as package:
package.writestr('manifest.yaml', yaml.safe_dump(manifest))
raw_archive = archive.getvalue()
if kind == 'legacy':
return raw_archive, {}
from langbot_plugin.certification import create_envelope, write_envelope
signing_key = Ed25519PrivateKey.generate()
signed_archive = write_envelope(
raw_archive,
create_envelope(
raw_archive,
'wrong-key' if kind == 'invalid_shared' else 'ephemeral',
signing_key.sign,
),
)
trusted_key = signing_key.public_key().public_bytes(
serialization.Encoding.Raw,
serialization.PublicFormat.Raw,
)
return signed_archive, {'ephemeral': base64.b64encode(trusted_key).decode('ascii')}
def _normalized_digest(archive: bytes) -> str:
from langbot_plugin.certification import normalized_zip_digest
return normalized_zip_digest(archive)
@@ -0,0 +1,64 @@
"""Exercise actual installed SDK RPC cancellation, not a mocked cancel contract."""
import asyncio
from types import SimpleNamespace
import pytest
from langbot_plugin.entities.io.actions.enums import RuntimeToPluginAction
from langbot_plugin.runtime.io.handler import ActionResponse
from tests.integration.plugin.test_rag_file_transfer_protocol import BINDING, protocol_stack
@pytest.mark.asyncio
@pytest.mark.parametrize('disconnect', [False, True], ids=['cancel-waiter', 'close-host-connection'])
async def test_b1_host_cancellation_does_not_fence_remote_ingest(tmp_path, monkeypatch, disconnect):
async with protocol_stack(tmp_path, monkeypatch, 'shared', BINDING) as stack:
entered, release, finished = asyncio.Event(), asyncio.Event(), asyncio.Event()
vectors = set()
selected = SimpleNamespace(_runtime_plugin_handler=stack.bridge)
monkeypatch.setattr(stack.runtime.plugin_mgr, '_get_connected_rag_plugin', lambda *_: (selected, 'engine'))
@stack.plugin.action(RuntimeToPluginAction.INGEST_DOCUMENT)
async def ingest(data):
entered.set()
await release.wait()
vectors.add('host-id')
finished.set()
return ActionResponse.success({'document_id': 'host-id', 'status': 'completed'})
@stack.plugin.action(RuntimeToPluginAction.DELETE_DOCUMENT)
async def delete(data):
vectors.discard(data['document_id'])
return ActionResponse.success({'success': True})
async def send_ingest():
with stack.core.installation_scope(BINDING):
return await stack.core.rag_ingest_document('tester', 'engine', {})
local = asyncio.create_task(send_ingest())
try:
await asyncio.wait_for(entered.wait(), 5)
if disconnect:
await stack.core.close()
result = await asyncio.gather(local, return_exceptions=True)
assert isinstance(result[0], Exception)
else:
local.cancel()
with pytest.raises(asyncio.CancelledError):
await local
assert not finished.is_set()
# The SDK has no ingest/delete barrier: a new caller can receive a
# confirmed deletion while the old plugin action can still write.
# Use the surviving Runtime->plugin connection after Host disconnect.
result = await stack.bridge.rag_delete_document('kb-a', 'host-id')
assert result == {'success': True}
assert vectors == set()
release.set()
await asyncio.wait_for(finished.wait(), 5)
assert vectors == {'host-id'}
finally:
release.set()
if not local.done():
local.cancel()
await asyncio.gather(local, return_exceptions=True)