test(core): provision real plugin, CLI and SeekDB contracts (#2636)

* test(core): provision official plugin CLI and SeekDB contracts

* test(core): trigger report guard tests and trim fixture checks

tests/README.md

@@ -160,6 +160,10 @@ The test suite uses a centralized fixture system that provides:
 3. **Coverage**: Tests cover happy paths, edge cases, and error conditions
 4. **Extensibility**: Easy to add new tests by reusing existing fixtures
 
+See [provisioned dependency contracts](../docs/DEPENDENCY_CONTRACT_TESTS.md) for
+the real compiled CLI, pinned official plugin identity helpers, and embedded
+SeekDB CI jobs, including local prerequisites and verification boundaries.
+
 ## Running Tests
 
 ### Quick self-test for developers

tests/fixtures/legacy_plugin_identity_sources.json

@@ -0,0 +1,19 @@
+{
+  "repository": "langbot-app/langbot-plugins",
+  "revision": "7ad54b1a139d88113bc55128326800ed29779737",
+  "contract": "Current dedicated-workspace and trusted shared-installation identity; see each plugin SHARED_RUNTIME.md.",
+  "files": {
+    "Runner/dify-agent/components/runner/default.py": "0bcaff74c64c89be2bddb4be5871ce148a814676b50987a65f95228ecb7da8fa",
+    "Runner/dify-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a",
+    "Runner/dify-agent/pkg/dify_client.py": "607ed8d610d916e5b64ed20699e3de6e2216b238a5fecc9cabcdb75533b8dc56",
+    "Runner/n8n-agent/components/runner/default.py": "2719ec0a8ff779e90e61bd0209653fb9a1eab4b552877cfcbd4d3c9ce803fcf1",
+    "Runner/n8n-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a",
+    "Runner/n8n-agent/pkg/n8n_client.py": "15149b2d4162b51a3a7427d639908130ea4179014067f04a459a8a71ca105c15",
+    "Runner/coze-agent/components/runner/default.py": "23f222bd1bf94682d44df021d56e6e69801fbe8ea5bac99693d0f1e2bce8f5e8",
+    "Runner/coze-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a",
+    "Runner/coze-agent/pkg/coze_client.py": "379cb7f07f9931cc9de4cc40f8ebddea171c2a05383e662f6bd88ecdc5b3c317",
+    "Runner/tbox-agent/components/runner/default.py": "4c9fdf8cc10c916915036112ded8768a619a827ca4b5e6a3f05881a7d0e4492f",
+    "Runner/tbox-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a",
+    "Runner/tbox-agent/pkg/tbox_client.py": "0f09a3c70f773bc6377fd5b55a41d186dd15487d46aab3e61b837c08e15100a0"
+  }
+}

tests/unit_tests/agent/test_legacy_identity_boundary.py

@@ -1,12 +1,15 @@
 """Native identity provenance across the Host/SDK and persisted resume boundary."""
 
-import ast
-from pathlib import Path
+import hashlib
+import json
+from contextlib import nullcontext
 from types import SimpleNamespace
 from unittest.mock import AsyncMock, patch
 
 import pytest
 from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
+from langbot_plugin.api.proxies.invocation import bind_invocation
+from langbot_plugin.entities.io.context import InstallationBinding
 from langbot.pkg.agent.runner.query_entry_adapter import QueryEntryAdapter
 from langbot.pkg.agent.runner.context_builder import RunnerContextBuilder
 from langbot.pkg.agent.runner.host_models import AgentEventEnvelope
@@ -16,6 +19,7 @@ from langbot.pkg.entity.persistence.base import Base
 from sqlalchemy.ext.asyncio import create_async_engine
 from tests.unit_tests.agent import test_event_first_protocol as event_fixtures
 from tests.unit_tests.agent.test_context_validation import TestContextValidation as Helpers
+from tests.utils.legacy_plugin_contract import load_identity_helper, verified_plugin_root
 
 
 @pytest.fixture
@@ -192,35 +196,30 @@ async def test_tbox_resume_without_persisted_identity_fails_closed(mock_query):
     assert context['runtime']['metadata']['bot_id'] is None
 
 
+@pytest.fixture(scope='session')
+def official_plugin_root():
+    return verified_plugin_root()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    'runner,folder,method',
+    'runner,folder,method,error_name,local_identity',
     [
-        ('DifyAgent', 'dify-agent', '_get_user_tag'),
-        ('N8nAgent', 'n8n-agent', '_get_user_tag'),
-        ('CozeAgent', 'coze-agent', '_get_user_id'),
-        ('TboxAgent', 'tbox-agent', '_get_user_id'),
+        ('DifyAgent', 'dify-agent', '_get_user_tag', 'DifyConfigError', 'person_Session_007'),
+        ('N8nAgent', 'n8n-agent', '_get_user_tag', 'N8nConfigError', 'person_Session_007'),
+        ('CozeAgent', 'coze-agent', '_get_user_id', 'CozeConfigError', 'person_launcher-123'),
+        ('TboxAgent', 'tbox-agent', '_get_user_id', 'TboxConfigError', 'bot-uuid-123'),
     ],
 )
-@pytest.mark.parametrize('missing', [False, True])
-async def test_real_plugin_identity_helper_with_host_payload(mock_query, runner, folder, method, missing):
-    # Execute the actual pure helper AST without loading vendor clients or making network calls.
-    root = Path(__file__).resolve().parents[3].parent / 'langbot-plugins-411-migration'
-    path = root / 'Runner' / folder / 'components/runner/default.py'
-    if not path.exists():
-        pytest.skip('sibling plugin source checkout is required for cross-repository contract gate')
-    tree = ast.parse(path.read_text())
-    function = next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == method)
-
-    class ConfigError(Exception):
-        def __init__(self, message, **kwargs):
-            super().__init__(message)
-
-    namespace = {
-        'RunnerContext': RunnerContext,
-        **{name: ConfigError for name in ('DifyConfigError', 'N8nConfigError', 'CozeConfigError', 'TboxConfigError')},
-    }
-    exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), 'exec'), namespace)
+@pytest.mark.parametrize('missing', [False, True], ids=['trusted-identity', 'missing-identity'])
+@pytest.mark.parametrize('profile', ['dedicated', 'shared'])
+async def test_real_plugin_identity_helper_with_host_payload(
+    mock_query, monkeypatch, official_plugin_root, runner, folder, method, error_name, local_identity, missing, profile
+):
+    # Keep exact Host identity selection above; current supported plugins namespace
+    # that identity to prevent collisions across Workspaces and installations.
+    helper, config_error = load_identity_helper(official_plugin_root, folder, method, error_name)
+    monkeypatch.setenv('LANGBOT_PLUGIN_RUNTIME_PROFILE', profile)
     mock_query.session.launcher_id = 'Session_007'
     event = QueryEntryAdapter.query_to_event(mock_query)
     if missing:
@@ -229,13 +228,34 @@ async def test_real_plugin_identity_helper_with_host_payload(mock_query, runner,
     source = 'legacy-bot' if runner == 'TboxAgent' else 'legacy-session'
     context, _ = await build(event, runner, source)
     sdk = RunnerContext.model_validate(context)
-    if missing:
-        with pytest.raises(ConfigError, match='trusted Host identity'):
-            namespace[method](None, sdk)
-    else:
-        expected = (
-            mock_query.bot_uuid
-            if runner == 'TboxAgent'
-            else ('person_launcher-123' if runner == 'CozeAgent' else 'person_Session_007')
+    handler = SimpleNamespace()
+    consumer = SimpleNamespace(_plugin_runtime_handler=handler)
+    binding = InstallationBinding(
+        instance_uuid='identity-instance',
+        workspace_uuid=event.workspace_id,
+        installation_uuid='00000000-0000-4000-8000-000000000007',
+        runtime_revision=1,
+        artifact_digest='a' * 64,
+    )
+    if profile == 'shared':
+        # A stale connection binding must never win over the active invocation.
+        handler.bound_action_context = binding.model_copy(
+            update={'workspace_uuid': 'foreign-workspace', 'installation_uuid': 'foreign-installation'}
         )
-        assert namespace[method](None, sdk) == expected
+    authority = bind_invocation(handler, binding=binding) if profile == 'shared' else nullcontext()
+    with authority:
+        if missing:
+            with pytest.raises(config_error, match='trusted Host identity') as caught:
+                helper(consumer, sdk)
+            assert caught.value.code == f'{folder.removesuffix("-agent")}.identity_unavailable'
+        else:
+            # The expected local IDs above are independent of the actual Host
+            # payload and plugin output, preserving session-vs-event provenance.
+            scope = (
+                ['identity-instance', event.workspace_id, '00000000-0000-4000-8000-000000000007']
+                if profile == 'shared'
+                else [event.workspace_id]
+            )
+            encoded = json.dumps([scope, local_identity], ensure_ascii=False, separators=(',', ':')).encode()
+            expected = 'lb_' + hashlib.sha256(encoded).hexdigest()
+            assert helper(consumer, sdk) == expected

tests/unit_tests/quality/test_required_contract_report.py

@@ -0,0 +1,72 @@
+"""The optional-dependency CI guard must reject a false-green JUnit report."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+
+
+SCRIPT = Path(__file__).resolve().parents[3] / 'scripts' / 'assert-required-tests.py'
+SPEC = importlib.util.spec_from_file_location('required_contract_report', SCRIPT)
+assert SPEC is not None and SPEC.loader is not None
+REPORT = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(REPORT)
+
+
+def write_report(tmp_path, cases):
+    path = tmp_path / 'report.xml'
+    path.write_text(f'<testsuites><testsuite>{cases}</testsuite></testsuites>')
+    return path
+
+
+def test_accepts_exact_passing_cases(tmp_path):
+    path = write_report(
+        tmp_path, '<testcase classname="contracts" name="first"/><testcase classname="contracts" name="second"/>'
+    )
+    assert REPORT.verify_report(path, 'contracts', 2) == 2
+
+
+@pytest.mark.parametrize('outcome', ['skipped', 'failure', 'error'])
+def test_rejects_every_nonpassing_outcome(tmp_path, outcome):
+    path = write_report(tmp_path, f'<testcase classname="contracts" name="required"><{outcome}/></testcase>')
+    with pytest.raises(ValueError, match=outcome):
+        REPORT.verify_report(path, 'contracts', 1)
+
+
+@pytest.mark.parametrize('count', [0, 2])
+def test_rejects_changed_case_count(tmp_path, count):
+    path = write_report(tmp_path, '<testcase classname="contracts" name="case"/>' * count)
+    with pytest.raises(ValueError, match='Expected 1 tests'):
+        REPORT.verify_report(path, 'contracts', 1)
+
+
+def test_rejects_wrong_module(tmp_path):
+    path = write_report(tmp_path, '<testcase classname="unrelated" name="case"/>')
+    with pytest.raises(ValueError, match='Unexpected test module'):
+        REPORT.verify_report(path, 'contracts', 1)
+
+
+def test_rejects_missing_report(tmp_path):
+    with pytest.raises(FileNotFoundError):
+        REPORT.verify_report(tmp_path / 'missing.xml', 'contracts', 1)
+
+
+def test_command_line_success_and_failure(tmp_path):
+    path = write_report(tmp_path, '<testcase classname="contracts" name="case"/>')
+    command = [sys.executable, str(SCRIPT), str(path), '--module', 'contracts', '--expected-tests']
+    good = subprocess.run([*command, '1'], capture_output=True, text=True)
+    assert good.returncode == 0
+    assert '1 required tests passed, 0 skipped' in good.stdout
+    bad = subprocess.run([*command, '0'], capture_output=True, text=True)
+    assert bad.returncode != 0
+    assert 'must be positive' in bad.stderr
+
+
+def test_rejects_wrong_function(tmp_path):
+    path = write_report(tmp_path, '<testcase classname="contracts" name="unrelated"/>')
+    with pytest.raises(ValueError, match='Unexpected test name'):
+        REPORT.verify_report(path, 'contracts', 1, 'test_required')

tests/utils/legacy_plugin_contract.py

@@ -0,0 +1,61 @@
+"""Load identity consumers from integrity-checked official plugin source."""
+
+import ast
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+
+import pytest
+from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
+
+
+SOURCE_LOCK = Path(__file__).resolve().parents[1] / 'fixtures' / 'legacy_plugin_identity_sources.json'
+
+
+def verified_plugin_root() -> Path:
+    """Allow optional local execution, but never skip an explicitly provisioned source."""
+    configured = os.environ.get('LANGBOT_LEGACY_PLUGIN_ROOT')
+    if configured is None:
+        pytest.skip('set LANGBOT_LEGACY_PLUGIN_ROOT to the official revision in legacy_plugin_identity_sources.json')
+    if not configured.strip():
+        pytest.fail('LANGBOT_LEGACY_PLUGIN_ROOT is set but empty', pytrace=False)
+    root = Path(configured).resolve()
+    lock = json.loads(SOURCE_LOCK.read_text())
+    for relative, expected in lock['files'].items():
+        path = root / relative
+        if not path.resolve().is_relative_to(root) or not path.is_file():
+            pytest.fail(f'Official plugin fixture is missing or outside its root: {relative}', pytrace=False)
+        actual = hashlib.sha256(path.read_bytes()).hexdigest()
+        if actual != expected:
+            pytest.fail(
+                f'Official plugin fixture hash mismatch: {relative}; expected {lock["repository"]}@{lock["revision"]}',
+                pytrace=False,
+            )
+    return root
+
+
+def load_identity_helper(root: Path, folder: str, method: str, error_name: str):
+    """Execute actual identity code and its error class without vendor client I/O."""
+    plugin = root / 'Runner' / folder
+    path = plugin / 'components/runner/default.py'
+    tree = ast.parse(path.read_text())
+    runner_classes = [node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == 'DefaultRunner']
+    assert len(runner_classes) == 1, f'{path}: expected one DefaultRunner'
+    methods = [node for node in runner_classes[0].body if isinstance(node, ast.FunctionDef) and node.name == method]
+    assert len(methods) == 1, f'{path}: expected one {method}'
+    client_path = plugin / 'pkg' / f'{folder.removesuffix("-agent")}_client.py'
+    client_tree = ast.parse(client_path.read_text())
+    errors = [node for node in client_tree.body if isinstance(node, ast.ClassDef) and node.name == error_name]
+    assert len(errors) == 1, f'{client_path}: expected one {error_name}'
+
+    scoped_path = plugin / 'pkg/scoped_identity.py'
+    spec = importlib.util.spec_from_file_location(f'identity_contract_{folder.replace("-", "_")}', scoped_path)
+    assert spec is not None and spec.loader is not None
+    scoped_module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(scoped_module)
+    namespace = {'RunnerContext': RunnerContext, 'scoped_identity': scoped_module.scoped_identity}
+    exec(compile(ast.Module(body=errors, type_ignores=[]), str(client_path), 'exec'), namespace)
+    exec(compile(ast.Module(body=methods, type_ignores=[]), str(path), 'exec'), namespace)
+    return namespace[method], namespace[error_name]