From 438cbd41a91bde11b11a26fed8f0c471cb2adb01 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <1051233107@qq.com> Date: Sat, 17 Oct 2026 09:13:44 +0800 Subject: [PATCH] test(core): provision real plugin, CLI and SeekDB contracts (#2636) * test(core): provision official plugin CLI and SeekDB contracts * test(core): trigger report guard tests and trim fixture checks --- .github/workflows/optional-contracts.yml | 215 ++++++++++++++++++ .github/workflows/run-tests.yml | 1 + docs/DEPENDENCY_CONTRACT_TESTS.md | 112 +++++++++ scripts/assert-required-tests.py | 39 ++++ tests/README.md | 4 + .../legacy_plugin_identity_sources.json | 19 ++ .../agent/test_legacy_identity_boundary.py | 90 +++++--- .../quality/test_required_contract_report.py | 72 ++++++ tests/utils/legacy_plugin_contract.py | 61 +++++ 9 files changed, 578 insertions(+), 35 deletions(-) create mode 100644 .github/workflows/optional-contracts.yml create mode 100644 docs/DEPENDENCY_CONTRACT_TESTS.md create mode 100644 scripts/assert-required-tests.py create mode 100644 tests/fixtures/legacy_plugin_identity_sources.json create mode 100644 tests/unit_tests/quality/test_required_contract_report.py create mode 100644 tests/utils/legacy_plugin_contract.py diff --git a/.github/workflows/optional-contracts.yml b/.github/workflows/optional-contracts.yml new file mode 100644 index 000000000..f7e18f890 --- /dev/null +++ b/.github/workflows/optional-contracts.yml @@ -0,0 +1,215 @@ +name: Dependency Contracts + +on: + pull_request: + types: [opened, ready_for_review, synchronize] + paths: + - 'src/langbot/**' + - 'tests/**' + - 'pyproject.toml' + - 'pytest.ini' + - 'uv.lock' + - 'scripts/assert-required-tests.py' + - '.github/workflows/optional-contracts.yml' + push: + branches: [master, develop] + paths: + - 'src/langbot/**' + - 'tests/**' + - 'pyproject.toml' + - 'pytest.ini' + - 'uv.lock' + - 'scripts/assert-required-tests.py' + - '.github/workflows/optional-contracts.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + compiled-cli: + name: Compiled CLI HTTP Contract + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + env: + GOTOOLCHAIN: local + CGO_ENABLED: '0' + + steps: + - name: Checkout Core + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Checkout official CLI source + uses: actions/checkout@v4 + with: + repository: langbot-app/langbot-cli + # lbctl v0.2.0; advance only after verifying the Core HTTP contract. + ref: 3bd20eecd4771b1fe5d0dca61d8b5523182e7bd9 + path: .contract-sources/langbot-cli + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@v5 + with: + # Match the pinned CLI's release toolchain. + go-version: '1.26.8' + cache-dependency-path: .contract-sources/langbot-cli/go.sum + + - name: Build pinned CLI + working-directory: .contract-sources/langbot-cli + env: + LANGBOT_CLI_BIN: ${{ runner.temp }}/langbot-cli/lbctl + run: | + test "$(git rev-parse HEAD)" = 3bd20eecd4771b1fe5d0dca61d8b5523182e7bd9 + go mod download + go mod verify + mkdir -p "$(dirname "$LANGBOT_CLI_BIN")" + go build -mod=readonly -trimpath -o "$LANGBOT_CLI_BIN" ./cmd/lbctl + test -x "$LANGBOT_CLI_BIN" + go version -m "$LANGBOT_CLI_BIN" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install Core dependencies + run: uv sync --dev --frozen + + - name: Run required compiled CLI contract + env: + LANGBOT_CLI_BIN: ${{ runner.temp }}/langbot-cli/lbctl + run: | + uv run --no-sync pytest \ + tests/unit_tests/api/service/test_cli_diagnostics.py::test_compiled_cli_against_core_http \ + -q --tb=short -rs --junitxml=compiled-cli-results.xml + python scripts/assert-required-tests.py compiled-cli-results.xml \ + --module tests.unit_tests.api.service.test_cli_diagnostics \ + --name-prefix test_compiled_cli_against_core_http --expected-tests 1 + + - name: Upload compiled CLI contract result + if: always() + uses: actions/upload-artifact@v4 + with: + name: compiled-cli-contract-result + path: compiled-cli-results.xml + if-no-files-found: ignore + + plugin-identity: + name: Official Plugin Identity Contract + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout Core + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Read official source pin + id: plugin-source + run: | + python3 - <<'PY' + import json + import os + import re + from pathlib import Path + + lock = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text()) + assert lock['repository'] == 'langbot-app/langbot-plugins' + assert re.fullmatch(r'[0-9a-f]{40}', lock['revision']) + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f"revision={lock['revision']}\n") + PY + + - name: Checkout pinned official plugin sources + uses: actions/checkout@v4 + with: + repository: langbot-app/langbot-plugins + ref: ${{ steps.plugin-source.outputs.revision }} + path: .contract-sources/langbot-plugins + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install Core dependencies + run: uv sync --dev --frozen + + - name: Run required identity consumer contracts + env: + LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins + run: | + uv run --no-sync pytest \ + tests/unit_tests/agent/test_legacy_identity_boundary.py::test_real_plugin_identity_helper_with_host_payload \ + -q --tb=short -rs --junitxml=plugin-identity-results.xml + python scripts/assert-required-tests.py plugin-identity-results.xml \ + --module tests.unit_tests.agent.test_legacy_identity_boundary \ + --name-prefix test_real_plugin_identity_helper_with_host_payload --expected-tests 16 + + - name: Upload plugin identity results + if: always() + uses: actions/upload-artifact@v4 + with: + name: plugin-identity-contract-results + path: plugin-identity-results.xml + if-no-files-found: ignore + + seekdb: + name: Embedded SeekDB Contract + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout Core + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Install locked SeekDB extra + run: uv sync --dev --extra seekdb --frozen + + - name: Run required embedded SeekDB contracts + env: + SEEKDB_TEST_TMP: ${{ runner.temp }}/langbot-seekdb-contract + run: | + uv run --no-sync python - <<'PY' + from importlib.metadata import version + import pyseekdb + import pylibseekdb + + print(f"pyseekdb={version('pyseekdb')}, pylibseekdb={version('pylibseekdb')}") + PY + uv run --no-sync pytest tests/integration/vector/test_seekdb.py \ + -m slow -q --tb=short -rs --durations=5 \ + --basetemp="$SEEKDB_TEST_TMP" --junitxml=seekdb-contract-results.xml + python scripts/assert-required-tests.py seekdb-contract-results.xml \ + --module tests.integration.vector.test_seekdb --name-prefix test_ --expected-tests 2 + + - name: Upload SeekDB results and startup logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: seekdb-contract-results + path: | + seekdb-contract-results.xml + ${{ runner.temp }}/langbot-seekdb-contract/**/log/*.log + if-no-files-found: ignore diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 51653e2f3..7f5e173c3 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -11,6 +11,7 @@ on: - 'uv.lock' - 'run_tests.sh' - 'scripts/test-*.sh' + - 'scripts/assert-required-tests.py' push: branches: - master diff --git a/docs/DEPENDENCY_CONTRACT_TESTS.md b/docs/DEPENDENCY_CONTRACT_TESTS.md new file mode 100644 index 000000000..48168d529 --- /dev/null +++ b/docs/DEPENDENCY_CONTRACT_TESTS.md @@ -0,0 +1,112 @@ +# Provisioned dependency contracts + +`.github/workflows/optional-contracts.yml` runs three isolated jobs for optional +product dependencies. Every job provisions its dependency and rejects missing, +skipped, failed, or errored required cases. These are tests of actual integration +boundaries; removing them or substituting fake external implementations would +lose coverage. No production credentials or external model accounts are needed. + +The fixtures are deliberately pinned for reproducibility. They are not a promise +that the latest release of every external repository passes. Update pins only +after rerunning the relevant contracts. This workflow is separate from the +latest-LocalAgent Space-resolution work in PR #2632. + +## Official plugin identity boundary + +The source lock in `tests/fixtures/legacy_plugin_identity_sources.json` records +the official repository, immutable revision, and SHA-256 of the consumed Python +source files. CI checks out that revision with read-only access +and no persisted Git credentials. Tests verify file digests before executing the +actual identity methods, their genuine configuration-error classes, and their +real `scoped_identity` dependency. Vendor HTTP clients are not constructed. + +The original test expected bare upstream user IDs and extracted only one method +from a hard-coded sibling checkout. Supported current plugins intentionally +namespace IDs by Workspace or trusted installation, so merely provisioning that +old test produced false failures. The current fixture is: + +- Official repository: `langbot-app/langbot-plugins` +- Revision: `7ad54b1a139d88113bc55128326800ed29779737` +- DifyAgent 0.2.3; N8nAgent, CozeAgent and TboxAgent 0.2.2 +- All exceed Core's current migration minimums (0.1.10; TboxAgent 0.1.8) + +The 19 Host provenance/resume tests retain their exact raw-identity assertions. +The consumer boundary adds dedicated/shared profiles to the former eight cases: +16 real-plugin cases verify the independently specified session/event/bot source, +current documented namespace, and rejection of missing trusted identity. Shared +cases use the real SDK `bind_invocation` and `InstallationBinding`. They are not +full plugin-installation, vendor API, or migration-admission tests. + +The documented semantic change is in each pinned plugin's `SHARED_RUNTIME.md`, +for example [Dify's upstream identity migration](https://github.com/langbot-app/langbot-plugins/blob/7ad54b1a139d88113bc55128326800ed29779737/Runner/dify-agent/SHARED_RUNTIME.md#upstream-identity-migration). +An older bare-ID source revision also passed the old tests, but its manifest +versions are below Core's migration minimums. It is intentionally not used to +claim current compatibility. + +To run with an official checkout at the locked revision: + +```bash +export LANGBOT_LEGACY_PLUGIN_ROOT=/absolute/path/to/langbot-plugins +uv run --frozen pytest tests/unit_tests/agent/test_legacy_identity_boundary.py -q -rs +``` + +An unset variable allows an explicit local prerequisite skip. An empty variable, +a missing file, a path escaping the fixture root, or a hash mismatch fails. +CI selects the 16 consumer cases and requires all 16 to pass without skips. + +## Embedded SeekDB + +The job installs `uv sync --dev --extra seekdb --frozen`, using the versions and +wheel hashes already committed in `uv.lock`: pyseekdb 1.4.0.post1 and +pylibseekdb 1.4.0. It runs the existing two slow tests against a real embedded +engine, covering upsert/text preservation and full-text/hybrid relevance order. +The tests supply their own small embedding vectors; no model download is needed. + +```bash +uv sync --dev --extra seekdb --frozen +uv run --no-sync pytest tests/integration/vector/test_seekdb.py \ + -m slow -q -rs --durations=5 --junitxml=seekdb-contract-results.xml +python scripts/assert-required-tests.py seekdb-contract-results.xml \ + --module tests.integration.vector.test_seekdb --expected-tests 2 +``` + +This isolated Linux job keeps native dependency startup failures visible instead +of treating them as skips. JUnit results and native startup logs are uploaded +when available. A missing optional package or native engine failure is not +counted as a passing contract. +## Compiled CLI HTTP contract + +The `Compiled CLI HTTP Contract` CI job runs the real `lbctl` process against a +loopback-bound Core HTTP server with temporary SQLite data and synthetic runtime +providers. It checks command parsing, API-key/workspace and capability discovery, +knowledge-engine/parser routes, sandbox diagnostics, pipeline extension updates, +pipeline execution, and monitoring reads. It does not need production accounts, +external model services, Docker, or a running LangBot deployment. + +CI builds the official [`langbot-app/langbot-cli` v0.2.0 source](https://github.com/langbot-app/langbot-cli/tree/3bd20eecd4771b1fe5d0dca61d8b5523182e7bd9), +pinned to commit `3bd20eecd4771b1fe5d0dca61d8b5523182e7bd9`, using Go 1.26.8. +Module versions and checksums come from that checkout's `go.mod` and `go.sum`. +The job verifies modules and builds with `CGO_ENABLED=0`, `-mod=readonly`, and +`-trimpath`. It requires exactly one passing JUnit case and rejects skips. + +To run locally with an official pinned CLI checkout and Go 1.26.8: + +```bash +# In the official langbot-cli checkout: +git checkout 3bd20eecd4771b1fe5d0dca61d8b5523182e7bd9 +export GOTOOLCHAIN=local CGO_ENABLED=0 +go mod download +go mod verify +go build -mod=readonly -trimpath -o ./bin/lbctl ./cmd/lbctl +export LANGBOT_CLI_BIN="$(pwd)/bin/lbctl" + +# In the LangBot Core checkout: +uv run --frozen pytest \ + tests/unit_tests/api/service/test_cli_diagnostics.py::test_compiled_cli_against_core_http \ + -q --tb=short -rs +``` + +Without `LANGBOT_CLI_BIN`, this test remains an explicit local skip. When set, it +must point to a real compiled executable. CI supplies an absolute path and treats +a missing binary, command failure, or skipped case as a failure. Update the source +pin deliberately and rerun the contract before accepting a newer CLI version. diff --git a/scripts/assert-required-tests.py b/scripts/assert-required-tests.py new file mode 100644 index 000000000..2ff1f6ced --- /dev/null +++ b/scripts/assert-required-tests.py @@ -0,0 +1,39 @@ +"""Fail closed when a provisioned dependency-contract lane skips or loses tests.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import xml.etree.ElementTree as ET + + +def verify_report(path: Path, expected_module: str, expected_tests: int, name_prefix: str = '') -> int: + cases = list(ET.parse(path).getroot().iter('testcase')) + if len(cases) != expected_tests: + raise ValueError(f'Expected {expected_tests} tests, found {len(cases)} in {path}') + for case in cases: + if case.get('classname') != expected_module: + raise ValueError(f'Unexpected test module: {case.get("classname")!r}') + if not case.get('name', '').startswith(name_prefix): + raise ValueError(f'Unexpected test name: {case.get("name")!r}') + for outcome in ('skipped', 'failure', 'error'): + if case.find(outcome) is not None: + raise ValueError(f'Required test {case.get("name")!r} reported {outcome}') + return len(cases) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('report', type=Path) + parser.add_argument('--module', required=True) + parser.add_argument('--expected-tests', type=int, required=True) + parser.add_argument('--name-prefix', default='') + args = parser.parse_args() + if args.expected_tests < 1: + parser.error('--expected-tests must be positive') + count = verify_report(args.report, args.module, args.expected_tests, args.name_prefix) + print(f'{args.module}: {count} required tests passed, 0 skipped') + + +if __name__ == '__main__': + main() diff --git a/tests/README.md b/tests/README.md index f3acd40b3..d87fc3d68 100644 --- a/tests/README.md +++ b/tests/README.md @@ -160,6 +160,10 @@ The test suite uses a centralized fixture system that provides: 3. **Coverage**: Tests cover happy paths, edge cases, and error conditions 4. **Extensibility**: Easy to add new tests by reusing existing fixtures +See [provisioned dependency contracts](../docs/DEPENDENCY_CONTRACT_TESTS.md) for +the real compiled CLI, pinned official plugin identity helpers, and embedded +SeekDB CI jobs, including local prerequisites and verification boundaries. + ## Running Tests ### Quick self-test for developers diff --git a/tests/fixtures/legacy_plugin_identity_sources.json b/tests/fixtures/legacy_plugin_identity_sources.json new file mode 100644 index 000000000..83cb7337f --- /dev/null +++ b/tests/fixtures/legacy_plugin_identity_sources.json @@ -0,0 +1,19 @@ +{ + "repository": "langbot-app/langbot-plugins", + "revision": "7ad54b1a139d88113bc55128326800ed29779737", + "contract": "Current dedicated-workspace and trusted shared-installation identity; see each plugin SHARED_RUNTIME.md.", + "files": { + "Runner/dify-agent/components/runner/default.py": "0bcaff74c64c89be2bddb4be5871ce148a814676b50987a65f95228ecb7da8fa", + "Runner/dify-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a", + "Runner/dify-agent/pkg/dify_client.py": "607ed8d610d916e5b64ed20699e3de6e2216b238a5fecc9cabcdb75533b8dc56", + "Runner/n8n-agent/components/runner/default.py": "2719ec0a8ff779e90e61bd0209653fb9a1eab4b552877cfcbd4d3c9ce803fcf1", + "Runner/n8n-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a", + "Runner/n8n-agent/pkg/n8n_client.py": "15149b2d4162b51a3a7427d639908130ea4179014067f04a459a8a71ca105c15", + "Runner/coze-agent/components/runner/default.py": "23f222bd1bf94682d44df021d56e6e69801fbe8ea5bac99693d0f1e2bce8f5e8", + "Runner/coze-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a", + "Runner/coze-agent/pkg/coze_client.py": "379cb7f07f9931cc9de4cc40f8ebddea171c2a05383e662f6bd88ecdc5b3c317", + "Runner/tbox-agent/components/runner/default.py": "4c9fdf8cc10c916915036112ded8768a619a827ca4b5e6a3f05881a7d0e4492f", + "Runner/tbox-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a", + "Runner/tbox-agent/pkg/tbox_client.py": "0f09a3c70f773bc6377fd5b55a41d186dd15487d46aab3e61b837c08e15100a0" + } +} diff --git a/tests/unit_tests/agent/test_legacy_identity_boundary.py b/tests/unit_tests/agent/test_legacy_identity_boundary.py index 241b818b4..216ade894 100644 --- a/tests/unit_tests/agent/test_legacy_identity_boundary.py +++ b/tests/unit_tests/agent/test_legacy_identity_boundary.py @@ -1,12 +1,15 @@ """Native identity provenance across the Host/SDK and persisted resume boundary.""" -import ast -from pathlib import Path +import hashlib +import json +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest from langbot_plugin.api.entities.builtin.runner.context import RunnerContext +from langbot_plugin.api.proxies.invocation import bind_invocation +from langbot_plugin.entities.io.context import InstallationBinding from langbot.pkg.agent.runner.query_entry_adapter import QueryEntryAdapter from langbot.pkg.agent.runner.context_builder import RunnerContextBuilder from langbot.pkg.agent.runner.host_models import AgentEventEnvelope @@ -16,6 +19,7 @@ from langbot.pkg.entity.persistence.base import Base from sqlalchemy.ext.asyncio import create_async_engine from tests.unit_tests.agent import test_event_first_protocol as event_fixtures from tests.unit_tests.agent.test_context_validation import TestContextValidation as Helpers +from tests.utils.legacy_plugin_contract import load_identity_helper, verified_plugin_root @pytest.fixture @@ -192,35 +196,30 @@ async def test_tbox_resume_without_persisted_identity_fails_closed(mock_query): assert context['runtime']['metadata']['bot_id'] is None +@pytest.fixture(scope='session') +def official_plugin_root(): + return verified_plugin_root() + + @pytest.mark.asyncio @pytest.mark.parametrize( - 'runner,folder,method', + 'runner,folder,method,error_name,local_identity', [ - ('DifyAgent', 'dify-agent', '_get_user_tag'), - ('N8nAgent', 'n8n-agent', '_get_user_tag'), - ('CozeAgent', 'coze-agent', '_get_user_id'), - ('TboxAgent', 'tbox-agent', '_get_user_id'), + ('DifyAgent', 'dify-agent', '_get_user_tag', 'DifyConfigError', 'person_Session_007'), + ('N8nAgent', 'n8n-agent', '_get_user_tag', 'N8nConfigError', 'person_Session_007'), + ('CozeAgent', 'coze-agent', '_get_user_id', 'CozeConfigError', 'person_launcher-123'), + ('TboxAgent', 'tbox-agent', '_get_user_id', 'TboxConfigError', 'bot-uuid-123'), ], ) -@pytest.mark.parametrize('missing', [False, True]) -async def test_real_plugin_identity_helper_with_host_payload(mock_query, runner, folder, method, missing): - # Execute the actual pure helper AST without loading vendor clients or making network calls. - root = Path(__file__).resolve().parents[3].parent / 'langbot-plugins-411-migration' - path = root / 'Runner' / folder / 'components/runner/default.py' - if not path.exists(): - pytest.skip('sibling plugin source checkout is required for cross-repository contract gate') - tree = ast.parse(path.read_text()) - function = next(node for node in ast.walk(tree) if isinstance(node, ast.FunctionDef) and node.name == method) - - class ConfigError(Exception): - def __init__(self, message, **kwargs): - super().__init__(message) - - namespace = { - 'RunnerContext': RunnerContext, - **{name: ConfigError for name in ('DifyConfigError', 'N8nConfigError', 'CozeConfigError', 'TboxConfigError')}, - } - exec(compile(ast.Module(body=[function], type_ignores=[]), str(path), 'exec'), namespace) +@pytest.mark.parametrize('missing', [False, True], ids=['trusted-identity', 'missing-identity']) +@pytest.mark.parametrize('profile', ['dedicated', 'shared']) +async def test_real_plugin_identity_helper_with_host_payload( + mock_query, monkeypatch, official_plugin_root, runner, folder, method, error_name, local_identity, missing, profile +): + # Keep exact Host identity selection above; current supported plugins namespace + # that identity to prevent collisions across Workspaces and installations. + helper, config_error = load_identity_helper(official_plugin_root, folder, method, error_name) + monkeypatch.setenv('LANGBOT_PLUGIN_RUNTIME_PROFILE', profile) mock_query.session.launcher_id = 'Session_007' event = QueryEntryAdapter.query_to_event(mock_query) if missing: @@ -229,13 +228,34 @@ async def test_real_plugin_identity_helper_with_host_payload(mock_query, runner, source = 'legacy-bot' if runner == 'TboxAgent' else 'legacy-session' context, _ = await build(event, runner, source) sdk = RunnerContext.model_validate(context) - if missing: - with pytest.raises(ConfigError, match='trusted Host identity'): - namespace[method](None, sdk) - else: - expected = ( - mock_query.bot_uuid - if runner == 'TboxAgent' - else ('person_launcher-123' if runner == 'CozeAgent' else 'person_Session_007') + handler = SimpleNamespace() + consumer = SimpleNamespace(_plugin_runtime_handler=handler) + binding = InstallationBinding( + instance_uuid='identity-instance', + workspace_uuid=event.workspace_id, + installation_uuid='00000000-0000-4000-8000-000000000007', + runtime_revision=1, + artifact_digest='a' * 64, + ) + if profile == 'shared': + # A stale connection binding must never win over the active invocation. + handler.bound_action_context = binding.model_copy( + update={'workspace_uuid': 'foreign-workspace', 'installation_uuid': 'foreign-installation'} ) - assert namespace[method](None, sdk) == expected + authority = bind_invocation(handler, binding=binding) if profile == 'shared' else nullcontext() + with authority: + if missing: + with pytest.raises(config_error, match='trusted Host identity') as caught: + helper(consumer, sdk) + assert caught.value.code == f'{folder.removesuffix("-agent")}.identity_unavailable' + else: + # The expected local IDs above are independent of the actual Host + # payload and plugin output, preserving session-vs-event provenance. + scope = ( + ['identity-instance', event.workspace_id, '00000000-0000-4000-8000-000000000007'] + if profile == 'shared' + else [event.workspace_id] + ) + encoded = json.dumps([scope, local_identity], ensure_ascii=False, separators=(',', ':')).encode() + expected = 'lb_' + hashlib.sha256(encoded).hexdigest() + assert helper(consumer, sdk) == expected diff --git a/tests/unit_tests/quality/test_required_contract_report.py b/tests/unit_tests/quality/test_required_contract_report.py new file mode 100644 index 000000000..0c7c5cab5 --- /dev/null +++ b/tests/unit_tests/quality/test_required_contract_report.py @@ -0,0 +1,72 @@ +"""The optional-dependency CI guard must reject a false-green JUnit report.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys + +import pytest + + +SCRIPT = Path(__file__).resolve().parents[3] / 'scripts' / 'assert-required-tests.py' +SPEC = importlib.util.spec_from_file_location('required_contract_report', SCRIPT) +assert SPEC is not None and SPEC.loader is not None +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) + + +def write_report(tmp_path, cases): + path = tmp_path / 'report.xml' + path.write_text(f'{cases}') + return path + + +def test_accepts_exact_passing_cases(tmp_path): + path = write_report( + tmp_path, '' + ) + assert REPORT.verify_report(path, 'contracts', 2) == 2 + + +@pytest.mark.parametrize('outcome', ['skipped', 'failure', 'error']) +def test_rejects_every_nonpassing_outcome(tmp_path, outcome): + path = write_report(tmp_path, f'<{outcome}/>') + with pytest.raises(ValueError, match=outcome): + REPORT.verify_report(path, 'contracts', 1) + + +@pytest.mark.parametrize('count', [0, 2]) +def test_rejects_changed_case_count(tmp_path, count): + path = write_report(tmp_path, '' * count) + with pytest.raises(ValueError, match='Expected 1 tests'): + REPORT.verify_report(path, 'contracts', 1) + + +def test_rejects_wrong_module(tmp_path): + path = write_report(tmp_path, '') + with pytest.raises(ValueError, match='Unexpected test module'): + REPORT.verify_report(path, 'contracts', 1) + + +def test_rejects_missing_report(tmp_path): + with pytest.raises(FileNotFoundError): + REPORT.verify_report(tmp_path / 'missing.xml', 'contracts', 1) + + +def test_command_line_success_and_failure(tmp_path): + path = write_report(tmp_path, '') + command = [sys.executable, str(SCRIPT), str(path), '--module', 'contracts', '--expected-tests'] + good = subprocess.run([*command, '1'], capture_output=True, text=True) + assert good.returncode == 0 + assert '1 required tests passed, 0 skipped' in good.stdout + bad = subprocess.run([*command, '0'], capture_output=True, text=True) + assert bad.returncode != 0 + assert 'must be positive' in bad.stderr + + +def test_rejects_wrong_function(tmp_path): + path = write_report(tmp_path, '') + with pytest.raises(ValueError, match='Unexpected test name'): + REPORT.verify_report(path, 'contracts', 1, 'test_required') diff --git a/tests/utils/legacy_plugin_contract.py b/tests/utils/legacy_plugin_contract.py new file mode 100644 index 000000000..0056ce05a --- /dev/null +++ b/tests/utils/legacy_plugin_contract.py @@ -0,0 +1,61 @@ +"""Load identity consumers from integrity-checked official plugin source.""" + +import ast +import hashlib +import importlib.util +import json +import os +from pathlib import Path + +import pytest +from langbot_plugin.api.entities.builtin.runner.context import RunnerContext + + +SOURCE_LOCK = Path(__file__).resolve().parents[1] / 'fixtures' / 'legacy_plugin_identity_sources.json' + + +def verified_plugin_root() -> Path: + """Allow optional local execution, but never skip an explicitly provisioned source.""" + configured = os.environ.get('LANGBOT_LEGACY_PLUGIN_ROOT') + if configured is None: + pytest.skip('set LANGBOT_LEGACY_PLUGIN_ROOT to the official revision in legacy_plugin_identity_sources.json') + if not configured.strip(): + pytest.fail('LANGBOT_LEGACY_PLUGIN_ROOT is set but empty', pytrace=False) + root = Path(configured).resolve() + lock = json.loads(SOURCE_LOCK.read_text()) + for relative, expected in lock['files'].items(): + path = root / relative + if not path.resolve().is_relative_to(root) or not path.is_file(): + pytest.fail(f'Official plugin fixture is missing or outside its root: {relative}', pytrace=False) + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + pytest.fail( + f'Official plugin fixture hash mismatch: {relative}; expected {lock["repository"]}@{lock["revision"]}', + pytrace=False, + ) + return root + + +def load_identity_helper(root: Path, folder: str, method: str, error_name: str): + """Execute actual identity code and its error class without vendor client I/O.""" + plugin = root / 'Runner' / folder + path = plugin / 'components/runner/default.py' + tree = ast.parse(path.read_text()) + runner_classes = [node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == 'DefaultRunner'] + assert len(runner_classes) == 1, f'{path}: expected one DefaultRunner' + methods = [node for node in runner_classes[0].body if isinstance(node, ast.FunctionDef) and node.name == method] + assert len(methods) == 1, f'{path}: expected one {method}' + client_path = plugin / 'pkg' / f'{folder.removesuffix("-agent")}_client.py' + client_tree = ast.parse(client_path.read_text()) + errors = [node for node in client_tree.body if isinstance(node, ast.ClassDef) and node.name == error_name] + assert len(errors) == 1, f'{client_path}: expected one {error_name}' + + scoped_path = plugin / 'pkg/scoped_identity.py' + spec = importlib.util.spec_from_file_location(f'identity_contract_{folder.replace("-", "_")}', scoped_path) + assert spec is not None and spec.loader is not None + scoped_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(scoped_module) + namespace = {'RunnerContext': RunnerContext, 'scoped_identity': scoped_module.scoped_identity} + exec(compile(ast.Module(body=errors, type_ignores=[]), str(client_path), 'exec'), namespace) + exec(compile(ast.Module(body=methods, type_ignores=[]), str(path), 'exec'), namespace) + return namespace[method], namespace[error_name]