From 51b4de966d7b6291215f9d130334898b20495002 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sat, 17 Oct 2026 19:31:10 +0800 Subject: [PATCH 1/2] test: require pinned LocalAgent contracts in CI --- .github/workflows/run-tests.yml | 47 +++++++++++++++- .../libs/dify_service_api/v1/client_test.py | 17 ------ tests/e2e/test_local_runner_fake_provider.py | 56 +++++++++++++++---- .../legacy_plugin_identity_sources.json | 22 ++++++++ tests/utils/legacy_plugin_contract.py | 6 +- 5 files changed, 117 insertions(+), 31 deletions(-) delete mode 100644 src/langbot/libs/dify_service_api/v1/client_test.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1912f9c6e..59bfaf991 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -196,6 +196,30 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Read official plugin source pin + id: plugin-source + run: | + python3 - <<'PY' + import json + import os + import re + from pathlib import Path + + lock = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text()) + assert lock['repository'] == 'langbot-app/langbot-plugins' + assert re.fullmatch(r'[0-9a-f]{40}', lock['revision']) + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f"revision={lock['revision']}\n") + PY + + - name: Checkout pinned official plugin sources + uses: actions/checkout@v4 + with: + repository: langbot-app/langbot-plugins + ref: ${{ steps.plugin-source.outputs.revision }} + path: .contract-sources/langbot-plugins + persist-credentials: false + - name: Set up Python uses: actions/setup-python@v5 with: @@ -208,7 +232,28 @@ jobs: run: uv sync --dev - name: Run E2E startup tests - run: uv run pytest tests/e2e -q --tb=short + run: uv run pytest tests/e2e --ignore=tests/e2e/test_local_runner_fake_provider.py -q --tb=short + + - name: Check Docker for the required LocalAgent Box contract + run: docker info + + - name: Run required LocalAgent contracts + env: + LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins + run: | + uv run --no-sync pytest tests/e2e/test_local_runner_fake_provider.py \ + -q --tb=short -rs --junitxml=local-agent-results.xml + python scripts/assert-required-tests.py local-agent-results.xml \ + --module tests.e2e.test_local_runner_fake_provider \ + --name-prefix test_local_runner_ --expected-tests 6 + + - name: Upload LocalAgent contract results + if: always() + uses: actions/upload-artifact@v4 + with: + name: local-agent-contract-results + path: local-agent-results.xml + if-no-files-found: ignore - name: E2E Test Summary if: always() diff --git a/src/langbot/libs/dify_service_api/v1/client_test.py b/src/langbot/libs/dify_service_api/v1/client_test.py deleted file mode 100644 index 2695b2ea4..000000000 --- a/src/langbot/libs/dify_service_api/v1/client_test.py +++ /dev/null @@ -1,17 +0,0 @@ -from . import client - -import asyncio - -import os - - -class TestDifyClient: - async def test_chat_messages(self): - cln = client.DifyClient(api_key=os.getenv('DIFY_API_KEY')) - - resp = await cln.chat_messages(inputs={}, query='Who are you?', user_id='test') - print(resp) - - -if __name__ == '__main__': - asyncio.run(TestDifyClient().test_chat_messages()) diff --git a/tests/e2e/test_local_runner_fake_provider.py b/tests/e2e/test_local_runner_fake_provider.py index 37a4b9044..196b185e4 100644 --- a/tests/e2e/test_local_runner_fake_provider.py +++ b/tests/e2e/test_local_runner_fake_provider.py @@ -1,7 +1,7 @@ """E2E coverage for the official Local Agent runner with fake Host resources. These tests start the real LangBot application and the real SDK Plugin Runtime, -load the consolidated ``langbot-plugin-demo/Runner/LocalAgent`` plugin, and verify Local Agent paths +load the pinned official ``langbot-plugins/Runner/LocalAgent`` plugin, and verify Local Agent paths that must cross Host run-scoped APIs without calling any external provider. """ @@ -13,6 +13,7 @@ import os import shutil import socket import sqlite3 +import stat import subprocess import sys import tempfile @@ -23,7 +24,7 @@ from typing import Any import pytest from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories -from tests.e2e.utils.process_manager import find_project_root +from tests.utils.legacy_plugin_contract import verified_plugin_root pytestmark = pytest.mark.e2e @@ -43,16 +44,15 @@ def _free_port() -> int: def _local_agent_repo() -> Path: - """Return the sibling local-agent repository used by this workspace E2E.""" - project_root = find_project_root() - return project_root.parent / 'langbot-plugin-demo' / 'Runner' / 'LocalAgent' + """Use the same official source pin as the plugin identity contracts.""" + return verified_plugin_root(files_key='local_agent_files') / 'Runner' / 'LocalAgent' def _package_local_agent_plugin(tmpdir: Path) -> Path: - """Package the sibling Local Agent plugin for the real local-install flow.""" + """Package the verified official plugin for the real local-install flow.""" local_agent_src = _local_agent_repo() if not (local_agent_src / 'manifest.yaml').exists(): - pytest.skip(f'local-agent repository not found at {local_agent_src}') + pytest.fail(f'LocalAgent manifest not found at {local_agent_src}', pytrace=False) package_source = tmpdir / 'local-agent-package' ignore = shutil.ignore_patterns( @@ -275,7 +275,12 @@ def local_agent_e2e_tmpdir(): """Create temporary directory for Local Agent E2E testing.""" tmpdir = Path(tempfile.mkdtemp(prefix='langbot_local_agent_e2e_')) yield tmpdir - shutil.rmtree(tmpdir, ignore_errors=True) + # The runtime seals its artifact/dependency directories as read-only. + # Restore only this fixture's own directories so teardown removes them. + for path in tmpdir.rglob('*'): + if path.is_dir() and not path.is_symlink(): + path.chmod(path.stat().st_mode | stat.S_IWUSR) + shutil.rmtree(tmpdir) @pytest.fixture(scope='session') @@ -1050,6 +1055,7 @@ def test_local_runner_owns_box_reuse_and_explicit_files( super().__init__() self.box = box self.bindings = [] + self.outboxes = [] async def get_resolved_tool_catalog(self, *args, **kwargs): return [{'name': 'exec', 'source': 'native', 'source_id': None}] @@ -1066,19 +1072,31 @@ def test_local_runner_owns_box_reuse_and_explicit_files( self.bindings.append((binding.session_id, binding.run_id)) source = next(iter(binding.imported.values()))['path'] outbox = f'/workspace/outbox/{binding.io_scope}' + self.outboxes.append(outbox) # Shell arguments are Host-issued paths, never model input. import shlex + directory_mode = '-m 700 ' if len(self.bindings) == 2 else '' result = await self.box.execute_tool( { - 'command': f'mkdir -p {shlex.quote(outbox)} && cp {shlex.quote(source)} {shlex.quote(outbox + "/answer.txt")}' + 'command': ( + f'mkdir {directory_mode}-p {shlex.quote(outbox)} && ' + f'cp {shlex.quote(source)} {shlex.quote(outbox + "/answer.txt")} && ' + f"stat -c '%u %a' {shlex.quote(outbox)} {shlex.quote(outbox + '/answer.txt')}" + ) }, query, ) assert result['ok'], result + ownership = result['stdout'].strip().splitlines() + assert ownership[-1] == '0 600', ownership + if len(self.bindings) == 2: + assert ownership[0] == '0 700', ownership return result async def probe(ap): + import shlex + backend = TestDockerBackend(ap.logger) if not await backend.is_available(): pytest.skip('Docker is required for the real Box probe') @@ -1120,6 +1138,12 @@ def test_local_runner_owns_box_reuse_and_explicit_files( output = [component for message in messages for component in (message.attachments or [])] assert len(output) == 1, messages assert base64.b64decode(output[0].base64) == f'run-{index}'.encode() + inspection = await box.execute_in_context( + context, + {'cmd': f'ls -A {shlex.quote(manager.outboxes[-1])}', 'session_id': manager.bindings[-1][0]}, + ) + assert inspection.exit_code == 0, inspection + assert inspection.stdout.strip() == '', inspection.stdout status = await RunnerBoxService(box).status(context) assert status['used'] == 1 and status['remaining'] == 0, status assert manager.bindings[0][0] == manager.bindings[1][0] @@ -1129,6 +1153,18 @@ def test_local_runner_owns_box_reuse_and_explicit_files( with pytest.raises(BoxCapacityExceededError): await RunnerBoxService(box).acquire(context, {'reuse_key': 'different'}) finally: - await box.shutdown() + try: + if manager.bindings: + # Only this fixture's temporary Workspace is mounted here. + cleanup = await box.execute_in_context( + context, + { + 'cmd': f'chown -hR {os.getuid()}:{os.getgid()} /workspace/outbox', + 'session_id': manager.bindings[-1][0], + }, + ) + assert cleanup.exit_code == 0, cleanup + finally: + await box.shutdown() _run_local_agent_probe(local_agent_e2e_tmpdir, probe) diff --git a/tests/fixtures/legacy_plugin_identity_sources.json b/tests/fixtures/legacy_plugin_identity_sources.json index 83cb7337f..62c60b864 100644 --- a/tests/fixtures/legacy_plugin_identity_sources.json +++ b/tests/fixtures/legacy_plugin_identity_sources.json @@ -15,5 +15,27 @@ "Runner/tbox-agent/components/runner/default.py": "4c9fdf8cc10c916915036112ded8768a619a827ca4b5e6a3f05881a7d0e4492f", "Runner/tbox-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a", "Runner/tbox-agent/pkg/tbox_client.py": "0f09a3c70f773bc6377fd5b55a41d186dd15487d46aab3e61b837c08e15100a0" + }, + "local_agent_files": { + "Runner/LocalAgent/assets/icon.svg": "7e1cb51793e898fca810ec72ba6fe9af377074f52139cdd56c93e87f3459f964", + "Runner/LocalAgent/components/runner/default.py": "c63dc2697c61184e496d2fbed9808ccac23fb706a7dcccf34292da2b8e0079b9", + "Runner/LocalAgent/components/runner/default.yaml": "c1364cf0f211162bb701e27ed92b351fd9ff4cfa0b4469cfeb83ad58136cdf47", + "Runner/LocalAgent/main.py": "0cde45d7f6f411563fbded09be70d0a497c7c7999b5eaa27f83052383348277f", + "Runner/LocalAgent/manifest.yaml": "e71e03f2670dade9ae43e58ddf787bb87254ba9f05e1a691822679f3641bb885", + "Runner/LocalAgent/pkg/__init__.py": "ce5142b4971e8d7945fb3c202bacbc28011b033de0ddcb612d5d29106d6985ec", + "Runner/LocalAgent/pkg/agent_core/__init__.py": "56e95f2391f796e17aee6a168603853ea78b59df97a136bfa33532aede2fa413", + "Runner/LocalAgent/pkg/agent_core/langbot.py": "dc16818ac81dacf37dd6ee799dec4fcd8235ffab4602ede30d29633e78626fc0", + "Runner/LocalAgent/pkg/agent_core/loop.py": "47451d5935d79b0a632fc87f3d94a7d0ecf1b50621141ea8b1031dd87755d5ad", + "Runner/LocalAgent/pkg/agent_core/types.py": "df82b1a17cd2b73f65ebbde22ab8f204ffd27b8d68814b404a1ecce9988e1fcf", + "Runner/LocalAgent/pkg/box.py": "82c05a55bd59b41e844f4613563e518768921fe4fd34bcb4c14c86e395ccd021", + "Runner/LocalAgent/pkg/config.py": "8dee31aecbcdba4c1e79e1735499c3c7ee826ec395032b045169652b38216faf", + "Runner/LocalAgent/pkg/context_pipeline.py": "4c6da3c48faf28985b8bad013fe15106705cd3e025f0c55708b463e4ea07eb05", + "Runner/LocalAgent/pkg/messages.py": "620a0a225e95c5f2c8c6e977801413e553d4a1da3fb2ddea14e527ad41f7a3af", + "Runner/LocalAgent/pkg/model_calling.py": "29989b102c8f649e1e65c7c5719761ed39d357d37d57bfa321976d48fb11b76c", + "Runner/LocalAgent/pkg/rag.py": "ea407af8d4f6fe5b00cde72c7d85ccc541d6fc46254188c372351d93da4a1f0f", + "Runner/LocalAgent/pkg/run_assembly.py": "ab08c6a18a108cfc6048f158b09ad874577451072dce0bbaa3aa9d15c33a3afe", + "Runner/LocalAgent/pyproject.toml": "ef14f2ed5d4d78591d2f1c0ac0f05856d558dae71da330a203abf3160356e6b1", + "Runner/LocalAgent/requirements.txt": "dd4542c179fa98c8341e7a0b582eb741a70633cd621f4509d09a9ff5c7e52c76", + "Runner/LocalAgent/uv.lock": "4c3c7655ef8c3984e7b720dea346ea67b4d29e8c6d543037779008c3bca074a8" } } diff --git a/tests/utils/legacy_plugin_contract.py b/tests/utils/legacy_plugin_contract.py index 0056ce05a..5d4fb15d8 100644 --- a/tests/utils/legacy_plugin_contract.py +++ b/tests/utils/legacy_plugin_contract.py @@ -1,4 +1,4 @@ -"""Load identity consumers from integrity-checked official plugin source.""" +"""Load contract fixtures from integrity-checked official plugin source.""" import ast import hashlib @@ -14,7 +14,7 @@ from langbot_plugin.api.entities.builtin.runner.context import RunnerContext SOURCE_LOCK = Path(__file__).resolve().parents[1] / 'fixtures' / 'legacy_plugin_identity_sources.json' -def verified_plugin_root() -> Path: +def verified_plugin_root(*, files_key: str = 'files') -> Path: """Allow optional local execution, but never skip an explicitly provisioned source.""" configured = os.environ.get('LANGBOT_LEGACY_PLUGIN_ROOT') if configured is None: @@ -23,7 +23,7 @@ def verified_plugin_root() -> Path: pytest.fail('LANGBOT_LEGACY_PLUGIN_ROOT is set but empty', pytrace=False) root = Path(configured).resolve() lock = json.loads(SOURCE_LOCK.read_text()) - for relative, expected in lock['files'].items(): + for relative, expected in lock[files_key].items(): path = root / relative if not path.resolve().is_relative_to(root) or not path.is_file(): pytest.fail(f'Official plugin fixture is missing or outside its root: {relative}', pytrace=False) From f2c0a437f862c34bb3d12586eb63815029bbb2e7 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sat, 17 Oct 2026 19:56:53 +0800 Subject: [PATCH 2/2] test(ci): require PostgreSQL monitoring session contract --- .github/workflows/run-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 59bfaf991..d0e5e1998 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -155,6 +155,7 @@ jobs: tests/integration/persistence/test_rag_interrupted_ingestion.py \ tests/integration/persistence/test_migration_branch_convergence.py \ tests/integration/persistence/test_runner_timestamps_postgres.py \ + tests/unit_tests/api/service/test_monitoring_sessions.py \ -k postgres -q --tb=short -rs --durations=10 \ --junitxml=postgres-contract-results.xml python - <<'PY' @@ -168,6 +169,7 @@ jobs: 'test_rag_interrupted_ingestion': 22, 'test_migration_branch_convergence': 7, 'test_runner_timestamps_postgres': 32, + 'test_monitoring_sessions': 1, } counts = Counter(case.get('classname', '').rsplit('.', 1)[-1] for case in cases) for module, minimum in expected.items():