Merge pull request #2704 from langbot-app/test/quality-local-runner-execution-20261017

test: require LocalAgent and monitoring PostgreSQL contracts in CI
2026-10-22 08:25:21 +08:00 · 2026-10-17 20:36:33 +08:00
parent 3a026cc7dc ab29e83ca0
commit eaaf44f87a
5 changed files with 119 additions and 31 deletions
@@ -155,6 +155,7 @@ jobs:
            tests/integration/persistence/test_rag_interrupted_ingestion.py \
            tests/integration/persistence/test_migration_branch_convergence.py \
            tests/integration/persistence/test_runner_timestamps_postgres.py \
+            tests/unit_tests/api/service/test_monitoring_sessions.py \
            -k postgres -q --tb=short -rs --durations=10 \
            --junitxml=postgres-contract-results.xml
          python - <<'PY'
@@ -168,6 +169,7 @@ jobs:
              'test_rag_interrupted_ingestion': 22,
              'test_migration_branch_convergence': 6,
              'test_runner_timestamps_postgres': 32,
+              'test_monitoring_sessions': 1,
          }
          counts = Counter(case.get('classname', '').rsplit('.', 1)[-1] for case in cases)
          for module, minimum in expected.items():
@@ -203,6 +205,30 @@ jobs:
      - name: Checkout code
        uses: actions/checkout@v4

+      - name: Read official plugin source pin
+        id: plugin-source
+        run: |
+          python3 - <<'PY'
+          import json
+          import os
+          import re
+          from pathlib import Path
+
+          lock = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())
+          assert lock['repository'] == 'langbot-app/langbot-plugins'
+          assert re.fullmatch(r'[0-9a-f]{40}', lock['revision'])
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
+              output.write(f"revision={lock['revision']}\n")
+          PY
+
+      - name: Checkout pinned official plugin sources
+        uses: actions/checkout@v4
+        with:
+          repository: langbot-app/langbot-plugins
+          ref: ${{ steps.plugin-source.outputs.revision }}
+          path: .contract-sources/langbot-plugins
+          persist-credentials: false
+
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
@@ -215,7 +241,28 @@ jobs:
        run: uv sync --dev

      - name: Run E2E startup tests
-        run: uv run pytest tests/e2e -q --tb=short
+        run: uv run pytest tests/e2e --ignore=tests/e2e/test_local_runner_fake_provider.py -q --tb=short
+
+      - name: Check Docker for the required LocalAgent Box contract
+        run: docker info
+
+      - name: Run required LocalAgent contracts
+        env:
+          LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins
+        run: |
+          uv run --no-sync pytest tests/e2e/test_local_runner_fake_provider.py \
+            -q --tb=short -rs --junitxml=local-agent-results.xml
+          python scripts/assert-required-tests.py local-agent-results.xml \
+            --module tests.e2e.test_local_runner_fake_provider \
+            --name-prefix test_local_runner_ --expected-tests 6
+
+      - name: Upload LocalAgent contract results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: local-agent-contract-results
+          path: local-agent-results.xml
+          if-no-files-found: ignore

      - name: E2E Test Summary
        if: always()
@@ -1,17 +0,0 @@
-from . import client
-
-import asyncio
-
-import os
-
-
-class TestDifyClient:
-    async def test_chat_messages(self):
-        cln = client.DifyClient(api_key=os.getenv('DIFY_API_KEY'))
-
-        resp = await cln.chat_messages(inputs={}, query='Who are you?', user_id='test')
-        print(resp)
-
-
-if __name__ == '__main__':
-    asyncio.run(TestDifyClient().test_chat_messages())
@@ -1,7 +1,7 @@
 """E2E coverage for the official Local Agent runner with fake Host resources.

 These tests start the real LangBot application and the real SDK Plugin Runtime,
-load the consolidated ``langbot-plugin-demo/Runner/LocalAgent`` plugin, and verify Local Agent paths
+load the pinned official ``langbot-plugins/Runner/LocalAgent`` plugin, and verify Local Agent paths
 that must cross Host run-scoped APIs without calling any external provider.
 """

@@ -13,6 +13,7 @@ import os
 import shutil
 import socket
 import sqlite3
+import stat
 import subprocess
 import sys
 import tempfile
@@ -23,7 +24,7 @@ from typing import Any
 import pytest

 from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories
-from tests.e2e.utils.process_manager import find_project_root
+from tests.utils.legacy_plugin_contract import verified_plugin_root

 pytestmark = pytest.mark.e2e

@@ -43,16 +44,15 @@ def _free_port() -> int:


 def _local_agent_repo() -> Path:
-    """Return the sibling local-agent repository used by this workspace E2E."""
-    project_root = find_project_root()
-    return project_root.parent / 'langbot-plugin-demo' / 'Runner' / 'LocalAgent'
+    """Use the same official source pin as the plugin identity contracts."""
+    return verified_plugin_root(files_key='local_agent_files') / 'Runner' / 'LocalAgent'


 def _package_local_agent_plugin(tmpdir: Path) -> Path:
-    """Package the sibling Local Agent plugin for the real local-install flow."""
+    """Package the verified official plugin for the real local-install flow."""
    local_agent_src = _local_agent_repo()
    if not (local_agent_src / 'manifest.yaml').exists():
-        pytest.skip(f'local-agent repository not found at {local_agent_src}')
+        pytest.fail(f'LocalAgent manifest not found at {local_agent_src}', pytrace=False)

    package_source = tmpdir / 'local-agent-package'
    ignore = shutil.ignore_patterns(
@@ -275,7 +275,12 @@ def local_agent_e2e_tmpdir():
    """Create temporary directory for Local Agent E2E testing."""
    tmpdir = Path(tempfile.mkdtemp(prefix='langbot_local_agent_e2e_'))
    yield tmpdir
-    shutil.rmtree(tmpdir, ignore_errors=True)
+    # The runtime seals its artifact/dependency directories as read-only.
+    # Restore only this fixture's own directories so teardown removes them.
+    for path in tmpdir.rglob('*'):
+        if path.is_dir() and not path.is_symlink():
+            path.chmod(path.stat().st_mode | stat.S_IWUSR)
+    shutil.rmtree(tmpdir)


@pytest.fixture(scope='session')
@@ -1050,6 +1055,7 @@ def test_local_runner_owns_box_reuse_and_explicit_files(
            super().__init__()
            self.box = box
            self.bindings = []
+            self.outboxes = []

        async def get_resolved_tool_catalog(self, *args, **kwargs):
            return [{'name': 'exec', 'source': 'native', 'source_id': None}]
@@ -1066,19 +1072,31 @@ def test_local_runner_owns_box_reuse_and_explicit_files(
            self.bindings.append((binding.session_id, binding.run_id))
            source = next(iter(binding.imported.values()))['path']
            outbox = f'/workspace/outbox/{binding.io_scope}'
+            self.outboxes.append(outbox)
            # Shell arguments are Host-issued paths, never model input.
            import shlex

+            directory_mode = '-m 700 ' if len(self.bindings) == 2 else ''
            result = await self.box.execute_tool(
                {
-                    'command': f'mkdir -p {shlex.quote(outbox)} && cp {shlex.quote(source)} {shlex.quote(outbox + "/answer.txt")}'
+                    'command': (
+                        f'mkdir {directory_mode}-p {shlex.quote(outbox)} && '
+                        f'cp {shlex.quote(source)} {shlex.quote(outbox + "/answer.txt")} && '
+                        f"stat -c '%u %a' {shlex.quote(outbox)} {shlex.quote(outbox + '/answer.txt')}"
+                    )
                },
                query,
            )
            assert result['ok'], result
+            ownership = result['stdout'].strip().splitlines()
+            assert ownership[-1] == '0 600', ownership
+            if len(self.bindings) == 2:
+                assert ownership[0] == '0 700', ownership
            return result

    async def probe(ap):
+        import shlex
+
        backend = TestDockerBackend(ap.logger)
        if not await backend.is_available():
            pytest.skip('Docker is required for the real Box probe')
@@ -1120,6 +1138,12 @@ def test_local_runner_owns_box_reuse_and_explicit_files(
                output = [component for message in messages for component in (message.attachments or [])]
                assert len(output) == 1, messages
                assert base64.b64decode(output[0].base64) == f'run-{index}'.encode()
+                inspection = await box.execute_in_context(
+                    context,
+                    {'cmd': f'ls -A {shlex.quote(manager.outboxes[-1])}', 'session_id': manager.bindings[-1][0]},
+                )
+                assert inspection.exit_code == 0, inspection
+                assert inspection.stdout.strip() == '', inspection.stdout
                status = await RunnerBoxService(box).status(context)
                assert status['used'] == 1 and status['remaining'] == 0, status
            assert manager.bindings[0][0] == manager.bindings[1][0]
@@ -1129,6 +1153,18 @@ def test_local_runner_owns_box_reuse_and_explicit_files(
            with pytest.raises(BoxCapacityExceededError):
                await RunnerBoxService(box).acquire(context, {'reuse_key': 'different'})
        finally:
-            await box.shutdown()
+            try:
+                if manager.bindings:
+                    # Only this fixture's temporary Workspace is mounted here.
+                    cleanup = await box.execute_in_context(
+                        context,
+                        {
+                            'cmd': f'chown -hR {os.getuid()}:{os.getgid()} /workspace/outbox',
+                            'session_id': manager.bindings[-1][0],
+                        },
+                    )
+                    assert cleanup.exit_code == 0, cleanup
+            finally:
+                await box.shutdown()

    _run_local_agent_probe(local_agent_e2e_tmpdir, probe)
@@ -15,5 +15,27 @@
    "Runner/tbox-agent/components/runner/default.py": "4c9fdf8cc10c916915036112ded8768a619a827ca4b5e6a3f05881a7d0e4492f",
    "Runner/tbox-agent/pkg/scoped_identity.py": "5d03d0c56aaf3301bc5d594905ecf862151fa2abc73daba1a6596f0fdc16374a",
    "Runner/tbox-agent/pkg/tbox_client.py": "0f09a3c70f773bc6377fd5b55a41d186dd15487d46aab3e61b837c08e15100a0"
+  },
+  "local_agent_files": {
+    "Runner/LocalAgent/assets/icon.svg": "7e1cb51793e898fca810ec72ba6fe9af377074f52139cdd56c93e87f3459f964",
+    "Runner/LocalAgent/components/runner/default.py": "c63dc2697c61184e496d2fbed9808ccac23fb706a7dcccf34292da2b8e0079b9",
+    "Runner/LocalAgent/components/runner/default.yaml": "c1364cf0f211162bb701e27ed92b351fd9ff4cfa0b4469cfeb83ad58136cdf47",
+    "Runner/LocalAgent/main.py": "0cde45d7f6f411563fbded09be70d0a497c7c7999b5eaa27f83052383348277f",
+    "Runner/LocalAgent/manifest.yaml": "e71e03f2670dade9ae43e58ddf787bb87254ba9f05e1a691822679f3641bb885",
+    "Runner/LocalAgent/pkg/__init__.py": "ce5142b4971e8d7945fb3c202bacbc28011b033de0ddcb612d5d29106d6985ec",
+    "Runner/LocalAgent/pkg/agent_core/__init__.py": "56e95f2391f796e17aee6a168603853ea78b59df97a136bfa33532aede2fa413",
+    "Runner/LocalAgent/pkg/agent_core/langbot.py": "dc16818ac81dacf37dd6ee799dec4fcd8235ffab4602ede30d29633e78626fc0",
+    "Runner/LocalAgent/pkg/agent_core/loop.py": "47451d5935d79b0a632fc87f3d94a7d0ecf1b50621141ea8b1031dd87755d5ad",
+    "Runner/LocalAgent/pkg/agent_core/types.py": "df82b1a17cd2b73f65ebbde22ab8f204ffd27b8d68814b404a1ecce9988e1fcf",
+    "Runner/LocalAgent/pkg/box.py": "82c05a55bd59b41e844f4613563e518768921fe4fd34bcb4c14c86e395ccd021",
+    "Runner/LocalAgent/pkg/config.py": "8dee31aecbcdba4c1e79e1735499c3c7ee826ec395032b045169652b38216faf",
+    "Runner/LocalAgent/pkg/context_pipeline.py": "4c6da3c48faf28985b8bad013fe15106705cd3e025f0c55708b463e4ea07eb05",
+    "Runner/LocalAgent/pkg/messages.py": "620a0a225e95c5f2c8c6e977801413e553d4a1da3fb2ddea14e527ad41f7a3af",
+    "Runner/LocalAgent/pkg/model_calling.py": "29989b102c8f649e1e65c7c5719761ed39d357d37d57bfa321976d48fb11b76c",
+    "Runner/LocalAgent/pkg/rag.py": "ea407af8d4f6fe5b00cde72c7d85ccc541d6fc46254188c372351d93da4a1f0f",
+    "Runner/LocalAgent/pkg/run_assembly.py": "ab08c6a18a108cfc6048f158b09ad874577451072dce0bbaa3aa9d15c33a3afe",
+    "Runner/LocalAgent/pyproject.toml": "ef14f2ed5d4d78591d2f1c0ac0f05856d558dae71da330a203abf3160356e6b1",
+    "Runner/LocalAgent/requirements.txt": "dd4542c179fa98c8341e7a0b582eb741a70633cd621f4509d09a9ff5c7e52c76",
+    "Runner/LocalAgent/uv.lock": "4c3c7655ef8c3984e7b720dea346ea67b4d29e8c6d543037779008c3bca074a8"
  }
 }
@@ -1,4 +1,4 @@
-"""Load identity consumers from integrity-checked official plugin source."""
+"""Load contract fixtures from integrity-checked official plugin source."""

 import ast
 import hashlib
@@ -14,7 +14,7 @@ from langbot_plugin.api.entities.builtin.runner.context import RunnerContext
 SOURCE_LOCK = Path(__file__).resolve().parents[1] / 'fixtures' / 'legacy_plugin_identity_sources.json'


-def verified_plugin_root() -> Path:
+def verified_plugin_root(*, files_key: str = 'files') -> Path:
    """Allow optional local execution, but never skip an explicitly provisioned source."""
    configured = os.environ.get('LANGBOT_LEGACY_PLUGIN_ROOT')
    if configured is None:
@@ -23,7 +23,7 @@ def verified_plugin_root() -> Path:
        pytest.fail('LANGBOT_LEGACY_PLUGIN_ROOT is set but empty', pytrace=False)
    root = Path(configured).resolve()
    lock = json.loads(SOURCE_LOCK.read_text())
-    for relative, expected in lock['files'].items():
+    for relative, expected in lock[files_key].items():
        path = root / relative
        if not path.resolve().is_relative_to(root) or not path.is_file():
            pytest.fail(f'Official plugin fixture is missing or outside its root: {relative}', pytrace=False)