Merge pull request #2705 from langbot-app/test/required-contract-suites

test(ci): decouple required suites from test names and counts

.github/workflows/optional-contracts.yml

@@ -9,7 +9,6 @@ on:
       - 'pyproject.toml'
       - 'pytest.ini'
       - 'uv.lock'
-      - 'scripts/assert-required-tests.py'
       - '.github/workflows/optional-contracts.yml'
   push:
     branches: [master, develop]
@@ -19,7 +18,6 @@ on:
       - 'pyproject.toml'
       - 'pytest.ini'
       - 'uv.lock'
-      - 'scripts/assert-required-tests.py'
       - '.github/workflows/optional-contracts.yml'
   workflow_dispatch:
 
@@ -87,12 +85,8 @@ jobs:
         env:
           LANGBOT_CLI_BIN: ${{ runner.temp }}/langbot-cli/lbctl
         run: |
-          uv run --no-sync pytest \
-            tests/unit_tests/api/service/test_cli_diagnostics.py::test_compiled_cli_against_core_http \
+          uv run --no-sync pytest --required-suite compiled_cli \
             -q --tb=short -rs --junitxml=compiled-cli-results.xml
-          python scripts/assert-required-tests.py compiled-cli-results.xml \
-            --module tests.unit_tests.api.service.test_cli_diagnostics \
-            --name-prefix test_compiled_cli_against_core_http --expected-tests 1
 
       - name: Upload compiled CLI contract result
         if: always()
@@ -151,12 +145,8 @@ jobs:
         env:
           LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins
         run: |
-          uv run --no-sync pytest \
-            tests/unit_tests/agent/test_legacy_identity_boundary.py::test_real_plugin_identity_helper_with_host_payload \
+          uv run --no-sync pytest --required-suite plugin_identity \
             -q --tb=short -rs --junitxml=plugin-identity-results.xml
-          python scripts/assert-required-tests.py plugin-identity-results.xml \
-            --module tests.unit_tests.agent.test_legacy_identity_boundary \
-            --name-prefix test_real_plugin_identity_helper_with_host_payload --expected-tests 16
 
       - name: Upload plugin identity results
         if: always()
@@ -198,11 +188,9 @@ jobs:
 
           print(f"pyseekdb={version('pyseekdb')}, pylibseekdb={version('pylibseekdb')}")
           PY
-          uv run --no-sync pytest tests/integration/vector/test_seekdb.py \
-            -m slow -q --tb=short -rs --durations=5 \
+          uv run --no-sync pytest --required-suite seekdb \
+            -q --tb=short -rs --durations=5 \
             --basetemp="$SEEKDB_TEST_TMP" --junitxml=seekdb-contract-results.xml
-          python scripts/assert-required-tests.py seekdb-contract-results.xml \
-            --module tests.integration.vector.test_seekdb --name-prefix test_ --expected-tests 2
 
       - name: Upload SeekDB results and startup logs
         if: always()

.github/workflows/run-tests.yml

@@ -11,7 +11,7 @@ on:
       - 'uv.lock'
       - 'run_tests.sh'
       - 'scripts/test-*.sh'
-      - 'scripts/assert-required-tests.py'
+      - 'pytest.ini'
   push:
     branches:
       - master
@@ -50,11 +50,8 @@ jobs:
 
       - name: Run required security regressions
         run: |
-          uv run pytest tests/test_cwe94_debug_exec.py \
+          uv run pytest --required-suite security \
             -q --tb=short -rs --junitxml=cwe94-results.xml
-          python scripts/assert-required-tests.py cwe94-results.xml \
-            --module tests.test_cwe94_debug_exec \
-            --name-prefix test_no_ --expected-tests 2
 
       - name: Upload security regression results
         if: always()
@@ -149,45 +146,8 @@ jobs:
 
       - name: Run required PostgreSQL contracts
         run: |
-          uv run pytest \
-            tests/integration/persistence/test_pipeline_admission_postgres.py \
-            tests/integration/persistence/test_rag_document_identity.py \
-            tests/integration/persistence/test_rag_interrupted_ingestion.py \
-            tests/integration/persistence/test_migration_branch_convergence.py \
-            tests/integration/persistence/test_runner_timestamps_postgres.py \
-            tests/unit_tests/api/service/test_monitoring_sessions.py \
-            -k postgres -q --tb=short -rs --durations=10 \
-            --junitxml=postgres-contract-results.xml
-          python - <<'PY'
-          from collections import Counter
-          import xml.etree.ElementTree as ET
-
-          cases = list(ET.parse('postgres-contract-results.xml').getroot().iter('testcase'))
-          expected = {
-              'test_pipeline_admission_postgres': 27,
-              'test_rag_document_identity': 34,
-              'test_rag_interrupted_ingestion': 22,
-              'test_migration_branch_convergence': 6,
-              'test_runner_timestamps_postgres': 32,
-              'test_monitoring_sessions': 1,
-          }
-          counts = Counter(case.get('classname', '').rsplit('.', 1)[-1] for case in cases)
-          for module, minimum in expected.items():
-              assert counts[module] >= minimum, f'{module}: expected at least {minimum} cases, got {counts[module]}'
-          convergence_cases = {
-              case.get('name') for case in cases
-              if case.get('classname', '').rsplit('.', 1)[-1] == 'test_migration_branch_convergence'
-          }
-          assert 'test_empty_database_startup_schema_then_real_migrations[postgresql]' in convergence_cases, (
-              'The fresh-schema migration and idempotency contract must execute'
-          )
-          unsuccessful = [
-              case.get('name') for case in cases
-              if any(case.find(tag) is not None for tag in ('skipped', 'failure', 'error'))
-          ]
-          assert not unsuccessful, f'PostgreSQL contracts must pass without skips: {unsuccessful}'
-          print(f'PostgreSQL contracts: {len(cases)} passed, 0 skipped; {dict(counts)}')
-          PY
+          uv run pytest --required-suite postgres \
+            -q --tb=short -rs --durations=10 --junitxml=postgres-contract-results.xml
 
       - name: Upload PostgreSQL contract results
         if: always()
@@ -241,7 +201,7 @@ jobs:
         run: uv sync --dev
 
       - name: Run E2E startup tests
-        run: uv run pytest tests/e2e --ignore=tests/e2e/test_local_runner_fake_provider.py -q --tb=short
+        run: uv run pytest tests/e2e -m "not local_agent" -q --tb=short
 
       - name: Check Docker for the required LocalAgent Box contract
         run: docker info
@@ -250,11 +210,8 @@ jobs:
         env:
           LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins
         run: |
-          uv run --no-sync pytest tests/e2e/test_local_runner_fake_provider.py \
+          uv run --no-sync pytest --required-suite local_agent \
             -q --tb=short -rs --junitxml=local-agent-results.xml
-          python scripts/assert-required-tests.py local-agent-results.xml \
-            --module tests.e2e.test_local_runner_fake_provider \
-            --name-prefix test_local_runner_ --expected-tests 6
 
       - name: Upload LocalAgent contract results
         if: always()