Merge pull request #2705 from langbot-app/test/required-contract-suites

test(ci): decouple required suites from test names and counts

tests/conftest.py

@@ -0,0 +1,3 @@
+"""Repository-wide test execution policy."""
+
+pytest_plugins = ['tests.utils.required_suites']

tests/e2e/test_local_runner_fake_provider.py

@@ -26,7 +26,7 @@ import pytest
 from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories
 from tests.utils.legacy_plugin_contract import verified_plugin_root
 
-pytestmark = pytest.mark.e2e
+pytestmark = [pytest.mark.e2e, pytest.mark.local_agent]
 
 
 LOCAL_RUNNER_ID = 'plugin:langbot-team/LocalAgent/default'
@@ -557,6 +557,7 @@ def _run_local_agent_probe(tmpdir: Path, probe):
     return asyncio.run(_run())
 
 
+@pytest.mark.contract('agent_ledger')
 def test_local_runner_uses_host_fake_provider_and_persists_ledger(
     local_agent_e2e_tmpdir,
     local_agent_e2e_config_path,
@@ -619,6 +620,7 @@ def test_local_runner_uses_host_fake_provider_and_persists_ledger(
         conn.close()
 
 
+@pytest.mark.contract('agent_tools')
 def test_local_runner_executes_authorized_tool_loop_through_host_action(
     local_agent_e2e_tmpdir,
     local_agent_e2e_config_path,
@@ -688,6 +690,7 @@ def test_local_runner_executes_authorized_tool_loop_through_host_action(
         conn.close()
 
 
+@pytest.mark.contract('agent_rag')
 def test_local_runner_retrieves_authorized_rag_context_through_host_action(
     local_agent_e2e_tmpdir,
     local_agent_e2e_config_path,
@@ -762,6 +765,7 @@ def test_local_runner_retrieves_authorized_rag_context_through_host_action(
         conn.close()
 
 
+@pytest.mark.contract('agent_compaction')
 def test_local_runner_compacts_history_and_persists_checkpoint(
     local_agent_e2e_tmpdir,
     local_agent_e2e_config_path,
@@ -855,6 +859,7 @@ def test_local_runner_compacts_history_and_persists_checkpoint(
         conn.close()
 
 
+@pytest.mark.contract('agent_multi_turn')
 def test_local_runner_combines_rag_compaction_and_multi_turn_tool_loop(
     local_agent_e2e_tmpdir,
     local_agent_e2e_config_path,
@@ -1017,6 +1022,7 @@ def test_local_runner_combines_rag_compaction_and_multi_turn_tool_loop(
         conn.close()
 
 
+@pytest.mark.contract('agent_box')
 def test_local_runner_owns_box_reuse_and_explicit_files(
     local_agent_e2e_tmpdir,
     local_agent_e2e_config_path,

tests/integration/persistence/test_migration_branch_convergence.py

@@ -39,7 +39,7 @@ ACCOUNT = '41100000-0000-4000-8000-000000000002'
 NOW = datetime.datetime(2026, 9, 1)
 
 
-@pytest.fixture(params=['sqlite', pytest.param('postgresql', marks=pytest.mark.slow)])
+@pytest.fixture(params=['sqlite', pytest.param('postgresql', marks=[pytest.mark.slow, pytest.mark.postgres])])
 async def convergence_engine(request, tmp_path):
     if request.param == 'sqlite':
         engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "convergence.db"}')
@@ -354,6 +354,7 @@ async def _assert_merged_schema_and_data(engine, source):
         ('beta', None),
     ],
 )
+@pytest.mark.contract('migration_history')
 async def test_historical_schema_and_populated_branch_upgrade(convergence_engine, source, checkpoint):
     engine = convergence_engine
     fixture = json.loads((FIXTURES / f'{source}_schema.json').read_text())
@@ -395,6 +396,7 @@ async def test_historical_schema_and_populated_branch_upgrade(convergence_engine
 
 
 @pytest.mark.asyncio
+@pytest.mark.contract('migration_fresh_schema')
 async def test_empty_database_startup_schema_then_real_migrations(convergence_engine):
     engine = convergence_engine
     async with engine.begin() as conn:

tests/integration/persistence/test_pipeline_admission_postgres.py

@@ -12,7 +12,11 @@ from langbot.pkg.persistence.pipeline_admission import lock_pipeline_admission
 from langbot.pkg.entity.persistence.agent_interaction import AgentInteraction
 
 URL = os.environ.get('LANGBOT_ADMISSION_TEST_URL')
-pytestmark = [pytest.mark.asyncio, pytest.mark.skipif(not URL, reason='disposable PostgreSQL URL required')]
+pytestmark = [
+    pytest.mark.postgres,
+    pytest.mark.asyncio,
+    pytest.mark.skipif(not URL, reason='disposable PostgreSQL URL required'),
+]
 
 
 @pytest.fixture
@@ -111,6 +115,7 @@ async def test_real_writer_waits_and_revalidates_after_transaction(env, outcome)
         assert (await writer)[0]['status'] == 'pending'
 
 
+@pytest.mark.contract('admission_isolation')
 async def test_real_writer_first_blocks_migration_and_other_tenant_isolated(env):
     await write(env)
     async with env.pm.tenant_uow(base.WS) as uow:

tests/integration/persistence/test_rag_document_identity.py

@@ -54,7 +54,7 @@ DOCUMENT_IDENTITY_REVISION = '0025_rag_document_identity'
 CONTEXT = ExecutionContext(instance_uuid='instance-a', workspace_uuid='workspace-a', placement_generation=5)
 
 
-@pytest_asyncio.fixture(params=['sqlite', 'postgres'])
+@pytest_asyncio.fixture(params=['sqlite', pytest.param('postgres', marks=pytest.mark.postgres)])
 async def database(request, tmp_path):
     admin = None
     schema = 'ke_identity_' + uuid.uuid4().hex
@@ -183,6 +183,7 @@ async def read_file_row(runtime, file_id='host-id'):
 @pytest.mark.parametrize(
     'engine_id', ['dify-upstream', 'fastgpt-collection', 'ragflow-upstream', 'host-id', ' opaque ID ']
 )
+@pytest.mark.contract('rag_identity')
 async def test_ingest_persists_exact_engine_identity_and_restart_delete(runtime, engine_id):
     file = await seed(runtime)
     runtime.ap.plugin_connector.call_rag_ingest.return_value['document_id'] = engine_id

tests/integration/persistence/test_rag_interrupted_ingestion.py

@@ -25,6 +25,7 @@ def attach_task_manager(runtime):
 
 
 @pytest.mark.asyncio
+@pytest.mark.contract('rag_recovery')
 async def test_cancelled_ingestion_retains_recovery_resources(runtime):
     """Local cancellation is interrupted observation, not remote quiescence."""
     file = await seed(runtime)

tests/integration/persistence/test_runner_timestamps_postgres.py

@@ -25,7 +25,12 @@ from langbot.pkg.entity.persistence.base import Base
 from langbot.pkg.entity.persistence.event_log import EventLog
 from langbot.pkg.entity.persistence.transcript import Transcript
 
-pytestmark = [pytest.mark.integration, pytest.mark.slow, pytest.mark.asyncio]
+pytestmark = [
+    pytest.mark.postgres,
+    pytest.mark.integration,
+    pytest.mark.slow,
+    pytest.mark.asyncio,
+]
 
 UTC = dt.timezone.utc
 NOW = dt.datetime(2026, 9, 22, 4, 5, 6, 123000, tzinfo=UTC)
@@ -110,6 +115,7 @@ async def _legacy_run(engine, *, status='queued'):
         )
 
 
+@pytest.mark.contract('runner_timestamps')
 async def test_run_lifecycle_deadline_events_and_stats(pg_engine):
     store = run_ledger_store.RunLedgerStore(pg_engine)
     deadline = NOW + dt.timedelta(minutes=5)

tests/integration/vector/test_seekdb.py

@@ -14,16 +14,14 @@ import uuid
 
 import pytest
 
-pytest.importorskip('pyseekdb')
-
-from langbot.pkg.vector.vdbs.seekdb import SeekDBVectorDatabase
-
-
-pytestmark = [pytest.mark.integration, pytest.mark.slow]
+pytestmark = [pytest.mark.integration, pytest.mark.slow, pytest.mark.seekdb]
 
 
 @pytest.fixture
 async def backend(tmp_path):
+    pytest.importorskip('pyseekdb')
+    from langbot.pkg.vector.vdbs.seekdb import SeekDBVectorDatabase
+
     app = SimpleNamespace(
         instance_config=SimpleNamespace(
             data={
@@ -51,6 +49,7 @@ async def backend(tmp_path):
 
 
 @pytest.mark.asyncio
+@pytest.mark.contract('vector_round_trip')
 async def test_upsert_and_text_round_trip(backend) -> None:
     database, collection = backend
     original = 'He said "hello".\nC:\\notes\\file.txt isn\'t empty. 中文'
@@ -80,6 +79,7 @@ async def test_upsert_and_text_round_trip(backend) -> None:
 
 
 @pytest.mark.asyncio
+@pytest.mark.contract('hybrid_relevance')
 async def test_full_text_and_hybrid_results_keep_relevance_order(backend) -> None:
     database, collection = backend
     documents = [

tests/test_cwe94_debug_exec.py

@@ -12,12 +12,17 @@ This test verifies that:
 import ast
 import pathlib
 
+import pytest
+
+pytestmark = pytest.mark.security
+
 # Resolve project root (one level up from tests/)
 _PROJECT_ROOT = pathlib.Path(__file__).resolve().parent.parent
 
 VULN_FILE = _PROJECT_ROOT / 'src' / 'langbot' / 'pkg' / 'api' / 'http' / 'controller' / 'groups' / 'system.py'
 
 
+@pytest.mark.contract('debug_exec_absent')
 def test_no_exec_call_in_system_controller():
     """Verify there is no exec() call in system.py that takes user input."""
     with open(VULN_FILE, 'r') as f:
@@ -38,6 +43,7 @@ def test_no_exec_call_in_system_controller():
     )
 
 
+@pytest.mark.contract('debug_route_absent')
 def test_no_debug_exec_route():
     """Verify the /debug/exec route is not registered."""
     with open(VULN_FILE, 'r') as f:

tests/unit_tests/agent/test_legacy_identity_boundary.py

@@ -205,14 +205,43 @@ def official_plugin_root():
 @pytest.mark.parametrize(
     'runner,folder,method,error_name,local_identity',
     [
-        ('DifyAgent', 'dify-agent', '_get_user_tag', 'DifyConfigError', 'person_Session_007'),
-        ('N8nAgent', 'n8n-agent', '_get_user_tag', 'N8nConfigError', 'person_Session_007'),
-        ('CozeAgent', 'coze-agent', '_get_user_id', 'CozeConfigError', 'person_launcher-123'),
-        ('TboxAgent', 'tbox-agent', '_get_user_id', 'TboxConfigError', 'bot-uuid-123'),
+        pytest.param(
+            'DifyAgent',
+            'dify-agent',
+            '_get_user_tag',
+            'DifyConfigError',
+            'person_Session_007',
+            marks=pytest.mark.contract('dify_identity'),
+        ),
+        pytest.param(
+            'N8nAgent',
+            'n8n-agent',
+            '_get_user_tag',
+            'N8nConfigError',
+            'person_Session_007',
+            marks=pytest.mark.contract('n8n_identity'),
+        ),
+        pytest.param(
+            'CozeAgent',
+            'coze-agent',
+            '_get_user_id',
+            'CozeConfigError',
+            'person_launcher-123',
+            marks=pytest.mark.contract('coze_identity'),
+        ),
+        pytest.param(
+            'TboxAgent',
+            'tbox-agent',
+            '_get_user_id',
+            'TboxConfigError',
+            'bot-uuid-123',
+            marks=pytest.mark.contract('tbox_identity'),
+        ),
     ],
 )
 @pytest.mark.parametrize('missing', [False, True], ids=['trusted-identity', 'missing-identity'])
 @pytest.mark.parametrize('profile', ['dedicated', 'shared'])
+@pytest.mark.plugin_identity
 async def test_real_plugin_identity_helper_with_host_payload(
     mock_query, monkeypatch, official_plugin_root, runner, folder, method, error_name, local_identity, missing, profile
 ):

tests/unit_tests/api/service/test_cli_diagnostics.py

@@ -236,6 +236,8 @@ async def test_extension_discovery_preserves_empty_bindings_and_redacts_plugins(
     ap.plugin_connector.require_workspace_context.assert_awaited_once_with(request_context())
 
 
+@pytest.mark.compiled_cli
+@pytest.mark.contract('cli_http')
 async def test_compiled_cli_against_core_http(service, tmp_path):  # noqa: F811
     """Optional cross-repo check: real HTTP/SQLite/Pipeline, synthetic runtime providers."""
     import json

tests/unit_tests/api/service/test_monitoring_sessions.py

@@ -21,6 +21,8 @@ pytestmark = pytest.mark.asyncio
 
 
 @pytest.mark.asyncio(loop_scope='module')
+@pytest.mark.postgres
+@pytest.mark.contract('monitoring_sessions')
 async def test_postgres_upgrade_rls_and_concurrent_bot_counts(cloud_database):  # noqa: F811
     import asyncio
     import importlib

tests/unit_tests/quality/test_required_contract_report.py

@@ -1,72 +1,128 @@
-"""The optional-dependency CI guard must reject a false-green JUnit report."""
+"""Exercise the required-suite policy through real pytest collection and execution."""
 
-from __future__ import annotations
-
-import importlib.util
 from pathlib import Path
+import os
 import subprocess
 import sys
 
 import pytest
 
 
-SCRIPT = Path(__file__).resolve().parents[3] / 'scripts' / 'assert-required-tests.py'
-SPEC = importlib.util.spec_from_file_location('required_contract_report', SCRIPT)
-assert SPEC is not None and SPEC.loader is not None
-REPORT = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(REPORT)
+ROOT = Path(__file__).resolve().parents[3]
+PASSING = """
+import pytest
+
+@pytest.mark.security
+@pytest.mark.contract('debug_exec_absent')
+@pytest.mark.parametrize('value', [1, 2, 3])
+def test_renamed_behavior(value):
+    assert value > 0
+
+@pytest.mark.security
+@pytest.mark.contract('debug_route_absent')
+def test_another_name():
+    pass
+
+def test_unrelated():
+    assert False
+"""
 
 
-def write_report(tmp_path, cases):
-    path = tmp_path / 'report.xml'
-    path.write_text(f'<testsuites><testsuite>{cases}</testsuite></testsuites>')
-    return path
-
-
-def test_accepts_exact_passing_cases(tmp_path):
-    path = write_report(
-        tmp_path, '<testcase classname="contracts" name="first"/><testcase classname="contracts" name="second"/>'
+def run_suite(tmp_path, source, *args):
+    (tmp_path / 'moved').mkdir(exist_ok=True)
+    (tmp_path / 'moved/test_any_name.py').write_text(source)
+    (tmp_path / 'pytest.ini').write_text('[pytest]\nmarkers =\n    security: suite\n    contract(name): behavior\n')
+    return subprocess.run(
+        [
+            sys.executable,
+            '-m',
+            'pytest',
+            '-p',
+            'tests.utils.required_suites',
+            '--required-suite',
+            'security',
+            '-q',
+            *args,
+        ],
+        cwd=tmp_path,
+        env={**os.environ, 'PYTHONPATH': str(ROOT)},
+        capture_output=True,
+        text=True,
+        timeout=30,
     )
-    assert REPORT.verify_report(path, 'contracts', 2) == 2
 
 
-@pytest.mark.parametrize('outcome', ['skipped', 'failure', 'error'])
-def test_rejects_every_nonpassing_outcome(tmp_path, outcome):
-    path = write_report(tmp_path, f'<testcase classname="contracts" name="required"><{outcome}/></testcase>')
-    with pytest.raises(ValueError, match=outcome):
-        REPORT.verify_report(path, 'contracts', 1)
+def test_renames_moves_and_parameter_growth_need_no_ci_edit(tmp_path):
+    result = run_suite(tmp_path, PASSING)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert '4 passed, 1 deselected' in result.stdout
+    result = run_suite(tmp_path, PASSING.replace('[1, 2, 3]', '[1, 2, 3, 4]'))
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert '5 passed, 1 deselected' in result.stdout
 
 
-@pytest.mark.parametrize('count', [0, 2])
-def test_rejects_changed_case_count(tmp_path, count):
-    path = write_report(tmp_path, '<testcase classname="contracts" name="case"/>' * count)
-    with pytest.raises(ValueError, match='Expected 1 tests'):
-        REPORT.verify_report(path, 'contracts', 1)
+@pytest.mark.parametrize(
+    'statement',
+    [
+        "pytest.skip('dependency unavailable')",
+        "pytest.xfail('known bug')",
+        'assert False',
+        "raise RuntimeError('broken')",
+    ],
+)
+def test_required_cases_must_actually_pass(tmp_path, statement):
+    result = run_suite(tmp_path, PASSING.replace('    pass', f'    {statement}'))
+    assert result.returncode != 0
+    assert 'all selected cases must pass' in result.stdout
 
 
-def test_rejects_wrong_module(tmp_path):
-    path = write_report(tmp_path, '<testcase classname="unrelated" name="case"/>')
-    with pytest.raises(ValueError, match='Unexpected test module'):
-        REPORT.verify_report(path, 'contracts', 1)
+def test_unexpected_xpass_is_not_a_required_pass(tmp_path):
+    source = PASSING.replace('def test_another_name():', '@pytest.mark.xfail(strict=False)\ndef test_another_name():')
+    result = run_suite(tmp_path, source)
+    assert result.returncode != 0
+    assert 'all selected cases must pass' in result.stdout
 
 
-def test_rejects_missing_report(tmp_path):
-    with pytest.raises(FileNotFoundError):
-        REPORT.verify_report(tmp_path / 'missing.xml', 'contracts', 1)
+def test_missing_behavior_cannot_be_replaced_by_more_unrelated_cases(tmp_path):
+    source = PASSING.replace("@pytest.mark.contract('debug_route_absent')", '')
+    result = run_suite(tmp_path, source)
+    assert result.returncode != 0
+    assert "missing contracts: ['debug_route_absent']" in result.stderr
 
 
-def test_command_line_success_and_failure(tmp_path):
-    path = write_report(tmp_path, '<testcase classname="contracts" name="case"/>')
-    command = [sys.executable, str(SCRIPT), str(path), '--module', 'contracts', '--expected-tests']
-    good = subprocess.run([*command, '1'], capture_output=True, text=True)
-    assert good.returncode == 0
-    assert '1 required tests passed, 0 skipped' in good.stdout
-    bad = subprocess.run([*command, '0'], capture_output=True, text=True)
-    assert bad.returncode != 0
-    assert 'must be positive' in bad.stderr
+def test_empty_suite_fails(tmp_path):
+    result = run_suite(tmp_path, 'def test_unrelated(): pass')
+    assert result.returncode != 0
+    assert 'missing contracts' in result.stderr
 
 
-def test_rejects_wrong_function(tmp_path):
-    path = write_report(tmp_path, '<testcase classname="contracts" name="unrelated"/>')
-    with pytest.raises(ValueError, match='Unexpected test name'):
-        REPORT.verify_report(path, 'contracts', 1, 'test_required')
+@pytest.mark.parametrize('args', [('-k', 'renamed'), ('-m', 'security'), ('--deselect', 'moved/test_any_name.py')])
+def test_required_suite_rejects_partial_filters(tmp_path, args):
+    result = run_suite(tmp_path, PASSING, *args)
+    assert result.returncode != 0
+    assert 'cannot be combined' in result.stderr
+
+
+@pytest.mark.parametrize('phase', ['setup', 'teardown'])
+def test_fixture_skips_also_fail(tmp_path, phase):
+    fixture = '\n@pytest.fixture(autouse=True)\ndef dependency():\n'
+    if phase == 'teardown':
+        fixture += '    yield\n'
+    fixture += "    pytest.skip('dependency unavailable')\n"
+    result = run_suite(tmp_path, PASSING + fixture)
+    assert result.returncode != 0
+    assert 'all selected cases must pass' in result.stdout
+
+
+def test_optional_local_execution_still_allows_skips(tmp_path):
+    (tmp_path / 'test_optional.py').write_text("import pytest\ndef test_optional(): pytest.skip('no database')")
+    result = subprocess.run(
+        [sys.executable, '-m', 'pytest', '-p', 'tests.utils.required_suites', '-q'],
+        cwd=tmp_path,
+        env={**os.environ, 'PYTHONPATH': str(ROOT)},
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert '1 skipped' in result.stdout

tests/utils/required_suites.py

@@ -0,0 +1,71 @@
+"""Required suites select explicit markers and enforce execution, not case counts."""
+
+import pytest
+
+
+# These are behavior obligations, independent of file names and parametrization.
+REQUIRED_CONTRACTS = {
+    'postgres': {
+        'admission_isolation',
+        'rag_identity',
+        'rag_recovery',
+        'migration_history',
+        'migration_fresh_schema',
+        'runner_timestamps',
+        'monitoring_sessions',
+    },
+    'local_agent': {'agent_ledger', 'agent_tools', 'agent_rag', 'agent_compaction', 'agent_multi_turn', 'agent_box'},
+    'security': {'debug_exec_absent', 'debug_route_absent'},
+    'compiled_cli': {'cli_http'},
+    'plugin_identity': {'dify_identity', 'n8n_identity', 'coze_identity', 'tbox_identity'},
+    'seekdb': {'vector_round_trip', 'hybrid_relevance'},
+}
+
+
+def pytest_addoption(parser):
+    parser.addoption('--required-suite', choices=tuple(REQUIRED_CONTRACTS), help='Run a mandatory dependency suite')
+
+
+def pytest_configure(config):
+    suite = config.getoption('--required-suite')
+    if suite:
+        if config.option.markexpr or config.option.keyword or config.option.deselect:
+            raise pytest.UsageError('--required-suite cannot be combined with -m, -k or --deselect')
+        config.option.markexpr = suite
+        config.pluginmanager.register(RequiredSuite(suite), 'required-suite-policy')
+
+
+class RequiredSuite:
+    def __init__(self, suite):
+        self.suite = suite
+        self.items = {}
+        self.passed = set()
+        self.unsuccessful = set()
+
+    def pytest_collection_finish(self, session):
+        for item in session.items:
+            self.items[item.nodeid] = {mark.args[0] for mark in item.iter_markers('contract')}
+        covered = set().union(*self.items.values())
+        missing = REQUIRED_CONTRACTS[self.suite] - covered
+        if missing:
+            raise pytest.UsageError(f'Required suite {self.suite} is missing contracts: {sorted(missing)}')
+
+    def pytest_runtest_logreport(self, report):
+        if report.nodeid not in self.items:
+            return
+        if report.failed or report.skipped or hasattr(report, 'wasxfail'):
+            self.unsuccessful.add(report.nodeid)
+        if report.when == 'call' and report.passed:
+            self.passed.add(report.nodeid)
+
+    def pytest_sessionfinish(self, session, exitstatus):
+        if session.config.option.collectonly:
+            return
+        incomplete = set(self.items) - self.passed
+        if exitstatus == 0 and (incomplete or self.unsuccessful or not self.items):
+            session.exitstatus = pytest.ExitCode.TESTS_FAILED
+        reporter = session.config.pluginmanager.get_plugin('terminalreporter')
+        if reporter and (incomplete or self.unsuccessful):
+            reporter.write_sep('=', f'Required suite {self.suite}: all selected cases must pass without skips/xfails')
+            for nodeid in sorted(incomplete | self.unsuccessful):
+                reporter.write_line(nodeid)