Merge pull request #2705 from langbot-app/test/required-contract-suites

test(ci): decouple required suites from test names and counts

tests/unit_tests/agent/test_legacy_identity_boundary.py

@@ -205,14 +205,43 @@ def official_plugin_root():
 @pytest.mark.parametrize(
     'runner,folder,method,error_name,local_identity',
     [
-        ('DifyAgent', 'dify-agent', '_get_user_tag', 'DifyConfigError', 'person_Session_007'),
-        ('N8nAgent', 'n8n-agent', '_get_user_tag', 'N8nConfigError', 'person_Session_007'),
-        ('CozeAgent', 'coze-agent', '_get_user_id', 'CozeConfigError', 'person_launcher-123'),
-        ('TboxAgent', 'tbox-agent', '_get_user_id', 'TboxConfigError', 'bot-uuid-123'),
+        pytest.param(
+            'DifyAgent',
+            'dify-agent',
+            '_get_user_tag',
+            'DifyConfigError',
+            'person_Session_007',
+            marks=pytest.mark.contract('dify_identity'),
+        ),
+        pytest.param(
+            'N8nAgent',
+            'n8n-agent',
+            '_get_user_tag',
+            'N8nConfigError',
+            'person_Session_007',
+            marks=pytest.mark.contract('n8n_identity'),
+        ),
+        pytest.param(
+            'CozeAgent',
+            'coze-agent',
+            '_get_user_id',
+            'CozeConfigError',
+            'person_launcher-123',
+            marks=pytest.mark.contract('coze_identity'),
+        ),
+        pytest.param(
+            'TboxAgent',
+            'tbox-agent',
+            '_get_user_id',
+            'TboxConfigError',
+            'bot-uuid-123',
+            marks=pytest.mark.contract('tbox_identity'),
+        ),
     ],
 )
 @pytest.mark.parametrize('missing', [False, True], ids=['trusted-identity', 'missing-identity'])
 @pytest.mark.parametrize('profile', ['dedicated', 'shared'])
+@pytest.mark.plugin_identity
 async def test_real_plugin_identity_helper_with_host_payload(
     mock_query, monkeypatch, official_plugin_root, runner, folder, method, error_name, local_identity, missing, profile
 ):

tests/unit_tests/api/service/test_cli_diagnostics.py

@@ -236,6 +236,8 @@ async def test_extension_discovery_preserves_empty_bindings_and_redacts_plugins(
     ap.plugin_connector.require_workspace_context.assert_awaited_once_with(request_context())
 
 
+@pytest.mark.compiled_cli
+@pytest.mark.contract('cli_http')
 async def test_compiled_cli_against_core_http(service, tmp_path):  # noqa: F811
     """Optional cross-repo check: real HTTP/SQLite/Pipeline, synthetic runtime providers."""
     import json

tests/unit_tests/api/service/test_monitoring_sessions.py

@@ -21,6 +21,8 @@ pytestmark = pytest.mark.asyncio
 
 
 @pytest.mark.asyncio(loop_scope='module')
+@pytest.mark.postgres
+@pytest.mark.contract('monitoring_sessions')
 async def test_postgres_upgrade_rls_and_concurrent_bot_counts(cloud_database):  # noqa: F811
     import asyncio
     import importlib

tests/unit_tests/quality/test_required_contract_report.py

@@ -1,72 +1,128 @@
-"""The optional-dependency CI guard must reject a false-green JUnit report."""
+"""Exercise the required-suite policy through real pytest collection and execution."""
 
-from __future__ import annotations
-
-import importlib.util
 from pathlib import Path
+import os
 import subprocess
 import sys
 
 import pytest
 
 
-SCRIPT = Path(__file__).resolve().parents[3] / 'scripts' / 'assert-required-tests.py'
-SPEC = importlib.util.spec_from_file_location('required_contract_report', SCRIPT)
-assert SPEC is not None and SPEC.loader is not None
-REPORT = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(REPORT)
+ROOT = Path(__file__).resolve().parents[3]
+PASSING = """
+import pytest
+
+@pytest.mark.security
+@pytest.mark.contract('debug_exec_absent')
+@pytest.mark.parametrize('value', [1, 2, 3])
+def test_renamed_behavior(value):
+    assert value > 0
+
+@pytest.mark.security
+@pytest.mark.contract('debug_route_absent')
+def test_another_name():
+    pass
+
+def test_unrelated():
+    assert False
+"""
 
 
-def write_report(tmp_path, cases):
-    path = tmp_path / 'report.xml'
-    path.write_text(f'<testsuites><testsuite>{cases}</testsuite></testsuites>')
-    return path
-
-
-def test_accepts_exact_passing_cases(tmp_path):
-    path = write_report(
-        tmp_path, '<testcase classname="contracts" name="first"/><testcase classname="contracts" name="second"/>'
+def run_suite(tmp_path, source, *args):
+    (tmp_path / 'moved').mkdir(exist_ok=True)
+    (tmp_path / 'moved/test_any_name.py').write_text(source)
+    (tmp_path / 'pytest.ini').write_text('[pytest]\nmarkers =\n    security: suite\n    contract(name): behavior\n')
+    return subprocess.run(
+        [
+            sys.executable,
+            '-m',
+            'pytest',
+            '-p',
+            'tests.utils.required_suites',
+            '--required-suite',
+            'security',
+            '-q',
+            *args,
+        ],
+        cwd=tmp_path,
+        env={**os.environ, 'PYTHONPATH': str(ROOT)},
+        capture_output=True,
+        text=True,
+        timeout=30,
     )
-    assert REPORT.verify_report(path, 'contracts', 2) == 2
 
 
-@pytest.mark.parametrize('outcome', ['skipped', 'failure', 'error'])
-def test_rejects_every_nonpassing_outcome(tmp_path, outcome):
-    path = write_report(tmp_path, f'<testcase classname="contracts" name="required"><{outcome}/></testcase>')
-    with pytest.raises(ValueError, match=outcome):
-        REPORT.verify_report(path, 'contracts', 1)
+def test_renames_moves_and_parameter_growth_need_no_ci_edit(tmp_path):
+    result = run_suite(tmp_path, PASSING)
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert '4 passed, 1 deselected' in result.stdout
+    result = run_suite(tmp_path, PASSING.replace('[1, 2, 3]', '[1, 2, 3, 4]'))
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert '5 passed, 1 deselected' in result.stdout
 
 
-@pytest.mark.parametrize('count', [0, 2])
-def test_rejects_changed_case_count(tmp_path, count):
-    path = write_report(tmp_path, '<testcase classname="contracts" name="case"/>' * count)
-    with pytest.raises(ValueError, match='Expected 1 tests'):
-        REPORT.verify_report(path, 'contracts', 1)
+@pytest.mark.parametrize(
+    'statement',
+    [
+        "pytest.skip('dependency unavailable')",
+        "pytest.xfail('known bug')",
+        'assert False',
+        "raise RuntimeError('broken')",
+    ],
+)
+def test_required_cases_must_actually_pass(tmp_path, statement):
+    result = run_suite(tmp_path, PASSING.replace('    pass', f'    {statement}'))
+    assert result.returncode != 0
+    assert 'all selected cases must pass' in result.stdout
 
 
-def test_rejects_wrong_module(tmp_path):
-    path = write_report(tmp_path, '<testcase classname="unrelated" name="case"/>')
-    with pytest.raises(ValueError, match='Unexpected test module'):
-        REPORT.verify_report(path, 'contracts', 1)
+def test_unexpected_xpass_is_not_a_required_pass(tmp_path):
+    source = PASSING.replace('def test_another_name():', '@pytest.mark.xfail(strict=False)\ndef test_another_name():')
+    result = run_suite(tmp_path, source)
+    assert result.returncode != 0
+    assert 'all selected cases must pass' in result.stdout
 
 
-def test_rejects_missing_report(tmp_path):
-    with pytest.raises(FileNotFoundError):
-        REPORT.verify_report(tmp_path / 'missing.xml', 'contracts', 1)
+def test_missing_behavior_cannot_be_replaced_by_more_unrelated_cases(tmp_path):
+    source = PASSING.replace("@pytest.mark.contract('debug_route_absent')", '')
+    result = run_suite(tmp_path, source)
+    assert result.returncode != 0
+    assert "missing contracts: ['debug_route_absent']" in result.stderr
 
 
-def test_command_line_success_and_failure(tmp_path):
-    path = write_report(tmp_path, '<testcase classname="contracts" name="case"/>')
-    command = [sys.executable, str(SCRIPT), str(path), '--module', 'contracts', '--expected-tests']
-    good = subprocess.run([*command, '1'], capture_output=True, text=True)
-    assert good.returncode == 0
-    assert '1 required tests passed, 0 skipped' in good.stdout
-    bad = subprocess.run([*command, '0'], capture_output=True, text=True)
-    assert bad.returncode != 0
-    assert 'must be positive' in bad.stderr
+def test_empty_suite_fails(tmp_path):
+    result = run_suite(tmp_path, 'def test_unrelated(): pass')
+    assert result.returncode != 0
+    assert 'missing contracts' in result.stderr
 
 
-def test_rejects_wrong_function(tmp_path):
-    path = write_report(tmp_path, '<testcase classname="contracts" name="unrelated"/>')
-    with pytest.raises(ValueError, match='Unexpected test name'):
-        REPORT.verify_report(path, 'contracts', 1, 'test_required')
+@pytest.mark.parametrize('args', [('-k', 'renamed'), ('-m', 'security'), ('--deselect', 'moved/test_any_name.py')])
+def test_required_suite_rejects_partial_filters(tmp_path, args):
+    result = run_suite(tmp_path, PASSING, *args)
+    assert result.returncode != 0
+    assert 'cannot be combined' in result.stderr
+
+
+@pytest.mark.parametrize('phase', ['setup', 'teardown'])
+def test_fixture_skips_also_fail(tmp_path, phase):
+    fixture = '\n@pytest.fixture(autouse=True)\ndef dependency():\n'
+    if phase == 'teardown':
+        fixture += '    yield\n'
+    fixture += "    pytest.skip('dependency unavailable')\n"
+    result = run_suite(tmp_path, PASSING + fixture)
+    assert result.returncode != 0
+    assert 'all selected cases must pass' in result.stdout
+
+
+def test_optional_local_execution_still_allows_skips(tmp_path):
+    (tmp_path / 'test_optional.py').write_text("import pytest\ndef test_optional(): pytest.skip('no database')")
+    result = subprocess.run(
+        [sys.executable, '-m', 'pytest', '-p', 'tests.utils.required_suites', '-q'],
+        cwd=tmp_path,
+        env={**os.environ, 'PYTHONPATH': str(ROOT)},
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert '1 skipped' in result.stdout