diff --git a/.github/workflows/optional-contracts.yml b/.github/workflows/optional-contracts.yml
index f7e18f890..2a2809b6a 100644
--- a/.github/workflows/optional-contracts.yml
+++ b/.github/workflows/optional-contracts.yml
@@ -9,7 +9,6 @@ on:
- 'pyproject.toml'
- 'pytest.ini'
- 'uv.lock'
- - 'scripts/assert-required-tests.py'
- '.github/workflows/optional-contracts.yml'
push:
branches: [master, develop]
@@ -19,7 +18,6 @@ on:
- 'pyproject.toml'
- 'pytest.ini'
- 'uv.lock'
- - 'scripts/assert-required-tests.py'
- '.github/workflows/optional-contracts.yml'
workflow_dispatch:
@@ -87,12 +85,8 @@ jobs:
env:
LANGBOT_CLI_BIN: ${{ runner.temp }}/langbot-cli/lbctl
run: |
- uv run --no-sync pytest \
- tests/unit_tests/api/service/test_cli_diagnostics.py::test_compiled_cli_against_core_http \
+ uv run --no-sync pytest --required-suite compiled_cli \
-q --tb=short -rs --junitxml=compiled-cli-results.xml
- python scripts/assert-required-tests.py compiled-cli-results.xml \
- --module tests.unit_tests.api.service.test_cli_diagnostics \
- --name-prefix test_compiled_cli_against_core_http --expected-tests 1
- name: Upload compiled CLI contract result
if: always()
@@ -151,12 +145,8 @@ jobs:
env:
LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins
run: |
- uv run --no-sync pytest \
- tests/unit_tests/agent/test_legacy_identity_boundary.py::test_real_plugin_identity_helper_with_host_payload \
+ uv run --no-sync pytest --required-suite plugin_identity \
-q --tb=short -rs --junitxml=plugin-identity-results.xml
- python scripts/assert-required-tests.py plugin-identity-results.xml \
- --module tests.unit_tests.agent.test_legacy_identity_boundary \
- --name-prefix test_real_plugin_identity_helper_with_host_payload --expected-tests 16
- name: Upload plugin identity results
if: always()
@@ -198,11 +188,9 @@ jobs:
print(f"pyseekdb={version('pyseekdb')}, pylibseekdb={version('pylibseekdb')}")
PY
- uv run --no-sync pytest tests/integration/vector/test_seekdb.py \
- -m slow -q --tb=short -rs --durations=5 \
+ uv run --no-sync pytest --required-suite seekdb \
+ -q --tb=short -rs --durations=5 \
--basetemp="$SEEKDB_TEST_TMP" --junitxml=seekdb-contract-results.xml
- python scripts/assert-required-tests.py seekdb-contract-results.xml \
- --module tests.integration.vector.test_seekdb --name-prefix test_ --expected-tests 2
- name: Upload SeekDB results and startup logs
if: always()
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 583dfd5a5..bd54e498f 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -11,7 +11,7 @@ on:
- 'uv.lock'
- 'run_tests.sh'
- 'scripts/test-*.sh'
- - 'scripts/assert-required-tests.py'
+ - 'pytest.ini'
push:
branches:
- master
@@ -50,11 +50,8 @@ jobs:
- name: Run required security regressions
run: |
- uv run pytest tests/test_cwe94_debug_exec.py \
+ uv run pytest --required-suite security \
-q --tb=short -rs --junitxml=cwe94-results.xml
- python scripts/assert-required-tests.py cwe94-results.xml \
- --module tests.test_cwe94_debug_exec \
- --name-prefix test_no_ --expected-tests 2
- name: Upload security regression results
if: always()
@@ -149,45 +146,8 @@ jobs:
- name: Run required PostgreSQL contracts
run: |
- uv run pytest \
- tests/integration/persistence/test_pipeline_admission_postgres.py \
- tests/integration/persistence/test_rag_document_identity.py \
- tests/integration/persistence/test_rag_interrupted_ingestion.py \
- tests/integration/persistence/test_migration_branch_convergence.py \
- tests/integration/persistence/test_runner_timestamps_postgres.py \
- tests/unit_tests/api/service/test_monitoring_sessions.py \
- -k postgres -q --tb=short -rs --durations=10 \
- --junitxml=postgres-contract-results.xml
- python - <<'PY'
- from collections import Counter
- import xml.etree.ElementTree as ET
-
- cases = list(ET.parse('postgres-contract-results.xml').getroot().iter('testcase'))
- expected = {
- 'test_pipeline_admission_postgres': 27,
- 'test_rag_document_identity': 34,
- 'test_rag_interrupted_ingestion': 22,
- 'test_migration_branch_convergence': 6,
- 'test_runner_timestamps_postgres': 32,
- 'test_monitoring_sessions': 1,
- }
- counts = Counter(case.get('classname', '').rsplit('.', 1)[-1] for case in cases)
- for module, minimum in expected.items():
- assert counts[module] >= minimum, f'{module}: expected at least {minimum} cases, got {counts[module]}'
- convergence_cases = {
- case.get('name') for case in cases
- if case.get('classname', '').rsplit('.', 1)[-1] == 'test_migration_branch_convergence'
- }
- assert 'test_empty_database_startup_schema_then_real_migrations[postgresql]' in convergence_cases, (
- 'The fresh-schema migration and idempotency contract must execute'
- )
- unsuccessful = [
- case.get('name') for case in cases
- if any(case.find(tag) is not None for tag in ('skipped', 'failure', 'error'))
- ]
- assert not unsuccessful, f'PostgreSQL contracts must pass without skips: {unsuccessful}'
- print(f'PostgreSQL contracts: {len(cases)} passed, 0 skipped; {dict(counts)}')
- PY
+ uv run pytest --required-suite postgres \
+ -q --tb=short -rs --durations=10 --junitxml=postgres-contract-results.xml
- name: Upload PostgreSQL contract results
if: always()
@@ -241,7 +201,7 @@ jobs:
run: uv sync --dev
- name: Run E2E startup tests
- run: uv run pytest tests/e2e --ignore=tests/e2e/test_local_runner_fake_provider.py -q --tb=short
+ run: uv run pytest tests/e2e -m "not local_agent" -q --tb=short
- name: Check Docker for the required LocalAgent Box contract
run: docker info
@@ -250,11 +210,8 @@ jobs:
env:
LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins
run: |
- uv run --no-sync pytest tests/e2e/test_local_runner_fake_provider.py \
+ uv run --no-sync pytest --required-suite local_agent \
-q --tb=short -rs --junitxml=local-agent-results.xml
- python scripts/assert-required-tests.py local-agent-results.xml \
- --module tests.e2e.test_local_runner_fake_provider \
- --name-prefix test_local_runner_ --expected-tests 6
- name: Upload LocalAgent contract results
if: always()
diff --git a/docs/DEPENDENCY_CONTRACT_TESTS.md b/docs/DEPENDENCY_CONTRACT_TESTS.md
index 48168d529..6c038c99f 100644
--- a/docs/DEPENDENCY_CONTRACT_TESTS.md
+++ b/docs/DEPENDENCY_CONTRACT_TESTS.md
@@ -47,12 +47,12 @@ To run with an official checkout at the locked revision:
```bash
export LANGBOT_LEGACY_PLUGIN_ROOT=/absolute/path/to/langbot-plugins
-uv run --frozen pytest tests/unit_tests/agent/test_legacy_identity_boundary.py -q -rs
+uv run --frozen pytest --required-suite plugin_identity -q -rs
```
An unset variable allows an explicit local prerequisite skip. An empty variable,
a missing file, a path escaping the fixture root, or a hash mismatch fails.
-CI selects the 16 consumer cases and requires all 16 to pass without skips.
+CI selects the `plugin_identity` suite and requires every selected case to pass without skips.
## Embedded SeekDB
@@ -64,10 +64,8 @@ The tests supply their own small embedding vectors; no model download is needed.
```bash
uv sync --dev --extra seekdb --frozen
-uv run --no-sync pytest tests/integration/vector/test_seekdb.py \
- -m slow -q -rs --durations=5 --junitxml=seekdb-contract-results.xml
-python scripts/assert-required-tests.py seekdb-contract-results.xml \
- --module tests.integration.vector.test_seekdb --expected-tests 2
+uv run --no-sync pytest --required-suite seekdb \
+ -q -rs --durations=5 --junitxml=seekdb-contract-results.xml
```
This isolated Linux job keeps native dependency startup failures visible instead
@@ -87,7 +85,7 @@ CI builds the official [`langbot-app/langbot-cli` v0.2.0 source](https://github.
pinned to commit `3bd20eecd4771b1fe5d0dca61d8b5523182e7bd9`, using Go 1.26.8.
Module versions and checksums come from that checkout's `go.mod` and `go.sum`.
The job verifies modules and builds with `CGO_ENABLED=0`, `-mod=readonly`, and
-`-trimpath`. It requires exactly one passing JUnit case and rejects skips.
+`-trimpath`. It requires the CLI HTTP behavior contract and rejects skips.
To run locally with an official pinned CLI checkout and Go 1.26.8:
@@ -101,12 +99,35 @@ go build -mod=readonly -trimpath -o ./bin/lbctl ./cmd/lbctl
export LANGBOT_CLI_BIN="$(pwd)/bin/lbctl"
# In the LangBot Core checkout:
-uv run --frozen pytest \
- tests/unit_tests/api/service/test_cli_diagnostics.py::test_compiled_cli_against_core_http \
- -q --tb=short -rs
+uv run --frozen pytest --required-suite compiled_cli -q --tb=short -rs
```
Without `LANGBOT_CLI_BIN`, this test remains an explicit local skip. When set, it
must point to a real compiled executable. CI supplies an absolute path and treats
a missing binary, command failure, or skipped case as a failure. Update the source
pin deliberately and rerun the contract before accepting a newer CLI version.
+
+## Required suite policy
+
+CI provisions dependencies and invokes `pytest --required-suite SUITE`. The
+registered suites are `postgres`, `local_agent`, `security`, `compiled_cli`,
+`plugin_identity`, and `seekdb`. Tests carry explicit suite markers; mixed
+SQLite/PostgreSQL fixtures mark only their PostgreSQL parameter. New marked
+cases are discovered automatically, without editing workflow file lists or counts.
+
+`tests/utils/required_suites.py` lists the stable behavior obligations for each
+suite. `@pytest.mark.contract('name')` identifies the behavior a case exercises.
+Only a new or retired behavior obligation changes this policy; renaming, moving,
+or parameterizing an existing test does not. Unmarked sibling tests are not run
+by a required suite.
+
+Required execution fails if a behavior obligation is absent, no cases are
+selected, or any selected case fails, errors, skips, or xfails (including fixture
+setup/teardown). Partial `-k`, `-m`, and `--deselect` filters are rejected.
+`--collect-only` validates selection without claiming execution. Without
+`--required-suite`, optional local prerequisites may still skip as before.
+JUnit remains an artifact, not a second source of the expected test inventory.
+
+These checks prevent missing execution; they cannot prove that a behavior marker
+is truthful or that an assertion detects defects. Review and mutation checks
+still establish assertion quality.
diff --git a/pytest.ini b/pytest.ini
index d86cbba9f..8f543757a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -28,6 +28,13 @@ addopts =
# Markers
markers =
+ contract(name): stable behavior obligation for required suites
+ postgres: required postgres execution suite
+ local_agent: required local_agent execution suite
+ security: required security execution suite
+ compiled_cli: required compiled_cli execution suite
+ plugin_identity: required plugin_identity execution suite
+ seekdb: required seekdb execution suite
asyncio: mark test as async
unit: mark test as unit test
integration: mark test as integration test
diff --git a/scripts/assert-required-tests.py b/scripts/assert-required-tests.py
deleted file mode 100644
index 2ff1f6ced..000000000
--- a/scripts/assert-required-tests.py
+++ /dev/null
@@ -1,39 +0,0 @@
-"""Fail closed when a provisioned dependency-contract lane skips or loses tests."""
-
-from __future__ import annotations
-
-import argparse
-from pathlib import Path
-import xml.etree.ElementTree as ET
-
-
-def verify_report(path: Path, expected_module: str, expected_tests: int, name_prefix: str = '') -> int:
- cases = list(ET.parse(path).getroot().iter('testcase'))
- if len(cases) != expected_tests:
- raise ValueError(f'Expected {expected_tests} tests, found {len(cases)} in {path}')
- for case in cases:
- if case.get('classname') != expected_module:
- raise ValueError(f'Unexpected test module: {case.get("classname")!r}')
- if not case.get('name', '').startswith(name_prefix):
- raise ValueError(f'Unexpected test name: {case.get("name")!r}')
- for outcome in ('skipped', 'failure', 'error'):
- if case.find(outcome) is not None:
- raise ValueError(f'Required test {case.get("name")!r} reported {outcome}')
- return len(cases)
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('report', type=Path)
- parser.add_argument('--module', required=True)
- parser.add_argument('--expected-tests', type=int, required=True)
- parser.add_argument('--name-prefix', default='')
- args = parser.parse_args()
- if args.expected_tests < 1:
- parser.error('--expected-tests must be positive')
- count = verify_report(args.report, args.module, args.expected_tests, args.name_prefix)
- print(f'{args.module}: {count} required tests passed, 0 skipped')
-
-
-if __name__ == '__main__':
- main()
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 000000000..8e63694e3
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,3 @@
+"""Repository-wide test execution policy."""
+
+pytest_plugins = ['tests.utils.required_suites']
diff --git a/tests/e2e/test_local_runner_fake_provider.py b/tests/e2e/test_local_runner_fake_provider.py
index 196b185e4..cbb953bad 100644
--- a/tests/e2e/test_local_runner_fake_provider.py
+++ b/tests/e2e/test_local_runner_fake_provider.py
@@ -26,7 +26,7 @@ import pytest
from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories
from tests.utils.legacy_plugin_contract import verified_plugin_root
-pytestmark = pytest.mark.e2e
+pytestmark = [pytest.mark.e2e, pytest.mark.local_agent]
LOCAL_RUNNER_ID = 'plugin:langbot-team/LocalAgent/default'
@@ -557,6 +557,7 @@ def _run_local_agent_probe(tmpdir: Path, probe):
return asyncio.run(_run())
+@pytest.mark.contract('agent_ledger')
def test_local_runner_uses_host_fake_provider_and_persists_ledger(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
@@ -619,6 +620,7 @@ def test_local_runner_uses_host_fake_provider_and_persists_ledger(
conn.close()
+@pytest.mark.contract('agent_tools')
def test_local_runner_executes_authorized_tool_loop_through_host_action(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
@@ -688,6 +690,7 @@ def test_local_runner_executes_authorized_tool_loop_through_host_action(
conn.close()
+@pytest.mark.contract('agent_rag')
def test_local_runner_retrieves_authorized_rag_context_through_host_action(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
@@ -762,6 +765,7 @@ def test_local_runner_retrieves_authorized_rag_context_through_host_action(
conn.close()
+@pytest.mark.contract('agent_compaction')
def test_local_runner_compacts_history_and_persists_checkpoint(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
@@ -855,6 +859,7 @@ def test_local_runner_compacts_history_and_persists_checkpoint(
conn.close()
+@pytest.mark.contract('agent_multi_turn')
def test_local_runner_combines_rag_compaction_and_multi_turn_tool_loop(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
@@ -1017,6 +1022,7 @@ def test_local_runner_combines_rag_compaction_and_multi_turn_tool_loop(
conn.close()
+@pytest.mark.contract('agent_box')
def test_local_runner_owns_box_reuse_and_explicit_files(
local_agent_e2e_tmpdir,
local_agent_e2e_config_path,
diff --git a/tests/integration/persistence/test_migration_branch_convergence.py b/tests/integration/persistence/test_migration_branch_convergence.py
index 903b2794a..58e4a5796 100644
--- a/tests/integration/persistence/test_migration_branch_convergence.py
+++ b/tests/integration/persistence/test_migration_branch_convergence.py
@@ -39,7 +39,7 @@ ACCOUNT = '41100000-0000-4000-8000-000000000002'
NOW = datetime.datetime(2026, 9, 1)
-@pytest.fixture(params=['sqlite', pytest.param('postgresql', marks=pytest.mark.slow)])
+@pytest.fixture(params=['sqlite', pytest.param('postgresql', marks=[pytest.mark.slow, pytest.mark.postgres])])
async def convergence_engine(request, tmp_path):
if request.param == 'sqlite':
engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "convergence.db"}')
@@ -354,6 +354,7 @@ async def _assert_merged_schema_and_data(engine, source):
('beta', None),
],
)
+@pytest.mark.contract('migration_history')
async def test_historical_schema_and_populated_branch_upgrade(convergence_engine, source, checkpoint):
engine = convergence_engine
fixture = json.loads((FIXTURES / f'{source}_schema.json').read_text())
@@ -395,6 +396,7 @@ async def test_historical_schema_and_populated_branch_upgrade(convergence_engine
@pytest.mark.asyncio
+@pytest.mark.contract('migration_fresh_schema')
async def test_empty_database_startup_schema_then_real_migrations(convergence_engine):
engine = convergence_engine
async with engine.begin() as conn:
diff --git a/tests/integration/persistence/test_pipeline_admission_postgres.py b/tests/integration/persistence/test_pipeline_admission_postgres.py
index 6c95eae63..865a158eb 100644
--- a/tests/integration/persistence/test_pipeline_admission_postgres.py
+++ b/tests/integration/persistence/test_pipeline_admission_postgres.py
@@ -12,7 +12,11 @@ from langbot.pkg.persistence.pipeline_admission import lock_pipeline_admission
from langbot.pkg.entity.persistence.agent_interaction import AgentInteraction
URL = os.environ.get('LANGBOT_ADMISSION_TEST_URL')
-pytestmark = [pytest.mark.asyncio, pytest.mark.skipif(not URL, reason='disposable PostgreSQL URL required')]
+pytestmark = [
+ pytest.mark.postgres,
+ pytest.mark.asyncio,
+ pytest.mark.skipif(not URL, reason='disposable PostgreSQL URL required'),
+]
@pytest.fixture
@@ -111,6 +115,7 @@ async def test_real_writer_waits_and_revalidates_after_transaction(env, outcome)
assert (await writer)[0]['status'] == 'pending'
+@pytest.mark.contract('admission_isolation')
async def test_real_writer_first_blocks_migration_and_other_tenant_isolated(env):
await write(env)
async with env.pm.tenant_uow(base.WS) as uow:
diff --git a/tests/integration/persistence/test_rag_document_identity.py b/tests/integration/persistence/test_rag_document_identity.py
index 5a729dd87..21cc6d7dc 100644
--- a/tests/integration/persistence/test_rag_document_identity.py
+++ b/tests/integration/persistence/test_rag_document_identity.py
@@ -54,7 +54,7 @@ DOCUMENT_IDENTITY_REVISION = '0025_rag_document_identity'
CONTEXT = ExecutionContext(instance_uuid='instance-a', workspace_uuid='workspace-a', placement_generation=5)
-@pytest_asyncio.fixture(params=['sqlite', 'postgres'])
+@pytest_asyncio.fixture(params=['sqlite', pytest.param('postgres', marks=pytest.mark.postgres)])
async def database(request, tmp_path):
admin = None
schema = 'ke_identity_' + uuid.uuid4().hex
@@ -183,6 +183,7 @@ async def read_file_row(runtime, file_id='host-id'):
@pytest.mark.parametrize(
'engine_id', ['dify-upstream', 'fastgpt-collection', 'ragflow-upstream', 'host-id', ' opaque ID ']
)
+@pytest.mark.contract('rag_identity')
async def test_ingest_persists_exact_engine_identity_and_restart_delete(runtime, engine_id):
file = await seed(runtime)
runtime.ap.plugin_connector.call_rag_ingest.return_value['document_id'] = engine_id
diff --git a/tests/integration/persistence/test_rag_interrupted_ingestion.py b/tests/integration/persistence/test_rag_interrupted_ingestion.py
index aa6b53dc6..8e6d77c26 100644
--- a/tests/integration/persistence/test_rag_interrupted_ingestion.py
+++ b/tests/integration/persistence/test_rag_interrupted_ingestion.py
@@ -25,6 +25,7 @@ def attach_task_manager(runtime):
@pytest.mark.asyncio
+@pytest.mark.contract('rag_recovery')
async def test_cancelled_ingestion_retains_recovery_resources(runtime):
"""Local cancellation is interrupted observation, not remote quiescence."""
file = await seed(runtime)
diff --git a/tests/integration/persistence/test_runner_timestamps_postgres.py b/tests/integration/persistence/test_runner_timestamps_postgres.py
index 6cc988b7b..732b40490 100644
--- a/tests/integration/persistence/test_runner_timestamps_postgres.py
+++ b/tests/integration/persistence/test_runner_timestamps_postgres.py
@@ -25,7 +25,12 @@ from langbot.pkg.entity.persistence.base import Base
from langbot.pkg.entity.persistence.event_log import EventLog
from langbot.pkg.entity.persistence.transcript import Transcript
-pytestmark = [pytest.mark.integration, pytest.mark.slow, pytest.mark.asyncio]
+pytestmark = [
+ pytest.mark.postgres,
+ pytest.mark.integration,
+ pytest.mark.slow,
+ pytest.mark.asyncio,
+]
UTC = dt.timezone.utc
NOW = dt.datetime(2026, 9, 22, 4, 5, 6, 123000, tzinfo=UTC)
@@ -110,6 +115,7 @@ async def _legacy_run(engine, *, status='queued'):
)
+@pytest.mark.contract('runner_timestamps')
async def test_run_lifecycle_deadline_events_and_stats(pg_engine):
store = run_ledger_store.RunLedgerStore(pg_engine)
deadline = NOW + dt.timedelta(minutes=5)
diff --git a/tests/integration/vector/test_seekdb.py b/tests/integration/vector/test_seekdb.py
index 668072aba..6cf919bde 100644
--- a/tests/integration/vector/test_seekdb.py
+++ b/tests/integration/vector/test_seekdb.py
@@ -14,16 +14,14 @@ import uuid
import pytest
-pytest.importorskip('pyseekdb')
-
-from langbot.pkg.vector.vdbs.seekdb import SeekDBVectorDatabase
-
-
-pytestmark = [pytest.mark.integration, pytest.mark.slow]
+pytestmark = [pytest.mark.integration, pytest.mark.slow, pytest.mark.seekdb]
@pytest.fixture
async def backend(tmp_path):
+ pytest.importorskip('pyseekdb')
+ from langbot.pkg.vector.vdbs.seekdb import SeekDBVectorDatabase
+
app = SimpleNamespace(
instance_config=SimpleNamespace(
data={
@@ -51,6 +49,7 @@ async def backend(tmp_path):
@pytest.mark.asyncio
+@pytest.mark.contract('vector_round_trip')
async def test_upsert_and_text_round_trip(backend) -> None:
database, collection = backend
original = 'He said "hello".\nC:\\notes\\file.txt isn\'t empty. 中文'
@@ -80,6 +79,7 @@ async def test_upsert_and_text_round_trip(backend) -> None:
@pytest.mark.asyncio
+@pytest.mark.contract('hybrid_relevance')
async def test_full_text_and_hybrid_results_keep_relevance_order(backend) -> None:
database, collection = backend
documents = [
diff --git a/tests/test_cwe94_debug_exec.py b/tests/test_cwe94_debug_exec.py
index b31085981..ba91931a6 100644
--- a/tests/test_cwe94_debug_exec.py
+++ b/tests/test_cwe94_debug_exec.py
@@ -12,12 +12,17 @@ This test verifies that:
import ast
import pathlib
+import pytest
+
+pytestmark = pytest.mark.security
+
# Resolve project root (one level up from tests/)
_PROJECT_ROOT = pathlib.Path(__file__).resolve().parent.parent
VULN_FILE = _PROJECT_ROOT / 'src' / 'langbot' / 'pkg' / 'api' / 'http' / 'controller' / 'groups' / 'system.py'
+@pytest.mark.contract('debug_exec_absent')
def test_no_exec_call_in_system_controller():
"""Verify there is no exec() call in system.py that takes user input."""
with open(VULN_FILE, 'r') as f:
@@ -38,6 +43,7 @@ def test_no_exec_call_in_system_controller():
)
+@pytest.mark.contract('debug_route_absent')
def test_no_debug_exec_route():
"""Verify the /debug/exec route is not registered."""
with open(VULN_FILE, 'r') as f:
diff --git a/tests/unit_tests/agent/test_legacy_identity_boundary.py b/tests/unit_tests/agent/test_legacy_identity_boundary.py
index 216ade894..d5347acfb 100644
--- a/tests/unit_tests/agent/test_legacy_identity_boundary.py
+++ b/tests/unit_tests/agent/test_legacy_identity_boundary.py
@@ -205,14 +205,43 @@ def official_plugin_root():
@pytest.mark.parametrize(
'runner,folder,method,error_name,local_identity',
[
- ('DifyAgent', 'dify-agent', '_get_user_tag', 'DifyConfigError', 'person_Session_007'),
- ('N8nAgent', 'n8n-agent', '_get_user_tag', 'N8nConfigError', 'person_Session_007'),
- ('CozeAgent', 'coze-agent', '_get_user_id', 'CozeConfigError', 'person_launcher-123'),
- ('TboxAgent', 'tbox-agent', '_get_user_id', 'TboxConfigError', 'bot-uuid-123'),
+ pytest.param(
+ 'DifyAgent',
+ 'dify-agent',
+ '_get_user_tag',
+ 'DifyConfigError',
+ 'person_Session_007',
+ marks=pytest.mark.contract('dify_identity'),
+ ),
+ pytest.param(
+ 'N8nAgent',
+ 'n8n-agent',
+ '_get_user_tag',
+ 'N8nConfigError',
+ 'person_Session_007',
+ marks=pytest.mark.contract('n8n_identity'),
+ ),
+ pytest.param(
+ 'CozeAgent',
+ 'coze-agent',
+ '_get_user_id',
+ 'CozeConfigError',
+ 'person_launcher-123',
+ marks=pytest.mark.contract('coze_identity'),
+ ),
+ pytest.param(
+ 'TboxAgent',
+ 'tbox-agent',
+ '_get_user_id',
+ 'TboxConfigError',
+ 'bot-uuid-123',
+ marks=pytest.mark.contract('tbox_identity'),
+ ),
],
)
@pytest.mark.parametrize('missing', [False, True], ids=['trusted-identity', 'missing-identity'])
@pytest.mark.parametrize('profile', ['dedicated', 'shared'])
+@pytest.mark.plugin_identity
async def test_real_plugin_identity_helper_with_host_payload(
mock_query, monkeypatch, official_plugin_root, runner, folder, method, error_name, local_identity, missing, profile
):
diff --git a/tests/unit_tests/api/service/test_cli_diagnostics.py b/tests/unit_tests/api/service/test_cli_diagnostics.py
index 95b85e709..a78965f5b 100644
--- a/tests/unit_tests/api/service/test_cli_diagnostics.py
+++ b/tests/unit_tests/api/service/test_cli_diagnostics.py
@@ -236,6 +236,8 @@ async def test_extension_discovery_preserves_empty_bindings_and_redacts_plugins(
ap.plugin_connector.require_workspace_context.assert_awaited_once_with(request_context())
+@pytest.mark.compiled_cli
+@pytest.mark.contract('cli_http')
async def test_compiled_cli_against_core_http(service, tmp_path): # noqa: F811
"""Optional cross-repo check: real HTTP/SQLite/Pipeline, synthetic runtime providers."""
import json
diff --git a/tests/unit_tests/api/service/test_monitoring_sessions.py b/tests/unit_tests/api/service/test_monitoring_sessions.py
index 868f51589..891171ba3 100644
--- a/tests/unit_tests/api/service/test_monitoring_sessions.py
+++ b/tests/unit_tests/api/service/test_monitoring_sessions.py
@@ -21,6 +21,8 @@ pytestmark = pytest.mark.asyncio
@pytest.mark.asyncio(loop_scope='module')
+@pytest.mark.postgres
+@pytest.mark.contract('monitoring_sessions')
async def test_postgres_upgrade_rls_and_concurrent_bot_counts(cloud_database): # noqa: F811
import asyncio
import importlib
diff --git a/tests/unit_tests/quality/test_required_contract_report.py b/tests/unit_tests/quality/test_required_contract_report.py
index 0c7c5cab5..79add8c29 100644
--- a/tests/unit_tests/quality/test_required_contract_report.py
+++ b/tests/unit_tests/quality/test_required_contract_report.py
@@ -1,72 +1,128 @@
-"""The optional-dependency CI guard must reject a false-green JUnit report."""
+"""Exercise the required-suite policy through real pytest collection and execution."""
-from __future__ import annotations
-
-import importlib.util
from pathlib import Path
+import os
import subprocess
import sys
import pytest
-SCRIPT = Path(__file__).resolve().parents[3] / 'scripts' / 'assert-required-tests.py'
-SPEC = importlib.util.spec_from_file_location('required_contract_report', SCRIPT)
-assert SPEC is not None and SPEC.loader is not None
-REPORT = importlib.util.module_from_spec(SPEC)
-SPEC.loader.exec_module(REPORT)
+ROOT = Path(__file__).resolve().parents[3]
+PASSING = """
+import pytest
+
+@pytest.mark.security
+@pytest.mark.contract('debug_exec_absent')
+@pytest.mark.parametrize('value', [1, 2, 3])
+def test_renamed_behavior(value):
+ assert value > 0
+
+@pytest.mark.security
+@pytest.mark.contract('debug_route_absent')
+def test_another_name():
+ pass
+
+def test_unrelated():
+ assert False
+"""
-def write_report(tmp_path, cases):
- path = tmp_path / 'report.xml'
- path.write_text(f'{cases}')
- return path
-
-
-def test_accepts_exact_passing_cases(tmp_path):
- path = write_report(
- tmp_path, ''
+def run_suite(tmp_path, source, *args):
+ (tmp_path / 'moved').mkdir(exist_ok=True)
+ (tmp_path / 'moved/test_any_name.py').write_text(source)
+ (tmp_path / 'pytest.ini').write_text('[pytest]\nmarkers =\n security: suite\n contract(name): behavior\n')
+ return subprocess.run(
+ [
+ sys.executable,
+ '-m',
+ 'pytest',
+ '-p',
+ 'tests.utils.required_suites',
+ '--required-suite',
+ 'security',
+ '-q',
+ *args,
+ ],
+ cwd=tmp_path,
+ env={**os.environ, 'PYTHONPATH': str(ROOT)},
+ capture_output=True,
+ text=True,
+ timeout=30,
)
- assert REPORT.verify_report(path, 'contracts', 2) == 2
-@pytest.mark.parametrize('outcome', ['skipped', 'failure', 'error'])
-def test_rejects_every_nonpassing_outcome(tmp_path, outcome):
- path = write_report(tmp_path, f'<{outcome}/>')
- with pytest.raises(ValueError, match=outcome):
- REPORT.verify_report(path, 'contracts', 1)
+def test_renames_moves_and_parameter_growth_need_no_ci_edit(tmp_path):
+ result = run_suite(tmp_path, PASSING)
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert '4 passed, 1 deselected' in result.stdout
+ result = run_suite(tmp_path, PASSING.replace('[1, 2, 3]', '[1, 2, 3, 4]'))
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert '5 passed, 1 deselected' in result.stdout
-@pytest.mark.parametrize('count', [0, 2])
-def test_rejects_changed_case_count(tmp_path, count):
- path = write_report(tmp_path, '' * count)
- with pytest.raises(ValueError, match='Expected 1 tests'):
- REPORT.verify_report(path, 'contracts', 1)
+@pytest.mark.parametrize(
+ 'statement',
+ [
+ "pytest.skip('dependency unavailable')",
+ "pytest.xfail('known bug')",
+ 'assert False',
+ "raise RuntimeError('broken')",
+ ],
+)
+def test_required_cases_must_actually_pass(tmp_path, statement):
+ result = run_suite(tmp_path, PASSING.replace(' pass', f' {statement}'))
+ assert result.returncode != 0
+ assert 'all selected cases must pass' in result.stdout
-def test_rejects_wrong_module(tmp_path):
- path = write_report(tmp_path, '')
- with pytest.raises(ValueError, match='Unexpected test module'):
- REPORT.verify_report(path, 'contracts', 1)
+def test_unexpected_xpass_is_not_a_required_pass(tmp_path):
+ source = PASSING.replace('def test_another_name():', '@pytest.mark.xfail(strict=False)\ndef test_another_name():')
+ result = run_suite(tmp_path, source)
+ assert result.returncode != 0
+ assert 'all selected cases must pass' in result.stdout
-def test_rejects_missing_report(tmp_path):
- with pytest.raises(FileNotFoundError):
- REPORT.verify_report(tmp_path / 'missing.xml', 'contracts', 1)
+def test_missing_behavior_cannot_be_replaced_by_more_unrelated_cases(tmp_path):
+ source = PASSING.replace("@pytest.mark.contract('debug_route_absent')", '')
+ result = run_suite(tmp_path, source)
+ assert result.returncode != 0
+ assert "missing contracts: ['debug_route_absent']" in result.stderr
-def test_command_line_success_and_failure(tmp_path):
- path = write_report(tmp_path, '')
- command = [sys.executable, str(SCRIPT), str(path), '--module', 'contracts', '--expected-tests']
- good = subprocess.run([*command, '1'], capture_output=True, text=True)
- assert good.returncode == 0
- assert '1 required tests passed, 0 skipped' in good.stdout
- bad = subprocess.run([*command, '0'], capture_output=True, text=True)
- assert bad.returncode != 0
- assert 'must be positive' in bad.stderr
+def test_empty_suite_fails(tmp_path):
+ result = run_suite(tmp_path, 'def test_unrelated(): pass')
+ assert result.returncode != 0
+ assert 'missing contracts' in result.stderr
-def test_rejects_wrong_function(tmp_path):
- path = write_report(tmp_path, '')
- with pytest.raises(ValueError, match='Unexpected test name'):
- REPORT.verify_report(path, 'contracts', 1, 'test_required')
+@pytest.mark.parametrize('args', [('-k', 'renamed'), ('-m', 'security'), ('--deselect', 'moved/test_any_name.py')])
+def test_required_suite_rejects_partial_filters(tmp_path, args):
+ result = run_suite(tmp_path, PASSING, *args)
+ assert result.returncode != 0
+ assert 'cannot be combined' in result.stderr
+
+
+@pytest.mark.parametrize('phase', ['setup', 'teardown'])
+def test_fixture_skips_also_fail(tmp_path, phase):
+ fixture = '\n@pytest.fixture(autouse=True)\ndef dependency():\n'
+ if phase == 'teardown':
+ fixture += ' yield\n'
+ fixture += " pytest.skip('dependency unavailable')\n"
+ result = run_suite(tmp_path, PASSING + fixture)
+ assert result.returncode != 0
+ assert 'all selected cases must pass' in result.stdout
+
+
+def test_optional_local_execution_still_allows_skips(tmp_path):
+ (tmp_path / 'test_optional.py').write_text("import pytest\ndef test_optional(): pytest.skip('no database')")
+ result = subprocess.run(
+ [sys.executable, '-m', 'pytest', '-p', 'tests.utils.required_suites', '-q'],
+ cwd=tmp_path,
+ env={**os.environ, 'PYTHONPATH': str(ROOT)},
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert '1 skipped' in result.stdout
diff --git a/tests/utils/required_suites.py b/tests/utils/required_suites.py
new file mode 100644
index 000000000..d752e10c6
--- /dev/null
+++ b/tests/utils/required_suites.py
@@ -0,0 +1,71 @@
+"""Required suites select explicit markers and enforce execution, not case counts."""
+
+import pytest
+
+
+# These are behavior obligations, independent of file names and parametrization.
+REQUIRED_CONTRACTS = {
+ 'postgres': {
+ 'admission_isolation',
+ 'rag_identity',
+ 'rag_recovery',
+ 'migration_history',
+ 'migration_fresh_schema',
+ 'runner_timestamps',
+ 'monitoring_sessions',
+ },
+ 'local_agent': {'agent_ledger', 'agent_tools', 'agent_rag', 'agent_compaction', 'agent_multi_turn', 'agent_box'},
+ 'security': {'debug_exec_absent', 'debug_route_absent'},
+ 'compiled_cli': {'cli_http'},
+ 'plugin_identity': {'dify_identity', 'n8n_identity', 'coze_identity', 'tbox_identity'},
+ 'seekdb': {'vector_round_trip', 'hybrid_relevance'},
+}
+
+
+def pytest_addoption(parser):
+ parser.addoption('--required-suite', choices=tuple(REQUIRED_CONTRACTS), help='Run a mandatory dependency suite')
+
+
+def pytest_configure(config):
+ suite = config.getoption('--required-suite')
+ if suite:
+ if config.option.markexpr or config.option.keyword or config.option.deselect:
+ raise pytest.UsageError('--required-suite cannot be combined with -m, -k or --deselect')
+ config.option.markexpr = suite
+ config.pluginmanager.register(RequiredSuite(suite), 'required-suite-policy')
+
+
+class RequiredSuite:
+ def __init__(self, suite):
+ self.suite = suite
+ self.items = {}
+ self.passed = set()
+ self.unsuccessful = set()
+
+ def pytest_collection_finish(self, session):
+ for item in session.items:
+ self.items[item.nodeid] = {mark.args[0] for mark in item.iter_markers('contract')}
+ covered = set().union(*self.items.values())
+ missing = REQUIRED_CONTRACTS[self.suite] - covered
+ if missing:
+ raise pytest.UsageError(f'Required suite {self.suite} is missing contracts: {sorted(missing)}')
+
+ def pytest_runtest_logreport(self, report):
+ if report.nodeid not in self.items:
+ return
+ if report.failed or report.skipped or hasattr(report, 'wasxfail'):
+ self.unsuccessful.add(report.nodeid)
+ if report.when == 'call' and report.passed:
+ self.passed.add(report.nodeid)
+
+ def pytest_sessionfinish(self, session, exitstatus):
+ if session.config.option.collectonly:
+ return
+ incomplete = set(self.items) - self.passed
+ if exitstatus == 0 and (incomplete or self.unsuccessful or not self.items):
+ session.exitstatus = pytest.ExitCode.TESTS_FAILED
+ reporter = session.config.pluginmanager.get_plugin('terminalreporter')
+ if reporter and (incomplete or self.unsuccessful):
+ reporter.write_sep('=', f'Required suite {self.suite}: all selected cases must pass without skips/xfails')
+ for nodeid in sorted(incomplete | self.unsuccessful):
+ reporter.write_line(nodeid)