test(regression): report asserted behavior coverage and observed HTTP routes

tools/README.md

@@ -474,3 +474,48 @@ Keep this longer test outside the one-minute merge gate while its budgets and
 repeatability are being evaluated. The same profile can be selected by the
 release-deployment Job, which supplies its ledger path and process IDs; that
 fresh-deployment path still needs separate end-to-end validation.
+
+## Business behavior coverage
+
+`regression_contracts.json` declares behavior obligations independently of how
+many requests a workload repeats. It includes explicit unimplemented risks
+(cancellation, restart recovery, credential revocation, Workspace isolation,
+PostgreSQL parity, real adapters and embeddings). This is coverage of the
+registered scope, not a percentage of all product behavior or API endpoints.
+
+Runners record named assertions as they execute. A green legacy request without
+assertion evidence is **incomplete**. An evaluated failing assertion counts as
+covered and failed; a timeout before an assertion does not count. A contract with
+multiple required assertions passes only when all are observed. Repeated
+observations cannot hide an incomplete or failed turn. History contracts require
+a subsequent turn, not merely the initial reply. Database/Box-event inspections
+are reported separately as gray-box evidence.
+
+Reports distinguish `passed`, `failed`, `incomplete`, `skipped`, `not_run` (a test
+exists but did not execute) and `missing_test`. Coverage and pass rate have separate
+denominators. The catalog fingerprint identifies the registered denominator.
+New case names with no contract appear as unclassified, and malformed evidence
+is reported as an error instead of silently accepted.
+
+The source route inventory discovers current static RouterGroup HTTP declarations
+and flags routes without a linked behavior contract. Links describe test intent;
+they are not observed HTTP hits or proof that every method behavior is tested.
+Dynamic routes, WebSockets and platform-dispatched routes are outside this static
+inventory. Full source locations and any unresolved declarations are retained
+in `coverage.json`; new declarations appear without editing the case list.
+
+```sh
+python tools/regression_coverage.py --run .qa/smoke/suite \
+  --run .qa/endurance/suite --output .qa/behavior-coverage
+```
+
+`coverage.json` and `coverage.md` are written by the Suite. CI consumers can
+publish these files as artifacts. Uncovered contracts do not add a merge gate. A missing or corrupt evidence
+file is a reporting failure; the report remains available for inspection.
+
+Pass `--http-log /path/to/core.log` to include observed methods, declared route
+templates and status counts from Hypercorn access records. Query strings and
+resource identifiers are not retained in this report. Unmatched requests are
+counted by method/status; webhook sub-paths are not separate declared routes.
+Missing logs or logs without recognizable access records are evidence errors.
+HTTP hits remain separate from assertion coverage.

tools/regression_contracts.json

@@ -0,0 +1,283 @@
+{
+  "version": 1,
+  "scope": "Registered CLI/HTTP Bot, conversation, model, tool and retrieval behavior obligations plus explicit adjacent-risk gaps. This is not all product features, all API endpoints, code coverage, or real-model quality coverage. Case repetition does not increase the denominator.",
+  "contracts": [
+    {"id": "configuration.identity",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "CLI resolves the configured Workspace identity",
+     "bindings": [{"profile": "smoke", "case": "identity", "checks": ["response"]}]
+    },
+    {"id": "configuration.plugins",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "Installed LocalAgent is visible through the plugin API",
+     "bindings": [{"profile": "smoke", "case": "plugins", "checks": ["response"]}],
+     "routes": ["GET /api/v1/plugins"]
+    },
+    {"id": "configuration.provider_readback",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "Created provider is visible through the public API",
+     "bindings": [{"profile": "smoke", "case": "provider_readback", "checks": ["response"]}],
+     "routes": ["GET /api/v1/provider/providers/<provider_uuid>"]
+    },
+    {"id": "configuration.model_readback",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "Created model is visible through the public API",
+     "bindings": [{"profile": "smoke", "case": "model_readback", "checks": ["response"]}],
+     "routes": ["GET /api/v1/provider/models/llm/<model_uuid>"]
+    },
+    {"id": "configuration.pipeline_readback",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "Pipeline readback includes its configured model",
+     "bindings": [{"profile": "smoke", "case": "pipeline_readback", "checks": ["response"]}],
+     "routes": ["GET /api/v1/pipelines/<pipeline_uuid>"]
+    },
+    {"id": "configuration.kb_readback",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "Knowledge-base identity and engine match configuration",
+     "bindings": [{"profile": "smoke", "case": "kb_readback", "checks": ["response", "fields"]}],
+     "routes": ["GET /api/v1/knowledge/bases/<knowledge_base_uuid>"]
+    },
+    {"id": "configuration.document_readback",
+     "area": "configuration", "kind": "blackbox", "priority": "P1",
+     "description": "Uploaded document completes ingestion in its owning knowledge base",
+     "bindings": [{"profile": "smoke", "case": "document_readback", "checks": ["response", "fields"]}],
+     "routes": ["GET /api/v1/knowledge/bases/<knowledge_base_uuid>/files"]
+    },
+    {"id": "ingress.missing_signature",
+     "area": "ingress", "kind": "blackbox", "priority": "P0",
+     "description": "Unsigned bot messages are rejected",
+     "bindings": [{"profile": "smoke", "case": "missing_signature", "checks": ["response", "fields"]}]
+    },
+    {"id": "ingress.bad_signature",
+     "area": "ingress", "kind": "blackbox", "priority": "P0",
+     "description": "Messages with an invalid signature are rejected",
+     "bindings": [{"profile": "smoke", "case": "bad_signature", "checks": ["response", "fields"]}]
+    },
+    {"id": "conversation.chat",
+     "area": "conversation", "kind": "blackbox", "priority": "P1",
+     "description": "Pipeline returns the expected chat response",
+     "bindings": [{"profile": "smoke", "case": "chat", "checks": ["response"]}, {"profile": "endurance", "case": "chat", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "conversation.unicode",
+     "area": "conversation", "kind": "blackbox", "priority": "P1",
+     "description": "Unicode input and streamed output survive the full conversation chain",
+     "bindings": [{"profile": "smoke", "case": "unicode", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "model.slow",
+     "area": "model", "kind": "blackbox", "priority": "P1",
+     "description": "Slow streamed model output reaches the caller",
+     "bindings": [{"profile": "smoke", "case": "slow", "checks": ["response", "model_sequence"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "model.fallback401",
+     "area": "model", "kind": "blackbox", "priority": "P1",
+     "description": "Authentication failure falls back to the configured secondary model",
+     "bindings": [{"profile": "smoke", "case": "fallback401", "checks": ["response", "model_sequence"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "model.fallback429",
+     "area": "model", "kind": "blackbox", "priority": "P1",
+     "description": "Rate limiting falls back to the configured secondary model",
+     "bindings": [{"profile": "smoke", "case": "fallback429", "checks": ["response", "model_sequence"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "model.fallback503",
+     "area": "model", "kind": "blackbox", "priority": "P1",
+     "description": "Unavailable primary model falls back to the secondary model",
+     "bindings": [{"profile": "smoke", "case": "fallback503", "checks": ["response", "model_sequence"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "tools.exec_error",
+     "area": "tools", "kind": "blackbox", "priority": "P1",
+     "description": "Tool error output is delivered to the model and handled",
+     "bindings": [{"profile": "smoke", "case": "exec_error", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "tools.parallel",
+     "area": "tools", "kind": "blackbox", "priority": "P1",
+     "description": "Parallel tool calls return matched results to the model",
+     "bindings": [{"profile": "smoke", "case": "parallel", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "delivery.unicode_export",
+     "area": "delivery", "kind": "blackbox", "priority": "P1",
+     "description": "Exported Unicode file name and bytes survive delivery",
+     "bindings": [{"profile": "smoke", "case": "unicode_export", "checks": ["response", "attachment"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "retrieval.other_rag",
+     "area": "retrieval", "kind": "blackbox", "priority": "P1",
+     "description": "A second knowledge base supplies its own content to the model",
+     "bindings": [{"profile": "smoke", "case": "other_rag", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "tools.box",
+     "area": "tools", "kind": "blackbox", "priority": "P1",
+     "description": "A real Box write/read round trip reaches the final reply",
+     "bindings": [{"profile": "smoke", "case": "box", "checks": ["response"]}, {"profile": "endurance", "case": "box", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "delivery.box_export",
+     "area": "delivery", "kind": "blackbox", "priority": "P1",
+     "description": "Exported file bytes match the generated fixture",
+     "bindings": [{"profile": "smoke", "case": "box_export", "checks": ["response", "attachment"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "retrieval.rag_chat",
+     "area": "retrieval", "kind": "blackbox", "priority": "P1",
+     "description": "Retrieved fixture content reaches the model and final reply",
+     "bindings": [{"profile": "smoke", "case": "rag_chat", "checks": ["response"]}, {"profile": "endurance", "case": "rag_chat", "checks": ["response"]}],
+     "routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
+    },
+    {"id": "retrieval.rag_retrieve",
+     "area": "retrieval", "kind": "blackbox", "priority": "P0",
+     "description": "Retrieval returns the owning knowledge-base marker without the other fixture",
+     "bindings": [{"profile": "smoke", "case": "rag_retrieve", "checks": ["response", "fields", "isolation"]}],
+     "routes": ["POST /api/v1/knowledge/bases/<knowledge_base_uuid>/retrieve"]
+    },
+    {"id": "retrieval.retrieve_other",
+     "area": "retrieval", "kind": "blackbox", "priority": "P0",
+     "description": "Retrieval from the second knowledge base excludes the first fixture",
+     "bindings": [{"profile": "smoke", "case": "retrieve_other", "checks": ["response", "fields", "isolation"]}],
+     "routes": ["POST /api/v1/knowledge/bases/<knowledge_base_uuid>/retrieve"]
+    },
+    {"id": "retrieval.retrieve_empty",
+     "area": "retrieval", "kind": "blackbox", "priority": "P1",
+     "description": "Empty knowledge base returns an empty retrieval result",
+     "bindings": [{"profile": "smoke", "case": "retrieve_empty", "checks": ["response", "fields"]}],
+     "routes": ["POST /api/v1/knowledge/bases/<knowledge_base_uuid>/retrieve"]
+    },
+    {"id": "conversation.group_chat_history",
+     "area": "conversation", "kind": "blackbox", "priority": "P0",
+     "description": "Multi-turn chat preserves the prior reply without mixing group history",
+     "bindings": [{"profile": "smoke", "case": "group_chat_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "chat", "checks": ["response"], "min_turn": 2}]
+    },
+    {"id": "conversation.group_box_history",
+     "area": "conversation", "kind": "blackbox", "priority": "P1",
+     "description": "Multi-turn Box conversation preserves group history",
+     "bindings": [{"profile": "smoke", "case": "group_box_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "box", "checks": ["response"], "min_turn": 2}]
+    },
+    {"id": "conversation.group_rag_history",
+     "area": "conversation", "kind": "blackbox", "priority": "P1",
+     "description": "Multi-turn RAG conversation preserves group history",
+     "bindings": [{"profile": "smoke", "case": "group_rag_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "rag_chat", "checks": ["response"], "min_turn": 2}]
+    },
+    {"id": "auth.invalid_credentials",
+     "area": "auth", "kind": "blackbox", "priority": "P0",
+     "description": "Missing and invalid API credentials cannot access Workspace resources",
+     "bindings": []
+    },
+    {"id": "auth.revocation",
+     "area": "auth", "kind": "blackbox", "priority": "P0",
+     "description": "Revoked credentials stop authorizing new requests",
+     "bindings": []
+    },
+    {"id": "workspace.read_isolation",
+     "area": "workspace", "kind": "blackbox", "priority": "P0",
+     "description": "One Workspace cannot read another Workspace resource",
+     "bindings": []
+    },
+    {"id": "workspace.write_isolation",
+     "area": "workspace", "kind": "blackbox", "priority": "P0",
+     "description": "One Workspace cannot change another Workspace resource",
+     "bindings": []
+    },
+    {"id": "execution.cancel_cleanup",
+     "area": "execution", "kind": "blackbox", "priority": "P0",
+     "description": "Cancelling a running execution reaches cancelled state and releases its work",
+     "bindings": []
+    },
+    {"id": "execution.timeout_cleanup",
+     "area": "execution", "kind": "blackbox", "priority": "P0",
+     "description": "Deadline expiry stops child work and allows a subsequent request",
+     "bindings": []
+    },
+    {"id": "execution.retry_side_effects",
+     "area": "execution", "kind": "blackbox", "priority": "P0",
+     "description": "Retries do not duplicate externally visible side effects",
+     "bindings": []
+    },
+    {"id": "runtime.restart_recovery",
+     "area": "runtime", "kind": "blackbox", "priority": "P0",
+     "description": "A runtime restart produces a known outcome and subsequent execution works",
+     "bindings": []
+    },
+    {"id": "storage.restart_history",
+     "area": "storage", "kind": "blackbox", "priority": "P1",
+     "description": "Conversation history remains usable across a Core restart",
+     "bindings": []
+    },
+    {"id": "model.stream_disconnect",
+     "area": "model", "kind": "blackbox", "priority": "P1",
+     "description": "A partial model stream disconnect is handled without a false successful completion",
+     "bindings": []
+    },
+    {"id": "model.exhausted_fallback",
+     "area": "model", "kind": "blackbox", "priority": "P1",
+     "description": "Failure of every configured model reaches a visible failed outcome",
+     "bindings": []
+    },
+    {"id": "tools.path_escape",
+     "area": "tools", "kind": "blackbox", "priority": "P0",
+     "description": "Box operations reject access outside their authorized filesystem scope",
+     "bindings": []
+    },
+    {"id": "tools.timeout_recovery",
+     "area": "tools", "kind": "blackbox", "priority": "P1",
+     "description": "Timed-out tools stop running and the next tool can execute",
+     "bindings": []
+    },
+    {"id": "delivery.callback_retry",
+     "area": "delivery", "kind": "blackbox", "priority": "P1",
+     "description": "Failed delivery callbacks retry according to policy without duplicate effects",
+     "bindings": []
+    },
+    {"id": "retrieval.update_delete",
+     "area": "retrieval", "kind": "blackbox", "priority": "P1",
+     "description": "Updated/deleted documents change retrieval results without stale leakage",
+     "bindings": []
+    },
+    {"id": "plugins.upgrade_recovery",
+     "area": "plugins", "kind": "blackbox", "priority": "P1",
+     "description": "Plugin upgrade failure preserves a usable installation",
+     "bindings": []
+    },
+    {"id": "agent.direct_execution",
+     "area": "agent", "kind": "blackbox", "priority": "P1",
+     "description": "Standalone Agent entrypoint completes through the public API",
+     "bindings": []
+    },
+    {"id": "persistence.postgres_parity",
+     "area": "persistence", "kind": "blackbox", "priority": "P1",
+     "description": "The same externally visible run contract holds on PostgreSQL",
+     "bindings": []
+    },
+    {"id": "platform.real_adapter",
+     "area": "platform", "kind": "blackbox", "priority": "P2",
+     "description": "An actual messaging-platform adapter receives and delivers the fixture",
+     "bindings": []
+    },
+    {"id": "retrieval.semantic_embedding",
+     "area": "retrieval", "kind": "blackbox", "priority": "P2",
+     "description": "A real embedding model retrieves relevant fixed documents",
+     "bindings": []
+    },
+    {"id": "execution.persisted_terminal",
+     "area": "execution", "kind": "graybox", "priority": "P0",
+     "description": "Each executed request has a unique completed persisted run and terminal event",
+     "bindings": [{"profile": "smoke", "case": "chat", "checks": ["ledger"]}, {"profile": "endurance", "case": "chat", "checks": ["ledger"]}]
+    },
+    {"id": "tools.persisted_box_results",
+     "area": "tools", "kind": "graybox", "priority": "P0",
+     "description": "Persisted Box events match backend, exit status and output",
+     "bindings": [{"profile": "smoke", "case": "box", "checks": ["ledger"]}, {"profile": "endurance", "case": "box", "checks": ["ledger"]}]
+    },
+    {"id": "ingress.no_run_on_reject",
+     "area": "ingress", "kind": "graybox", "priority": "P0",
+     "description": "Rejected ingress does not create an AgentRun",
+     "bindings": [{"profile": "smoke", "case": "missing_signature", "checks": ["no_run"]}]
+    }
+  ]
+}

tools/regression_coverage.py

@@ -0,0 +1,349 @@
+#!/usr/bin/env python3
+"""Report explicitly asserted business contracts, independently of pass rate."""
+
+import argparse
+import ast
+from collections import Counter
+import json
+import hashlib
+import re
+from urllib.parse import urlsplit
+from pathlib import Path
+
+CATALOG = Path(__file__).with_name('regression_contracts.json')
+STATES = ('passed', 'failed', 'incomplete', 'skipped', 'not_run', 'missing_test')
+
+
+def checked(row, name, action):
+    """Record an evaluated assertion; transport failures leave it unverified."""
+    assertions = row.setdefault('assertions', {})
+    try:
+        result = action()
+    except ValueError:
+        assertions[name] = 'failed'
+        raise
+    assertions[name] = 'passed'
+    return result
+
+
+def check_response(data, expected, forbidden=()):
+    text = json.dumps(data, ensure_ascii=False)
+    if expected not in text:
+        raise ValueError('Response does not contain the expected business result')
+    if any(value in text for value in forbidden):
+        raise ValueError('Response contains forbidden content')
+
+
+def load_catalog(path=CATALOG):
+    catalog = json.loads(Path(path).read_text())
+    if catalog.get('version') != 1 or not catalog.get('scope'):
+        raise ValueError('Coverage catalog requires version 1 and an explicit scope')
+    seen = set()
+    for item in catalog['contracts']:
+        if item['id'] in seen or item['kind'] not in ('blackbox', 'graybox'):
+            raise ValueError('Coverage contract IDs must be unique and kinds explicit')
+        seen.add(item['id'])
+        if not item['description'] or not item['area'] or item['priority'] not in ('P0', 'P1', 'P2'):
+            raise ValueError('Coverage contracts require a description, area and priority')
+        for binding in item['bindings']:
+            if not binding['profile'] or not binding['case'] or not binding['checks']:
+                raise ValueError('Coverage bindings require a profile, case and actual assertions')
+    return catalog
+
+
+def collect(directory):
+    directory = Path(directory)
+    profile = json.loads((directory / 'profile.json').read_text())
+    suite = profile.get('coverage_suite', 'unclassified')
+    sources = (
+        [directory / 'cases.json']
+        if 'smoke' in profile
+        else [directory / phase['name'] / 'requests.jsonl' for phase in profile.get('phases', [])]
+    )
+    rows, errors = [], []
+    for path in sources:
+        if not path.exists():
+            errors.append(f'{path}: missing executed-case evidence')
+            continue
+        try:
+            data = (
+                [json.loads(line) for line in path.read_text().splitlines()]
+                if path.suffix == '.jsonl'
+                else json.loads(path.read_text())
+            )
+            if not isinstance(data, list) or any(
+                not isinstance(row, dict)
+                or row.get('status') not in ('passed', 'ok', 'failed', 'unknown', 'skipped')
+                or not isinstance(row.get('case'), str)
+                or not isinstance(row.get('assertions', {}), dict)
+                or any(value not in ('passed', 'failed') for value in row.get('assertions', {}).values())
+                or not isinstance(row.get('turn', 0), int)
+                for row in data
+            ):
+                raise ValueError('Malformed executed-case evidence')
+            if not data:
+                errors.append(f'{path}: empty executed-case evidence')
+            rows.extend({**row, 'profile': suite, 'source': str(path)} for row in data)
+        except (ValueError, OSError) as exc:
+            errors.append(f'{path}: {exc}')
+    missing = {case['name'] for case in profile.get('cases', [])} - {row['case'] for row in rows}
+    if missing:
+        errors.append(f'{directory}: no evidence for planned cases: {sorted(missing)}')
+    return rows, errors
+
+
+def route_inventory(catalog, root=None):
+    """Discover RouterGroup HTTP declarations; links express test intent, not endpoint hits."""
+    root = Path(root) if root else Path(__file__).resolve().parents[1]
+    routes, unresolved = [], []
+    for path in sorted((root / 'src/langbot/pkg').rglob('*.py')):
+        if 'group_class' not in path.read_text():
+            continue
+        tree = ast.parse(path.read_text())
+        for cls in (node for node in ast.walk(tree) if isinstance(node, ast.ClassDef)):
+            groups = [
+                d
+                for d in cls.decorator_list
+                if isinstance(d, ast.Call) and isinstance(d.func, ast.Attribute) and d.func.attr == 'group_class'
+            ]
+            if not groups:
+                continue
+            try:
+                prefix = ast.literal_eval(groups[0].args[1])
+            except (ValueError, IndexError):
+                unresolved.append(f'{path.relative_to(root)}:{cls.lineno}')
+                continue
+            for func in (node for node in ast.walk(cls) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))):
+                for dec in func.decorator_list:
+                    if (
+                        not isinstance(dec, ast.Call)
+                        or not isinstance(dec.func, ast.Attribute)
+                        or dec.func.attr != 'route'
+                    ):
+                        continue
+                    location = f'{path.relative_to(root)}:{func.lineno}'
+                    try:
+                        rule = ast.literal_eval(dec.args[0])
+                        methods = next((ast.literal_eval(k.value) for k in dec.keywords if k.arg == 'methods'), ['GET'])
+                        if not isinstance(prefix, str) or not isinstance(rule, str) or not isinstance(methods, list):
+                            raise ValueError('Dynamic route declaration')
+                    except (ValueError, IndexError):
+                        unresolved.append(location)
+                        continue
+                    for method in methods:
+                        key = f'{method.upper()} {prefix}{rule}'
+                        routes.append(
+                            {
+                                'route': key,
+                                'source': location,
+                                'contracts': [c['id'] for c in catalog['contracts'] if key in c.get('routes', [])],
+                            }
+                        )
+    return {
+        'scope': 'Static RouterGroup HTTP declarations only; excludes WebSockets, platform-dispatched and dynamic routes. Contract links are intent, not HTTP hit coverage.',
+        'routes': routes,
+        'unresolved': unresolved,
+    }
+
+
+def http_observations(paths, inventory):
+    """Map Hypercorn access records to declared routes without retaining request URLs."""
+    patterns = []
+    for route in inventory['routes']:
+        method, path = route['route'].split(' ', 1)
+        parts = re.split(r'(<[^>]+>)', path)
+        pattern = ''.join(
+            ('.+' if part.startswith('<path:') else '[^/]+')
+            if part.startswith('<') and part.endswith('>')
+            else re.escape(part)
+            for part in parts
+        )
+        literal_length = sum(len(part) for part in parts if not part.startswith('<'))
+        patterns.append((literal_length, method, re.compile(pattern), route['route']))
+    patterns.sort(key=lambda item: item[0], reverse=True)
+    access = re.compile(r'\b(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) (\S+) (?:HTTP/)?[123]\.[01] (\d{3})\b')
+    matched, unclassified, records, errors = {}, Counter(), 0, []
+    for path in paths:
+        try:
+            with Path(path).open() as stream:
+                for line in stream:
+                    match = access.search(line)
+                    if not match:
+                        continue
+                    method, target, status = match.groups()
+                    records += 1
+                    request_path = urlsplit(target).path
+                    route = next(
+                        (
+                            name
+                            for _, expected_method, pattern, name in patterns
+                            if method == expected_method and pattern.fullmatch(request_path)
+                        ),
+                        None,
+                    )
+                    if route is None:
+                        unclassified[f'{method} {status}'] += 1
+                    else:
+                        matched.setdefault(route, Counter())[status] += 1
+        except (OSError, ValueError) as exc:
+            errors.append(f'{path}: {exc}')
+    if paths and not records:
+        errors.append('No HTTP access records were recognized')
+    return {
+        'scope': 'Observed HTTP methods and declared route templates; this does not prove behavior assertions or adapter sub-path coverage.',
+        'records': records,
+        'routes': [{'route': route, 'statuses': dict(statuses)} for route, statuses in sorted(matched.items())],
+        'unclassified': dict(unclassified),
+        'evidence_errors': errors,
+    }
+
+
+def counts(items):
+    status = Counter(item['status'] for item in items)
+    decided = status['passed'] + status['failed']
+    return {
+        'total': len(items),
+        'decided': decided,
+        'coverage_percent': round(100 * decided / len(items), 2) if items else None,
+        'pass_percent': round(100 * status['passed'] / decided, 2) if decided else None,
+        'status': {name: status[name] for name in STATES},
+    }
+
+
+def report(catalog, rows, errors=()):
+    items, bound = [], set()
+    for contract in catalog['contracts']:
+        observations = []
+        for binding in contract['bindings']:
+            bound.add((binding['profile'], binding['case']))
+            for row in rows:
+                if (row['profile'], row['case']) != (binding['profile'], binding['case']):
+                    continue
+                if row.get('turn', 0) < binding.get('min_turn', 0):
+                    continue
+                checks = [row.get('assertions', {}).get(name) for name in binding['checks']]
+                status = (
+                    'failed'
+                    if 'failed' in checks
+                    else 'skipped'
+                    if row['status'] == 'skipped'
+                    else 'passed'
+                    if all(value == 'passed' for value in checks)
+                    else 'incomplete'
+                )
+                observations.append({'status': status, 'case': row['case'], 'profile': row['profile']})
+        statuses = {row['status'] for row in observations}
+        status = (
+            'missing_test'
+            if not contract['bindings']
+            else 'not_run'
+            if not observations
+            else 'failed'
+            if 'failed' in statuses
+            else 'skipped'
+            if statuses == {'skipped'}
+            else 'passed'
+            if statuses == {'passed'}
+            else 'incomplete'
+        )
+        items.append({**contract, 'status': status, 'observations': dict(Counter(x['status'] for x in observations))})
+    return {
+        'version': 1,
+        'scope': catalog['scope'],
+        'catalog_version': catalog['version'],
+        'catalog_sha256': hashlib.sha256(json.dumps(catalog, sort_keys=True).encode()).hexdigest(),
+        'complete_evidence': not errors,
+        'kinds': {kind: counts([x for x in items if x['kind'] == kind]) for kind in ('blackbox', 'graybox')},
+        'areas': {
+            area: counts([x for x in items if x['kind'] == 'blackbox' and x['area'] == area])
+            for area in sorted({x['area'] for x in items if x['kind'] == 'blackbox'})
+        },
+        'contracts': items,
+        'unclassified_cases': [
+            {'profile': p, 'case': c} for p, c in sorted({(row['profile'], row['case']) for row in rows} - bound)
+        ],
+        'evidence_errors': list(errors),
+    }
+
+
+def write_report(directories, output, catalog_path=CATALOG, http_logs=()):
+    rows, errors = [], []
+    for directory in directories:
+        try:
+            found, invalid = collect(directory)
+            rows.extend(found)
+            errors.extend(invalid)
+        except (ValueError, OSError, KeyError) as exc:
+            errors.append(f'{directory}: {exc}')
+    catalog = load_catalog(catalog_path)
+    data = report(catalog, rows, errors)
+    data['route_inventory'] = route_inventory(catalog)
+    if http_logs:
+        data['http_observations'] = http_observations(http_logs, data['route_inventory'])
+        errors.extend(data['http_observations']['evidence_errors'])
+        data['evidence_errors'] = list(errors)
+        data['complete_evidence'] = not errors
+    output = Path(output)
+    output.mkdir(parents=True, exist_ok=True)
+    (output / 'coverage.json').write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
+    lines = [
+        '# Business behavior coverage',
+        '',
+        data['scope'],
+        '',
+        'Coverage counts evaluated contracts (pass or fail), not requests or code lines.',
+        'Incomplete evidence and skipped/not-run contracts do not count. Gray-box evidence is separate.',
+        '',
+        '| Evidence | Decided / registered | Coverage | Pass rate |',
+        '|---|---:|---:|---:|',
+    ]
+    for kind, values in data['kinds'].items():
+        lines.append(
+            f'| {kind} | {values["decided"]}/{values["total"]} | {values["coverage_percent"]}% | {values["pass_percent"]}% |'
+        )
+    lines += ['', '| Priority | Contract | Area | Evidence | Status |', '|---|---|---|---|---|']
+    for item in sorted(data['contracts'], key=lambda x: (x['priority'], x['area'], x['id'])):
+        lines.append(
+            f'| {item["priority"]} | {item["id"]}: {item["description"]} | {item["area"]} | {item["kind"]} | {item["status"]} |'
+        )
+    inventory = data['route_inventory']
+    unlinked = sum(not r['contracts'] for r in inventory['routes'])
+    lines += [
+        '',
+        f'Route inventory: {len(inventory["routes"])} HTTP declarations; {unlinked} have no linked behavior contract; {len(inventory["unresolved"])} need manual classification.',
+        inventory['scope'],
+        'Full route list and source locations are in coverage.json.',
+    ]
+    if http_logs:
+        observed = data['http_observations']
+        lines += [
+            '',
+            f'Observed HTTP: {len(observed["routes"])} declared routes, {observed["records"]} access records.',
+            observed['scope'],
+            '| Route | HTTP statuses and counts |',
+            '|---|---|',
+        ]
+        lines.extend(f'| {row["route"]} | {row["statuses"]} |' for row in observed['routes'])
+        lines.append(f'Unclassified method/status counts: {observed["unclassified"]}')
+    for item in data['unclassified_cases']:
+        lines.append(f'\nUnclassified case: {item["profile"]}/{item["case"]}')
+    for error in errors:
+        lines.append(f'\nEvidence error: {error}')
+    (output / 'coverage.md').write_text('\n'.join(lines) + '\n')
+    return data
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--run', type=Path, action='append', required=True)
+    parser.add_argument('--output', type=Path, required=True)
+    parser.add_argument('--http-log', type=Path, action='append', default=[])
+    args = parser.parse_args()
+    data = write_report(args.run, args.output, http_logs=args.http_log)
+    print(json.dumps(data['kinds'], indent=2))
+    if data['evidence_errors']:
+        raise SystemExit(1)
+
+
+if __name__ == '__main__':
+    main()

tools/regression_profiles/endurance.json

@@ -509,5 +509,6 @@
       "pids": 8,
       "count": 0
     }
-  }
+  },
+  "coverage_suite": "endurance"
 }

tools/regression_profiles/smoke.json

@@ -1640,5 +1640,6 @@
       "langbot-team/GeneralParsers"
     ],
     "mock_records": "${mock_records}"
-  }
+  },
+  "coverage_suite": "smoke"
 }

tools/regression_smoke.py

@@ -12,6 +12,7 @@ import urllib.request
 from uuid import uuid4
 
 from regression_traffic import group_request, read_endpoint_snapshot
+from regression_coverage import checked, check_response
 
 
 class UnresolvedRun(RuntimeError):
@@ -241,27 +242,39 @@ def run_smoke(suite):
                     offset = mock_path.stat().st_size if case.get('mock_requests') and mock_path.exists() else 0
                     try:
                         if rejected:
-                            data = rejected_group(case, group)
+                            data = checked(row, 'response', lambda: rejected_group(case, group))
                             count = connection.execute(
                                 'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND conversation_id=? AND id>?',
                                 (suite.variables['workspace_uuid'], 'group_' + group, after_id),
                             ).fetchone()[0]
-                            if count:
-                                raise ValueError('Rejected ingress created an AgentRun')
+                            checked(row, 'no_run', lambda: check_fields({'runs': count}, {'runs': 0}))
                         elif is_group:
-                            data = group_request(
-                                case, group, row['member_id'], turn, token, previous[group], suite.env
+                            data = checked(
+                                row,
+                                'response',
+                                lambda: group_request(
+                                    case, group, row['member_id'], turn, token, previous[group], suite.env
+                                ),
                             )['data']
                         else:
                             data = suite.call(case['args'], case.get('body'))
-                            if case['expect'] not in json.dumps(data, ensure_ascii=False):
-                                raise ValueError('Expected business response was not returned')
+                            checked(row, 'response', lambda: check_response(data, case['expect']))
                         row['output'] = data
-                        if any(value in json.dumps(data, ensure_ascii=False) for value in case.get('forbid', [])):
-                            raise ValueError('Response contains forbidden content from another knowledge base')
+                        if case.get('forbid'):
+                            checked(row, 'isolation', lambda: check_response(data, case['expect'], case['forbid']))
                         row['checks'] = {}
                         if case.get('fields'):
-                            row['checks']['fields'] = check_fields(data, case['fields'])
+                            row['checks']['fields'] = checked(row, 'fields', lambda: check_fields(data, case['fields']))
+                        if case.get('attachment'):
+                            row['checks']['attachment'] = checked(
+                                row, 'attachment', lambda: check_attachment(data, case['attachment'])
+                            )
+                        if case.get('mock_requests'):
+                            row['checks']['mock_requests'] = checked(
+                                row,
+                                'model_sequence',
+                                lambda: check_mock_requests(mock_path, offset, case['mock_requests']),
+                            )
                         if runner:
                             if not isinstance(data, dict):
                                 raise ValueError('Runner response must be an object')
@@ -276,13 +289,7 @@ def run_smoke(suite):
                                 settings.get('timeout_seconds', 30),
                             )
                             row['run'] = run
-                            row['checks'].update(check_run(connection, run, case))
-                            if case.get('attachment'):
-                                row['checks']['attachment'] = check_attachment(data, case['attachment'])
-                        if case.get('mock_requests'):
-                            row['checks']['mock_requests'] = check_mock_requests(
-                                mock_path, offset, case['mock_requests']
-                            )
+                            row['checks'].update(checked(row, 'ledger', lambda: check_run(connection, run, case)))
                         row['status'] = 'passed'
                         previous[group] = token
                     except urllib.error.HTTPError as exc:

tools/regression_suite.py

@@ -80,6 +80,7 @@ def validate(profile):
         'smoke',
         'verification',
         'resource_limits',
+        'coverage_suite',
     }
     if extra := profile.keys() - allowed:
         raise ValueError(f'unknown profile fields: {sorted(extra)}')
@@ -91,6 +92,8 @@ def validate(profile):
             not isinstance(profile[field], list) or any(not isinstance(item, dict) for item in profile[field])
         ):
             raise ValueError(f'{field} must be an array of objects')
+    if 'coverage_suite' in profile and not re.fullmatch(r'[a-z][a-z0-9_-]*', str(profile['coverage_suite'])):
+        raise ValueError('coverage_suite must be a stable suite identifier')
     cli = profile.get('cli', {})
     if not isinstance(cli, dict) or cli.keys() - {'config', 'executable'}:
         raise ValueError('cli accepts config and executable')
@@ -778,6 +781,11 @@ def main():
                         result['errors'].extend(suite.cleanup())
                     except (RuntimeError, ValueError, OSError, KeyError) as exc:
                         result['errors'].append(f'cleanup failed: {exc}')
+            if profile.get('coverage_suite'):
+                from regression_coverage import write_report
+
+                coverage = write_report([suite.output], suite.output)
+                result['errors'].extend(coverage['evidence_errors'])
             result['passed'] = not result['errors']
             suite.write('summary.json', result)
         print(json.dumps(result, ensure_ascii=False, indent=2))

tools/regression_traffic.py

@@ -3,6 +3,7 @@
 
 import argparse
 from collections import Counter
+from regression_coverage import checked, check_response
 from concurrent.futures import ThreadPoolExecutor
 import json
 import hashlib
@@ -308,9 +309,11 @@ def main():
         data = row['output']['data']
         checks = {}
         if case.get('fields'):
-            checks['fields'] = check_fields(data, case['fields'])
-        if any(value in json.dumps(data, ensure_ascii=False) for value in case.get('forbid', [])):
-            raise ValueError('Response contains forbidden content')
+            checks['fields'] = checked(row, 'fields', lambda: check_fields(data, case['fields']))
+        if case.get('forbid'):
+            checked(row, 'isolation', lambda: check_response(data, case['expect'], case['forbid']))
+        if case.get('attachment'):
+            checks['attachment'] = checked(row, 'attachment', lambda: check_attachment(data, case['attachment']))
         runner = is_group or case.get('args', [])[:2] == ['pipeline', 'run']
         row['requires_runner'] = runner
         if connection and runner:
@@ -330,13 +333,16 @@ def main():
                 args.verification_timeout,
             )
             row['run'] = run
-            checks.update(check_run(connection, run, case))
-            if case.get('attachment'):
-                checks['attachment'] = check_attachment(data, case['attachment'])
-            with lock:
-                if run['run_id'] in verified_ids:
-                    raise ValueError('Two requests matched the same AgentRun')
-                verified_ids.add(run['run_id'])
+
+            def verify_ledger():
+                result = check_run(connection, run, case)
+                with lock:
+                    if run['run_id'] in verified_ids:
+                        raise ValueError('Two requests matched the same AgentRun')
+                    verified_ids.add(run['run_id'])
+                return result
+
+            checks.update(checked(row, 'ledger', verify_ledger))
         row['checks'] = checks
         row['verification_seconds'] = time.monotonic() - began
 
@@ -415,9 +421,8 @@ def main():
                 row['output'] = json.loads(redact(result.stdout))
                 if not isinstance(row['output'], dict):
                     raise ValueError('CLI response must be an object')
-                if result.returncode == 0 and case['expect'] in json.dumps(
-                    row['output'].get('data'), ensure_ascii=False
-                ):
+                if result.returncode == 0:
+                    checked(row, 'response', lambda: check_response(row['output'].get('data'), case['expect']))
                     verify(row, case, connection, after_id)
                     row['status'] = 'ok'
                 elif 'result_unknown' in result.stdout:
@@ -483,7 +488,11 @@ def main():
                     elapsed=dispatch - started,
                     schedule_lag_seconds=max(0, dispatch - group['due']),
                 )
-                row['output'] = group_request(case, group['id'], member, group['turn'], token, group['previous'])
+                row['output'] = checked(
+                    row,
+                    'response',
+                    lambda: group_request(case, group['id'], member, group['turn'], token, group['previous']),
+                )
                 row['seconds'] = time.monotonic() - dispatch
                 verify(row, case, connection, after_id, is_group=True)
                 row['status'] = 'ok'

tools/test_regression_coverage.py

@@ -0,0 +1,236 @@
+"""Coverage must measure evaluated obligations, not green request counters."""
+
+import copy
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+from regression_coverage import checked, collect, http_observations, load_catalog, report, route_inventory, write_report
+
+
+def catalog():
+    return {
+        'version': 1,
+        'scope': 'Test scope',
+        'contracts': [
+            {
+                'id': 'reply',
+                'area': 'conversation',
+                'kind': 'blackbox',
+                'priority': 'P0',
+                'description': 'Reply and payload agree',
+                'bindings': [{'profile': 'smoke', 'case': 'chat', 'checks': ['response', 'fields']}],
+            },
+            {
+                'id': 'terminal',
+                'area': 'execution',
+                'kind': 'graybox',
+                'priority': 'P0',
+                'description': 'Persisted terminal state',
+                'bindings': [{'profile': 'smoke', 'case': 'chat', 'checks': ['ledger']}],
+            },
+            {
+                'id': 'recovery',
+                'area': 'execution',
+                'kind': 'blackbox',
+                'priority': 'P0',
+                'description': 'Restart recovery',
+                'bindings': [],
+            },
+        ],
+    }
+
+
+def row(**overrides):
+    return {'profile': 'smoke', 'case': 'chat', 'status': 'passed', **overrides}
+
+
+class CoverageTest(unittest.TestCase):
+    def test_old_green_rows_and_transport_failures_are_not_coverage(self):
+        for r in [row(), row(status='unknown'), row(assertions={'response': 'passed'})]:
+            data = report(catalog(), [r])
+            self.assertEqual(data['kinds']['blackbox']['decided'], 0)
+            self.assertEqual(data['contracts'][0]['status'], 'incomplete')
+        r = {}
+        with self.assertRaises(TimeoutError):
+            checked(r, 'response', lambda: (_ for _ in ()).throw(TimeoutError('transport')))
+        self.assertNotIn('response', r['assertions'])
+
+    def test_a_failed_assertion_is_covered_but_not_passed(self):
+        r = row()
+        with self.assertRaises(ValueError):
+            checked(r, 'response', lambda: (_ for _ in ()).throw(ValueError('wrong reply')))
+        data = report(catalog(), [r])
+        self.assertEqual(data['kinds']['blackbox']['coverage_percent'], 50)
+        self.assertEqual(data['kinds']['blackbox']['pass_percent'], 0)
+        self.assertEqual(data['contracts'][0]['status'], 'failed')
+
+    def test_graybox_failure_does_not_erase_external_assertions(self):
+        data = report(
+            catalog(),
+            [
+                row(
+                    status='failed',
+                    assertions={
+                        'response': 'passed',
+                        'fields': 'passed',
+                        'ledger': 'failed',
+                    },
+                )
+            ],
+        )
+        self.assertEqual(data['contracts'][0]['status'], 'passed')
+        self.assertEqual(data['contracts'][1]['status'], 'failed')
+        self.assertEqual(data['contracts'][2]['status'], 'missing_test')
+
+    def test_repetition_does_not_inflate_coverage_or_hide_incomplete_turns(self):
+        good = row(assertions={'response': 'passed', 'fields': 'passed'})
+        self.assertEqual(report(catalog(), [good])['kinds'], report(catalog(), [good] * 100)['kinds'])
+        mixed = report(catalog(), [good, row(status='unknown')])
+        self.assertEqual(mixed['contracts'][0]['status'], 'incomplete')
+        self.assertEqual(report(catalog(), [row(status='skipped')])['contracts'][0]['status'], 'skipped')
+        self.assertEqual(report(catalog(), [])['contracts'][0]['status'], 'not_run')
+
+    def test_first_turn_does_not_prove_history_and_unclassified_cases_are_visible(self):
+        c = catalog()
+        c['contracts'][0]['bindings'][0]['min_turn'] = 2
+        r = row(turn=1, assertions={'response': 'passed', 'fields': 'passed'})
+        data = report(c, [r, row(case='new_case')])
+        self.assertEqual(data['contracts'][0]['status'], 'not_run')
+        self.assertEqual(data['unclassified_cases'], [{'profile': 'smoke', 'case': 'new_case'}])
+
+    def test_partial_evidence_produces_an_error_and_a_report(self):
+        with tempfile.TemporaryDirectory() as d:
+            root = Path(d)
+            (root / 'profile.json').write_text(json.dumps({'coverage_suite': 'smoke', 'smoke': {}}))
+            (root / 'cases.json').write_text('[{"case":')
+            rows, errors = collect(root)
+            self.assertEqual(rows, [])
+            self.assertEqual(len(errors), 1)
+            result = write_report([root], root)
+            self.assertTrue(result['evidence_errors'])
+            self.assertEqual(result['kinds']['blackbox']['decided'], 0)
+            self.assertTrue((root / 'coverage.md').is_file())
+
+    def test_missing_phase_evidence_cannot_hide_behind_a_successful_phase(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'profile.json').write_text(
+                json.dumps(
+                    {
+                        'coverage_suite': 'smoke',
+                        'phases': [{'name': 'first'}, {'name': 'second'}],
+                    }
+                )
+            )
+            (root / 'first').mkdir()
+            (root / 'first/requests.jsonl').write_text(
+                json.dumps(
+                    row(
+                        assertions={'response': 'passed', 'fields': 'passed'},
+                    )
+                )
+                + '\n'
+            )
+            rows, errors = collect(root)
+            self.assertEqual(len(rows), 1)
+            self.assertEqual(len(errors), 1)
+            self.assertFalse(report(catalog(), rows, errors)['complete_evidence'])
+
+    def test_empty_and_absent_planned_cases_are_explicit_evidence_errors(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'profile.json').write_text(
+                json.dumps(
+                    {
+                        'coverage_suite': 'smoke',
+                        'smoke': {},
+                        'cases': [{'name': 'chat'}],
+                    }
+                )
+            )
+            (root / 'cases.json').write_text('[]')
+            rows, errors = collect(root)
+            self.assertEqual(rows, [])
+            self.assertTrue(any('empty' in error for error in errors))
+            self.assertTrue(any('planned cases' in error for error in errors))
+
+    def test_http_hits_use_method_templates_and_do_not_retain_private_urls(self):
+        inventory = {
+            'routes': [
+                {'route': value}
+                for value in (
+                    'GET /things/<uuid>',
+                    'GET /things/new',
+                    'POST /things/<uuid>',
+                    'POST /bots/<id>/<path:path>',
+                )
+            ]
+        }
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'access.log'
+            path.write_text(
+                '\n'.join(
+                    [
+                        'GET /things/new?token=secret 1.1 200',
+                        'GET /things/private-id 1.1 404',
+                        'POST /things/private-id 1.1 403',
+                        'POST /bots/private-bot/reset 1.1 200',
+                        'GET /unknown/private-id 1.1 404',
+                        'application log is not a request',
+                    ]
+                )
+            )
+            result = http_observations([path], inventory)
+            self.assertEqual(result['records'], 5)
+            self.assertEqual(len(result['routes']), 4)
+            self.assertEqual(result['unclassified'], {'GET 404': 1})
+            encoded = json.dumps(result)
+            for value in ('private-id', 'private-bot', 'secret'):
+                self.assertNotIn(value, encoded)
+            self.assertTrue(http_observations([path.with_name('missing')], inventory)['evidence_errors'])
+
+    def test_new_routes_are_discovered_without_adding_test_cases(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source = root / 'src/langbot/pkg/example.py'
+            source.parent.mkdir(parents=True)
+            source.write_text("""
+@group.group_class('example', '/api/example')
+class Routes:
+    async def initialize(self):
+        @self.route('/new', methods=['POST'])
+        async def new_operation(): pass
+        @self.route(dynamic_path)
+        async def dynamic_operation(): pass
+""")
+            inventory = route_inventory(catalog(), root)
+            self.assertEqual(inventory['routes'][0]['route'], 'POST /api/example/new')
+            self.assertEqual(inventory['routes'][0]['contracts'], [])
+            self.assertEqual(len(inventory['unresolved']), 1)
+
+    def test_catalog_bindings_name_real_cases_and_required_checks(self):
+        c = load_catalog()
+        profiles = {}
+        routes = {r['route'] for r in route_inventory(c)['routes']}
+        for contract in c['contracts']:
+            self.assertTrue(set(contract.get('routes', [])) <= routes)
+            for binding in contract['bindings']:
+                name = binding['profile']
+                if name not in profiles:
+                    path = Path(__file__).with_name('regression_profiles') / f'{name}.json'
+                    profiles[name] = {x['name']: x for x in json.loads(path.read_text())['cases']}
+                self.assertIn(binding['case'], profiles[name])
+                self.assertTrue(binding['checks'])
+        broken = copy.deepcopy(c)
+        broken['contracts'].append(broken['contracts'][0])
+        with tempfile.TemporaryDirectory() as d:
+            path = Path(d) / 'catalog.json'
+            path.write_text(json.dumps(broken))
+            with self.assertRaises(ValueError):
+                load_catalog(path)
+
+
+if __name__ == '__main__':
+    unittest.main()