From 81996e07b7a34a1e0b5d41568e6bf60d1da3f8d5 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:26:48 +0800 Subject: [PATCH] test(regression): report asserted behavior coverage and observed HTTP routes --- tools/README.md | 45 +++ tools/regression_contracts.json | 283 ++++++++++++++++++ tools/regression_coverage.py | 349 +++++++++++++++++++++++ tools/regression_profiles/endurance.json | 3 +- tools/regression_profiles/smoke.json | 3 +- tools/regression_smoke.py | 41 +-- tools/regression_suite.py | 8 + tools/regression_traffic.py | 37 ++- tools/test_regression_coverage.py | 236 +++++++++++++++ 9 files changed, 972 insertions(+), 33 deletions(-) create mode 100644 tools/regression_contracts.json create mode 100644 tools/regression_coverage.py create mode 100644 tools/test_regression_coverage.py diff --git a/tools/README.md b/tools/README.md index 1b3c00e82..3b33cfcc7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -474,3 +474,48 @@ Keep this longer test outside the one-minute merge gate while its budgets and repeatability are being evaluated. The same profile can be selected by the release-deployment Job, which supplies its ledger path and process IDs; that fresh-deployment path still needs separate end-to-end validation. + +## Business behavior coverage + +`regression_contracts.json` declares behavior obligations independently of how +many requests a workload repeats. It includes explicit unimplemented risks +(cancellation, restart recovery, credential revocation, Workspace isolation, +PostgreSQL parity, real adapters and embeddings). This is coverage of the +registered scope, not a percentage of all product behavior or API endpoints. + +Runners record named assertions as they execute. A green legacy request without +assertion evidence is **incomplete**. An evaluated failing assertion counts as +covered and failed; a timeout before an assertion does not count. A contract with +multiple required assertions passes only when all are observed. Repeated +observations cannot hide an incomplete or failed turn. History contracts require +a subsequent turn, not merely the initial reply. Database/Box-event inspections +are reported separately as gray-box evidence. + +Reports distinguish `passed`, `failed`, `incomplete`, `skipped`, `not_run` (a test +exists but did not execute) and `missing_test`. Coverage and pass rate have separate +denominators. The catalog fingerprint identifies the registered denominator. +New case names with no contract appear as unclassified, and malformed evidence +is reported as an error instead of silently accepted. + +The source route inventory discovers current static RouterGroup HTTP declarations +and flags routes without a linked behavior contract. Links describe test intent; +they are not observed HTTP hits or proof that every method behavior is tested. +Dynamic routes, WebSockets and platform-dispatched routes are outside this static +inventory. Full source locations and any unresolved declarations are retained +in `coverage.json`; new declarations appear without editing the case list. + +```sh +python tools/regression_coverage.py --run .qa/smoke/suite \ + --run .qa/endurance/suite --output .qa/behavior-coverage +``` + +`coverage.json` and `coverage.md` are written by the Suite. CI consumers can +publish these files as artifacts. Uncovered contracts do not add a merge gate. A missing or corrupt evidence +file is a reporting failure; the report remains available for inspection. + +Pass `--http-log /path/to/core.log` to include observed methods, declared route +templates and status counts from Hypercorn access records. Query strings and +resource identifiers are not retained in this report. Unmatched requests are +counted by method/status; webhook sub-paths are not separate declared routes. +Missing logs or logs without recognizable access records are evidence errors. +HTTP hits remain separate from assertion coverage. diff --git a/tools/regression_contracts.json b/tools/regression_contracts.json new file mode 100644 index 000000000..22334eec1 --- /dev/null +++ b/tools/regression_contracts.json @@ -0,0 +1,283 @@ +{ + "version": 1, + "scope": "Registered CLI/HTTP Bot, conversation, model, tool and retrieval behavior obligations plus explicit adjacent-risk gaps. This is not all product features, all API endpoints, code coverage, or real-model quality coverage. Case repetition does not increase the denominator.", + "contracts": [ + {"id": "configuration.identity", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "CLI resolves the configured Workspace identity", + "bindings": [{"profile": "smoke", "case": "identity", "checks": ["response"]}] + }, + {"id": "configuration.plugins", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "Installed LocalAgent is visible through the plugin API", + "bindings": [{"profile": "smoke", "case": "plugins", "checks": ["response"]}], + "routes": ["GET /api/v1/plugins"] + }, + {"id": "configuration.provider_readback", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "Created provider is visible through the public API", + "bindings": [{"profile": "smoke", "case": "provider_readback", "checks": ["response"]}], + "routes": ["GET /api/v1/provider/providers/"] + }, + {"id": "configuration.model_readback", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "Created model is visible through the public API", + "bindings": [{"profile": "smoke", "case": "model_readback", "checks": ["response"]}], + "routes": ["GET /api/v1/provider/models/llm/"] + }, + {"id": "configuration.pipeline_readback", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "Pipeline readback includes its configured model", + "bindings": [{"profile": "smoke", "case": "pipeline_readback", "checks": ["response"]}], + "routes": ["GET /api/v1/pipelines/"] + }, + {"id": "configuration.kb_readback", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "Knowledge-base identity and engine match configuration", + "bindings": [{"profile": "smoke", "case": "kb_readback", "checks": ["response", "fields"]}], + "routes": ["GET /api/v1/knowledge/bases/"] + }, + {"id": "configuration.document_readback", + "area": "configuration", "kind": "blackbox", "priority": "P1", + "description": "Uploaded document completes ingestion in its owning knowledge base", + "bindings": [{"profile": "smoke", "case": "document_readback", "checks": ["response", "fields"]}], + "routes": ["GET /api/v1/knowledge/bases//files"] + }, + {"id": "ingress.missing_signature", + "area": "ingress", "kind": "blackbox", "priority": "P0", + "description": "Unsigned bot messages are rejected", + "bindings": [{"profile": "smoke", "case": "missing_signature", "checks": ["response", "fields"]}] + }, + {"id": "ingress.bad_signature", + "area": "ingress", "kind": "blackbox", "priority": "P0", + "description": "Messages with an invalid signature are rejected", + "bindings": [{"profile": "smoke", "case": "bad_signature", "checks": ["response", "fields"]}] + }, + {"id": "conversation.chat", + "area": "conversation", "kind": "blackbox", "priority": "P1", + "description": "Pipeline returns the expected chat response", + "bindings": [{"profile": "smoke", "case": "chat", "checks": ["response"]}, {"profile": "endurance", "case": "chat", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "conversation.unicode", + "area": "conversation", "kind": "blackbox", "priority": "P1", + "description": "Unicode input and streamed output survive the full conversation chain", + "bindings": [{"profile": "smoke", "case": "unicode", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "model.slow", + "area": "model", "kind": "blackbox", "priority": "P1", + "description": "Slow streamed model output reaches the caller", + "bindings": [{"profile": "smoke", "case": "slow", "checks": ["response", "model_sequence"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "model.fallback401", + "area": "model", "kind": "blackbox", "priority": "P1", + "description": "Authentication failure falls back to the configured secondary model", + "bindings": [{"profile": "smoke", "case": "fallback401", "checks": ["response", "model_sequence"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "model.fallback429", + "area": "model", "kind": "blackbox", "priority": "P1", + "description": "Rate limiting falls back to the configured secondary model", + "bindings": [{"profile": "smoke", "case": "fallback429", "checks": ["response", "model_sequence"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "model.fallback503", + "area": "model", "kind": "blackbox", "priority": "P1", + "description": "Unavailable primary model falls back to the secondary model", + "bindings": [{"profile": "smoke", "case": "fallback503", "checks": ["response", "model_sequence"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "tools.exec_error", + "area": "tools", "kind": "blackbox", "priority": "P1", + "description": "Tool error output is delivered to the model and handled", + "bindings": [{"profile": "smoke", "case": "exec_error", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "tools.parallel", + "area": "tools", "kind": "blackbox", "priority": "P1", + "description": "Parallel tool calls return matched results to the model", + "bindings": [{"profile": "smoke", "case": "parallel", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "delivery.unicode_export", + "area": "delivery", "kind": "blackbox", "priority": "P1", + "description": "Exported Unicode file name and bytes survive delivery", + "bindings": [{"profile": "smoke", "case": "unicode_export", "checks": ["response", "attachment"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "retrieval.other_rag", + "area": "retrieval", "kind": "blackbox", "priority": "P1", + "description": "A second knowledge base supplies its own content to the model", + "bindings": [{"profile": "smoke", "case": "other_rag", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "tools.box", + "area": "tools", "kind": "blackbox", "priority": "P1", + "description": "A real Box write/read round trip reaches the final reply", + "bindings": [{"profile": "smoke", "case": "box", "checks": ["response"]}, {"profile": "endurance", "case": "box", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "delivery.box_export", + "area": "delivery", "kind": "blackbox", "priority": "P1", + "description": "Exported file bytes match the generated fixture", + "bindings": [{"profile": "smoke", "case": "box_export", "checks": ["response", "attachment"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "retrieval.rag_chat", + "area": "retrieval", "kind": "blackbox", "priority": "P1", + "description": "Retrieved fixture content reaches the model and final reply", + "bindings": [{"profile": "smoke", "case": "rag_chat", "checks": ["response"]}, {"profile": "endurance", "case": "rag_chat", "checks": ["response"]}], + "routes": ["POST /api/v1/pipelines//run"] + }, + {"id": "retrieval.rag_retrieve", + "area": "retrieval", "kind": "blackbox", "priority": "P0", + "description": "Retrieval returns the owning knowledge-base marker without the other fixture", + "bindings": [{"profile": "smoke", "case": "rag_retrieve", "checks": ["response", "fields", "isolation"]}], + "routes": ["POST /api/v1/knowledge/bases//retrieve"] + }, + {"id": "retrieval.retrieve_other", + "area": "retrieval", "kind": "blackbox", "priority": "P0", + "description": "Retrieval from the second knowledge base excludes the first fixture", + "bindings": [{"profile": "smoke", "case": "retrieve_other", "checks": ["response", "fields", "isolation"]}], + "routes": ["POST /api/v1/knowledge/bases//retrieve"] + }, + {"id": "retrieval.retrieve_empty", + "area": "retrieval", "kind": "blackbox", "priority": "P1", + "description": "Empty knowledge base returns an empty retrieval result", + "bindings": [{"profile": "smoke", "case": "retrieve_empty", "checks": ["response", "fields"]}], + "routes": ["POST /api/v1/knowledge/bases//retrieve"] + }, + {"id": "conversation.group_chat_history", + "area": "conversation", "kind": "blackbox", "priority": "P0", + "description": "Multi-turn chat preserves the prior reply without mixing group history", + "bindings": [{"profile": "smoke", "case": "group_chat_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "chat", "checks": ["response"], "min_turn": 2}] + }, + {"id": "conversation.group_box_history", + "area": "conversation", "kind": "blackbox", "priority": "P1", + "description": "Multi-turn Box conversation preserves group history", + "bindings": [{"profile": "smoke", "case": "group_box_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "box", "checks": ["response"], "min_turn": 2}] + }, + {"id": "conversation.group_rag_history", + "area": "conversation", "kind": "blackbox", "priority": "P1", + "description": "Multi-turn RAG conversation preserves group history", + "bindings": [{"profile": "smoke", "case": "group_rag_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "rag_chat", "checks": ["response"], "min_turn": 2}] + }, + {"id": "auth.invalid_credentials", + "area": "auth", "kind": "blackbox", "priority": "P0", + "description": "Missing and invalid API credentials cannot access Workspace resources", + "bindings": [] + }, + {"id": "auth.revocation", + "area": "auth", "kind": "blackbox", "priority": "P0", + "description": "Revoked credentials stop authorizing new requests", + "bindings": [] + }, + {"id": "workspace.read_isolation", + "area": "workspace", "kind": "blackbox", "priority": "P0", + "description": "One Workspace cannot read another Workspace resource", + "bindings": [] + }, + {"id": "workspace.write_isolation", + "area": "workspace", "kind": "blackbox", "priority": "P0", + "description": "One Workspace cannot change another Workspace resource", + "bindings": [] + }, + {"id": "execution.cancel_cleanup", + "area": "execution", "kind": "blackbox", "priority": "P0", + "description": "Cancelling a running execution reaches cancelled state and releases its work", + "bindings": [] + }, + {"id": "execution.timeout_cleanup", + "area": "execution", "kind": "blackbox", "priority": "P0", + "description": "Deadline expiry stops child work and allows a subsequent request", + "bindings": [] + }, + {"id": "execution.retry_side_effects", + "area": "execution", "kind": "blackbox", "priority": "P0", + "description": "Retries do not duplicate externally visible side effects", + "bindings": [] + }, + {"id": "runtime.restart_recovery", + "area": "runtime", "kind": "blackbox", "priority": "P0", + "description": "A runtime restart produces a known outcome and subsequent execution works", + "bindings": [] + }, + {"id": "storage.restart_history", + "area": "storage", "kind": "blackbox", "priority": "P1", + "description": "Conversation history remains usable across a Core restart", + "bindings": [] + }, + {"id": "model.stream_disconnect", + "area": "model", "kind": "blackbox", "priority": "P1", + "description": "A partial model stream disconnect is handled without a false successful completion", + "bindings": [] + }, + {"id": "model.exhausted_fallback", + "area": "model", "kind": "blackbox", "priority": "P1", + "description": "Failure of every configured model reaches a visible failed outcome", + "bindings": [] + }, + {"id": "tools.path_escape", + "area": "tools", "kind": "blackbox", "priority": "P0", + "description": "Box operations reject access outside their authorized filesystem scope", + "bindings": [] + }, + {"id": "tools.timeout_recovery", + "area": "tools", "kind": "blackbox", "priority": "P1", + "description": "Timed-out tools stop running and the next tool can execute", + "bindings": [] + }, + {"id": "delivery.callback_retry", + "area": "delivery", "kind": "blackbox", "priority": "P1", + "description": "Failed delivery callbacks retry according to policy without duplicate effects", + "bindings": [] + }, + {"id": "retrieval.update_delete", + "area": "retrieval", "kind": "blackbox", "priority": "P1", + "description": "Updated/deleted documents change retrieval results without stale leakage", + "bindings": [] + }, + {"id": "plugins.upgrade_recovery", + "area": "plugins", "kind": "blackbox", "priority": "P1", + "description": "Plugin upgrade failure preserves a usable installation", + "bindings": [] + }, + {"id": "agent.direct_execution", + "area": "agent", "kind": "blackbox", "priority": "P1", + "description": "Standalone Agent entrypoint completes through the public API", + "bindings": [] + }, + {"id": "persistence.postgres_parity", + "area": "persistence", "kind": "blackbox", "priority": "P1", + "description": "The same externally visible run contract holds on PostgreSQL", + "bindings": [] + }, + {"id": "platform.real_adapter", + "area": "platform", "kind": "blackbox", "priority": "P2", + "description": "An actual messaging-platform adapter receives and delivers the fixture", + "bindings": [] + }, + {"id": "retrieval.semantic_embedding", + "area": "retrieval", "kind": "blackbox", "priority": "P2", + "description": "A real embedding model retrieves relevant fixed documents", + "bindings": [] + }, + {"id": "execution.persisted_terminal", + "area": "execution", "kind": "graybox", "priority": "P0", + "description": "Each executed request has a unique completed persisted run and terminal event", + "bindings": [{"profile": "smoke", "case": "chat", "checks": ["ledger"]}, {"profile": "endurance", "case": "chat", "checks": ["ledger"]}] + }, + {"id": "tools.persisted_box_results", + "area": "tools", "kind": "graybox", "priority": "P0", + "description": "Persisted Box events match backend, exit status and output", + "bindings": [{"profile": "smoke", "case": "box", "checks": ["ledger"]}, {"profile": "endurance", "case": "box", "checks": ["ledger"]}] + }, + {"id": "ingress.no_run_on_reject", + "area": "ingress", "kind": "graybox", "priority": "P0", + "description": "Rejected ingress does not create an AgentRun", + "bindings": [{"profile": "smoke", "case": "missing_signature", "checks": ["no_run"]}] + } + ] +} diff --git a/tools/regression_coverage.py b/tools/regression_coverage.py new file mode 100644 index 000000000..719511ce2 --- /dev/null +++ b/tools/regression_coverage.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +"""Report explicitly asserted business contracts, independently of pass rate.""" + +import argparse +import ast +from collections import Counter +import json +import hashlib +import re +from urllib.parse import urlsplit +from pathlib import Path + +CATALOG = Path(__file__).with_name('regression_contracts.json') +STATES = ('passed', 'failed', 'incomplete', 'skipped', 'not_run', 'missing_test') + + +def checked(row, name, action): + """Record an evaluated assertion; transport failures leave it unverified.""" + assertions = row.setdefault('assertions', {}) + try: + result = action() + except ValueError: + assertions[name] = 'failed' + raise + assertions[name] = 'passed' + return result + + +def check_response(data, expected, forbidden=()): + text = json.dumps(data, ensure_ascii=False) + if expected not in text: + raise ValueError('Response does not contain the expected business result') + if any(value in text for value in forbidden): + raise ValueError('Response contains forbidden content') + + +def load_catalog(path=CATALOG): + catalog = json.loads(Path(path).read_text()) + if catalog.get('version') != 1 or not catalog.get('scope'): + raise ValueError('Coverage catalog requires version 1 and an explicit scope') + seen = set() + for item in catalog['contracts']: + if item['id'] in seen or item['kind'] not in ('blackbox', 'graybox'): + raise ValueError('Coverage contract IDs must be unique and kinds explicit') + seen.add(item['id']) + if not item['description'] or not item['area'] or item['priority'] not in ('P0', 'P1', 'P2'): + raise ValueError('Coverage contracts require a description, area and priority') + for binding in item['bindings']: + if not binding['profile'] or not binding['case'] or not binding['checks']: + raise ValueError('Coverage bindings require a profile, case and actual assertions') + return catalog + + +def collect(directory): + directory = Path(directory) + profile = json.loads((directory / 'profile.json').read_text()) + suite = profile.get('coverage_suite', 'unclassified') + sources = ( + [directory / 'cases.json'] + if 'smoke' in profile + else [directory / phase['name'] / 'requests.jsonl' for phase in profile.get('phases', [])] + ) + rows, errors = [], [] + for path in sources: + if not path.exists(): + errors.append(f'{path}: missing executed-case evidence') + continue + try: + data = ( + [json.loads(line) for line in path.read_text().splitlines()] + if path.suffix == '.jsonl' + else json.loads(path.read_text()) + ) + if not isinstance(data, list) or any( + not isinstance(row, dict) + or row.get('status') not in ('passed', 'ok', 'failed', 'unknown', 'skipped') + or not isinstance(row.get('case'), str) + or not isinstance(row.get('assertions', {}), dict) + or any(value not in ('passed', 'failed') for value in row.get('assertions', {}).values()) + or not isinstance(row.get('turn', 0), int) + for row in data + ): + raise ValueError('Malformed executed-case evidence') + if not data: + errors.append(f'{path}: empty executed-case evidence') + rows.extend({**row, 'profile': suite, 'source': str(path)} for row in data) + except (ValueError, OSError) as exc: + errors.append(f'{path}: {exc}') + missing = {case['name'] for case in profile.get('cases', [])} - {row['case'] for row in rows} + if missing: + errors.append(f'{directory}: no evidence for planned cases: {sorted(missing)}') + return rows, errors + + +def route_inventory(catalog, root=None): + """Discover RouterGroup HTTP declarations; links express test intent, not endpoint hits.""" + root = Path(root) if root else Path(__file__).resolve().parents[1] + routes, unresolved = [], [] + for path in sorted((root / 'src/langbot/pkg').rglob('*.py')): + if 'group_class' not in path.read_text(): + continue + tree = ast.parse(path.read_text()) + for cls in (node for node in ast.walk(tree) if isinstance(node, ast.ClassDef)): + groups = [ + d + for d in cls.decorator_list + if isinstance(d, ast.Call) and isinstance(d.func, ast.Attribute) and d.func.attr == 'group_class' + ] + if not groups: + continue + try: + prefix = ast.literal_eval(groups[0].args[1]) + except (ValueError, IndexError): + unresolved.append(f'{path.relative_to(root)}:{cls.lineno}') + continue + for func in (node for node in ast.walk(cls) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))): + for dec in func.decorator_list: + if ( + not isinstance(dec, ast.Call) + or not isinstance(dec.func, ast.Attribute) + or dec.func.attr != 'route' + ): + continue + location = f'{path.relative_to(root)}:{func.lineno}' + try: + rule = ast.literal_eval(dec.args[0]) + methods = next((ast.literal_eval(k.value) for k in dec.keywords if k.arg == 'methods'), ['GET']) + if not isinstance(prefix, str) or not isinstance(rule, str) or not isinstance(methods, list): + raise ValueError('Dynamic route declaration') + except (ValueError, IndexError): + unresolved.append(location) + continue + for method in methods: + key = f'{method.upper()} {prefix}{rule}' + routes.append( + { + 'route': key, + 'source': location, + 'contracts': [c['id'] for c in catalog['contracts'] if key in c.get('routes', [])], + } + ) + return { + 'scope': 'Static RouterGroup HTTP declarations only; excludes WebSockets, platform-dispatched and dynamic routes. Contract links are intent, not HTTP hit coverage.', + 'routes': routes, + 'unresolved': unresolved, + } + + +def http_observations(paths, inventory): + """Map Hypercorn access records to declared routes without retaining request URLs.""" + patterns = [] + for route in inventory['routes']: + method, path = route['route'].split(' ', 1) + parts = re.split(r'(<[^>]+>)', path) + pattern = ''.join( + ('.+' if part.startswith('') + else re.escape(part) + for part in parts + ) + literal_length = sum(len(part) for part in parts if not part.startswith('<')) + patterns.append((literal_length, method, re.compile(pattern), route['route'])) + patterns.sort(key=lambda item: item[0], reverse=True) + access = re.compile(r'\b(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS) (\S+) (?:HTTP/)?[123]\.[01] (\d{3})\b') + matched, unclassified, records, errors = {}, Counter(), 0, [] + for path in paths: + try: + with Path(path).open() as stream: + for line in stream: + match = access.search(line) + if not match: + continue + method, target, status = match.groups() + records += 1 + request_path = urlsplit(target).path + route = next( + ( + name + for _, expected_method, pattern, name in patterns + if method == expected_method and pattern.fullmatch(request_path) + ), + None, + ) + if route is None: + unclassified[f'{method} {status}'] += 1 + else: + matched.setdefault(route, Counter())[status] += 1 + except (OSError, ValueError) as exc: + errors.append(f'{path}: {exc}') + if paths and not records: + errors.append('No HTTP access records were recognized') + return { + 'scope': 'Observed HTTP methods and declared route templates; this does not prove behavior assertions or adapter sub-path coverage.', + 'records': records, + 'routes': [{'route': route, 'statuses': dict(statuses)} for route, statuses in sorted(matched.items())], + 'unclassified': dict(unclassified), + 'evidence_errors': errors, + } + + +def counts(items): + status = Counter(item['status'] for item in items) + decided = status['passed'] + status['failed'] + return { + 'total': len(items), + 'decided': decided, + 'coverage_percent': round(100 * decided / len(items), 2) if items else None, + 'pass_percent': round(100 * status['passed'] / decided, 2) if decided else None, + 'status': {name: status[name] for name in STATES}, + } + + +def report(catalog, rows, errors=()): + items, bound = [], set() + for contract in catalog['contracts']: + observations = [] + for binding in contract['bindings']: + bound.add((binding['profile'], binding['case'])) + for row in rows: + if (row['profile'], row['case']) != (binding['profile'], binding['case']): + continue + if row.get('turn', 0) < binding.get('min_turn', 0): + continue + checks = [row.get('assertions', {}).get(name) for name in binding['checks']] + status = ( + 'failed' + if 'failed' in checks + else 'skipped' + if row['status'] == 'skipped' + else 'passed' + if all(value == 'passed' for value in checks) + else 'incomplete' + ) + observations.append({'status': status, 'case': row['case'], 'profile': row['profile']}) + statuses = {row['status'] for row in observations} + status = ( + 'missing_test' + if not contract['bindings'] + else 'not_run' + if not observations + else 'failed' + if 'failed' in statuses + else 'skipped' + if statuses == {'skipped'} + else 'passed' + if statuses == {'passed'} + else 'incomplete' + ) + items.append({**contract, 'status': status, 'observations': dict(Counter(x['status'] for x in observations))}) + return { + 'version': 1, + 'scope': catalog['scope'], + 'catalog_version': catalog['version'], + 'catalog_sha256': hashlib.sha256(json.dumps(catalog, sort_keys=True).encode()).hexdigest(), + 'complete_evidence': not errors, + 'kinds': {kind: counts([x for x in items if x['kind'] == kind]) for kind in ('blackbox', 'graybox')}, + 'areas': { + area: counts([x for x in items if x['kind'] == 'blackbox' and x['area'] == area]) + for area in sorted({x['area'] for x in items if x['kind'] == 'blackbox'}) + }, + 'contracts': items, + 'unclassified_cases': [ + {'profile': p, 'case': c} for p, c in sorted({(row['profile'], row['case']) for row in rows} - bound) + ], + 'evidence_errors': list(errors), + } + + +def write_report(directories, output, catalog_path=CATALOG, http_logs=()): + rows, errors = [], [] + for directory in directories: + try: + found, invalid = collect(directory) + rows.extend(found) + errors.extend(invalid) + except (ValueError, OSError, KeyError) as exc: + errors.append(f'{directory}: {exc}') + catalog = load_catalog(catalog_path) + data = report(catalog, rows, errors) + data['route_inventory'] = route_inventory(catalog) + if http_logs: + data['http_observations'] = http_observations(http_logs, data['route_inventory']) + errors.extend(data['http_observations']['evidence_errors']) + data['evidence_errors'] = list(errors) + data['complete_evidence'] = not errors + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + (output / 'coverage.json').write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') + lines = [ + '# Business behavior coverage', + '', + data['scope'], + '', + 'Coverage counts evaluated contracts (pass or fail), not requests or code lines.', + 'Incomplete evidence and skipped/not-run contracts do not count. Gray-box evidence is separate.', + '', + '| Evidence | Decided / registered | Coverage | Pass rate |', + '|---|---:|---:|---:|', + ] + for kind, values in data['kinds'].items(): + lines.append( + f'| {kind} | {values["decided"]}/{values["total"]} | {values["coverage_percent"]}% | {values["pass_percent"]}% |' + ) + lines += ['', '| Priority | Contract | Area | Evidence | Status |', '|---|---|---|---|---|'] + for item in sorted(data['contracts'], key=lambda x: (x['priority'], x['area'], x['id'])): + lines.append( + f'| {item["priority"]} | {item["id"]}: {item["description"]} | {item["area"]} | {item["kind"]} | {item["status"]} |' + ) + inventory = data['route_inventory'] + unlinked = sum(not r['contracts'] for r in inventory['routes']) + lines += [ + '', + f'Route inventory: {len(inventory["routes"])} HTTP declarations; {unlinked} have no linked behavior contract; {len(inventory["unresolved"])} need manual classification.', + inventory['scope'], + 'Full route list and source locations are in coverage.json.', + ] + if http_logs: + observed = data['http_observations'] + lines += [ + '', + f'Observed HTTP: {len(observed["routes"])} declared routes, {observed["records"]} access records.', + observed['scope'], + '| Route | HTTP statuses and counts |', + '|---|---|', + ] + lines.extend(f'| {row["route"]} | {row["statuses"]} |' for row in observed['routes']) + lines.append(f'Unclassified method/status counts: {observed["unclassified"]}') + for item in data['unclassified_cases']: + lines.append(f'\nUnclassified case: {item["profile"]}/{item["case"]}') + for error in errors: + lines.append(f'\nEvidence error: {error}') + (output / 'coverage.md').write_text('\n'.join(lines) + '\n') + return data + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--run', type=Path, action='append', required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--http-log', type=Path, action='append', default=[]) + args = parser.parse_args() + data = write_report(args.run, args.output, http_logs=args.http_log) + print(json.dumps(data['kinds'], indent=2)) + if data['evidence_errors']: + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/regression_profiles/endurance.json b/tools/regression_profiles/endurance.json index 25480525a..5b87ddea7 100644 --- a/tools/regression_profiles/endurance.json +++ b/tools/regression_profiles/endurance.json @@ -509,5 +509,6 @@ "pids": 8, "count": 0 } - } + }, + "coverage_suite": "endurance" } diff --git a/tools/regression_profiles/smoke.json b/tools/regression_profiles/smoke.json index 123dfa331..c8342eb7f 100644 --- a/tools/regression_profiles/smoke.json +++ b/tools/regression_profiles/smoke.json @@ -1640,5 +1640,6 @@ "langbot-team/GeneralParsers" ], "mock_records": "${mock_records}" - } + }, + "coverage_suite": "smoke" } diff --git a/tools/regression_smoke.py b/tools/regression_smoke.py index e462844ec..7734ef87c 100644 --- a/tools/regression_smoke.py +++ b/tools/regression_smoke.py @@ -12,6 +12,7 @@ import urllib.request from uuid import uuid4 from regression_traffic import group_request, read_endpoint_snapshot +from regression_coverage import checked, check_response class UnresolvedRun(RuntimeError): @@ -241,27 +242,39 @@ def run_smoke(suite): offset = mock_path.stat().st_size if case.get('mock_requests') and mock_path.exists() else 0 try: if rejected: - data = rejected_group(case, group) + data = checked(row, 'response', lambda: rejected_group(case, group)) count = connection.execute( 'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND conversation_id=? AND id>?', (suite.variables['workspace_uuid'], 'group_' + group, after_id), ).fetchone()[0] - if count: - raise ValueError('Rejected ingress created an AgentRun') + checked(row, 'no_run', lambda: check_fields({'runs': count}, {'runs': 0})) elif is_group: - data = group_request( - case, group, row['member_id'], turn, token, previous[group], suite.env + data = checked( + row, + 'response', + lambda: group_request( + case, group, row['member_id'], turn, token, previous[group], suite.env + ), )['data'] else: data = suite.call(case['args'], case.get('body')) - if case['expect'] not in json.dumps(data, ensure_ascii=False): - raise ValueError('Expected business response was not returned') + checked(row, 'response', lambda: check_response(data, case['expect'])) row['output'] = data - if any(value in json.dumps(data, ensure_ascii=False) for value in case.get('forbid', [])): - raise ValueError('Response contains forbidden content from another knowledge base') + if case.get('forbid'): + checked(row, 'isolation', lambda: check_response(data, case['expect'], case['forbid'])) row['checks'] = {} if case.get('fields'): - row['checks']['fields'] = check_fields(data, case['fields']) + row['checks']['fields'] = checked(row, 'fields', lambda: check_fields(data, case['fields'])) + if case.get('attachment'): + row['checks']['attachment'] = checked( + row, 'attachment', lambda: check_attachment(data, case['attachment']) + ) + if case.get('mock_requests'): + row['checks']['mock_requests'] = checked( + row, + 'model_sequence', + lambda: check_mock_requests(mock_path, offset, case['mock_requests']), + ) if runner: if not isinstance(data, dict): raise ValueError('Runner response must be an object') @@ -276,13 +289,7 @@ def run_smoke(suite): settings.get('timeout_seconds', 30), ) row['run'] = run - row['checks'].update(check_run(connection, run, case)) - if case.get('attachment'): - row['checks']['attachment'] = check_attachment(data, case['attachment']) - if case.get('mock_requests'): - row['checks']['mock_requests'] = check_mock_requests( - mock_path, offset, case['mock_requests'] - ) + row['checks'].update(checked(row, 'ledger', lambda: check_run(connection, run, case))) row['status'] = 'passed' previous[group] = token except urllib.error.HTTPError as exc: diff --git a/tools/regression_suite.py b/tools/regression_suite.py index ee6ea0da0..b29de516d 100644 --- a/tools/regression_suite.py +++ b/tools/regression_suite.py @@ -80,6 +80,7 @@ def validate(profile): 'smoke', 'verification', 'resource_limits', + 'coverage_suite', } if extra := profile.keys() - allowed: raise ValueError(f'unknown profile fields: {sorted(extra)}') @@ -91,6 +92,8 @@ def validate(profile): not isinstance(profile[field], list) or any(not isinstance(item, dict) for item in profile[field]) ): raise ValueError(f'{field} must be an array of objects') + if 'coverage_suite' in profile and not re.fullmatch(r'[a-z][a-z0-9_-]*', str(profile['coverage_suite'])): + raise ValueError('coverage_suite must be a stable suite identifier') cli = profile.get('cli', {}) if not isinstance(cli, dict) or cli.keys() - {'config', 'executable'}: raise ValueError('cli accepts config and executable') @@ -778,6 +781,11 @@ def main(): result['errors'].extend(suite.cleanup()) except (RuntimeError, ValueError, OSError, KeyError) as exc: result['errors'].append(f'cleanup failed: {exc}') + if profile.get('coverage_suite'): + from regression_coverage import write_report + + coverage = write_report([suite.output], suite.output) + result['errors'].extend(coverage['evidence_errors']) result['passed'] = not result['errors'] suite.write('summary.json', result) print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/tools/regression_traffic.py b/tools/regression_traffic.py index cbd0c5bc9..fdf6d8518 100644 --- a/tools/regression_traffic.py +++ b/tools/regression_traffic.py @@ -3,6 +3,7 @@ import argparse from collections import Counter +from regression_coverage import checked, check_response from concurrent.futures import ThreadPoolExecutor import json import hashlib @@ -308,9 +309,11 @@ def main(): data = row['output']['data'] checks = {} if case.get('fields'): - checks['fields'] = check_fields(data, case['fields']) - if any(value in json.dumps(data, ensure_ascii=False) for value in case.get('forbid', [])): - raise ValueError('Response contains forbidden content') + checks['fields'] = checked(row, 'fields', lambda: check_fields(data, case['fields'])) + if case.get('forbid'): + checked(row, 'isolation', lambda: check_response(data, case['expect'], case['forbid'])) + if case.get('attachment'): + checks['attachment'] = checked(row, 'attachment', lambda: check_attachment(data, case['attachment'])) runner = is_group or case.get('args', [])[:2] == ['pipeline', 'run'] row['requires_runner'] = runner if connection and runner: @@ -330,13 +333,16 @@ def main(): args.verification_timeout, ) row['run'] = run - checks.update(check_run(connection, run, case)) - if case.get('attachment'): - checks['attachment'] = check_attachment(data, case['attachment']) - with lock: - if run['run_id'] in verified_ids: - raise ValueError('Two requests matched the same AgentRun') - verified_ids.add(run['run_id']) + + def verify_ledger(): + result = check_run(connection, run, case) + with lock: + if run['run_id'] in verified_ids: + raise ValueError('Two requests matched the same AgentRun') + verified_ids.add(run['run_id']) + return result + + checks.update(checked(row, 'ledger', verify_ledger)) row['checks'] = checks row['verification_seconds'] = time.monotonic() - began @@ -415,9 +421,8 @@ def main(): row['output'] = json.loads(redact(result.stdout)) if not isinstance(row['output'], dict): raise ValueError('CLI response must be an object') - if result.returncode == 0 and case['expect'] in json.dumps( - row['output'].get('data'), ensure_ascii=False - ): + if result.returncode == 0: + checked(row, 'response', lambda: check_response(row['output'].get('data'), case['expect'])) verify(row, case, connection, after_id) row['status'] = 'ok' elif 'result_unknown' in result.stdout: @@ -483,7 +488,11 @@ def main(): elapsed=dispatch - started, schedule_lag_seconds=max(0, dispatch - group['due']), ) - row['output'] = group_request(case, group['id'], member, group['turn'], token, group['previous']) + row['output'] = checked( + row, + 'response', + lambda: group_request(case, group['id'], member, group['turn'], token, group['previous']), + ) row['seconds'] = time.monotonic() - dispatch verify(row, case, connection, after_id, is_group=True) row['status'] = 'ok' diff --git a/tools/test_regression_coverage.py b/tools/test_regression_coverage.py new file mode 100644 index 000000000..7a2a50a4b --- /dev/null +++ b/tools/test_regression_coverage.py @@ -0,0 +1,236 @@ +"""Coverage must measure evaluated obligations, not green request counters.""" + +import copy +import json +from pathlib import Path +import tempfile +import unittest + +from regression_coverage import checked, collect, http_observations, load_catalog, report, route_inventory, write_report + + +def catalog(): + return { + 'version': 1, + 'scope': 'Test scope', + 'contracts': [ + { + 'id': 'reply', + 'area': 'conversation', + 'kind': 'blackbox', + 'priority': 'P0', + 'description': 'Reply and payload agree', + 'bindings': [{'profile': 'smoke', 'case': 'chat', 'checks': ['response', 'fields']}], + }, + { + 'id': 'terminal', + 'area': 'execution', + 'kind': 'graybox', + 'priority': 'P0', + 'description': 'Persisted terminal state', + 'bindings': [{'profile': 'smoke', 'case': 'chat', 'checks': ['ledger']}], + }, + { + 'id': 'recovery', + 'area': 'execution', + 'kind': 'blackbox', + 'priority': 'P0', + 'description': 'Restart recovery', + 'bindings': [], + }, + ], + } + + +def row(**overrides): + return {'profile': 'smoke', 'case': 'chat', 'status': 'passed', **overrides} + + +class CoverageTest(unittest.TestCase): + def test_old_green_rows_and_transport_failures_are_not_coverage(self): + for r in [row(), row(status='unknown'), row(assertions={'response': 'passed'})]: + data = report(catalog(), [r]) + self.assertEqual(data['kinds']['blackbox']['decided'], 0) + self.assertEqual(data['contracts'][0]['status'], 'incomplete') + r = {} + with self.assertRaises(TimeoutError): + checked(r, 'response', lambda: (_ for _ in ()).throw(TimeoutError('transport'))) + self.assertNotIn('response', r['assertions']) + + def test_a_failed_assertion_is_covered_but_not_passed(self): + r = row() + with self.assertRaises(ValueError): + checked(r, 'response', lambda: (_ for _ in ()).throw(ValueError('wrong reply'))) + data = report(catalog(), [r]) + self.assertEqual(data['kinds']['blackbox']['coverage_percent'], 50) + self.assertEqual(data['kinds']['blackbox']['pass_percent'], 0) + self.assertEqual(data['contracts'][0]['status'], 'failed') + + def test_graybox_failure_does_not_erase_external_assertions(self): + data = report( + catalog(), + [ + row( + status='failed', + assertions={ + 'response': 'passed', + 'fields': 'passed', + 'ledger': 'failed', + }, + ) + ], + ) + self.assertEqual(data['contracts'][0]['status'], 'passed') + self.assertEqual(data['contracts'][1]['status'], 'failed') + self.assertEqual(data['contracts'][2]['status'], 'missing_test') + + def test_repetition_does_not_inflate_coverage_or_hide_incomplete_turns(self): + good = row(assertions={'response': 'passed', 'fields': 'passed'}) + self.assertEqual(report(catalog(), [good])['kinds'], report(catalog(), [good] * 100)['kinds']) + mixed = report(catalog(), [good, row(status='unknown')]) + self.assertEqual(mixed['contracts'][0]['status'], 'incomplete') + self.assertEqual(report(catalog(), [row(status='skipped')])['contracts'][0]['status'], 'skipped') + self.assertEqual(report(catalog(), [])['contracts'][0]['status'], 'not_run') + + def test_first_turn_does_not_prove_history_and_unclassified_cases_are_visible(self): + c = catalog() + c['contracts'][0]['bindings'][0]['min_turn'] = 2 + r = row(turn=1, assertions={'response': 'passed', 'fields': 'passed'}) + data = report(c, [r, row(case='new_case')]) + self.assertEqual(data['contracts'][0]['status'], 'not_run') + self.assertEqual(data['unclassified_cases'], [{'profile': 'smoke', 'case': 'new_case'}]) + + def test_partial_evidence_produces_an_error_and_a_report(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / 'profile.json').write_text(json.dumps({'coverage_suite': 'smoke', 'smoke': {}})) + (root / 'cases.json').write_text('[{"case":') + rows, errors = collect(root) + self.assertEqual(rows, []) + self.assertEqual(len(errors), 1) + result = write_report([root], root) + self.assertTrue(result['evidence_errors']) + self.assertEqual(result['kinds']['blackbox']['decided'], 0) + self.assertTrue((root / 'coverage.md').is_file()) + + def test_missing_phase_evidence_cannot_hide_behind_a_successful_phase(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'profile.json').write_text( + json.dumps( + { + 'coverage_suite': 'smoke', + 'phases': [{'name': 'first'}, {'name': 'second'}], + } + ) + ) + (root / 'first').mkdir() + (root / 'first/requests.jsonl').write_text( + json.dumps( + row( + assertions={'response': 'passed', 'fields': 'passed'}, + ) + ) + + '\n' + ) + rows, errors = collect(root) + self.assertEqual(len(rows), 1) + self.assertEqual(len(errors), 1) + self.assertFalse(report(catalog(), rows, errors)['complete_evidence']) + + def test_empty_and_absent_planned_cases_are_explicit_evidence_errors(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'profile.json').write_text( + json.dumps( + { + 'coverage_suite': 'smoke', + 'smoke': {}, + 'cases': [{'name': 'chat'}], + } + ) + ) + (root / 'cases.json').write_text('[]') + rows, errors = collect(root) + self.assertEqual(rows, []) + self.assertTrue(any('empty' in error for error in errors)) + self.assertTrue(any('planned cases' in error for error in errors)) + + def test_http_hits_use_method_templates_and_do_not_retain_private_urls(self): + inventory = { + 'routes': [ + {'route': value} + for value in ( + 'GET /things/', + 'GET /things/new', + 'POST /things/', + 'POST /bots//', + ) + ] + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'access.log' + path.write_text( + '\n'.join( + [ + 'GET /things/new?token=secret 1.1 200', + 'GET /things/private-id 1.1 404', + 'POST /things/private-id 1.1 403', + 'POST /bots/private-bot/reset 1.1 200', + 'GET /unknown/private-id 1.1 404', + 'application log is not a request', + ] + ) + ) + result = http_observations([path], inventory) + self.assertEqual(result['records'], 5) + self.assertEqual(len(result['routes']), 4) + self.assertEqual(result['unclassified'], {'GET 404': 1}) + encoded = json.dumps(result) + for value in ('private-id', 'private-bot', 'secret'): + self.assertNotIn(value, encoded) + self.assertTrue(http_observations([path.with_name('missing')], inventory)['evidence_errors']) + + def test_new_routes_are_discovered_without_adding_test_cases(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'src/langbot/pkg/example.py' + source.parent.mkdir(parents=True) + source.write_text(""" +@group.group_class('example', '/api/example') +class Routes: + async def initialize(self): + @self.route('/new', methods=['POST']) + async def new_operation(): pass + @self.route(dynamic_path) + async def dynamic_operation(): pass +""") + inventory = route_inventory(catalog(), root) + self.assertEqual(inventory['routes'][0]['route'], 'POST /api/example/new') + self.assertEqual(inventory['routes'][0]['contracts'], []) + self.assertEqual(len(inventory['unresolved']), 1) + + def test_catalog_bindings_name_real_cases_and_required_checks(self): + c = load_catalog() + profiles = {} + routes = {r['route'] for r in route_inventory(c)['routes']} + for contract in c['contracts']: + self.assertTrue(set(contract.get('routes', [])) <= routes) + for binding in contract['bindings']: + name = binding['profile'] + if name not in profiles: + path = Path(__file__).with_name('regression_profiles') / f'{name}.json' + profiles[name] = {x['name']: x for x in json.loads(path.read_text())['cases']} + self.assertIn(binding['case'], profiles[name]) + self.assertTrue(binding['checks']) + broken = copy.deepcopy(c) + broken['contracts'].append(broken['contracts'][0]) + with tempfile.TemporaryDirectory() as d: + path = Path(d) / 'catalog.json' + path.write_text(json.dumps(broken)) + with self.assertRaises(ValueError): + load_catalog(path) + + +if __name__ == '__main__': + unittest.main()