From 1fe750fd1cc5aa7952387a4c3877661274db1ca7 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:11:11 +0800 Subject: [PATCH] test(tools): add finite core smoke with terminal run and file oracles --- tools/README.md | 47 +++ tools/mock_llm_scenarios.json | 62 +++- tools/regression_job.py | 10 +- tools/regression_profiles/smoke.json | 512 +++++++++++++++++++++++++++ tools/regression_smoke.py | 209 +++++++++++ tools/regression_suite.py | 88 ++++- tools/test_regression_smoke.py | 104 ++++++ 7 files changed, 1026 insertions(+), 6 deletions(-) create mode 100644 tools/regression_profiles/smoke.json create mode 100644 tools/regression_smoke.py create mode 100644 tools/test_regression_smoke.py diff --git a/tools/README.md b/tools/README.md index d4c3a01e9..939876cbc 100644 --- a/tools/README.md +++ b/tools/README.md @@ -290,3 +290,50 @@ quality. Run the Job plumbing checks on Linux with: ```sh uv run --no-sync python -m unittest discover -s tools -p 'test_*.py' -v ``` + +## Finite core smoke + +Use [`regression_profiles/smoke.json`](regression_profiles/smoke.json) for a short +functional check on an existing Core. It provisions fresh providers, models, +Pipelines, a knowledge base and a signed HTTP Bot through `lbctl`, ingests the +HTML fixture through GeneralParsers, then sends nine requests across six cases: +chat, two Docker Box exec calls, explicit file export, RAG chat, direct retrieval, +and two interleaved groups with two turns each. Group members change between +turns; the mock checks prior user/reply history and rejects cross-group history. + +Start `mock_llm.py` with the bundled scenarios as described above. Copy the smoke +profile and configure the CLI executable/config, service URLs, plugin archives, +and `vars.core_db` to the target Core's existing SQLite `langbot.db`. Keep fixture +paths valid relative to the copied profile. Set `plugins: []` to reuse installed +plugins; LocalAgent, LangRAG and GeneralParsers must still be enabled and initialized. +The embedding endpoint must serve the configured model (`all-MiniLM-L6-v2` by +default). Generation is deterministic mock traffic; Core, plugins, Box, document +parsing and retrieval execute normally. + +```sh +uv run --no-sync python tools/regression_suite.py validate --profile /path/to/smoke.json +uv run --no-sync python tools/regression_suite.py run \ + --profile /path/to/smoke.json --output .qa/smoke-001 +``` + +A correct reply alone does not pass. Each Pipeline/group request must match one +new AgentRun in the authenticated workspace with `completed` status and a persisted +`run.completed` event. Box cases additionally check actual Docker execution, +exit codes and sentinel output; export checks the delivered filename and exact +attachment bytes. The ledger is opened read-only. This first version requires +local SQLite access; remote-only Core and PostgreSQL ledgers are not supported. +The observation timeout does not extend the Runner deadline or retry requests. +Unknown outcomes fail the suite; later turns of that group are skipped while +independent cases continue. + +`summary.json` contains the verdict; exit code 0 means all checks and required +cleanup passed. `cases.json` records per-request evidence, `environment.json` +records identity/capabilities/plugins, and `owned.json` tracks created resources. +Successful runs delete those API resources. Failures retain them by default; use +`uv run --no-sync python tools/regression_suite.py cleanup --profile /path/to/smoke.json --output .qa/smoke-001` +after inspection. Box sessions and AgentRun records follow the reused service's +normal retention policy. Load override flags are rejected for smoke. + +To use a fresh deployment, select this suite in `job.json`; the Job supplies its +SQLite path and uses mock embedding. Smoke skips load warmup and timed phases. +The dedicated Docker daemon requirement above still applies. diff --git a/tools/mock_llm_scenarios.json b/tools/mock_llm_scenarios.json index e8455ba2d..40791e5bd 100644 --- a/tools/mock_llm_scenarios.json +++ b/tools/mock_llm_scenarios.json @@ -16,5 +16,65 @@ "mock-429": {"status": 429, "steps": [{"content": "unused"}]}, "mock-503": {"status": 503, "steps": [{"content": "unused"}]}, "mock-slow": {"delay_seconds": 2, "steps": [{"content": "LANGBOT_SLOW_OK"}]}, - "mock-slow-stream": {"chunk_delay_seconds": 0.2, "steps": [{"content": "LANGBOT_SLOW_STREAM_OK"}]} + "mock-slow-stream": {"chunk_delay_seconds": 0.2, "steps": [{"content": "LANGBOT_SLOW_STREAM_OK"}]}, + "mock-smoke-box": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf LANGBOT_BOX_SENTINEL > /tmp/langbot-smoke.txt; cat /tmp/langbot-smoke.txt" + } + } + ] + }, + { + "expect_tool_output": "LANGBOT_BOX_SENTINEL", + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "cat /tmp/langbot-smoke.txt" + } + } + ] + }, + { + "expect_tool_output": "LANGBOT_BOX_SENTINEL", + "content": "LANGBOT_BOX_OK" + } + ] + }, + "mock-smoke-export": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf LANGBOT_EXPORT_SENTINEL > /tmp/langbot-smoke-output.txt; cat /tmp/langbot-smoke-output.txt" + } + } + ] + }, + { + "expect_tool_output": "LANGBOT_EXPORT_SENTINEL", + "tool_calls": [ + { + "name": "export_box_files", + "arguments": { + "paths": [ + "/tmp/langbot-smoke-output.txt" + ] + } + } + ] + }, + { + "expect_tool_output": "accepted", + "content": "LANGBOT_EXPORT_OK" + } + ] + } } diff --git a/tools/regression_job.py b/tools/regression_job.py index 1fc46afd0..754dd4166 100644 --- a/tools/regression_job.py +++ b/tools/regression_job.py @@ -304,6 +304,8 @@ class Job: self.save() profile = self.suite_profile profile.setdefault('vars', {})['core_base_url'] = endpoint + if 'smoke' in profile: + profile['smoke']['ledger_db'] = str(Path(self.state['record']['data_dir']) / 'langbot.db') profile['cli'] = {'executable': self.cli[0], 'config': str(self.cli_config)} profile['observe'] = {'health': health, 'pids': {}, 'containers': list(containers.values())} for service, container in containers.items(): @@ -315,6 +317,8 @@ class Job: self.redactions.extend(suite.secrets) suite.write('profile.json', profile) suite.prepare() + if 'smoke' in profile: + return suite # Prime every configured business case before measuring steady load. for case in profile['cases']: case = resolve(case, suite.variables) @@ -441,7 +445,11 @@ def main(): if args.profile is None: raise ValueError('--profile is required for run/validate') profile, suite, path = load_profile(args.profile.resolve()) - for phase in suite['phases']: + if 'smoke' in suite and any( + getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group') + ): + raise ValueError('Load overrides are not applicable to smoke mode') + for phase in suite.get('phases', []): for flag, field in [ ('duration', 'duration_seconds'), ('rps', 'rps'), diff --git a/tools/regression_profiles/smoke.json b/tools/regression_profiles/smoke.json new file mode 100644 index 000000000..d261d654e --- /dev/null +++ b/tools/regression_profiles/smoke.json @@ -0,0 +1,512 @@ +{ + "version": 1, + "cli": { + "executable": "lbctl", + "config": "./lbctl.yaml" + }, + "vars": { + "plugin_dir": "/path/to/plugin-zips", + "mock_base_url": "http://127.0.0.1:18080/v1", + "embedding_base_url": "http://127.0.0.1:15410/v1", + "core_health": "http://127.0.0.1:5300/healthz", + "core_base_url": "http://127.0.0.1:5300", + "core_db": "/path/to/core/data/langbot.db" + }, + "plugins": [ + { + "source": "local", + "file": "${plugin_dir}/LocalAgent.zip" + }, + { + "source": "local", + "file": "${plugin_dir}/LangRAG.zip" + }, + { + "source": "local", + "file": "${plugin_dir}/GeneralParsers.zip" + } + ], + "resources": [ + { + "id": "mock_provider", + "kind": "provider", + "body": { + "name": "regression mock", + "requester": "openai-chat-completions", + "base_url": "${mock_base_url}", + "api_keys": [ + "test-only" + ] + } + }, + { + "id": "embedding_provider", + "kind": "provider", + "body": { + "name": "regression embedding", + "requester": "openai-chat-completions", + "base_url": "${embedding_base_url}", + "api_keys": [ + "test-only" + ] + } + }, + { + "id": "chat_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-chat", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "box_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-box", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "export_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-export", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "rag_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-rag", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "embedding_model", + "kind": "model", + "type": "embedding", + "body": { + "name": "all-MiniLM-L6-v2", + "provider_uuid": "${embedding_provider}" + } + }, + { + "id": "knowledge_base", + "kind": "knowledge-base", + "body": { + "name": "regression parser golden", + "knowledge_engine_plugin_id": "langbot-team/LangRAG", + "creation_settings": { + "embedding_model_uuid": "${embedding_model}", + "index_type": "chunk", + "chunk_size": 512, + "overlap": 50 + }, + "retrieval_settings": { + "search_type": "vector", + "top_k": 5, + "query_rewrite": "off", + "rerank": "off" + } + }, + "documents": [ + { + "file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html", + "parser_plugin_id": "langbot-team/GeneralParsers" + } + ] + }, + { + "id": "chat_pipeline", + "kind": "pipeline", + "body": { + "name": "regression chat", + "description": "Normal deterministic chat", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${chat_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "box_pipeline", + "kind": "pipeline", + "body": { + "name": "regression Box", + "description": "Real Box write and read", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${box_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "rag_pipeline", + "kind": "pipeline", + "body": { + "name": "regression RAG", + "description": "Retrieved model input oracle", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${rag_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [ + "${knowledge_base}" + ], + "box-enabled": false, + "timeout": 30, + "retrieval-top-k": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "chat_bot", + "kind": "bot", + "body": { + "name": "regression group chat", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${chat_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + }, + { + "id": "export_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke file export", + "description": "Explicit Box file delivery", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${export_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + } + ], + "cases": [ + { + "name": "chat", + "args": [ + "pipeline", + "run", + "${chat_pipeline}", + "--message", + "Run the normal QA scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_CHAT_OK" + }, + { + "name": "box", + "args": [ + "pipeline", + "run", + "${box_pipeline}", + "--message", + "Run the normal QA scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_BOX_OK", + "box_exec": { + "count": 2, + "backend": "docker", + "stdout": "LANGBOT_BOX_SENTINEL" + } + }, + { + "name": "box_export", + "args": [ + "pipeline", + "run", + "${export_pipeline}", + "--message", + "Run the file export QA scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_EXPORT_OK", + "box_exec": { + "count": 1, + "backend": "docker", + "stdout": "LANGBOT_EXPORT_SENTINEL" + }, + "attachment": { + "name": "langbot-smoke-output.txt", + "text": "LANGBOT_EXPORT_SENTINEL" + } + }, + { + "name": "rag_chat", + "args": [ + "pipeline", + "run", + "${rag_pipeline}", + "--message", + "What is the documented parser-rag sentinel?", + "--timeout", + "90s" + ], + "expect": "LANGBOT_RAG_OK" + }, + { + "name": "rag_retrieve", + "args": [ + "knowledge-base", + "retrieve", + "${knowledge_base}" + ], + "body": { + "query": "What is the parser-rag golden sentinel and parser-engine pair?", + "retrieval_settings": { + "search_type": "vector", + "top_k": 5 + } + }, + "expect": "aurora-parser-rag-9137" + }, + { + "name": "group_history", + "http_group": { + "url": "${core_base_url}/bots/${chat_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_CHAT_OK" + } + ], + "observe": { + "health": { + "core": "${core_health}", + "mock": "http://127.0.0.1:18080/healthz", + "box": "http://127.0.0.1:5410/readyz" + }, + "pids": {}, + "pid_files": {}, + "containers": [] + }, + "cleanup": "on-success", + "smoke": { + "ledger_db": "${core_db}", + "timeout_seconds": 30, + "group_turns": 2, + "groups_per_case": 2, + "required_plugins": [ + "langbot-team/LocalAgent", + "langbot-team/LangRAG", + "langbot-team/GeneralParsers" + ] + } +} diff --git a/tools/regression_smoke.py b/tools/regression_smoke.py new file mode 100644 index 000000000..cc14b5ff3 --- /dev/null +++ b/tools/regression_smoke.py @@ -0,0 +1,209 @@ +"""Run finite CLI/group smoke cases with an independent read-only AgentRun oracle.""" + +from collections import Counter +from contextlib import closing +import base64 +import json +from pathlib import Path +import sqlite3 +import time +import urllib.error +from uuid import uuid4 + +from regression_traffic import group_request, read_endpoint_snapshot + + +class UnresolvedRun(RuntimeError): + """The request cannot yet be matched to one authoritative terminal run.""" + + +def completed_run(connection, workspace, conversation, after_id, timeout): + deadline = time.monotonic() + timeout + while True: + rows = connection.execute( + 'SELECT run_id, status, status_reason, finished_at FROM agent_run ' + 'WHERE workspace_id=? AND conversation_id=? AND id>? ORDER BY id', + (workspace, conversation, after_id), + ).fetchall() + if len(rows) > 1: + raise UnresolvedRun('More than one AgentRun matches this request') + if rows and (rows[0]['finished_at'] or rows[0]['status'] in {'completed', 'failed', 'timeout', 'cancelled'}): + return dict(rows[0]) + if time.monotonic() >= deadline: + raise UnresolvedRun('No unique terminal AgentRun before the observation deadline; not retried') + time.sleep(0.1) + + +def check_run(connection, run, case): + if run['status'] != 'completed': + raise ValueError(f'AgentRun {run["run_id"]} ended as {run["status"]}: {run["status_reason"]}') + events = [ + (kind, json.loads(data or '{}')) + for kind, data in connection.execute( + 'SELECT type, data_json FROM agent_run_event WHERE run_id=? ORDER BY sequence', (run['run_id'],) + ) + ] + if any(not isinstance(data, dict) for _kind, data in events): + raise ValueError('AgentRun event payload must be an object') + if not any(kind == 'run.completed' for kind, _data in events): + raise ValueError('Completed AgentRun is missing its persisted run.completed event') + checks = {'terminal_event': True} + if expected := case.get('box_exec'): + calls = [ + data.get('result', {}) + for kind, data in events + if kind == 'tool.call.completed' and data.get('tool_name') == 'exec' + ] + if len(calls) != expected['count'] or any( + not isinstance(item, dict) + or item.get('backend') != expected['backend'] + or item.get('exit_code') != 0 + or not isinstance(item.get('stdout'), str) + or expected['stdout'] not in item['stdout'] + for item in calls + ): + raise ValueError('Box execution count/backend/exit code/output did not match the oracle') + checks['box_exec'] = {'calls': len(calls), 'backend': expected['backend'], 'output_matched': True} + return checks + + +def check_attachment(data, expected): + replies = data.get('replies') + if not isinstance(replies, list) or any( + not isinstance(reply, dict) + or not isinstance(reply.get('message_chain'), list) + or any(not isinstance(part, dict) for part in reply['message_chain']) + for reply in replies + ): + raise ValueError('Malformed file delivery response') + attachments = [part for reply in replies for part in reply.get('message_chain', []) if part.get('type') == 'File'] + if len(attachments) != 1 or attachments[0].get('name') != expected['name']: + raise ValueError('Expected exactly one file attachment with the configured name') + encoded = attachments[0].get('base64') + if not isinstance(encoded, str): + raise ValueError('Attachment must contain base64 bytes') + content = base64.b64decode(encoded, validate=True) + if content != expected['text'].encode('utf-8'): + raise ValueError('Exported attachment bytes differ from the expected fixture') + return {'name': expected['name'], 'bytes': len(content), 'content_matched': True} + + +def run_smoke(suite): + # Imported here because Suite dispatches to this module after profile validation. + from regression_suite import resolve + + settings = suite.profile['smoke'] + path = Path(suite.path(settings['ledger_db'])).resolve() + if not path.is_file(): + raise ValueError('smoke.ledger_db must be an existing target Core SQLite database') + rows, errors, health = [], [], [] + started = time.monotonic() + + def redact(value): + for secret in suite.secrets: + value = value.replace(secret, '[REDACTED]') + return value + + def sample_health(phase): + for name, url in resolve(suite.profile['observe']['health'], suite.variables).items(): + row = {'phase': phase, 'endpoint': name} + try: + row['result'] = read_endpoint_snapshot(url, timeout_seconds=3) + except (RuntimeError, OSError, ValueError) as exc: + row['error'] = redact(str(exc)) + errors.append(f'{phase}/{name}: health check failed') + health.append(row) + suite.write('health.json', health) + + sample_health('before') + # shortcut: local SQLite only; use a terminal-run API when lbctl exposes one. + with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=5)) as connection: + connection.row_factory = sqlite3.Row + connection.execute('PRAGMA query_only=ON') + for original in suite.profile['cases']: + case = resolve(original, suite.variables) + is_group = 'http_group' in case + runner = is_group or case.get('args', [])[:2] == ['pipeline', 'run'] + groups = ( + [f'qa-smoke-{uuid4().hex}' for _ in range(settings.get('groups_per_case', 2))] if is_group else [None] + ) + previous = {group: 'none' for group in groups} + retired = set() + for turn in range(1, (settings.get('group_turns', 2) if is_group else 1) + 1): + for group in groups: + row = {'case': case['name'], 'status': 'failed', 'requires_runner': runner} + if is_group: + row.update(group_id=group, turn=turn, member_id=f'qa-member-{turn % 2:03d}') + if group in retired: + row.update(status='skipped', error='Earlier turn failed or is unknown; group not retried') + rows.append(row) + continue + before = time.monotonic() + after_id = connection.execute('SELECT COALESCE(MAX(id),0) FROM agent_run').fetchone()[0] + token = uuid4().hex + try: + if is_group: + data = group_request( + case, group, row['member_id'], turn, token, previous[group], suite.env + )['data'] + else: + data = suite.call(case['args'], case.get('body')) + if case['expect'] not in json.dumps(data, ensure_ascii=False): + raise ValueError('Expected business response was not returned') + row['output'] = data + if runner: + if not isinstance(data, dict): + raise ValueError('Runner response must be an object') + session = data.get('session_id') + if not isinstance(session, str) or not session: + raise UnresolvedRun('Reply has no session ID for AgentRun reconciliation') + run = completed_run( + connection, + suite.variables['workspace_uuid'], + 'group_' + session if is_group else session, + after_id, + settings.get('timeout_seconds', 30), + ) + row['run'] = run + row['checks'] = check_run(connection, run, case) + if case.get('attachment'): + row['checks']['attachment'] = check_attachment(data, case['attachment']) + row['status'] = 'passed' + previous[group] = token + except urllib.error.HTTPError as exc: + row.update( + status='unknown' if exc.code >= 500 else 'failed', error=f'HTTP {exc.code}; not retried' + ) + except (UnresolvedRun, urllib.error.URLError, TimeoutError) as exc: + row.update(status='unknown', error=redact(str(exc))) + except (RuntimeError, ValueError, OSError, KeyError, sqlite3.Error) as exc: + message = redact(str(exc)) + row.update( + status='unknown' + if 'outcome unknown' in message or 'result_unknown' in message + else 'failed', + error=message, + ) + row['seconds'] = time.monotonic() - before + if row['status'] != 'passed': + errors.append(f'{case["name"]}: {row["status"]}: {row["error"]}') + if is_group: + retired.add(group) + rows.append(row) + suite.write('cases.json', rows) + suite.write( + 'progress.json', + {'completed_requests': len(rows), 'status': dict(Counter(x['status'] for x in rows))}, + ) + suite.write('cases.json', rows) + sample_health('after') + return { + 'mode': 'smoke', + 'phases': [], + 'cases': rows, + 'errors': errors, + 'elapsed_seconds': time.monotonic() - started, + 'status': dict(Counter(row['status'] for row in rows)), + 'case_count': len(suite.profile['cases']), + 'requests': len(rows), + } diff --git a/tools/regression_suite.py b/tools/regression_suite.py index ad7e02d96..d2345e46e 100644 --- a/tools/regression_suite.py +++ b/tools/regression_suite.py @@ -66,7 +66,19 @@ def number(value, label, *, minimum=0, integer=False): def validate(profile): if not isinstance(profile, dict) or profile.get('version') != 1: raise ValueError('profile version must be 1') - allowed = {'version', 'cli', 'vars', 'plugins', 'resources', 'cases', 'phases', 'observe', 'cleanup', 'thresholds'} + allowed = { + 'version', + 'cli', + 'vars', + 'plugins', + 'resources', + 'cases', + 'phases', + 'observe', + 'cleanup', + 'thresholds', + 'smoke', + } if extra := profile.keys() - allowed: raise ValueError(f'unknown profile fields: {sorted(extra)}') for field in ('cli', 'vars', 'observe', 'thresholds'): @@ -131,7 +143,7 @@ def validate(profile): raise ValueError('cases must be a non-empty array') names = set() for case in cases: - if case.keys() - {'name', 'weight', 'args', 'expect', 'body', 'http_group'}: + if case.keys() - {'name', 'weight', 'args', 'expect', 'body', 'http_group', 'box_exec', 'attachment'}: raise ValueError('unknown case field') if not isinstance(case.get('name'), str) or not case['name'] or case['name'] in names: raise ValueError('case names must be non-empty and unique') @@ -162,10 +174,54 @@ def validate(profile): resolve(case, variables) if 'body' in case and (not isinstance(case['body'], dict) or '--file' in case['args']): raise ValueError('case body must be an object and cannot coexist with --file') + if 'box_exec' in case: + check = case['box_exec'] + if not isinstance(check, dict) or set(check) != {'count', 'backend', 'stdout'}: + raise ValueError('box_exec requires count, backend and stdout') + number(check['count'], 'box_exec.count', minimum=1, integer=True) + if any(not isinstance(check[k], str) or not check[k] for k in ('backend', 'stdout')): + raise ValueError('box_exec backend/stdout must be non-empty strings') + if 'smoke' not in profile: + raise ValueError('box_exec checks require smoke mode') + if 'http_group' not in case and case.get('args', [])[:2] != ['pipeline', 'run']: + raise ValueError('box_exec requires a Pipeline or group case') + if 'attachment' in case: + check = case['attachment'] + if ( + 'smoke' not in profile + or case.get('args', [])[:2] != ['pipeline', 'run'] + or not isinstance(check, dict) + or set(check) != {'name', 'text'} + or any(not isinstance(value, str) or not value for value in check.values()) + ): + raise ValueError('attachment requires name/text on a smoke Pipeline case') if sum(case.get('weight', 1) for case in cases) > 1000: raise ValueError('total case weight must be <= 1000') - phases = profile.get('phases') - if not isinstance(phases, list) or not phases: + smoke = profile.get('smoke') + if 'smoke' in profile: + if not isinstance(smoke, dict) or smoke.keys() - { + 'ledger_db', + 'timeout_seconds', + 'group_turns', + 'groups_per_case', + 'required_plugins', + }: + raise ValueError('invalid smoke settings') + if not isinstance(smoke.get('ledger_db'), str) or not smoke['ledger_db']: + raise ValueError('smoke.ledger_db must name the target Core SQLite database') + resolve(smoke, variables) + number(smoke.get('timeout_seconds', 30), 'smoke timeout_seconds', minimum=0.001) + number(smoke.get('group_turns', 2), 'smoke group_turns', minimum=2, integer=True) + number(smoke.get('groups_per_case', 2), 'smoke groups_per_case', minimum=2, integer=True) + if smoke.get('group_turns', 2) > 10 or smoke.get('groups_per_case', 2) > 10: + raise ValueError('smoke group_turns/groups_per_case must be <= 10') + required = smoke.get('required_plugins', []) + if not isinstance(required, list) or any(not isinstance(p, str) or not p for p in required): + raise ValueError('smoke.required_plugins must be an array of plugin IDs') + if profile.get('phases') or profile.get('thresholds'): + raise ValueError('smoke uses fixed case counts; load phases/thresholds are not applicable') + phases = profile.get('phases', []) + if not isinstance(phases, list) or (smoke is None and not phases): raise ValueError('phases must be a non-empty array') names = set() for phase in phases: @@ -341,11 +397,25 @@ class Suite: self.write('owned.json', {'workspace_uuid': self.variables['workspace_uuid'], 'resources': self.owned}) def prepare(self): + if 'smoke' in self.profile and not Path(self.path(self.profile['smoke']['ledger_db'])).is_file(): + raise ValueError('smoke.ledger_db must be an existing target Core SQLite database') identity = self.call(['whoami']) self.variables['workspace_uuid'] = identity['workspace_uuid'] self.journal() for plugin in self.profile.get('plugins', []): self.call(['plugin', 'install', plugin['source'], '--file', self.path(plugin['file']), '--wait']) + if 'smoke' in self.profile: + capabilities = self.call(['capabilities']) + plugins = self.call(['plugin', 'list']) + ready = { + f'{item["author"]}/{item["name"]}' + for item in plugins['plugins'] + if item.get('enabled') and item.get('status') == 'initialized' + } + missing = set(self.profile['smoke'].get('required_plugins', [])) - ready + self.write('environment.json', {'identity': identity, 'capabilities': capabilities, 'plugins': plugins}) + if missing: + raise RuntimeError(f'Required plugins are not initialized: {sorted(missing)}') for item in self.profile.get('resources', []): kind = item['kind'] body = resolve(item['body'], self.variables) @@ -399,6 +469,8 @@ class Suite: if key not in self.profile.get('vars', {}) and key != 'group_secret' }, ) + if 'smoke' in self.profile: + return warmups = [] for case in self.profile['cases']: if 'http_group' in case: @@ -428,6 +500,10 @@ class Suite: return failures def run(self): + if 'smoke' in self.profile: + from regression_smoke import run_smoke + + return run_smoke(self) cases = [] for original in self.profile['cases']: case = resolve(original, self.variables) @@ -513,6 +589,10 @@ def main(): args = parser.parse_args() try: profile = json.loads(args.profile.read_text()) + if 'smoke' in profile and any( + getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group') + ): + raise ValueError('Load overrides are not applicable to smoke mode') for phase in profile.get('phases', []): for flag, field in [ ('duration', 'duration_seconds'), diff --git a/tools/test_regression_smoke.py b/tools/test_regression_smoke.py new file mode 100644 index 000000000..96404a4b8 --- /dev/null +++ b/tools/test_regression_smoke.py @@ -0,0 +1,104 @@ +"""Challenge the smoke verdict independently of a successful model reply.""" + +import base64 +import copy +import json +from pathlib import Path +import sqlite3 +import unittest + +from regression_smoke import UnresolvedRun, check_attachment, check_run, completed_run +from regression_suite import validate + + +class SmokeOracleTest(unittest.TestCase): + def setUp(self): + self.db = sqlite3.connect(':memory:') + self.db.row_factory = sqlite3.Row + self.addCleanup(self.db.close) + self.db.executescript(""" + CREATE TABLE agent_run (id INTEGER, run_id TEXT, workspace_id TEXT, + conversation_id TEXT, status TEXT, status_reason TEXT, finished_at TEXT); + CREATE TABLE agent_run_event (run_id TEXT, sequence INTEGER, type TEXT, data_json TEXT); + INSERT INTO agent_run VALUES (1, 'run', 'workspace', 'conversation', 'completed', '', 'now'); + INSERT INTO agent_run_event VALUES ('run', 1, 'run.completed', '{}'); + """) + + def run_row(self, workspace='workspace', after=0): + return completed_run(self.db, workspace, 'conversation', after, 0) + + def test_terminal_status_and_event_are_both_required(self): + self.assertTrue(check_run(self.db, self.run_row(), {})['terminal_event']) + self.db.execute("UPDATE agent_run SET status='failed'") + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), {}) + self.db.execute("UPDATE agent_run SET status='completed'") + self.db.execute('DELETE FROM agent_run_event') + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), {}) + + def test_missing_stale_wrong_workspace_and_ambiguous_runs_are_unknown(self): + for workspace, after in [('other', 0), ('workspace', 1)]: + with self.subTest(workspace=workspace, after=after), self.assertRaises(UnresolvedRun): + self.run_row(workspace, after) + self.db.execute("UPDATE agent_run SET status='running', finished_at=NULL") + with self.assertRaises(UnresolvedRun): + self.run_row() + self.db.execute('INSERT INTO agent_run SELECT * FROM agent_run') + with self.assertRaises(UnresolvedRun): + self.run_row() + + def test_box_requires_actual_backend_success_and_output(self): + case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'sentinel'}} + good = {'backend': 'docker', 'exit_code': 0, 'stdout': 'sentinel'} + for result, passes in [ + (good, True), + ({**good, 'backend': 'host'}, False), + ({**good, 'exit_code': 1}, False), + ({**good, 'stdout': ''}, False), + (None, False), + ]: + self.db.execute('DELETE FROM agent_run_event WHERE sequence=2') + self.db.execute( + 'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)', + ('run', 2, 'tool.call.completed', json.dumps({'tool_name': 'exec', 'result': result})), + ) + with self.subTest(result=result): + if passes: + check_run(self.db, self.run_row(), case) + else: + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), case) + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), {'box_exec': {**case['box_exec'], 'count': 2}}) + + def test_attachment_requires_exact_bytes_and_single_file(self): + expected = {'name': 'out.txt', 'text': 'sentinel'} + part = {'type': 'File', 'name': 'out.txt', 'base64': base64.b64encode(b'sentinel').decode()} + data = {'replies': [{'message_chain': [part]}]} + self.assertTrue(check_attachment(data, expected)['content_matched']) + for parts in [ + [], + [part, part], + [{**part, 'base64': '!!!'}], + [{**part, 'base64': 'd3Jvbmc='}], + [{**part, 'name': 'other'}], + [None], + ]: + with self.subTest(parts=parts), self.assertRaises(ValueError): + check_attachment({'replies': [{'message_chain': parts}]}, expected) + + def test_profile_supports_mixed_cases_and_rejects_ignored_oracles(self): + profile = json.loads(Path(__file__).with_name('regression_profiles').joinpath('smoke.json').read_text()) + validate(profile) + for changes in [{'thresholds': {'min_rps_ratio': 1}}, {'smoke': {'ledger_db': ''}}, {'phases': None}]: + with self.subTest(changes=changes), self.assertRaises(ValueError): + validate({**profile, **changes}) + wrong = copy.deepcopy(profile) + wrong['cases'][4]['box_exec'] = {'count': 1, 'backend': 'docker', 'stdout': 'x'} + with self.assertRaises(ValueError): + validate(wrong) + + +if __name__ == '__main__': + unittest.main()