mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-11-01 13:25:18 +08:00
test(tools): add finite core smoke with terminal run and file oracles
This commit is contained in:
huanghuoguoguo
@@ -290,3 +290,50 @@ quality. Run the Job plumbing checks on Linux with:
|
||||
```sh
|
||||
uv run --no-sync python -m unittest discover -s tools -p 'test_*.py' -v
|
||||
```
|
||||
|
||||
## Finite core smoke
|
||||
|
||||
Use [`regression_profiles/smoke.json`](regression_profiles/smoke.json) for a short
|
||||
functional check on an existing Core. It provisions fresh providers, models,
|
||||
Pipelines, a knowledge base and a signed HTTP Bot through `lbctl`, ingests the
|
||||
HTML fixture through GeneralParsers, then sends nine requests across six cases:
|
||||
chat, two Docker Box exec calls, explicit file export, RAG chat, direct retrieval,
|
||||
and two interleaved groups with two turns each. Group members change between
|
||||
turns; the mock checks prior user/reply history and rejects cross-group history.
|
||||
|
||||
Start `mock_llm.py` with the bundled scenarios as described above. Copy the smoke
|
||||
profile and configure the CLI executable/config, service URLs, plugin archives,
|
||||
and `vars.core_db` to the target Core's existing SQLite `langbot.db`. Keep fixture
|
||||
paths valid relative to the copied profile. Set `plugins: []` to reuse installed
|
||||
plugins; LocalAgent, LangRAG and GeneralParsers must still be enabled and initialized.
|
||||
The embedding endpoint must serve the configured model (`all-MiniLM-L6-v2` by
|
||||
default). Generation is deterministic mock traffic; Core, plugins, Box, document
|
||||
parsing and retrieval execute normally.
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_suite.py validate --profile /path/to/smoke.json
|
||||
uv run --no-sync python tools/regression_suite.py run \
|
||||
--profile /path/to/smoke.json --output .qa/smoke-001
|
||||
```
|
||||
|
||||
A correct reply alone does not pass. Each Pipeline/group request must match one
|
||||
new AgentRun in the authenticated workspace with `completed` status and a persisted
|
||||
`run.completed` event. Box cases additionally check actual Docker execution,
|
||||
exit codes and sentinel output; export checks the delivered filename and exact
|
||||
attachment bytes. The ledger is opened read-only. This first version requires
|
||||
local SQLite access; remote-only Core and PostgreSQL ledgers are not supported.
|
||||
The observation timeout does not extend the Runner deadline or retry requests.
|
||||
Unknown outcomes fail the suite; later turns of that group are skipped while
|
||||
independent cases continue.
|
||||
|
||||
`summary.json` contains the verdict; exit code 0 means all checks and required
|
||||
cleanup passed. `cases.json` records per-request evidence, `environment.json`
|
||||
records identity/capabilities/plugins, and `owned.json` tracks created resources.
|
||||
Successful runs delete those API resources. Failures retain them by default; use
|
||||
`uv run --no-sync python tools/regression_suite.py cleanup --profile /path/to/smoke.json --output .qa/smoke-001`
|
||||
after inspection. Box sessions and AgentRun records follow the reused service's
|
||||
normal retention policy. Load override flags are rejected for smoke.
|
||||
|
||||
To use a fresh deployment, select this suite in `job.json`; the Job supplies its
|
||||
SQLite path and uses mock embedding. Smoke skips load warmup and timed phases.
|
||||
The dedicated Docker daemon requirement above still applies.
|
||||
|
||||
@@ -16,5 +16,65 @@
|
||||
"mock-429": {"status": 429, "steps": [{"content": "unused"}]},
|
||||
"mock-503": {"status": 503, "steps": [{"content": "unused"}]},
|
||||
"mock-slow": {"delay_seconds": 2, "steps": [{"content": "LANGBOT_SLOW_OK"}]},
|
||||
"mock-slow-stream": {"chunk_delay_seconds": 0.2, "steps": [{"content": "LANGBOT_SLOW_STREAM_OK"}]}
|
||||
"mock-slow-stream": {"chunk_delay_seconds": 0.2, "steps": [{"content": "LANGBOT_SLOW_STREAM_OK"}]},
|
||||
"mock-smoke-box": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf LANGBOT_BOX_SENTINEL > /tmp/langbot-smoke.txt; cat /tmp/langbot-smoke.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "LANGBOT_BOX_SENTINEL",
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "cat /tmp/langbot-smoke.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "LANGBOT_BOX_SENTINEL",
|
||||
"content": "LANGBOT_BOX_OK"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-export": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf LANGBOT_EXPORT_SENTINEL > /tmp/langbot-smoke-output.txt; cat /tmp/langbot-smoke-output.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "LANGBOT_EXPORT_SENTINEL",
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "export_box_files",
|
||||
"arguments": {
|
||||
"paths": [
|
||||
"/tmp/langbot-smoke-output.txt"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "accepted",
|
||||
"content": "LANGBOT_EXPORT_OK"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -304,6 +304,8 @@ class Job:
|
||||
self.save()
|
||||
profile = self.suite_profile
|
||||
profile.setdefault('vars', {})['core_base_url'] = endpoint
|
||||
if 'smoke' in profile:
|
||||
profile['smoke']['ledger_db'] = str(Path(self.state['record']['data_dir']) / 'langbot.db')
|
||||
profile['cli'] = {'executable': self.cli[0], 'config': str(self.cli_config)}
|
||||
profile['observe'] = {'health': health, 'pids': {}, 'containers': list(containers.values())}
|
||||
for service, container in containers.items():
|
||||
@@ -315,6 +317,8 @@ class Job:
|
||||
self.redactions.extend(suite.secrets)
|
||||
suite.write('profile.json', profile)
|
||||
suite.prepare()
|
||||
if 'smoke' in profile:
|
||||
return suite
|
||||
# Prime every configured business case before measuring steady load.
|
||||
for case in profile['cases']:
|
||||
case = resolve(case, suite.variables)
|
||||
@@ -441,7 +445,11 @@ def main():
|
||||
if args.profile is None:
|
||||
raise ValueError('--profile is required for run/validate')
|
||||
profile, suite, path = load_profile(args.profile.resolve())
|
||||
for phase in suite['phases']:
|
||||
if 'smoke' in suite and any(
|
||||
getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group')
|
||||
):
|
||||
raise ValueError('Load overrides are not applicable to smoke mode')
|
||||
for phase in suite.get('phases', []):
|
||||
for flag, field in [
|
||||
('duration', 'duration_seconds'),
|
||||
('rps', 'rps'),
|
||||
|
||||
@@ -0,0 +1,512 @@
|
||||
{
|
||||
"version": 1,
|
||||
"cli": {
|
||||
"executable": "lbctl",
|
||||
"config": "./lbctl.yaml"
|
||||
},
|
||||
"vars": {
|
||||
"plugin_dir": "/path/to/plugin-zips",
|
||||
"mock_base_url": "http://127.0.0.1:18080/v1",
|
||||
"embedding_base_url": "http://127.0.0.1:15410/v1",
|
||||
"core_health": "http://127.0.0.1:5300/healthz",
|
||||
"core_base_url": "http://127.0.0.1:5300",
|
||||
"core_db": "/path/to/core/data/langbot.db"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"source": "local",
|
||||
"file": "${plugin_dir}/LocalAgent.zip"
|
||||
},
|
||||
{
|
||||
"source": "local",
|
||||
"file": "${plugin_dir}/LangRAG.zip"
|
||||
},
|
||||
{
|
||||
"source": "local",
|
||||
"file": "${plugin_dir}/GeneralParsers.zip"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "mock_provider",
|
||||
"kind": "provider",
|
||||
"body": {
|
||||
"name": "regression mock",
|
||||
"requester": "openai-chat-completions",
|
||||
"base_url": "${mock_base_url}",
|
||||
"api_keys": [
|
||||
"test-only"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "embedding_provider",
|
||||
"kind": "provider",
|
||||
"body": {
|
||||
"name": "regression embedding",
|
||||
"requester": "openai-chat-completions",
|
||||
"base_url": "${embedding_base_url}",
|
||||
"api_keys": [
|
||||
"test-only"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chat_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-chat",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "box_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-smoke-box",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "export_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-smoke-export",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rag_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-rag",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "embedding_model",
|
||||
"kind": "model",
|
||||
"type": "embedding",
|
||||
"body": {
|
||||
"name": "all-MiniLM-L6-v2",
|
||||
"provider_uuid": "${embedding_provider}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "knowledge_base",
|
||||
"kind": "knowledge-base",
|
||||
"body": {
|
||||
"name": "regression parser golden",
|
||||
"knowledge_engine_plugin_id": "langbot-team/LangRAG",
|
||||
"creation_settings": {
|
||||
"embedding_model_uuid": "${embedding_model}",
|
||||
"index_type": "chunk",
|
||||
"chunk_size": 512,
|
||||
"overlap": 50
|
||||
},
|
||||
"retrieval_settings": {
|
||||
"search_type": "vector",
|
||||
"top_k": 5,
|
||||
"query_rewrite": "off",
|
||||
"rerank": "off"
|
||||
}
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html",
|
||||
"parser_plugin_id": "langbot-team/GeneralParsers"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "chat_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "regression chat",
|
||||
"description": "Normal deterministic chat",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${chat_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [],
|
||||
"box-enabled": false,
|
||||
"timeout": 30,
|
||||
"max-round": 20
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
},
|
||||
"trigger": {
|
||||
"group-respond-rules": {
|
||||
"prefix": [
|
||||
"ai "
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "box_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "regression Box",
|
||||
"description": "Real Box write and read",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${box_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [],
|
||||
"box-enabled": true,
|
||||
"box-session-id-template": "{launcher_type}_{launcher_id}",
|
||||
"timeout": 30,
|
||||
"max-tool-iterations": 5
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rag_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "regression RAG",
|
||||
"description": "Retrieved model input oracle",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${rag_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [
|
||||
"${knowledge_base}"
|
||||
],
|
||||
"box-enabled": false,
|
||||
"timeout": 30,
|
||||
"retrieval-top-k": 5
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chat_bot",
|
||||
"kind": "bot",
|
||||
"body": {
|
||||
"name": "regression group chat",
|
||||
"description": "Synthetic signed group ingress",
|
||||
"adapter": "http_bot",
|
||||
"enable": true,
|
||||
"adapter_config": {
|
||||
"inbound_secret": "${group_secret}",
|
||||
"signature_required": true,
|
||||
"callback_url": "http://127.0.0.1:9/unused-sync-only",
|
||||
"default_session_type": "group",
|
||||
"callback_timeout": 20,
|
||||
"callback_max_retries": 0
|
||||
},
|
||||
"event_bindings": [
|
||||
{
|
||||
"event_pattern": "message.received",
|
||||
"target_type": "pipeline",
|
||||
"target_uuid": "${chat_pipeline}",
|
||||
"enabled": true,
|
||||
"filters": []
|
||||
}
|
||||
],
|
||||
"plugin_processors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "export_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "smoke file export",
|
||||
"description": "Explicit Box file delivery",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${export_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [],
|
||||
"box-enabled": true,
|
||||
"box-session-id-template": "{launcher_type}_{launcher_id}",
|
||||
"timeout": 30,
|
||||
"max-tool-iterations": 5
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "chat",
|
||||
"args": [
|
||||
"pipeline",
|
||||
"run",
|
||||
"${chat_pipeline}",
|
||||
"--message",
|
||||
"Run the normal QA scenario",
|
||||
"--timeout",
|
||||
"90s"
|
||||
],
|
||||
"expect": "LANGBOT_CHAT_OK"
|
||||
},
|
||||
{
|
||||
"name": "box",
|
||||
"args": [
|
||||
"pipeline",
|
||||
"run",
|
||||
"${box_pipeline}",
|
||||
"--message",
|
||||
"Run the normal QA scenario",
|
||||
"--timeout",
|
||||
"90s"
|
||||
],
|
||||
"expect": "LANGBOT_BOX_OK",
|
||||
"box_exec": {
|
||||
"count": 2,
|
||||
"backend": "docker",
|
||||
"stdout": "LANGBOT_BOX_SENTINEL"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "box_export",
|
||||
"args": [
|
||||
"pipeline",
|
||||
"run",
|
||||
"${export_pipeline}",
|
||||
"--message",
|
||||
"Run the file export QA scenario",
|
||||
"--timeout",
|
||||
"90s"
|
||||
],
|
||||
"expect": "LANGBOT_EXPORT_OK",
|
||||
"box_exec": {
|
||||
"count": 1,
|
||||
"backend": "docker",
|
||||
"stdout": "LANGBOT_EXPORT_SENTINEL"
|
||||
},
|
||||
"attachment": {
|
||||
"name": "langbot-smoke-output.txt",
|
||||
"text": "LANGBOT_EXPORT_SENTINEL"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rag_chat",
|
||||
"args": [
|
||||
"pipeline",
|
||||
"run",
|
||||
"${rag_pipeline}",
|
||||
"--message",
|
||||
"What is the documented parser-rag sentinel?",
|
||||
"--timeout",
|
||||
"90s"
|
||||
],
|
||||
"expect": "LANGBOT_RAG_OK"
|
||||
},
|
||||
{
|
||||
"name": "rag_retrieve",
|
||||
"args": [
|
||||
"knowledge-base",
|
||||
"retrieve",
|
||||
"${knowledge_base}"
|
||||
],
|
||||
"body": {
|
||||
"query": "What is the parser-rag golden sentinel and parser-engine pair?",
|
||||
"retrieval_settings": {
|
||||
"search_type": "vector",
|
||||
"top_k": 5
|
||||
}
|
||||
},
|
||||
"expect": "aurora-parser-rag-9137"
|
||||
},
|
||||
{
|
||||
"name": "group_history",
|
||||
"http_group": {
|
||||
"url": "${core_base_url}/bots/${chat_bot}/sync",
|
||||
"secret_env": "LANGBOT_QA_GROUP_SECRET",
|
||||
"message": "ai What is the documented parser-rag sentinel?"
|
||||
},
|
||||
"expect": "LANGBOT_CHAT_OK"
|
||||
}
|
||||
],
|
||||
"observe": {
|
||||
"health": {
|
||||
"core": "${core_health}",
|
||||
"mock": "http://127.0.0.1:18080/healthz",
|
||||
"box": "http://127.0.0.1:5410/readyz"
|
||||
},
|
||||
"pids": {},
|
||||
"pid_files": {},
|
||||
"containers": []
|
||||
},
|
||||
"cleanup": "on-success",
|
||||
"smoke": {
|
||||
"ledger_db": "${core_db}",
|
||||
"timeout_seconds": 30,
|
||||
"group_turns": 2,
|
||||
"groups_per_case": 2,
|
||||
"required_plugins": [
|
||||
"langbot-team/LocalAgent",
|
||||
"langbot-team/LangRAG",
|
||||
"langbot-team/GeneralParsers"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Run finite CLI/group smoke cases with an independent read-only AgentRun oracle."""
|
||||
|
||||
from collections import Counter
|
||||
from contextlib import closing
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import time
|
||||
import urllib.error
|
||||
from uuid import uuid4
|
||||
|
||||
from regression_traffic import group_request, read_endpoint_snapshot
|
||||
|
||||
|
||||
class UnresolvedRun(RuntimeError):
|
||||
"""The request cannot yet be matched to one authoritative terminal run."""
|
||||
|
||||
|
||||
def completed_run(connection, workspace, conversation, after_id, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
rows = connection.execute(
|
||||
'SELECT run_id, status, status_reason, finished_at FROM agent_run '
|
||||
'WHERE workspace_id=? AND conversation_id=? AND id>? ORDER BY id',
|
||||
(workspace, conversation, after_id),
|
||||
).fetchall()
|
||||
if len(rows) > 1:
|
||||
raise UnresolvedRun('More than one AgentRun matches this request')
|
||||
if rows and (rows[0]['finished_at'] or rows[0]['status'] in {'completed', 'failed', 'timeout', 'cancelled'}):
|
||||
return dict(rows[0])
|
||||
if time.monotonic() >= deadline:
|
||||
raise UnresolvedRun('No unique terminal AgentRun before the observation deadline; not retried')
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def check_run(connection, run, case):
|
||||
if run['status'] != 'completed':
|
||||
raise ValueError(f'AgentRun {run["run_id"]} ended as {run["status"]}: {run["status_reason"]}')
|
||||
events = [
|
||||
(kind, json.loads(data or '{}'))
|
||||
for kind, data in connection.execute(
|
||||
'SELECT type, data_json FROM agent_run_event WHERE run_id=? ORDER BY sequence', (run['run_id'],)
|
||||
)
|
||||
]
|
||||
if any(not isinstance(data, dict) for _kind, data in events):
|
||||
raise ValueError('AgentRun event payload must be an object')
|
||||
if not any(kind == 'run.completed' for kind, _data in events):
|
||||
raise ValueError('Completed AgentRun is missing its persisted run.completed event')
|
||||
checks = {'terminal_event': True}
|
||||
if expected := case.get('box_exec'):
|
||||
calls = [
|
||||
data.get('result', {})
|
||||
for kind, data in events
|
||||
if kind == 'tool.call.completed' and data.get('tool_name') == 'exec'
|
||||
]
|
||||
if len(calls) != expected['count'] or any(
|
||||
not isinstance(item, dict)
|
||||
or item.get('backend') != expected['backend']
|
||||
or item.get('exit_code') != 0
|
||||
or not isinstance(item.get('stdout'), str)
|
||||
or expected['stdout'] not in item['stdout']
|
||||
for item in calls
|
||||
):
|
||||
raise ValueError('Box execution count/backend/exit code/output did not match the oracle')
|
||||
checks['box_exec'] = {'calls': len(calls), 'backend': expected['backend'], 'output_matched': True}
|
||||
return checks
|
||||
|
||||
|
||||
def check_attachment(data, expected):
|
||||
replies = data.get('replies')
|
||||
if not isinstance(replies, list) or any(
|
||||
not isinstance(reply, dict)
|
||||
or not isinstance(reply.get('message_chain'), list)
|
||||
or any(not isinstance(part, dict) for part in reply['message_chain'])
|
||||
for reply in replies
|
||||
):
|
||||
raise ValueError('Malformed file delivery response')
|
||||
attachments = [part for reply in replies for part in reply.get('message_chain', []) if part.get('type') == 'File']
|
||||
if len(attachments) != 1 or attachments[0].get('name') != expected['name']:
|
||||
raise ValueError('Expected exactly one file attachment with the configured name')
|
||||
encoded = attachments[0].get('base64')
|
||||
if not isinstance(encoded, str):
|
||||
raise ValueError('Attachment must contain base64 bytes')
|
||||
content = base64.b64decode(encoded, validate=True)
|
||||
if content != expected['text'].encode('utf-8'):
|
||||
raise ValueError('Exported attachment bytes differ from the expected fixture')
|
||||
return {'name': expected['name'], 'bytes': len(content), 'content_matched': True}
|
||||
|
||||
|
||||
def run_smoke(suite):
|
||||
# Imported here because Suite dispatches to this module after profile validation.
|
||||
from regression_suite import resolve
|
||||
|
||||
settings = suite.profile['smoke']
|
||||
path = Path(suite.path(settings['ledger_db'])).resolve()
|
||||
if not path.is_file():
|
||||
raise ValueError('smoke.ledger_db must be an existing target Core SQLite database')
|
||||
rows, errors, health = [], [], []
|
||||
started = time.monotonic()
|
||||
|
||||
def redact(value):
|
||||
for secret in suite.secrets:
|
||||
value = value.replace(secret, '[REDACTED]')
|
||||
return value
|
||||
|
||||
def sample_health(phase):
|
||||
for name, url in resolve(suite.profile['observe']['health'], suite.variables).items():
|
||||
row = {'phase': phase, 'endpoint': name}
|
||||
try:
|
||||
row['result'] = read_endpoint_snapshot(url, timeout_seconds=3)
|
||||
except (RuntimeError, OSError, ValueError) as exc:
|
||||
row['error'] = redact(str(exc))
|
||||
errors.append(f'{phase}/{name}: health check failed')
|
||||
health.append(row)
|
||||
suite.write('health.json', health)
|
||||
|
||||
sample_health('before')
|
||||
# shortcut: local SQLite only; use a terminal-run API when lbctl exposes one.
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=5)) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute('PRAGMA query_only=ON')
|
||||
for original in suite.profile['cases']:
|
||||
case = resolve(original, suite.variables)
|
||||
is_group = 'http_group' in case
|
||||
runner = is_group or case.get('args', [])[:2] == ['pipeline', 'run']
|
||||
groups = (
|
||||
[f'qa-smoke-{uuid4().hex}' for _ in range(settings.get('groups_per_case', 2))] if is_group else [None]
|
||||
)
|
||||
previous = {group: 'none' for group in groups}
|
||||
retired = set()
|
||||
for turn in range(1, (settings.get('group_turns', 2) if is_group else 1) + 1):
|
||||
for group in groups:
|
||||
row = {'case': case['name'], 'status': 'failed', 'requires_runner': runner}
|
||||
if is_group:
|
||||
row.update(group_id=group, turn=turn, member_id=f'qa-member-{turn % 2:03d}')
|
||||
if group in retired:
|
||||
row.update(status='skipped', error='Earlier turn failed or is unknown; group not retried')
|
||||
rows.append(row)
|
||||
continue
|
||||
before = time.monotonic()
|
||||
after_id = connection.execute('SELECT COALESCE(MAX(id),0) FROM agent_run').fetchone()[0]
|
||||
token = uuid4().hex
|
||||
try:
|
||||
if is_group:
|
||||
data = group_request(
|
||||
case, group, row['member_id'], turn, token, previous[group], suite.env
|
||||
)['data']
|
||||
else:
|
||||
data = suite.call(case['args'], case.get('body'))
|
||||
if case['expect'] not in json.dumps(data, ensure_ascii=False):
|
||||
raise ValueError('Expected business response was not returned')
|
||||
row['output'] = data
|
||||
if runner:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError('Runner response must be an object')
|
||||
session = data.get('session_id')
|
||||
if not isinstance(session, str) or not session:
|
||||
raise UnresolvedRun('Reply has no session ID for AgentRun reconciliation')
|
||||
run = completed_run(
|
||||
connection,
|
||||
suite.variables['workspace_uuid'],
|
||||
'group_' + session if is_group else session,
|
||||
after_id,
|
||||
settings.get('timeout_seconds', 30),
|
||||
)
|
||||
row['run'] = run
|
||||
row['checks'] = check_run(connection, run, case)
|
||||
if case.get('attachment'):
|
||||
row['checks']['attachment'] = check_attachment(data, case['attachment'])
|
||||
row['status'] = 'passed'
|
||||
previous[group] = token
|
||||
except urllib.error.HTTPError as exc:
|
||||
row.update(
|
||||
status='unknown' if exc.code >= 500 else 'failed', error=f'HTTP {exc.code}; not retried'
|
||||
)
|
||||
except (UnresolvedRun, urllib.error.URLError, TimeoutError) as exc:
|
||||
row.update(status='unknown', error=redact(str(exc)))
|
||||
except (RuntimeError, ValueError, OSError, KeyError, sqlite3.Error) as exc:
|
||||
message = redact(str(exc))
|
||||
row.update(
|
||||
status='unknown'
|
||||
if 'outcome unknown' in message or 'result_unknown' in message
|
||||
else 'failed',
|
||||
error=message,
|
||||
)
|
||||
row['seconds'] = time.monotonic() - before
|
||||
if row['status'] != 'passed':
|
||||
errors.append(f'{case["name"]}: {row["status"]}: {row["error"]}')
|
||||
if is_group:
|
||||
retired.add(group)
|
||||
rows.append(row)
|
||||
suite.write('cases.json', rows)
|
||||
suite.write(
|
||||
'progress.json',
|
||||
{'completed_requests': len(rows), 'status': dict(Counter(x['status'] for x in rows))},
|
||||
)
|
||||
suite.write('cases.json', rows)
|
||||
sample_health('after')
|
||||
return {
|
||||
'mode': 'smoke',
|
||||
'phases': [],
|
||||
'cases': rows,
|
||||
'errors': errors,
|
||||
'elapsed_seconds': time.monotonic() - started,
|
||||
'status': dict(Counter(row['status'] for row in rows)),
|
||||
'case_count': len(suite.profile['cases']),
|
||||
'requests': len(rows),
|
||||
}
|
||||
@@ -66,7 +66,19 @@ def number(value, label, *, minimum=0, integer=False):
|
||||
def validate(profile):
|
||||
if not isinstance(profile, dict) or profile.get('version') != 1:
|
||||
raise ValueError('profile version must be 1')
|
||||
allowed = {'version', 'cli', 'vars', 'plugins', 'resources', 'cases', 'phases', 'observe', 'cleanup', 'thresholds'}
|
||||
allowed = {
|
||||
'version',
|
||||
'cli',
|
||||
'vars',
|
||||
'plugins',
|
||||
'resources',
|
||||
'cases',
|
||||
'phases',
|
||||
'observe',
|
||||
'cleanup',
|
||||
'thresholds',
|
||||
'smoke',
|
||||
}
|
||||
if extra := profile.keys() - allowed:
|
||||
raise ValueError(f'unknown profile fields: {sorted(extra)}')
|
||||
for field in ('cli', 'vars', 'observe', 'thresholds'):
|
||||
@@ -131,7 +143,7 @@ def validate(profile):
|
||||
raise ValueError('cases must be a non-empty array')
|
||||
names = set()
|
||||
for case in cases:
|
||||
if case.keys() - {'name', 'weight', 'args', 'expect', 'body', 'http_group'}:
|
||||
if case.keys() - {'name', 'weight', 'args', 'expect', 'body', 'http_group', 'box_exec', 'attachment'}:
|
||||
raise ValueError('unknown case field')
|
||||
if not isinstance(case.get('name'), str) or not case['name'] or case['name'] in names:
|
||||
raise ValueError('case names must be non-empty and unique')
|
||||
@@ -162,10 +174,54 @@ def validate(profile):
|
||||
resolve(case, variables)
|
||||
if 'body' in case and (not isinstance(case['body'], dict) or '--file' in case['args']):
|
||||
raise ValueError('case body must be an object and cannot coexist with --file')
|
||||
if 'box_exec' in case:
|
||||
check = case['box_exec']
|
||||
if not isinstance(check, dict) or set(check) != {'count', 'backend', 'stdout'}:
|
||||
raise ValueError('box_exec requires count, backend and stdout')
|
||||
number(check['count'], 'box_exec.count', minimum=1, integer=True)
|
||||
if any(not isinstance(check[k], str) or not check[k] for k in ('backend', 'stdout')):
|
||||
raise ValueError('box_exec backend/stdout must be non-empty strings')
|
||||
if 'smoke' not in profile:
|
||||
raise ValueError('box_exec checks require smoke mode')
|
||||
if 'http_group' not in case and case.get('args', [])[:2] != ['pipeline', 'run']:
|
||||
raise ValueError('box_exec requires a Pipeline or group case')
|
||||
if 'attachment' in case:
|
||||
check = case['attachment']
|
||||
if (
|
||||
'smoke' not in profile
|
||||
or case.get('args', [])[:2] != ['pipeline', 'run']
|
||||
or not isinstance(check, dict)
|
||||
or set(check) != {'name', 'text'}
|
||||
or any(not isinstance(value, str) or not value for value in check.values())
|
||||
):
|
||||
raise ValueError('attachment requires name/text on a smoke Pipeline case')
|
||||
if sum(case.get('weight', 1) for case in cases) > 1000:
|
||||
raise ValueError('total case weight must be <= 1000')
|
||||
phases = profile.get('phases')
|
||||
if not isinstance(phases, list) or not phases:
|
||||
smoke = profile.get('smoke')
|
||||
if 'smoke' in profile:
|
||||
if not isinstance(smoke, dict) or smoke.keys() - {
|
||||
'ledger_db',
|
||||
'timeout_seconds',
|
||||
'group_turns',
|
||||
'groups_per_case',
|
||||
'required_plugins',
|
||||
}:
|
||||
raise ValueError('invalid smoke settings')
|
||||
if not isinstance(smoke.get('ledger_db'), str) or not smoke['ledger_db']:
|
||||
raise ValueError('smoke.ledger_db must name the target Core SQLite database')
|
||||
resolve(smoke, variables)
|
||||
number(smoke.get('timeout_seconds', 30), 'smoke timeout_seconds', minimum=0.001)
|
||||
number(smoke.get('group_turns', 2), 'smoke group_turns', minimum=2, integer=True)
|
||||
number(smoke.get('groups_per_case', 2), 'smoke groups_per_case', minimum=2, integer=True)
|
||||
if smoke.get('group_turns', 2) > 10 or smoke.get('groups_per_case', 2) > 10:
|
||||
raise ValueError('smoke group_turns/groups_per_case must be <= 10')
|
||||
required = smoke.get('required_plugins', [])
|
||||
if not isinstance(required, list) or any(not isinstance(p, str) or not p for p in required):
|
||||
raise ValueError('smoke.required_plugins must be an array of plugin IDs')
|
||||
if profile.get('phases') or profile.get('thresholds'):
|
||||
raise ValueError('smoke uses fixed case counts; load phases/thresholds are not applicable')
|
||||
phases = profile.get('phases', [])
|
||||
if not isinstance(phases, list) or (smoke is None and not phases):
|
||||
raise ValueError('phases must be a non-empty array')
|
||||
names = set()
|
||||
for phase in phases:
|
||||
@@ -341,11 +397,25 @@ class Suite:
|
||||
self.write('owned.json', {'workspace_uuid': self.variables['workspace_uuid'], 'resources': self.owned})
|
||||
|
||||
def prepare(self):
|
||||
if 'smoke' in self.profile and not Path(self.path(self.profile['smoke']['ledger_db'])).is_file():
|
||||
raise ValueError('smoke.ledger_db must be an existing target Core SQLite database')
|
||||
identity = self.call(['whoami'])
|
||||
self.variables['workspace_uuid'] = identity['workspace_uuid']
|
||||
self.journal()
|
||||
for plugin in self.profile.get('plugins', []):
|
||||
self.call(['plugin', 'install', plugin['source'], '--file', self.path(plugin['file']), '--wait'])
|
||||
if 'smoke' in self.profile:
|
||||
capabilities = self.call(['capabilities'])
|
||||
plugins = self.call(['plugin', 'list'])
|
||||
ready = {
|
||||
f'{item["author"]}/{item["name"]}'
|
||||
for item in plugins['plugins']
|
||||
if item.get('enabled') and item.get('status') == 'initialized'
|
||||
}
|
||||
missing = set(self.profile['smoke'].get('required_plugins', [])) - ready
|
||||
self.write('environment.json', {'identity': identity, 'capabilities': capabilities, 'plugins': plugins})
|
||||
if missing:
|
||||
raise RuntimeError(f'Required plugins are not initialized: {sorted(missing)}')
|
||||
for item in self.profile.get('resources', []):
|
||||
kind = item['kind']
|
||||
body = resolve(item['body'], self.variables)
|
||||
@@ -399,6 +469,8 @@ class Suite:
|
||||
if key not in self.profile.get('vars', {}) and key != 'group_secret'
|
||||
},
|
||||
)
|
||||
if 'smoke' in self.profile:
|
||||
return
|
||||
warmups = []
|
||||
for case in self.profile['cases']:
|
||||
if 'http_group' in case:
|
||||
@@ -428,6 +500,10 @@ class Suite:
|
||||
return failures
|
||||
|
||||
def run(self):
|
||||
if 'smoke' in self.profile:
|
||||
from regression_smoke import run_smoke
|
||||
|
||||
return run_smoke(self)
|
||||
cases = []
|
||||
for original in self.profile['cases']:
|
||||
case = resolve(original, self.variables)
|
||||
@@ -513,6 +589,10 @@ def main():
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
profile = json.loads(args.profile.read_text())
|
||||
if 'smoke' in profile and any(
|
||||
getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group')
|
||||
):
|
||||
raise ValueError('Load overrides are not applicable to smoke mode')
|
||||
for phase in profile.get('phases', []):
|
||||
for flag, field in [
|
||||
('duration', 'duration_seconds'),
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Challenge the smoke verdict independently of a successful model reply."""
|
||||
|
||||
import base64
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import unittest
|
||||
|
||||
from regression_smoke import UnresolvedRun, check_attachment, check_run, completed_run
|
||||
from regression_suite import validate
|
||||
|
||||
|
||||
class SmokeOracleTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.db = sqlite3.connect(':memory:')
|
||||
self.db.row_factory = sqlite3.Row
|
||||
self.addCleanup(self.db.close)
|
||||
self.db.executescript("""
|
||||
CREATE TABLE agent_run (id INTEGER, run_id TEXT, workspace_id TEXT,
|
||||
conversation_id TEXT, status TEXT, status_reason TEXT, finished_at TEXT);
|
||||
CREATE TABLE agent_run_event (run_id TEXT, sequence INTEGER, type TEXT, data_json TEXT);
|
||||
INSERT INTO agent_run VALUES (1, 'run', 'workspace', 'conversation', 'completed', '', 'now');
|
||||
INSERT INTO agent_run_event VALUES ('run', 1, 'run.completed', '{}');
|
||||
""")
|
||||
|
||||
def run_row(self, workspace='workspace', after=0):
|
||||
return completed_run(self.db, workspace, 'conversation', after, 0)
|
||||
|
||||
def test_terminal_status_and_event_are_both_required(self):
|
||||
self.assertTrue(check_run(self.db, self.run_row(), {})['terminal_event'])
|
||||
self.db.execute("UPDATE agent_run SET status='failed'")
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), {})
|
||||
self.db.execute("UPDATE agent_run SET status='completed'")
|
||||
self.db.execute('DELETE FROM agent_run_event')
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), {})
|
||||
|
||||
def test_missing_stale_wrong_workspace_and_ambiguous_runs_are_unknown(self):
|
||||
for workspace, after in [('other', 0), ('workspace', 1)]:
|
||||
with self.subTest(workspace=workspace, after=after), self.assertRaises(UnresolvedRun):
|
||||
self.run_row(workspace, after)
|
||||
self.db.execute("UPDATE agent_run SET status='running', finished_at=NULL")
|
||||
with self.assertRaises(UnresolvedRun):
|
||||
self.run_row()
|
||||
self.db.execute('INSERT INTO agent_run SELECT * FROM agent_run')
|
||||
with self.assertRaises(UnresolvedRun):
|
||||
self.run_row()
|
||||
|
||||
def test_box_requires_actual_backend_success_and_output(self):
|
||||
case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'sentinel'}}
|
||||
good = {'backend': 'docker', 'exit_code': 0, 'stdout': 'sentinel'}
|
||||
for result, passes in [
|
||||
(good, True),
|
||||
({**good, 'backend': 'host'}, False),
|
||||
({**good, 'exit_code': 1}, False),
|
||||
({**good, 'stdout': ''}, False),
|
||||
(None, False),
|
||||
]:
|
||||
self.db.execute('DELETE FROM agent_run_event WHERE sequence=2')
|
||||
self.db.execute(
|
||||
'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)',
|
||||
('run', 2, 'tool.call.completed', json.dumps({'tool_name': 'exec', 'result': result})),
|
||||
)
|
||||
with self.subTest(result=result):
|
||||
if passes:
|
||||
check_run(self.db, self.run_row(), case)
|
||||
else:
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), case)
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), {'box_exec': {**case['box_exec'], 'count': 2}})
|
||||
|
||||
def test_attachment_requires_exact_bytes_and_single_file(self):
|
||||
expected = {'name': 'out.txt', 'text': 'sentinel'}
|
||||
part = {'type': 'File', 'name': 'out.txt', 'base64': base64.b64encode(b'sentinel').decode()}
|
||||
data = {'replies': [{'message_chain': [part]}]}
|
||||
self.assertTrue(check_attachment(data, expected)['content_matched'])
|
||||
for parts in [
|
||||
[],
|
||||
[part, part],
|
||||
[{**part, 'base64': '!!!'}],
|
||||
[{**part, 'base64': 'd3Jvbmc='}],
|
||||
[{**part, 'name': 'other'}],
|
||||
[None],
|
||||
]:
|
||||
with self.subTest(parts=parts), self.assertRaises(ValueError):
|
||||
check_attachment({'replies': [{'message_chain': parts}]}, expected)
|
||||
|
||||
def test_profile_supports_mixed_cases_and_rejects_ignored_oracles(self):
|
||||
profile = json.loads(Path(__file__).with_name('regression_profiles').joinpath('smoke.json').read_text())
|
||||
validate(profile)
|
||||
for changes in [{'thresholds': {'min_rps_ratio': 1}}, {'smoke': {'ledger_db': ''}}, {'phases': None}]:
|
||||
with self.subTest(changes=changes), self.assertRaises(ValueError):
|
||||
validate({**profile, **changes})
|
||||
wrong = copy.deepcopy(profile)
|
||||
wrong['cases'][4]['box_exec'] = {'count': 1, 'backend': 'docker', 'stdout': 'x'}
|
||||
with self.assertRaises(ValueError):
|
||||
validate(wrong)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user