mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-30 04:15:45 +08:00
Merge pull request #2784 from langbot-app/ci/daily-endurance
ci(regression): exercise monorepo plugins in daily functional cycles
This commit is contained in:
huanghuoguoguo
@@ -67,14 +67,21 @@ jobs:
|
||||
python - <<'PY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
path = Path('.qa/core-smoke/gate/gate.json')
|
||||
root = Path('.qa/core-smoke')
|
||||
path = root / 'gate/gate.json'
|
||||
manifest = root / 'plugins/manifest.json'
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
if manifest.exists():
|
||||
for plugin in json.loads(manifest.read_text())['plugins']:
|
||||
out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text())
|
||||
out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
|
||||
f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n")
|
||||
if data.get('infrastructure_error'):
|
||||
out.write('Preparation or teardown failed; inspect service logs.\n')
|
||||
for error in data.get('coverage_errors', []):
|
||||
out.write(f"Coverage evidence error: {error}\n")
|
||||
out.write('Service/dependency preparation is outside the 60-second execution budget.\n')
|
||||
else:
|
||||
out.write('Core smoke did not produce a verdict; inspect preparation failure.\n')
|
||||
@@ -92,6 +99,12 @@ jobs:
|
||||
.qa/core-smoke/gate/suite/summary.json
|
||||
.qa/core-smoke/gate/suite/health.json
|
||||
.qa/core-smoke/gate/suite/cleanup-checks.json
|
||||
.qa/core-smoke/gate/suite/coverage.json
|
||||
.qa/core-smoke/gate/suite/coverage.md
|
||||
.qa/core-smoke/gate/suite/cycles.json
|
||||
.qa/core-smoke/gate/suite/execution.json
|
||||
.qa/core-smoke/plugins/manifest.json
|
||||
.qa/core-smoke/plugins/*-build.log
|
||||
.qa/core-smoke/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
name: Daily Endurance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
|
||||
- cron: '23 19 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
scenario_groups:
|
||||
description: 'Optional local groups, space separated; empty runs only base cases'
|
||||
type: string
|
||||
default: 'faults resources lifecycle plugins'
|
||||
extra_plugins:
|
||||
description: 'Additional source plugin paths; clear with the plugins group to run only base plugins'
|
||||
type: string
|
||||
default: 'misc/EssentialCommands misc/FAQManager'
|
||||
restart_services:
|
||||
description: 'Check owned Core and Runtime restarts once before resource sampling'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: daily-endurance-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
endurance:
|
||||
name: Daily regression (30m)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install locked dependencies
|
||||
run: uv sync --dev --frozen
|
||||
- name: Prepare CLI and sandbox image
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
mkdir -p .endurance-bin
|
||||
gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \
|
||||
--pattern lbctl_linux_amd64 --pattern checksums.txt
|
||||
cd .endurance-bin
|
||||
sha256sum --check --ignore-missing checksums.txt
|
||||
chmod +x lbctl_linux_amd64
|
||||
docker pull python:3.12-alpine
|
||||
- name: Run repeated scenarios and recovery checks
|
||||
env:
|
||||
SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle plugins' || inputs.scenario_groups }}
|
||||
EXTRA_PLUGINS: ${{ github.event_name == 'schedule' && 'misc/EssentialCommands misc/FAQManager' || inputs.extra_plugins }}
|
||||
RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }}
|
||||
run: |
|
||||
read -r -a groups <<< "$SCENARIO_GROUPS"
|
||||
read -r -a extras <<< "$EXTRA_PLUGINS"
|
||||
args=()
|
||||
if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi
|
||||
uv run --no-sync python tools/regression_ci.py \
|
||||
--suite daily --scenario-groups "${groups[@]}" "${args[@]}" \
|
||||
--lbctl .endurance-bin/lbctl_linux_amd64 --plugins plugins --extra-plugins "${extras[@]}" --output .qa/daily
|
||||
- name: Publish observations
|
||||
if: always()
|
||||
run: |
|
||||
python - <<'PY_SUMMARY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path('.qa/daily')
|
||||
def read(path):
|
||||
source = root / path
|
||||
return json.loads(source.read_text()) if source.exists() else {}
|
||||
|
||||
gate = read('gate/gate.json')
|
||||
suite = read('gate/suite/summary.json')
|
||||
manifest = read('plugins/manifest.json')
|
||||
coverage = read('gate/suite/coverage.json')
|
||||
passed = gate.get('passed') is True and suite.get('passed') is True
|
||||
outer_failed = any(gate.get(key) for key in ('infrastructure_error', 'timed_out', 'resource_errors', 'coverage_errors'))
|
||||
outcome = 'PASS' if passed else 'INCOMPLETE' if suite.get('outcome') == 'INCOMPLETE' and not outer_failed else 'FAIL / incomplete'
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n")
|
||||
out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n')
|
||||
out.write('Real Core/Runtime/source plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
|
||||
out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n")
|
||||
restart = gate.get('restart_checks')
|
||||
if restart:
|
||||
out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n")
|
||||
for error in restart.get('errors', []):
|
||||
out.write(f"Restart check error: {error}\n")
|
||||
out.write(f"Plugin selection: `{manifest.get('selection', 'not reported')}`.\n\n")
|
||||
for plugin in manifest.get('plugins', []):
|
||||
out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
|
||||
out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n")
|
||||
out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n")
|
||||
data = suite.get('test_data', {})
|
||||
out.write(f"Observed conversations: {data.get('observed_conversations', 'not reported')}; AgentRuns: {data.get('observed_agent_runs', 'not reported')}; group sessions: {data.get('group_sessions', 'not reported')}; owned API resources: {data.get('owned_api_resources', 'not reported')}.\n\n")
|
||||
out.write('One Workspace; each pipeline reuses one Box per CI run. This does not test cross-group Box file isolation or tenant isolation.\n\n')
|
||||
observations = suite.get('observations', {})
|
||||
if 'successful_requests_per_second' in observations:
|
||||
out.write(f"Successful throughput: {observations['successful_requests_per_second']:.3f} RPS.\n\n")
|
||||
if observations.get('step_p95_seconds'):
|
||||
out.write('| Case | Full-step P95 (s) |\n|---|---:|\n')
|
||||
for name, p95 in observations['step_p95_seconds'].items():
|
||||
out.write(f"| {name} | {p95} |\n")
|
||||
for kind, values in coverage.get('kinds', {}).items():
|
||||
out.write(f"\n{kind} behavior evidence: {values['decided']} / {values['total']}; statuses: {values['status']}.\n")
|
||||
if suite.get('resources', {}).get('targets'):
|
||||
out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
|
||||
for target, metrics in suite['resources']['targets'].items():
|
||||
for metric, values in metrics.items():
|
||||
out.write(f"| {target} | {metric} | {values.get('growth', 'unavailable')} | {values.get('allowance', 'unavailable')} |\n")
|
||||
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
|
||||
if gate.get('timed_out'):
|
||||
errors.append('Suite exceeded its 1860-second outer process guard.')
|
||||
if gate.get('infrastructure_error'):
|
||||
errors.append(gate['infrastructure_error'])
|
||||
if not gate or not suite:
|
||||
errors.append('Missing verdict; inspect preparation and service logs.')
|
||||
for error in errors:
|
||||
out.write(f"\n- {error}\n")
|
||||
PY_SUMMARY
|
||||
- name: Preserve verdicts and observations
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: daily-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
.qa/daily/gate/gate.json
|
||||
.qa/daily/gate/resources.jsonl
|
||||
.qa/daily/gate/suite.log
|
||||
.qa/daily/gate/suite/summary.json
|
||||
.qa/daily/gate/suite/cases.json
|
||||
.qa/daily/gate/suite/health.json
|
||||
.qa/daily/gate/suite/cleanup-checks.json
|
||||
.qa/daily/gate/suite/resources.jsonl
|
||||
.qa/daily/gate/suite/cycles.json
|
||||
.qa/daily/gate/suite/execution.json
|
||||
.qa/daily/gate/suite/coverage.json
|
||||
.qa/daily/gate/suite/coverage.md
|
||||
.qa/daily/plugins/manifest.json
|
||||
.qa/daily/plugins/*-build.log
|
||||
.qa/daily/restart-checks/profile.json
|
||||
.qa/daily/restart-checks/cases.json
|
||||
.qa/daily/restart-checks/summary.json
|
||||
.qa/daily/restart-checks/coverage.json
|
||||
.qa/daily/restart-checks/coverage.md
|
||||
.qa/daily/restart-checks/cleanup-checks.json
|
||||
.qa/daily/restart-checks/owned.json
|
||||
.qa/daily/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
+102
-6
@@ -367,9 +367,14 @@ phases. The dedicated Docker daemon requirement above still applies.
|
||||
|
||||
`Core Smoke` runs regression-tool tests and a real Core smoke in parallel on
|
||||
GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate,
|
||||
uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl`
|
||||
uses the SDK from `uv.lock`, plugins built from the same checkout, `lbctl`
|
||||
v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials
|
||||
or access to a developer's running environment.
|
||||
The SDK CLI builds LocalAgent, LangRAG and GeneralParsers into `.lbpkg` archives.
|
||||
`plugins/manifest.json` records selection mode, plugin versions, archive hashes
|
||||
and exact installation/readiness readback. Build logs are preserved alongside it.
|
||||
The published route remains available when `--plugins` is omitted; see
|
||||
[published plugin preparation and replay](published-plugins.md).
|
||||
|
||||
Dependency installation, service startup, sandbox image download and plugin
|
||||
installation are preparation. After readiness, the gate allows **60 seconds total**
|
||||
@@ -396,11 +401,102 @@ inspect its retained ownership journal before cleanup. CI tears down its own
|
||||
service processes and owned sandboxes afterward.
|
||||
|
||||
The workflow always creates its two checks on PRs and cancels superseded runs.
|
||||
After the workflow is merged, configure `Regression tools` and `Core smoke (60s)`
|
||||
as required status checks in the repository ruleset. A green workflow alone does
|
||||
not enforce merge protection. Longer load and soak tests use the existing Suite
|
||||
and Job entry points with separately calibrated latency, throughput and resource
|
||||
budgets; the one-minute smoke does not replace them.
|
||||
The existing required check names remain `Regression tools` and `Core smoke (60s)`.
|
||||
`Daily Endurance` is independent and is not required for merging. Longer load and
|
||||
soak tests use the existing Suite and Job entry points with separately calibrated
|
||||
latency, throughput and resource budgets; the one-minute smoke does not replace them.
|
||||
|
||||
## Daily regression
|
||||
|
||||
`Daily Endurance` runs independently of PR checks at 19:23 UTC (03:23
|
||||
Asia/Shanghai) on the default branch. GitHub may delay scheduled starts. It also
|
||||
supports **Run workflow** with a selected branch. The schedule becomes active
|
||||
only after the workflow is merged into the default branch; it is not a required
|
||||
merge check.
|
||||
|
||||
The workflow reuses the disposable smoke stack and builds plugins from this checkout.
|
||||
`--suite daily` consumes 46 base cases plus the selected local groups, requires its first pass, then
|
||||
repeats ordered cases within one **1800-second** window. The clock starts before
|
||||
creating scenario resources and includes a 120-second reserve for drain/API
|
||||
cleanup and a 60-second cooldown. Service and plugin preparation precede this
|
||||
clock. A separate 1860-second process guard allows startup and final reporting;
|
||||
it does not grant extra scenario time. The whole job retains its 45-minute limit.
|
||||
|
||||
The scheduled job enables all local groups (including `plugins`) and the optional one-time restart
|
||||
checks. Manual dispatch exposes both switches. `--scenario-groups` accepts a
|
||||
space-separated selection; supplying the flag with no values runs only base
|
||||
cases. `scenarios` and `daily` enable all three groups when the flag is omitted.
|
||||
|
||||
| Group | Additional behavior |
|
||||
|---|---|
|
||||
| `faults` | Cancel/timeout cleanup, model retry side effects, partial stream failure, exhausted fallback, Box path rejection |
|
||||
| `resources` | API-key revocation, direct Agent execution, document deletion with a surviving retrieval control |
|
||||
| `plugins` | EssentialCommands version command without model/AgentRun execution; FAQManager Page API CRUD and search freshness |
|
||||
| `lifecycle` | Signed HTTP callback 5xx retry identity and subsequent delivery |
|
||||
|
||||
For example, select resource and callback cases with
|
||||
`--suite scenarios --scenario-groups resources lifecycle`. Add
|
||||
`--restart-services` to check Core history after restart and a Runtime restart
|
||||
during an active request. These checks run only against services owned by this
|
||||
disposable runner, once before the fixed-PID resource sampler starts. Their
|
||||
elapsed time is deducted from both scenario and outer process budgets; the
|
||||
resource baseline never joins measurements from old and new PIDs. Failed restart
|
||||
checks retain their own evidence and fail the run before repeated traffic starts.
|
||||
|
||||
Credential revocation and restart diagnostics use an ephemeral user created
|
||||
through the normal initialization/login APIs in the runner's new private database.
|
||||
Generic Suite callers must provide `LANGBOT_QA_USER_TOKEN` for credential tests;
|
||||
missing prerequisites are skipped and produce an incomplete result. Disabled
|
||||
groups remain unexecuted in coverage, and a saved profile cannot execute a case
|
||||
whose group is disabled. Expected failed/cancelled/timeout Runs are recorded by
|
||||
exact ID and status; extra, missing or nonterminal Runs still fail reconciliation.
|
||||
|
||||
Fault cases reuse one Box per case and suite, with unique files per invocation.
|
||||
This bounds normal sandbox growth while preserving cancellation side-effect
|
||||
checks. The callback receiver uses the standard library and loopback; Core and
|
||||
the runner must share a network namespace. Document deletion and callback retry
|
||||
identity have separate narrow contracts: they do not establish document update
|
||||
semantics or exactly-once effects at an arbitrary callback receiver. Plugin
|
||||
upgrade rollback and cases needing additional environments remain explicit gaps.
|
||||
|
||||
The daily profile uses the existing endurance resource allowances with a
|
||||
180-second warmup before baseline sampling. Each pipeline that enables Box uses
|
||||
one stable reuse key unique to this CI run, so conversations created by repeated
|
||||
cycles reuse that pipeline's sandbox. This measures repeated tool use; it does not test
|
||||
cross-group Box file isolation. The profile uses one Workspace and does not
|
||||
establish isolation between tenants. Conversation history and AgentRuns still
|
||||
accumulate and are reported. Resource excesses remain failures.
|
||||
|
||||
Throughput and full-step P95 are observations. Business assertions, unknown
|
||||
results, health failures, incomplete evidence and resource limits determine the
|
||||
verdict. Summaries include first-pass completion, cycles, plugin installation
|
||||
readback, final ledger reconciliation and coverage. Manifests, case results,
|
||||
coverage, cycles, execution stages, resource samples and service logs are retained
|
||||
for 30 days, including failures. Hosted runners do not provide a controlled
|
||||
performance baseline. No automatic comparison between runs is enabled.
|
||||
|
||||
To run the same entrypoint on a disposable Linux host with an idle Docker daemon:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \
|
||||
--lbctl /path/to/lbctl --plugins plugins \
|
||||
--extra-plugins misc/EssentialCommands misc/FAQManager --output .qa/daily-001
|
||||
```
|
||||
|
||||
`--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared
|
||||
expanded profile once under a 300-second guard. `--suite endurance` retains the
|
||||
separate ten-minute load profile under a 900-second guard; its CI entry point also
|
||||
reports throughput/P95 without applying their thresholds. All four modes use
|
||||
published plugins when `--plugins` is omitted. Both workflows explicitly select
|
||||
`--plugins plugins` so source changes are tested in the same PR. Daily additionally
|
||||
selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the
|
||||
space-separated `extra_plugins` list or leave it empty. `--extra-plugins` requires
|
||||
source mode and accepts categorized plugin paths, without removing the three base
|
||||
plugins. Missing or incompatible packages fail preparation, with no silent fallback.
|
||||
Existing conversation Pipelines allow only the base plugins; plugin-specific cases
|
||||
bind their own extensions to avoid changing unrelated prompts/tools. The suite attempts owned API cleanup
|
||||
even after failures. The CI owner then stops its service processes and removes
|
||||
its Box containers; a process guard timeout can interrupt API cleanup.
|
||||
|
||||
## Sustained regression and baseline comparison
|
||||
|
||||
|
||||
+169
-25
@@ -1,12 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate."""
|
||||
"""Prepare a disposable Linux CI stack from this checkout, then run a regression suite."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -15,20 +14,60 @@ import time
|
||||
import yaml
|
||||
|
||||
from regression_gate import supervise
|
||||
from regression_plugins import install_published_plugins, prepare_published_plugins
|
||||
from regression_coverage import write_report
|
||||
from regression_plugins import PLUGIN_IDS, install_plugins, prepare_published_plugins, prepare_source_plugins
|
||||
from regression_local_cases import GROUPS, api, require
|
||||
from regression_suite import Suite
|
||||
from regression_traffic import owned_box_containers, read_endpoint_snapshot
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900}
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402
|
||||
|
||||
|
||||
def suite_profile(name, run_id, groups=()):
|
||||
profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json')
|
||||
if name in ('scenarios', 'daily'):
|
||||
from regression_scenarios import expanded_profile
|
||||
|
||||
profile = expanded_profile(groups)
|
||||
else:
|
||||
profile = json.loads(profile_path.read_text())
|
||||
for resource in profile['resources']:
|
||||
if resource['kind'] == 'pipeline':
|
||||
resource['body'].setdefault(
|
||||
'extensions_preferences',
|
||||
{
|
||||
'enable_all_plugins': False,
|
||||
'plugins': [dict(zip(('author', 'name'), plugin.split('/'))) for plugin in PLUGIN_IDS],
|
||||
},
|
||||
)
|
||||
if name == 'smoke':
|
||||
profile['smoke']['group_turns'] = 2
|
||||
if name == 'daily':
|
||||
limits = json.loads((profile_path.parent / 'endurance.json').read_text())['resource_limits']
|
||||
limits['warmup_seconds'] = 180
|
||||
profile.update(
|
||||
execution={'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60},
|
||||
resource_limits=limits,
|
||||
)
|
||||
for resource in profile['resources']:
|
||||
if resource['kind'] != 'pipeline':
|
||||
continue
|
||||
for config in resource['body']['config']['ai']['runner_config'].values():
|
||||
if config.get('box-enabled'):
|
||||
# This workload measures repeated tool use; each pipeline owns one sandbox per CI run.
|
||||
config['box-session-id-template'] = f'qa-{run_id}-{resource["id"]}'
|
||||
profile.update(cleanup='always', performance_mode='observe')
|
||||
return profile, profile_path
|
||||
|
||||
|
||||
def ready(url, processes, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if any(p.poll() is not None for p in processes):
|
||||
raise RuntimeError('A smoke dependency exited; inspect service logs')
|
||||
raise RuntimeError('A regression dependency exited; inspect service logs')
|
||||
try:
|
||||
read_endpoint_snapshot(url, timeout_seconds=1)
|
||||
return
|
||||
@@ -47,6 +86,7 @@ def run(args):
|
||||
private = root / '.private'
|
||||
private.mkdir(mode=0o700)
|
||||
processes, logs = [], []
|
||||
owned_services = {}
|
||||
key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3))
|
||||
env = dict(
|
||||
os.environ,
|
||||
@@ -60,18 +100,66 @@ def run(args):
|
||||
)
|
||||
|
||||
def start(name, command):
|
||||
log = (root / f'{name}.log').open('w')
|
||||
log = (root / f'{name}.log').open('a')
|
||||
logs.append(log)
|
||||
process = subprocess.Popen(
|
||||
command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True
|
||||
)
|
||||
processes.append(process)
|
||||
owned_services[name] = (process, command)
|
||||
return process.pid
|
||||
|
||||
def restart_owned(name, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
def remaining():
|
||||
value = deadline - time.monotonic()
|
||||
if value <= 0:
|
||||
raise TimeoutError(f'{name} restart exceeded its deadline')
|
||||
return value
|
||||
|
||||
require(name in {'core', 'runtime'}, "Only this runner's Core and Runtime can be restarted")
|
||||
process, command = owned_services[name]
|
||||
old_pid = process.pid
|
||||
require(process.poll() is None, f'{name} exited before the planned restart')
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=min(5, remaining()))
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait(timeout=remaining())
|
||||
processes.remove(process)
|
||||
pids[name] = start(name, command)
|
||||
if name == 'runtime':
|
||||
ready('http://127.0.0.1:15340/healthz', processes, timeout=remaining())
|
||||
ready(profile['observe']['health']['core'], processes, timeout=remaining())
|
||||
while True:
|
||||
previous_timeout = suite.timeout
|
||||
try:
|
||||
suite.timeout = lambda maximum: min(maximum, remaining())
|
||||
listed = suite.call(['plugin', 'list'])
|
||||
except RuntimeError:
|
||||
remaining()
|
||||
time.sleep(min(0.2, remaining()))
|
||||
continue
|
||||
finally:
|
||||
suite.timeout = previous_timeout
|
||||
initialized = {
|
||||
f'{item["author"]}/{item["name"]}'
|
||||
for item in listed['plugins']
|
||||
if item.get('enabled') and item.get('status') == 'initialized'
|
||||
}
|
||||
if set(profile['smoke']['required_plugins']) <= initialized:
|
||||
break
|
||||
time.sleep(min(0.2, remaining()))
|
||||
return {'service': name, 'old_pid': old_pid, 'new_pid': pids[name], 'old_exit_code': process.returncode}
|
||||
|
||||
try:
|
||||
published = None
|
||||
if args.plugins is None:
|
||||
published = prepare_published_plugins(root / 'plugins', args.plugin_manifest)
|
||||
packages = (
|
||||
prepare_source_plugins(root / 'plugins', args.plugins, args.extra_plugins)
|
||||
if args.plugins is not None
|
||||
else prepare_published_plugins(root / 'plugins', args.plugin_manifest)
|
||||
)
|
||||
config_path = create_minimal_config(private, port=15301)
|
||||
create_test_directories(private)
|
||||
config = yaml.safe_load(config_path.read_text())
|
||||
@@ -82,8 +170,12 @@ def run(args):
|
||||
host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine'
|
||||
)
|
||||
config_path.write_text(yaml.safe_dump(config))
|
||||
profile_path = ROOT / 'tools/regression_profiles/smoke.json'
|
||||
profile = json.loads(profile_path.read_text())
|
||||
groups = args.scenario_groups
|
||||
if groups is None:
|
||||
groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else []
|
||||
if 'plugins' in groups and not {'misc/EssentialCommands', 'misc/FAQManager'} <= set(args.extra_plugins):
|
||||
groups.remove('plugins')
|
||||
profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups)
|
||||
cli_config = private / 'lbctl.yaml'
|
||||
cli_config.write_text(
|
||||
yaml.safe_dump(
|
||||
@@ -109,8 +201,8 @@ def run(args):
|
||||
mock_records=str(root / 'mock-requests.jsonl'),
|
||||
)
|
||||
profile['plugins'] = []
|
||||
# Two turns retain the history/isolation oracle while leaving room for setup and cleanup.
|
||||
profile['smoke']['group_turns'] = 2
|
||||
if 'smoke' in profile:
|
||||
profile['smoke']['required_plugins'] = [entry['id'] for entry in packages['plugins']]
|
||||
for resource in profile['resources']:
|
||||
if resource['kind'] == 'model' and resource.get('type') == 'embedding':
|
||||
resource['body']['name'] = 'mock-embedding'
|
||||
@@ -165,28 +257,51 @@ def run(args):
|
||||
},
|
||||
'pids': pids,
|
||||
}
|
||||
if args.suite in ('daily', 'endurance'):
|
||||
profile['observe']['box_data_root'] = str(private / 'box')
|
||||
for url in profile['observe']['health'].values():
|
||||
ready(url, processes)
|
||||
preparation = root / 'preparation'
|
||||
preparation.mkdir()
|
||||
suite = Suite(profile, profile_path, preparation, env)
|
||||
identity = suite.call(['whoami'])
|
||||
suite.variables['workspace_uuid'] = identity['workspace_uuid']
|
||||
# Bind the CLI to the newly bootstrapped Workspace before installing plugins.
|
||||
cli_data = yaml.safe_load(cli_config.read_text())
|
||||
cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid']
|
||||
cli_config.write_text(yaml.safe_dump(cli_data))
|
||||
if published is not None:
|
||||
install_published_plugins(suite.call, root / 'plugins', published)
|
||||
else:
|
||||
for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'):
|
||||
source = args.plugins / directory
|
||||
package_source = private / source.name
|
||||
shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__'))
|
||||
archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source)
|
||||
suite.call(['plugin', 'install', 'local', '--file', archive, '--wait'])
|
||||
generated = private / 'smoke.json'
|
||||
if 'resources' in groups or args.restart_services:
|
||||
# Account creation is restricted to this runner's newly created private database.
|
||||
credentials = {'user': '[email protected]', 'password': secrets.token_urlsafe(32)}
|
||||
suite.secrets.append(credentials['password'])
|
||||
status, response = api(suite, 'POST', '/api/v1/user/init', credentials, headers={})
|
||||
require(status == 200 and response.get('code') == 0, 'Disposable test account initialization failed')
|
||||
status, response = api(suite, 'POST', '/api/v1/user/auth', credentials, headers={})
|
||||
require(status == 200 and response.get('code') == 0, 'Disposable test account authentication failed')
|
||||
env['LANGBOT_QA_USER_TOKEN'] = response['data']['token']
|
||||
suite.env['LANGBOT_QA_USER_TOKEN'] = env['LANGBOT_QA_USER_TOKEN']
|
||||
suite.secrets.append(env['LANGBOT_QA_USER_TOKEN'])
|
||||
install_plugins(suite.call, root / 'plugins', packages)
|
||||
restart_result = None
|
||||
restart_elapsed = 0
|
||||
if args.restart_services:
|
||||
from regression_lifecycle_cases import run_restart_checks
|
||||
|
||||
started = time.monotonic()
|
||||
restart_result = run_restart_checks(profile, profile_path, root / 'restart-checks', env, restart_owned)
|
||||
restart_elapsed = time.monotonic() - started
|
||||
if not restart_result['passed']:
|
||||
raise RuntimeError('Owned-service restart checks failed; inspect restart-checks/summary.json')
|
||||
if 'execution' in profile:
|
||||
profile['execution']['total_seconds'] -= restart_elapsed
|
||||
require(
|
||||
profile['execution']['total_seconds']
|
||||
> profile['execution']['drain_seconds'] + profile['execution']['cooldown_seconds'],
|
||||
'Restart checks consumed the scenario execution budget',
|
||||
)
|
||||
generated = private / f'{args.suite}.json'
|
||||
generated.write_text(json.dumps(profile, indent=2))
|
||||
return supervise(
|
||||
result = supervise(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tools/regression_suite.py'),
|
||||
@@ -197,10 +312,21 @@ def run(args):
|
||||
str(root / 'gate/suite'),
|
||||
],
|
||||
root / 'gate',
|
||||
60,
|
||||
SUITE_BUDGETS[args.suite] - restart_elapsed,
|
||||
pids,
|
||||
env,
|
||||
)
|
||||
evidence = [root / 'gate/suite']
|
||||
if restart_result is not None:
|
||||
evidence.append(root / 'restart-checks')
|
||||
result['restart_checks'] = restart_result
|
||||
result['restart_elapsed_seconds'] = restart_elapsed
|
||||
coverage = write_report(evidence, root / 'gate/suite', http_logs=[root / 'core.log'])
|
||||
result['coverage_errors'] = coverage['evidence_errors']
|
||||
result['scenario_groups'] = groups
|
||||
result['passed'] = result['passed'] and not result['coverage_errors']
|
||||
(root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
return result
|
||||
finally:
|
||||
for process in reversed(processes):
|
||||
if process.poll() is None:
|
||||
@@ -222,9 +348,24 @@ def main():
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--lbctl', type=Path, required=True)
|
||||
plugins = parser.add_mutually_exclusive_group()
|
||||
plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout')
|
||||
plugins.add_argument('--plugins', type=Path, help='Build and install plugins from this source root')
|
||||
plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run')
|
||||
parser.add_argument(
|
||||
'--extra-plugins', nargs='*', default=[], help='Additional paths under --plugins, e.g. misc/HelloPlugin'
|
||||
)
|
||||
parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke')
|
||||
parser.add_argument(
|
||||
'--scenario-groups',
|
||||
nargs='*',
|
||||
choices=tuple(GROUPS),
|
||||
help='Local groups for scenarios/daily (default: all); an empty list selects only the base cases',
|
||||
)
|
||||
parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once')
|
||||
args = parser.parse_args()
|
||||
if args.extra_plugins and args.plugins is None:
|
||||
parser.error('--extra-plugins requires --plugins')
|
||||
if args.suite not in ('scenarios', 'daily') and (args.scenario_groups is not None or args.restart_services):
|
||||
parser.error('Optional scenarios and restarts require --suite scenarios or --suite daily')
|
||||
if args.output.exists():
|
||||
parser.error('--output must be a new directory')
|
||||
try:
|
||||
@@ -233,6 +374,9 @@ def main():
|
||||
verdict = args.output / 'gate' / 'gate.json'
|
||||
result = json.loads(verdict.read_text()) if verdict.exists() else {}
|
||||
result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}')
|
||||
restart_summary = args.output / 'restart-checks/summary.json'
|
||||
if restart_summary.exists():
|
||||
result['restart_checks'] = json.loads(restart_summary.read_text())
|
||||
verdict.parent.mkdir(parents=True, exist_ok=True)
|
||||
verdict.write_text(json.dumps(result, indent=2) + '\n')
|
||||
raise
|
||||
|
||||
@@ -240,6 +240,17 @@
|
||||
"description": "Updated/deleted documents change retrieval results without stale leakage",
|
||||
"bindings": []
|
||||
},
|
||||
{"id": "plugins.command_dispatch",
|
||||
"area": "plugins", "kind": "blackbox", "priority": "P1",
|
||||
"description": "An installed command returns the Core version through signed Bot ingress without invoking an agent or model",
|
||||
"bindings": [{"profile": "scenarios", "case": "plugins.command_version", "checks": ["command_plugin_bindings", "command_version", "command_no_run", "command_no_model", "command_bot_cleanup"]}]
|
||||
},
|
||||
{"id": "plugins.page_storage_crud",
|
||||
"area": "plugins", "kind": "blackbox", "priority": "P1",
|
||||
"description": "An installed Page API persists FAQ creates and updates and removes deleted entries from reads and search",
|
||||
"bindings": [{"profile": "scenarios", "case": "plugins.faq_crud", "checks": ["faq_create", "faq_read", "faq_search", "faq_update", "faq_update_read", "faq_update_search", "faq_old_answer_absent", "faq_delete"]}],
|
||||
"routes": ["POST /api/v1/plugins/<author>/<plugin_name>/page-api"]
|
||||
},
|
||||
{"id": "plugins.upgrade_recovery",
|
||||
"area": "plugins", "kind": "blackbox", "priority": "P1",
|
||||
"description": "Plugin upgrade failure preserves a usable installation",
|
||||
|
||||
@@ -17,7 +17,7 @@ import urllib.request
|
||||
import yaml
|
||||
|
||||
from regression_suite import Suite, number, resolve, validate
|
||||
from regression_plugins import install_published_plugins, prepare_published_plugins
|
||||
from regression_plugins import install_plugins, prepare_published_plugins
|
||||
from regression_traffic import owned_box_containers
|
||||
|
||||
|
||||
@@ -360,7 +360,7 @@ class Job:
|
||||
if published is not None:
|
||||
self.state['phase'] = 'plugins'
|
||||
self.save()
|
||||
install_published_plugins(suite.call, self.output / 'plugins', published)
|
||||
install_plugins(suite.call, self.output / 'plugins', published)
|
||||
self.state['phase'] = 'resources'
|
||||
self.save()
|
||||
suite.write('profile.json', profile)
|
||||
|
||||
@@ -13,6 +13,7 @@ GROUPS = {
|
||||
'faults': 'regression_execution_cases',
|
||||
'resources': 'regression_resource_cases',
|
||||
'lifecycle': 'regression_lifecycle_cases',
|
||||
'plugins': 'regression_plugin_cases',
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""Local plugin business checks through signed Bot ingress and the Page API."""
|
||||
|
||||
from contextlib import closing
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from regression_coverage import checked
|
||||
from regression_lifecycle_cases import ledger
|
||||
from regression_local_cases import api, require
|
||||
from regression_resource_cases import cleanup, journal
|
||||
from regression_smoke import check_mock_requests, signed_headers
|
||||
from regression_suite import resolve
|
||||
|
||||
GROUP = 'plugins'
|
||||
CASE_NAMES = {'plugins.command_version', 'plugins.faq_crud'}
|
||||
COMMAND_PLUGINS = [
|
||||
{'author': 'langbot-team', 'name': name}
|
||||
for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands')
|
||||
]
|
||||
FAQ_API = '/api/v1/plugins/RockChinQ/FAQManager/page-api'
|
||||
|
||||
|
||||
def add_cases(profile):
|
||||
pipeline = copy.deepcopy(next(item for item in profile['resources'] if item['id'] == 'chat_pipeline'))
|
||||
pipeline['id'] = 'plugin_command_pipeline'
|
||||
pipeline['body']['name'] = 'regression plugin commands'
|
||||
pipeline['body']['extensions_preferences'] = {
|
||||
'enable_all_plugins': False,
|
||||
'plugins': copy.deepcopy(COMMAND_PLUGINS),
|
||||
}
|
||||
profile['resources'].append(pipeline)
|
||||
for name, expected in (
|
||||
('plugins.command_version', 'LANGBOT_PLUGIN_COMMAND_OK'),
|
||||
('plugins.faq_crud', 'LANGBOT_PLUGIN_FAQ_OK'),
|
||||
):
|
||||
profile['cases'].append({'name': name, 'scenario': name, 'stage': 'plugins', 'expect': expected})
|
||||
|
||||
|
||||
def data(suite, method, path, body=None, *, headers=None):
|
||||
status, response = api(suite, method, path, body, headers=headers)
|
||||
require(
|
||||
status == 200 and isinstance(response, dict) and response.get('code') == 0,
|
||||
f'{method} {path} did not return a successful response (HTTP {status})',
|
||||
)
|
||||
result = response.get('data')
|
||||
if method == 'DELETE' and result is None:
|
||||
return {}
|
||||
require(isinstance(result, dict), f'{method} {path} returned malformed data')
|
||||
return result
|
||||
|
||||
|
||||
def command_version(suite, case, row):
|
||||
pipeline = data(suite, 'GET', f'/api/v1/pipelines/{suite.variables["plugin_command_pipeline"]}')['pipeline']
|
||||
preferences = pipeline.get('extensions_preferences', {})
|
||||
checked(
|
||||
row,
|
||||
'command_plugin_bindings',
|
||||
lambda: require(
|
||||
preferences.get('enable_all_plugins') is False and preferences.get('plugins') == COMMAND_PLUGINS,
|
||||
'Command pipeline must bind only the three base plugins and EssentialCommands',
|
||||
),
|
||||
)
|
||||
version = data(suite, 'GET', '/api/v1/system/info').get('version')
|
||||
require(isinstance(version, str) and bool(version), 'Core did not report its version')
|
||||
records = Path(suite.path(suite.profile['smoke']['mock_records']))
|
||||
require(records.is_file(), 'Command model-call evidence requires the dedicated mock recording file')
|
||||
body = resolve(
|
||||
copy.deepcopy(next(item['body'] for item in suite.profile['resources'] if item['id'] == 'chat_bot')),
|
||||
suite.variables,
|
||||
)
|
||||
body['name'] = 'qa-plugin-command-' + uuid4().hex
|
||||
body['adapter_config']['default_session_type'] = 'person'
|
||||
body['event_bindings'] = [
|
||||
{
|
||||
'event_pattern': 'message.received',
|
||||
'target_type': 'pipeline',
|
||||
'target_uuid': suite.variables['plugin_command_pipeline'],
|
||||
'enabled': True,
|
||||
'filters': [],
|
||||
}
|
||||
]
|
||||
body['plugin_processors'] = []
|
||||
owned = {'kind': 'bot', 'name': body['name'], 'state': 'creating'}
|
||||
row.setdefault('owned_resources', []).append(owned)
|
||||
journal(suite, row)
|
||||
|
||||
def delete_bot():
|
||||
if 'uuid' not in owned:
|
||||
bots = data(suite, 'GET', '/api/v1/platform/bots')['bots']
|
||||
matches = [bot for bot in bots if bot.get('name') == owned['name']]
|
||||
require(len(matches) == 1, 'Created command Bot identity is unresolved; inspect the resource journal')
|
||||
owned['uuid'] = matches[0]['uuid']
|
||||
journal(suite, row)
|
||||
path = f'/api/v1/platform/bots/{owned["uuid"]}'
|
||||
status, response = api(suite, 'GET', path)
|
||||
if status != 404:
|
||||
require(
|
||||
status == 200
|
||||
and response.get('code') == 0
|
||||
and response.get('data', {}).get('bot', {}).get('name') == owned['name'],
|
||||
'Cannot confirm ownership of the command Bot before cleanup',
|
||||
)
|
||||
data(suite, 'DELETE', path)
|
||||
status, _ = api(suite, 'GET', path)
|
||||
require(status == 404, 'Command Bot remains visible after cleanup')
|
||||
owned['state'] = 'deleted'
|
||||
|
||||
try:
|
||||
created = data(suite, 'POST', '/api/v1/platform/bots', body)
|
||||
owned.update(uuid=created['uuid'], state='created')
|
||||
journal(suite, row)
|
||||
session = 'qa-plugin-version-' + uuid4().hex
|
||||
payload = {
|
||||
'session_id': session,
|
||||
'session_type': 'person',
|
||||
'message': [{'type': 'Plain', 'text': '!version'}],
|
||||
}
|
||||
offset = records.stat().st_size
|
||||
reply = data(
|
||||
suite,
|
||||
'POST',
|
||||
f'/bots/{owned["uuid"]}/sync',
|
||||
payload,
|
||||
headers=signed_headers(json.dumps(payload).encode(), body['adapter_config']['inbound_secret']),
|
||||
)
|
||||
row['command_response'] = reply
|
||||
parts = reply.get('message')
|
||||
checked(
|
||||
row,
|
||||
'command_version',
|
||||
lambda: require(
|
||||
reply.get('session_id') == session
|
||||
and bool(reply.get('reply_to'))
|
||||
and isinstance(parts, list)
|
||||
and all(isinstance(part, dict) and part.get('type') == 'Plain' for part in parts)
|
||||
and ''.join(part.get('text', '') for part in parts) == f'[bot] LangBot Version: {version}',
|
||||
'Version command did not return the exact Core version in the originating session',
|
||||
),
|
||||
)
|
||||
with closing(ledger(suite)) as connection:
|
||||
count = connection.execute(
|
||||
'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND bot_id=? AND id>?',
|
||||
(suite.variables['workspace_uuid'], owned['uuid'], row['after_id']),
|
||||
).fetchone()[0]
|
||||
checked(row, 'command_no_run', lambda: require(count == 0, 'Version command unexpectedly created an AgentRun'))
|
||||
checked(
|
||||
row,
|
||||
'command_no_model',
|
||||
lambda: check_mock_requests(records, offset, [], timeout=suite.timeout(3)),
|
||||
)
|
||||
return {'result': case['expect'], 'version': version, 'agent_runs': count, 'model_requests': 0}
|
||||
finally:
|
||||
cleanup(suite, row, 'command_bot_cleanup', delete_bot)
|
||||
|
||||
|
||||
def faq_crud(suite, case, row):
|
||||
def page(method, endpoint, body=None):
|
||||
return data(
|
||||
suite, 'POST', FAQ_API, {'page_id': 'manager', 'endpoint': endpoint, 'method': method, 'body': body}
|
||||
)
|
||||
|
||||
def entries():
|
||||
values = page('GET', '/entries').get('entries')
|
||||
require(
|
||||
isinstance(values, list) and all(isinstance(item, dict) and item.get('id') for item in values),
|
||||
'FAQ entries response is malformed',
|
||||
)
|
||||
return values
|
||||
|
||||
previous = {item['id'] for item in entries()}
|
||||
marker = 'qa-faq-' + uuid4().hex
|
||||
question, before, after = marker + '-question', marker + '-before', marker + '-after'
|
||||
owned = {'kind': 'faq-entry', 'question': question, 'state': 'creating'}
|
||||
row.setdefault('owned_resources', []).append(owned)
|
||||
journal(suite, row)
|
||||
|
||||
def expect_entry(values, answer):
|
||||
expected = {'id': owned['id'], 'question': question, 'answer': answer}
|
||||
require(values == [expected], 'FAQ content or entry identity differs from the submitted value')
|
||||
|
||||
def delete_entry():
|
||||
current = entries()
|
||||
if 'id' not in owned:
|
||||
matches = [item for item in current if item.get('question') == question and item['id'] not in previous]
|
||||
require(len(matches) <= 1, 'Created FAQ identity is ambiguous; inspect the resource journal')
|
||||
if not matches:
|
||||
owned['state'] = 'absent'
|
||||
return
|
||||
owned['id'] = matches[0]['id']
|
||||
journal(suite, row)
|
||||
require(owned['id'] not in previous, 'Refusing to delete a pre-existing FAQ entry')
|
||||
matches = [item for item in current if item['id'] == owned['id']]
|
||||
if matches:
|
||||
require(
|
||||
len(matches) == 1 and matches[0].get('question') == question, 'FAQ ownership changed before cleanup'
|
||||
)
|
||||
deleted = page('DELETE', '/entries', {'id': owned['id']})
|
||||
require(deleted.get('deleted') == owned['id'], 'FAQ deletion returned a different entry ID')
|
||||
require(not any(item['id'] == owned['id'] for item in entries()), 'Deleted FAQ remains visible')
|
||||
require(page('POST', '/search', {'query': marker}).get('results') == [], 'Deleted FAQ remains searchable')
|
||||
owned['state'] = 'deleted'
|
||||
|
||||
try:
|
||||
created = page('POST', '/entries', {'question': question, 'answer': before}).get('entry')
|
||||
require(isinstance(created, dict) and bool(created.get('id')), 'FAQ creation did not return an entry ID')
|
||||
owned.update(id=created['id'], state='created')
|
||||
journal(suite, row)
|
||||
require(owned['id'] not in previous, 'FAQ creation reused a pre-existing entry ID')
|
||||
checked(row, 'faq_create', lambda: expect_entry([created], before))
|
||||
checked(
|
||||
row, 'faq_read', lambda: expect_entry([item for item in entries() if item['id'] == owned['id']], before)
|
||||
)
|
||||
checked(
|
||||
row, 'faq_search', lambda: expect_entry(page('POST', '/search', {'query': before}).get('results'), before)
|
||||
)
|
||||
updated = page('PUT', '/entries', {'id': owned['id'], 'question': question, 'answer': after}).get('entry')
|
||||
checked(row, 'faq_update', lambda: expect_entry([updated], after))
|
||||
checked(
|
||||
row,
|
||||
'faq_update_read',
|
||||
lambda: expect_entry([item for item in entries() if item['id'] == owned['id']], after),
|
||||
)
|
||||
checked(
|
||||
row,
|
||||
'faq_update_search',
|
||||
lambda: expect_entry(page('POST', '/search', {'query': after}).get('results'), after),
|
||||
)
|
||||
checked(
|
||||
row,
|
||||
'faq_old_answer_absent',
|
||||
lambda: require(
|
||||
page('POST', '/search', {'query': before}).get('results') == [], 'FAQ search returned stale content'
|
||||
),
|
||||
)
|
||||
checked(row, 'faq_delete', delete_entry)
|
||||
return {'result': case['expect'], 'entry_id': owned['id'], 'deleted': True}
|
||||
finally:
|
||||
if owned['state'] != 'deleted':
|
||||
cleanup(suite, row, 'faq_cleanup', delete_entry)
|
||||
else:
|
||||
journal(suite, row)
|
||||
|
||||
|
||||
def run_case(suite, case, row):
|
||||
require(case['scenario'] in CASE_NAMES, 'Unknown plugin scenario')
|
||||
return (command_version if case['scenario'] == 'plugins.command_version' else faq_crud)(suite, case, row)
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Resolve official published regression plugins once, then install the recorded bytes."""
|
||||
"""Prepare source or published regression plugins and verify installed artifacts."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
@@ -6,6 +6,10 @@ import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
@@ -15,6 +19,7 @@ import yaml
|
||||
|
||||
MARKETPLACE = 'https://space.langbot.app'
|
||||
PLUGIN_IDS = ('langbot-team/LocalAgent', 'langbot-team/LangRAG', 'langbot-team/GeneralParsers')
|
||||
PLUGIN_PATHS = ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers')
|
||||
METADATA_LIMIT = 1024 * 1024
|
||||
PACKAGE_LIMIT = 64 * 1024 * 1024
|
||||
|
||||
@@ -83,7 +88,7 @@ def check_archive(content, plugin_id, version):
|
||||
):
|
||||
raise ValueError('package manifest identity/version differs from the selected release')
|
||||
except (zipfile.BadZipFile, yaml.YAMLError, KeyError, ValueError, RuntimeError) as exc:
|
||||
raise ValueError(f'Invalid published package for {plugin_id}@{version}: {exc}') from exc
|
||||
raise ValueError(f'Invalid plugin package for {plugin_id}@{version}: {exc}') from exc
|
||||
|
||||
|
||||
def prepare_published_plugins(directory, replay=None):
|
||||
@@ -141,8 +146,61 @@ def prepare_published_plugins(directory, replay=None):
|
||||
return manifest
|
||||
|
||||
|
||||
def install_published_plugins(call, directory, manifest):
|
||||
"""Install the downloaded packages and require exact, initialized CLI readback."""
|
||||
def prepare_source_plugins(directory, source, extra_plugins=()):
|
||||
"""Build selected plugin directories with the SDK CLI, without publishing."""
|
||||
directory, source = Path(directory).resolve(), Path(source).resolve()
|
||||
paths = [*PLUGIN_PATHS, *extra_plugins]
|
||||
if len(paths) != len(set(paths)):
|
||||
raise ValueError('Duplicate source plugin selection')
|
||||
for path in paths:
|
||||
if (
|
||||
not re.fullmatch(r'(Runner|KnowledgeEngine|misc)/[A-Za-z0-9_.-]+', path)
|
||||
or not (source / path).resolve().is_relative_to(source)
|
||||
or not (source / path / 'manifest.yaml').is_file()
|
||||
):
|
||||
raise ValueError(f'Invalid source plugin path: {path}')
|
||||
directory.mkdir(parents=True, exist_ok=False)
|
||||
manifest = {'version': 1, 'selection': 'source', 'plugins': []}
|
||||
write_manifest(directory, manifest)
|
||||
for path in paths:
|
||||
metadata = yaml.safe_load((source / path / 'manifest.yaml').read_text())['metadata']
|
||||
plugin_id = f'{metadata["author"]}/{metadata["name"]}'
|
||||
if any(entry['id'] == plugin_id for entry in manifest['plugins']):
|
||||
raise ValueError(f'Duplicate source plugin identity: {plugin_id}')
|
||||
if not all(re.fullmatch(r'[A-Za-z0-9_.-]+', metadata[key]) for key in ('author', 'name')):
|
||||
raise ValueError(f'Invalid source plugin identity: {plugin_id}')
|
||||
version = release_version(metadata['version'])
|
||||
entry = {
|
||||
'id': plugin_id,
|
||||
'version': version,
|
||||
'source': path,
|
||||
'archive': f'{metadata["author"]}-{metadata["name"]}-{version}.lbpkg',
|
||||
}
|
||||
manifest['plugins'].append(entry)
|
||||
write_manifest(directory, manifest)
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
package_source = Path(temporary) / 'plugin'
|
||||
shutil.copytree(
|
||||
source / path, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')
|
||||
)
|
||||
with (directory / f'{metadata["name"]}-build.log').open('w') as log:
|
||||
subprocess.run(
|
||||
[sys.executable, '-m', 'langbot_plugin.cli.__init__', 'build', '--output', str(directory)],
|
||||
cwd=package_source,
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
check=True,
|
||||
timeout=180,
|
||||
)
|
||||
content = (directory / entry['archive']).read_bytes()
|
||||
check_archive(content, plugin_id, version)
|
||||
entry['sha256'] = hashlib.sha256(content).hexdigest()
|
||||
write_manifest(directory, manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def install_plugins(call, directory, manifest):
|
||||
"""Install recorded packages and require exact, initialized CLI readback."""
|
||||
directory = Path(directory)
|
||||
for entry in manifest['plugins']:
|
||||
archive = directory / entry['archive']
|
||||
@@ -151,7 +209,7 @@ def install_published_plugins(call, directory, manifest):
|
||||
try:
|
||||
call(['plugin', 'install', 'local', '--file', str(archive), '--wait'])
|
||||
except RuntimeError as exc:
|
||||
raise RuntimeError(f'Published plugin install failed: {entry["id"]}@{entry["version"]}: {exc}') from exc
|
||||
raise RuntimeError(f'Plugin install failed: {entry["id"]}@{entry["version"]}: {exc}') from exc
|
||||
response = call(['plugin', 'list'])
|
||||
plugins = response.get('plugins') if isinstance(response, dict) else None
|
||||
if not isinstance(plugins, list) or any(not isinstance(item, dict) for item in plugins):
|
||||
@@ -165,6 +223,4 @@ def install_published_plugins(call, directory, manifest):
|
||||
or matches[0].get('enabled') is not True
|
||||
or matches[0].get('status') != 'initialized'
|
||||
):
|
||||
raise RuntimeError(
|
||||
f'Published plugin is not ready at the selected version: {entry["id"]}: {entry["installed"]}'
|
||||
)
|
||||
raise RuntimeError(f'Plugin is not ready at the selected version: {entry["id"]}: {entry["installed"]}')
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Exercise the daily workflow's published verdict, including incomplete runs."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
from regression_ci import SUITE_BUDGETS, suite_profile
|
||||
from regression_suite import validate
|
||||
|
||||
|
||||
class CIProfileTest(unittest.TestCase):
|
||||
def test_local_groups_are_explicit_and_smoke_rejects_them(self):
|
||||
profile, _ = suite_profile('daily', 'test-run', ['resources'])
|
||||
self.assertEqual(profile['scenario_groups'], ['resources'])
|
||||
self.assertIn('auth_revocation', {case['name'] for case in profile['cases']})
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
process = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(Path(__file__).with_name('regression_ci.py')),
|
||||
'--suite',
|
||||
'smoke',
|
||||
'--scenario-groups',
|
||||
'resources',
|
||||
'--lbctl',
|
||||
'unused',
|
||||
'--output',
|
||||
str(Path(directory) / 'not-created'),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
self.assertEqual(process.returncode, 2)
|
||||
self.assertIn('require --suite scenarios or --suite daily', process.stderr)
|
||||
self.assertFalse((Path(directory) / 'not-created').exists())
|
||||
|
||||
def test_suite_selection_uses_shared_cases_and_one_daily_budget(self):
|
||||
for name in ('smoke', 'scenarios', 'daily', 'endurance'):
|
||||
with self.subTest(suite=name):
|
||||
profile, path = suite_profile(name, 'test-run')
|
||||
profile['observe'] = {
|
||||
'health': {'core': 'http://localhost/healthz'},
|
||||
'pids': {'core': 1, 'runtime': 2, 'box': 3},
|
||||
'box_data_root': '/tmp/test-box',
|
||||
}
|
||||
validate(profile)
|
||||
self.assertTrue(path.exists())
|
||||
self.assertEqual(profile['cleanup'], 'always')
|
||||
self.assertEqual(profile['performance_mode'], 'observe')
|
||||
if name == 'smoke':
|
||||
self.assertEqual(profile['smoke']['group_turns'], 2)
|
||||
self.assertEqual(SUITE_BUDGETS[name], 60)
|
||||
elif name in ('scenarios', 'daily'):
|
||||
self.assertEqual(len(profile['cases']), 46)
|
||||
self.assertEqual(profile['coverage_suite'], 'scenarios')
|
||||
self.assertEqual(profile['smoke']['group_turns'], 3)
|
||||
else:
|
||||
self.assertEqual(profile['phases'][0]['duration_seconds'], 540)
|
||||
self.assertEqual('execution' in profile, name == 'daily')
|
||||
daily, _ = suite_profile('daily', 'test-run')
|
||||
self.assertEqual(daily['execution'], {'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60})
|
||||
endurance, _ = suite_profile('endurance', 'test-run')
|
||||
self.assertEqual(daily['resource_limits']['warmup_seconds'], 180)
|
||||
self.assertEqual(daily['resource_limits']['targets'], endurance['resource_limits']['targets'])
|
||||
self.assertEqual(daily['resource_limits']['containers'], endurance['resource_limits']['containers'])
|
||||
|
||||
def test_daily_reuses_boxes_per_pipeline_and_separates_runs(self):
|
||||
def keys(name, run_id):
|
||||
profile, _ = suite_profile(name, run_id)
|
||||
return {
|
||||
resource['id']: runner['box-session-id-template']
|
||||
for resource in profile['resources']
|
||||
if resource['kind'] == 'pipeline'
|
||||
for runner in resource['body']['config']['ai']['runner_config'].values()
|
||||
if runner.get('box-enabled')
|
||||
}
|
||||
|
||||
first, second = keys('daily', 'run-a'), keys('daily', 'run-b')
|
||||
self.assertTrue(first)
|
||||
self.assertEqual(len(set(first.values())), len(first))
|
||||
self.assertFalse(set(first.values()) & set(second.values()))
|
||||
self.assertEqual(first, keys('daily', 'run-a'))
|
||||
self.assertTrue(all(value == '{launcher_type}_{launcher_id}' for value in keys('scenarios', 'run-a').values()))
|
||||
|
||||
|
||||
class DailySummaryTest(unittest.TestCase):
|
||||
def test_summary_reports_observations_without_hiding_outer_failures(self):
|
||||
workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml'
|
||||
steps = yaml.safe_load(workflow.read_text())['jobs']['endurance']['steps']
|
||||
command = next(step['run'] for step in steps if step.get('name') == 'Publish observations')
|
||||
suite = {
|
||||
'passed': True,
|
||||
'outcome': 'PASS',
|
||||
'errors': [],
|
||||
'first_pass_complete': True,
|
||||
'cycles': [{'complete': True}],
|
||||
'requests': 86,
|
||||
'ledger_closeout': {'matched': True},
|
||||
'observations': {'successful_requests_per_second': 1.0, 'step_p95_seconds': {'chat': 0.5}},
|
||||
}
|
||||
for gate, actual_suite, expected in [
|
||||
({'passed': True}, suite, 'Daily regression: PASS'),
|
||||
({'passed': False, 'infrastructure_error': 'teardown failed'}, suite, 'teardown failed'),
|
||||
({'passed': False, 'infrastructure_error': 'published plugin install failed'}, None, 'not confirmed'),
|
||||
({'passed': False, 'timed_out': True}, suite, '1860-second outer process guard'),
|
||||
({'passed': False, 'coverage_errors': ['missing access records']}, suite, 'missing access records'),
|
||||
({'passed': False}, {**suite, 'passed': False, 'outcome': 'INCOMPLETE'}, 'Daily regression: INCOMPLETE'),
|
||||
(None, None, 'Missing verdict'),
|
||||
]:
|
||||
with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
target = root / '.qa/daily'
|
||||
(target / 'plugins').mkdir(parents=True)
|
||||
(target / 'plugins/manifest.json').write_text(
|
||||
json.dumps({'plugins': [{'id': 'langbot-team/LocalAgent', 'version': '1.2.3'}]})
|
||||
)
|
||||
if gate is not None:
|
||||
(target / 'gate/suite').mkdir(parents=True)
|
||||
(target / 'gate/gate.json').write_text(json.dumps(gate))
|
||||
if actual_suite is not None:
|
||||
(target / 'gate/suite/summary.json').write_text(json.dumps(actual_suite))
|
||||
summary = root / 'summary.md'
|
||||
env = {
|
||||
**os.environ,
|
||||
'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'],
|
||||
'GITHUB_STEP_SUMMARY': str(summary),
|
||||
'GITHUB_SHA': 'core-revision',
|
||||
}
|
||||
subprocess.run(['bash', '-e', '-c', command], cwd=root, env=env, check=True, capture_output=True)
|
||||
text = summary.read_text()
|
||||
self.assertIn(expected, text)
|
||||
self.assertIn('langbot-team/[email protected]', text)
|
||||
self.assertIn('not confirmed', text)
|
||||
if gate and gate.get('passed'):
|
||||
self.assertIn('Final ledger reconciliation: True', text)
|
||||
self.assertIn('1.000 RPS', text)
|
||||
else:
|
||||
self.assertNotIn('Daily regression: PASS', text)
|
||||
|
||||
def test_core_summary_records_preparation_failure_and_selected_plugins(self):
|
||||
workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml'
|
||||
steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps']
|
||||
command = next(step['run'] for step in steps if step.get('name') == 'Publish gate result')
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
target = root / '.qa/core-smoke'
|
||||
(target / 'plugins').mkdir(parents=True)
|
||||
(target / 'gate').mkdir()
|
||||
(target / 'plugins/manifest.json').write_text(
|
||||
json.dumps({'plugins': [{'id': 'langbot-team/LocalAgent', 'version': '1.2.3'}]})
|
||||
)
|
||||
(target / 'gate/gate.json').write_text(
|
||||
json.dumps({'passed': False, 'infrastructure_error': 'incompatible'})
|
||||
)
|
||||
summary = root / 'summary.md'
|
||||
subprocess.run(
|
||||
['bash', '-e', '-c', command],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env={
|
||||
**os.environ,
|
||||
'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'],
|
||||
'GITHUB_STEP_SUMMARY': str(summary),
|
||||
},
|
||||
)
|
||||
text = summary.read_text()
|
||||
self.assertIn('Core smoke: FAIL', text)
|
||||
self.assertIn('Preparation or teardown failed', text)
|
||||
self.assertIn('langbot-team/[email protected]', text)
|
||||
self.assertIn('not confirmed', text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Challenge plugin business oracles and cleanup without starting target services."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import regression_plugin_cases as plugins
|
||||
from regression_scenarios import expanded_profile
|
||||
|
||||
|
||||
def success(value=None):
|
||||
return 200, {'code': 0, 'data': value}
|
||||
|
||||
|
||||
class PluginCasesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.root = Path(temporary.name)
|
||||
self.profile = expanded_profile()
|
||||
plugins.add_cases(self.profile)
|
||||
self.records = self.root / 'mock.jsonl'
|
||||
self.records.write_text(
|
||||
json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'earlier'}, 'status': 200}) + '\n'
|
||||
)
|
||||
self.database = self.root / 'ledger.db'
|
||||
with sqlite3.connect(self.database) as connection:
|
||||
connection.execute('CREATE TABLE agent_run (id INTEGER, workspace_id TEXT, bot_id TEXT)')
|
||||
self.profile['smoke'].update(ledger_db=str(self.database), mock_records=str(self.records))
|
||||
self.suite = SimpleNamespace(
|
||||
profile=self.profile,
|
||||
variables={
|
||||
'workspace_uuid': 'workspace',
|
||||
'chat_pipeline': 'original-pipeline',
|
||||
'plugin_command_pipeline': 'isolated-pipeline',
|
||||
'group_secret': 'private-signing-secret',
|
||||
},
|
||||
secrets=['private-signing-secret'],
|
||||
env={},
|
||||
path=lambda value: value,
|
||||
timeout=lambda value: value,
|
||||
write=lambda name, value: (self.root / name).write_text(json.dumps(value)),
|
||||
)
|
||||
|
||||
def case(self, scenario):
|
||||
return next(case for case in self.profile['cases'] if case.get('scenario') == scenario)
|
||||
|
||||
def test_profile_adds_only_one_isolated_pipeline_and_two_scenarios(self):
|
||||
original = expanded_profile()
|
||||
profile = copy.deepcopy(original)
|
||||
plugins.add_cases(profile)
|
||||
self.assertEqual(profile['resources'][:-1], original['resources'])
|
||||
self.assertEqual(profile['cases'][:-2], original['cases'])
|
||||
pipeline = profile['resources'][-1]
|
||||
self.assertEqual(pipeline['id'], 'plugin_command_pipeline')
|
||||
self.assertEqual(
|
||||
pipeline['body']['extensions_preferences'],
|
||||
{
|
||||
'enable_all_plugins': False,
|
||||
'plugins': [
|
||||
{'author': 'langbot-team', 'name': name}
|
||||
for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands')
|
||||
],
|
||||
},
|
||||
)
|
||||
self.assertEqual({case['scenario'] for case in profile['cases'][-2:]}, plugins.CASE_NAMES)
|
||||
self.assertTrue(all('http_group' not in case for case in profile['cases'][-2:]))
|
||||
|
||||
def command_api(self, failure=None):
|
||||
state = {}
|
||||
|
||||
def request(suite, method, path, body=None, headers=None):
|
||||
if path == '/api/v1/pipelines/isolated-pipeline':
|
||||
return success(
|
||||
{
|
||||
'pipeline': {
|
||||
'extensions_preferences': {
|
||||
'enable_all_plugins': failure == 'bindings',
|
||||
'plugins': copy.deepcopy(plugins.COMMAND_PLUGINS),
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
if path == '/api/v1/system/info':
|
||||
return success({'version': '9.5.1'})
|
||||
if path == '/api/v1/platform/bots' and method == 'POST':
|
||||
self.assertEqual(body['event_bindings'][0]['target_uuid'], 'isolated-pipeline')
|
||||
self.assertEqual(body['plugin_processors'], [])
|
||||
self.assertEqual(body['adapter_config']['inbound_secret'], 'private-signing-secret')
|
||||
state.update(uuid='owned-bot', name=body['name'])
|
||||
return success({'uuid': 'owned-bot'})
|
||||
if path == '/bots/owned-bot/sync':
|
||||
self.assertEqual(body['session_type'], 'person')
|
||||
self.assertEqual(body['message'], [{'type': 'Plain', 'text': '!version'}])
|
||||
signed = headers['X-LB-Timestamp'].encode() + b'.' + json.dumps(body).encode()
|
||||
expected = 'sha256=' + hmac.new(b'private-signing-secret', signed, hashlib.sha256).hexdigest()
|
||||
self.assertEqual(headers['X-LB-Signature'], expected)
|
||||
if failure == 'run':
|
||||
with sqlite3.connect(self.database) as connection:
|
||||
connection.execute("INSERT INTO agent_run VALUES (1, 'workspace', 'owned-bot')")
|
||||
if failure == 'model':
|
||||
with self.records.open('a') as stream:
|
||||
stream.write(
|
||||
json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'wrong'}, 'status': 200})
|
||||
+ '\n'
|
||||
)
|
||||
return success(
|
||||
{
|
||||
'session_id': 'wrong-session' if failure == 'session' else body['session_id'],
|
||||
'reply_to': 'in_message',
|
||||
'message': [
|
||||
{
|
||||
'type': 'Plain',
|
||||
'text': '[bot] LangBot Version: ' + ('9.4.0' if failure == 'version' else '9.5.1'),
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
if path == '/api/v1/platform/bots/owned-bot':
|
||||
if method == 'DELETE':
|
||||
state['deleted'] = True
|
||||
return success() # The real Bot DELETE returns data=null.
|
||||
return (404, {'code': -1}) if state.get('deleted') else success({'bot': dict(state)})
|
||||
self.fail(f'Unexpected command API call: {method} {path}')
|
||||
|
||||
return request, state
|
||||
|
||||
def test_command_checks_signed_reply_without_requiring_a_run(self):
|
||||
request, state = self.command_api()
|
||||
row = {'after_id': 0}
|
||||
with patch.object(plugins, 'api', side_effect=request):
|
||||
result = plugins.run_case(self.suite, self.case('plugins.command_version'), row)
|
||||
self.assertEqual(
|
||||
result, {'result': 'LANGBOT_PLUGIN_COMMAND_OK', 'version': '9.5.1', 'agent_runs': 0, 'model_requests': 0}
|
||||
)
|
||||
self.assertTrue(state['deleted'])
|
||||
self.assertNotIn('scenario_runs', row)
|
||||
self.assertTrue(all(value == 'passed' for value in row['assertions'].values()))
|
||||
|
||||
def test_command_rejects_bad_reply_model_calls_runs_and_plugin_leakage(self):
|
||||
for failure in ('version', 'session', 'model', 'run', 'bindings'):
|
||||
with self.subTest(failure=failure):
|
||||
with sqlite3.connect(self.database) as connection:
|
||||
connection.execute('DELETE FROM agent_run')
|
||||
request, state = self.command_api(failure)
|
||||
row = {'after_id': 0}
|
||||
with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError):
|
||||
plugins.run_case(self.suite, self.case('plugins.command_version'), row)
|
||||
if failure != 'bindings':
|
||||
self.assertTrue(state['deleted'])
|
||||
self.assertIn('failed', row['assertions'].values())
|
||||
|
||||
def faq_api(self, failure=None, cleanup_failure=False):
|
||||
unrelated = {'id': 'unrelated', 'question': 'Existing question', 'answer': 'Existing answer'}
|
||||
state = {'entries': [unrelated], 'created': 0, 'deletes': []}
|
||||
|
||||
def request(suite, method, path, body=None, headers=None):
|
||||
self.assertEqual((method, path, body['page_id']), ('POST', plugins.FAQ_API, 'manager'))
|
||||
verb, endpoint, payload = body['method'], body['endpoint'], body['body']
|
||||
if (verb, endpoint) == ('GET', '/entries'):
|
||||
return success({'entries': copy.deepcopy(state['entries'])})
|
||||
if (verb, endpoint) == ('POST', '/entries'):
|
||||
state['created'] += 1
|
||||
entry = {'id': 'owned-faq', **payload}
|
||||
state['entries'].append(entry)
|
||||
state['original'] = copy.deepcopy(entry)
|
||||
if failure == 'lost_create':
|
||||
raise ConnectionResetError('creation response lost')
|
||||
return success({'entry': copy.deepcopy(entry)})
|
||||
if (verb, endpoint) == ('PUT', '/entries'):
|
||||
self.assertEqual(payload['id'], 'owned-faq')
|
||||
if failure == 'update_error':
|
||||
raise ValueError('primary update failure')
|
||||
if failure != 'stale_update':
|
||||
state['entries'][1].update(payload)
|
||||
return success({'entry': copy.deepcopy(payload)})
|
||||
if (verb, endpoint) == ('POST', '/search'):
|
||||
rows = list(state['entries'])
|
||||
if failure == 'stale_search' and state['deletes']:
|
||||
rows.append(state['original'])
|
||||
return success(
|
||||
{
|
||||
'results': copy.deepcopy(
|
||||
[
|
||||
item
|
||||
for item in rows
|
||||
if payload['query'] in item['question'] or payload['query'] in item['answer']
|
||||
]
|
||||
)
|
||||
}
|
||||
)
|
||||
if (verb, endpoint) == ('DELETE', '/entries'):
|
||||
self.assertEqual(payload, {'id': 'owned-faq'})
|
||||
if cleanup_failure:
|
||||
raise RuntimeError('cleanup private-signing-secret failed')
|
||||
state['deletes'].append(payload['id'])
|
||||
state['entries'] = [item for item in state['entries'] if item['id'] != payload['id']]
|
||||
return success({'deleted': payload['id']})
|
||||
self.fail(f'Unexpected Page API call: {body}')
|
||||
|
||||
return request, state
|
||||
|
||||
def test_faq_crud_checks_content_and_removes_only_its_entry(self):
|
||||
request, state = self.faq_api()
|
||||
row = {}
|
||||
with patch.object(plugins, 'api', side_effect=request):
|
||||
result = plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
||||
self.assertEqual(result, {'result': 'LANGBOT_PLUGIN_FAQ_OK', 'entry_id': 'owned-faq', 'deleted': True})
|
||||
self.assertEqual(state['deletes'], ['owned-faq'])
|
||||
self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated'])
|
||||
self.assertTrue(all(value == 'passed' for value in row['assertions'].values()))
|
||||
|
||||
def test_faq_rejects_stale_readback_and_search(self):
|
||||
for failure, assertion in (('stale_update', 'faq_update_read'), ('stale_search', 'faq_delete')):
|
||||
with self.subTest(failure=failure):
|
||||
request, state = self.faq_api(failure)
|
||||
row = {}
|
||||
with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError):
|
||||
plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
||||
self.assertEqual(row['assertions'][assertion], 'failed')
|
||||
self.assertEqual(state['deletes'], ['owned-faq'])
|
||||
self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated'])
|
||||
|
||||
def test_faq_lost_create_response_recovers_owned_id_without_retrying_creation(self):
|
||||
request, state = self.faq_api('lost_create')
|
||||
row = {}
|
||||
with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ConnectionResetError):
|
||||
plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
||||
self.assertEqual(state['created'], 1)
|
||||
self.assertEqual(state['deletes'], ['owned-faq'])
|
||||
self.assertEqual(row['owned_resources'][0]['state'], 'deleted')
|
||||
|
||||
def test_faq_preserves_original_error_and_redacts_cleanup_error(self):
|
||||
request, _ = self.faq_api('update_error', cleanup_failure=True)
|
||||
row = {}
|
||||
with (
|
||||
patch.object(plugins, 'api', side_effect=request),
|
||||
self.assertRaisesRegex(ValueError, 'primary update failure'),
|
||||
):
|
||||
plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
||||
self.assertEqual(row['cleanup_errors'], ['RuntimeError: cleanup [REDACTED] failed'])
|
||||
evidence = (self.root / row['resource_journal']).read_text()
|
||||
self.assertNotIn('private-signing-secret', evidence)
|
||||
self.assertIn('owned-faq', evidence)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Exercise published resolution, replay integrity and installation readback."""
|
||||
"""Exercise plugin selection, artifact integrity and installation readback."""
|
||||
|
||||
from collections import Counter
|
||||
import copy
|
||||
@@ -12,6 +12,8 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
import zipfile
|
||||
|
||||
import yaml
|
||||
|
||||
import regression_plugins as plugins
|
||||
|
||||
|
||||
@@ -27,6 +29,66 @@ def archive(plugin_id, version='1.2.3'):
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
class SourcePluginsTests(unittest.TestCase):
|
||||
def test_selected_sources_build_in_isolation_and_record_actual_artifacts(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
source, output = root / 'sources', root / 'packages'
|
||||
paths = (*plugins.PLUGIN_PATHS, 'misc/ExtraPlugin')
|
||||
for path in paths:
|
||||
folder = source / path
|
||||
folder.mkdir(parents=True)
|
||||
with zipfile.ZipFile(io.BytesIO(archive('test/' + folder.name))) as package:
|
||||
package.extractall(folder)
|
||||
|
||||
def build(args, *, cwd, **kwargs):
|
||||
self.assertEqual(args[1:5], ['-m', 'langbot_plugin.cli.__init__', 'build', '--output'])
|
||||
self.assertFalse(cwd.is_relative_to(source))
|
||||
metadata = yaml.safe_load((cwd / 'manifest.yaml').read_text())['metadata']
|
||||
(cwd / 'generated.html').write_text('Generated only in the isolated build copy')
|
||||
filename = f'{metadata["author"]}-{metadata["name"]}-{metadata["version"]}.lbpkg'
|
||||
(Path(args[-1]) / filename).write_bytes(archive('test/' + metadata['name']))
|
||||
|
||||
with patch.object(plugins.subprocess, 'run', side_effect=build):
|
||||
manifest = plugins.prepare_source_plugins(output, source, ['misc/ExtraPlugin'])
|
||||
self.assertEqual(manifest['selection'], 'source')
|
||||
self.assertEqual([entry['source'] for entry in manifest['plugins']], list(paths))
|
||||
for entry in manifest['plugins']:
|
||||
self.assertEqual(
|
||||
entry['sha256'], plugins.hashlib.sha256((output / entry['archive']).read_bytes()).hexdigest()
|
||||
)
|
||||
self.assertFalse((source / entry['source'] / 'generated.html').exists())
|
||||
|
||||
def test_invalid_or_duplicate_selection_is_rejected_before_build(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
for path in plugins.PLUGIN_PATHS:
|
||||
(root / path).mkdir(parents=True)
|
||||
(root / path / 'manifest.yaml').write_text('kind: Plugin')
|
||||
(root / 'misc/escape').symlink_to(root.parent, target_is_directory=True)
|
||||
for selected in (['../outside'], ['/tmp/plugin'], ['misc/escape'], [plugins.PLUGIN_PATHS[0]]):
|
||||
with self.subTest(selected=selected), patch.object(plugins.subprocess, 'run') as build:
|
||||
with self.assertRaises(ValueError):
|
||||
plugins.prepare_source_plugins(root / 'output', root, selected)
|
||||
build.assert_not_called()
|
||||
|
||||
def test_wrong_built_identity_fails_without_installation(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
for path in plugins.PLUGIN_PATHS:
|
||||
folder = root / path
|
||||
folder.mkdir(parents=True)
|
||||
with zipfile.ZipFile(io.BytesIO(archive('test/' + folder.name))) as package:
|
||||
package.extractall(folder)
|
||||
|
||||
def build(args, **kwargs):
|
||||
(Path(args[-1]) / 'test-LocalAgent-1.2.3.lbpkg').write_bytes(archive('test/WrongPlugin'))
|
||||
|
||||
with patch.object(plugins.subprocess, 'run', side_effect=build):
|
||||
with self.assertRaisesRegex(ValueError, 'identity/version differs'):
|
||||
plugins.prepare_source_plugins(root / 'output', root)
|
||||
|
||||
|
||||
class PublishedPluginsTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
@@ -95,7 +157,7 @@ class PublishedPluginsTests(unittest.TestCase):
|
||||
self.assertEqual(args, ['plugin', 'list'])
|
||||
return {'plugins': installed}
|
||||
|
||||
plugins.install_published_plugins(call, directory, manifest)
|
||||
plugins.install_plugins(call, directory, manifest)
|
||||
recorded = json.loads((directory / 'manifest.json').read_text())
|
||||
self.assertTrue(all(entry['installed'][0]['status'] == 'initialized' for entry in recorded['plugins']))
|
||||
self.requests.clear()
|
||||
@@ -178,13 +240,13 @@ class PublishedPluginsTests(unittest.TestCase):
|
||||
}
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, 'not ready at the selected version'):
|
||||
plugins.install_published_plugins(call, directory, manifest)
|
||||
plugins.install_plugins(call, directory, manifest)
|
||||
self.assertEqual(len(calls), 2)
|
||||
recorded = json.loads((directory / 'manifest.json').read_text())
|
||||
self.assertEqual(recorded['plugins'][0]['installed'][0]['version'], '0.0.1')
|
||||
(directory / manifest['plugins'][0]['archive']).write_bytes(b'changed after download')
|
||||
with self.assertRaisesRegex(ValueError, 'changed before installation'):
|
||||
plugins.install_published_plugins(call, directory, manifest)
|
||||
plugins.install_plugins(call, directory, manifest)
|
||||
self.assertEqual(len(calls), 2)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user