From 21be11766b6978b438339b2c55b614c5ffb9cc5e Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:46:53 +0800 Subject: [PATCH] ci(regression): build monorepo plugins and exercise local command and page APIs --- .github/workflows/core-smoke.yml | 3 +- .github/workflows/daily-endurance.yml | 18 +- tools/README.md | 27 ++- tools/regression_ci.py | 41 +++-- tools/regression_contracts.json | 11 ++ tools/regression_job.py | 4 +- tools/regression_local_cases.py | 1 + tools/regression_plugin_cases.py | 248 +++++++++++++++++++++++++ tools/regression_plugins.py | 72 +++++++- tools/test_regression_plugin_cases.py | 255 ++++++++++++++++++++++++++ tools/test_regression_plugins.py | 70 ++++++- 11 files changed, 706 insertions(+), 44 deletions(-) create mode 100644 tools/regression_plugin_cases.py create mode 100644 tools/test_regression_plugin_cases.py diff --git a/.github/workflows/core-smoke.yml b/.github/workflows/core-smoke.yml index a3169692b..f440c6b5e 100644 --- a/.github/workflows/core-smoke.yml +++ b/.github/workflows/core-smoke.yml @@ -73,7 +73,7 @@ jobs: with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: if manifest.exists(): for plugin in json.loads(manifest.read_text())['plugins']: - out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") + out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") if path.exists(): data = json.loads(path.read_text()) out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; " @@ -104,6 +104,7 @@ jobs: .qa/core-smoke/gate/suite/cycles.json .qa/core-smoke/gate/suite/execution.json .qa/core-smoke/plugins/manifest.json + .qa/core-smoke/plugins/*-build.log .qa/core-smoke/*.log if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index d6dc3eb41..f11f86482 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -9,7 +9,11 @@ on: scenario_groups: description: 'Optional local groups, space separated; empty runs only base cases' type: string - default: 'faults resources lifecycle' + default: 'faults resources lifecycle plugins' + extra_plugins: + description: 'Additional source plugin paths; clear with the plugins group to run only base plugins' + type: string + default: 'misc/EssentialCommands misc/FAQManager' restart_services: description: 'Check owned Core and Runtime restarts once before resource sampling' type: boolean @@ -52,15 +56,17 @@ jobs: docker pull python:3.12-alpine - name: Run repeated scenarios and recovery checks env: - SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle' || inputs.scenario_groups }} + SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle plugins' || inputs.scenario_groups }} + EXTRA_PLUGINS: ${{ github.event_name == 'schedule' && 'misc/EssentialCommands misc/FAQManager' || inputs.extra_plugins }} RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }} run: | read -r -a groups <<< "$SCENARIO_GROUPS" + read -r -a extras <<< "$EXTRA_PLUGINS" args=() if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi uv run --no-sync python tools/regression_ci.py \ --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \ - --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily + --lbctl .endurance-bin/lbctl_linux_amd64 --plugins plugins --extra-plugins "${extras[@]}" --output .qa/daily - name: Publish observations if: always() run: | @@ -83,15 +89,16 @@ jobs: with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n") out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n') - out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n') + out.write('Real Core/Runtime/source plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n') out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n") restart = gate.get('restart_checks') if restart: out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n") for error in restart.get('errors', []): out.write(f"Restart check error: {error}\n") + out.write(f"Plugin selection: `{manifest.get('selection', 'not reported')}`.\n\n") for plugin in manifest.get('plugins', []): - out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") + out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n") out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n") data = suite.get('test_data', {}) @@ -140,6 +147,7 @@ jobs: .qa/daily/gate/suite/coverage.json .qa/daily/gate/suite/coverage.md .qa/daily/plugins/manifest.json + .qa/daily/plugins/*-build.log .qa/daily/restart-checks/profile.json .qa/daily/restart-checks/cases.json .qa/daily/restart-checks/summary.json diff --git a/tools/README.md b/tools/README.md index 193643ef1..cdb11e2f4 100644 --- a/tools/README.md +++ b/tools/README.md @@ -367,12 +367,13 @@ phases. The dedicated Docker daemon requirement above still applies. `Core Smoke` runs regression-tool tests and a real Core smoke in parallel on GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate, -uses the SDK from `uv.lock`, the latest published official plugins, `lbctl` +uses the SDK from `uv.lock`, plugins built from the same checkout, `lbctl` v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials or access to a developer's running environment. -Plugin versions are resolved once per run, installed from the downloaded release -archives and recorded with hashes and installation readback in `plugins/manifest.json`. -An unavailable or incompatible published package fails preparation; see +The SDK CLI builds LocalAgent, LangRAG and GeneralParsers into `.lbpkg` archives. +`plugins/manifest.json` records selection mode, plugin versions, archive hashes +and exact installation/readiness readback. Build logs are preserved alongside it. +The published route remains available when `--plugins` is omitted; see [published plugin preparation and replay](published-plugins.md). Dependency installation, service startup, sandbox image download and plugin @@ -413,7 +414,7 @@ supports **Run workflow** with a selected branch. The schedule becomes active only after the workflow is merged into the default branch; it is not a required merge check. -The workflow reuses the disposable smoke stack and latest published plugins. +The workflow reuses the disposable smoke stack and builds plugins from this checkout. `--suite daily` consumes 46 base cases plus the selected local groups, requires its first pass, then repeats ordered cases within one **1800-second** window. The clock starts before creating scenario resources and includes a 120-second reserve for drain/API @@ -421,7 +422,7 @@ cleanup and a 60-second cooldown. Service and plugin preparation precede this clock. A separate 1860-second process guard allows startup and final reporting; it does not grant extra scenario time. The whole job retains its 45-minute limit. -The scheduled job enables all local groups and the optional one-time restart +The scheduled job enables all local groups (including `plugins`) and the optional one-time restart checks. Manual dispatch exposes both switches. `--scenario-groups` accepts a space-separated selection; supplying the flag with no values runs only base cases. `scenarios` and `daily` enable all three groups when the flag is omitted. @@ -430,6 +431,7 @@ cases. `scenarios` and `daily` enable all three groups when the flag is omitted. |---|---| | `faults` | Cancel/timeout cleanup, model retry side effects, partial stream failure, exhausted fallback, Box path rejection | | `resources` | API-key revocation, direct Agent execution, document deletion with a surviving retrieval control | +| `plugins` | EssentialCommands version command without model/AgentRun execution; FAQManager Page API CRUD and search freshness | | `lifecycle` | Signed HTTP callback 5xx retry identity and subsequent delivery | For example, select resource and callback cases with @@ -477,15 +479,22 @@ To run the same entrypoint on a disposable Linux host with an idle Docker daemon ```sh uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \ - --lbctl /path/to/lbctl --output .qa/daily-001 + --lbctl /path/to/lbctl --plugins plugins \ + --extra-plugins misc/EssentialCommands misc/FAQManager --output .qa/daily-001 ``` `--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared expanded profile once under a 300-second guard. `--suite endurance` retains the separate ten-minute load profile under a 900-second guard; its CI entry point also reports throughput/P95 without applying their thresholds. All four modes use -published plugins by default. Explicit `--plugins` remains available for source -development; neither workflow selects it. The suite attempts owned API cleanup +published plugins when `--plugins` is omitted. Both workflows explicitly select +`--plugins plugins` so source changes are tested in the same PR. Daily additionally +selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the +space-separated `extra_plugins` list or leave it empty. `--extra-plugins` requires +source mode and accepts categorized plugin paths, without removing the three base +plugins. Missing or incompatible packages fail preparation, with no silent fallback. +Existing conversation Pipelines allow only the base plugins; plugin-specific cases +bind their own extensions to avoid changing unrelated prompts/tools. The suite attempts owned API cleanup even after failures. The CI owner then stops its service processes and removes its Box containers; a process guard timeout can interrupt API cleanup. diff --git a/tools/regression_ci.py b/tools/regression_ci.py index facb0517b..f83c9c885 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -6,7 +6,6 @@ import json import os from pathlib import Path import secrets -import shutil import signal import subprocess import sys @@ -16,7 +15,7 @@ import yaml from regression_gate import supervise from regression_coverage import write_report -from regression_plugins import install_published_plugins, prepare_published_plugins +from regression_plugins import PLUGIN_IDS, install_plugins, prepare_published_plugins, prepare_source_plugins from regression_local_cases import GROUPS, api, require from regression_suite import Suite from regression_traffic import owned_box_containers, read_endpoint_snapshot @@ -35,6 +34,15 @@ def suite_profile(name, run_id, groups=()): profile = expanded_profile(groups) else: profile = json.loads(profile_path.read_text()) + for resource in profile['resources']: + if resource['kind'] == 'pipeline': + resource['body'].setdefault( + 'extensions_preferences', + { + 'enable_all_plugins': False, + 'plugins': [dict(zip(('author', 'name'), plugin.split('/'))) for plugin in PLUGIN_IDS], + }, + ) if name == 'smoke': profile['smoke']['group_turns'] = 2 if name == 'daily': @@ -147,9 +155,11 @@ def run(args): return {'service': name, 'old_pid': old_pid, 'new_pid': pids[name], 'old_exit_code': process.returncode} try: - published = None - if args.plugins is None: - published = prepare_published_plugins(root / 'plugins', args.plugin_manifest) + packages = ( + prepare_source_plugins(root / 'plugins', args.plugins, args.extra_plugins) + if args.plugins is not None + else prepare_published_plugins(root / 'plugins', args.plugin_manifest) + ) config_path = create_minimal_config(private, port=15301) create_test_directories(private) config = yaml.safe_load(config_path.read_text()) @@ -163,6 +173,8 @@ def run(args): groups = args.scenario_groups if groups is None: groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else [] + if 'plugins' in groups and not {'misc/EssentialCommands', 'misc/FAQManager'} <= set(args.extra_plugins): + groups.remove('plugins') profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups) cli_config = private / 'lbctl.yaml' cli_config.write_text( @@ -189,6 +201,8 @@ def run(args): mock_records=str(root / 'mock-requests.jsonl'), ) profile['plugins'] = [] + if 'smoke' in profile: + profile['smoke']['required_plugins'] = [entry['id'] for entry in packages['plugins']] for resource in profile['resources']: if resource['kind'] == 'model' and resource.get('type') == 'embedding': resource['body']['name'] = 'mock-embedding' @@ -267,15 +281,7 @@ def run(args): env['LANGBOT_QA_USER_TOKEN'] = response['data']['token'] suite.env['LANGBOT_QA_USER_TOKEN'] = env['LANGBOT_QA_USER_TOKEN'] suite.secrets.append(env['LANGBOT_QA_USER_TOKEN']) - if published is not None: - install_published_plugins(suite.call, root / 'plugins', published) - else: - for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'): - source = args.plugins / directory - package_source = private / source.name - shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')) - archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source) - suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) + install_plugins(suite.call, root / 'plugins', packages) restart_result = None restart_elapsed = 0 if args.restart_services: @@ -342,8 +348,11 @@ def main(): parser.add_argument('--output', type=Path, required=True) parser.add_argument('--lbctl', type=Path, required=True) plugins = parser.add_mutually_exclusive_group() - plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout') + plugins.add_argument('--plugins', type=Path, help='Build and install plugins from this source root') plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run') + parser.add_argument( + '--extra-plugins', nargs='*', default=[], help='Additional paths under --plugins, e.g. misc/HelloPlugin' + ) parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke') parser.add_argument( '--scenario-groups', @@ -353,6 +362,8 @@ def main(): ) parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once') args = parser.parse_args() + if args.extra_plugins and args.plugins is None: + parser.error('--extra-plugins requires --plugins') if args.suite not in ('scenarios', 'daily') and (args.scenario_groups is not None or args.restart_services): parser.error('Optional scenarios and restarts require --suite scenarios or --suite daily') if args.output.exists(): diff --git a/tools/regression_contracts.json b/tools/regression_contracts.json index a12488748..da894dcbc 100644 --- a/tools/regression_contracts.json +++ b/tools/regression_contracts.json @@ -240,6 +240,17 @@ "description": "Updated/deleted documents change retrieval results without stale leakage", "bindings": [] }, + {"id": "plugins.command_dispatch", + "area": "plugins", "kind": "blackbox", "priority": "P1", + "description": "An installed command returns the Core version through signed Bot ingress without invoking an agent or model", + "bindings": [{"profile": "scenarios", "case": "plugins.command_version", "checks": ["command_plugin_bindings", "command_version", "command_no_run", "command_no_model", "command_bot_cleanup"]}] + }, + {"id": "plugins.page_storage_crud", + "area": "plugins", "kind": "blackbox", "priority": "P1", + "description": "An installed Page API persists FAQ creates and updates and removes deleted entries from reads and search", + "bindings": [{"profile": "scenarios", "case": "plugins.faq_crud", "checks": ["faq_create", "faq_read", "faq_search", "faq_update", "faq_update_read", "faq_update_search", "faq_old_answer_absent", "faq_delete"]}], + "routes": ["POST /api/v1/plugins///page-api"] + }, {"id": "plugins.upgrade_recovery", "area": "plugins", "kind": "blackbox", "priority": "P1", "description": "Plugin upgrade failure preserves a usable installation", diff --git a/tools/regression_job.py b/tools/regression_job.py index 775afc7a5..df671539d 100644 --- a/tools/regression_job.py +++ b/tools/regression_job.py @@ -17,7 +17,7 @@ import urllib.request import yaml from regression_suite import Suite, number, resolve, validate -from regression_plugins import install_published_plugins, prepare_published_plugins +from regression_plugins import install_plugins, prepare_published_plugins from regression_traffic import owned_box_containers @@ -360,7 +360,7 @@ class Job: if published is not None: self.state['phase'] = 'plugins' self.save() - install_published_plugins(suite.call, self.output / 'plugins', published) + install_plugins(suite.call, self.output / 'plugins', published) self.state['phase'] = 'resources' self.save() suite.write('profile.json', profile) diff --git a/tools/regression_local_cases.py b/tools/regression_local_cases.py index 84f3501a2..dc25dcff2 100644 --- a/tools/regression_local_cases.py +++ b/tools/regression_local_cases.py @@ -13,6 +13,7 @@ GROUPS = { 'faults': 'regression_execution_cases', 'resources': 'regression_resource_cases', 'lifecycle': 'regression_lifecycle_cases', + 'plugins': 'regression_plugin_cases', } diff --git a/tools/regression_plugin_cases.py b/tools/regression_plugin_cases.py new file mode 100644 index 000000000..2c59fbb93 --- /dev/null +++ b/tools/regression_plugin_cases.py @@ -0,0 +1,248 @@ +"""Local plugin business checks through signed Bot ingress and the Page API.""" + +from contextlib import closing +import copy +import json +from pathlib import Path +from uuid import uuid4 + +from regression_coverage import checked +from regression_lifecycle_cases import ledger +from regression_local_cases import api, require +from regression_resource_cases import cleanup, journal +from regression_smoke import check_mock_requests, signed_headers +from regression_suite import resolve + +GROUP = 'plugins' +CASE_NAMES = {'plugins.command_version', 'plugins.faq_crud'} +COMMAND_PLUGINS = [ + {'author': 'langbot-team', 'name': name} + for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands') +] +FAQ_API = '/api/v1/plugins/RockChinQ/FAQManager/page-api' + + +def add_cases(profile): + pipeline = copy.deepcopy(next(item for item in profile['resources'] if item['id'] == 'chat_pipeline')) + pipeline['id'] = 'plugin_command_pipeline' + pipeline['body']['name'] = 'regression plugin commands' + pipeline['body']['extensions_preferences'] = { + 'enable_all_plugins': False, + 'plugins': copy.deepcopy(COMMAND_PLUGINS), + } + profile['resources'].append(pipeline) + for name, expected in ( + ('plugins.command_version', 'LANGBOT_PLUGIN_COMMAND_OK'), + ('plugins.faq_crud', 'LANGBOT_PLUGIN_FAQ_OK'), + ): + profile['cases'].append({'name': name, 'scenario': name, 'stage': 'plugins', 'expect': expected}) + + +def data(suite, method, path, body=None, *, headers=None): + status, response = api(suite, method, path, body, headers=headers) + require( + status == 200 and isinstance(response, dict) and response.get('code') == 0, + f'{method} {path} did not return a successful response (HTTP {status})', + ) + result = response.get('data') + if method == 'DELETE' and result is None: + return {} + require(isinstance(result, dict), f'{method} {path} returned malformed data') + return result + + +def command_version(suite, case, row): + pipeline = data(suite, 'GET', f'/api/v1/pipelines/{suite.variables["plugin_command_pipeline"]}')['pipeline'] + preferences = pipeline.get('extensions_preferences', {}) + checked( + row, + 'command_plugin_bindings', + lambda: require( + preferences.get('enable_all_plugins') is False and preferences.get('plugins') == COMMAND_PLUGINS, + 'Command pipeline must bind only the three base plugins and EssentialCommands', + ), + ) + version = data(suite, 'GET', '/api/v1/system/info').get('version') + require(isinstance(version, str) and bool(version), 'Core did not report its version') + records = Path(suite.path(suite.profile['smoke']['mock_records'])) + require(records.is_file(), 'Command model-call evidence requires the dedicated mock recording file') + body = resolve( + copy.deepcopy(next(item['body'] for item in suite.profile['resources'] if item['id'] == 'chat_bot')), + suite.variables, + ) + body['name'] = 'qa-plugin-command-' + uuid4().hex + body['adapter_config']['default_session_type'] = 'person' + body['event_bindings'] = [ + { + 'event_pattern': 'message.received', + 'target_type': 'pipeline', + 'target_uuid': suite.variables['plugin_command_pipeline'], + 'enabled': True, + 'filters': [], + } + ] + body['plugin_processors'] = [] + owned = {'kind': 'bot', 'name': body['name'], 'state': 'creating'} + row.setdefault('owned_resources', []).append(owned) + journal(suite, row) + + def delete_bot(): + if 'uuid' not in owned: + bots = data(suite, 'GET', '/api/v1/platform/bots')['bots'] + matches = [bot for bot in bots if bot.get('name') == owned['name']] + require(len(matches) == 1, 'Created command Bot identity is unresolved; inspect the resource journal') + owned['uuid'] = matches[0]['uuid'] + journal(suite, row) + path = f'/api/v1/platform/bots/{owned["uuid"]}' + status, response = api(suite, 'GET', path) + if status != 404: + require( + status == 200 + and response.get('code') == 0 + and response.get('data', {}).get('bot', {}).get('name') == owned['name'], + 'Cannot confirm ownership of the command Bot before cleanup', + ) + data(suite, 'DELETE', path) + status, _ = api(suite, 'GET', path) + require(status == 404, 'Command Bot remains visible after cleanup') + owned['state'] = 'deleted' + + try: + created = data(suite, 'POST', '/api/v1/platform/bots', body) + owned.update(uuid=created['uuid'], state='created') + journal(suite, row) + session = 'qa-plugin-version-' + uuid4().hex + payload = { + 'session_id': session, + 'session_type': 'person', + 'message': [{'type': 'Plain', 'text': '!version'}], + } + offset = records.stat().st_size + reply = data( + suite, + 'POST', + f'/bots/{owned["uuid"]}/sync', + payload, + headers=signed_headers(json.dumps(payload).encode(), body['adapter_config']['inbound_secret']), + ) + row['command_response'] = reply + parts = reply.get('message') + checked( + row, + 'command_version', + lambda: require( + reply.get('session_id') == session + and bool(reply.get('reply_to')) + and isinstance(parts, list) + and all(isinstance(part, dict) and part.get('type') == 'Plain' for part in parts) + and ''.join(part.get('text', '') for part in parts) == f'[bot] LangBot Version: {version}', + 'Version command did not return the exact Core version in the originating session', + ), + ) + with closing(ledger(suite)) as connection: + count = connection.execute( + 'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND bot_id=? AND id>?', + (suite.variables['workspace_uuid'], owned['uuid'], row['after_id']), + ).fetchone()[0] + checked(row, 'command_no_run', lambda: require(count == 0, 'Version command unexpectedly created an AgentRun')) + checked( + row, + 'command_no_model', + lambda: check_mock_requests(records, offset, [], timeout=suite.timeout(3)), + ) + return {'result': case['expect'], 'version': version, 'agent_runs': count, 'model_requests': 0} + finally: + cleanup(suite, row, 'command_bot_cleanup', delete_bot) + + +def faq_crud(suite, case, row): + def page(method, endpoint, body=None): + return data( + suite, 'POST', FAQ_API, {'page_id': 'manager', 'endpoint': endpoint, 'method': method, 'body': body} + ) + + def entries(): + values = page('GET', '/entries').get('entries') + require( + isinstance(values, list) and all(isinstance(item, dict) and item.get('id') for item in values), + 'FAQ entries response is malformed', + ) + return values + + previous = {item['id'] for item in entries()} + marker = 'qa-faq-' + uuid4().hex + question, before, after = marker + '-question', marker + '-before', marker + '-after' + owned = {'kind': 'faq-entry', 'question': question, 'state': 'creating'} + row.setdefault('owned_resources', []).append(owned) + journal(suite, row) + + def expect_entry(values, answer): + expected = {'id': owned['id'], 'question': question, 'answer': answer} + require(values == [expected], 'FAQ content or entry identity differs from the submitted value') + + def delete_entry(): + current = entries() + if 'id' not in owned: + matches = [item for item in current if item.get('question') == question and item['id'] not in previous] + require(len(matches) <= 1, 'Created FAQ identity is ambiguous; inspect the resource journal') + if not matches: + owned['state'] = 'absent' + return + owned['id'] = matches[0]['id'] + journal(suite, row) + require(owned['id'] not in previous, 'Refusing to delete a pre-existing FAQ entry') + matches = [item for item in current if item['id'] == owned['id']] + if matches: + require( + len(matches) == 1 and matches[0].get('question') == question, 'FAQ ownership changed before cleanup' + ) + deleted = page('DELETE', '/entries', {'id': owned['id']}) + require(deleted.get('deleted') == owned['id'], 'FAQ deletion returned a different entry ID') + require(not any(item['id'] == owned['id'] for item in entries()), 'Deleted FAQ remains visible') + require(page('POST', '/search', {'query': marker}).get('results') == [], 'Deleted FAQ remains searchable') + owned['state'] = 'deleted' + + try: + created = page('POST', '/entries', {'question': question, 'answer': before}).get('entry') + require(isinstance(created, dict) and bool(created.get('id')), 'FAQ creation did not return an entry ID') + owned.update(id=created['id'], state='created') + journal(suite, row) + require(owned['id'] not in previous, 'FAQ creation reused a pre-existing entry ID') + checked(row, 'faq_create', lambda: expect_entry([created], before)) + checked( + row, 'faq_read', lambda: expect_entry([item for item in entries() if item['id'] == owned['id']], before) + ) + checked( + row, 'faq_search', lambda: expect_entry(page('POST', '/search', {'query': before}).get('results'), before) + ) + updated = page('PUT', '/entries', {'id': owned['id'], 'question': question, 'answer': after}).get('entry') + checked(row, 'faq_update', lambda: expect_entry([updated], after)) + checked( + row, + 'faq_update_read', + lambda: expect_entry([item for item in entries() if item['id'] == owned['id']], after), + ) + checked( + row, + 'faq_update_search', + lambda: expect_entry(page('POST', '/search', {'query': after}).get('results'), after), + ) + checked( + row, + 'faq_old_answer_absent', + lambda: require( + page('POST', '/search', {'query': before}).get('results') == [], 'FAQ search returned stale content' + ), + ) + checked(row, 'faq_delete', delete_entry) + return {'result': case['expect'], 'entry_id': owned['id'], 'deleted': True} + finally: + if owned['state'] != 'deleted': + cleanup(suite, row, 'faq_cleanup', delete_entry) + else: + journal(suite, row) + + +def run_case(suite, case, row): + require(case['scenario'] in CASE_NAMES, 'Unknown plugin scenario') + return (command_version if case['scenario'] == 'plugins.command_version' else faq_crud)(suite, case, row) diff --git a/tools/regression_plugins.py b/tools/regression_plugins.py index af75d9341..327cfd66c 100644 --- a/tools/regression_plugins.py +++ b/tools/regression_plugins.py @@ -1,4 +1,4 @@ -"""Resolve official published regression plugins once, then install the recorded bytes.""" +"""Prepare source or published regression plugins and verify installed artifacts.""" from datetime import datetime, timezone import hashlib @@ -6,6 +6,10 @@ import io import json from pathlib import Path import re +import shutil +import subprocess +import sys +import tempfile import urllib.error import urllib.request import zipfile @@ -15,6 +19,7 @@ import yaml MARKETPLACE = 'https://space.langbot.app' PLUGIN_IDS = ('langbot-team/LocalAgent', 'langbot-team/LangRAG', 'langbot-team/GeneralParsers') +PLUGIN_PATHS = ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers') METADATA_LIMIT = 1024 * 1024 PACKAGE_LIMIT = 64 * 1024 * 1024 @@ -83,7 +88,7 @@ def check_archive(content, plugin_id, version): ): raise ValueError('package manifest identity/version differs from the selected release') except (zipfile.BadZipFile, yaml.YAMLError, KeyError, ValueError, RuntimeError) as exc: - raise ValueError(f'Invalid published package for {plugin_id}@{version}: {exc}') from exc + raise ValueError(f'Invalid plugin package for {plugin_id}@{version}: {exc}') from exc def prepare_published_plugins(directory, replay=None): @@ -141,8 +146,61 @@ def prepare_published_plugins(directory, replay=None): return manifest -def install_published_plugins(call, directory, manifest): - """Install the downloaded packages and require exact, initialized CLI readback.""" +def prepare_source_plugins(directory, source, extra_plugins=()): + """Build selected plugin directories with the SDK CLI, without publishing.""" + directory, source = Path(directory).resolve(), Path(source).resolve() + paths = [*PLUGIN_PATHS, *extra_plugins] + if len(paths) != len(set(paths)): + raise ValueError('Duplicate source plugin selection') + for path in paths: + if ( + not re.fullmatch(r'(Runner|KnowledgeEngine|misc)/[A-Za-z0-9_.-]+', path) + or not (source / path).resolve().is_relative_to(source) + or not (source / path / 'manifest.yaml').is_file() + ): + raise ValueError(f'Invalid source plugin path: {path}') + directory.mkdir(parents=True, exist_ok=False) + manifest = {'version': 1, 'selection': 'source', 'plugins': []} + write_manifest(directory, manifest) + for path in paths: + metadata = yaml.safe_load((source / path / 'manifest.yaml').read_text())['metadata'] + plugin_id = f'{metadata["author"]}/{metadata["name"]}' + if any(entry['id'] == plugin_id for entry in manifest['plugins']): + raise ValueError(f'Duplicate source plugin identity: {plugin_id}') + if not all(re.fullmatch(r'[A-Za-z0-9_.-]+', metadata[key]) for key in ('author', 'name')): + raise ValueError(f'Invalid source plugin identity: {plugin_id}') + version = release_version(metadata['version']) + entry = { + 'id': plugin_id, + 'version': version, + 'source': path, + 'archive': f'{metadata["author"]}-{metadata["name"]}-{version}.lbpkg', + } + manifest['plugins'].append(entry) + write_manifest(directory, manifest) + with tempfile.TemporaryDirectory() as temporary: + package_source = Path(temporary) / 'plugin' + shutil.copytree( + source / path, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__') + ) + with (directory / f'{metadata["name"]}-build.log').open('w') as log: + subprocess.run( + [sys.executable, '-m', 'langbot_plugin.cli.__init__', 'build', '--output', str(directory)], + cwd=package_source, + stdout=log, + stderr=subprocess.STDOUT, + check=True, + timeout=180, + ) + content = (directory / entry['archive']).read_bytes() + check_archive(content, plugin_id, version) + entry['sha256'] = hashlib.sha256(content).hexdigest() + write_manifest(directory, manifest) + return manifest + + +def install_plugins(call, directory, manifest): + """Install recorded packages and require exact, initialized CLI readback.""" directory = Path(directory) for entry in manifest['plugins']: archive = directory / entry['archive'] @@ -151,7 +209,7 @@ def install_published_plugins(call, directory, manifest): try: call(['plugin', 'install', 'local', '--file', str(archive), '--wait']) except RuntimeError as exc: - raise RuntimeError(f'Published plugin install failed: {entry["id"]}@{entry["version"]}: {exc}') from exc + raise RuntimeError(f'Plugin install failed: {entry["id"]}@{entry["version"]}: {exc}') from exc response = call(['plugin', 'list']) plugins = response.get('plugins') if isinstance(response, dict) else None if not isinstance(plugins, list) or any(not isinstance(item, dict) for item in plugins): @@ -165,6 +223,4 @@ def install_published_plugins(call, directory, manifest): or matches[0].get('enabled') is not True or matches[0].get('status') != 'initialized' ): - raise RuntimeError( - f'Published plugin is not ready at the selected version: {entry["id"]}: {entry["installed"]}' - ) + raise RuntimeError(f'Plugin is not ready at the selected version: {entry["id"]}: {entry["installed"]}') diff --git a/tools/test_regression_plugin_cases.py b/tools/test_regression_plugin_cases.py new file mode 100644 index 000000000..52258ce02 --- /dev/null +++ b/tools/test_regression_plugin_cases.py @@ -0,0 +1,255 @@ +"""Challenge plugin business oracles and cleanup without starting target services.""" + +import copy +import hashlib +import hmac +import json +from pathlib import Path +import sqlite3 +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import regression_plugin_cases as plugins +from regression_scenarios import expanded_profile + + +def success(value=None): + return 200, {'code': 0, 'data': value} + + +class PluginCasesTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.profile = expanded_profile() + plugins.add_cases(self.profile) + self.records = self.root / 'mock.jsonl' + self.records.write_text( + json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'earlier'}, 'status': 200}) + '\n' + ) + self.database = self.root / 'ledger.db' + with sqlite3.connect(self.database) as connection: + connection.execute('CREATE TABLE agent_run (id INTEGER, workspace_id TEXT, bot_id TEXT)') + self.profile['smoke'].update(ledger_db=str(self.database), mock_records=str(self.records)) + self.suite = SimpleNamespace( + profile=self.profile, + variables={ + 'workspace_uuid': 'workspace', + 'chat_pipeline': 'original-pipeline', + 'plugin_command_pipeline': 'isolated-pipeline', + 'group_secret': 'private-signing-secret', + }, + secrets=['private-signing-secret'], + env={}, + path=lambda value: value, + timeout=lambda value: value, + write=lambda name, value: (self.root / name).write_text(json.dumps(value)), + ) + + def case(self, scenario): + return next(case for case in self.profile['cases'] if case.get('scenario') == scenario) + + def test_profile_adds_only_one_isolated_pipeline_and_two_scenarios(self): + original = expanded_profile() + profile = copy.deepcopy(original) + plugins.add_cases(profile) + self.assertEqual(profile['resources'][:-1], original['resources']) + self.assertEqual(profile['cases'][:-2], original['cases']) + pipeline = profile['resources'][-1] + self.assertEqual(pipeline['id'], 'plugin_command_pipeline') + self.assertEqual( + pipeline['body']['extensions_preferences'], + { + 'enable_all_plugins': False, + 'plugins': [ + {'author': 'langbot-team', 'name': name} + for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands') + ], + }, + ) + self.assertEqual({case['scenario'] for case in profile['cases'][-2:]}, plugins.CASE_NAMES) + self.assertTrue(all('http_group' not in case for case in profile['cases'][-2:])) + + def command_api(self, failure=None): + state = {} + + def request(suite, method, path, body=None, headers=None): + if path == '/api/v1/pipelines/isolated-pipeline': + return success( + { + 'pipeline': { + 'extensions_preferences': { + 'enable_all_plugins': failure == 'bindings', + 'plugins': copy.deepcopy(plugins.COMMAND_PLUGINS), + } + } + } + ) + if path == '/api/v1/system/info': + return success({'version': '9.5.1'}) + if path == '/api/v1/platform/bots' and method == 'POST': + self.assertEqual(body['event_bindings'][0]['target_uuid'], 'isolated-pipeline') + self.assertEqual(body['plugin_processors'], []) + self.assertEqual(body['adapter_config']['inbound_secret'], 'private-signing-secret') + state.update(uuid='owned-bot', name=body['name']) + return success({'uuid': 'owned-bot'}) + if path == '/bots/owned-bot/sync': + self.assertEqual(body['session_type'], 'person') + self.assertEqual(body['message'], [{'type': 'Plain', 'text': '!version'}]) + signed = headers['X-LB-Timestamp'].encode() + b'.' + json.dumps(body).encode() + expected = 'sha256=' + hmac.new(b'private-signing-secret', signed, hashlib.sha256).hexdigest() + self.assertEqual(headers['X-LB-Signature'], expected) + if failure == 'run': + with sqlite3.connect(self.database) as connection: + connection.execute("INSERT INTO agent_run VALUES (1, 'workspace', 'owned-bot')") + if failure == 'model': + with self.records.open('a') as stream: + stream.write( + json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'wrong'}, 'status': 200}) + + '\n' + ) + return success( + { + 'session_id': 'wrong-session' if failure == 'session' else body['session_id'], + 'reply_to': 'in_message', + 'message': [ + { + 'type': 'Plain', + 'text': '[bot] LangBot Version: ' + ('9.4.0' if failure == 'version' else '9.5.1'), + } + ], + } + ) + if path == '/api/v1/platform/bots/owned-bot': + if method == 'DELETE': + state['deleted'] = True + return success() # The real Bot DELETE returns data=null. + return (404, {'code': -1}) if state.get('deleted') else success({'bot': dict(state)}) + self.fail(f'Unexpected command API call: {method} {path}') + + return request, state + + def test_command_checks_signed_reply_without_requiring_a_run(self): + request, state = self.command_api() + row = {'after_id': 0} + with patch.object(plugins, 'api', side_effect=request): + result = plugins.run_case(self.suite, self.case('plugins.command_version'), row) + self.assertEqual( + result, {'result': 'LANGBOT_PLUGIN_COMMAND_OK', 'version': '9.5.1', 'agent_runs': 0, 'model_requests': 0} + ) + self.assertTrue(state['deleted']) + self.assertNotIn('scenario_runs', row) + self.assertTrue(all(value == 'passed' for value in row['assertions'].values())) + + def test_command_rejects_bad_reply_model_calls_runs_and_plugin_leakage(self): + for failure in ('version', 'session', 'model', 'run', 'bindings'): + with self.subTest(failure=failure): + with sqlite3.connect(self.database) as connection: + connection.execute('DELETE FROM agent_run') + request, state = self.command_api(failure) + row = {'after_id': 0} + with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError): + plugins.run_case(self.suite, self.case('plugins.command_version'), row) + if failure != 'bindings': + self.assertTrue(state['deleted']) + self.assertIn('failed', row['assertions'].values()) + + def faq_api(self, failure=None, cleanup_failure=False): + unrelated = {'id': 'unrelated', 'question': 'Existing question', 'answer': 'Existing answer'} + state = {'entries': [unrelated], 'created': 0, 'deletes': []} + + def request(suite, method, path, body=None, headers=None): + self.assertEqual((method, path, body['page_id']), ('POST', plugins.FAQ_API, 'manager')) + verb, endpoint, payload = body['method'], body['endpoint'], body['body'] + if (verb, endpoint) == ('GET', '/entries'): + return success({'entries': copy.deepcopy(state['entries'])}) + if (verb, endpoint) == ('POST', '/entries'): + state['created'] += 1 + entry = {'id': 'owned-faq', **payload} + state['entries'].append(entry) + state['original'] = copy.deepcopy(entry) + if failure == 'lost_create': + raise ConnectionResetError('creation response lost') + return success({'entry': copy.deepcopy(entry)}) + if (verb, endpoint) == ('PUT', '/entries'): + self.assertEqual(payload['id'], 'owned-faq') + if failure == 'update_error': + raise ValueError('primary update failure') + if failure != 'stale_update': + state['entries'][1].update(payload) + return success({'entry': copy.deepcopy(payload)}) + if (verb, endpoint) == ('POST', '/search'): + rows = list(state['entries']) + if failure == 'stale_search' and state['deletes']: + rows.append(state['original']) + return success( + { + 'results': copy.deepcopy( + [ + item + for item in rows + if payload['query'] in item['question'] or payload['query'] in item['answer'] + ] + ) + } + ) + if (verb, endpoint) == ('DELETE', '/entries'): + self.assertEqual(payload, {'id': 'owned-faq'}) + if cleanup_failure: + raise RuntimeError('cleanup private-signing-secret failed') + state['deletes'].append(payload['id']) + state['entries'] = [item for item in state['entries'] if item['id'] != payload['id']] + return success({'deleted': payload['id']}) + self.fail(f'Unexpected Page API call: {body}') + + return request, state + + def test_faq_crud_checks_content_and_removes_only_its_entry(self): + request, state = self.faq_api() + row = {} + with patch.object(plugins, 'api', side_effect=request): + result = plugins.run_case(self.suite, self.case('plugins.faq_crud'), row) + self.assertEqual(result, {'result': 'LANGBOT_PLUGIN_FAQ_OK', 'entry_id': 'owned-faq', 'deleted': True}) + self.assertEqual(state['deletes'], ['owned-faq']) + self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated']) + self.assertTrue(all(value == 'passed' for value in row['assertions'].values())) + + def test_faq_rejects_stale_readback_and_search(self): + for failure, assertion in (('stale_update', 'faq_update_read'), ('stale_search', 'faq_delete')): + with self.subTest(failure=failure): + request, state = self.faq_api(failure) + row = {} + with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError): + plugins.run_case(self.suite, self.case('plugins.faq_crud'), row) + self.assertEqual(row['assertions'][assertion], 'failed') + self.assertEqual(state['deletes'], ['owned-faq']) + self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated']) + + def test_faq_lost_create_response_recovers_owned_id_without_retrying_creation(self): + request, state = self.faq_api('lost_create') + row = {} + with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ConnectionResetError): + plugins.run_case(self.suite, self.case('plugins.faq_crud'), row) + self.assertEqual(state['created'], 1) + self.assertEqual(state['deletes'], ['owned-faq']) + self.assertEqual(row['owned_resources'][0]['state'], 'deleted') + + def test_faq_preserves_original_error_and_redacts_cleanup_error(self): + request, _ = self.faq_api('update_error', cleanup_failure=True) + row = {} + with ( + patch.object(plugins, 'api', side_effect=request), + self.assertRaisesRegex(ValueError, 'primary update failure'), + ): + plugins.run_case(self.suite, self.case('plugins.faq_crud'), row) + self.assertEqual(row['cleanup_errors'], ['RuntimeError: cleanup [REDACTED] failed']) + evidence = (self.root / row['resource_journal']).read_text() + self.assertNotIn('private-signing-secret', evidence) + self.assertIn('owned-faq', evidence) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_plugins.py b/tools/test_regression_plugins.py index c8c3bcf3d..6ae525ebe 100644 --- a/tools/test_regression_plugins.py +++ b/tools/test_regression_plugins.py @@ -1,4 +1,4 @@ -"""Exercise published resolution, replay integrity and installation readback.""" +"""Exercise plugin selection, artifact integrity and installation readback.""" from collections import Counter import copy @@ -12,6 +12,8 @@ import unittest from unittest.mock import patch import zipfile +import yaml + import regression_plugins as plugins @@ -27,6 +29,66 @@ def archive(plugin_id, version='1.2.3'): return output.getvalue() +class SourcePluginsTests(unittest.TestCase): + def test_selected_sources_build_in_isolation_and_record_actual_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, output = root / 'sources', root / 'packages' + paths = (*plugins.PLUGIN_PATHS, 'misc/ExtraPlugin') + for path in paths: + folder = source / path + folder.mkdir(parents=True) + with zipfile.ZipFile(io.BytesIO(archive('test/' + folder.name))) as package: + package.extractall(folder) + + def build(args, *, cwd, **kwargs): + self.assertEqual(args[1:5], ['-m', 'langbot_plugin.cli.__init__', 'build', '--output']) + self.assertFalse(cwd.is_relative_to(source)) + metadata = yaml.safe_load((cwd / 'manifest.yaml').read_text())['metadata'] + (cwd / 'generated.html').write_text('Generated only in the isolated build copy') + filename = f'{metadata["author"]}-{metadata["name"]}-{metadata["version"]}.lbpkg' + (Path(args[-1]) / filename).write_bytes(archive('test/' + metadata['name'])) + + with patch.object(plugins.subprocess, 'run', side_effect=build): + manifest = plugins.prepare_source_plugins(output, source, ['misc/ExtraPlugin']) + self.assertEqual(manifest['selection'], 'source') + self.assertEqual([entry['source'] for entry in manifest['plugins']], list(paths)) + for entry in manifest['plugins']: + self.assertEqual( + entry['sha256'], plugins.hashlib.sha256((output / entry['archive']).read_bytes()).hexdigest() + ) + self.assertFalse((source / entry['source'] / 'generated.html').exists()) + + def test_invalid_or_duplicate_selection_is_rejected_before_build(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for path in plugins.PLUGIN_PATHS: + (root / path).mkdir(parents=True) + (root / path / 'manifest.yaml').write_text('kind: Plugin') + (root / 'misc/escape').symlink_to(root.parent, target_is_directory=True) + for selected in (['../outside'], ['/tmp/plugin'], ['misc/escape'], [plugins.PLUGIN_PATHS[0]]): + with self.subTest(selected=selected), patch.object(plugins.subprocess, 'run') as build: + with self.assertRaises(ValueError): + plugins.prepare_source_plugins(root / 'output', root, selected) + build.assert_not_called() + + def test_wrong_built_identity_fails_without_installation(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for path in plugins.PLUGIN_PATHS: + folder = root / path + folder.mkdir(parents=True) + with zipfile.ZipFile(io.BytesIO(archive('test/' + folder.name))) as package: + package.extractall(folder) + + def build(args, **kwargs): + (Path(args[-1]) / 'test-LocalAgent-1.2.3.lbpkg').write_bytes(archive('test/WrongPlugin')) + + with patch.object(plugins.subprocess, 'run', side_effect=build): + with self.assertRaisesRegex(ValueError, 'identity/version differs'): + plugins.prepare_source_plugins(root / 'output', root) + + class PublishedPluginsTests(unittest.TestCase): @classmethod def setUpClass(cls): @@ -95,7 +157,7 @@ class PublishedPluginsTests(unittest.TestCase): self.assertEqual(args, ['plugin', 'list']) return {'plugins': installed} - plugins.install_published_plugins(call, directory, manifest) + plugins.install_plugins(call, directory, manifest) recorded = json.loads((directory / 'manifest.json').read_text()) self.assertTrue(all(entry['installed'][0]['status'] == 'initialized' for entry in recorded['plugins'])) self.requests.clear() @@ -178,13 +240,13 @@ class PublishedPluginsTests(unittest.TestCase): } with self.assertRaisesRegex(RuntimeError, 'not ready at the selected version'): - plugins.install_published_plugins(call, directory, manifest) + plugins.install_plugins(call, directory, manifest) self.assertEqual(len(calls), 2) recorded = json.loads((directory / 'manifest.json').read_text()) self.assertEqual(recorded['plugins'][0]['installed'][0]['version'], '0.0.1') (directory / manifest['plugins'][0]['archive']).write_bytes(b'changed after download') with self.assertRaisesRegex(ValueError, 'changed before installation'): - plugins.install_published_plugins(call, directory, manifest) + plugins.install_plugins(call, directory, manifest) self.assertEqual(len(calls), 2)