ci(regression): build monorepo plugins and exercise local command and page APIs

2026-11-02 05:45:20 +08:00 · 2026-10-28 20:46:53 +08:00
parent 1f842381df
commit 21be11766b
11 changed files with 706 additions and 44 deletions
@@ -73,7 +73,7 @@ jobs:
          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
              if manifest.exists():
                  for plugin in json.loads(manifest.read_text())['plugins']:
-                      out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
+                      out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
              if path.exists():
                  data = json.loads(path.read_text())
                  out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
@@ -104,6 +104,7 @@ jobs:
            .qa/core-smoke/gate/suite/cycles.json
            .qa/core-smoke/gate/suite/execution.json
            .qa/core-smoke/plugins/manifest.json
+            .qa/core-smoke/plugins/*-build.log
            .qa/core-smoke/*.log
          if-no-files-found: warn
          retention-days: 7
@@ -9,7 +9,11 @@ on:
      scenario_groups:
        description: 'Optional local groups, space separated; empty runs only base cases'
        type: string
-        default: 'faults resources lifecycle'
+        default: 'faults resources lifecycle plugins'
+      extra_plugins:
+        description: 'Additional source plugin paths; clear with the plugins group to run only base plugins'
+        type: string
+        default: 'misc/EssentialCommands misc/FAQManager'
      restart_services:
        description: 'Check owned Core and Runtime restarts once before resource sampling'
        type: boolean
@@ -52,15 +56,17 @@ jobs:
          docker pull python:3.12-alpine
      - name: Run repeated scenarios and recovery checks
        env:
-          SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle' || inputs.scenario_groups }}
+          SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle plugins' || inputs.scenario_groups }}
+          EXTRA_PLUGINS: ${{ github.event_name == 'schedule' && 'misc/EssentialCommands misc/FAQManager' || inputs.extra_plugins }}
          RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }}
        run: |
          read -r -a groups <<< "$SCENARIO_GROUPS"
+          read -r -a extras <<< "$EXTRA_PLUGINS"
          args=()
          if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi
          uv run --no-sync python tools/regression_ci.py \
            --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \
-            --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily
+            --lbctl .endurance-bin/lbctl_linux_amd64 --plugins plugins --extra-plugins "${extras[@]}" --output .qa/daily
      - name: Publish observations
        if: always()
        run: |
@@ -83,15 +89,16 @@ jobs:
          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
              out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n")
              out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n')
-              out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
+              out.write('Real Core/Runtime/source plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
              out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n")
              restart = gate.get('restart_checks')
              if restart:
                  out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n")
                  for error in restart.get('errors', []):
                      out.write(f"Restart check error: {error}\n")
+              out.write(f"Plugin selection: `{manifest.get('selection', 'not reported')}`.\n\n")
              for plugin in manifest.get('plugins', []):
-                  out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
+                  out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
              out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n")
              out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n")
              data = suite.get('test_data', {})
@@ -140,6 +147,7 @@ jobs:
            .qa/daily/gate/suite/coverage.json
            .qa/daily/gate/suite/coverage.md
            .qa/daily/plugins/manifest.json
+            .qa/daily/plugins/*-build.log
            .qa/daily/restart-checks/profile.json
            .qa/daily/restart-checks/cases.json
            .qa/daily/restart-checks/summary.json
@@ -367,12 +367,13 @@ phases. The dedicated Docker daemon requirement above still applies.

 `Core Smoke` runs regression-tool tests and a real Core smoke in parallel on
 GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate,
-uses the SDK from `uv.lock`, the latest published official plugins, `lbctl`
+uses the SDK from `uv.lock`, plugins built from the same checkout, `lbctl`
 v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials
 or access to a developer's running environment.
-Plugin versions are resolved once per run, installed from the downloaded release
-archives and recorded with hashes and installation readback in `plugins/manifest.json`.
-An unavailable or incompatible published package fails preparation; see
+The SDK CLI builds LocalAgent, LangRAG and GeneralParsers into `.lbpkg` archives.
+`plugins/manifest.json` records selection mode, plugin versions, archive hashes
+and exact installation/readiness readback. Build logs are preserved alongside it.
+The published route remains available when `--plugins` is omitted; see
 [published plugin preparation and replay](published-plugins.md).

 Dependency installation, service startup, sandbox image download and plugin
@@ -413,7 +414,7 @@ supports **Run workflow** with a selected branch. The schedule becomes active
 only after the workflow is merged into the default branch; it is not a required
 merge check.

-The workflow reuses the disposable smoke stack and latest published plugins.
+The workflow reuses the disposable smoke stack and builds plugins from this checkout.
 `--suite daily` consumes 46 base cases plus the selected local groups, requires its first pass, then
 repeats ordered cases within one **1800-second** window. The clock starts before
 creating scenario resources and includes a 120-second reserve for drain/API
@@ -421,7 +422,7 @@ cleanup and a 60-second cooldown. Service and plugin preparation precede this
 clock. A separate 1860-second process guard allows startup and final reporting;
 it does not grant extra scenario time. The whole job retains its 45-minute limit.

-The scheduled job enables all local groups and the optional one-time restart
+The scheduled job enables all local groups (including `plugins`) and the optional one-time restart
 checks. Manual dispatch exposes both switches. `--scenario-groups` accepts a
 space-separated selection; supplying the flag with no values runs only base
 cases. `scenarios` and `daily` enable all three groups when the flag is omitted.
@@ -430,6 +431,7 @@ cases. `scenarios` and `daily` enable all three groups when the flag is omitted.
 |---|---|
 | `faults` | Cancel/timeout cleanup, model retry side effects, partial stream failure, exhausted fallback, Box path rejection |
 | `resources` | API-key revocation, direct Agent execution, document deletion with a surviving retrieval control |
+| `plugins` | EssentialCommands version command without model/AgentRun execution; FAQManager Page API CRUD and search freshness |
 | `lifecycle` | Signed HTTP callback 5xx retry identity and subsequent delivery |

 For example, select resource and callback cases with
@@ -477,15 +479,22 @@ To run the same entrypoint on a disposable Linux host with an idle Docker daemon

 ```sh
 uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \
-  --lbctl /path/to/lbctl --output .qa/daily-001
+  --lbctl /path/to/lbctl --plugins plugins \
+  --extra-plugins misc/EssentialCommands misc/FAQManager --output .qa/daily-001
 ```

 `--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared
 expanded profile once under a 300-second guard. `--suite endurance` retains the
 separate ten-minute load profile under a 900-second guard; its CI entry point also
 reports throughput/P95 without applying their thresholds. All four modes use
-published plugins by default. Explicit `--plugins` remains available for source
-development; neither workflow selects it. The suite attempts owned API cleanup
+published plugins when `--plugins` is omitted. Both workflows explicitly select
+`--plugins plugins` so source changes are tested in the same PR. Daily additionally
+selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the
+space-separated `extra_plugins` list or leave it empty. `--extra-plugins` requires
+source mode and accepts categorized plugin paths, without removing the three base
+plugins. Missing or incompatible packages fail preparation, with no silent fallback.
+Existing conversation Pipelines allow only the base plugins; plugin-specific cases
+bind their own extensions to avoid changing unrelated prompts/tools. The suite attempts owned API cleanup
 even after failures. The CI owner then stops its service processes and removes
 its Box containers; a process guard timeout can interrupt API cleanup.

@@ -6,7 +6,6 @@ import json
 import os
 from pathlib import Path
 import secrets
-import shutil
 import signal
 import subprocess
 import sys
@@ -16,7 +15,7 @@ import yaml

 from regression_gate import supervise
 from regression_coverage import write_report
-from regression_plugins import install_published_plugins, prepare_published_plugins
+from regression_plugins import PLUGIN_IDS, install_plugins, prepare_published_plugins, prepare_source_plugins
 from regression_local_cases import GROUPS, api, require
 from regression_suite import Suite
 from regression_traffic import owned_box_containers, read_endpoint_snapshot
@@ -35,6 +34,15 @@ def suite_profile(name, run_id, groups=()):
        profile = expanded_profile(groups)
    else:
        profile = json.loads(profile_path.read_text())
+    for resource in profile['resources']:
+        if resource['kind'] == 'pipeline':
+            resource['body'].setdefault(
+                'extensions_preferences',
+                {
+                    'enable_all_plugins': False,
+                    'plugins': [dict(zip(('author', 'name'), plugin.split('/'))) for plugin in PLUGIN_IDS],
+                },
+            )
    if name == 'smoke':
        profile['smoke']['group_turns'] = 2
    if name == 'daily':
@@ -147,9 +155,11 @@ def run(args):
        return {'service': name, 'old_pid': old_pid, 'new_pid': pids[name], 'old_exit_code': process.returncode}

    try:
-        published = None
-        if args.plugins is None:
-            published = prepare_published_plugins(root / 'plugins', args.plugin_manifest)
+        packages = (
+            prepare_source_plugins(root / 'plugins', args.plugins, args.extra_plugins)
+            if args.plugins is not None
+            else prepare_published_plugins(root / 'plugins', args.plugin_manifest)
+        )
        config_path = create_minimal_config(private, port=15301)
        create_test_directories(private)
        config = yaml.safe_load(config_path.read_text())
@@ -163,6 +173,8 @@ def run(args):
        groups = args.scenario_groups
        if groups is None:
            groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else []
+            if 'plugins' in groups and not {'misc/EssentialCommands', 'misc/FAQManager'} <= set(args.extra_plugins):
+                groups.remove('plugins')
        profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups)
        cli_config = private / 'lbctl.yaml'
        cli_config.write_text(
@@ -189,6 +201,8 @@ def run(args):
            mock_records=str(root / 'mock-requests.jsonl'),
        )
        profile['plugins'] = []
+        if 'smoke' in profile:
+            profile['smoke']['required_plugins'] = [entry['id'] for entry in packages['plugins']]
        for resource in profile['resources']:
            if resource['kind'] == 'model' and resource.get('type') == 'embedding':
                resource['body']['name'] = 'mock-embedding'
@@ -267,15 +281,7 @@ def run(args):
            env['LANGBOT_QA_USER_TOKEN'] = response['data']['token']
            suite.env['LANGBOT_QA_USER_TOKEN'] = env['LANGBOT_QA_USER_TOKEN']
            suite.secrets.append(env['LANGBOT_QA_USER_TOKEN'])
-        if published is not None:
-            install_published_plugins(suite.call, root / 'plugins', published)
-        else:
-            for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'):
-                source = args.plugins / directory
-                package_source = private / source.name
-                shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__'))
-                archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source)
-                suite.call(['plugin', 'install', 'local', '--file', archive, '--wait'])
+        install_plugins(suite.call, root / 'plugins', packages)
        restart_result = None
        restart_elapsed = 0
        if args.restart_services:
@@ -342,8 +348,11 @@ def main():
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--lbctl', type=Path, required=True)
    plugins = parser.add_mutually_exclusive_group()
-    plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout')
+    plugins.add_argument('--plugins', type=Path, help='Build and install plugins from this source root')
    plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run')
+    parser.add_argument(
+        '--extra-plugins', nargs='*', default=[], help='Additional paths under --plugins, e.g. misc/HelloPlugin'
+    )
    parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke')
    parser.add_argument(
        '--scenario-groups',
@@ -353,6 +362,8 @@ def main():
    )
    parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once')
    args = parser.parse_args()
+    if args.extra_plugins and args.plugins is None:
+        parser.error('--extra-plugins requires --plugins')
    if args.suite not in ('scenarios', 'daily') and (args.scenario_groups is not None or args.restart_services):
        parser.error('Optional scenarios and restarts require --suite scenarios or --suite daily')
    if args.output.exists():
@@ -240,6 +240,17 @@
     "description": "Updated/deleted documents change retrieval results without stale leakage",
     "bindings": []
    },
+    {"id": "plugins.command_dispatch",
+     "area": "plugins", "kind": "blackbox", "priority": "P1",
+     "description": "An installed command returns the Core version through signed Bot ingress without invoking an agent or model",
+     "bindings": [{"profile": "scenarios", "case": "plugins.command_version", "checks": ["command_plugin_bindings", "command_version", "command_no_run", "command_no_model", "command_bot_cleanup"]}]
+    },
+    {"id": "plugins.page_storage_crud",
+     "area": "plugins", "kind": "blackbox", "priority": "P1",
+     "description": "An installed Page API persists FAQ creates and updates and removes deleted entries from reads and search",
+     "bindings": [{"profile": "scenarios", "case": "plugins.faq_crud", "checks": ["faq_create", "faq_read", "faq_search", "faq_update", "faq_update_read", "faq_update_search", "faq_old_answer_absent", "faq_delete"]}],
+     "routes": ["POST /api/v1/plugins/<author>/<plugin_name>/page-api"]
+    },
    {"id": "plugins.upgrade_recovery",
     "area": "plugins", "kind": "blackbox", "priority": "P1",
     "description": "Plugin upgrade failure preserves a usable installation",
@@ -17,7 +17,7 @@ import urllib.request
 import yaml

 from regression_suite import Suite, number, resolve, validate
-from regression_plugins import install_published_plugins, prepare_published_plugins
+from regression_plugins import install_plugins, prepare_published_plugins
 from regression_traffic import owned_box_containers


@@ -360,7 +360,7 @@ class Job:
        if published is not None:
            self.state['phase'] = 'plugins'
            self.save()
-            install_published_plugins(suite.call, self.output / 'plugins', published)
+            install_plugins(suite.call, self.output / 'plugins', published)
            self.state['phase'] = 'resources'
            self.save()
        suite.write('profile.json', profile)
@@ -13,6 +13,7 @@ GROUPS = {
    'faults': 'regression_execution_cases',
    'resources': 'regression_resource_cases',
    'lifecycle': 'regression_lifecycle_cases',
+    'plugins': 'regression_plugin_cases',
 }


@@ -0,0 +1,248 @@
+"""Local plugin business checks through signed Bot ingress and the Page API."""
+
+from contextlib import closing
+import copy
+import json
+from pathlib import Path
+from uuid import uuid4
+
+from regression_coverage import checked
+from regression_lifecycle_cases import ledger
+from regression_local_cases import api, require
+from regression_resource_cases import cleanup, journal
+from regression_smoke import check_mock_requests, signed_headers
+from regression_suite import resolve
+
+GROUP = 'plugins'
+CASE_NAMES = {'plugins.command_version', 'plugins.faq_crud'}
+COMMAND_PLUGINS = [
+    {'author': 'langbot-team', 'name': name}
+    for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands')
+]
+FAQ_API = '/api/v1/plugins/RockChinQ/FAQManager/page-api'
+
+
+def add_cases(profile):
+    pipeline = copy.deepcopy(next(item for item in profile['resources'] if item['id'] == 'chat_pipeline'))
+    pipeline['id'] = 'plugin_command_pipeline'
+    pipeline['body']['name'] = 'regression plugin commands'
+    pipeline['body']['extensions_preferences'] = {
+        'enable_all_plugins': False,
+        'plugins': copy.deepcopy(COMMAND_PLUGINS),
+    }
+    profile['resources'].append(pipeline)
+    for name, expected in (
+        ('plugins.command_version', 'LANGBOT_PLUGIN_COMMAND_OK'),
+        ('plugins.faq_crud', 'LANGBOT_PLUGIN_FAQ_OK'),
+    ):
+        profile['cases'].append({'name': name, 'scenario': name, 'stage': 'plugins', 'expect': expected})
+
+
+def data(suite, method, path, body=None, *, headers=None):
+    status, response = api(suite, method, path, body, headers=headers)
+    require(
+        status == 200 and isinstance(response, dict) and response.get('code') == 0,
+        f'{method} {path} did not return a successful response (HTTP {status})',
+    )
+    result = response.get('data')
+    if method == 'DELETE' and result is None:
+        return {}
+    require(isinstance(result, dict), f'{method} {path} returned malformed data')
+    return result
+
+
+def command_version(suite, case, row):
+    pipeline = data(suite, 'GET', f'/api/v1/pipelines/{suite.variables["plugin_command_pipeline"]}')['pipeline']
+    preferences = pipeline.get('extensions_preferences', {})
+    checked(
+        row,
+        'command_plugin_bindings',
+        lambda: require(
+            preferences.get('enable_all_plugins') is False and preferences.get('plugins') == COMMAND_PLUGINS,
+            'Command pipeline must bind only the three base plugins and EssentialCommands',
+        ),
+    )
+    version = data(suite, 'GET', '/api/v1/system/info').get('version')
+    require(isinstance(version, str) and bool(version), 'Core did not report its version')
+    records = Path(suite.path(suite.profile['smoke']['mock_records']))
+    require(records.is_file(), 'Command model-call evidence requires the dedicated mock recording file')
+    body = resolve(
+        copy.deepcopy(next(item['body'] for item in suite.profile['resources'] if item['id'] == 'chat_bot')),
+        suite.variables,
+    )
+    body['name'] = 'qa-plugin-command-' + uuid4().hex
+    body['adapter_config']['default_session_type'] = 'person'
+    body['event_bindings'] = [
+        {
+            'event_pattern': 'message.received',
+            'target_type': 'pipeline',
+            'target_uuid': suite.variables['plugin_command_pipeline'],
+            'enabled': True,
+            'filters': [],
+        }
+    ]
+    body['plugin_processors'] = []
+    owned = {'kind': 'bot', 'name': body['name'], 'state': 'creating'}
+    row.setdefault('owned_resources', []).append(owned)
+    journal(suite, row)
+
+    def delete_bot():
+        if 'uuid' not in owned:
+            bots = data(suite, 'GET', '/api/v1/platform/bots')['bots']
+            matches = [bot for bot in bots if bot.get('name') == owned['name']]
+            require(len(matches) == 1, 'Created command Bot identity is unresolved; inspect the resource journal')
+            owned['uuid'] = matches[0]['uuid']
+            journal(suite, row)
+        path = f'/api/v1/platform/bots/{owned["uuid"]}'
+        status, response = api(suite, 'GET', path)
+        if status != 404:
+            require(
+                status == 200
+                and response.get('code') == 0
+                and response.get('data', {}).get('bot', {}).get('name') == owned['name'],
+                'Cannot confirm ownership of the command Bot before cleanup',
+            )
+            data(suite, 'DELETE', path)
+        status, _ = api(suite, 'GET', path)
+        require(status == 404, 'Command Bot remains visible after cleanup')
+        owned['state'] = 'deleted'
+
+    try:
+        created = data(suite, 'POST', '/api/v1/platform/bots', body)
+        owned.update(uuid=created['uuid'], state='created')
+        journal(suite, row)
+        session = 'qa-plugin-version-' + uuid4().hex
+        payload = {
+            'session_id': session,
+            'session_type': 'person',
+            'message': [{'type': 'Plain', 'text': '!version'}],
+        }
+        offset = records.stat().st_size
+        reply = data(
+            suite,
+            'POST',
+            f'/bots/{owned["uuid"]}/sync',
+            payload,
+            headers=signed_headers(json.dumps(payload).encode(), body['adapter_config']['inbound_secret']),
+        )
+        row['command_response'] = reply
+        parts = reply.get('message')
+        checked(
+            row,
+            'command_version',
+            lambda: require(
+                reply.get('session_id') == session
+                and bool(reply.get('reply_to'))
+                and isinstance(parts, list)
+                and all(isinstance(part, dict) and part.get('type') == 'Plain' for part in parts)
+                and ''.join(part.get('text', '') for part in parts) == f'[bot] LangBot Version: {version}',
+                'Version command did not return the exact Core version in the originating session',
+            ),
+        )
+        with closing(ledger(suite)) as connection:
+            count = connection.execute(
+                'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND bot_id=? AND id>?',
+                (suite.variables['workspace_uuid'], owned['uuid'], row['after_id']),
+            ).fetchone()[0]
+        checked(row, 'command_no_run', lambda: require(count == 0, 'Version command unexpectedly created an AgentRun'))
+        checked(
+            row,
+            'command_no_model',
+            lambda: check_mock_requests(records, offset, [], timeout=suite.timeout(3)),
+        )
+        return {'result': case['expect'], 'version': version, 'agent_runs': count, 'model_requests': 0}
+    finally:
+        cleanup(suite, row, 'command_bot_cleanup', delete_bot)
+
+
+def faq_crud(suite, case, row):
+    def page(method, endpoint, body=None):
+        return data(
+            suite, 'POST', FAQ_API, {'page_id': 'manager', 'endpoint': endpoint, 'method': method, 'body': body}
+        )
+
+    def entries():
+        values = page('GET', '/entries').get('entries')
+        require(
+            isinstance(values, list) and all(isinstance(item, dict) and item.get('id') for item in values),
+            'FAQ entries response is malformed',
+        )
+        return values
+
+    previous = {item['id'] for item in entries()}
+    marker = 'qa-faq-' + uuid4().hex
+    question, before, after = marker + '-question', marker + '-before', marker + '-after'
+    owned = {'kind': 'faq-entry', 'question': question, 'state': 'creating'}
+    row.setdefault('owned_resources', []).append(owned)
+    journal(suite, row)
+
+    def expect_entry(values, answer):
+        expected = {'id': owned['id'], 'question': question, 'answer': answer}
+        require(values == [expected], 'FAQ content or entry identity differs from the submitted value')
+
+    def delete_entry():
+        current = entries()
+        if 'id' not in owned:
+            matches = [item for item in current if item.get('question') == question and item['id'] not in previous]
+            require(len(matches) <= 1, 'Created FAQ identity is ambiguous; inspect the resource journal')
+            if not matches:
+                owned['state'] = 'absent'
+                return
+            owned['id'] = matches[0]['id']
+            journal(suite, row)
+        require(owned['id'] not in previous, 'Refusing to delete a pre-existing FAQ entry')
+        matches = [item for item in current if item['id'] == owned['id']]
+        if matches:
+            require(
+                len(matches) == 1 and matches[0].get('question') == question, 'FAQ ownership changed before cleanup'
+            )
+            deleted = page('DELETE', '/entries', {'id': owned['id']})
+            require(deleted.get('deleted') == owned['id'], 'FAQ deletion returned a different entry ID')
+        require(not any(item['id'] == owned['id'] for item in entries()), 'Deleted FAQ remains visible')
+        require(page('POST', '/search', {'query': marker}).get('results') == [], 'Deleted FAQ remains searchable')
+        owned['state'] = 'deleted'
+
+    try:
+        created = page('POST', '/entries', {'question': question, 'answer': before}).get('entry')
+        require(isinstance(created, dict) and bool(created.get('id')), 'FAQ creation did not return an entry ID')
+        owned.update(id=created['id'], state='created')
+        journal(suite, row)
+        require(owned['id'] not in previous, 'FAQ creation reused a pre-existing entry ID')
+        checked(row, 'faq_create', lambda: expect_entry([created], before))
+        checked(
+            row, 'faq_read', lambda: expect_entry([item for item in entries() if item['id'] == owned['id']], before)
+        )
+        checked(
+            row, 'faq_search', lambda: expect_entry(page('POST', '/search', {'query': before}).get('results'), before)
+        )
+        updated = page('PUT', '/entries', {'id': owned['id'], 'question': question, 'answer': after}).get('entry')
+        checked(row, 'faq_update', lambda: expect_entry([updated], after))
+        checked(
+            row,
+            'faq_update_read',
+            lambda: expect_entry([item for item in entries() if item['id'] == owned['id']], after),
+        )
+        checked(
+            row,
+            'faq_update_search',
+            lambda: expect_entry(page('POST', '/search', {'query': after}).get('results'), after),
+        )
+        checked(
+            row,
+            'faq_old_answer_absent',
+            lambda: require(
+                page('POST', '/search', {'query': before}).get('results') == [], 'FAQ search returned stale content'
+            ),
+        )
+        checked(row, 'faq_delete', delete_entry)
+        return {'result': case['expect'], 'entry_id': owned['id'], 'deleted': True}
+    finally:
+        if owned['state'] != 'deleted':
+            cleanup(suite, row, 'faq_cleanup', delete_entry)
+        else:
+            journal(suite, row)
+
+
+def run_case(suite, case, row):
+    require(case['scenario'] in CASE_NAMES, 'Unknown plugin scenario')
+    return (command_version if case['scenario'] == 'plugins.command_version' else faq_crud)(suite, case, row)
@@ -1,4 +1,4 @@
-"""Resolve official published regression plugins once, then install the recorded bytes."""
+"""Prepare source or published regression plugins and verify installed artifacts."""

 from datetime import datetime, timezone
 import hashlib
@@ -6,6 +6,10 @@ import io
 import json
 from pathlib import Path
 import re
+import shutil
+import subprocess
+import sys
+import tempfile
 import urllib.error
 import urllib.request
 import zipfile
@@ -15,6 +19,7 @@ import yaml

 MARKETPLACE = 'https://space.langbot.app'
 PLUGIN_IDS = ('langbot-team/LocalAgent', 'langbot-team/LangRAG', 'langbot-team/GeneralParsers')
+PLUGIN_PATHS = ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers')
 METADATA_LIMIT = 1024 * 1024
 PACKAGE_LIMIT = 64 * 1024 * 1024

@@ -83,7 +88,7 @@ def check_archive(content, plugin_id, version):
        ):
            raise ValueError('package manifest identity/version differs from the selected release')
    except (zipfile.BadZipFile, yaml.YAMLError, KeyError, ValueError, RuntimeError) as exc:
-        raise ValueError(f'Invalid published package for {plugin_id}@{version}: {exc}') from exc
+        raise ValueError(f'Invalid plugin package for {plugin_id}@{version}: {exc}') from exc


 def prepare_published_plugins(directory, replay=None):
@@ -141,8 +146,61 @@ def prepare_published_plugins(directory, replay=None):
    return manifest


-def install_published_plugins(call, directory, manifest):
-    """Install the downloaded packages and require exact, initialized CLI readback."""
+def prepare_source_plugins(directory, source, extra_plugins=()):
+    """Build selected plugin directories with the SDK CLI, without publishing."""
+    directory, source = Path(directory).resolve(), Path(source).resolve()
+    paths = [*PLUGIN_PATHS, *extra_plugins]
+    if len(paths) != len(set(paths)):
+        raise ValueError('Duplicate source plugin selection')
+    for path in paths:
+        if (
+            not re.fullmatch(r'(Runner|KnowledgeEngine|misc)/[A-Za-z0-9_.-]+', path)
+            or not (source / path).resolve().is_relative_to(source)
+            or not (source / path / 'manifest.yaml').is_file()
+        ):
+            raise ValueError(f'Invalid source plugin path: {path}')
+    directory.mkdir(parents=True, exist_ok=False)
+    manifest = {'version': 1, 'selection': 'source', 'plugins': []}
+    write_manifest(directory, manifest)
+    for path in paths:
+        metadata = yaml.safe_load((source / path / 'manifest.yaml').read_text())['metadata']
+        plugin_id = f'{metadata["author"]}/{metadata["name"]}'
+        if any(entry['id'] == plugin_id for entry in manifest['plugins']):
+            raise ValueError(f'Duplicate source plugin identity: {plugin_id}')
+        if not all(re.fullmatch(r'[A-Za-z0-9_.-]+', metadata[key]) for key in ('author', 'name')):
+            raise ValueError(f'Invalid source plugin identity: {plugin_id}')
+        version = release_version(metadata['version'])
+        entry = {
+            'id': plugin_id,
+            'version': version,
+            'source': path,
+            'archive': f'{metadata["author"]}-{metadata["name"]}-{version}.lbpkg',
+        }
+        manifest['plugins'].append(entry)
+        write_manifest(directory, manifest)
+        with tempfile.TemporaryDirectory() as temporary:
+            package_source = Path(temporary) / 'plugin'
+            shutil.copytree(
+                source / path, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')
+            )
+            with (directory / f'{metadata["name"]}-build.log').open('w') as log:
+                subprocess.run(
+                    [sys.executable, '-m', 'langbot_plugin.cli.__init__', 'build', '--output', str(directory)],
+                    cwd=package_source,
+                    stdout=log,
+                    stderr=subprocess.STDOUT,
+                    check=True,
+                    timeout=180,
+                )
+        content = (directory / entry['archive']).read_bytes()
+        check_archive(content, plugin_id, version)
+        entry['sha256'] = hashlib.sha256(content).hexdigest()
+        write_manifest(directory, manifest)
+    return manifest
+
+
+def install_plugins(call, directory, manifest):
+    """Install recorded packages and require exact, initialized CLI readback."""
    directory = Path(directory)
    for entry in manifest['plugins']:
        archive = directory / entry['archive']
@@ -151,7 +209,7 @@ def install_published_plugins(call, directory, manifest):
        try:
            call(['plugin', 'install', 'local', '--file', str(archive), '--wait'])
        except RuntimeError as exc:
-            raise RuntimeError(f'Published plugin install failed: {entry["id"]}@{entry["version"]}: {exc}') from exc
+            raise RuntimeError(f'Plugin install failed: {entry["id"]}@{entry["version"]}: {exc}') from exc
        response = call(['plugin', 'list'])
        plugins = response.get('plugins') if isinstance(response, dict) else None
        if not isinstance(plugins, list) or any(not isinstance(item, dict) for item in plugins):
@@ -165,6 +223,4 @@ def install_published_plugins(call, directory, manifest):
            or matches[0].get('enabled') is not True
            or matches[0].get('status') != 'initialized'
        ):
-            raise RuntimeError(
-                f'Published plugin is not ready at the selected version: {entry["id"]}: {entry["installed"]}'
-            )
+            raise RuntimeError(f'Plugin is not ready at the selected version: {entry["id"]}: {entry["installed"]}')
@@ -0,0 +1,255 @@
+"""Challenge plugin business oracles and cleanup without starting target services."""
+
+import copy
+import hashlib
+import hmac
+import json
+from pathlib import Path
+import sqlite3
+import tempfile
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+import regression_plugin_cases as plugins
+from regression_scenarios import expanded_profile
+
+
+def success(value=None):
+    return 200, {'code': 0, 'data': value}
+
+
+class PluginCasesTests(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        self.root = Path(temporary.name)
+        self.profile = expanded_profile()
+        plugins.add_cases(self.profile)
+        self.records = self.root / 'mock.jsonl'
+        self.records.write_text(
+            json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'earlier'}, 'status': 200}) + '\n'
+        )
+        self.database = self.root / 'ledger.db'
+        with sqlite3.connect(self.database) as connection:
+            connection.execute('CREATE TABLE agent_run (id INTEGER, workspace_id TEXT, bot_id TEXT)')
+        self.profile['smoke'].update(ledger_db=str(self.database), mock_records=str(self.records))
+        self.suite = SimpleNamespace(
+            profile=self.profile,
+            variables={
+                'workspace_uuid': 'workspace',
+                'chat_pipeline': 'original-pipeline',
+                'plugin_command_pipeline': 'isolated-pipeline',
+                'group_secret': 'private-signing-secret',
+            },
+            secrets=['private-signing-secret'],
+            env={},
+            path=lambda value: value,
+            timeout=lambda value: value,
+            write=lambda name, value: (self.root / name).write_text(json.dumps(value)),
+        )
+
+    def case(self, scenario):
+        return next(case for case in self.profile['cases'] if case.get('scenario') == scenario)
+
+    def test_profile_adds_only_one_isolated_pipeline_and_two_scenarios(self):
+        original = expanded_profile()
+        profile = copy.deepcopy(original)
+        plugins.add_cases(profile)
+        self.assertEqual(profile['resources'][:-1], original['resources'])
+        self.assertEqual(profile['cases'][:-2], original['cases'])
+        pipeline = profile['resources'][-1]
+        self.assertEqual(pipeline['id'], 'plugin_command_pipeline')
+        self.assertEqual(
+            pipeline['body']['extensions_preferences'],
+            {
+                'enable_all_plugins': False,
+                'plugins': [
+                    {'author': 'langbot-team', 'name': name}
+                    for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands')
+                ],
+            },
+        )
+        self.assertEqual({case['scenario'] for case in profile['cases'][-2:]}, plugins.CASE_NAMES)
+        self.assertTrue(all('http_group' not in case for case in profile['cases'][-2:]))
+
+    def command_api(self, failure=None):
+        state = {}
+
+        def request(suite, method, path, body=None, headers=None):
+            if path == '/api/v1/pipelines/isolated-pipeline':
+                return success(
+                    {
+                        'pipeline': {
+                            'extensions_preferences': {
+                                'enable_all_plugins': failure == 'bindings',
+                                'plugins': copy.deepcopy(plugins.COMMAND_PLUGINS),
+                            }
+                        }
+                    }
+                )
+            if path == '/api/v1/system/info':
+                return success({'version': '9.5.1'})
+            if path == '/api/v1/platform/bots' and method == 'POST':
+                self.assertEqual(body['event_bindings'][0]['target_uuid'], 'isolated-pipeline')
+                self.assertEqual(body['plugin_processors'], [])
+                self.assertEqual(body['adapter_config']['inbound_secret'], 'private-signing-secret')
+                state.update(uuid='owned-bot', name=body['name'])
+                return success({'uuid': 'owned-bot'})
+            if path == '/bots/owned-bot/sync':
+                self.assertEqual(body['session_type'], 'person')
+                self.assertEqual(body['message'], [{'type': 'Plain', 'text': '!version'}])
+                signed = headers['X-LB-Timestamp'].encode() + b'.' + json.dumps(body).encode()
+                expected = 'sha256=' + hmac.new(b'private-signing-secret', signed, hashlib.sha256).hexdigest()
+                self.assertEqual(headers['X-LB-Signature'], expected)
+                if failure == 'run':
+                    with sqlite3.connect(self.database) as connection:
+                        connection.execute("INSERT INTO agent_run VALUES (1, 'workspace', 'owned-bot')")
+                if failure == 'model':
+                    with self.records.open('a') as stream:
+                        stream.write(
+                            json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'wrong'}, 'status': 200})
+                            + '\n'
+                        )
+                return success(
+                    {
+                        'session_id': 'wrong-session' if failure == 'session' else body['session_id'],
+                        'reply_to': 'in_message',
+                        'message': [
+                            {
+                                'type': 'Plain',
+                                'text': '[bot] LangBot Version: ' + ('9.4.0' if failure == 'version' else '9.5.1'),
+                            }
+                        ],
+                    }
+                )
+            if path == '/api/v1/platform/bots/owned-bot':
+                if method == 'DELETE':
+                    state['deleted'] = True
+                    return success()  # The real Bot DELETE returns data=null.
+                return (404, {'code': -1}) if state.get('deleted') else success({'bot': dict(state)})
+            self.fail(f'Unexpected command API call: {method} {path}')
+
+        return request, state
+
+    def test_command_checks_signed_reply_without_requiring_a_run(self):
+        request, state = self.command_api()
+        row = {'after_id': 0}
+        with patch.object(plugins, 'api', side_effect=request):
+            result = plugins.run_case(self.suite, self.case('plugins.command_version'), row)
+        self.assertEqual(
+            result, {'result': 'LANGBOT_PLUGIN_COMMAND_OK', 'version': '9.5.1', 'agent_runs': 0, 'model_requests': 0}
+        )
+        self.assertTrue(state['deleted'])
+        self.assertNotIn('scenario_runs', row)
+        self.assertTrue(all(value == 'passed' for value in row['assertions'].values()))
+
+    def test_command_rejects_bad_reply_model_calls_runs_and_plugin_leakage(self):
+        for failure in ('version', 'session', 'model', 'run', 'bindings'):
+            with self.subTest(failure=failure):
+                with sqlite3.connect(self.database) as connection:
+                    connection.execute('DELETE FROM agent_run')
+                request, state = self.command_api(failure)
+                row = {'after_id': 0}
+                with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError):
+                    plugins.run_case(self.suite, self.case('plugins.command_version'), row)
+                if failure != 'bindings':
+                    self.assertTrue(state['deleted'])
+                self.assertIn('failed', row['assertions'].values())
+
+    def faq_api(self, failure=None, cleanup_failure=False):
+        unrelated = {'id': 'unrelated', 'question': 'Existing question', 'answer': 'Existing answer'}
+        state = {'entries': [unrelated], 'created': 0, 'deletes': []}
+
+        def request(suite, method, path, body=None, headers=None):
+            self.assertEqual((method, path, body['page_id']), ('POST', plugins.FAQ_API, 'manager'))
+            verb, endpoint, payload = body['method'], body['endpoint'], body['body']
+            if (verb, endpoint) == ('GET', '/entries'):
+                return success({'entries': copy.deepcopy(state['entries'])})
+            if (verb, endpoint) == ('POST', '/entries'):
+                state['created'] += 1
+                entry = {'id': 'owned-faq', **payload}
+                state['entries'].append(entry)
+                state['original'] = copy.deepcopy(entry)
+                if failure == 'lost_create':
+                    raise ConnectionResetError('creation response lost')
+                return success({'entry': copy.deepcopy(entry)})
+            if (verb, endpoint) == ('PUT', '/entries'):
+                self.assertEqual(payload['id'], 'owned-faq')
+                if failure == 'update_error':
+                    raise ValueError('primary update failure')
+                if failure != 'stale_update':
+                    state['entries'][1].update(payload)
+                return success({'entry': copy.deepcopy(payload)})
+            if (verb, endpoint) == ('POST', '/search'):
+                rows = list(state['entries'])
+                if failure == 'stale_search' and state['deletes']:
+                    rows.append(state['original'])
+                return success(
+                    {
+                        'results': copy.deepcopy(
+                            [
+                                item
+                                for item in rows
+                                if payload['query'] in item['question'] or payload['query'] in item['answer']
+                            ]
+                        )
+                    }
+                )
+            if (verb, endpoint) == ('DELETE', '/entries'):
+                self.assertEqual(payload, {'id': 'owned-faq'})
+                if cleanup_failure:
+                    raise RuntimeError('cleanup private-signing-secret failed')
+                state['deletes'].append(payload['id'])
+                state['entries'] = [item for item in state['entries'] if item['id'] != payload['id']]
+                return success({'deleted': payload['id']})
+            self.fail(f'Unexpected Page API call: {body}')
+
+        return request, state
+
+    def test_faq_crud_checks_content_and_removes_only_its_entry(self):
+        request, state = self.faq_api()
+        row = {}
+        with patch.object(plugins, 'api', side_effect=request):
+            result = plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
+        self.assertEqual(result, {'result': 'LANGBOT_PLUGIN_FAQ_OK', 'entry_id': 'owned-faq', 'deleted': True})
+        self.assertEqual(state['deletes'], ['owned-faq'])
+        self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated'])
+        self.assertTrue(all(value == 'passed' for value in row['assertions'].values()))
+
+    def test_faq_rejects_stale_readback_and_search(self):
+        for failure, assertion in (('stale_update', 'faq_update_read'), ('stale_search', 'faq_delete')):
+            with self.subTest(failure=failure):
+                request, state = self.faq_api(failure)
+                row = {}
+                with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError):
+                    plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
+                self.assertEqual(row['assertions'][assertion], 'failed')
+                self.assertEqual(state['deletes'], ['owned-faq'])
+                self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated'])
+
+    def test_faq_lost_create_response_recovers_owned_id_without_retrying_creation(self):
+        request, state = self.faq_api('lost_create')
+        row = {}
+        with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ConnectionResetError):
+            plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
+        self.assertEqual(state['created'], 1)
+        self.assertEqual(state['deletes'], ['owned-faq'])
+        self.assertEqual(row['owned_resources'][0]['state'], 'deleted')
+
+    def test_faq_preserves_original_error_and_redacts_cleanup_error(self):
+        request, _ = self.faq_api('update_error', cleanup_failure=True)
+        row = {}
+        with (
+            patch.object(plugins, 'api', side_effect=request),
+            self.assertRaisesRegex(ValueError, 'primary update failure'),
+        ):
+            plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
+        self.assertEqual(row['cleanup_errors'], ['RuntimeError: cleanup [REDACTED] failed'])
+        evidence = (self.root / row['resource_journal']).read_text()
+        self.assertNotIn('private-signing-secret', evidence)
+        self.assertIn('owned-faq', evidence)
+
+
+if __name__ == '__main__':
+    unittest.main()
@@ -1,4 +1,4 @@
-"""Exercise published resolution, replay integrity and installation readback."""
+"""Exercise plugin selection, artifact integrity and installation readback."""

 from collections import Counter
 import copy
@@ -12,6 +12,8 @@ import unittest
 from unittest.mock import patch
 import zipfile

+import yaml
+
 import regression_plugins as plugins


@@ -27,6 +29,66 @@ def archive(plugin_id, version='1.2.3'):
    return output.getvalue()


+class SourcePluginsTests(unittest.TestCase):
+    def test_selected_sources_build_in_isolation_and_record_actual_artifacts(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            source, output = root / 'sources', root / 'packages'
+            paths = (*plugins.PLUGIN_PATHS, 'misc/ExtraPlugin')
+            for path in paths:
+                folder = source / path
+                folder.mkdir(parents=True)
+                with zipfile.ZipFile(io.BytesIO(archive('test/' + folder.name))) as package:
+                    package.extractall(folder)
+
+            def build(args, *, cwd, **kwargs):
+                self.assertEqual(args[1:5], ['-m', 'langbot_plugin.cli.__init__', 'build', '--output'])
+                self.assertFalse(cwd.is_relative_to(source))
+                metadata = yaml.safe_load((cwd / 'manifest.yaml').read_text())['metadata']
+                (cwd / 'generated.html').write_text('Generated only in the isolated build copy')
+                filename = f'{metadata["author"]}-{metadata["name"]}-{metadata["version"]}.lbpkg'
+                (Path(args[-1]) / filename).write_bytes(archive('test/' + metadata['name']))
+
+            with patch.object(plugins.subprocess, 'run', side_effect=build):
+                manifest = plugins.prepare_source_plugins(output, source, ['misc/ExtraPlugin'])
+            self.assertEqual(manifest['selection'], 'source')
+            self.assertEqual([entry['source'] for entry in manifest['plugins']], list(paths))
+            for entry in manifest['plugins']:
+                self.assertEqual(
+                    entry['sha256'], plugins.hashlib.sha256((output / entry['archive']).read_bytes()).hexdigest()
+                )
+                self.assertFalse((source / entry['source'] / 'generated.html').exists())
+
+    def test_invalid_or_duplicate_selection_is_rejected_before_build(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            for path in plugins.PLUGIN_PATHS:
+                (root / path).mkdir(parents=True)
+                (root / path / 'manifest.yaml').write_text('kind: Plugin')
+            (root / 'misc/escape').symlink_to(root.parent, target_is_directory=True)
+            for selected in (['../outside'], ['/tmp/plugin'], ['misc/escape'], [plugins.PLUGIN_PATHS[0]]):
+                with self.subTest(selected=selected), patch.object(plugins.subprocess, 'run') as build:
+                    with self.assertRaises(ValueError):
+                        plugins.prepare_source_plugins(root / 'output', root, selected)
+                    build.assert_not_called()
+
+    def test_wrong_built_identity_fails_without_installation(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            root = Path(temporary)
+            for path in plugins.PLUGIN_PATHS:
+                folder = root / path
+                folder.mkdir(parents=True)
+                with zipfile.ZipFile(io.BytesIO(archive('test/' + folder.name))) as package:
+                    package.extractall(folder)
+
+            def build(args, **kwargs):
+                (Path(args[-1]) / 'test-LocalAgent-1.2.3.lbpkg').write_bytes(archive('test/WrongPlugin'))
+
+            with patch.object(plugins.subprocess, 'run', side_effect=build):
+                with self.assertRaisesRegex(ValueError, 'identity/version differs'):
+                    plugins.prepare_source_plugins(root / 'output', root)
+
+
 class PublishedPluginsTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
@@ -95,7 +157,7 @@ class PublishedPluginsTests(unittest.TestCase):
            self.assertEqual(args, ['plugin', 'list'])
            return {'plugins': installed}

-        plugins.install_published_plugins(call, directory, manifest)
+        plugins.install_plugins(call, directory, manifest)
        recorded = json.loads((directory / 'manifest.json').read_text())
        self.assertTrue(all(entry['installed'][0]['status'] == 'initialized' for entry in recorded['plugins']))
        self.requests.clear()
@@ -178,13 +240,13 @@ class PublishedPluginsTests(unittest.TestCase):
            }

        with self.assertRaisesRegex(RuntimeError, 'not ready at the selected version'):
-            plugins.install_published_plugins(call, directory, manifest)
+            plugins.install_plugins(call, directory, manifest)
        self.assertEqual(len(calls), 2)
        recorded = json.loads((directory / 'manifest.json').read_text())
        self.assertEqual(recorded['plugins'][0]['installed'][0]['version'], '0.0.1')
        (directory / manifest['plugins'][0]['archive']).write_bytes(b'changed after download')
        with self.assertRaisesRegex(ValueError, 'changed before installation'):
-            plugins.install_published_plugins(call, directory, manifest)
+            plugins.install_plugins(call, directory, manifest)
        self.assertEqual(len(calls), 2)