diff --git a/docs/LANG16_LOCAL_QA.md b/docs/LANG16_LOCAL_QA.md new file mode 100644 index 000000000..b27c7d6a5 --- /dev/null +++ b/docs/LANG16_LOCAL_QA.md @@ -0,0 +1,281 @@ +# LANG-16: release 4.11 local QA + +This report separates reproduced defects from unverified user reports. A bounded local run does not +rule out an overnight leak or establish the reporter's exact network failure. + +## Revisions, authorization and isolation + +- Upgrade source: v4.10.10, `a63808caa6f10bc11f711373cabc11ffb6519b00`, SDK 0.5.7. +- Target/before baseline: official v4.11.0, `502b38130f9878b405498b93e4203593573eb19b`, SDK 0.7.11. +- Core task branch: `qa/lang-16-v4.11.0`. This document accompanies the tested core changes. +- LocalAgent task branch: `agent/langbot-local-qa/7acb7953fc8d`, fix commit + `f33146a0c263f643d1bba803c640af2ac7906f46` in `langbot-app/langbot-plugins`. +- macOS 27.0 arm64, 36 GiB RAM, Python 3.12.14, Node 26.10.0, pnpm 8.9.2. +- Dedicated task worktrees; existing core/RT/Box/frontend on 5399, 5400/5401, 5410, 3000 were left alone. +- Test core 15399, RT 15400/15401, Box 15410, Vite 13000, OneBot peers 12280/12281, + synthetic marketplace 15420. Every startup uses new data/config/plugin/cache/log directories. +- Only synthetic OneBot peers, never real QQ destinations. Live model tests use the member-authorized + gateway with deepseek-chat and 128 output tokens; secrets entered through getpass, never committed. + The credential database/journals were deleted after testing. Raw config/log files are not deliverables. +- Single concurrent stack, two-second sampling, 2 GiB total owned RSS stop limit (3 GiB for browser tests), + CPU >200% for five samples stops a run; duration hard cap 900 seconds. 100% CPU means one logical core. + A deliberately blocking plugin is bounded to three seconds. No unbounded deadlock injection. +- Cleanup records owned process identities, checks the Multica daemon PID and verifies survivors. + Docker cleanup is restricted to containers with mounts below the freshly created probe directory. +- Updated authorization permits commits/pushes to task branches. No external PR, merge or deployment. + +## 1–2. Migration installation and diagnostics + +Both real marketplace paths installed LocalAgent 0.2.1 successfully on this Mac: +`baseline-20261017-152549` migration (12.13 s) and `152708` extensions (11.11 s), 866380-byte package. +They share the downloader, proxy-aware HTTP client and 15-second timeout. Migration prefers pinned +0.2.1 and may fall back to latest when absent; extension installation can request latest or pinned. +The reported path discrepancy did not reproduce. A progress screenshot alone does not prove success. + +`baseline-20261017-193519` uses real core APIs/downloader against a synthetic loopback HTTP server, +with NO_PROXY for localhost to avoid the host proxy changing disconnects into 502 responses: + +| Injected condition | Migration diagnostic | Migration / extension completion | +| --- | --- | --- | +| HTTP 503 | http_status, HTTP 503 | 1.07 / 1.02 s | +| Peer closes without response | incomplete_response | 1.02 / 1.03 s | +| Truncated body, 18/9999 bytes | incomplete_response | 1.04 / 1.03 s | +| 17-second silent response | read_timeout | 15.19 / 15.19 s | +| Listener offline/refused connection | connect_error | 1.04 / 0.02 s | + +All ten tasks reached done; migration pipelines stayed blocked with categorized errors. The new task +metadata allowlists diagnostic kind and HTTP status, excluding exception text, URLs, headers and bodies. +Unit cases deliberately embed fake credentials to verify they cannot leak through the new field. +This does not change download retries/timeouts or claim all existing extension error text is sanitized. + +`baseline-20261017-193742`: real Edge browser, first-run wizard skipped, migration reviewed, automatic +installation attempted, failure details expanded. Visible `Network diagnostic: http_status · HTTP 503`, +0 migrated/1 needs attention, no page errors; screenshot visually inspected. Locale label added for all +existing migration locales. MCP remains curated and exposes no migration tool; HTTP/task schema documented. + +These are synthetic network faults, not verified mainland-China direct-network measurements. The Mac's +actual marketplace route succeeded. A separate proxy-disabled HTTPS trace returned loc=JP and the +route used utun4 (tunnel), so this host still cannot establish mainland direct-network behavior. +No VPN/system proxy setting was changed. Exact user proxy/DNS/network conditions remain unknown. + +## 3. Unrelated ingress and eager Box creation: two independent defects + +4.10.10 default pipeline resolution used routed_by_rule=False. Migration creates a catch-all binding +with empty filters; 4.11 incorrectly marked every binding explicitly routed, bypassing group response +rules. `botmgr.py` now marks only bindings with filters as explicitly routed. Explicit filtered routes +retain their legacy override behavior. Empty-filter bindings now obey configured group response rules. + +- Before `baseline-20261017-151907`: unmentioned ordinary chat reached the handler. +- After `152221`: ordinary chat, @another user and 30 unrelated burst messages filtered; keyword, + @bot and private controls entered chat. All 35 inbound rows terminal, query cached/queued zero. +- `153000`: real marketplace LocalAgent with no authorized model; 32 unrelated inputs filtered, + three positive controls failed with the expected model-selection error; no pending rows remained. + +Busy-session steering happens before the normal stage chain. Controller now runs the same response gate +on a copied message before offering it to the active Runner. Failed claims preserve the original chain. +Random admission draws are cached per query so queue rechecks cannot increase admission probability. +These steering details have automated coverage; successful live steering acceptance is not claimed. + +Separate LocalAgent defect: preparing Box whenever sandbox tools were enabled created a sandbox for a +plain answer even when no tool ran. The plugin now prepares eagerly only for incoming attachments; +sandbox call_tool acquires lazily under a lock. Non-sandbox tools delegate without acquisition. A tool +result carries sandbox_outbox so the model can discover the lazy-created delivery directory. + +Real model + Docker + OneBot comparison (same prompts, gateway/model): + +| Run | Before input | Unrelated chat | Plain QA_OK, no tools | Actual exec | +| --- | ---: | ---: | ---: | ---: | +| Before, `154144`, marketplace 0.2.1 | 0 | 0 | 1 | 1 | +| After, `192121`, task-built package | 0 | 0 | 0 | 1 | + +Numbers are Box session counts. The tool executed `printf LANG16_TOOL_OK` successfully; messages ended +success and queue/cache returned to zero. Sandbox limits: 1 CPU, 512 MiB, 128 PIDs, read-only rootfs, +mount under task directory. Owned processes/container were removed. Marketplace and repository sources +have other differences, so this is a behavior comparison, not a throughput claim. Attachment import, +scoped export and concurrent acquisition are regression-tested; the live prompts had no attachments. + +## Real 4.10.10 database upgrade + +`baseline-20261017-193212` started actual 4.10.10 in its locked SDK environment, initialized a synthetic +account, pipeline and disabled OneBot bot, then stopped cleanly. `193444` copied its database AND instance +identity into a new 4.11 data directory. Alembic moved from `0021_merge_reasoning_config` to +`0037_metadata_text`; real migration downloaded/activated LocalAgent and migrated the pipeline. +The original default bot binding became message.received with filters=[]. Enabling the bot and sending +ordinary chat filtered it; @bot reached Runner and failed only because this test intentionally had no +model. Both inbound records were terminal. This is synthetic old data, not the reporter's database. +An initial DB-only copy (`193241`) correctly failed the instance identity guard and is excluded. + +## 4. Runtime/plugin fault isolation + +`baseline-20261017-152407`: terminate only owned RT at 20 s, restart at 27 s. Core health stayed 200; +RT-connected false at 22.98–27.13 s, true by 29.22 s; reconnect delays 1/2/4 seconds. + +The extended matrix uses real installed synthetic SDK Runner packages in dedicated worker processes: +handler RuntimeError, bounded timeout, three-second busy loop, OneBot disconnect/reconnect, os._exit(17), +initialization exception and invalid dependency specification. A separate healthy plugin receives control +messages and GET /plugins is timed at each stage. No cloud model is used in this fault/load fixture. +This does not establish fault isolation for certified shared-runtime pools or arbitrary user plugins. + +The baseline does show cross-plugin latency coupling: healthy controls after RAISE/TIMEOUT/EXIT arrived +about 0.40 s after fault initiation (sent at +0.30 s), but BLOCK's healthy response arrived at +3.169 s. +Core/RT health stayed responsive. SDK 0.7.11 `runtime/plugin/mgr.py:3166` serially emits the event to +every initialized plugin in scope without first excluding plugins with no EventListener components; +Core `plugin/connector.py:2344` serially visits installation bindings. Both QA plugins declare only a +Runner, so the unrelated worker's event RPC waits behind its busy event loop. This is latency coupling, +not a reproduced whole-RT crash. Fixing admission at the SDK layer needs its source worktree; +`multica repo checkout https://github.com/langbot-app/langbot-plugin-sdk` returned +`repo is not configured for this workspace`. Installed dependency files were not patched as a substitute. + +Causal configuration control `baseline-20261017-200408`: disable enable_all_plugins and bind each +pipeline only to its required QA plugin. The same BLOCK control arrived at +0.393 s, versus +3.191 s +with all plugins enabled; other controls were +0.428–0.448 s. Both database and monitoring executions +API showed zero running queries at all post-fault checkpoints. This is a verified temporary workaround +when a pipeline does not need the other plugins, not a replacement for an SDK fix. No existing user +pipeline configuration was changed. + +## 5, 7–9. Terminal states and resources + +A malformed response config missing random reproduced a stale monitoring row: `152004` had an empty +queue but the inbound row stayed pending after a real KeyError; a separate assistant error row existed. +`MonitoringHelper.record_query_error` now finishes the original inbound record before optional error +details. `152315` retest finished both rows as error with an empty queue. Unit coverage also fails the +optional detail write. This is a concrete stale-state defect, not proof of the reported 242 tasks' cause. + +Empty startup `150152`, approximately 90 seconds, all four health endpoints HTTP 200: + +| Process | Peak / final RSS MiB | Mean CPU after 30 s | +| --- | --- | ---: | +| Core | 538.0 / 531.7 | 0.33% | +| RT | 63.5 / 63.5 | 0.03% | +| Box | 73.2 / 73.2 | 0.03% | +| Vite frontend | 350.9 / 346.8 | 0.10% | + +The load preserves the release default per-conversation limiter (60 messages / 60 seconds, drop). +The 160-message burst exercises the 100-query cache bound; evicted queued queries have not yet +created monitoring rows. Every ten seconds the soak sends 50 unrelated group messages and one +keyword control in a different group. Extended matched measurements are recorded below. The frontend is a development server, not a +production build. RSS/threads/file descriptors/process counts and runtime health task/queue/session +counters are sampled; these are not heap profiles. Long-lived production memory exhaustion and the +reported 67694–68034-second pending workload must not be declared disproved by minute-scale probes. + +## Host shutdown cancellation and lease recovery + +`baseline-20261017-200457` runs a real QA Runner waiting for 60 seconds. Before shutdown, the actual +monitoring executions API reports one running pipeline query. The supervisor sends TERM only to its +recorded core PID, while RT remains available for orderly shutdown, then cleans the remaining owned +processes. The inbound row is still pending immediately after host shutdown; this path relies on the +existing execution lease, not the new stage-error patch. + +`baseline-20261017-200536` starts a fresh isolated core from that stopped synthetic database plus its +instance identity. At 20:07:23 local time, its lease maintenance logs `Marked 1 interrupted executions +as failed after host lease expiry`; the row is error (about 130 seconds after shutdown, consistent with +120-second lease and 20-second maintenance interval). This validates shutdown cancellation/recovery, +not a nonexistent public OneBot task-cancel endpoint or cleanup of legacy rows without leases. + +## Matched load measurements + +Before `baseline-20261017-193933` uses unmodified 502b3813; after `195149` uses product fixes at +079e945b. Each completed about 731 seconds, including faults, a 160-message burst, 60 cycles of +50 unrelated + 1 keyword input at ten-second intervals, and 20 seconds idle recovery: 3232 inputs total. +Both use SDK 0.7.11 and the same synthetic Runner behavior. The machine remains shared with existing +user services; rate-limit window alignment and restart backoff jitter are not microbenchmark-controlled. + +| Process | Soak mean CPU before / after | Soak peak RSS MiB before / after | +| --- | ---: | ---: | +| Core | 66.23% / 30.95% | 557.73 / 558.84 | +| RT | 4.89% / 2.05% | 65.19 / 65.03 | +| RT workers, summed | 1.53% / 0.65% | 165.67 / 177.83 | +| Box | 0.01% / 0.02% | 73.48 / 73.31 | +| Vite | 0.10% / 0.12% | 274.62 / 276.59 | + +- Unrelated inputs: 3000 before / 3000 after. Before, all 3000 produced replies (660 actual Runner + replies, 2340 rate-limit notices); after, zero produced replies. All 60 keyword soak + controls produced Runner replies in both runs. +- Query cache peaked at 100 (99 queued, 1 executing) in the burst; final cached/queued=0 in both. + Sixty queued inputs were evicted before monitoring-row creation. The 3172 recorded inbound rows + ended as 3168 success + 3 error + 1 pending before, versus 3168 success + 4 error after. +- The remaining before pending row was the disconnected peer's failed reply, not active work. +- Application completed-task retention stayed bounded at 200. Workers returned to two running + processes; transient initialization retries peaked at three. Final core/RT/worker/Box/web descriptors + were 48/16/14/9/30 in both runs. No monotonic RSS/worker growth or OOM reproduced in this window. +- Recovery mean core CPU was 0.50% / 0.55%, RT 0.10% / 0.08%. Peak RSS was essentially unchanged; + lower final RSS across processes is not evidence of a heap-leak fix (OS residency can change). +- All four services had zero failed health probes after first readiness; no resource threshold fired; + owned process survivors were empty. GET /plugins max was 25 / 23 ms, zero failures. +- Cross-plugin BLOCK latency remained: healthy response at +3.169 / +3.191 s from fault initiation, + versus about +0.40 s for other fault controls. These core fixes do not repair that SDK coupling. + +The soak excludes actual LLM/history growth and has no sandbox workload; real LocalAgent/Box behavior +was tested separately above. Container CPU/RSS was not profiled; recorded container settings are limits, +not measured usage. No claim is made about the reporter's 18-hour workload or full memory-leak absence. + +## 6. Groq request schema + +Local reconstruction showed provider_specific_fields passed through outbound history/tool calls; the +pinned LiteLLM Groq adapter preserves non-null keys. The Groq-only request-boundary fix deep-copies +outgoing messages and strips that metadata from messages/tool calls. Stored history retains Gemini +signatures for fallback, and other providers retain serialization. A proxy identified solely as OpenAI +without a Groq model/provider identity is outside this fix's scope. + +Tests cover explicit/inferred Groq, streaming/non-streaming, tools, immutable history and provider +fallback metadata. Two tests send real LiteLLM HTTP/JSON and SSE requests to a synthetic loopback +server. They are protocol regressions, not real Groq calls. Real Groq remains explicitly deferred. + +## Validation and reproduction + +- Core focused suite: 255 passed, 41 warnings, including controller, response rules, monitoring failure, + queue lifecycle, migration service and LiteLLM. +- LocalAgent suite: 254 passed, 10 warnings. `test_shared_release.py` excluded because it asserts SDK + exactly 0.7.4 while the target release pins 0.7.11; this is not a full plugin release-gate pass. +- Frontend TypeScript noEmit passed; Ruff and git diff --check passed. +- Evidence is attached to LANG-16; no raw credentials, database or full application logs are published. + +Reproduce the focused core suite from the task checkout: + +```sh +.venv/bin/pytest tests/unit_tests/pipeline/test_controller.py tests/unit_tests/pipeline/test_controller_tenancy.py tests/unit_tests/pipeline/test_resprule.py tests/unit_tests/pipeline/test_monitoring_failure.py tests/unit_tests/pipeline/test_pool.py tests/unit_tests/pipeline/test_query_pool.py tests/unit_tests/pipeline/test_pipeline_migration_runtime.py tests/unit_tests/platform/test_botmgr_execution_trace.py tests/unit_tests/provider/test_litellmchat.py tests/unit_tests/api/service/test_pipeline_migration.py +``` + +For LocalAgent, run `python -m pytest tests --ignore=tests/test_shared_release.py` from its directory +using the target SDK 0.7.11 environment. Run `node_modules/.bin/tsc --noEmit` from the web directory. + +Run probes from the task checkout with locked dependencies and free isolated ports: + +```sh +.venv/bin/python tests/manual/lang16_baseline.py +LANG16_ONEBOT=1 .venv/bin/python tests/manual/lang16_baseline.py +LANG16_NETWORK=1 LANG16_SECONDS=150 .venv/bin/python tests/manual/lang16_baseline.py +LANG16_MATRIX=1 LANG16_SOAK_ROUNDS=60 LANG16_SECONDS=900 .venv/bin/python tests/manual/lang16_baseline.py +``` + +Use LANG16_CORE_SOURCE for a separate unmodified baseline worktree; use LANG16_CORE_PYTHON for old +locked dependencies. LANG16_UPGRADE=old seeds old data; after uses LANG16_SEED_DB and +LANG16_UPGRADE_ACCOUNT from that isolated run. The supervisor copies only task-owned synthetic data. +Live model runner requires LANG16_LLM_BASE_URL and prompts for the key; never place a key in arguments. +LANG16_PLUGIN_PACKAGE selects the task-built package. Review environment/config before running; +only this task's directories and recorded resource identities may be cleaned. + +Excluded exploratory runs: initial Box default-port conflict; incomplete UI run interrupted by platform; +invalid Runner config nesting; network harness missing description; browser RSS threshold; incorrect UI +wizard/tour/wait selectors; initial matrix control accidentally repeated fault keywords (`192332`); before-run !qa command parsing (`193812`, intentionally stopped). +They are not acceptance evidence. A resource limit fired safely during browser setup, not production load. + +## Final issue-by-issue disposition + +| Item | Disposition | Boundary | +| --- | --- | --- | +| 1 Installation detail | Redacted network diagnostics implemented; real API and browser failure flow verified | Original user network incident itself not reproduced | +| 2 Installation path discrepancy | Both actual marketplace paths succeed; ten synthetic fault/path cases terminate | Mainland route blocked: proxy-disabled egress is JP through utun4 | +| 3 Box/unrelated messages | Catch-all gate, busy steering gate and LocalAgent lazy Box fixes delivered | Explicit filtered bindings keep their override; live steering acceptance/attachments are not covered by live prompts | +| 4 RT coupling | Handler/timeout/exit/init/dependency cases tested; whole-RT crash not reproduced | Cross-plugin BLOCK latency reproduced; SDK checkout unavailable; explicit plugin binding is a verified workaround | +| 5 Memory exhaustion | Not reproduced in bounded process-separated runs | No heap or overnight real-LLM-history soak; not a leak-free guarantee | +| 6 Groq | Outbound schema correction and local JSON/SSE regressions delivered | Real Groq explicitly deferred, not passed | +| 7 Processing records | Stage-exception and disconnected-reply stale rows fixed and retested | No bulk rewriting of historical pending records; host-shutdown recovery recorded separately | +| 8 Backlog CPU/memory | Matched workload CPU reduction measured; bounded queue and recovery verified | Peak memory unchanged; cannot attribute reporter's exact resource spike | +| 9 Startup/ongoing resources | Cold/idle/message/fault/burst/ten-minute soak and recovery measured | Development frontend and synthetic load; limits described above | + +Remaining support requests go to Mika: register langbot-app/langbot-plugin-sdk for a task checkout to +repair the reproduced cross-plugin wait, and provide an authorized isolated mainland egress on this Mac +for the network acceptance gap. No change to the existing VPN or user services is authorized or needed +for the completed tests. No new paid resource/account permission is assumed. Until those are resolved +(or explicitly deferred), this is a partial delivery and LANG-16 must not be marked fully accepted. diff --git a/tests/manual/lang16_baseline.py b/tests/manual/lang16_baseline.py new file mode 100644 index 000000000..bb39d8ecb --- /dev/null +++ b/tests/manual/lang16_baseline.py @@ -0,0 +1,344 @@ +"""Bounded local startup probe; never uses an existing deployment's data.""" + +import json +import os +from pathlib import Path +import secrets +import socket +import subprocess +import sys +import time +import urllib.request + +import psutil +import yaml + + +repo = Path(__file__).resolve().parents[2] +root = repo.parent / 'evidence' / ('baseline-' + time.strftime('%Y%m%d-%H%M%S')) +root.mkdir(parents=True, mode=0o700) +ports = [15399, 15400, 15401, 15410, 13000] +if os.environ.get('LANG16_ONEBOT') == '1': + ports.append(12280) +if os.environ.get('LANG16_NETWORK') == '1': + ports.append(15420) +if os.environ.get('LANG16_MATRIX') == '1': + ports.extend([12280, 12281]) +if os.environ.get('LANG16_UPGRADE'): + ports.append(12280) +if os.environ.get('LANG16_SEED_DB'): + ports.extend([12280, 12281]) +ports = sorted(set(ports)) +for port in ports: + with socket.socket() as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(('127.0.0.1', port)) +env = {k: os.environ[k] for k in ('PATH', 'HOME', 'LANG', 'SYSTEMROOT') if k in os.environ} +for key in ('XDG_CACHE_HOME', 'UV_CACHE_DIR', 'TMPDIR'): + directory = root / key.lower() + directory.mkdir() + env[key] = str(directory) +env.update( + LANGBOT_DATA_ROOT=str(root / 'data'), + PYTHONUNBUFFERED='1', + NO_PROXY='127.0.0.1,localhost', + LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=secrets.token_hex(32), + VITE_API_PROXY_TARGET='http://127.0.0.1:15399', +) +core_source = Path(os.environ.get('LANG16_CORE_SOURCE', str(repo))).resolve() +env['PYTHONPATH'] = str(core_source / 'src') +core_python = os.environ.get('LANG16_CORE_PYTHON', sys.executable) +config = yaml.safe_load((core_source / 'src/langbot/templates/config.yaml').read_text()) +config['api'].update(host='127.0.0.1', port=15399, webhook_prefix='http://127.0.0.1:15399') +config['plugin'].update( + runtime_ws_url='ws://127.0.0.1:15400/control/ws', display_plugin_debug_url='ws://127.0.0.1:15401/plugin/debug/ws' +) +config['space']['disable_telemetry'] = True +if os.environ.get('LANG16_NETWORK') == '1': + config['space']['url'] = 'http://127.0.0.1:15420' + config['space']['disable_models_service'] = True +config['box']['runtime']['endpoint'] = 'ws://127.0.0.1:15410' +if os.environ.get('LANG16_BOX') == '1': + config['box']['backend'] = 'docker' + config['box']['local']['host_root'] = str(root / 'data/box') + config['box']['local']['allowed_mount_roots'] = [str(root / 'data/box')] + config['box']['limits']['max_sessions'] = 1 + config['box']['default_memory_mb'] = 256 +config['database']['sqlite']['path'] = str(root / 'data/langbot.db') +(root / 'data').mkdir() +if os.environ.get('LANG16_SEED_DB'): + import shutil + + seed = Path(os.environ['LANG16_SEED_DB']) + shutil.copy2(seed, root / 'data/langbot.db') + # Preserve this synthetic instance's identity as a real data-directory upgrade does. + (root / 'data/labels').mkdir() + shutil.copy2(seed.parent / 'labels/instance_id.json', root / 'data/labels/instance_id.json') +(root / 'data/config.yaml').write_text(yaml.safe_dump(config)) +rss_limit = (3 if os.environ.get('LANG16_NETWORK') == '1' else 2) * 1024**3 +owned = {} +handles = [] +samples = [] +result = { + 'sha': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=core_source, text=True).strip(), + 'core_source': str(core_source), + 'ports': ports, + 'limits': {'rss_bytes': rss_limit, 'cpu_percent': 200, 'cpu_samples': 5}, + 'health': {}, + 'reason': 'duration_complete', +} + + +def launch(name, args, cwd): + log = (root / (name + '.log')).open('w') + handles.append(log) + child_env = dict(env) + if name == 'network' and os.environ.get('LANG16_NETWORK_UI_ONLY'): + child_env['LANG16_NETWORK_UI_ONLY'] = os.environ['LANG16_NETWORK_UI_ONLY'] + if name == 'upgrade': + for key in ('LANG16_UPGRADE', 'LANG16_UPGRADE_ACCOUNT'): + if key in os.environ: + child_env[key] = os.environ[key] + if name == 'matrix' and os.environ.get('LANG16_BOUND'): + child_env['LANG16_BOUND'] = os.environ['LANG16_BOUND'] + if name == 'matrix' and os.environ.get('LANG16_CANCEL'): + child_env['LANG16_CANCEL'] = os.environ['LANG16_CANCEL'] + if name == 'matrix' and os.environ.get('LANG16_SOAK_ROUNDS'): + child_env['LANG16_SOAK_ROUNDS'] = os.environ['LANG16_SOAK_ROUNDS'] + if name == 'onebot': + for key in ('LANG16_LLM_KEY', 'LANG16_LLM_MODEL', 'LANG16_LLM_BASE_URL', 'LANG16_PLUGIN_PACKAGE'): + if key in os.environ: + child_env[key] = os.environ[key] + proc = subprocess.Popen(args, cwd=cwd, env=child_env, stdout=log, stderr=subprocess.STDOUT) + owned[proc.pid] = (name, psutil.Process(proc.pid)) + return proc + + +try: + rt_process = launch( + 'rt', + [core_python, '-m', 'langbot_plugin.cli', 'rt', '--ws-control-port', '15400', '--ws-debug-port', '15401'], + root, + ) + launch( + 'box', + [core_python, '-m', 'langbot_plugin.cli', 'box', '--host', '127.0.0.1', '--ws-control-port', '15410'], + root, + ) + core_process = launch( + 'core', [core_python, str(core_source / 'main.py'), '--standalone-runtime', '--standalone-box'], root + ) + launch( + 'web', + [str(repo / 'web/node_modules/.bin/vite'), '--host', '127.0.0.1', '--port', '13000', '--strictPort'], + repo / 'web', + ) + start = time.monotonic() + result['started_monotonic'] = start + hot = 0 + ui_started = False + onebot_started = False + rt_stopped = False + rt_restarted = False + install_started = False + network_started = False + matrix_started = False + upgrade_started = False + while time.monotonic() - start < min(900, int(os.environ.get('LANG16_SECONDS', '90'))): + if core_process.poll() is not None: + result['reason'] = 'core_exited' + break + for parent_role, parent in list(owned.values()): + try: + for child in parent.children(recursive=True): + owned.setdefault(child.pid, (parent_role.split('-child')[0] + '-child', child)) + except psutil.NoSuchProcess: + pass + rows = [] + for pid, (name, proc) in owned.items(): + try: + rows.append( + { + 'pid': pid, + 'role': name, + 'rss': proc.memory_info().rss, + 'cpu': proc.cpu_percent(), + 'threads': proc.num_threads(), + 'fds': proc.num_fds(), + } + ) + except psutil.NoSuchProcess: + pass + sample = {'seconds': round(time.monotonic() - start, 2), 'processes': rows} + sample['probes'] = {} + samples.append(sample) + for name, port, path in [ + ('core', 15399, '/healthz'), + ('rt', 15400, '/healthz'), + ('box', 15410, '/healthz'), + ('web', 13000, '/'), + ]: + try: + with urllib.request.urlopen(f'http://127.0.0.1:{port}{path}', timeout=0.5) as response: + body = response.read() + entry = {'status': response.status, 'seconds': sample['seconds']} + if name != 'web': + entry['body'] = json.loads(body) + result['health'][name] = entry + sample['probes'][name] = entry + except Exception: + sample['probes'][name] = {'status': None} + if os.environ.get('LANG16_RT_FAULT') == '1': + if sample['seconds'] >= 20 and not rt_stopped: + daemon_pid = json.loads(subprocess.check_output(['multica', 'daemon', 'status', '--output', 'json']))[ + 'pid' + ] + if rt_process.pid == daemon_pid: + raise RuntimeError('Refusing to terminate daemon') + rt_process.terminate() + rt_stopped = True + if sample['seconds'] >= 27 and rt_stopped and not rt_restarted: + if rt_process.poll() is None: + raise RuntimeError('Original owned RT has not stopped') + rt_process = launch( + 'rt-restarted', + [ + sys.executable, + '-m', + 'langbot_plugin.cli', + 'rt', + '--ws-control-port', + '15400', + '--ws-debug-port', + '15401', + ], + root, + ) + rt_restarted = True + if os.environ.get('LANG16_UI') == '1' and not ui_started and {'core', 'web'} <= result['health'].keys(): + launch('ui', ['node', str(repo / 'tests/manual/lang16_ui.mjs')], root) + ui_started = True + if ( + os.environ.get('LANG16_ONEBOT') == '1' + and not onebot_started + and {'core', 'rt', 'box'} <= result['health'].keys() + ): + args = [sys.executable, str(repo / 'tests/manual/lang16_onebot.py')] + if os.environ.get('LANG16_BOX') == '1': + args.append('--box') + if os.environ.get('LANG16_RUNNER') == '1': + args.append('--installed') + if os.environ.get('LANG16_FAULT') == '1': + args.append('--fault') + launch('onebot', args, root) + onebot_started = True + if ( + os.environ.get('LANG16_INSTALL') + and not install_started + and {'core', 'rt', 'box'} <= result['health'].keys() + ): + launch( + 'install', + [sys.executable, str(repo / 'tests/manual/lang16_install.py'), os.environ['LANG16_INSTALL']], + root, + ) + install_started = True + if ( + os.environ.get('LANG16_NETWORK') == '1' + and not network_started + and {'core', 'rt'} <= result['health'].keys() + ): + launch('network', [sys.executable, str(repo / 'tests/manual/lang16_network.py')], root) + network_started = True + if os.environ.get('LANG16_MATRIX') == '1' and not matrix_started and {'core', 'rt'} <= result['health'].keys(): + launch('matrix', [sys.executable, str(repo / 'tests/manual/lang16_matrix.py')], root) + matrix_started = True + if os.environ.get('LANG16_UPGRADE') and not upgrade_started and {'core', 'rt'} <= result['health'].keys(): + launch('upgrade', [sys.executable, str(repo / 'tests/manual/lang16_upgrade.py')], root) + upgrade_started = True + if upgrade_started and (root / 'upgrade-result.json').exists(): + result['reason'] = 'upgrade_probe_finished' + break + if os.environ.get('LANG16_CANCEL') == '1' and (root / 'cancel-ready').exists(): + # This is the recorded task-owned core, never an existing service. + daemon_pid = json.loads(subprocess.check_output(['multica', 'daemon', 'status', '--output', 'json']))['pid'] + if core_process.pid == daemon_pid: + raise RuntimeError('Refusing to terminate daemon') + core_process.terminate() + try: + core_process.wait(timeout=10) + result['reason'] = 'cancellation_probe_finished' + except subprocess.TimeoutExpired: + result['reason'] = 'cancellation_shutdown_timeout' + break + if matrix_started and (root / 'matrix-finished').exists(): + result['reason'] = 'matrix_probe_finished' + break + if network_started and (root / 'network-finished').exists(): + result['reason'] = 'network_probe_finished' + break + if install_started and (root / 'install-result.json').exists(): + result['reason'] = 'install_probe_finished' + break + hot = hot + 1 if sum(row['cpu'] for row in rows) > 200 else 0 + if sum(row['rss'] for row in rows) > rss_limit or hot >= 5: + result['reason'] = 'resource_threshold' + break + time.sleep(2) +finally: + daemon = json.loads(subprocess.check_output(['multica', 'daemon', 'status', '--output', 'json']))['pid'] + for pid, (_, proc) in reversed(list(owned.items())): + if pid == daemon: + raise RuntimeError('Refusing to terminate daemon') + try: + if proc.is_running(): + proc.terminate() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs([p for _, p in owned.values()], timeout=8) + for proc in alive: + if proc.pid != daemon and proc.is_running(): + proc.kill() + psutil.wait_procs(alive, timeout=3) + # Only containers whose bind mounts are inside this newly-created probe + # directory belong to this run; unrelated containers must never be stopped. + if os.environ.get('LANG16_BOX') == '1': + ids = subprocess.check_output(['docker', 'ps', '-aq'], text=True).split() + owned_containers = [] + for cid in ids: + mounts = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .Mounts}}', cid])) + if any(str(m.get('Source', '')).startswith(str(root) + '/') for m in mounts): + owned_containers.append(cid) + result['owned_containers'] = owned_containers + for cid in owned_containers: + subprocess.run(['docker', 'rm', '-f', cid], check=True, stdout=subprocess.DEVNULL) + for handle in handles: + handle.close() + if os.environ.get('LANG16_LLM_KEY'): + # Remove the task-owned credential database, including SQLite journals. + for database_file in (root / 'data').glob('langbot.db*'): + database_file.unlink() + for log_file in root.glob('*.log'): + log_file.write_text(log_file.read_text().replace(os.environ['LANG16_LLM_KEY'], '[REDACTED]')) + if os.environ.get('LANG16_CANCEL') == '1' or os.environ.get('LANG16_SEED_DB'): + import sqlite3 + + with sqlite3.connect(root / 'data/langbot.db') as db: + result['post_shutdown_messages'] = list( + db.execute('select role,status,message_content from monitoring_messages order by timestamp') + ) + result['samples'] = samples + result['owned_pids'] = list(owned) + result['survivors'] = [pid for pid, (_, p) in owned.items() if p.is_running()] + (root / 'metrics.json').write_text(json.dumps(result, indent=2)) + print( + json.dumps( + { + 'evidence': str(root), + 'reason': result['reason'], + 'health': result['health'], + 'survivors': result['survivors'], + }, + indent=2, + ) + ) diff --git a/tests/manual/lang16_install.py b/tests/manual/lang16_install.py new file mode 100644 index 000000000..3df207cd0 --- /dev/null +++ b/tests/manual/lang16_install.py @@ -0,0 +1,74 @@ +"""Real marketplace install through isolated Core APIs; no model calls.""" + +import asyncio +import json +from pathlib import Path +import secrets +import sqlite3 +import subprocess +import sys +import time + +import httpx + + +async def main(): + mode = sys.argv[1] + result = {'mode': mode, 'requested_plugin': 'langbot-team/LocalAgent@0.2.1'} + async with httpx.AsyncClient(base_url='http://127.0.0.1:15399', timeout=20, trust_env=False) as client: + + async def api(method, path, data=None): + response = await client.request(method, '/api/v1' + path, json=data) + body = response.json() + if response.status_code >= 400 or body.get('code') != 0: + raise RuntimeError(f'{path}: HTTP {response.status_code}: {body.get("msg")}') + return body.get('data', {}) + + try: + account = {'user': 'lang16-install@example.invalid', 'password': secrets.token_urlsafe(24)} + await api('POST', '/user/init', account) + auth = await api('POST', '/user/auth', account) + client.headers['Authorization'] = 'Bearer ' + auth['token'] + if mode == 'migration': + pipeline = (await api('POST', '/pipelines', {'name': '4.10.10 synthetic', 'description': 'QA'}))['uuid'] + repo = Path(__file__).resolve().parents[2] + old_config = subprocess.check_output( + ['git', 'show', 'v4.10.10:src/langbot/templates/default-pipeline-config.json'], cwd=repo + ).decode() + with sqlite3.connect('data/langbot.db') as db: + db.execute( + 'update legacy_pipelines set config=?, for_version=? where uuid=?', + (old_config, '4.10.10', pipeline), + ) + preview = await api('GET', '/pipelines/_/migration/preview') + result['preview'] = [ + {k: v for k, v in item.items() if k != 'preview_token'} for item in preview['items'] + ] + task = await api( + 'POST', '/pipelines/_/migration/execute', {'confirmed': True, 'all': True, 'install_plugins': True} + ) + else: + task = await api( + 'POST', + '/plugins/install/marketplace', + {'plugin_author': 'langbot-team', 'plugin_name': 'LocalAgent', 'plugin_version': '0.2.1'}, + ) + start = time.monotonic() + while time.monotonic() - start < 120: + current = await api('GET', '/system/tasks/' + str(task['task_id'])) + result['task'] = { + 'runtime': {k: v for k, v in current.get('runtime', {}).items() if k != 'exception_traceback'}, + 'context': {k: v for k, v in current.get('context', {}).items() if k != 'log'}, + } + if current.get('runtime', {}).get('done'): + break + await asyncio.sleep(1) + result['seconds'] = round(time.monotonic() - start, 2) + result['plugins'] = await api('GET', '/plugins') + except Exception as exc: + result['probe_error'] = str(exc) + finally: + Path('install-result.json').write_text(json.dumps(result, ensure_ascii=False, indent=2)) + + +asyncio.run(main()) diff --git a/tests/manual/lang16_llm.py b/tests/manual/lang16_llm.py new file mode 100644 index 000000000..150328fce --- /dev/null +++ b/tests/manual/lang16_llm.py @@ -0,0 +1,31 @@ +"""Bounded live model probe; credentials enter via hidden terminal input only.""" + +import getpass +import os +from pathlib import Path +import subprocess +import sys +import httpx + +base_url = os.environ['LANG16_LLM_BASE_URL'].rstrip('/') +key = getpass.getpass('Test API key (hidden): ') +with httpx.Client(timeout=20) as client: + response = client.get(base_url + '/models', headers={'Authorization': 'Bearer ' + key}) + if response.status_code != 200: + raise SystemExit('Model listing HTTP ' + str(response.status_code)) + names = {m['id'] for m in response.json()['data']} +selected = next((m for m in ('deepseek-chat', 'gpt-4o-mini', 'gpt-4.1-mini', 'gpt-4.1-nano') if m in names), None) +if not selected: + print('Available model IDs:', ', '.join(sorted(names)), flush=True) + raise SystemExit('No eligible small test model available') +print('Selected test model:', selected, flush=True) +env = dict( + os.environ, + LANG16_LLM_KEY=key, + LANG16_LLM_MODEL=selected, + LANG16_ONEBOT='1', + LANG16_RUNNER='1', + LANG16_BOX='1', + LANG16_SECONDS='120', +) +raise SystemExit(subprocess.call([sys.executable, str(Path(__file__).with_name('lang16_baseline.py'))], env=env)) diff --git a/tests/manual/lang16_matrix.py b/tests/manual/lang16_matrix.py new file mode 100644 index 000000000..fec7b2280 --- /dev/null +++ b/tests/manual/lang16_matrix.py @@ -0,0 +1,301 @@ +"""Real dedicated-worker faults and bounded OneBot load with synthetic QA runners.""" + +import asyncio +import io +import json +import os +from pathlib import Path +import secrets +import sqlite3 +import time +import zipfile + +import httpx +import websockets +import yaml + +RUNNER = """import asyncio, os, time +from langbot_plugin.api.definition.components.runner.runner import Runner +from langbot_plugin.api.entities.builtin.provider.message import Message +from langbot_plugin.api.entities.builtin.runner import RunnerResult +class Probe(Runner): + async def run(self, ctx): + text = ctx.input.to_text() + if 'RAISE' in text: raise RuntimeError('LANG16 handler fault') + if 'EXIT' in text: os._exit(17) + if 'BLOCK' in text: + until = time.monotonic() + 3 + while time.monotonic() < until: pass + if 'TIMEOUT' in text: await asyncio.wait_for(asyncio.sleep(3), 0.2) + if 'SLOW' in text: await asyncio.sleep(0.1) + if 'WAIT' in text: await asyncio.sleep(60) + yield RunnerResult.message_completed(ctx.run_id, Message(role='assistant', content='LANG16_HEALTHY')) + yield RunnerResult.run_completed(ctx.run_id) +""" + + +def package(name, init_failure=False, dependency_failure=False): + manifest = { + 'apiVersion': 'v1', + 'kind': 'Plugin', + 'metadata': { + 'author': 'lang16-qa', + 'name': name, + 'version': '0.0.1', + 'label': {'en_US': name}, + 'description': {'en_US': 'Isolated synthetic QA fixture'}, + 'icon': 'assets/icon.svg', + }, + 'spec': {'config': [], 'components': {'Runner': {'fromDirs': [{'path': 'components/runner/'}]}}}, + 'execution': {'python': {'path': 'main.py', 'attr': 'ProbePlugin'}}, + } + component = { + 'apiVersion': 'langbot/v1', + 'kind': 'Runner', + 'metadata': {'name': 'default', 'label': {'en_US': 'Probe'}, 'description': {'en_US': 'No model calls'}}, + 'spec': { + 'usages': ['agent'], + 'config': [], + 'capabilities': {'streaming': True, 'tool_calling': False}, + 'permissions': {}, + }, + 'execution': {'python': {'path': 'default.py', 'attr': 'Probe'}}, + } + main = 'from langbot_plugin.api.definition.plugin import BasePlugin\nclass ProbePlugin(BasePlugin):\n async def initialize(self):\n ' + main += "raise RuntimeError('LANG16 initialization fault')\n" if init_failure else 'pass\n' + stream = io.BytesIO() + with zipfile.ZipFile(stream, 'w') as z: + for path, data in { + 'manifest.yaml': yaml.safe_dump(manifest), + 'main.py': main, + 'components/runner/default.yaml': yaml.safe_dump(component), + 'components/runner/default.py': RUNNER, + 'requirements.txt': 'not a valid requirement === !!!' if dependency_failure else '', + 'assets/icon.svg': '', + }.items(): + z.writestr(path, data) + return stream.getvalue() + + +async def main(): + results = { + 'synthetic_plugins': True, + 'stages': [], + 'sent': 0, + 'explicit_plugin_bindings': os.environ.get('LANG16_BOUND') == '1', + } + async with httpx.AsyncClient(base_url='http://127.0.0.1:15399', timeout=15, trust_env=False) as client: + + async def api(method, path, data=None): + r = await client.request(method, '/api/v1' + path, json=data) + b = r.json() + if r.status_code >= 400 or b.get('code') != 0: + raise RuntimeError(f'{path}: HTTP {r.status_code}: {b.get("msg")}') + return b.get('data', {}) + + async def snapshot(label): + health = (await client.get('/healthz')).json() + list_started = time.monotonic() + try: + plugin_list = await api('GET', '/plugins') + listing = {'ok': True, 'count': len(plugin_list.get('plugins', []))} + except Exception as exc: + listing = {'ok': False, 'error_type': type(exc).__name__} + listing['seconds'] = round(time.monotonic() - list_started, 3) + with sqlite3.connect('data/langbot.db') as db: + states = list(db.execute('select role,status,count(*) from monitoring_messages group by role,status')) + active = await api('GET', '/monitoring/executions?status=running&source=pipeline') + results['stages'].append( + { + 'label': label, + 'monotonic': time.monotonic(), + 'health': health, + 'message_states': states, + 'plugin_list': listing, + 'api_running_queries': active['total'], + } + ) + Path('matrix-result.json').write_text(json.dumps(results, indent=2)) + + async def install(name, **kwargs): + response = await client.post( + '/api/v1/plugins/install/local', files={'file': (name + '.lbpkg', package(name, **kwargs))} + ) + task = response.json()['data']['task_id'] + for _ in range(45): + state = (await api('GET', '/system/tasks/' + str(task)))['runtime'] + if state['done']: + return {'name': name, 'done': True, 'failed': bool(state.get('exception'))} + await asyncio.sleep(1) + return {'name': name, 'done': False} + + account = {'user': 'qa-matrix@example.invalid', 'password': secrets.token_urlsafe(24)} + await api('POST', '/user/init', account) + client.headers['Authorization'] = 'Bearer ' + (await api('POST', '/user/auth', account))['token'] + results['installations'] = [await install('HealthyProbe'), await install('FaultProbe')] + + async def bot(name, port): + pipeline = (await api('POST', '/pipelines', {'name': name, 'description': 'QA fixture'}))['uuid'] + config = (await api('GET', '/pipelines/' + pipeline))['pipeline']['config'] + config['trigger']['group-respond-rules'] = {'at': True, 'prefix': ['qa:'], 'regexp': [], 'random': 0} + runner = 'plugin:lang16-qa/' + name + '/default' + config['ai'] = {'runner': {'id': runner, 'expire-time': 0}, 'runner_config': {runner: {}}} + payload = {'config': config} + if os.environ.get('LANG16_BOUND') == '1': + payload['extensions_preferences'] = { + 'enable_all_plugins': False, + 'enable_all_mcp_servers': True, + 'plugins': [{'author': 'lang16-qa', 'name': name}], + 'mcp_servers': [], + } + await api('PUT', '/pipelines/' + pipeline, payload) + b = await api( + 'POST', + '/platform/bots', + { + 'name': name, + 'description': 'QA', + 'adapter': 'aiocqhttp', + 'enable': True, + 'adapter_config': {'host': '127.0.0.1', 'port': port, 'access-token': ''}, + 'event_bindings': [ + { + 'id': 'default', + 'event_pattern': 'message.received', + 'target_type': 'pipeline', + 'target_uuid': pipeline, + 'filters': [], + 'enabled': True, + } + ], + 'plugin_processors': [], + }, + ) + await api('PUT', '/platform/bots/' + b['uuid'], {'enable': True}) + + await bot('HealthyProbe', 12280) + await bot('FaultProbe', 12281) + await asyncio.sleep(1) + async with ( + websockets.connect( + 'ws://127.0.0.1:12280/ws/', additional_headers={'X-Self-ID': '910001', 'X-Client-Role': 'Universal'} + ) as healthy, + websockets.connect( + 'ws://127.0.0.1:12281/ws/', additional_headers={'X-Self-ID': '910001', 'X-Client-Role': 'Universal'} + ) as fault, + ): + replies = [] + + async def responder(ws, owner): + async for raw in ws: + r = json.loads(raw) + if r.get('action') == 'send_msg': + replies.append( + {'owner': owner, 'time': time.monotonic(), 'message': r.get('params', {}).get('message')} + ) + await ws.send( + json.dumps({'status': 'ok', 'retcode': 0, 'data': {'message_id': 1}, 'echo': r.get('echo')}) + ) + + responders = [ + asyncio.create_task(responder(healthy, 'healthy')), + asyncio.create_task(responder(fault, 'fault')), + ] + + async def send(ws, text, group=910100): + results['sent'] += 1 + await ws.send( + json.dumps( + { + 'time': int(time.time()), + 'self_id': 910001, + 'post_type': 'message', + 'message_type': 'group', + 'sub_type': 'normal', + 'message_id': results['sent'], + 'group_id': group, + 'user_id': 910002, + 'message': text, + 'raw_message': text, + 'sender': {'user_id': 910002, 'nickname': 'QA', 'role': 'member'}, + } + ) + ) + + try: + await send(healthy, 'qa: HEALTHY') + await asyncio.sleep(2) + await snapshot('healthy_baseline') + assert any('LANG16_HEALTHY' in str(r['message']) for r in replies), 'Healthy runner did not execute' + if os.environ.get('LANG16_CANCEL') == '1': + await send(healthy, 'qa: WAIT', 910500) + await asyncio.sleep(1) + await snapshot('before_cancel') + Path('cancel-ready').touch() + return + for kind in ('RAISE', 'TIMEOUT', 'BLOCK'): + results.setdefault('fault_started', {})[kind] = time.monotonic() + await send(fault, 'qa: ' + kind) + await asyncio.sleep(0.3) + await send(healthy, 'qa: HEALTHY control', 910101) + await asyncio.sleep(5) + await snapshot(kind) + await send(fault, 'qa: SLOW', 910400) + await asyncio.sleep(0.05) + await fault.close() + await asyncio.sleep(6) + await snapshot('onebot_disconnect') + async with websockets.connect( + 'ws://127.0.0.1:12281/ws/', + additional_headers={'X-Self-ID': '910001', 'X-Client-Role': 'Universal'}, + ) as reconnected: + responders.append(asyncio.create_task(responder(reconnected, 'fault_reconnected'))) + await send(reconnected, 'qa: HEALTHY reconnect', 910401) + await asyncio.sleep(2) + await snapshot('onebot_reconnected') + results.setdefault('fault_started', {})['EXIT'] = time.monotonic() + await send(reconnected, 'qa: EXIT', 910402) + await asyncio.sleep(0.3) + await send(healthy, 'qa: HEALTHY control', 910101) + await asyncio.sleep(5) + await snapshot('EXIT') + if os.environ.get('LANG16_BOUND') == '1': + results['complete'] = True + results['replies'] = replies + Path('matrix-result.json').write_text(json.dumps(results, indent=2)) + return + results['installations'] += [ + await install('InitFailure', init_failure=True), + await install('DependencyFailure', dependency_failure=True), + ] + await send(healthy, 'qa: HEALTHY after installs') + await asyncio.sleep(3) + await snapshot('installation_faults') + for _ in range(160): + await send(healthy, 'qa: SLOW', 910200) + await asyncio.sleep(1) + await snapshot('burst_peak') + await asyncio.sleep(25) + await snapshot('burst_drained') + for i in range(int(os.environ.get('LANG16_SOAK_ROUNDS', '6'))): + for _ in range(50): + await send(healthy, 'ordinary unrelated chat', 910300) + await send(healthy, 'qa: HEALTHY soak', 910301) + await asyncio.sleep(10) + await snapshot('soak_' + str(i)) + await asyncio.sleep(20) + await snapshot('idle_recovery') + results['complete'] = True + results['replies'] = replies + Path('matrix-result.json').write_text(json.dumps(results, indent=2)) + finally: + for task in responders: + task.cancel() + await asyncio.gather(*responders, return_exceptions=True) + Path('matrix-finished').touch() + + +try: + asyncio.run(main()) +finally: + Path('matrix-finished').touch() diff --git a/tests/manual/lang16_migration_ui.mjs b/tests/manual/lang16_migration_ui.mjs new file mode 100644 index 000000000..16e65a102 --- /dev/null +++ b/tests/manual/lang16_migration_ui.mjs @@ -0,0 +1,81 @@ +// Actual migration UI against the isolated Core and synthetic HTTP 503 marketplace. +import { createRequire } from "node:module"; +import { writeFile } from "node:fs/promises"; +const require = createRequire( + new URL("../../web/package.json", import.meta.url), +); +const { chromium } = require("@playwright/test"); +const browser = await chromium.launch({ + channel: "msedge", + headless: true, + args: ["--renderer-process-limit=2", "--disable-gpu"], +}); +const result = { errors: [] }; +let page; +try { + page = await browser.newPage({ + viewport: { width: 1280, height: 900 }, + locale: "en-US", + }); + page.on("pageerror", (error) => result.errors.push(error.message)); + await page.addInitScript((token) => { + localStorage.setItem("token", token); + localStorage.setItem("langbot_language", "en-US"); + localStorage.setItem("userEmail", "qa-network@example.invalid"); + }, process.env.LANG16_UI_TOKEN); + await page.goto("http://127.0.0.1:13000/home/pipelines", { + waitUntil: "domcontentloaded", + timeout: 20000, + }); + await page + .getByRole("button", { name: /^(Skip|跳过|Review migration|检查迁移)$/ }) + .first() + .waitFor({ timeout: 20000 }); + if (page.url().includes("/wizard")) { + await page.getByRole("button", { name: /^(Skip|跳过)$/ }).click(); + await page + .getByRole("dialog") + .getByRole("button", { name: /^(OK|确定)$/ }) + .click(); + await page.waitForURL((url) => !url.pathname.includes("/wizard")); + await page.goto("http://127.0.0.1:13000/home/pipelines", { + waitUntil: "domcontentloaded", + }); + } + const tourSkip = page.locator('[data-guided-tour-action="skip"]'); + try { + await tourSkip.waitFor({ state: "visible", timeout: 10000 }); + await tourSkip.click(); + } catch {} + await page + .getByRole("button", { name: /Review migration|检查迁移/ }) + .click({ timeout: 15000 }); + await page + .getByRole("button", { + name: /Install plugins and migrate|自动安装插件并迁移/, + }) + .click(); + await page + .getByText(/marketplace could not serve|插件市场暂时无法/) + .first() + .waitFor({ timeout: 15000 }); + await page + .getByRole("button", { name: /Installation details|安装详情/ }) + .click(); + await page.getByText(/http_status.*HTTP 503/).waitFor({ timeout: 5000 }); + result.body = await page.locator("body").innerText(); + result.diagnosticVisible = + result.body.includes("HTTP 503") && result.body.includes("http_status"); + await page.screenshot({ path: "migration-error.png", fullPage: true }); +} catch (error) { + result.failure = error.message; + if (page) { + result.url = page.url(); + result.body = await page.locator("body").innerText(); + await page.screenshot({ path: "migration-failure.png", fullPage: true }); + } + throw error; +} finally { + await writeFile("migration-ui.json", JSON.stringify(result, null, 2)); + await browser.close(); +} diff --git a/tests/manual/lang16_network.py b/tests/manual/lang16_network.py new file mode 100644 index 000000000..f5f8ad5df --- /dev/null +++ b/tests/manual/lang16_network.py @@ -0,0 +1,120 @@ +"""Real migration/extension API failures against a synthetic local marketplace.""" + +import asyncio +import json +import os +from pathlib import Path +import secrets +import sqlite3 +import subprocess +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import httpx + +mode = 'http503' + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + if mode == 'timeout': + time.sleep(17) + if mode == 'disconnect': + self.connection.close() + return + self.send_response(503 if mode == 'http503' else 200) + self.send_header('Content-Length', '9999' if mode == 'truncated' else '0') + self.end_headers() + if mode == 'truncated': + self.wfile.write(b'incomplete package') + self.wfile.flush() + self.close_connection = True + + +async def main(): + global mode + server = ThreadingHTTPServer(('127.0.0.1', 15420), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + results = [] + try: + async with httpx.AsyncClient(base_url='http://127.0.0.1:15399', timeout=20, trust_env=False) as client: + + async def api(method, path, data=None): + r = await client.request(method, '/api/v1' + path, json=data) + b = r.json() + if r.status_code >= 400 or b.get('code') != 0: + raise RuntimeError(f'{path}: HTTP {r.status_code}') + return b.get('data', {}) + + account = {'user': 'qa-network@example.invalid', 'password': secrets.token_urlsafe(24)} + await api('POST', '/user/init', account) + token = (await api('POST', '/user/auth', account))['token'] + client.headers['Authorization'] = 'Bearer ' + token + pipeline = ( + await api('POST', '/pipelines', {'name': 'Network failure probe', 'description': 'isolated QA'}) + )['uuid'] + old = subprocess.check_output( + ['git', 'show', 'v4.10.10:src/langbot/templates/default-pipeline-config.json'], + cwd=Path(__file__).resolve().parents[2], + ).decode() + with sqlite3.connect('data/langbot.db') as db: + db.execute( + 'update legacy_pipelines set config=?,for_version=? where uuid=?', (old, '4.10.10', pipeline) + ) + ui = await asyncio.create_subprocess_exec( + 'node', + str(Path(__file__).with_name('lang16_migration_ui.mjs')), + env=dict(os.environ, LANG16_UI_TOKEN=token), + ) + results.append({'ui_exit_code': await ui.wait()}) + if os.environ.get('LANG16_NETWORK_UI_ONLY') == '1': + Path('network-result.json').write_text(json.dumps(results, indent=2)) + return + for scenario in ('http503', 'disconnect', 'truncated', 'timeout', 'offline'): + mode = scenario + if scenario == 'offline': + server.shutdown() + server.server_close() + for route in ('migration', 'extension'): + start = time.monotonic() + if route == 'migration': + task = await api( + 'POST', + '/pipelines/_/migration/execute', + {'confirmed': True, 'all': True, 'install_plugins': True}, + ) + else: + task = await api( + 'POST', + '/plugins/install/marketplace', + {'plugin_author': 'langbot-team', 'plugin_name': 'LocalAgent', 'plugin_version': '0.2.1'}, + ) + for _ in range(45): + state = await api('GET', '/system/tasks/' + str(task['task_id'])) + if state['runtime']['done']: + break + await asyncio.sleep(1) + runtime = state['runtime'] + results.append( + { + 'scenario': scenario, + 'route': route, + 'seconds': round(time.monotonic() - start, 2), + 'done': runtime['done'], + 'state': runtime['state'], + 'exception_type': str(runtime.get('exception', '')).split(':')[0][:150], + 'result': runtime.get('result'), + } + ) + Path('network-result.json').write_text(json.dumps(results, indent=2)) + finally: + server.shutdown() + server.server_close() + Path('network-finished').touch() + + +asyncio.run(main()) diff --git a/tests/manual/lang16_onebot.py b/tests/manual/lang16_onebot.py new file mode 100644 index 000000000..758bfbd89 --- /dev/null +++ b/tests/manual/lang16_onebot.py @@ -0,0 +1,224 @@ +"""Real isolated Core/OneBot ingress probe with a synthetic protocol peer, no QQ traffic.""" + +import asyncio +import os +import json +from pathlib import Path +import secrets +import sqlite3 +import time +import sys + +import httpx +import websockets + + +async def main(): + async with httpx.AsyncClient(base_url='http://127.0.0.1:15399', timeout=15, trust_env=False) as client: + + async def api(method, path, data=None): + response = await client.request(method, '/api/v1' + path, json=data) + body = response.json() + if response.status_code >= 400 or body.get('code') != 0: + raise RuntimeError(f'{method} {path}: {response.status_code} {body.get("msg")}') + return body.get('data', {}) + + account = {'user': 'lang16@example.invalid', 'password': secrets.token_urlsafe(24)} + await api('POST', '/user/init', account) + auth = await api('POST', '/user/auth', account) + client.headers['Authorization'] = 'Bearer ' + auth['token'] + if '--installed' in sys.argv: + if os.environ.get('LANG16_PLUGIN_PACKAGE'): + package = Path(os.environ['LANG16_PLUGIN_PACKAGE']) + response = await client.post( + '/api/v1/plugins/install/local', files={'file': (package.name, package.read_bytes())} + ) + response.raise_for_status() + payload = response.json() + if payload.get('code') != 0: + raise RuntimeError('Local plugin upload failed') + task = payload['data'] + else: + task = await api( + 'POST', + '/plugins/install/marketplace', + {'plugin_author': 'langbot-team', 'plugin_name': 'LocalAgent', 'plugin_version': '0.2.1'}, + ) + for _ in range(60): + state = (await api('GET', '/system/tasks/' + str(task['task_id'])))['runtime'] + if state.get('done'): + if state.get('exception'): + raise RuntimeError('LocalAgent installation failed') + break + await asyncio.sleep(1) + else: + raise RuntimeError('LocalAgent installation deadline') + pipeline = (await api('POST', '/pipelines', {'name': 'LANG16 response gate', 'description': 'isolated probe'}))[ + 'uuid' + ] + config = (await api('GET', '/pipelines/' + pipeline))['pipeline']['config'] + config['trigger']['group-respond-rules'] = {'at': True, 'prefix': ['!qa'], 'regexp': []} + if '--fault' not in sys.argv: + config['trigger']['group-respond-rules']['random'] = 0 + if '--installed' in sys.argv: + config['ai']['runner']['id'] = 'plugin:langbot-team/LocalAgent/default' + # Exercise real Runner failure/cleanup without models or sandboxes. + config['ai']['runner_config'] = { + 'plugin:langbot-team/LocalAgent/default': {'box-enabled': '--box' in sys.argv} + } + if os.environ.get('LANG16_LLM_KEY'): + model = await api( + 'POST', + '/provider/models/llm', + { + 'name': os.environ['LANG16_LLM_MODEL'], + 'abilities': ['func_call'], + 'extra_args': {'max_tokens': 128}, + 'provider': { + 'requester': 'openai-chat-completions', + 'base_url': os.environ['LANG16_LLM_BASE_URL'], + 'api_keys': [os.environ['LANG16_LLM_KEY']], + }, + }, + ) + runner = config['ai']['runner_config']['plugin:langbot-team/LocalAgent/default'] + runner.update( + { + 'model': {'primary': model['uuid'], 'fallbacks': [], 'reasoning': {}}, + 'enable-all-tools': False, + 'tools': ['exec'], + 'max-tool-iterations': 2, + } + ) + await api('PUT', '/pipelines/' + pipeline, {'config': config}) + bot = await api( + 'POST', + '/platform/bots', + { + 'name': 'LANG16 OneBot', + 'description': 'synthetic protocol peer only', + 'adapter': 'aiocqhttp', + 'enable': True, + 'adapter_config': {'host': '127.0.0.1', 'port': 12280, 'access-token': ''}, + 'event_bindings': [ + { + 'id': 'default-migrated', + 'event_pattern': 'message.received', + 'target_type': 'pipeline', + 'target_uuid': pipeline, + 'filters': [], + 'enabled': True, + 'description': 'Migrated from default pipeline binding', + } + ], + 'plugin_processors': [], + }, + ) + # The release create API loads but does not run the adapter; applying + # enable through the update API starts it without restarting Core. + await api('PUT', '/platform/bots/' + bot['uuid'], {'enable': True}) + await asyncio.sleep(1) + async with websockets.connect( + 'ws://127.0.0.1:12280/ws/', + additional_headers={ + 'X-Self-ID': '900001', + 'X-Client-Role': 'Universal', + }, + ) as ws: + actions = [] + box_snapshots = ( + [{'case': 'before', 'sessions': await api('GET', '/box/sessions')}] if '--box' in sys.argv else [] + ) + + async def respond(): + async for raw in ws: + request = json.loads(raw) + actions.append(request.get('action')) + data = {'message_id': 1} + if request.get('action') == 'get_login_info': + data = {'user_id': 900001, 'nickname': 'LANG16'} + await ws.send(json.dumps({'status': 'ok', 'retcode': 0, 'data': data, 'echo': request.get('echo')})) + + responder = asyncio.create_task(respond()) + cases = [('unrelated', 'ordinary chat'), ('keyword', '!qa hello'), ('mention', '[CQ:at,qq=900001]hello')] + if '--box' in sys.argv: + cases = [('unrelated', 'ordinary chat'), ('keyword', '!qa Reply exactly QA_OK without using tools.')] + if os.environ.get('LANG16_LLM_KEY'): + cases.append(('tool', '!qa Use exec to run printf LANG16_TOOL_OK and report its output.')) + if '--fault' not in sys.argv and '--box' not in sys.argv: + cases += [('other_mention', '[CQ:at,qq=900003]hello'), ('private', 'hello')] + try: + for i, (name, content) in enumerate(cases): + await ws.send( + json.dumps( + { + 'time': int(time.time()), + 'self_id': 900001, + 'post_type': 'message', + 'message_type': 'private' if name == 'private' else 'group', + 'sub_type': 'friend' if name == 'private' else 'normal', + 'message_id': 100 + i, + 'group_id': 900100 if '--box' in sys.argv else 900100 + i, + 'user_id': 900002, + 'message': content, + 'raw_message': content, + 'sender': {'user_id': 900002, 'nickname': 'QA', 'card': '', 'role': 'member'}, + } + ) + ) + await asyncio.sleep(20 if os.environ.get('LANG16_LLM_KEY') else (8 if '--box' in sys.argv else 2)) + if '--box' in sys.argv: + box_snapshots.append({'case': name, 'sessions': await api('GET', '/box/sessions')}) + if '--fault' not in sys.argv and '--box' not in sys.argv: + for i in range(30): + await ws.send( + json.dumps( + { + 'time': int(time.time()), + 'self_id': 900001, + 'post_type': 'message', + 'message_type': 'group', + 'sub_type': 'normal', + 'message_id': 200 + i, + 'group_id': 900200, + 'user_id': 900002, + 'message': 'unrelated burst', + 'raw_message': 'unrelated burst', + 'sender': {'user_id': 900002, 'nickname': 'QA', 'role': 'member'}, + } + ) + ) + await asyncio.sleep(4) + finally: + responder.cancel() + try: + await responder + except asyncio.CancelledError: + pass + health = (await client.get('/healthz')).json() + with sqlite3.connect('data/langbot.db') as db: + tables = [row[0] for row in db.execute("select name from sqlite_master where type='table'")] + monitoring = {} + for table in tables: + if table in ('monitoring_messages', 'monitoring_sessions'): + db.row_factory = sqlite3.Row + monitoring[table] = [dict(row) for row in db.execute('select * from ' + table)] + Path('onebot-result.json').write_text( + json.dumps( + { + 'cases': cases, + 'box_snapshots': box_snapshots, + 'live_model': os.environ.get('LANG16_LLM_MODEL'), + 'installed_runner_no_model': '--installed' in sys.argv, + 'actions': actions, + 'health': health, + 'monitoring': monitoring, + 'pipeline': pipeline, + 'bot': bot, + }, + indent=2, + ) + ) + + +asyncio.run(main()) diff --git a/tests/manual/lang16_ui.mjs b/tests/manual/lang16_ui.mjs new file mode 100644 index 000000000..dfd430d04 --- /dev/null +++ b/tests/manual/lang16_ui.mjs @@ -0,0 +1,36 @@ +// Real local UI navigation, without touching an existing browser profile. +import { createRequire } from "node:module"; +import { writeFile } from "node:fs/promises"; +const require = createRequire( + new URL("../../web/package.json", import.meta.url), +); +const { chromium } = require("@playwright/test"); +const browser = await chromium.launch({ channel: "msedge", headless: true }); +try { + const page = await browser.newPage({ + viewport: { width: 1280, height: 900 }, + }); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.goto("http://127.0.0.1:13000/login", { + waitUntil: "networkidle", + timeout: 20000, + }); + await page.screenshot({ path: "login.png", fullPage: true }); + const login = await page.locator("body").innerText(); + await page.goto("http://127.0.0.1:13000/register", { + waitUntil: "networkidle", + timeout: 20000, + }); + await page.screenshot({ path: "register.png", fullPage: true }); + await writeFile( + "ui.json", + JSON.stringify( + { login, register: await page.locator("body").innerText(), errors }, + null, + 2, + ), + ); +} finally { + await browser.close(); +} diff --git a/tests/manual/lang16_upgrade.py b/tests/manual/lang16_upgrade.py new file mode 100644 index 000000000..1c5e11e8f --- /dev/null +++ b/tests/manual/lang16_upgrade.py @@ -0,0 +1,109 @@ +"""Seed an actual 4.10.10 database, then validate its 4.11 upgrade through APIs.""" + +import asyncio +import json +import os +from pathlib import Path +import secrets +import sqlite3 +import time + +import httpx +import websockets + + +async def main(): + result = {'mode': os.environ['LANG16_UPGRADE']} + async with httpx.AsyncClient(base_url='http://127.0.0.1:15399', timeout=20, trust_env=False) as client: + + async def api(method, path, data=None): + r = await client.request(method, '/api/v1' + path, json=data) + b = r.json() + if r.status_code >= 400 or b.get('code') != 0: + raise RuntimeError(f'{path}: HTTP {r.status_code}: {b.get("msg")}') + return b.get('data', {}) + + if result['mode'] == 'old': + account = {'user': 'qa-upgrade@example.invalid', 'password': secrets.token_urlsafe(24)} + await api('POST', '/user/init', account) + Path('upgrade-account.json').write_text(json.dumps(account)) + Path('upgrade-account.json').chmod(0o600) + else: + account = json.loads(Path(os.environ['LANG16_UPGRADE_ACCOUNT']).read_text()) + client.headers['Authorization'] = 'Bearer ' + (await api('POST', '/user/auth', account))['token'] + if result['mode'] == 'old': + pipeline = (await api('POST', '/pipelines', {'name': 'Real 4.10.10 seed', 'description': 'QA upgrade'}))[ + 'uuid' + ] + bot = await api( + 'POST', + '/platform/bots', + { + 'name': 'Upgrade OneBot', + 'description': 'QA', + 'adapter': 'aiocqhttp', + 'enable': False, + 'adapter_config': {'host': '127.0.0.1', 'port': 12280, 'access-token': ''}, + 'use_pipeline_uuid': pipeline, + }, + ) + result.update(pipeline=pipeline, bot=bot['uuid']) + else: + preview = await api('GET', '/pipelines/_/migration/preview') + result['preview'] = [{k: v for k, v in x.items() if k != 'preview_token'} for x in preview['items']] + task = await api( + 'POST', '/pipelines/_/migration/execute', {'confirmed': True, 'all': True, 'install_plugins': True} + ) + for _ in range(60): + state = (await api('GET', '/system/tasks/' + str(task['task_id'])))['runtime'] + if state['done']: + break + await asyncio.sleep(1) + result['task'] = {k: v for k, v in state.items() if k != 'exception_traceback'} + bots = (await api('GET', '/platform/bots'))['bots'] + result['bindings'] = [b.get('event_bindings') for b in bots] + await api('PUT', '/platform/bots/' + bots[0]['uuid'], {'enable': True}) + await asyncio.sleep(1) + async with websockets.connect( + 'ws://127.0.0.1:12280/ws/', additional_headers={'X-Self-ID': '920001', 'X-Client-Role': 'Universal'} + ) as ws: + + async def respond(): + async for raw in ws: + r = json.loads(raw) + await ws.send( + json.dumps({'status': 'ok', 'retcode': 0, 'data': {'message_id': 1}, 'echo': r.get('echo')}) + ) + + responder = asyncio.create_task(respond()) + try: + for i, text in enumerate(['unrelated chat', '[CQ:at,qq=920001]hello']): + await ws.send( + json.dumps( + { + 'time': int(time.time()), + 'self_id': 920001, + 'post_type': 'message', + 'message_type': 'group', + 'sub_type': 'normal', + 'message_id': i + 1, + 'group_id': 920100, + 'user_id': 920002, + 'message': text, + 'raw_message': text, + 'sender': {'user_id': 920002, 'nickname': 'QA', 'role': 'member'}, + } + ) + ) + await asyncio.sleep(3) + finally: + responder.cancel() + await asyncio.gather(responder, return_exceptions=True) + with sqlite3.connect('data/langbot.db') as db: + result['alembic_revision'] = list(db.execute('select version_num from alembic_version')) + result['messages'] = list(db.execute('select role,status,message_content from monitoring_messages')) + result['health'] = (await client.get('/healthz')).json() + Path('upgrade-result.json').write_text(json.dumps(result, indent=2)) + + +asyncio.run(main())