feat(agent-runner): enforce 4.x host-owned execution

2026-07-25 05:46:13 +00:00 · 2026-07-12 20:36:32 +08:00
parent e6384aae5d
commit 99d9c227f9
171 changed files with 6958 additions and 5385 deletions
@@ -522,7 +522,6 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) {
    prompt: [{ role: "system", content: "You are a deterministic QA assistant. Reply exactly as instructed." }],
    "remove-think": false,
    "knowledge-bases": [],
-    "box-session-id-template": "{launcher_type}_{launcher_id}",
    "retrieval-top-k": 5,
    "rerank-model": "",
    "rerank-top-k": 5,
@@ -112,7 +112,9 @@ function parseJsonEnv(key, fallback) {
 }

 function positiveNumberEnv(key, fallback) {
-  const value = Number(env[key] || "");
+  const raw = env[key];
+  if (raw === undefined || raw === "") return fallback;
+  const value = Number(raw);
  return Number.isFinite(value) && value >= 0 ? value : fallback;
 }

@@ -17,7 +17,7 @@ env:
 automation: skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs
 steps:
  - "Run `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run` first; remove `--dry-run` after checking the planned evidence directory."
-  - "Automation resolves LANGBOT_REPO, defaulting to ../LangBot, and imports the sibling SDK from LANGBOT_PLUGIN_SDK_REPO or ../langbot-plugin-sdk/src."
+  - "Automation resolves LANGBOT_REPO, defaulting to the parent LangBot checkout, and imports the sibling SDK from LANGBOT_PLUGIN_SDK_REPO or ../../langbot-plugin-sdk/src."
  - "Automation checks run status sets, terminal status validation, ledger table/index DDL, and a minimal synchronous insert/read path."
 checks:
  - "automation-result.json status is pass."
@@ -33,6 +33,7 @@ automation_expected_runner_id: "plugin:qa/agent-runner/default"
 automation_prompt: "hello-live"
 automation_expected_text: "QA_AGENT_RUNNER_OK:hello-live"
 automation_response_timeout_ms: "120000"
+automation_debug_chat_response_p95_ms: "120000"
 automation_reset_debug_chat: "1"
 setup_automation:
  - "case:agent-runner-live-install"
@@ -18,7 +18,7 @@ env:
 automation: skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs
 steps:
  - "Run `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run` first; remove `--dry-run` after checking the SDK repo target and evidence directory."
-  - "Automation resolves LANGBOT_PLUGIN_SDK_REPO, defaulting to ../langbot-plugin-sdk when the env var is unset."
+  - "Automation resolves LANGBOT_PLUGIN_SDK_REPO, defaulting to ../../langbot-plugin-sdk when the env var is unset."
  - "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_agent_runner.py and tests/runtime/test_pull_api_handlers.py."
 checks:
  - "automation-result.json status is pass."
@@ -28,7 +28,7 @@ evidence_required:
  - filesystem
 diagnostics:
  - "This probe does not open the WebUI; it runs SDK pytest targets directly."
-  - "env_issue means LANGBOT_PLUGIN_SDK_REPO/default ../langbot-plugin-sdk did not resolve, rtk/uv was unavailable, or the expected test files are missing."
+  - "env_issue means LANGBOT_PLUGIN_SDK_REPO/default ../../langbot-plugin-sdk did not resolve, rtk/uv was unavailable, or the expected test files are missing."
  - "fail means the existing SDK runtime pytest target failed or timed out."
 success_patterns:
  - "pytest passed"
@@ -37,7 +37,7 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":650,"context-reserve-tokens":180,"context-keep-recent-tokens":120,"context-summary-tokens":220,"max-tool-iterations":4,"tool-execution-mode":"serial"}'
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_runner_config: "1"
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
@@ -33,7 +33,7 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_runner_config_patch_json: '{"context-window-tokens":225,"context-reserve-tokens":50,"context-keep-recent-tokens":30,"context-summary-tokens":105,"knowledge-bases":[]}'
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_runner_config: "1"
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
@@ -38,7 +38,7 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":5,"tool-execution-mode":"serial"}'
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_runner_config: "1"
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
@@ -38,7 +38,7 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":3,"tool-execution-mode":"parallel"}'
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_runner_config: "1"
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
@@ -34,7 +34,7 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":3,"tool-execution-mode":"serial"}'
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_runner_config: "1"
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
@@ -35,7 +35,7 @@ automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
 automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":2,"tool-execution-mode":"serial"}'
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_runner_config: "1"
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
@@ -32,7 +32,7 @@ automation_env:
 automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
 automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
 automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default"
-automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot","name":"local-agent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}'
+automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}'
 automation_restore_extensions: "1"
 automation_reset_debug_chat: "1"
 automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result."
@@ -80,13 +80,17 @@ async function main() {
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });
  const startedAt = new Date();
-  const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot");
+  const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
  const stdoutLog = join(evidenceDir, "probe-stdout.log");
  const stderrLog = join(evidenceDir, "probe-stderr.log");
  const automationResultJson = join(evidenceDir, "automation-result.json");
  const resultJson = join(evidenceDir, "result.json");
  const timeoutMs = Number(env.LANGBOT_ASYNC_DB_READINESS_TIMEOUT_MS || "5000");
-  const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script], cwd: langbotRepo };
+  const command = {
+    executable: "rtk",
+    args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script],
+    cwd: langbotRepo,
+  };
  const result = {
    source: "automation",
    probe: "aiosqlite-readiness",
@@ -142,15 +142,20 @@ async function main() {
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });
  const startedAt = new Date();
-  const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot");
-  const sdkSrc = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src");
+  const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
+  const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
+  const sdkSrc = resolve(sdkRepo, "src");
  const fixturePath = resolve(root, "skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json");
  const stdoutLog = join(evidenceDir, "probe-stdout.log");
  const stderrLog = join(evidenceDir, "probe-stderr.log");
  const automationResultJson = join(evidenceDir, "automation-result.json");
  const resultJson = join(evidenceDir, "result.json");
  const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
-  const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script, fixturePath], cwd: langbotRepo };
+  const command = {
+    executable: "rtk",
+    args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, fixturePath],
+    cwd: langbotRepo,
+  };
  const result = {
    source: "automation",
    probe: "agent-runner-behavior-matrix",
@@ -129,7 +129,7 @@ async function main() {
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });
  const startedAt = new Date();
-  const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk");
+  const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
  const sdkSrc = resolve(sdkRepo, "src");
  const fixturePath = resolve(root, "skills/langbot-testing/fixtures/plugins/qa-agent-runner");
  const stdoutLog = join(evidenceDir, "probe-stdout.log");
@@ -137,7 +137,7 @@ async function main() {
  const automationResultJson = join(evidenceDir, "automation-result.json");
  const resultJson = join(evidenceDir, "result.json");
  const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
-  const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script, fixturePath], cwd: sdkRepo };
+  const command = { executable: "rtk", args: ["uv", "run", "--no-sync", "python", "-c", script, fixturePath], cwd: sdkRepo };
  const result = {
    source: "automation",
    probe: "agent-runner-fixture-contract",
@@ -5,9 +5,9 @@ import { runPytestProbe } from "./pytest-probe.mjs";
 await runPytestProbe({
  caseId: "agent-runner-ledger-concurrency",
  repoEnvKey: "LANGBOT_REPO",
-  defaultRepo: "../LangBot",
+  defaultRepo: "..",
  pythonPathEnvKeys: ["LANGBOT_PLUGIN_SDK_REPO"],
-  defaultPythonPaths: ["../langbot-plugin-sdk/src"],
+  defaultPythonPaths: ["../../langbot-plugin-sdk/src"],
  description: "LangBot AgentRunner run ledger claim, lease, authorization, and runtime-admin pytest probe.",
  testTargets: [
    "tests/unit_tests/agent/test_run_ledger_store.py::test_create_queued_run_claim_renew_release",
@@ -152,15 +152,20 @@ async function main() {
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });
  const startedAt = new Date();
-  const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot");
-  const sdkSrc = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src");
+  const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
+  const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
+  const sdkSrc = resolve(sdkRepo, "src");
  const dbPath = join(evidenceDir, "ledger-contention.sqlite3");
  const stdoutLog = join(evidenceDir, "probe-stdout.log");
  const stderrLog = join(evidenceDir, "probe-stderr.log");
  const automationResultJson = join(evidenceDir, "automation-result.json");
  const resultJson = join(evidenceDir, "result.json");
  const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
-  const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script, dbPath], cwd: langbotRepo };
+  const command = {
+    executable: "rtk",
+    args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, dbPath],
+    cwd: langbotRepo,
+  };
  const result = {
    source: "automation",
    probe: "agent-runner-ledger-contention",
@@ -127,13 +127,18 @@ async function main() {
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });
  const startedAt = new Date();
-  const langbotRepo = resolveFromRoot(root, env.LANGBOT_REPO || "../LangBot");
-  const sdkSrc = resolveFromRoot(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src");
+  const langbotRepo = resolveFromRoot(root, env.LANGBOT_REPO || "..");
+  const sdkRepo = resolveFromRoot(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
+  const sdkSrc = resolve(sdkRepo, "src");
  const stdoutLog = join(evidenceDir, "probe-stdout.log");
  const stderrLog = join(evidenceDir, "probe-stderr.log");
  const automationResultJson = join(evidenceDir, "automation-result.json");
  const resultJson = join(evidenceDir, "result.json");
-  const command = { executable: "rtk", args: ["uv", "run", "python", "-c", probeScript], cwd: langbotRepo };
+  const command = {
+    executable: "rtk",
+    args: [resolve(langbotRepo, ".venv/bin/python"), "-c", probeScript],
+    cwd: langbotRepo,
+  };
  const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
  const result = {
    source: "automation",
@@ -119,14 +119,19 @@ async function main() {
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });
  const startedAt = new Date();
-  const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot");
-  const sdkSrc = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src");
+  const langbotRepo = resolve(root, env.LANGBOT_REPO || "..");
+  const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk");
+  const sdkSrc = resolve(sdkRepo, "src");
  const stdoutLog = join(evidenceDir, "probe-stdout.log");
  const stderrLog = join(evidenceDir, "probe-stderr.log");
  const automationResultJson = join(evidenceDir, "automation-result.json");
  const resultJson = join(evidenceDir, "result.json");
  const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000");
-  const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script], cwd: langbotRepo };
+  const command = {
+    executable: "rtk",
+    args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script],
+    cwd: langbotRepo,
+  };
  const result = {
    source: "automation",
    probe: "agent-runner-ledger-stress",
@@ -5,7 +5,7 @@ import { runPytestProbe } from "./pytest-probe.mjs";
 await runPytestProbe({
  caseId: "agent-runner-runtime-chaos",
  repoEnvKey: "LANGBOT_PLUGIN_SDK_REPO",
-  defaultRepo: "../langbot-plugin-sdk",
+  defaultRepo: "../../langbot-plugin-sdk",
  description: "LangBot plugin SDK AgentRunner runtime failure, timeout, forwarding, and pull API pytest probe.",
  testTargets: [
    "tests/runtime/plugin/test_mgr_agent_runner.py",
@@ -129,7 +129,7 @@ export async function runPytestProbe({
  const resultJson = join(evidenceDir, "result.json");
  const command = {
    executable: "rtk",
-    args: ["uv", "run", "pytest", "-q", ...testTargets],
+    args: ["uv", "run", "--no-sync", "pytest", "-q", ...testTargets],
    cwd: repoPath,
  };
  const result = {
@@ -3589,7 +3589,7 @@ test("MCP stdio tool-call case setups pipeline and registered MCP server", () =>
      JSON.parse(run.automation.env_defaults.LANGBOT_E2E_EXTENSIONS_PATCH_JSON),
      {
        enable_all_plugins: false,
-        bound_plugins: [{ author: "langbot", name: "local-agent" }],
+        bound_plugins: [{ author: "langbot-team", name: "LocalAgent" }],
        enable_all_mcp_servers: false,
        bound_mcp_servers: ["mcp-server-uuid"],
        enable_all_skills: false,
@@ -3695,6 +3695,10 @@ test("AgentRunner QA Debug Chat case uses dedicated pipeline env", () => {
    run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID,
    "plugin:qa/agent-runner/default",
  );
+  assert.equal(
+    run.automation.env_defaults.LANGBOT_E2E_DEBUG_CHAT_RESPONSE_P95_MS,
+    "120000",
+  );
  assert.deepEqual(
    run.setup_automation.map((item: { entry: string }) => item.entry),
    [