xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-07-17 01:46:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(agent-runner): close sandbox skill authoring flow

Browse Source
This commit is contained in:
huanghuoguoguo
2026-07-15 08:35:52 +08:00
parent d871baab68
commit 730d52bcd0
12 changed files with 313 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/pipeline-debug-chat.mjs
+26 -2
View File
@@ -239,6 +239,7 @@ async function runFilesystemChecks(checks) {
}
const contains = textList(check.contains);
const notContains = textList(check.not_contains || check.notContains);
const expectedJson = check.json_equals ?? check.jsonEquals;
const expectedExitCode = Number.isInteger(check.exit_code)
? check.exit_code
: Number.isInteger(check.expected_exit_code)
@@ -257,18 +258,31 @@ async function runFilesystemChecks(checks) {
}
const missing = contains.filter((needle) => !text.includes(needle));
const forbidden = notContains.filter((needle) => text.includes(needle));
let jsonError = "";
let jsonMatches = null;
if (expectedJson !== undefined) {
try {
jsonMatches = JSON.stringify(sortJson(JSON.parse(text))) === JSON.stringify(sortJson(expectedJson));
if (!jsonMatches) jsonError = "JSON content does not match json_equals.";
} catch (error) {
jsonMatches = false;
jsonError = `Invalid JSON: ${error.message}`;
}
}
const failed = missing.length > 0 || forbidden.length > 0 || jsonMatches === false;
results.push({
index,
status: missing.length || forbidden.length ? "fail" : "pass",
status: failed ? "fail" : "pass",
type: "file",
path,
missing,
forbidden,
json_matches: jsonMatches,
reason: missing.length
? `Missing expected text: ${missing.join(", ")}`
: forbidden.length
? `Found forbidden text: ${forbidden.join(", ")}`
: "",
: jsonError,
});
continue;
}
@@ -310,6 +324,16 @@ async function runFilesystemChecks(checks) {
};
}
function sortJson(value) {
if (Array.isArray(value)) return value.map(sortJson);
if (!value || typeof value !== "object") return value;
return Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, sortJson(value[key])]),
);
}
function pipelineIdFromUrl(url) {
if (!url) return "";
try {
skills/scripts/e2e/reset-sandbox-skill.mjs
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env node
import { env } from "node:process";
import {
apiJson,
ensureEvidence,
evidencePaths,
loadEnvFiles,
resetAndAuthLocalUser,
writeResult,
} from "./lib/langbot-e2e.mjs";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const caseId = "reset-sandbox-skill";
await loadEnvFiles();
const paths = evidencePaths(caseId);
await ensureEvidence(paths);
const nameArgument = process.argv.find((argument) => argument.startsWith("--name="));
const skillName = nameArgument?.slice("--name=".length).trim() || "";
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const result = {
source: "setup_automation",
case_id: caseId,
run_id: paths.runId,
status: "fail",
reason: "",
skill_name: skillName,
existed: false,
deleted: false,
evidence_collected: ["api_diagnostic"],
};
try {
if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");
if (!skillName || !/^[A-Za-z0-9_-]+$/.test(skillName)) {
throw new Error("--name must contain only letters, numbers, hyphens, or underscores.");
}
const user = env.LANGBOT_E2E_LOGIN_USER || "";
if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required.");
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
const listed = await apiJson(backendUrl, "/api/v1/skills", { token: auth.token });
if (listed.status >= 400 || listed.json.code !== 0) {
throw new Error(listed.json.msg || `Skill listing failed with HTTP ${listed.status}.`);
}
result.existed = (listed.json.data?.skills || []).some((skill) => skill?.name === skillName);
if (result.existed) {
const deleted = await apiJson(
backendUrl,
`/api/v1/skills/${encodeURIComponent(skillName)}`,
{ method: "DELETE", token: auth.token },
);
if (deleted.status >= 400 || deleted.json.code !== 0) {
throw new Error(deleted.json.msg || `Skill deletion failed with HTTP ${deleted.status}.`);
}
result.deleted = true;
}
result.status = "pass";
result.reason = result.deleted
? `Removed stale sandbox skill ${skillName}.`
: `Sandbox skill ${skillName} was already absent.`;
} catch (error) {
result.status = /ECONNREFUSED|fetch failed|not configured|required/.test(error.message)
? "env_issue"
: "fail";
result.reason = error.message;
}
await writeResult(paths, result);
console.log(JSON.stringify(result, null, 2));
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
skills/skills.index.json
+8 -2
View File
@@ -1621,8 +1621,14 @@
"edit"
],
"automation": "scripts/e2e/pipeline-debug-chat.mjs",
"setup_automation": [],
"setup_provides_env": [],
"setup_automation": [
"node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env",
"node:scripts/e2e/reset-sandbox-skill.mjs --name=lb-skill-edit-regression"
],
"setup_provides_env": [
"LANGBOT_LOCAL_AGENT_PIPELINE_URL",
"LANGBOT_LOCAL_AGENT_PIPELINE_NAME"
],
"evidence_required": [
"ui",
"backend_log",
skills/skills/langbot-testing/cases/sandbox-skill-authoring-edit-existing-e2e.yaml
+11 -5
View File
@@ -32,9 +32,15 @@ automation_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL
automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_expected_text: "UPDATED_MARKER_EDIT_REGRESSION"
automation_prompts_json: '[{"prompt":"Skill name is lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace. In that single command, remove old /workspace/lb-skill-edit-regression, create SKILL.md, scripts/use.py, and data/input.json. The JSON must contain numbers [1,2,3,4], factors [2,3,4], and marker CREATE_MARKER_EDIT_REGRESSION. The Python script must read data/input.json relative to __file__ and print exactly SANDBOX_SKILL_CREATE_OK sum=10 product=24 marker=CREATE_MARKER_EDIT_REGRESSION. Run python3 /workspace/lb-skill-edit-regression/scripts/use.py in the same command. After the exec succeeds, final answer exactly CREATE_OK:lb-skill-edit-regression.","expected_text":"CREATE_MARKER_EDIT_REGRESSION","response_timeout_ms":"180000"},{"prompt":"Continue with the same skill name lb-skill-edit-regression. Strictly call register_skill with path=/workspace/lb-skill-edit-regression and name=lb-skill-edit-regression, then call activate with skill_name=lb-skill-edit-regression. Do not call exec, read, write, edit, grep, or ls in this step. After both tools succeed, final answer exactly REGISTER_ACTIVATE_OK:lb-skill-edit-regression.","expected_text":"REGISTER_ACTIVATE_OK:lb-skill-edit-regression","response_timeout_ms":"180000"},{"prompt":"Continue with the same activated skill lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace/.skills/lb-skill-edit-regression. In that single exec command, run exactly this shell logic: set -e; overwrite SKILL.md with a heading and UPDATED_MARKER_EDIT_REGRESSION; overwrite data/input.json with numbers [5,6], factors [7,8], and marker UPDATED_MARKER_EDIT_REGRESSION; overwrite scripts/use.py so it reads data/input.json relative to __file__ and prints exactly SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION; run python3 scripts/use.py. Do not write exit(1). Do not call find, ls, read, grep as separate tools. If the single exec exits successfully and prints SANDBOX_SKILL_MODIFIED_OK, do not call more tools. Final answer exactly E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK.","expected_text":"UPDATED_MARKER_EDIT_REGRESSION","response_timeout_ms":"240000"}]'
automation_filesystem_checks_json: '[{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/SKILL.md","contains":"UPDATED_MARKER_EDIT_REGRESSION"},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/data/input.json","contains":["UPDATED_MARKER_EDIT_REGRESSION","\"numbers\": [5, 6]","\"factors\": [7, 8]"]},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/scripts/use.py","contains":["SANDBOX_SKILL_MODIFIED_OK"],"not_contains":["SANDBOX_SKILL_CREATE_OK","exit(1)"]},{"argv":["python3","scripts/use.py"],"cwd":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression","stdout_contains":"SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION","exit_code":0}]'
automation_expected_text: "E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK"
automation_prompts_json: '[{"prompt":"Skill name is lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace. In that single command, remove old /workspace/lb-skill-edit-regression, create SKILL.md, scripts/use.py, and data/input.json. The JSON must contain numbers [1,2,3,4], factors [2,3,4], and marker CREATE_MARKER_EDIT_REGRESSION. The Python script must read data/input.json relative to __file__ and print exactly SANDBOX_SKILL_CREATE_OK sum=10 product=24 marker=CREATE_MARKER_EDIT_REGRESSION. Run python3 /workspace/lb-skill-edit-regression/scripts/use.py in the same command. After the exec succeeds, final answer exactly CREATE_OK:lb-skill-edit-regression.","expected_text":"CREATE_OK:lb-skill-edit-regression","response_timeout_ms":"180000"},{"prompt":"Continue with the same skill name lb-skill-edit-regression. Strictly call register_skill with path=/workspace/lb-skill-edit-regression and name=lb-skill-edit-regression, then call activate with skill_name=lb-skill-edit-regression. Do not call exec, read, write, edit, grep, or ls in this step. After both tools succeed, final answer exactly REGISTER_ACTIVATE_OK:lb-skill-edit-regression.","expected_text":"REGISTER_ACTIVATE_OK:lb-skill-edit-regression","response_timeout_ms":"180000"},{"prompt":"Continue with the same activated skill lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace/.skills/lb-skill-edit-regression. In that single exec command, run exactly this shell logic: set -e; overwrite SKILL.md with a heading and UPDATED_MARKER_EDIT_REGRESSION; overwrite data/input.json with numbers [5,6], factors [7,8], and marker UPDATED_MARKER_EDIT_REGRESSION; overwrite scripts/use.py so it reads data/input.json relative to __file__ and prints exactly SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION; run python3 scripts/use.py. Do not write exit(1). Do not call find, ls, read, grep as separate tools. If the single exec exits successfully and prints SANDBOX_SKILL_MODIFIED_OK, do not call more tools. Final answer exactly E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK.","expected_text":"E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK","response_timeout_ms":"240000"}]'
setup_automation:
- "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"
- "node:scripts/e2e/reset-sandbox-skill.mjs --name=lb-skill-edit-regression"
setup_provides_env:
- LANGBOT_LOCAL_AGENT_PIPELINE_URL
- LANGBOT_LOCAL_AGENT_PIPELINE_NAME
automation_filesystem_checks_json: '[{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/SKILL.md","contains":"UPDATED_MARKER_EDIT_REGRESSION"},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/data/input.json","json_equals":{"numbers":[5,6],"factors":[7,8],"marker":"UPDATED_MARKER_EDIT_REGRESSION"}},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/scripts/use.py","contains":["SANDBOX_SKILL_MODIFIED_OK"],"not_contains":["SANDBOX_SKILL_CREATE_OK","exit(1)"]},{"argv":["python3","scripts/use.py"],"cwd":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression","stdout_contains":"SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION","exit_code":0}]'
automation_stream_output: "0"
automation_reset_debug_chat: "1"
automation_response_timeout_ms: "420000"
@@ -48,9 +54,9 @@ steps:
- "Run the automation prompt or the equivalent prompt pattern from references/sandbox-skill-authoring.md."
- "Capture the visible final response, backend logs, and the registered skill-store files."
checks:
- "UI: A Bot message, not only the user prompt, contains UPDATED_MARKER_EDIT_REGRESSION."
- "UI: A Bot message, not only the user prompt, contains E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK."
- "Logs: The model called exec, register_skill, activate, then a second exec whose workdir is /workspace/.skills/lb-skill-edit-regression."
- "Logs: The second exec stdout contains SANDBOX_SKILL_MODIFIED_OK and GREP_ALL_OK or equivalent grep success."
- "Logs: The second exec stdout contains SANDBOX_SKILL_MODIFIED_OK."
- "Automation: Filesystem checks pass for the registered skill-store files under LANGBOT_REPO/data/box/skills/lb-skill-edit-regression."
- "Skill store: SKILL.md and data/input.json contain UPDATED_MARKER_EDIT_REGRESSION."
- "Skill store: scripts/use.py is the modified script and prints SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION."