xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-09-16 14:57:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(runner): unify plugin execution across agents and event processors

Browse Source
This commit is contained in:
RockChinQ
2026-09-10 18:04:38 +08:00
parent 8903a40c41
commit f24a7c9bb2
223 changed files with 4091 additions and 3068 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/scripts/e2e/agent-run-ledger-audit.py
+195 -164
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Audit one persisted AgentRunner run without exposing authorization secrets."""
"""Audit one persisted Runner run without exposing authorization secrets."""
from __future__ import annotations
@@ -25,29 +25,29 @@ from agent_run_ledger_policy import (
def database_url(repo: pathlib.Path) -> str:
config = yaml.safe_load((repo / "data/config.yaml").read_text(encoding="utf-8")) or {}
database = config.get("database", {})
kind = database.get("use", "sqlite")
if kind == "sqlite":
path = pathlib.Path(database.get("sqlite", {}).get("path", "data/langbot.db"))
config = yaml.safe_load((repo / 'data/config.yaml').read_text(encoding='utf-8')) or {}
database = config.get('database', {})
kind = database.get('use', 'sqlite')
if kind == 'sqlite':
path = pathlib.Path(database.get('sqlite', {}).get('path', 'data/langbot.db'))
if not path.is_absolute():
path = repo / path
return f"sqlite+aiosqlite:///{path}"
if kind in {"postgres", "postgresql"}:
values = database.get("postgresql", {})
user = urllib.parse.quote_plus(str(values.get("user", "postgres")))
password = urllib.parse.quote_plus(str(values.get("password", "postgres")))
host = values.get("host", "127.0.0.1")
port = values.get("port", 5432)
name = values.get("database", "postgres")
return f"postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}"
raise RuntimeError(f"Unsupported database backend: {kind}")
return f'sqlite+aiosqlite:///{path}'
if kind in {'postgres', 'postgresql'}:
values = database.get('postgresql', {})
user = urllib.parse.quote_plus(str(values.get('user', 'postgres')))
password = urllib.parse.quote_plus(str(values.get('password', 'postgres')))
host = values.get('host', '127.0.0.1')
port = values.get('port', 5432)
name = values.get('database', 'postgres')
return f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}'
raise RuntimeError(f'Unsupported database backend: {kind}')
def parse_created_after(value: str | None) -> datetime.datetime | None:
if not value:
return None
parsed = datetime.datetime.fromisoformat(value.replace("Z", "+00:00"))
parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00'))
if parsed.tzinfo is not None:
parsed = parsed.astimezone(datetime.timezone.utc).replace(tzinfo=None)
return parsed
@@ -55,19 +55,19 @@ def parse_created_after(value: str | None) -> datetime.datetime | None:
def event_matches_tool_call(data_json: str | None, tool_name: str, parameters: dict | None) -> bool:
try:
data = json.loads(data_json or "{}")
data = json.loads(data_json or '{}')
except (TypeError, ValueError):
return False
if not isinstance(data, dict) or data.get("tool_name") != tool_name:
if not isinstance(data, dict) or data.get('tool_name') != tool_name:
return False
return parameters is None or data.get("parameters") == parameters
return parameters is None or data.get('parameters') == parameters
def collect_result_texts(value: object) -> list[str]:
texts: list[str] = []
if isinstance(value, dict):
for key, item in value.items():
if key == "text" and isinstance(item, str):
if key == 'text' and isinstance(item, str):
texts.append(item)
else:
texts.extend(collect_result_texts(item))
@@ -85,7 +85,7 @@ async def audit(
expected_tool_name: str | None = None,
expected_parameters: dict | None = None,
expected_result_text: str | None = None,
tool_authorization_mode: str = "strict",
tool_authorization_mode: str = 'strict',
) -> dict:
engine = create_async_engine(database_url(repo))
failures: list[dict] = []
@@ -93,82 +93,106 @@ async def audit(
try:
async with engine.connect() as connection:
if run_id:
run_row = (await connection.execute(
sqlalchemy.text("SELECT * FROM agent_run WHERE run_id = :run_id"),
{"run_id": run_id},
)).mappings().first()
run_row = (
(
await connection.execute(
sqlalchemy.text('SELECT * FROM agent_run WHERE run_id = :run_id'),
{'run_id': run_id},
)
)
.mappings()
.first()
)
elif expected_tool_name:
query = "SELECT * FROM agent_run"
query = 'SELECT * FROM agent_run'
params = {}
if created_after is not None:
query += " WHERE created_at >= :created_after"
params["created_after"] = created_after
query += " ORDER BY id DESC LIMIT 100"
query += ' WHERE created_at >= :created_after'
params['created_after'] = created_after
query += ' ORDER BY id DESC LIMIT 100'
candidates = (await connection.execute(sqlalchemy.text(query), params)).mappings().all()
run_row = None
for candidate in candidates:
started_rows = (await connection.execute(
sqlalchemy.text(
"SELECT data_json FROM agent_run_event "
"WHERE run_id = :run_id AND type = 'tool.call.started' ORDER BY sequence"
),
{"run_id": str(candidate["run_id"])},
)).mappings().all()
started_rows = (
(
await connection.execute(
sqlalchemy.text(
'SELECT data_json FROM agent_run_event '
"WHERE run_id = :run_id AND type = 'tool.call.started' ORDER BY sequence"
),
{'run_id': str(candidate['run_id'])},
)
)
.mappings()
.all()
)
if any(
event_matches_tool_call(row.get("data_json"), expected_tool_name, expected_parameters)
event_matches_tool_call(row.get('data_json'), expected_tool_name, expected_parameters)
for row in started_rows
):
run_row = candidate
break
else:
run_row = (await connection.execute(
sqlalchemy.text("SELECT * FROM agent_run ORDER BY id DESC LIMIT 1")
)).mappings().first()
run_row = (
(await connection.execute(sqlalchemy.text('SELECT * FROM agent_run ORDER BY id DESC LIMIT 1')))
.mappings()
.first()
)
if run_row is None:
status = "fail" if expected_tool_name else "env_issue"
status = 'fail' if expected_tool_name else 'env_issue'
return {
"status": status,
"reason": "No AgentRunner run contains the expected tool call." if expected_tool_name else "No matching AgentRunner run exists.",
"failures": [{"kind": "expected_tool_call_missing"}] if expected_tool_name else [],
"warnings": [],
'status': status,
'reason': 'No Runner run contains the expected tool call.'
if expected_tool_name
else 'No matching Runner run exists.',
'failures': [{'kind': 'expected_tool_call_missing'}] if expected_tool_name else [],
'warnings': [],
}
selected_run_id = str(run_row["run_id"])
event_rows = (await connection.execute(
sqlalchemy.text("SELECT sequence, type, data_json, metadata_json FROM agent_run_event WHERE run_id = :run_id ORDER BY sequence"),
{"run_id": selected_run_id},
)).mappings().all()
selected_run_id = str(run_row['run_id'])
event_rows = (
(
await connection.execute(
sqlalchemy.text(
'SELECT sequence, type, data_json, metadata_json FROM agent_run_event WHERE run_id = :run_id ORDER BY sequence'
),
{'run_id': selected_run_id},
)
)
.mappings()
.all()
)
finally:
await engine.dispose()
authorization = load_ledger_json(
run_row.get("authorization_json"),
field="agent_run.authorization_json",
run_row.get('authorization_json'),
field='agent_run.authorization_json',
failures=failures,
)
tools = authorization.get("resources", {}).get("tools", []) if isinstance(authorization, dict) else []
tools = authorization.get('resources', {}).get('tools', []) if isinstance(authorization, dict) else []
allowed_tools: dict[str, dict] = {}
incomplete_tool_metadata: list[dict] = []
for tool in tools if isinstance(tools, list) else []:
if not isinstance(tool, dict):
incomplete_tool_metadata.append({"tool_name": "", "missing": ["tool object"]})
incomplete_tool_metadata.append({'tool_name': '', 'missing': ['tool object']})
continue
name = str(tool.get("tool_name", ""))
name = str(tool.get('tool_name', ''))
missing = []
if not name:
missing.append("tool_name")
if not str(tool.get("description", "")).strip():
missing.append("description")
if not isinstance(tool.get("parameters"), dict):
missing.append("parameters")
if not (tool.get("source") or tool.get("tool_type") or tool.get("source_id")):
missing.append("owner")
missing.append('tool_name')
if not str(tool.get('description', '')).strip():
missing.append('description')
if not isinstance(tool.get('parameters'), dict):
missing.append('parameters')
if not (tool.get('source') or tool.get('tool_type') or tool.get('source_id')):
missing.append('owner')
if missing:
incomplete_tool_metadata.append({"tool_name": name, "missing": missing})
incomplete_tool_metadata.append({'tool_name': name, 'missing': missing})
if name:
allowed_tools[name] = tool
if incomplete_tool_metadata:
failures.append({"kind": "incomplete_tool_metadata", "tools": incomplete_tool_metadata})
failures.append({'kind': 'incomplete_tool_metadata', 'tools': incomplete_tool_metadata})
starts: dict[str, list[dict]] = {}
completions: dict[str, list[dict]] = {}
@@ -178,7 +202,7 @@ async def audit(
invalid_tool_argument_errors: list[dict] = []
successful_tool_completion_sequences: list[int] = []
forbidden_pattern = re.compile(
r"invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout",
r'invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout',
re.I,
)
@@ -189,55 +213,53 @@ async def audit(
return collected
for key, item in value.items():
normalized = str(key).lower()
if normalized in {"error", "code", "status", "reason", "error_message"} and item is not None and item != "":
if normalized in {'error', 'code', 'status', 'reason', 'error_message'} and item is not None and item != '':
collected.append(str(item))
if isinstance(item, dict):
collected.extend(error_surface(item))
return collected
for row in event_rows:
event_type = str(row["type"])
event_type = str(row['type'])
event_types.append(event_type)
before = len(failures)
data = load_ledger_json(
row.get("data_json"),
field=f"agent_run_event[{row['sequence']}].data_json",
row.get('data_json'),
field=f'agent_run_event[{row["sequence"]}].data_json',
failures=failures,
)
invalid_event_json += int(len(failures) > before)
if not isinstance(data, dict):
failures.append({"kind": "invalid_event_payload", "sequence": row["sequence"], "type": event_type})
failures.append({'kind': 'invalid_event_payload', 'sequence': row['sequence'], 'type': event_type})
continue
if event_type in {"tool.call.started", "tool.call.completed"}:
call_id = str(data.get("tool_call_id", ""))
item = {"sequence": row["sequence"], "tool_name": str(data.get("tool_name", "")), "data": data}
if event_type in {'tool.call.started', 'tool.call.completed'}:
call_id = str(data.get('tool_call_id', ''))
item = {'sequence': row['sequence'], 'tool_name': str(data.get('tool_name', '')), 'data': data}
if not call_id:
failures.append({"kind": "missing_tool_call_id", "sequence": row["sequence"], "type": event_type})
elif event_type == "tool.call.started":
failures.append({'kind': 'missing_tool_call_id', 'sequence': row['sequence'], 'type': event_type})
elif event_type == 'tool.call.started':
starts.setdefault(call_id, []).append(item)
else:
completions.setdefault(call_id, []).append(item)
if not data.get("error") and data.get("result") is not None:
successful_tool_completion_sequences.append(row["sequence"])
diagnostic_text = "\n".join(error_surface(data))
if event_type == "run.failed":
diagnostic_text += "\n" + json.dumps(data, ensure_ascii=True)
if not data.get('error') and data.get('result') is not None:
successful_tool_completion_sequences.append(row['sequence'])
diagnostic_text = '\n'.join(error_surface(data))
if event_type == 'run.failed':
diagnostic_text += '\n' + json.dumps(data, ensure_ascii=True)
match = forbidden_pattern.search(diagnostic_text)
if match:
suspicious_errors.append({"sequence": row["sequence"], "type": event_type, "signal": match.group(0)})
elif event_type == "tool.call.completed":
suspicious_errors.append({'sequence': row['sequence'], 'type': event_type, 'signal': match.group(0)})
elif event_type == 'tool.call.completed':
signal = invalid_tool_argument_error_signal(diagnostic_text)
if signal:
invalid_tool_argument_errors.append(
{"sequence": row["sequence"], "type": event_type, "signal": signal}
)
invalid_tool_argument_errors.append({'sequence': row['sequence'], 'type': event_type, 'signal': signal})
if run_row["status"] != "completed":
failures.append({"kind": "run_status", "actual": run_row["status"], "expected": "completed"})
if "run.completed" not in event_types:
failures.append({"kind": "missing_run_completed_event"})
if "run.failed" in event_types:
failures.append({"kind": "run_failed_event"})
if run_row['status'] != 'completed':
failures.append({'kind': 'run_status', 'actual': run_row['status'], 'expected': 'completed'})
if 'run.completed' not in event_types:
failures.append({'kind': 'missing_run_completed_event'})
if 'run.failed' in event_types:
failures.append({'kind': 'run_failed_event'})
all_call_ids = sorted(set(starts) | set(completions))
unauthorized_calls = []
@@ -245,14 +267,21 @@ async def audit(
started = starts.get(call_id, [])
completed = completions.get(call_id, [])
if len(started) != 1 or len(completed) != 1:
failures.append({"kind": "tool_call_pairing", "tool_call_id": call_id, "started": len(started), "completed": len(completed)})
failures.append(
{
'kind': 'tool_call_pairing',
'tool_call_id': call_id,
'started': len(started),
'completed': len(completed),
}
)
continue
if started[0]["tool_name"] != completed[0]["tool_name"]:
failures.append({"kind": "tool_name_mismatch", "tool_call_id": call_id})
if started[0]["sequence"] >= completed[0]["sequence"]:
failures.append({"kind": "tool_call_order", "tool_call_id": call_id})
if started[0]["tool_name"] not in allowed_tools:
unauthorized_calls.append({"tool_call_id": call_id, "tool_name": started[0]["tool_name"]})
if started[0]['tool_name'] != completed[0]['tool_name']:
failures.append({'kind': 'tool_name_mismatch', 'tool_call_id': call_id})
if started[0]['sequence'] >= completed[0]['sequence']:
failures.append({'kind': 'tool_call_order', 'tool_call_id': call_id})
if started[0]['tool_name'] not in allowed_tools:
unauthorized_calls.append({'tool_call_id': call_id, 'tool_name': started[0]['tool_name']})
authorization_failures, authorization_warnings = classify_tool_authorization(
unauthorized_calls,
authorization_mode=tool_authorization_mode,
@@ -263,20 +292,18 @@ async def audit(
invalid_tool_argument_errors,
successful_tool_completion_sequences=successful_tool_completion_sequences,
run_completed=(
run_row["status"] == "completed"
and "run.completed" in event_types
and "run.failed" not in event_types
run_row['status'] == 'completed' and 'run.completed' in event_types and 'run.failed' not in event_types
),
)
if unrecovered_argument_errors:
suspicious_errors.extend(unrecovered_argument_errors)
warnings.extend(recovered_argument_warnings)
if suspicious_errors:
failures.append({"kind": "forbidden_error_signals", "events": suspicious_errors})
failures.append({'kind': 'forbidden_error_signals', 'events': suspicious_errors})
if not event_rows:
failures.append({"kind": "missing_run_events"})
failures.append({'kind': 'missing_run_events'})
if not tools:
warnings.append({"kind": "no_authorized_tools", "reason": "The run authorization snapshot exposes no tools."})
warnings.append({'kind': 'no_authorized_tools', 'reason': 'The run authorization snapshot exposes no tools.'})
expected_call_summary = None
if expected_tool_name:
@@ -284,97 +311,101 @@ async def audit(
item
for items in starts.values()
for item in items
if item["tool_name"] == expected_tool_name
and (expected_parameters is None or item["data"].get("parameters") == expected_parameters)
if item['tool_name'] == expected_tool_name
and (expected_parameters is None or item['data'].get('parameters') == expected_parameters)
]
if len(matching_starts) != 1:
failures.append({"kind": "expected_tool_call_count", "actual": len(matching_starts), "expected": 1})
failures.append({'kind': 'expected_tool_call_count', 'actual': len(matching_starts), 'expected': 1})
matching_completions = []
for started in matching_starts:
call_id = str(started["data"].get("tool_call_id", ""))
call_id = str(started['data'].get('tool_call_id', ''))
matching_completions.extend(completions.get(call_id, []))
result_text_match = expected_result_text is None or any(
expected_result_text in collect_result_texts(completed["data"].get("result"))
expected_result_text in collect_result_texts(completed['data'].get('result'))
for completed in matching_completions
)
if expected_result_text is not None and not result_text_match:
failures.append({"kind": "expected_tool_result_text_missing"})
failures.append({'kind': 'expected_tool_result_text_missing'})
expected_call_summary = {
"tool_name": expected_tool_name,
"parameters_match_required": expected_parameters is not None,
"matched_started_count": len(matching_starts),
"matched_completed_count": len(matching_completions),
"result_text_match_required": expected_result_text is not None,
"result_text_match": result_text_match,
'tool_name': expected_tool_name,
'parameters_match_required': expected_parameters is not None,
'matched_started_count': len(matching_starts),
'matched_completed_count': len(matching_completions),
'result_text_match_required': expected_result_text is not None,
'result_text_match': result_text_match,
}
metrics = {
"event_count": len(event_rows),
"tool_call_started": sum(len(items) for items in starts.values()),
"tool_call_completed": sum(len(items) for items in completions.values()),
"tool_call_ids": len(all_call_ids),
"authorized_tool_count": len(allowed_tools),
"tool_authorization_mode": tool_authorization_mode,
"runner_native_tool_call_count": len(unauthorized_calls) if tool_authorization_mode == "runner-native" else 0,
"invalid_event_json": invalid_event_json,
"suspicious_error_count": len(suspicious_errors),
"recovered_tool_argument_error_count": len(recovered_argument_warnings),
'event_count': len(event_rows),
'tool_call_started': sum(len(items) for items in starts.values()),
'tool_call_completed': sum(len(items) for items in completions.values()),
'tool_call_ids': len(all_call_ids),
'authorized_tool_count': len(allowed_tools),
'tool_authorization_mode': tool_authorization_mode,
'runner_native_tool_call_count': len(unauthorized_calls) if tool_authorization_mode == 'runner-native' else 0,
'invalid_event_json': invalid_event_json,
'suspicious_error_count': len(suspicious_errors),
'recovered_tool_argument_error_count': len(recovered_argument_warnings),
}
return {
"status": "pass" if not failures else "fail",
"reason": "Agent run ledger audit passed." if not failures else f"Agent run ledger audit found {len(failures)} invariant failure(s).",
"run": {
"run_id": selected_run_id,
"runner_id": run_row["runner_id"],
"status": run_row["status"],
"created_at": str(run_row["created_at"]),
"finished_at": str(run_row["finished_at"]),
'status': 'pass' if not failures else 'fail',
'reason': 'Agent run ledger audit passed.'
if not failures
else f'Agent run ledger audit found {len(failures)} invariant failure(s).',
'run': {
'run_id': selected_run_id,
'runner_id': run_row['runner_id'],
'status': run_row['status'],
'created_at': str(run_row['created_at']),
'finished_at': str(run_row['finished_at']),
},
"metrics": metrics,
"expected_tool_call": expected_call_summary,
"failures": failures,
"warnings": warnings,
'metrics': metrics,
'expected_tool_call': expected_call_summary,
'failures': failures,
'warnings': warnings,
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--repo", required=True)
parser.add_argument("--run-id")
parser.add_argument("--created-after")
parser.add_argument("--expected-tool-name")
parser.add_argument("--expected-parameters-json")
parser.add_argument("--expected-result-text")
parser.add_argument('--repo', required=True)
parser.add_argument('--run-id')
parser.add_argument('--created-after')
parser.add_argument('--expected-tool-name')
parser.add_argument('--expected-parameters-json')
parser.add_argument('--expected-result-text')
parser.add_argument(
"--tool-authorization-mode",
choices=("strict", "runner-native"),
default="strict",
'--tool-authorization-mode',
choices=('strict', 'runner-native'),
default='strict',
)
parser.add_argument("--output", required=True)
parser.add_argument('--output', required=True)
args = parser.parse_args()
try:
expected_parameters = None
if args.expected_parameters_json:
expected_parameters = json.loads(args.expected_parameters_json)
if not isinstance(expected_parameters, dict):
raise ValueError("--expected-parameters-json must decode to an object")
raise ValueError('--expected-parameters-json must decode to an object')
if (expected_parameters is not None or args.expected_result_text) and not args.expected_tool_name:
raise ValueError("--expected-tool-name is required with expected parameters or result text")
report = asyncio.run(audit(
pathlib.Path(args.repo).resolve(),
args.run_id,
created_after=parse_created_after(args.created_after),
expected_tool_name=args.expected_tool_name,
expected_parameters=expected_parameters,
expected_result_text=args.expected_result_text,
tool_authorization_mode=args.tool_authorization_mode,
))
raise ValueError('--expected-tool-name is required with expected parameters or result text')
report = asyncio.run(
audit(
pathlib.Path(args.repo).resolve(),
args.run_id,
created_after=parse_created_after(args.created_after),
expected_tool_name=args.expected_tool_name,
expected_parameters=expected_parameters,
expected_result_text=args.expected_result_text,
tool_authorization_mode=args.tool_authorization_mode,
)
)
except Exception as exc: # noqa: BLE001 - probe must classify environment failures
report = {"status": "env_issue", "reason": str(exc), "failures": [], "warnings": []}
pathlib.Path(args.output).write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
report = {'status': 'env_issue', 'reason': str(exc), 'failures': [], 'warnings': []}
pathlib.Path(args.output).write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps(report))
return 0 if report["status"] == "pass" else 2 if report["status"] == "env_issue" else 1
return 0 if report['status'] == 'pass' else 2 if report['status'] == 'env_issue' else 1
if __name__ == "__main__":
if __name__ == '__main__':
sys.exit(main())
skills/scripts/e2e/agent-runner-health-visibility.mjs
+3 -3
View File
@@ -143,7 +143,7 @@ try {
}
if (!runner?.name) {
result.status = "blocked";
throw new Error("No registered AgentRunner is available for the UI check.");
throw new Error("No registered Runner is available for the UI check.");
}
const runnerConfigStage = runnerTab.stages.find(
@@ -155,7 +155,7 @@ try {
body: {
kind: "agent",
name: `Runner Health ${paths.runId.slice(-40)}`,
description: "Temporary AgentRunner health visibility fixture",
description: "Temporary Runner health visibility fixture",
emoji: "H",
component_ref: runner.name,
config: {
@@ -229,7 +229,7 @@ try {
}
result.status = "pass";
result.reason =
"Agent Runner settings visibly distinguished a registered runner from a stale binding.";
"Runner settings visibly distinguished a registered runner from a stale binding.";
} catch (error) {
if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail";
result.reason = result.reason || error.message;
skills/scripts/e2e/agent-runner-release-preflight.mjs
+464 -305
View File
@@ -24,7 +24,10 @@ function loadEnvDefaults(path) {
if (sep === -1) continue;
const key = line.slice(0, sep).trim();
if (env[key]) continue;
env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, "");
env[key] = line
.slice(sep + 1)
.trim()
.replace(/^["']|["']$/g, "");
}
}
@@ -46,12 +49,16 @@ function redactMessage(text) {
return String(text ?? "")
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
.replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]");
.replace(
/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi,
"$1=[redacted]",
);
}
function isEnvironmentError(message) {
return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i
.test(message);
return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i.test(
message,
);
}
loadEnvDefaults("skills/.env");
@@ -80,9 +87,15 @@ const targets = [
},
{
id: "acp-agent-runner",
expected_runner_id: "plugin:langbot-team/ACPAgentRunner/default",
pipeline_url: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_AGENT_RUNNER_PIPELINE_URL"),
pipeline_name: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", "LANGBOT_AGENT_RUNNER_PIPELINE_NAME"),
expected_runner_id: "plugin:langbot-team/ACPRunner/default",
pipeline_url: firstEnv(
"LANGBOT_ACP_RUNNER_PIPELINE_URL",
"LANGBOT_RUNNER_PIPELINE_URL",
),
pipeline_name: firstEnv(
"LANGBOT_ACP_RUNNER_PIPELINE_NAME",
"LANGBOT_RUNNER_PIPELINE_NAME",
),
require_func_call_model: false,
require_vision_model: false,
},
@@ -111,20 +124,29 @@ const result = {
automation_result_json: paths.automationResultJson,
result_json: paths.resultJson,
},
evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"],
evidence_collected: [
"ui",
"screenshot",
"console",
"network",
"api_diagnostic",
],
};
async function run() {
if (!backendUrl || !frontendUrl) {
result.status = "env_issue";
result.reason = "LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured.";
result.reason =
"LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured.";
return;
}
browser = await createBrowser(paths);
const { page } = browser;
await page.goto(frontendUrl, { waitUntil: "domcontentloaded" });
await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {});
await page
.waitForLoadState("networkidle", { timeout: 10_000 })
.catch(() => {});
const workspace = await ensureBrowserWorkspace(page, backendUrl);
if (workspace.status !== "pass") {
result.status = workspace.status;
@@ -132,309 +154,428 @@ async function run() {
return;
}
const diagnostic = await page.evaluate(async ({ backendUrl, targets, testModels }) => {
const blockers = [];
const envIssues = [];
const warnings = [];
const checks = [];
const diagnostic = await page.evaluate(
async ({ backendUrl, targets, testModels }) => {
const blockers = [];
const envIssues = [];
const warnings = [];
const checks = [];
const addCheck = (name, status, detail = {}) => {
checks.push({ name, status, ...detail });
if (status === "blocked") blockers.push({ name, ...detail });
if (status === "env_issue") envIssues.push({ name, ...detail });
};
const safeMessage = (value) => String(value ?? "")
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
.replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]");
const token = localStorage.getItem("token");
if (!token) {
addCheck("browser-auth", "blocked", { reason: "Browser profile has no localStorage token." });
return { authenticated: false, blockers, env_issues: envIssues, warnings, checks };
}
const headers = {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
"X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "",
};
const getJson = async (path) => {
const response = await fetch(`${backendUrl}${path}`, { headers });
return {
status: response.status,
json: await response.json().catch(() => ({})),
const addCheck = (name, status, detail = {}) => {
checks.push({ name, status, ...detail });
if (status === "blocked") blockers.push({ name, ...detail });
if (status === "env_issue") envIssues.push({ name, ...detail });
};
};
const postJson = async (path, body) => {
const response = await fetch(`${backendUrl}${path}`, {
method: "POST",
headers,
body: JSON.stringify(body),
});
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
const safeMessage = (value) =>
String(value ?? "")
.replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]")
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
.replace(
/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi,
"$1=[redacted]",
);
const tokenCheck = await getJson("/api/v1/user/check-token");
addCheck(
"browser-auth",
tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0 ? "pass" : "blocked",
{ http_status: tokenCheck.status, code: tokenCheck.json.code ?? null, reason: safeMessage(tokenCheck.json.msg || "") },
);
const systemInfo = await getJson("/api/v1/system/info");
addCheck(
"backend-system-info",
systemInfo.status < 400 ? "pass" : "env_issue",
{
http_status: systemInfo.status,
version: systemInfo.json.data?.version || systemInfo.json.data?.system?.version || "",
},
);
const pluginSystem = await getJson("/api/v1/system/status/plugin-system");
addCheck(
"plugin-system",
pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0 ? "pass" : "env_issue",
{
http_status: pluginSystem.status,
code: pluginSystem.json.code ?? null,
status: pluginSystem.json.data?.status || pluginSystem.json.data?.state || "",
reason: safeMessage(pluginSystem.json.msg || ""),
},
);
const boxStatus = await getJson("/api/v1/box/status");
addCheck(
"box-runtime",
boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0 ? "pass" : "env_issue",
{
http_status: boxStatus.status,
code: boxStatus.json.code ?? null,
status: boxStatus.json.data?.status || "",
backend: boxStatus.json.data?.backend || "",
reason: safeMessage(boxStatus.json.msg || ""),
},
);
const plugins = await getJson("/api/v1/plugins");
const installedPluginIds = (plugins.json.data?.plugins || [])
.map((plugin) => {
const metadata = plugin.manifest?.manifest?.metadata || plugin.manifest?.metadata || plugin.metadata || {};
return metadata.author && metadata.name ? `${metadata.author}/${metadata.name}` : "";
})
.filter(Boolean);
const requiredPlugins = ["langbot-team/LocalAgent", "langbot-team/ACPAgentRunner", "qa/plugin-smoke"];
const pluginPresence = Object.fromEntries(requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]));
for (const [id, present] of Object.entries(pluginPresence)) {
addCheck(`plugin:${id}`, present ? "pass" : "blocked", { plugin_id: id, reason: present ? "" : "Required plugin is not listed by /api/v1/plugins." });
}
const tools = await getJson("/api/v1/tools");
const toolNames = (tools.json.data?.tools || [])
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
.filter(Boolean)
.sort();
addCheck(
"tool:qa_plugin_echo",
toolNames.includes("qa_plugin_echo") ? "pass" : "blocked",
{ reason: toolNames.includes("qa_plugin_echo") ? "" : "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools." },
);
if (!toolNames.includes("qa_mcp_echo")) {
warnings.push({
name: "tool:qa_mcp_echo",
reason: "qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.",
});
}
const modelResponse = await getJson("/api/v1/provider/models/llm");
const models = (modelResponse.json.data?.models || []).map((model) => ({
uuid: model.uuid,
name: model.name,
abilities: Array.isArray(model.abilities) ? model.abilities : [],
provider_uuid: model.provider_uuid || model.provider?.uuid || "",
provider_name: model.provider_name || model.provider?.name || "",
requester: model.requester || model.provider?.requester || "",
}));
addCheck(
"llm-model-list",
modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0 ? "pass" : "env_issue",
{ http_status: modelResponse.status, model_count: models.length, reason: safeMessage(modelResponse.json.msg || "") },
);
const modelById = new Map(models.map((model) => [model.uuid, model]));
const pipelineList = await getJson("/api/v1/pipelines");
const pipelines = pipelineList.json.data?.pipelines || [];
addCheck(
"pipeline-list",
pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0 ? "pass" : "blocked",
{ http_status: pipelineList.status, pipeline_count: pipelines.length, reason: safeMessage(pipelineList.json.msg || "") },
);
const resolvedPipelines = [];
const modelTested = new Set();
for (const target of targets) {
let pipelineId = "";
let matchedBy = "";
if (target.pipeline_url) {
try {
pipelineId = new URL(target.pipeline_url).searchParams.get("id") || "";
matchedBy = pipelineId ? "url" : "";
} catch {
pipelineId = "";
}
}
if (!pipelineId && target.pipeline_name) {
const match = pipelines.find((pipeline) => pipeline.name === target.pipeline_name);
if (match) {
pipelineId = match.uuid;
matchedBy = "name";
}
}
if (!pipelineId) {
addCheck(`pipeline:${target.id}`, "blocked", {
target: target.id,
reason: "Required pipeline env is missing or could not resolve to a pipeline id.",
const token = localStorage.getItem("token");
if (!token) {
addCheck("browser-auth", "blocked", {
reason: "Browser profile has no localStorage token.",
});
continue;
return {
authenticated: false,
blockers,
env_issues: envIssues,
warnings,
checks,
};
}
const response = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`);
const pipeline = response.json.data?.pipeline;
if (response.status >= 400 || !pipeline) {
addCheck(`pipeline:${target.id}`, "blocked", {
target: target.id,
pipeline_id: pipelineId,
http_status: response.status,
reason: safeMessage(response.json.msg || "Could not load pipeline."),
});
continue;
}
const config = pipeline.config || {};
const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {};
const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {};
const runnerId = runner.id || "";
const runnerConfigs = aiConfig.runner_config && typeof aiConfig.runner_config === "object" ? aiConfig.runner_config : {};
const runnerConfig = runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" ? runnerConfigs[runnerId] : {};
const pipelineSummary = {
target: target.id,
pipeline_id: pipelineId,
pipeline_name: pipeline.name,
matched_by: matchedBy,
runner_id: runnerId,
expected_runner_id: target.expected_runner_id,
runner_config_keys: Object.keys(runnerConfig).sort(),
const headers = {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
"X-Workspace-Id":
localStorage.getItem("langbot_active_workspace_uuid") || "",
};
const getJson = async (path) => {
const response = await fetch(`${backendUrl}${path}`, { headers });
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
const postJson = async (path, body) => {
const response = await fetch(`${backendUrl}${path}`, {
method: "POST",
headers,
body: JSON.stringify(body),
});
return {
status: response.status,
json: await response.json().catch(() => ({})),
};
};
resolvedPipelines.push(pipelineSummary);
const tokenCheck = await getJson("/api/v1/user/check-token");
addCheck(
`pipeline:${target.id}:runner`,
runnerId === target.expected_runner_id ? "pass" : "blocked",
"browser-auth",
tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0
? "pass"
: "blocked",
{
...pipelineSummary,
reason: runnerId === target.expected_runner_id ? "" : `Expected ${target.expected_runner_id}, got ${runnerId || "<missing>"}.`,
http_status: tokenCheck.status,
code: tokenCheck.json.code ?? null,
reason: safeMessage(tokenCheck.json.msg || ""),
},
);
if (target.require_func_call_model || target.require_vision_model || (testModels && target.id === "local-agent")) {
const modelConfig = runnerConfig.model;
const primaryModelId = typeof modelConfig === "string"
? modelConfig
: modelConfig && typeof modelConfig === "object"
? modelConfig.primary || ""
const systemInfo = await getJson("/api/v1/system/info");
addCheck(
"backend-system-info",
systemInfo.status < 400 ? "pass" : "env_issue",
{
http_status: systemInfo.status,
version:
systemInfo.json.data?.version ||
systemInfo.json.data?.system?.version ||
"",
},
);
const pluginSystem = await getJson("/api/v1/system/status/plugin-system");
addCheck(
"plugin-system",
pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0
? "pass"
: "env_issue",
{
http_status: pluginSystem.status,
code: pluginSystem.json.code ?? null,
status:
pluginSystem.json.data?.status ||
pluginSystem.json.data?.state ||
"",
reason: safeMessage(pluginSystem.json.msg || ""),
},
);
const boxStatus = await getJson("/api/v1/box/status");
addCheck(
"box-runtime",
boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0
? "pass"
: "env_issue",
{
http_status: boxStatus.status,
code: boxStatus.json.code ?? null,
status: boxStatus.json.data?.status || "",
backend: boxStatus.json.data?.backend || "",
reason: safeMessage(boxStatus.json.msg || ""),
},
);
const plugins = await getJson("/api/v1/plugins");
const installedPluginIds = (plugins.json.data?.plugins || [])
.map((plugin) => {
const metadata =
plugin.manifest?.manifest?.metadata ||
plugin.manifest?.metadata ||
plugin.metadata ||
{};
return metadata.author && metadata.name
? `${metadata.author}/${metadata.name}`
: "";
if (!primaryModelId) {
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
...pipelineSummary,
reason: "Local-agent runner config has no primary model.",
});
continue;
}
const model = modelById.get(primaryModelId);
if (!model) {
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
...pipelineSummary,
model_uuid: primaryModelId,
reason: "Primary model is not listed by /api/v1/provider/models/llm.",
});
continue;
}
addCheck(`pipeline:${target.id}:primary-model`, "pass", {
...pipelineSummary,
model: {
uuid: model.uuid,
name: model.name,
abilities: model.abilities,
provider_name: model.provider_name,
requester: model.requester,
},
})
.filter(Boolean);
const requiredPlugins = [
"langbot-team/LocalAgent",
"langbot-team/ACPRunner",
"qa/plugin-smoke",
];
const pluginPresence = Object.fromEntries(
requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]),
);
for (const [id, present] of Object.entries(pluginPresence)) {
addCheck(`plugin:${id}`, present ? "pass" : "blocked", {
plugin_id: id,
reason: present
? ""
: "Required plugin is not listed by /api/v1/plugins.",
});
if (target.require_func_call_model) {
addCheck(
`pipeline:${target.id}:func-call-model`,
model.abilities.includes("func_call") ? "pass" : "env_issue",
{
model_uuid: model.uuid,
model_name: model.name,
abilities: model.abilities,
reason: model.abilities.includes("func_call") ? "" : "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.",
},
);
}
const tools = await getJson("/api/v1/tools");
const toolNames = (tools.json.data?.tools || [])
.map((tool) => tool.name || tool.tool_name || tool.function?.name || "")
.filter(Boolean)
.sort();
addCheck(
"tool:qa_plugin_echo",
toolNames.includes("qa_plugin_echo") ? "pass" : "blocked",
{
reason: toolNames.includes("qa_plugin_echo")
? ""
: "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools.",
},
);
if (!toolNames.includes("qa_mcp_echo")) {
warnings.push({
name: "tool:qa_mcp_echo",
reason:
"qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.",
});
}
const modelResponse = await getJson("/api/v1/provider/models/llm");
const models = (modelResponse.json.data?.models || []).map((model) => ({
uuid: model.uuid,
name: model.name,
abilities: Array.isArray(model.abilities) ? model.abilities : [],
provider_uuid: model.provider_uuid || model.provider?.uuid || "",
provider_name: model.provider_name || model.provider?.name || "",
requester: model.requester || model.provider?.requester || "",
}));
addCheck(
"llm-model-list",
modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0
? "pass"
: "env_issue",
{
http_status: modelResponse.status,
model_count: models.length,
reason: safeMessage(modelResponse.json.msg || ""),
},
);
const modelById = new Map(models.map((model) => [model.uuid, model]));
const pipelineList = await getJson("/api/v1/pipelines");
const pipelines = pipelineList.json.data?.pipelines || [];
addCheck(
"pipeline-list",
pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0
? "pass"
: "blocked",
{
http_status: pipelineList.status,
pipeline_count: pipelines.length,
reason: safeMessage(pipelineList.json.msg || ""),
},
);
const resolvedPipelines = [];
const modelTested = new Set();
for (const target of targets) {
let pipelineId = "";
let matchedBy = "";
if (target.pipeline_url) {
try {
pipelineId =
new URL(target.pipeline_url).searchParams.get("id") || "";
matchedBy = pipelineId ? "url" : "";
} catch {
pipelineId = "";
}
}
if (target.require_vision_model) {
addCheck(
`pipeline:${target.id}:vision-model`,
model.abilities.includes("vision") ? "pass" : "env_issue",
{
model_uuid: model.uuid,
model_name: model.name,
abilities: model.abilities,
reason: model.abilities.includes("vision") ? "" : "Release gate includes multimodal cases; the local-agent primary model must advertise vision.",
},
if (!pipelineId && target.pipeline_name) {
const match = pipelines.find(
(pipeline) => pipeline.name === target.pipeline_name,
);
if (match) {
pipelineId = match.uuid;
matchedBy = "name";
}
}
if (testModels && !modelTested.has(model.uuid)) {
modelTested.add(model.uuid);
const modelTest = await postJson(`/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`, { extra_args: {} });
const passed = modelTest.status < 400 && (modelTest.json.code ?? 0) === 0;
addCheck(
`model-test:${model.name}`,
passed ? "pass" : "env_issue",
{
model_uuid: model.uuid,
model_name: model.name,
http_status: modelTest.status,
code: modelTest.json.code ?? null,
reason: passed ? "" : safeMessage(modelTest.json.msg || modelTest.json.message || "Model test failed."),
if (!pipelineId) {
addCheck(`pipeline:${target.id}`, "blocked", {
target: target.id,
reason:
"Required pipeline env is missing or could not resolve to a pipeline id.",
});
continue;
}
const response = await getJson(
`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
);
const pipeline = response.json.data?.pipeline;
if (response.status >= 400 || !pipeline) {
addCheck(`pipeline:${target.id}`, "blocked", {
target: target.id,
pipeline_id: pipelineId,
http_status: response.status,
reason: safeMessage(
response.json.msg || "Could not load pipeline.",
),
});
continue;
}
const config = pipeline.config || {};
const aiConfig =
config.ai && typeof config.ai === "object" ? config.ai : {};
const runner =
aiConfig.runner && typeof aiConfig.runner === "object"
? aiConfig.runner
: {};
const runnerId = runner.id || "";
const runnerConfigs =
aiConfig.runner_config && typeof aiConfig.runner_config === "object"
? aiConfig.runner_config
: {};
const runnerConfig =
runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object"
? runnerConfigs[runnerId]
: {};
const pipelineSummary = {
target: target.id,
pipeline_id: pipelineId,
pipeline_name: pipeline.name,
matched_by: matchedBy,
runner_id: runnerId,
expected_runner_id: target.expected_runner_id,
runner_config_keys: Object.keys(runnerConfig).sort(),
};
resolvedPipelines.push(pipelineSummary);
addCheck(
`pipeline:${target.id}:runner`,
runnerId === target.expected_runner_id ? "pass" : "blocked",
{
...pipelineSummary,
reason:
runnerId === target.expected_runner_id
? ""
: `Expected ${target.expected_runner_id}, got ${runnerId || "<missing>"}.`,
},
);
if (
target.require_func_call_model ||
target.require_vision_model ||
(testModels && target.id === "local-agent")
) {
const modelConfig = runnerConfig.model;
const primaryModelId =
typeof modelConfig === "string"
? modelConfig
: modelConfig && typeof modelConfig === "object"
? modelConfig.primary || ""
: "";
if (!primaryModelId) {
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
...pipelineSummary,
reason: "Local-agent runner config has no primary model.",
});
continue;
}
const model = modelById.get(primaryModelId);
if (!model) {
addCheck(`pipeline:${target.id}:primary-model`, "blocked", {
...pipelineSummary,
model_uuid: primaryModelId,
reason:
"Primary model is not listed by /api/v1/provider/models/llm.",
});
continue;
}
addCheck(`pipeline:${target.id}:primary-model`, "pass", {
...pipelineSummary,
model: {
uuid: model.uuid,
name: model.name,
abilities: model.abilities,
provider_name: model.provider_name,
requester: model.requester,
},
);
});
if (target.require_func_call_model) {
addCheck(
`pipeline:${target.id}:func-call-model`,
model.abilities.includes("func_call") ? "pass" : "env_issue",
{
model_uuid: model.uuid,
model_name: model.name,
abilities: model.abilities,
reason: model.abilities.includes("func_call")
? ""
: "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.",
},
);
}
if (target.require_vision_model) {
addCheck(
`pipeline:${target.id}:vision-model`,
model.abilities.includes("vision") ? "pass" : "env_issue",
{
model_uuid: model.uuid,
model_name: model.name,
abilities: model.abilities,
reason: model.abilities.includes("vision")
? ""
: "Release gate includes multimodal cases; the local-agent primary model must advertise vision.",
},
);
}
if (testModels && !modelTested.has(model.uuid)) {
modelTested.add(model.uuid);
const modelTest = await postJson(
`/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`,
{ extra_args: {} },
);
const passed =
modelTest.status < 400 && (modelTest.json.code ?? 0) === 0;
addCheck(
`model-test:${model.name}`,
passed ? "pass" : "env_issue",
{
model_uuid: model.uuid,
model_name: model.name,
http_status: modelTest.status,
code: modelTest.json.code ?? null,
reason: passed
? ""
: safeMessage(
modelTest.json.msg ||
modelTest.json.message ||
"Model test failed.",
),
},
);
}
}
}
}
return {
authenticated: true,
blockers,
env_issues: envIssues,
warnings,
checks,
resolved_pipelines: resolvedPipelines,
tools: {
required: ["qa_plugin_echo"],
optional_before_register: ["qa_mcp_echo"],
present: toolNames.filter((name) => ["qa_plugin_echo", "qa_mcp_echo"].includes(name)),
},
models,
};
}, { backendUrl, targets, testModels });
return {
authenticated: true,
blockers,
env_issues: envIssues,
warnings,
checks,
resolved_pipelines: resolvedPipelines,
tools: {
required: ["qa_plugin_echo"],
optional_before_register: ["qa_mcp_echo"],
present: toolNames.filter((name) =>
["qa_plugin_echo", "qa_mcp_echo"].includes(name),
),
},
models,
};
},
{ backendUrl, targets, testModels },
);
diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") }));
diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") }));
await writeFile(diagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8");
diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({
...item,
reason: redactMessage(item.reason || ""),
}));
diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({
...item,
reason: redactMessage(item.reason || ""),
}));
await writeFile(
diagnosticPath,
`${JSON.stringify(diagnostic, null, 2)}\n`,
"utf8",
);
await safeScreenshot(page, paths.screenshot);
const blockers = diagnostic.blockers || [];
@@ -447,31 +588,49 @@ async function run() {
result.reason = `Preflight environment issue: ${envIssues.map((item) => item.name).join(", ")}`;
} else {
result.status = "pass";
result.reason = "Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready.";
result.reason =
"Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready.";
}
result.check_count = Array.isArray(diagnostic.checks) ? diagnostic.checks.length : 0;
result.warning_count = Array.isArray(diagnostic.warnings) ? diagnostic.warnings.length : 0;
result.check_count = Array.isArray(diagnostic.checks)
? diagnostic.checks.length
: 0;
result.warning_count = Array.isArray(diagnostic.warnings)
? diagnostic.warnings.length
: 0;
}
try {
await run();
} catch (error) {
const message = redactMessage(error instanceof Error ? error.message : String(error));
const message = redactMessage(
error instanceof Error ? error.message : String(error),
);
result.status = isEnvironmentError(message) ? "env_issue" : "fail";
result.reason = message;
await writeFile(diagnosticPath, `${JSON.stringify({
authenticated: false,
blockers: [],
env_issues: result.status === "env_issue" ? [{ name: "preflight-runtime", reason: message }] : [],
warnings: [],
checks: [
await writeFile(
diagnosticPath,
`${JSON.stringify(
{
name: "preflight-runtime",
status: result.status,
reason: message,
authenticated: false,
blockers: [],
env_issues:
result.status === "env_issue"
? [{ name: "preflight-runtime", reason: message }]
: [],
warnings: [],
checks: [
{
name: "preflight-runtime",
status: result.status,
reason: message,
},
],
},
],
}, null, 2)}\n`, "utf8").catch(() => {});
null,
2,
)}\n`,
"utf8",
).catch(() => {});
} finally {
if (browser) await browser.close().catch(() => {});
const finishedAt = new Date();
skills/scripts/e2e/agent_run_ledger_policy.py
+14 -15
View File
@@ -1,4 +1,4 @@
"""Policy helpers for classifying AgentRunner ledger error signals."""
"""Policy helpers for classifying Runner ledger error signals."""
from __future__ import annotations
@@ -7,7 +7,7 @@ import re
_INVALID_TOOL_ARGUMENT_PATTERN = re.compile(
r"invalid json arguments|\b\d+\s+validation errors?\s+for\s+[A-Za-z_][A-Za-z0-9_]*Args\b",
r'invalid json arguments|\b\d+\s+validation errors?\s+for\s+[A-Za-z_][A-Za-z0-9_]*Args\b',
re.IGNORECASE,
)
@@ -19,14 +19,14 @@ def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) ->
try:
return json.loads(value)
except (TypeError, ValueError) as exc:
failures.append({"kind": "invalid_json", "field": field, "reason": str(exc)})
failures.append({'kind': 'invalid_json', 'field': field, 'reason': str(exc)})
return {}
def invalid_tool_argument_error_signal(value: str) -> str:
"""Return the persisted signal for malformed model-supplied tool arguments."""
match = _INVALID_TOOL_ARGUMENT_PATTERN.search(value)
return match.group(0) if match else ""
return match.group(0) if match else ''
def classify_invalid_tool_argument_errors(
@@ -40,15 +40,14 @@ def classify_invalid_tool_argument_errors(
warnings: list[dict] = []
for event in events:
recovered = run_completed and any(
sequence > event["sequence"]
for sequence in successful_tool_completion_sequences
sequence > event['sequence'] for sequence in successful_tool_completion_sequences
)
if recovered:
warnings.append(
{
"kind": "recovered_tool_argument_error",
"event": event,
"reason": "The model continued with a later successful tool call and the run completed.",
'kind': 'recovered_tool_argument_error',
'event': event,
'reason': 'The model continued with a later successful tool call and the run completed.',
}
)
else:
@@ -64,15 +63,15 @@ def classify_tool_authorization(
"""Classify tool names absent from the Host authorization snapshot."""
if not calls:
return [], []
if authorization_mode == "runner-native":
if authorization_mode == 'runner-native':
return [], [
{
"kind": "runner_native_tool_calls",
"calls": calls,
"reason": (
"External runner tool telemetry is not a LangBot Host tool call; "
'kind': 'runner_native_tool_calls',
'calls': calls,
'reason': (
'External runner tool telemetry is not a LangBot Host tool call; '
"the runner's own permission system governs it."
),
}
]
return [{"kind": "unauthorized_tool_calls", "calls": calls}], []
return [{'kind': 'unauthorized_tool_calls', 'calls': calls}], []
skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs
+79 -35
View File
@@ -12,7 +12,7 @@ import {
writeResult,
} from "./lib/langbot-e2e.mjs";
const RUNNER_ID = "plugin:langbot-team/ACPAgentRunner/default";
const RUNNER_ID = "plugin:langbot-team/ACPRunner/default";
const DEFAULT_PIPELINE_NAME = "Agent QA ACP Claude Debug Chat";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const caseId = "ensure-acp-agent-runner-pipeline";
@@ -24,13 +24,18 @@ await ensureEvidence(paths);
const writeEnv = process.argv.includes("--write-env");
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
const sshTarget = env.LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET || "yhh@101.34.71.12";
const sshConnectTimeout = env.LANGBOT_ACP_AGENT_RUNNER_SSH_CONNECT_TIMEOUT || "8";
const sshPort = env.LANGBOT_ACP_AGENT_RUNNER_SSH_PORT || "22";
const sshIdentityFile = env.LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE || "";
const sshExtraOptions = env.LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS || "";
const remoteWorkspace = env.LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE || "/home/yhh/langbot-e2e/acp-workspace";
const pipelineName =
env.LANGBOT_E2E_CREATE_PIPELINE_NAME ||
env.LANGBOT_ACP_RUNNER_PIPELINE_NAME ||
DEFAULT_PIPELINE_NAME;
const sshTarget = env.LANGBOT_ACP_RUNNER_SSH_TARGET || "yhh@101.34.71.12";
const sshConnectTimeout = env.LANGBOT_ACP_RUNNER_SSH_CONNECT_TIMEOUT || "8";
const sshPort = env.LANGBOT_ACP_RUNNER_SSH_PORT || "22";
const sshIdentityFile = env.LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE || "";
const sshExtraOptions = env.LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS || "";
const remoteWorkspace =
env.LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE ||
"/home/yhh/langbot-e2e/acp-workspace";
const envLocalPath = resolve("skills/.env.local");
const result = {
@@ -64,7 +69,9 @@ try {
const user = env.LANGBOT_E2E_LOGIN_USER || "";
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
if (!user) {
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
throw new Error(
"LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.",
);
}
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
@@ -116,13 +123,13 @@ try {
if (writeEnv && result.pipeline_id) {
await upsertEnvLocal(envLocalPath, {
LANGBOT_E2E_LOGIN_USER: user,
LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET: sshTarget,
LANGBOT_ACP_AGENT_RUNNER_SSH_PORT: sshPort,
LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE: sshIdentityFile,
LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS: sshExtraOptions,
LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE: remoteWorkspace,
LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url,
LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName,
LANGBOT_ACP_RUNNER_SSH_TARGET: sshTarget,
LANGBOT_ACP_RUNNER_SSH_PORT: sshPort,
LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE: sshIdentityFile,
LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS: sshExtraOptions,
LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE: remoteWorkspace,
LANGBOT_ACP_RUNNER_PIPELINE_URL: result.pipeline_url,
LANGBOT_ACP_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName,
});
result.wrote_env = true;
}
@@ -133,10 +140,20 @@ try {
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) {
const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token });
async function ensurePipeline({
backendUrl,
token,
pipelineName,
runnerId,
runnerConfig,
}) {
const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", {
token,
});
if (isApiFailure(pipelineList)) {
return {
status: "fail",
@@ -155,7 +172,8 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
token,
body: {
name: pipelineName,
description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.",
description:
"Local QA pipeline for real ACP Claude Runner Debug Chat smoke tests.",
emoji: "QA",
},
});
@@ -167,7 +185,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
};
}
const pipelineId = createdResponse.json.data?.uuid || "";
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
{ token },
);
pipeline = loaded.json.data?.pipeline || null;
created = true;
}
@@ -179,7 +201,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
};
}
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{ token },
);
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
return {
status: "fail",
@@ -190,9 +216,15 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
}
pipeline = loaded.json.data.pipeline;
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
const config =
pipeline.config && typeof pipeline.config === "object"
? pipeline.config
: {};
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {};
const runnerConfigs =
ai.runner_config && typeof ai.runner_config === "object"
? ai.runner_config
: {};
const updatedConfig = {
...config,
ai: {
@@ -209,16 +241,21 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
},
};
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
method: "PUT",
token,
body: {
name: pipelineName,
description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
const updateResponse = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{
method: "PUT",
token,
body: {
name: pipelineName,
description:
"Local QA pipeline for real ACP Claude Runner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
},
},
});
);
if (isApiFailure(updateResponse)) {
return {
status: "fail",
@@ -230,7 +267,9 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
return {
status: "pass",
reason: created ? "ACP AgentRunner pipeline created and configured." : "ACP AgentRunner pipeline updated.",
reason: created
? "ACP Runner pipeline created and configured."
: "ACP Runner pipeline updated.",
pipeline_id: pipeline.uuid,
pipeline_name: pipelineName,
created,
@@ -239,7 +278,12 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
}
function isApiFailure(response) {
return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0);
return (
response.status >= 400 ||
(response.json &&
response.json.code !== undefined &&
response.json.code !== 0)
);
}
async function upsertEnvLocal(path, values) {
skills/scripts/e2e/ensure-local-agent-pipeline.mjs
+12 -12
View File
@@ -102,14 +102,16 @@ try {
backend_token_check: auth.check,
};
const pluginSetup = await ensureLocalAgentRunner({
const pluginSetup = await ensureLocalRunner({
backendUrl,
token: auth.token,
});
result.plugin_setup = pluginSetup;
if (pluginSetup.status !== "pass") {
result.status = pluginSetup.status === "env_issue" ? "env_issue" : "fail";
throw new Error(pluginSetup.reason || "Failed to prepare the LocalAgent runner plugin.");
throw new Error(
pluginSetup.reason || "Failed to prepare the LocalAgent runner plugin.",
);
}
const wizard = await skipWizard({ backendUrl, token: auth.token });
@@ -205,7 +207,7 @@ async function skipWizard({ backendUrl, token }) {
};
}
async function ensureLocalAgentRunner({ backendUrl, token }) {
async function ensureLocalRunner({ backendUrl, token }) {
const [author, name] = RUNNER_ID.replace(/^plugin:/, "").split("/");
const existingRunnerIds = await listRunnerIds(backendUrl, token);
if (existingRunnerIds.includes(RUNNER_ID)) {
@@ -264,10 +266,9 @@ async function ensureLocalAgentRunner({ backendUrl, token }) {
};
}
const spaceUrl = String(env.LANGBOT_SPACE_URL || "https://space.langbot.app").replace(
/\/$/,
"",
);
const spaceUrl = String(
env.LANGBOT_SPACE_URL || "https://space.langbot.app",
).replace(/\/$/, "");
let detailResponse;
try {
detailResponse = await fetch(
@@ -376,7 +377,8 @@ async function waitForRunnerRegistration({
}) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if ((await listRunnerIds(backendUrl, token)).includes(runnerId)) return true;
if ((await listRunnerIds(backendUrl, token)).includes(runnerId))
return true;
await sleep(1000);
}
return false;
@@ -501,8 +503,7 @@ async function ensureLocalAgentPipeline({
token,
body: {
name: pipelineName,
description:
"Local QA pipeline for AgentRunner Debug Chat smoke tests.",
description: "Local QA pipeline for Runner Debug Chat smoke tests.",
emoji: "QA",
},
});
@@ -640,8 +641,7 @@ async function ensureLocalAgentPipeline({
token,
body: {
name: pipelineName,
description:
"Local QA pipeline for AgentRunner Debug Chat smoke tests.",
description: "Local QA pipeline for Runner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
},
skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs
+65 -23
View File
@@ -24,7 +24,10 @@ await ensureEvidence(paths);
const writeEnv = process.argv.includes("--write-env");
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME;
const pipelineName =
env.LANGBOT_E2E_CREATE_PIPELINE_NAME ||
env.LANGBOT_QA_RUNNER_PIPELINE_NAME ||
DEFAULT_PIPELINE_NAME;
const envLocalPath = resolve("skills/.env.local");
const result = {
@@ -55,7 +58,9 @@ try {
const user = env.LANGBOT_E2E_LOGIN_USER || "";
const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
if (!user) {
throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.");
throw new Error(
"LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.",
);
}
const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
@@ -80,8 +85,8 @@ try {
if (writeEnv && result.pipeline_id) {
await upsertEnvLocal(envLocalPath, {
LANGBOT_E2E_LOGIN_USER: user,
LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url,
LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName,
LANGBOT_QA_RUNNER_PIPELINE_URL: result.pipeline_url,
LANGBOT_QA_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName,
});
result.wrote_env = true;
}
@@ -92,10 +97,20 @@ try {
console.log(JSON.stringify(result, null, 2));
}
process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1);
process.exit(
result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);
async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) {
const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token });
async function ensurePipeline({
backendUrl,
token,
pipelineName,
runnerId,
runnerConfig,
}) {
const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", {
token,
});
if (isApiFailure(pipelineList)) {
return {
status: "fail",
@@ -114,7 +129,8 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
token,
body: {
name: pipelineName,
description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.",
description:
"Local QA pipeline for deterministic QA Runner Debug Chat smoke tests.",
emoji: "QA",
},
});
@@ -126,7 +142,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
};
}
const pipelineId = createdResponse.json.data?.uuid || "";
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
{ token },
);
pipeline = loaded.json.data?.pipeline || null;
created = true;
}
@@ -138,7 +158,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
};
}
const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token });
const loaded = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{ token },
);
if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
return {
status: "fail",
@@ -149,9 +173,15 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
}
pipeline = loaded.json.data.pipeline;
const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {};
const config =
pipeline.config && typeof pipeline.config === "object"
? pipeline.config
: {};
const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {};
const runnerConfigs =
ai.runner_config && typeof ai.runner_config === "object"
? ai.runner_config
: {};
const updatedConfig = {
...config,
ai: {
@@ -168,16 +198,21 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
},
};
const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, {
method: "PUT",
token,
body: {
name: pipelineName,
description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
const updateResponse = await apiJson(
backendUrl,
`/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
{
method: "PUT",
token,
body: {
name: pipelineName,
description:
"Local QA pipeline for deterministic QA Runner Debug Chat smoke tests.",
emoji: "QA",
config: updatedConfig,
},
},
});
);
if (isApiFailure(updateResponse)) {
return {
status: "fail",
@@ -189,7 +224,9 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
return {
status: "pass",
reason: created ? "QA AgentRunner pipeline created and configured." : "QA AgentRunner pipeline updated.",
reason: created
? "QA Runner pipeline created and configured."
: "QA Runner pipeline updated.",
pipeline_id: pipeline.uuid,
pipeline_name: pipelineName,
created,
@@ -198,7 +235,12 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne
}
function isApiFailure(response) {
return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0);
return (
response.status >= 400 ||
(response.json &&
response.json.code !== undefined &&
response.json.code !== 0)
);
}
async function upsertEnvLocal(path, values) {
skills/scripts/e2e/wizard-runner-marketplace-catalog.mjs
+20 -13
View File
@@ -20,7 +20,10 @@ await loadEnvFiles();
const paths = evidencePaths(caseId);
await ensureEvidence(paths);
const mobileScreenshot = paths.screenshot.replace(/\.png$/, "-mobile.png");
const installedScreenshot = paths.screenshot.replace(/\.png$/, "-installed.png");
const installedScreenshot = paths.screenshot.replace(
/\.png$/,
"-installed.png",
);
const startedAt = new Date();
let frontendUrl = "";
@@ -84,7 +87,7 @@ try {
}
try {
const payload = request.postDataJSON();
if (payload?.component_filter === "AgentRunner") {
if (payload?.component_filter === "Runner") {
result.marketplace_request = {
endpoint: new URL(request.url()).pathname,
component_filter: payload.component_filter,
@@ -98,14 +101,17 @@ try {
});
page.on("response", async (response) => {
const pathname = new URL(response.url()).pathname;
if (!/\/api\/v1\/marketplace\/(extensions|plugins)\/search$/.test(pathname)) {
if (
!/\/api\/v1\/marketplace\/(extensions|plugins)\/search$/.test(pathname)
) {
return;
}
try {
const payload = await response.json();
const entries = payload?.data?.extensions || payload?.data?.plugins || [];
const localAgent = entries.find(
(entry) => `${entry.author}/${entry.name}` === "langbot-team/LocalAgent",
(entry) =>
`${entry.author}/${entry.name}` === "langbot-team/LocalAgent",
);
result.marketplace_response = {
endpoint: pathname,
@@ -228,7 +234,7 @@ try {
});
await browseLink.waitFor();
const href = await browseLink.getAttribute("href");
if (href !== "/home/extensions?type=plugin&component=AgentRunner") {
if (href !== "/home/extensions?type=plugin&component=Runner") {
throw new Error(`Unexpected Runner marketplace URL: ${href}`);
}
const nextButton = page.getByRole("button", {
@@ -238,9 +244,7 @@ try {
throw new Error("Wizard allowed continuing without an installed Runner.");
}
if (!result.marketplace_request) {
throw new Error(
"Wizard did not request the AgentRunner Marketplace catalog.",
);
throw new Error("Wizard did not request the Runner Marketplace catalog.");
}
if (
!result.marketplace_response?.local_agent_present ||
@@ -287,10 +291,14 @@ try {
.then(() => "failed"),
]);
if (installOutcome === "failed") {
throw new Error("LocalAgent installation failed before Runner registration.");
throw new Error(
"LocalAgent installation failed before Runner registration.",
);
}
if (await nextButton.isDisabled()) {
throw new Error("Create & Deploy remained disabled after LocalAgent installation.");
throw new Error(
"Create & Deploy remained disabled after LocalAgent installation.",
);
}
const [installedPluginsResponse, installedMetadataResponse] =
@@ -298,8 +306,7 @@ try {
apiJson(backendUrl, "/api/v1/plugins", { token }),
apiJson(backendUrl, "/api/v1/pipelines/_/metadata", { token }),
]);
const postInstallPlugins =
installedPluginsResponse.json.data?.plugins || [];
const postInstallPlugins = installedPluginsResponse.json.data?.plugins || [];
const installedRunnerStage = installedMetadataResponse.json.data?.configs
?.find((config) => config.name === "ai")
?.stages?.find((stage) => stage.name === "runner");
@@ -338,7 +345,7 @@ try {
}
result.status = "pass";
result.reason =
"A clean first-run instance discovered LocalAgent in the AgentRunner catalog, installed and registered it, selected it, and enabled Create & Deploy.";
"A clean first-run instance discovered LocalAgent in the Runner catalog, installed and registered it, selected it, and enabled Create & Deploy.";
} catch (error) {
if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail";
result.reason = result.reason || error.message;
skills/scripts/e2e/workspace-compatibility-preflight.mjs
+156 -45
View File
@@ -42,26 +42,83 @@ const result = {
};
const repositories = [
{ id: "langbot", directory: "LangBot", envKey: "LANGBOT_REPO", manifest: false },
{ id: "plugin-sdk", directory: "langbot-plugin-sdk", envKey: "LANGBOT_PLUGIN_SDK_REPO", manifest: false },
{ id: "agent-runner", directory: "langbot-agent-runner", envKey: "LANGBOT_AGENT_RUNNER_REPO", manifest: false },
{ id: "local-agent", directory: "langbot-local-agent", envKey: "LANGBOT_LOCAL_AGENT_REPO", identity: "langbot-team/LocalAgent" },
{ id: "control-plane", directory: "langbot-agent-control-plane", envKey: "LANGBOT_AGENT_CONTROL_PLANE_REPO", identity: "langbot/agent-control-plane" },
{ id: "longterm-memory", directory: "langbot-longterm-memory", envKey: "LANGBOT_LONGTERM_MEMORY_REPO", identity: "langbot-team/LongTermMemory" },
{ id: "parser", directory: "langbot-parser", envKey: "LANGBOT_PARSER_PLUGIN_REPO", identity: "langbot-team/GeneralParsers" },
{ id: "rag", directory: "langbot-rag", envKey: "LANGBOT_RAG_PLUGIN_REPO", identity: "langbot-team/LangRAG" },
{ id: "skill-authoring", directory: "langbot-skill-authoring", envKey: "LANGBOT_SKILL_AUTHORING_REPO", identity: "huanghuoguoguo/skill-authoring" },
{
id: "langbot",
directory: "LangBot",
envKey: "LANGBOT_REPO",
manifest: false,
},
{
id: "plugin-sdk",
directory: "langbot-plugin-sdk",
envKey: "LANGBOT_PLUGIN_SDK_REPO",
manifest: false,
},
{
id: "agent-runner",
directory: "langbot-agent-runner",
envKey: "LANGBOT_RUNNER_REPO",
manifest: false,
},
{
id: "local-agent",
directory: "langbot-local-agent",
envKey: "LANGBOT_LOCAL_AGENT_REPO",
identity: "langbot-team/LocalAgent",
},
{
id: "control-plane",
directory: "langbot-agent-control-plane",
envKey: "LANGBOT_AGENT_CONTROL_PLANE_REPO",
identity: "langbot/agent-control-plane",
},
{
id: "longterm-memory",
directory: "langbot-longterm-memory",
envKey: "LANGBOT_LONGTERM_MEMORY_REPO",
identity: "langbot-team/LongTermMemory",
},
{
id: "parser",
directory: "langbot-parser",
envKey: "LANGBOT_PARSER_PLUGIN_REPO",
identity: "langbot-team/GeneralParsers",
},
{
id: "rag",
directory: "langbot-rag",
envKey: "LANGBOT_RAG_PLUGIN_REPO",
identity: "langbot-team/LangRAG",
},
{
id: "skill-authoring",
directory: "langbot-skill-authoring",
envKey: "LANGBOT_SKILL_AUTHORING_REPO",
identity: "huanghuoguoguo/skill-authoring",
},
];
function run(command, args, options = {}) {
return new Promise((resolvePromise) => {
const child = spawn(command, args, { ...options, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
const child = spawn(command, args, {
...options,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
let stdout = "";
let stderr = "";
child.stdout.on("data", (chunk) => { stdout += chunk; });
child.stderr.on("data", (chunk) => { stderr += chunk; });
child.on("error", (error) => resolvePromise({ status: null, stdout, stderr, error }));
child.on("close", (status) => resolvePromise({ status, stdout, stderr, error: null }));
child.stdout.on("data", (chunk) => {
stdout += chunk;
});
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
child.on("error", (error) =>
resolvePromise({ status: null, stdout, stderr, error }),
);
child.on("close", (status) =>
resolvePromise({ status, stdout, stderr, error: null }),
);
});
}
@@ -71,31 +128,50 @@ function addCheck(name, status, detail = {}) {
try {
const langbotRepo = await resolveLangBotRepo();
const workspaceRoot = resolve(env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo));
const workspaceRoot = resolve(
env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo),
);
const resolved = {};
for (const repository of repositories) {
const path = resolve(env[repository.envKey] || (repository.id === "langbot" ? langbotRepo : join(workspaceRoot, repository.directory)));
const path = resolve(
env[repository.envKey] ||
(repository.id === "langbot"
? langbotRepo
: join(workspaceRoot, repository.directory)),
);
resolved[repository.id] = path;
try {
await access(join(path, ".git"));
} catch {
addCheck(`repo:${repository.id}`, "fail", { path, reason: "Git checkout is missing." });
addCheck(`repo:${repository.id}`, "fail", {
path,
reason: "Git checkout is missing.",
});
continue;
}
const branch = await run("git", ["branch", "--show-current"], { cwd: path });
const branch = await run("git", ["branch", "--show-current"], {
cwd: path,
});
const branchName = branch.stdout.trim();
const compatible = branch.status === 0 && /^(?:main|dev\/4\.11\.x)$/.test(branchName);
const compatible =
branch.status === 0 && /^(?:main|dev\/4\.11\.x)$/.test(branchName);
addCheck(`repo:${repository.id}`, compatible ? "pass" : "fail", {
path,
branch: branchName,
reason: compatible ? "" : "Expected main or dev/4.11.x compatibility branch.",
reason: compatible
? ""
: "Expected main or dev/4.11.x compatibility branch.",
});
const dirty = await run("git", ["status", "--short"], { cwd: path });
if (dirty.stdout.trim()) {
result.warnings.push({ name: `dirty:${repository.id}`, path, entries: dirty.stdout.trim().split(/\r?\n/).length });
result.warnings.push({
name: `dirty:${repository.id}`,
path,
entries: dirty.stdout.trim().split(/\r?\n/).length,
});
}
if (repository.identity) {
@@ -105,13 +181,20 @@ try {
const author = manifest.match(/^\s{2}author:\s*([^\s#]+)/m)?.[1] || "";
const name = manifest.match(/^\s{2}name:\s*([^\s#]+)/m)?.[1] || "";
const identity = `${author}/${name}`;
addCheck(`manifest:${repository.id}`, identity === repository.identity ? "pass" : "fail", {
path: manifestPath,
identity,
expected_identity: repository.identity,
});
addCheck(
`manifest:${repository.id}`,
identity === repository.identity ? "pass" : "fail",
{
path: manifestPath,
identity,
expected_identity: repository.identity,
},
);
} catch (error) {
addCheck(`manifest:${repository.id}`, "fail", { path: manifestPath, reason: error.message });
addCheck(`manifest:${repository.id}`, "fail", {
path: manifestPath,
reason: error.message,
});
}
}
}
@@ -121,41 +204,69 @@ try {
await access(python);
addCheck("langbot-venv", "pass", { python });
} catch {
addCheck("langbot-venv", "fail", { python, reason: "LangBot virtualenv Python is missing." });
addCheck("langbot-venv", "fail", {
python,
reason: "LangBot virtualenv Python is missing.",
});
}
const sdkSrc = join(resolved["plugin-sdk"], "src");
const importProbe = await run(python, ["-c", [
"import json, pathlib, langbot_plugin",
"from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput",
"from langbot_plugin.api.entities.builtin.agent_runner.result import AgentRunResult",
"print(json.dumps({'path': str(pathlib.Path(langbot_plugin.__file__).resolve()), 'entities': [AgentInput.__name__, AgentRunResult.__name__]}))",
].join("; ")], {
cwd: resolved.langbot,
env: { ...env, PYTHONPATH: [sdkSrc, env.PYTHONPATH].filter(Boolean).join(delimiter) },
});
const importProbe = await run(
python,
[
"-c",
[
"import json, pathlib, langbot_plugin",
"from langbot_plugin.api.entities.builtin.runner.input import AgentInput",
"from langbot_plugin.api.entities.builtin.runner.result import RunnerResult",
"print(json.dumps({'path': str(pathlib.Path(langbot_plugin.__file__).resolve()), 'entities': [AgentInput.__name__, RunnerResult.__name__]}))",
].join("; "),
],
{
cwd: resolved.langbot,
env: {
...env,
PYTHONPATH: [sdkSrc, env.PYTHONPATH].filter(Boolean).join(delimiter),
},
},
);
let importDetail = {};
try { importDetail = JSON.parse(importProbe.stdout.trim()); } catch { importDetail = { stderr: importProbe.stderr.trim() }; }
const localSdkLoaded = importProbe.status === 0 && resolve(importDetail.path || "").startsWith(resolve(sdkSrc));
try {
importDetail = JSON.parse(importProbe.stdout.trim());
} catch {
importDetail = { stderr: importProbe.stderr.trim() };
}
const localSdkLoaded =
importProbe.status === 0 &&
resolve(importDetail.path || "").startsWith(resolve(sdkSrc));
addCheck("local-sdk-import", localSdkLoaded ? "pass" : "fail", {
expected_root: resolve(sdkSrc),
...importDetail,
reason: localSdkLoaded ? "" : "langbot_plugin did not load from the workspace SDK source tree.",
reason: localSdkLoaded
? ""
: "langbot_plugin did not load from the workspace SDK source tree.",
});
const failures = result.checks.filter((check) => check.status === "fail");
result.status = failures.length === 0 ? "pass" : "fail";
result.reason = failures.length === 0
? `Workspace compatibility preflight passed with ${result.warnings.length} non-blocking dirty-worktree warning(s).`
: `Workspace compatibility preflight found ${failures.length} blocking check(s).`;
result.reason =
failures.length === 0
? `Workspace compatibility preflight passed with ${result.warnings.length} non-blocking dirty-worktree warning(s).`
: `Workspace compatibility preflight found ${failures.length} blocking check(s).`;
} catch (error) {
result.status = /missing|ENOENT|not found/i.test(error.message) ? "env_issue" : "fail";
result.status = /missing|ENOENT|not found/i.test(error.message)
? "env_issue"
: "fail";
result.reason = error.message;
} finally {
const finishedAt = new Date();
result.finished_at = finishedAt.toISOString();
result.finished_at_local = localIsoWithOffset(finishedAt);
await writeFile(detailsPath, `${JSON.stringify({ checks: result.checks, warnings: result.warnings }, null, 2)}\n`, "utf8");
await writeFile(
detailsPath,
`${JSON.stringify({ checks: result.checks, warnings: result.warnings }, null, 2)}\n`,
"utf8",
);
await writeResult(paths, result);
console.log(JSON.stringify(result, null, 2));
}