mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-08 18:47:14 +00:00
huanghuoguoguo
45 lines
2.1 KiB
YAML
45 lines
2.1 KiB
YAML
id: agent-run-ledger-audit
|
|
title: "Persisted AgentRunner run ledger passes end-to-end invariants"
|
|
mode: probe
|
|
area: agent
|
|
type: regression
|
|
priority: p0
|
|
risk: high
|
|
ci_eligible: false
|
|
tags:
|
|
- agent-runner
|
|
- ledger
|
|
- audit
|
|
- tools
|
|
skills:
|
|
- langbot-testing
|
|
automation: scripts/e2e/agent-run-ledger-audit.mjs
|
|
steps:
|
|
- "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted AgentRunner run."
|
|
- "For an external runner's own CLI tools, set LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE=runner-native; keep the default strict mode for Host tool calls."
|
|
- "Read the active LangBot database configuration and inspect the selected run and its ordered events."
|
|
- "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering."
|
|
- "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema."
|
|
- "Reject malformed persisted event JSON, unrecovered invalid tool arguments, timeout, forbidden, permission-denied, or unauthorized signals."
|
|
- "Record a malformed model tool-argument payload as a recovery warning only when a later tool call succeeds and the run completes."
|
|
checks:
|
|
- "ledger-audit.json status is pass."
|
|
- "metrics has equal tool_call_started and tool_call_completed counts."
|
|
- "failures is empty and no unauthorized tool call is present."
|
|
evidence_required:
|
|
- filesystem
|
|
- metrics
|
|
- api_diagnostic
|
|
diagnostics:
|
|
- "Run this immediately after a complex UI Agent task so latest-run selection cannot drift to unrelated traffic."
|
|
- "Use LANGBOT_AGENT_RUN_ID when multiple operators share the test instance."
|
|
- "runner-native mode still checks tool pairing, stable names, ordering, terminal state, and error signals; the external runner's permission system owns its local tools."
|
|
success_patterns:
|
|
- "Agent run ledger audit passed"
|
|
failure_patterns:
|
|
- "tool_call_pairing"
|
|
- "forbidden_error_signals"
|
|
- "unauthorized_tool_calls"
|
|
troubleshooting:
|
|
- box-runtime-silent-action-timeout
|