xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-10-30 20:35:22 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
test/behavior-evidence
LangBot/tools/regression_contracts.json
T

284 lines
15 KiB
JSON
Raw Permalink Blame History

{
"version": 1,
"scope": "Registered CLI/HTTP Bot, conversation, model, tool and retrieval behavior obligations plus explicit adjacent-risk gaps. This is not all product features, all API endpoints, code coverage, or real-model quality coverage. Case repetition does not increase the denominator.",
"contracts": [
{"id": "configuration.identity",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "CLI resolves the configured Workspace identity",
"bindings": [{"profile": "smoke", "case": "identity", "checks": ["response"]}]
},
{"id": "configuration.plugins",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "Installed LocalAgent is visible through the plugin API",
"bindings": [{"profile": "smoke", "case": "plugins", "checks": ["response"]}],
"routes": ["GET /api/v1/plugins"]
},
{"id": "configuration.provider_readback",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "Created provider is visible through the public API",
"bindings": [{"profile": "smoke", "case": "provider_readback", "checks": ["response"]}],
"routes": ["GET /api/v1/provider/providers/<provider_uuid>"]
},
{"id": "configuration.model_readback",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "Created model is visible through the public API",
"bindings": [{"profile": "smoke", "case": "model_readback", "checks": ["response"]}],
"routes": ["GET /api/v1/provider/models/llm/<model_uuid>"]
},
{"id": "configuration.pipeline_readback",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "Pipeline readback includes its configured model",
"bindings": [{"profile": "smoke", "case": "pipeline_readback", "checks": ["response"]}],
"routes": ["GET /api/v1/pipelines/<pipeline_uuid>"]
},
{"id": "configuration.kb_readback",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "Knowledge-base identity and engine match configuration",
"bindings": [{"profile": "smoke", "case": "kb_readback", "checks": ["response", "fields"]}],
"routes": ["GET /api/v1/knowledge/bases/<knowledge_base_uuid>"]
},
{"id": "configuration.document_readback",
"area": "configuration", "kind": "blackbox", "priority": "P1",
"description": "Uploaded document completes ingestion in its owning knowledge base",
"bindings": [{"profile": "smoke", "case": "document_readback", "checks": ["response", "fields"]}],
"routes": ["GET /api/v1/knowledge/bases/<knowledge_base_uuid>/files"]
},
{"id": "ingress.missing_signature",
"area": "ingress", "kind": "blackbox", "priority": "P0",
"description": "Unsigned bot messages are rejected",
"bindings": [{"profile": "smoke", "case": "missing_signature", "checks": ["response", "fields"]}]
},
{"id": "ingress.bad_signature",
"area": "ingress", "kind": "blackbox", "priority": "P0",
"description": "Messages with an invalid signature are rejected",
"bindings": [{"profile": "smoke", "case": "bad_signature", "checks": ["response", "fields"]}]
},
{"id": "conversation.chat",
"area": "conversation", "kind": "blackbox", "priority": "P1",
"description": "Pipeline returns the expected chat response",
"bindings": [{"profile": "smoke", "case": "chat", "checks": ["response"]}, {"profile": "endurance", "case": "chat", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "conversation.unicode",
"area": "conversation", "kind": "blackbox", "priority": "P1",
"description": "Unicode input and streamed output survive the full conversation chain",
"bindings": [{"profile": "smoke", "case": "unicode", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "model.slow",
"area": "model", "kind": "blackbox", "priority": "P1",
"description": "Slow streamed model output reaches the caller",
"bindings": [{"profile": "smoke", "case": "slow", "checks": ["response", "model_sequence"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "model.fallback401",
"area": "model", "kind": "blackbox", "priority": "P1",
"description": "Authentication failure falls back to the configured secondary model",
"bindings": [{"profile": "smoke", "case": "fallback401", "checks": ["response", "model_sequence"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "model.fallback429",
"area": "model", "kind": "blackbox", "priority": "P1",
"description": "Rate limiting falls back to the configured secondary model",
"bindings": [{"profile": "smoke", "case": "fallback429", "checks": ["response", "model_sequence"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "model.fallback503",
"area": "model", "kind": "blackbox", "priority": "P1",
"description": "Unavailable primary model falls back to the secondary model",
"bindings": [{"profile": "smoke", "case": "fallback503", "checks": ["response", "model_sequence"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "tools.exec_error",
"area": "tools", "kind": "blackbox", "priority": "P1",
"description": "Tool error output is delivered to the model and handled",
"bindings": [{"profile": "smoke", "case": "exec_error", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "tools.parallel",
"area": "tools", "kind": "blackbox", "priority": "P1",
"description": "Parallel tool calls return matched results to the model",
"bindings": [{"profile": "smoke", "case": "parallel", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "delivery.unicode_export",
"area": "delivery", "kind": "blackbox", "priority": "P1",
"description": "Exported Unicode file name and bytes survive delivery",
"bindings": [{"profile": "smoke", "case": "unicode_export", "checks": ["response", "attachment"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "retrieval.other_rag",
"area": "retrieval", "kind": "blackbox", "priority": "P1",
"description": "A second knowledge base supplies its own content to the model",
"bindings": [{"profile": "smoke", "case": "other_rag", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "tools.box",
"area": "tools", "kind": "blackbox", "priority": "P1",
"description": "A real Box write/read round trip reaches the final reply",
"bindings": [{"profile": "smoke", "case": "box", "checks": ["response"]}, {"profile": "endurance", "case": "box", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "delivery.box_export",
"area": "delivery", "kind": "blackbox", "priority": "P1",
"description": "Exported file bytes match the generated fixture",
"bindings": [{"profile": "smoke", "case": "box_export", "checks": ["response", "attachment"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "retrieval.rag_chat",
"area": "retrieval", "kind": "blackbox", "priority": "P1",
"description": "Retrieved fixture content reaches the model and final reply",
"bindings": [{"profile": "smoke", "case": "rag_chat", "checks": ["response"]}, {"profile": "endurance", "case": "rag_chat", "checks": ["response"]}],
"routes": ["POST /api/v1/pipelines/<pipeline_uuid>/run"]
},
{"id": "retrieval.rag_retrieve",
"area": "retrieval", "kind": "blackbox", "priority": "P0",
"description": "Retrieval returns the owning knowledge-base marker without the other fixture",
"bindings": [{"profile": "smoke", "case": "rag_retrieve", "checks": ["response", "fields", "isolation"]}],
"routes": ["POST /api/v1/knowledge/bases/<knowledge_base_uuid>/retrieve"]
},
{"id": "retrieval.retrieve_other",
"area": "retrieval", "kind": "blackbox", "priority": "P0",
"description": "Retrieval from the second knowledge base excludes the first fixture",
"bindings": [{"profile": "smoke", "case": "retrieve_other", "checks": ["response", "fields", "isolation"]}],
"routes": ["POST /api/v1/knowledge/bases/<knowledge_base_uuid>/retrieve"]
},
{"id": "retrieval.retrieve_empty",
"area": "retrieval", "kind": "blackbox", "priority": "P1",
"description": "Empty knowledge base returns an empty retrieval result",
"bindings": [{"profile": "smoke", "case": "retrieve_empty", "checks": ["response", "fields"]}],
"routes": ["POST /api/v1/knowledge/bases/<knowledge_base_uuid>/retrieve"]
},
{"id": "conversation.group_chat_history",
"area": "conversation", "kind": "blackbox", "priority": "P0",
"description": "Multi-turn chat preserves the prior reply without mixing group history",
"bindings": [{"profile": "smoke", "case": "group_chat_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "chat", "checks": ["response"], "min_turn": 2}]
},
{"id": "conversation.group_box_history",
"area": "conversation", "kind": "blackbox", "priority": "P1",
"description": "Multi-turn Box conversation preserves group history",
"bindings": [{"profile": "smoke", "case": "group_box_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "box", "checks": ["response"], "min_turn": 2}]
},
{"id": "conversation.group_rag_history",
"area": "conversation", "kind": "blackbox", "priority": "P1",
"description": "Multi-turn RAG conversation preserves group history",
"bindings": [{"profile": "smoke", "case": "group_rag_history", "checks": ["response"], "min_turn": 2}, {"profile": "endurance", "case": "rag_chat", "checks": ["response"], "min_turn": 2}]
},
{"id": "auth.invalid_credentials",
"area": "auth", "kind": "blackbox", "priority": "P0",
"description": "Missing and invalid API credentials cannot access Workspace resources",
"bindings": []
},
{"id": "auth.revocation",
"area": "auth", "kind": "blackbox", "priority": "P0",
"description": "Revoked credentials stop authorizing new requests",
"bindings": []
},
{"id": "workspace.read_isolation",
"area": "workspace", "kind": "blackbox", "priority": "P0",
"description": "One Workspace cannot read another Workspace resource",
"bindings": []
},
{"id": "workspace.write_isolation",
"area": "workspace", "kind": "blackbox", "priority": "P0",
"description": "One Workspace cannot change another Workspace resource",
"bindings": []
},
{"id": "execution.cancel_cleanup",
"area": "execution", "kind": "blackbox", "priority": "P0",
"description": "Cancelling a running execution reaches cancelled state and releases its work",
"bindings": []
},
{"id": "execution.timeout_cleanup",
"area": "execution", "kind": "blackbox", "priority": "P0",
"description": "Deadline expiry stops child work and allows a subsequent request",
"bindings": []
},
{"id": "execution.retry_side_effects",
"area": "execution", "kind": "blackbox", "priority": "P0",
"description": "Retries do not duplicate externally visible side effects",
"bindings": []
},
{"id": "runtime.restart_recovery",
"area": "runtime", "kind": "blackbox", "priority": "P0",
"description": "A runtime restart produces a known outcome and subsequent execution works",
"bindings": []
},
{"id": "storage.restart_history",
"area": "storage", "kind": "blackbox", "priority": "P1",
"description": "Conversation history remains usable across a Core restart",
"bindings": []
},
{"id": "model.stream_disconnect",
"area": "model", "kind": "blackbox", "priority": "P1",
"description": "A partial model stream disconnect is handled without a false successful completion",
"bindings": []
},
{"id": "model.exhausted_fallback",
"area": "model", "kind": "blackbox", "priority": "P1",
"description": "Failure of every configured model reaches a visible failed outcome",
"bindings": []
},
{"id": "tools.path_escape",
"area": "tools", "kind": "blackbox", "priority": "P0",
"description": "Box operations reject access outside their authorized filesystem scope",
"bindings": []
},
{"id": "tools.timeout_recovery",
"area": "tools", "kind": "blackbox", "priority": "P1",
"description": "Timed-out tools stop running and the next tool can execute",
"bindings": []
},
{"id": "delivery.callback_retry",
"area": "delivery", "kind": "blackbox", "priority": "P1",
"description": "Failed delivery callbacks retry according to policy without duplicate effects",
"bindings": []
},
{"id": "retrieval.update_delete",
"area": "retrieval", "kind": "blackbox", "priority": "P1",
"description": "Updated/deleted documents change retrieval results without stale leakage",
"bindings": []
},
{"id": "plugins.upgrade_recovery",
"area": "plugins", "kind": "blackbox", "priority": "P1",
"description": "Plugin upgrade failure preserves a usable installation",
"bindings": []
},
{"id": "agent.direct_execution",
"area": "agent", "kind": "blackbox", "priority": "P1",
"description": "Standalone Agent entrypoint completes through the public API",
"bindings": []
},
{"id": "persistence.postgres_parity",
"area": "persistence", "kind": "blackbox", "priority": "P1",
"description": "The same externally visible run contract holds on PostgreSQL",
"bindings": []
},
{"id": "platform.real_adapter",
"area": "platform", "kind": "blackbox", "priority": "P2",
"description": "An actual messaging-platform adapter receives and delivers the fixture",
"bindings": []
},
{"id": "retrieval.semantic_embedding",
"area": "retrieval", "kind": "blackbox", "priority": "P2",
"description": "A real embedding model retrieves relevant fixed documents",
"bindings": []
},
{"id": "execution.persisted_terminal",
"area": "execution", "kind": "graybox", "priority": "P0",
"description": "Each executed request has a unique completed persisted run and terminal event",
"bindings": [{"profile": "smoke", "case": "chat", "checks": ["ledger"]}, {"profile": "endurance", "case": "chat", "checks": ["ledger"]}]
},
{"id": "tools.persisted_box_results",
"area": "tools", "kind": "graybox", "priority": "P0",
"description": "Persisted Box events match backend, exit status and output",
"bindings": [{"profile": "smoke", "case": "box", "checks": ["ledger"]}, {"profile": "endurance", "case": "box", "checks": ["ledger"]}]
},
{"id": "ingress.no_run_on_reject",
"area": "ingress", "kind": "graybox", "priority": "P0",
"description": "Rejected ingress does not create an AgentRun",
"bindings": [{"profile": "smoke", "case": "missing_signature", "checks": ["no_run"]}]
}
]
}
Reference in New Issue View Git Blame Copy Permalink