test(ci): add daily Dify Cloud Agent regression

Add daily Dify Cloud Agent regression via Runner
2026-11-01 05:15:22 +08:00 · 2026-10-31 15:35:55 +08:00
parent a6ed298f2a
commit f750bb96ac
2 changed files with 213 additions and 0 deletions
@@ -249,3 +249,51 @@ jobs:
            .qa/api/*.log
          if-no-files-found: warn
          retention-days: 30
+
+  dify-cloud:
+    name: Daily Dify Cloud Agent regression
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: plugins/Runner
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Install locked Runner dependencies
+        run: uv sync --dev --frozen
+      - name: Run live Dify Agent regression
+        env:
+          DIFY_AGENT_API_KEY: ${{ secrets.DIFY_AGENT_API_KEY }}
+          DIFY_LIVE_OUTPUT: ${{ github.workspace }}/.qa/dify-live/summary.json
+        run: |
+          if [[ -z "$DIFY_AGENT_API_KEY" ]]; then
+            if [[ "$GITHUB_EVENT_NAME" == "schedule" ]]; then
+              echo "DIFY_AGENT_API_KEY is required for the scheduled live Dify regression." >&2
+              exit 1
+            fi
+            echo "DIFY_AGENT_API_KEY is not configured; skipping manual live Dify regression."
+            exit 0
+          fi
+          uv run --no-sync python http-certification-tests/live_dify_agent.py
+      - name: Publish Dify regression summary
+        if: always()
+        run: |
+          if [[ -f "$GITHUB_WORKSPACE/.qa/dify-live/summary.json" ]]; then
+            cat "$GITHUB_WORKSPACE/.qa/dify-live/summary.json" >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Preserve Dify regression evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: daily-dify-${{ github.run_id }}-${{ github.run_attempt }}
+          path: .qa/dify-live/summary.json
+          if-no-files-found: warn
+          retention-days: 30
@@ -0,0 +1,165 @@
+"""Live Dify Cloud regression for the Dify Agent Runner."""
+
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import json
+import os
+import sys
+from pathlib import Path
+
+from langbot_plugin.api.entities.builtin.runner import (
+    ActorContext,
+    AgentEventContext,
+    AgentInput,
+    AgentResources,
+    AgentRunState,
+    AgentRuntimeContext,
+    AgentTrigger,
+    ConversationContext,
+    DeliveryContext,
+    RunnerContext,
+)
+
+ROOT = Path(__file__).resolve().parents[1]
+PLUGIN = ROOT / "dify-agent"
+SENTINEL = "LANGBOT_DIFY_DAILY_OK"
+
+
+def _load_runner():
+    saved = {key: value for key, value in sys.modules.items() if key == "pkg" or key.startswith("pkg.")}
+    for key in saved:
+        del sys.modules[key]
+    sys.path.insert(0, str(PLUGIN))
+    try:
+        spec = importlib.util.spec_from_file_location("live_dify_agent_runner", PLUGIN / "components/runner/default.py")
+        if spec is None or spec.loader is None:
+            raise RuntimeError("cannot load Dify Agent runner")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module, saved
+    except Exception:
+        sys.path.remove(str(PLUGIN))
+        for key in list(sys.modules):
+            if key == "pkg" or key.startswith("pkg."):
+                del sys.modules[key]
+        sys.modules.update(saved)
+        raise
+
+
+def _context() -> RunnerContext:
+    return RunnerContext(
+        run_id="dify-cloud-daily-regression",
+        trigger=AgentTrigger(type="message.received"),
+        event=AgentEventContext(event_id="dify-cloud-daily-regression", event_type="message.received", source="ci"),
+        conversation=ConversationContext(conversation_id="daily-regression", session_id="daily-regression"),
+        input=AgentInput(text=f"Reply with exactly {SENTINEL} and nothing else. Do not call tools."),
+        delivery=DeliveryContext(surface="ci", supports_streaming=True),
+        resources=AgentResources(),
+        state=AgentRunState(),
+        runtime=AgentRuntimeContext(),
+        actor=ActorContext(actor_type="user", actor_id="daily-regression"),
+        config={},
+    )
+
+
+def _kind(result) -> str:
+    return getattr(result.type, "value", str(result.type))
+
+
+def _summary_path() -> Path:
+    return Path(os.environ.get("DIFY_LIVE_OUTPUT", ".qa/dify-live/summary.json"))
+
+
+def _write_summary(payload: dict) -> None:
+    path = _summary_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+
+async def _run() -> None:
+    api_key = os.environ.get("DIFY_AGENT_API_KEY", "")
+    base_url = os.environ.get("DIFY_AGENT_BASE_URL", "https://api.dify.ai/v1")
+    timeout = float(os.environ.get("DIFY_AGENT_TIMEOUT", "120"))
+    if not api_key:
+        raise RuntimeError("DIFY_AGENT_API_KEY is required for the live regression")
+
+    module, saved_modules = _load_runner()
+    try:
+        client = module.AsyncDifyClient(api_key=api_key, base_url=base_url, timeout=timeout)
+        event_types: list[str] = []
+
+        class ObservedClient:
+            base_url = client.base_url
+
+            async def chat_messages(self, **kwargs):
+                async for event in client.chat_messages(**kwargs):
+                    event_types.append(str(event.get("event", "")))
+                    yield event
+
+        runner = object.__new__(module.DefaultRunner)
+        context = _context()
+        results = [
+            result
+            async for result in runner._run_chat_or_agent(
+                context,
+                ObservedClient(),
+                {},
+                context.input.to_text(),
+                "langbot-daily-regression",
+                "",
+                [],
+                "agent",
+                False,
+            )
+        ]
+        final_chunks = [
+            result.data["chunk"]
+            for result in results
+            if _kind(result) == "message.delta" and result.data.get("chunk", {}).get("is_final")
+        ]
+        final_text = "".join(str(chunk.get("content") or "") for chunk in final_chunks)
+        # Agent apps may emit agent_message directly without a separate message event.
+        required_events = {"agent_message", "message_end"}
+        missing_events = sorted(required_events.difference(event_types))
+        if missing_events:
+            raise AssertionError(f"Dify Agent stream did not include required events: {missing_events}")
+        if len(final_chunks) != 1:
+            raise AssertionError(f"expected exactly one final message chunk, got {len(final_chunks)}")
+        if final_text.count(SENTINEL) != 1:
+            raise AssertionError("final Dify answer was missing or repeated")
+        if not any(_kind(result) == "run.completed" for result in results):
+            raise AssertionError("Dify Agent run did not complete")
+
+        _write_summary(
+            {
+                "passed": True,
+                "event_types": event_types,
+                "final_chunk_count": len(final_chunks),
+                "sentinel_count": final_text.count(SENTINEL),
+            }
+        )
+        print(f"Dify live regression passed: events={event_types}; final_chunks={len(final_chunks)}")
+    finally:
+        sys.path.remove(str(PLUGIN))
+        for key in list(sys.modules):
+            if key == "pkg" or key.startswith("pkg."):
+                del sys.modules[key]
+        sys.modules.update(saved_modules)
+
+
+def main() -> int:
+    try:
+        asyncio.run(_run())
+    except Exception as exc:
+        api_key = os.environ.get("DIFY_AGENT_API_KEY", "")
+        message = str(exc).replace(api_key, "<redacted>") if api_key else str(exc)
+        _write_summary({"passed": False, "error": message})
+        print(f"Dify live regression failed: {message}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())