diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index 0f721cf46..f9620d7dc 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -296,4 +296,97 @@ jobs: name: daily-dify-${{ github.run_id }}-${{ github.run_attempt }} path: .qa/dify-live/summary.json if-no-files-found: warn - retention-days: 30 \ No newline at end of file + retention-days: 30 + n8n-latest: + name: Daily latest n8n Agent Runner regression + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: plugins/Runner + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Start latest n8n container + run: | + mkdir -p "$GITHUB_WORKSPACE/.qa/n8n-live" + docker pull docker.n8n.io/n8nio/n8n:latest + docker image inspect docker.n8n.io/n8nio/n8n:latest --format '{{json .RepoDigests}}' > "$GITHUB_WORKSPACE/.qa/n8n-live/image-digests.json" + docker run -d --user node --name langbot-n8n-latest -p 5678:5678 \ + -e N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=false \ + -e N8N_SECURE_COOKIE=false \ + -e N8N_DIAGNOSTICS_ENABLED=false \ + -e N8N_PERSONALIZATION_ENABLED=false \ + -e N8N_VERSION_NOTIFICATIONS_ENABLED=false \ + --entrypoint /bin/sh \ + docker.n8n.io/n8nio/n8n:latest -c 'tail -f /dev/null' + - name: Import and publish smoke workflow + run: | + docker cp http-certification-tests/fixtures/n8n_latest_runner_smoke.json langbot-n8n-latest:/tmp/n8n_latest_runner_smoke.json + docker exec -u node langbot-n8n-latest n8n import:workflow --input=/tmp/n8n_latest_runner_smoke.json + docker exec -u node langbot-n8n-latest n8n export:workflow --all --output=/tmp/n8n_export.json + docker exec -u node langbot-n8n-latest sh -c 'cat /tmp/n8n_export.json' > /tmp/n8n_export.json + workflow_id=$(python3 -c 'import json; print(json.load(open("/tmp/n8n_export.json"))[0]["id"])') + echo "Imported workflow id: $workflow_id" + docker exec -u node langbot-n8n-latest n8n publish:workflow --id="$workflow_id" + docker exec -d -u node langbot-n8n-latest n8n start + for attempt in {1..60}; do + if curl -fsS http://127.0.0.1:5678/healthz >/dev/null; then + break + fi + if [[ "$attempt" == 60 ]]; then + echo 'latest n8n did not become ready' >&2 + exit 1 + fi + sleep 2 + done + docker exec -u node langbot-n8n-latest n8n --version > "$GITHUB_WORKSPACE/.qa/n8n-live/version.txt" + - name: Verify production Webhook + run: | + for attempt in {1..30}; do + if curl -fsS -X POST http://127.0.0.1:5678/webhook/langbot-n8n-smoke \ + -H 'Content-Type: application/json' \ + -d '{"chatInput":"direct-smoke","message":"direct-smoke","user_id":"direct-user","conversation_id":"direct-conversation","session_id":"direct-session"}' \ + | tee "$GITHUB_WORKSPACE/.qa/n8n-live/direct-response.json" \ + | grep -F 'LANGBOT_N8N_DAILY_OK'; then + break + fi + if [[ "$attempt" == 30 ]]; then + echo 'latest n8n production Webhook did not become ready' >&2 + exit 1 + fi + sleep 2 + done + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install Runner dependencies + run: uv sync --dev --frozen + - name: Run LangBot n8n DefaultRunner regression + env: + N8N_LIVE_OUTPUT: ${{ github.workspace }}/.qa/n8n-live/summary.json + run: uv run --no-sync python http-certification-tests/live_n8n_agent.py + - name: Collect latest n8n evidence + if: always() + run: docker logs langbot-n8n-latest > "$GITHUB_WORKSPACE/.qa/n8n-live/container.log" 2>&1 || true + - name: Remove latest n8n container + if: always() + run: docker rm -f langbot-n8n-latest >/dev/null 2>&1 || true + - name: Publish latest n8n summary + if: always() + run: | + if [[ -f "$GITHUB_WORKSPACE/.qa/n8n-live/summary.json" ]]; then + cat "$GITHUB_WORKSPACE/.qa/n8n-live/summary.json" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Preserve latest n8n evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: daily-n8n-latest-${{ github.run_id }}-${{ github.run_attempt }} + path: .qa/n8n-live + if-no-files-found: warn + retention-days: 30 diff --git a/plugins/Runner/http-certification-tests/fixtures/n8n_latest_runner_smoke.json b/plugins/Runner/http-certification-tests/fixtures/n8n_latest_runner_smoke.json new file mode 100644 index 000000000..507dd9e1f --- /dev/null +++ b/plugins/Runner/http-certification-tests/fixtures/n8n_latest_runner_smoke.json @@ -0,0 +1,41 @@ +{ + "name": "LangBot n8n latest Runner smoke", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "langbot-n8n-smoke", + "responseMode": "responseNode", + "options": {} + }, + "id": "langbot-n8n-webhook", + "name": "Webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [0, 0], + "webhookId": "langbot-n8n-smoke" + }, + { + "parameters": { + "respondWith": "json", + "responseBody": "={{ { response: 'LANGBOT_N8N_DAILY_OK|' + $json.body.chatInput + '|' + $json.body.message + '|' + $json.body.user_id + '|' + $json.body.conversation_id + '|' + $json.body.session_id } }}", + "options": {} + }, + "id": "langbot-n8n-response", + "name": "Respond to Webhook", + "type": "n8n-nodes-base.respondToWebhook", + "typeVersion": 1.1, + "position": [300, 0] + } + ], + "connections": { + "Webhook": { + "main": [[{"node": "Respond to Webhook", "type": "main", "index": 0}]] + } + }, + "active": false, + "settings": {"executionOrder": "v1"}, + "versionId": "langbot-n8n-latest-runner-smoke-v1", + "meta": {"templateCredsSetupCompleted": true}, + "tags": [] +} diff --git a/plugins/Runner/http-certification-tests/live_n8n_agent.py b/plugins/Runner/http-certification-tests/live_n8n_agent.py new file mode 100644 index 000000000..ee1440885 --- /dev/null +++ b/plugins/Runner/http-certification-tests/live_n8n_agent.py @@ -0,0 +1,152 @@ +"""Live latest n8n regression for the LangBot n8n Agent Runner.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import json +import os +import sys +from pathlib import Path + +from langbot_plugin.api.entities.builtin.runner import ( + ActorContext, + AgentEventContext, + AgentInput, + AgentResources, + AgentRunState, + AgentRuntimeContext, + AgentTrigger, + ConversationContext, + DeliveryContext, + RunnerContext, +) + +ROOT = Path(__file__).resolve().parents[1] +PLUGIN = ROOT / "n8n-agent" +SENTINEL = "LANGBOT_N8N_DAILY_OK" + + +def _load_runner(): + saved = {key: value for key, value in sys.modules.items() if key == "pkg" or key.startswith("pkg.")} + for key in saved: + del sys.modules[key] + sys.path.insert(0, str(PLUGIN)) + try: + spec = importlib.util.spec_from_file_location("live_n8n_agent_runner", PLUGIN / "components/runner/default.py") + if spec is None or spec.loader is None: + raise RuntimeError("cannot load n8n Agent Runner") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module, saved + except Exception: + sys.path.remove(str(PLUGIN)) + for key in list(sys.modules): + if key == "pkg" or key.startswith("pkg."): + del sys.modules[key] + sys.modules.update(saved) + raise + + +def _context(state: dict[str, str] | None = None) -> RunnerContext: + return RunnerContext( + run_id="n8n-latest-daily-regression", + trigger=AgentTrigger(type="message.received"), + event=AgentEventContext( + event_id="n8n-latest-daily-regression", + event_type="message.received", + source="ci", + ), + conversation=ConversationContext(conversation_id="daily-regression", session_id="daily-regression"), + input=AgentInput(text=f"Reply with exactly {SENTINEL}."), + delivery=DeliveryContext(surface="ci", supports_streaming=True), + resources=AgentResources(), + state=AgentRunState(conversation=state or {}), + runtime=AgentRuntimeContext(), + actor=ActorContext(actor_type="user", actor_id="daily-regression"), + config={ + "webhook-url": os.environ.get("N8N_WEBHOOK_URL", "http://127.0.0.1:5678/webhook/langbot-n8n-smoke"), + "auth-type": "none", + "output-key": "response", + "response-handling": "reply", + "timeout": 30, + }, + ) + + +def _kind(result) -> str: + return getattr(result.type, "value", str(result.type)) + + +def _summary_path() -> Path: + return Path(os.environ.get("N8N_LIVE_OUTPUT", ".qa/n8n-live/summary.json")) + + +def _write_summary(payload: dict) -> None: + path = _summary_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _assert_run(results, *, expect_state_updates: bool) -> str: + types = [_kind(result) for result in results] + final_chunks = [ + result.data["chunk"] + for result in results + if _kind(result) == "message.delta" and result.data.get("chunk", {}).get("is_final") + ] + if len(final_chunks) != 1: + raise AssertionError(f"expected one final chunk, got {len(final_chunks)}: {types}") + content = str(final_chunks[0].get("content") or "") + if not content.startswith(f"{SENTINEL}|Reply with exactly {SENTINEL}."): + raise AssertionError(f"unexpected n8n response: {content}") + if "|user_daily-regression|n8n_conversation_" not in content or "|n8n_session_" not in content: + raise AssertionError(f"Runner payload/state fields missing: {content}") + if types.count("state.updated") != (2 if expect_state_updates else 0): + raise AssertionError(f"unexpected state updates: {types}") + if types[-1] != "run.completed": + raise AssertionError(f"Runner did not complete: {types}") + return content + + +async def _run() -> None: + module, saved_modules = _load_runner() + try: + runner = object.__new__(module.DefaultRunner) + first = [result async for result in runner.run(_context())] + first_content = _assert_run(first, expect_state_updates=True) + state = {result.data["key"]: result.data["value"] for result in first if _kind(result) == "state.updated"} + second = [result async for result in runner.run(_context(state))] + second_content = _assert_run(second, expect_state_updates=False) + if second_content != first_content: + raise AssertionError("persisted n8n conversation/session state changed the response") + _write_summary( + { + "passed": True, + "runner": "DefaultRunner", + "turns": 2, + "state_keys": sorted(state), + "final_content": first_content, + } + ) + print(f"n8n latest Agent Runner regression passed: state_keys={sorted(state)}") + finally: + sys.path.remove(str(PLUGIN)) + for key in list(sys.modules): + if key == "pkg" or key.startswith("pkg."): + del sys.modules[key] + sys.modules.update(saved_modules) + + +def main() -> int: + try: + asyncio.run(_run()) + except Exception as exc: + _write_summary({"passed": False, "error": str(exc)}) + print(f"n8n latest Agent Runner regression failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugins/Runner/n8n-agent/components/runner/default.py b/plugins/Runner/n8n-agent/components/runner/default.py index ae03da355..d7fe03a61 100644 --- a/plugins/Runner/n8n-agent/components/runner/default.py +++ b/plugins/Runner/n8n-agent/components/runner/default.py @@ -6,6 +6,7 @@ Real n8n webhook integration supporting streaming and non-streaming responses. from __future__ import annotations import logging +import math import typing import uuid @@ -129,6 +130,16 @@ class DefaultRunner(Runner): if auth_type == "basic" and basic_encoding not in ("utf-8", "latin1"): raise N8nConfigError("basic-encoding must be utf-8 or latin1") + timeout_value = config.get("timeout", 120) + if timeout_value is None or isinstance(timeout_value, bool): + raise N8nConfigError("timeout must be a positive finite number") + try: + timeout = float(timeout_value) + except (TypeError, ValueError): + raise N8nConfigError("timeout must be a positive finite number") from None + if not math.isfinite(timeout) or timeout <= 0: + raise N8nConfigError("timeout must be a positive finite number") + return { "webhook_url": webhook_url, "auth_type": auth_type, @@ -141,7 +152,7 @@ class DefaultRunner(Runner): "header_name": config.get("header-name", ""), "header_value": config.get("header-value", ""), }, - "timeout": float(config.get("timeout", 120)), + "timeout": timeout, "output_key": config.get("output-key", "response"), "response_handling": response_handling, "langbot_assets_enabled": _to_bool(config.get("langbot-assets-enabled"), False), diff --git a/plugins/Runner/tests/test_native_parity_n8n.py b/plugins/Runner/tests/test_native_parity_n8n.py index ee6448ed7..5fdc47bb3 100644 --- a/plugins/Runner/tests/test_native_parity_n8n.py +++ b/plugins/Runner/tests/test_native_parity_n8n.py @@ -303,6 +303,16 @@ async def test_timeout_includes_waiting_for_response_headers(runner_module, monk assert results[0]["data"]["code"] == "n8n.timeout" +@pytest.mark.asyncio +@pytest.mark.parametrize("value", [None, True, 0, -1, "invalid", float("inf"), float("nan")]) +async def test_invalid_timeout_fails_before_http(runner_module, monkeypatch, value): + requests = wire(monkeypatch, Body([b'{"response":"ok"}'])) + results = await collect(runner_module, context({"timeout": value})) + assert [result["type"] for result in results] == ["run.failed"] + assert results[0]["data"]["code"] == "n8n.config_invalid" + assert requests == [] + + @pytest.mark.asyncio async def test_reply_timeout_is_total_not_only_per_chunk(runner_module, monkeypatch): class SlowBody(Body):