LangBot/skills/scripts/e2e/ensure-local-agent-pipeline.mjs

#!/usr/bin/env node

import { readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
import { env } from "node:process";
import {
  apiJson,
  bodyText,
  createBrowser,
  ensureEvidence,
  evidencePaths,
  loadEnvFiles,
  redact,
  resetAndAuthLocalUser,
  safeScreenshot,
  setBrowserToken,
  verifyBrowserToken,
  writeResult,
} from "./lib/langbot-e2e.mjs";

const RUNNER_ID = "plugin:langbot-team/LocalAgent/default";
const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000";
const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat";
const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026";
const DEFAULT_MODEL_TEST_LIMIT = 8;
const DEFAULT_MODEL_FALLBACK_COUNT = 3;
const DEFAULT_FAKE_MODEL_UUID = "langbot-e2e-fake-local-agent-model";
const DEFAULT_FAKE_PROVIDER_NAME = "LangBot E2E Fake OpenAI Provider";
const caseId = "ensure-local-agent-pipeline";

await loadEnvFiles();
const paths = evidencePaths(caseId);
await ensureEvidence(paths);

const writeEnv = process.argv.includes("--write-env");
const pipelineName =
  env.LANGBOT_E2E_CREATE_PIPELINE_NAME ||
  env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME ||
  DEFAULT_PIPELINE_NAME;
const frontendUrl = env.LANGBOT_FRONTEND_URL || "";
const backendUrl = env.LANGBOT_BACKEND_URL || "";
const envLocalPath = resolve("skills/.env.local");

const result = {
  source: "automation",
  case_id: caseId,
  run_id: paths.runId,
  status: "fail",
  reason: "",
  frontend_url: frontendUrl,
  backend_url: backendUrl,
  pipeline_name: pipelineName,
  pipeline_id: "",
  pipeline_url: "",
  runner_id: RUNNER_ID,
  selected_model_id: "",
  selected_model_name: "",
  fallback_model_ids: [],
  fake_provider: null,
  model_count: 0,
  space_model_count: 0,
  scanned_space_model_count: 0,
  tested_model_count: 0,
  model_tests: [],
  plugin_setup: null,
  created: false,
  updated: false,
  wrote_env: false,
  auth: null,
  wizard: null,
  browser_token_check: null,
  page_signal: "",
  evidence: {
    console_log: paths.consoleLog,
    network_log: paths.networkLog,
    screenshot: paths.screenshot,
    automation_result_json: paths.automationResultJson,
    result_json: paths.resultJson,
  },
  evidence_collected: ["api_diagnostic", "console", "network", "screenshot"],
};

let browser;

try {
  if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured.");
  if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured.");

  const user = env.LANGBOT_E2E_LOGIN_USER || "";
  const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD;
  if (!user) {
    result.status = "env_issue";
    throw new Error(
      "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.",
    );
  }

  const auth = await resetAndAuthLocalUser({ backendUrl, user, password });
  result.auth = {
    source: "local_recovery_login",
    user,
    backend_token_check: auth.check,
  };

  const pluginSetup = await ensureLocalAgentRunner({
    backendUrl,
    token: auth.token,
  });
  result.plugin_setup = pluginSetup;
  if (pluginSetup.status !== "pass") {
    result.status = pluginSetup.status === "env_issue" ? "env_issue" : "fail";
    throw new Error(pluginSetup.reason || "Failed to prepare the LocalAgent runner plugin.");
  }

  const wizard = await skipWizard({ backendUrl, token: auth.token });
  result.wizard = wizard;
  if (wizard.status !== "pass") {
    result.status = "fail";
    throw new Error(
      wizard.reason || "Failed to mark the local QA wizard as skipped.",
    );
  }

  const prepared = await ensureLocalAgentPipeline({
    backendUrl,
    token: auth.token,
    pipelineName,
    runnerId: RUNNER_ID,
  });
  Object.assign(result, prepared);
  if (result.pipeline_id) {
    result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(result.pipeline_id)}`;
  }

  if (writeEnv && result.pipeline_id) {
    await upsertEnvLocal(envLocalPath, {
      LANGBOT_E2E_LOGIN_USER: user,
      LANGBOT_PIPELINE_URL: result.pipeline_url,
      LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName,
      LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url,
      LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName,
      ...(result.selected_model_id
        ? {
            LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id,
            LANGBOT_E2E_MODEL_UUID: result.selected_model_id,
          }
        : {}),
    });
    result.wrote_env = true;
  }

  browser = await createBrowser(paths);
  const { page } = browser;
  await setBrowserToken(page, frontendUrl, auth.token);
  const browserCheck = await verifyBrowserToken(page, backendUrl);
  result.browser_token_check = browserCheck;
  if (!browserCheck.authenticated) {
    throw new Error(
      browserCheck.reason || "Browser token check failed after setup.",
    );
  }
  await page.goto(result.pipeline_url || frontendUrl, {
    waitUntil: "domcontentloaded",
  });
  await page
    .waitForLoadState("networkidle", { timeout: 10_000 })
    .catch(() => {});
  const text = await bodyText(page);
  result.page_signal =
    ["Pipelines", "流水线", pipelineName].find((signal) =>
      text.includes(signal),
    ) || "";
} catch (error) {
  result.status = result.status === "env_issue" ? "env_issue" : "fail";
  result.reason = result.reason || error.message;
} finally {
  if (browser?.page) await safeScreenshot(browser.page, paths.screenshot);
  if (browser) await browser.close().catch(() => {});
  await writeResult(paths, result);
  console.log(JSON.stringify(result, null, 2));
}

process.exit(
  result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1,
);

async function skipWizard({ backendUrl, token }) {
  const response = await apiJson(
    backendUrl,
    "/api/v1/system/wizard/completed",
    {
      method: "POST",
      token,
      body: { status: "skipped" },
    },
  );
  const ok = response.status < 400 && response.json.code === 0;
  return {
    status: ok ? "pass" : "fail",
    http_status: response.status,
    code: response.json.code ?? null,
    reason: ok
      ? "Wizard marked skipped for local QA."
      : response.json.msg || "Wizard status update failed.",
  };
}

async function ensureLocalAgentRunner({ backendUrl, token }) {
  const [author, name] = RUNNER_ID.replace(/^plugin:/, "").split("/");
  const existingRunnerIds = await listRunnerIds(backendUrl, token);
  if (existingRunnerIds.includes(RUNNER_ID)) {
    return {
      status: "pass",
      reason: "LocalAgent runner is already registered.",
      plugin_id: `${author}/${name}`,
      runner_id: RUNNER_ID,
      installed: false,
    };
  }

  const pluginsResponse = await apiJson(backendUrl, "/api/v1/plugins", {
    token,
  });
  if (isApiFailure(pluginsResponse)) {
    return {
      status: "env_issue",
      reason:
        pluginsResponse.json.msg ||
        "Failed to inspect installed plugins before preparing LocalAgent.",
    };
  }
  const pluginId = `${author}/${name}`;
  const pluginPresent = (pluginsResponse.json.data?.plugins || []).some(
    (plugin) => {
      const metadata =
        plugin.manifest?.manifest?.metadata ||
        plugin.manifest?.metadata ||
        plugin.metadata ||
        {};
      return `${metadata.author}/${metadata.name}` === pluginId;
    },
  );
  if (pluginPresent) {
    const registered = await waitForRunnerRegistration({
      backendUrl,
      token,
      runnerId: RUNNER_ID,
      timeoutMs: 30_000,
    });
    return registered
      ? {
          status: "pass",
          reason: "Installed LocalAgent plugin finished Runner registration.",
          plugin_id: pluginId,
          runner_id: RUNNER_ID,
          installed: false,
        }
      : {
          status: "fail",
          reason: `${pluginId} is installed but ${RUNNER_ID} did not register.`,
          plugin_id: pluginId,
          runner_id: RUNNER_ID,
          installed: false,
        };
  }

  const spaceUrl = String(env.LANGBOT_SPACE_URL || "https://space.langbot.app").replace(
    /\/$/,
    "",
  );
  let detailResponse;
  try {
    detailResponse = await fetch(
      `${spaceUrl}/api/v1/marketplace/plugins/${encodeURIComponent(author)}/${encodeURIComponent(name)}`,
    );
  } catch (error) {
    return {
      status: "env_issue",
      reason: `Could not reach LangBot Space for ${pluginId}: ${error.message}`,
    };
  }
  const detail = await detailResponse.json().catch(() => ({}));
  const version = detail?.data?.plugin?.latest_version || "";
  if (detailResponse.status >= 400 || detail.code !== 0 || !version) {
    return {
      status: "env_issue",
      reason:
        detail.msg ||
        `LangBot Space did not return an installable version for ${pluginId}.`,
    };
  }

  const install = await apiJson(
    backendUrl,
    "/api/v1/plugins/install/marketplace",
    {
      method: "POST",
      token,
      body: {
        plugin_author: author,
        plugin_name: name,
        plugin_version: version,
      },
    },
  );
  const taskId = install.json.data?.task_id ?? null;
  if (isApiFailure(install) || !taskId) {
    return {
      status: "fail",
      reason:
        install.json.msg ||
        `Marketplace install did not create a task for ${pluginId}.`,
      plugin_id: pluginId,
      version,
    };
  }
  const task = await waitForTask({ backendUrl, token, taskId });
  if (!taskComplete(task)) {
    return {
      status: taskFailed(task) ? "fail" : "env_issue",
      reason:
        task?.runtime?.exception ||
        task?.error ||
        `Marketplace install task for ${pluginId} did not complete.`,
      plugin_id: pluginId,
      version,
      task_id: taskId,
    };
  }
  const registered = await waitForRunnerRegistration({
    backendUrl,
    token,
    runnerId: RUNNER_ID,
    timeoutMs: 60_000,
  });
  return registered
    ? {
        status: "pass",
        reason: `Installed ${pluginId} ${version} and registered ${RUNNER_ID}.`,
        plugin_id: pluginId,
        runner_id: RUNNER_ID,
        version,
        task_id: taskId,
        installed: true,
      }
    : {
        status: "fail",
        reason: `Installed ${pluginId} ${version}, but ${RUNNER_ID} did not register.`,
        plugin_id: pluginId,
        runner_id: RUNNER_ID,
        version,
        task_id: taskId,
        installed: true,
      };
}

async function listRunnerIds(backendUrl, token) {
  const response = await apiJson(backendUrl, "/api/v1/pipelines/_/metadata", {
    token,
  });
  if (isApiFailure(response)) return [];
  return (response.json.data?.configs || [])
    .flatMap((section) => section.stages || [])
    .flatMap((stage) => stage.config || [])
    .filter((item) => item.name === "id")
    .flatMap((item) => item.options || [])
    .map((option) => option.name || option.value || option.id || "")
    .filter(Boolean);
}

async function waitForRunnerRegistration({
  backendUrl,
  token,
  runnerId,
  timeoutMs,
}) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if ((await listRunnerIds(backendUrl, token)).includes(runnerId)) return true;
    await sleep(1000);
  }
  return false;
}

async function waitForTask({ backendUrl, token, taskId }) {
  const deadline =
    Date.now() + Number(env.LANGBOT_PLUGIN_INSTALL_TIMEOUT_MS || 120_000);
  let task = null;
  while (Date.now() < deadline) {
    const response = await apiJson(
      backendUrl,
      `/api/v1/system/tasks/${encodeURIComponent(taskId)}`,
      { token },
    );
    task = response.json.data || response.json;
    if (taskComplete(task) || taskFailed(task)) return task;
    await sleep(1000);
  }
  return task;
}

function taskComplete(task) {
  const status = String(task?.status || task?.state || "").toLowerCase();
  const runtimeStatus = String(
    task?.runtime?.status || task?.runtime?.state || "",
  ).toLowerCase();
  return (
    ["done", "completed", "success", "succeeded", "finished"].includes(
      status,
    ) ||
    ["done", "completed", "success", "succeeded", "finished"].includes(
      runtimeStatus,
    ) ||
    task?.done === true ||
    task?.completed === true ||
    (task?.runtime?.done === true && !task?.runtime?.exception)
  );
}

function taskFailed(task) {
  const status = String(task?.status || task?.state || "").toLowerCase();
  const runtimeStatus = String(
    task?.runtime?.status || task?.runtime?.state || "",
  ).toLowerCase();
  return (
    ["failed", "error", "cancelled", "canceled"].includes(status) ||
    ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus) ||
    task?.failed === true ||
    Boolean(task?.error) ||
    Boolean(task?.runtime?.exception)
  );
}

function sleep(ms) {
  return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
}

async function ensureLocalAgentPipeline({
  backendUrl,
  token,
  pipelineName,
  runnerId,
}) {
  const fakeProviderBaseUrl =
    env.LANGBOT_E2E_FAKE_PROVIDER_BASE_URL ||
    env.LANGBOT_FAKE_PROVIDER_BASE_URL ||
    "";
  let fakeModel = null;
  if (fakeProviderBaseUrl) {
    fakeModel = await ensureFakeProviderModel({
      backendUrl,
      token,
      baseUrl: fakeProviderBaseUrl,
    });
    if (fakeModel.status !== "pass") return fakeModel;
  }

  const [pipelineList, modelList] = await Promise.all([
    apiJson(backendUrl, "/api/v1/pipelines", { token }),
    apiJson(backendUrl, "/api/v1/provider/models/llm", { token }),
  ]);

  if (isApiFailure(pipelineList)) {
    return {
      status: "fail",
      reason: pipelineList.json.msg || "Failed to list pipelines.",
      list_status: pipelineList.status,
    };
  }
  if (isApiFailure(modelList)) {
    return {
      status: "fail",
      reason: modelList.json.msg || "Failed to list LLM models.",
      model_status: modelList.status,
    };
  }

  const models = modelList.json.data?.models || [];
  const skippedModelIds = new Set(
    String(env.LANGBOT_E2E_SKIP_MODEL_UUIDS || "")
      .split(",")
      .map((item) => item.trim())
      .filter(Boolean),
  );
  const skippedModelNames = new Set(
    String(env.LANGBOT_E2E_SKIP_MODEL_NAMES || "")
      .split(",")
      .map((item) => item.trim())
      .filter(Boolean),
  );
  const spaceModels = models.filter(
    (model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid),
  );
  const pipelines = pipelineList.json.data?.pipelines || [];
  let pipeline = pipelines.find((item) => item.name === pipelineName) || null;
  let created = false;

  if (!pipeline) {
    const createdResponse = await apiJson(backendUrl, "/api/v1/pipelines", {
      method: "POST",
      token,
      body: {
        name: pipelineName,
        description:
          "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
        emoji: "QA",
      },
    });
    if (isApiFailure(createdResponse)) {
      return {
        status: "fail",
        reason: createdResponse.json.msg || "Failed to create pipeline.",
        create_status: createdResponse.status,
        model_count: models.length,
        space_model_count: spaceModels.length,
      };
    }
    const pipelineId = createdResponse.json.data?.uuid || "";
    const loaded = await apiJson(
      backendUrl,
      `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`,
      { token },
    );
    pipeline = loaded.json.data?.pipeline || null;
    created = true;
  }

  if (!pipeline?.uuid) {
    return {
      status: "fail",
      reason: "Pipeline was not created or resolved.",
      model_count: models.length,
      space_model_count: spaceModels.length,
    };
  }

  const loaded = await apiJson(
    backendUrl,
    `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
    { token },
  );
  if (isApiFailure(loaded) || !loaded.json.data?.pipeline) {
    return {
      status: "fail",
      reason: loaded.json.msg || "Failed to load pipeline.",
      get_status: loaded.status,
      pipeline_id: pipeline.uuid,
      model_count: models.length,
      space_model_count: spaceModels.length,
    };
  }
  pipeline = loaded.json.data.pipeline;

  const config =
    pipeline.config && typeof pipeline.config === "object"
      ? pipeline.config
      : {};
  const ai = config.ai && typeof config.ai === "object" ? config.ai : {};
  const runnerConfigs =
    ai.runner_config && typeof ai.runner_config === "object"
      ? ai.runner_config
      : {};
  const rawExistingLocalAgentConfig =
    runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object"
      ? runnerConfigs[runnerId]
      : {};
  const existingLocalAgentConfig = rawExistingLocalAgentConfig;
  const existingModel =
    existingLocalAgentConfig.model &&
    typeof existingLocalAgentConfig.model === "object"
      ? existingLocalAgentConfig.model
      : {};
  const requestedModelId =
    env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || "";
  const selected = fakeModel
    ? {
        status: "pass",
        reason: "",
        selected_model_id: fakeModel.model_uuid,
        selected_model_name: fakeModel.model_name,
        fallback_model_ids: [],
        scanned_space_model_count: 0,
        tested_model_count: 0,
        model_tests: [],
      }
    : await selectWorkingSpaceModel({
        backendUrl,
        token,
        models,
        skippedModelIds,
        skippedModelNames,
        requestedModelId,
        existingModelId: existingModel.primary || "",
      });
  const selectedModelId = selected.selected_model_id || "";
  const localAgentConfig = {
    timeout: 300,
    prompt: [{ role: "system", content: "You are a helpful assistant." }],
    "remove-think": false,
    "knowledge-bases": [],
    "retrieval-top-k": 5,
    "rerank-model": "",
    "rerank-top-k": 5,
    "max-tool-iterations": 20,
    "tool-execution-mode": "parallel",
    "max-tool-result-chars": 20000,
    "context-history-fetch-limit": 50,
    "context-window-tokens": 200000,
    "context-reserve-tokens": 16384,
    "context-keep-recent-tokens": 20000,
    "context-summary-tokens": 8000,
    // Current backend truncation still reads this field directly.
    "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10),
    model: {
      primary: selectedModelId,
      fallbacks: selected.fallback_model_ids || [],
    },
  };
  const updatedConfig = {
    ...config,
    ai: {
      ...ai,
      runner: {
        ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}),
        id: runnerId,
        "expire-time": 0,
      },
      runner_config: {
        ...runnerConfigs,
        [runnerId]: localAgentConfig,
      },
    },
  };

  const updateResponse = await apiJson(
    backendUrl,
    `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`,
    {
      method: "PUT",
      token,
      body: {
        name: pipelineName,
        description:
          "Local QA pipeline for AgentRunner Debug Chat smoke tests.",
        emoji: "QA",
        config: updatedConfig,
      },
    },
  );
  if (isApiFailure(updateResponse)) {
    return {
      status: "fail",
      reason: updateResponse.json.msg || "Failed to update pipeline config.",
      update_status: updateResponse.status,
      pipeline_id: pipeline.uuid,
      model_count: models.length,
      space_model_count: spaceModels.length,
      scanned_space_model_count: selected.scanned_space_model_count,
      tested_model_count: selected.tested_model_count,
      model_tests: selected.model_tests,
      selected_model_id: selectedModelId,
      selected_model_name: selected.selected_model_name,
      fallback_model_ids: selected.fallback_model_ids,
    };
  }

  return {
    status: selectedModelId ? "pass" : "env_issue",
    reason: selectedModelId
      ? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).`
      : selected.reason ||
        "No working Space LLM model is configured in this LangBot instance.",
    pipeline_id: pipeline.uuid,
    pipeline_name: pipelineName,
    model_count: models.length,
    space_model_count: spaceModels.length,
    scanned_space_model_count: selected.scanned_space_model_count,
    tested_model_count: selected.tested_model_count,
    model_tests: selected.model_tests,
    selected_model_id: selectedModelId,
    selected_model_name: selected.selected_model_name,
    fallback_model_ids: selected.fallback_model_ids,
    fake_provider: fakeModel,
    created,
    updated: true,
  };
}

async function ensureFakeProviderModel({ backendUrl, token, baseUrl }) {
  const modelUuid = env.LANGBOT_E2E_FAKE_MODEL_UUID || DEFAULT_FAKE_MODEL_UUID;
  const modelName =
    env.LANGBOT_E2E_FAKE_MODEL_NAME ||
    env.LANGBOT_FAKE_PROVIDER_MODEL ||
    env.LANGBOT_FAKE_PROVIDER_MODEL_NAME ||
    "langbot-e2e-fake-model";
  const providerName =
    env.LANGBOT_E2E_FAKE_PROVIDER_NAME || DEFAULT_FAKE_PROVIDER_NAME;
  const providerRequester =
    env.LANGBOT_E2E_FAKE_PROVIDER_REQUESTER || "openai-chat-completions";
  const apiKey =
    env.LANGBOT_E2E_FAKE_PROVIDER_API_KEY ||
    env.LANGBOT_FAKE_PROVIDER_API_KEY ||
    "fake-key";

  const providersResponse = await apiJson(
    backendUrl,
    "/api/v1/provider/providers",
    { token },
  );
  if (isApiFailure(providersResponse)) {
    return {
      status: "fail",
      reason:
        providersResponse.json.msg ||
        "Failed to list providers before creating fake provider.",
      provider_status: providersResponse.status,
    };
  }

  const normalizedBaseUrl = String(baseUrl || "").replace(/\/$/, "");
  const providers = providersResponse.json.data?.providers || [];
  let provider = providers.find(
    (item) =>
      item.name === providerName ||
      (item.requester === providerRequester &&
        String(item.base_url || "").replace(/\/$/, "") === normalizedBaseUrl),
  );
  const providerBody = {
    name: providerName,
    requester: providerRequester,
    base_url: normalizedBaseUrl,
    api_keys: [apiKey],
  };

  if (provider?.uuid) {
    const updateResponse = await apiJson(
      backendUrl,
      `/api/v1/provider/providers/${encodeURIComponent(provider.uuid)}`,
      {
        method: "PUT",
        token,
        body: providerBody,
      },
    );
    if (isApiFailure(updateResponse)) {
      return {
        status: "fail",
        reason: updateResponse.json.msg || "Failed to update fake provider.",
        provider_status: updateResponse.status,
      };
    }
  } else {
    const createProviderResponse = await apiJson(
      backendUrl,
      "/api/v1/provider/providers",
      {
        method: "POST",
        token,
        body: providerBody,
      },
    );
    if (isApiFailure(createProviderResponse)) {
      return {
        status: "fail",
        reason:
          createProviderResponse.json.msg || "Failed to create fake provider.",
        provider_status: createProviderResponse.status,
      };
    }
    provider = { uuid: createProviderResponse.json.data?.uuid || "" };
  }

  if (!provider?.uuid) {
    return {
      status: "fail",
      reason: "Fake provider did not return a provider uuid.",
    };
  }

  let resolvedModelUuid = modelUuid;
  let modelResponse = await apiJson(
    backendUrl,
    `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`,
    {
      token,
    },
  );
  if (modelResponse.status === 404) {
    const providerModelsResponse = await apiJson(
      backendUrl,
      `/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(provider.uuid)}`,
      { token },
    );
    if (!isApiFailure(providerModelsResponse)) {
      const existingModel = (
        providerModelsResponse.json.data?.models || []
      ).find((item) => item.name === modelName);
      if (existingModel?.uuid) {
        resolvedModelUuid = existingModel.uuid;
        modelResponse = await apiJson(
          backendUrl,
          `/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`,
          {
            token,
          },
        );
      }
    }
  }

  const modelBody = {
    name: modelName,
    provider_uuid: provider.uuid,
    abilities: ["func_call", "vision"],
    context_length: 8192,
    extra_args: {},
    prefered_ranking: 0,
  };
  if (modelResponse.status === 404) {
    const createModelResponse = await apiJson(
      backendUrl,
      "/api/v1/provider/models/llm",
      {
        method: "POST",
        token,
        body: {
          uuid: modelUuid,
          ...modelBody,
        },
      },
    );
    if (isApiFailure(createModelResponse)) {
      return {
        status: "fail",
        reason: createModelResponse.json.msg || "Failed to create fake model.",
        model_status: createModelResponse.status,
      };
    }
    resolvedModelUuid = createModelResponse.json.data?.uuid || modelUuid;
  } else if (isApiFailure(modelResponse)) {
    return {
      status: "fail",
      reason: modelResponse.json.msg || "Failed to load fake model.",
      model_status: modelResponse.status,
    };
  } else {
    const updateModelResponse = await apiJson(
      backendUrl,
      `/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`,
      {
        method: "PUT",
        token,
        body: modelBody,
      },
    );
    if (isApiFailure(updateModelResponse)) {
      return {
        status: "fail",
        reason: updateModelResponse.json.msg || "Failed to update fake model.",
        model_status: updateModelResponse.status,
      };
    }
  }

  return {
    status: "pass",
    provider_uuid: provider.uuid,
    provider_requester: providerRequester,
    base_url: normalizedBaseUrl,
    model_uuid: resolvedModelUuid,
    model_name: modelName,
  };
}

function isApiFailure(response) {
  return (
    response.status >= 400 ||
    (response.json.code !== undefined && response.json.code !== 0)
  );
}

function isSpaceModel(model) {
  const provider =
    model?.provider && typeof model.provider === "object" ? model.provider : {};
  return (
    model?.provider_uuid === SPACE_PROVIDER_UUID ||
    provider.uuid === SPACE_PROVIDER_UUID ||
    provider.requester === "space-chat-completions" ||
    provider.name === "LangBot Models"
  );
}

async function selectWorkingSpaceModel({
  backendUrl,
  token,
  models,
  skippedModelIds,
  skippedModelNames,
  requestedModelId,
  existingModelId,
}) {
  const modelTests = [];
  const testLimit = positiveInteger(
    env.LANGBOT_E2E_MODEL_TEST_LIMIT,
    DEFAULT_MODEL_TEST_LIMIT,
  );
  const fallbackCount = positiveInteger(
    env.LANGBOT_E2E_MODEL_FALLBACK_COUNT,
    DEFAULT_MODEL_FALLBACK_COUNT,
  );
  const workingModels = [];
  const spaceModels = rankModels(
    models.filter(
      (model) =>
        model.uuid &&
        isSpaceModel(model) &&
        !skippedModelIds.has(model.uuid) &&
        !skippedModelNames.has(model.name),
    ),
  );
  const requestedModel = requestedModelId
    ? spaceModels.find((model) => model.uuid === requestedModelId) || null
    : null;
  const existingModel = existingModelId
    ? spaceModels.find((model) => model.uuid === existingModelId) || null
    : null;
  const candidates = uniqueCandidates([
    ...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []),
    ...(existingModel
      ? [existingCandidate(existingModel, "existing-pipeline")]
      : []),
    ...spaceModels.map((model) => existingCandidate(model, "configured-space")),
  ]);

  let scanResult = { status: "skipped", models: [], reason: "" };
  if (env.LANGBOT_E2E_SCAN_SPACE_MODELS !== "false") {
    scanResult = await scanSpaceModels({ backendUrl, token });
    if (scanResult.status === "pass") {
      const knownNames = new Set(spaceModels.map((model) => model.name));
      candidates.push(
        ...scanResult.models
          .filter(
            (model) =>
              model.name &&
              !knownNames.has(model.name) &&
              !skippedModelNames.has(model.name),
          )
          .map((model) => scannedCandidate(model)),
      );
    }
  }

  const unique = uniqueCandidates(candidates);
  for (const candidate of unique.slice(0, testLimit)) {
    const test = await ensureAndTestModel({ backendUrl, token, candidate });
    modelTests.push(test);
    if (test.status === "pass" && test.model_uuid) {
      workingModels.push(test);
      if (workingModels.length >= fallbackCount + 1) break;
    }
  }

  if (workingModels.length > 0) {
    const [primary, ...fallbacks] = workingModels;
    return {
      status: "pass",
      reason: "",
      selected_model_id: primary.model_uuid,
      selected_model_name: primary.model_name,
      fallback_model_ids: fallbacks.map((model) => model.model_uuid),
      scanned_space_model_count: scanResult.models.length,
      tested_model_count: modelTests.length,
      model_tests: modelTests,
    };
  }

  const baseReason =
    unique.length === 0
      ? scanResult.reason || "No Space LLM model candidates are available."
      : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`;
  return {
    status: "env_issue",
    reason:
      requestedModelId && !requestedModel
        ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}`
        : baseReason,
    selected_model_id: "",
    selected_model_name: "",
    fallback_model_ids: [],
    scanned_space_model_count: scanResult.models.length,
    tested_model_count: modelTests.length,
    model_tests: modelTests,
  };
}

async function scanSpaceModels({ backendUrl, token }) {
  const response = await apiJson(
    backendUrl,
    `/api/v1/provider/providers/${encodeURIComponent(SPACE_PROVIDER_UUID)}/scan-models?type=llm`,
    { token },
  );
  if (isApiFailure(response)) {
    return {
      status: "env_issue",
      models: [],
      reason: safeReason(
        response.json.msg ||
          response.json.message ||
          "Failed to scan Space LLM models.",
      ),
    };
  }
  return {
    status: "pass",
    models: response.json.data?.models || [],
    reason: "",
  };
}

async function ensureAndTestModel({ backendUrl, token, candidate }) {
  let modelUuid = candidate.uuid || "";
  let created = false;
  if (!modelUuid) {
    const create = await apiJson(backendUrl, "/api/v1/provider/models/llm", {
      method: "POST",
      token,
      body: {
        name: candidate.name,
        provider_uuid: SPACE_PROVIDER_UUID,
        abilities: candidate.abilities || [],
        context_length: candidate.context_length ?? null,
        extra_args: {},
        prefered_ranking: positiveInteger(candidate.prefered_ranking, 0),
      },
    });
    modelUuid = create.json.data?.uuid || "";
    if (isApiFailure(create) || !modelUuid) {
      return modelTestResult(candidate, {
        status: "fail",
        reason: safeReason(
          create.json.msg || "Failed to create scanned Space model.",
        ),
        http_status: create.status,
      });
    }
    created = true;
  }

  const test = await apiJson(
    backendUrl,
    `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`,
    {
      method: "POST",
      token,
      body: { extra_args: {} },
    },
  );
  const passed = !isApiFailure(test);
  if (!passed && created) {
    await apiJson(
      backendUrl,
      `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`,
      {
        method: "DELETE",
        token,
      },
    ).catch(() => {});
  }
  return modelTestResult(candidate, {
    status: passed ? "pass" : "fail",
    reason: passed
      ? ""
      : safeReason(
          test.json.msg || test.json.message || "Space model test failed.",
        ),
    http_status: test.status,
    model_uuid: modelUuid,
    created,
  });
}

function modelTestResult(candidate, details) {
  return {
    source: candidate.source,
    model_uuid: details.model_uuid || candidate.uuid || "",
    model_name: candidate.name,
    status: details.status,
    reason: details.reason || "",
    http_status: details.http_status ?? null,
    created: Boolean(details.created),
  };
}

function existingCandidate(model, source) {
  return {
    source,
    uuid: model.uuid,
    name: model.name,
    abilities: model.abilities || [],
    context_length: model.context_length,
    prefered_ranking: model.prefered_ranking,
  };
}

function scannedCandidate(model) {
  return {
    source: "scanned-space",
    uuid: "",
    name: model.name || model.id,
    abilities: model.abilities || [],
    context_length: model.context_length,
    prefered_ranking: model.prefered_ranking,
  };
}

function uniqueCandidates(candidates) {
  const seen = new Set();
  const result = [];
  for (const candidate of candidates) {
    const key = candidate.uuid
      ? `uuid:${candidate.uuid}`
      : `name:${candidate.name}`;
    if (!candidate.name || seen.has(key)) continue;
    seen.add(key);
    result.push(candidate);
  }
  return result;
}

function rankModels(models) {
  return [...models].sort((left, right) => {
    const leftRank = Number.isFinite(Number(left.prefered_ranking))
      ? Number(left.prefered_ranking)
      : 9999;
    const rightRank = Number.isFinite(Number(right.prefered_ranking))
      ? Number(right.prefered_ranking)
      : 9999;
    if (leftRank !== rightRank) return leftRank - rightRank;
    return String(left.name || "").localeCompare(String(right.name || ""));
  });
}

function positiveInteger(value, fallback) {
  const parsed = Number(value);
  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
}

function safeReason(value) {
  return redact(String(value || "")).slice(0, 1000);
}

async function upsertEnvLocal(path, updates) {
  let text = "";
  try {
    text = await readFile(path, "utf8");
  } catch {
    text = "";
  }
  const lines = text.split(/\r?\n/);
  const seen = new Set();
  const next = lines.map((line) => {
    const trimmed = line.trim();
    const equals = trimmed.indexOf("=");
    if (equals <= 0 || trimmed.startsWith("#")) return line;
    const key = trimmed.slice(0, equals).trim();
    if (!(key in updates)) return line;
    seen.add(key);
    return `${key}=${updates[key]}`;
  });
  for (const [key, value] of Object.entries(updates)) {
    if (!seen.has(key)) next.push(`${key}=${value}`);
  }
  await writeFile(
    path,
    `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`,
    "utf8",
  );
}