LangBot/skills/skills/langbot-testing/probes/langbot-live-control-plane-api.mjs

#!/usr/bin/env node

import { mkdir, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { env, exit } from "node:process";

function pad(value, size = 2) {
  return String(value).padStart(size, "0");
}

function localIsoWithOffset(date = new Date()) {
  const offsetMinutes = -date.getTimezoneOffset();
  const sign = offsetMinutes >= 0 ? "+" : "-";
  const absolute = Math.abs(offsetMinutes);
  return [
    `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`,
    `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`,
    `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`,
  ].join("");
}

function timestampSlug(date = new Date()) {
  return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
}

function percentile(values, percentileValue) {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);
  const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1);
  return Number(sorted[index].toFixed(3));
}

function stats(values) {
  if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 };
  return {
    min: Number(Math.min(...values).toFixed(3)),
    p50: percentile(values, 50),
    p95: percentile(values, 95),
    p99: percentile(values, 99),
    max: Number(Math.max(...values).toFixed(3)),
  };
}

function joinUrl(baseUrl, path) {
  const base = baseUrl.replace(/\/+$/, "");
  const suffix = path.startsWith("/") ? path : `/${path}`;
  return `${base}${suffix}`;
}

function parseJsonObject(value, fallback) {
  if (!value) return fallback;
  try {
    const parsed = JSON.parse(value);
    return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : fallback;
  } catch {
    return fallback;
  }
}

function controlPlaneEndpoints() {
  return [
    {
      id: "healthz",
      path: "/healthz",
      expected_status: 200,
      expected_code: 0,
      p95_budget_ms: Number(env.LANGBOT_PERF_HEALTHZ_P95_MS || "500"),
      required_data_fields: [],
    },
    {
      id: "system_info",
      path: "/api/v1/system/info",
      expected_status: 200,
      expected_code: 0,
      p95_budget_ms: Number(env.LANGBOT_PERF_SYSTEM_INFO_P95_MS || "1000"),
      required_data_fields: ["version", "edition", "enable_marketplace"],
    },
  ];
}

async function fetchEndpoint(backendUrl, endpoint, timeoutMs) {
  const url = joinUrl(backendUrl, endpoint.path);
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), timeoutMs);
  const started = performance.now();
  let bodyText = "";
  let json = null;
  let jsonValid = false;
  let error = "";

  try {
    const response = await fetch(url, {
      method: "GET",
      headers: { "accept": "application/json" },
      signal: controller.signal,
    });
    bodyText = await response.text();
    try {
      json = bodyText ? JSON.parse(bodyText) : null;
      jsonValid = json !== null;
    } catch (parseError) {
      error = parseError instanceof Error ? parseError.message : String(parseError);
    }

    const data = json && typeof json === "object" && json.data && typeof json.data === "object" ? json.data : {};
    const missingFields = endpoint.required_data_fields.filter((field) => !(field in data));
    const statusOk = response.status === endpoint.expected_status;
    const codeOk = !json || typeof json !== "object" ? false : json.code === endpoint.expected_code;
    const shapeOk = jsonValid && missingFields.length === 0;
    const latencyMs = performance.now() - started;
    return {
      endpoint_id: endpoint.id,
      path: endpoint.path,
      url,
      status: response.status,
      ok: statusOk && codeOk && shapeOk,
      status_ok: statusOk,
      code_ok: codeOk,
      json_valid: jsonValid,
      missing_fields: missingFields,
      response_code: json && typeof json === "object" ? json.code : null,
      latency_ms: Number(latencyMs.toFixed(3)),
      error,
    };
  } catch (fetchError) {
    const latencyMs = performance.now() - started;
    return {
      endpoint_id: endpoint.id,
      path: endpoint.path,
      url,
      status: 0,
      ok: false,
      status_ok: false,
      code_ok: false,
      json_valid: false,
      missing_fields: endpoint.required_data_fields,
      response_code: null,
      latency_ms: Number(latencyMs.toFixed(3)),
      error: fetchError instanceof Error ? fetchError.message : String(fetchError),
    };
  } finally {
    clearTimeout(timeout);
  }
}

async function runBatches(backendUrl, endpoints, totalRequests, concurrency, timeoutMs) {
  const queue = Array.from({ length: totalRequests }, (_, index) => endpoints[index % endpoints.length]);
  const results = [];
  while (queue.length > 0) {
    const batch = queue.splice(0, concurrency);
    results.push(...await Promise.all(batch.map((endpoint) => fetchEndpoint(backendUrl, endpoint, timeoutMs))));
  }
  return results;
}

function endpointMetrics(endpoints, results) {
  return Object.fromEntries(endpoints.map((endpoint) => {
    const samples = results.filter((item) => item.endpoint_id === endpoint.id);
    const okSamples = samples.filter((item) => item.ok);
    return [
      endpoint.id,
      {
        path: endpoint.path,
        requests: samples.length,
        ok_count: okSamples.length,
        error_rate: samples.length === 0 ? 1 : Number(((samples.length - okSamples.length) / samples.length).toFixed(4)),
        latency_ms: stats(okSamples.map((item) => item.latency_ms)),
        p95_budget_ms: endpoint.p95_budget_ms,
      },
    ];
  }));
}

async function main() {
  const root = resolve(env.LBS_ROOT || process.cwd());
  const caseId = "langbot-live-control-plane-api";
  const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });

  const startedAt = new Date();
  const backendUrl = env.LANGBOT_BACKEND_URL || "";
  const endpoints = controlPlaneEndpoints();
  const configuredBudgets = parseJsonObject(env.LANGBOT_CONTROL_PLANE_P95_BUDGETS_JSON, {});
  for (const endpoint of endpoints) {
    const budget = configuredBudgets[endpoint.id];
    if (typeof budget === "number" && Number.isFinite(budget)) endpoint.p95_budget_ms = budget;
  }
  const totalRequests = Number(env.LANGBOT_CONTROL_PLANE_REQUESTS || "20");
  const concurrency = Number(env.LANGBOT_CONTROL_PLANE_CONCURRENCY || "4");
  const timeoutMs = Number(env.LANGBOT_CONTROL_PLANE_TIMEOUT_MS || "5000");
  const maxErrorRate = Number(env.LANGBOT_CONTROL_PLANE_MAX_ERROR_RATE || "0");
  const metricsPath = join(evidenceDir, "metrics.json");
  const endpointsPath = join(evidenceDir, "endpoints.json");
  const networkLogPath = join(evidenceDir, "network.log");
  const automationResultPath = join(evidenceDir, "automation-result.json");
  const resultPath = join(evidenceDir, "result.json");

  let status = "fail";
  let reason = "";
  let results = [];
  if (!backendUrl) {
    status = "env_issue";
    reason = "LANGBOT_BACKEND_URL is not configured.";
  } else {
    results = await runBatches(backendUrl, endpoints, totalRequests, concurrency, timeoutMs);
    const allConnectionFailures = results.length > 0 && results.every((item) => item.status === 0);
    if (allConnectionFailures) {
      status = "env_issue";
      reason = `Backend did not respond at ${backendUrl}.`;
    }
  }

  const okResults = results.filter((item) => item.ok);
  const statusCounts = {};
  for (const item of results) {
    const key = item.status === 0 ? "network_error" : String(item.status);
    statusCounts[key] = (statusCounts[key] || 0) + 1;
  }
  const perEndpoint = endpointMetrics(endpoints, results);
  const responseShapeFailures = results.filter((item) => !item.json_valid || item.missing_fields.length > 0 || !item.code_ok).length;
  const errorRate = results.length === 0 ? 1 : Number(((results.length - okResults.length) / results.length).toFixed(4));
  const thresholds = {
    error_rate: { actual: errorRate, max: maxErrorRate, pass: errorRate <= maxErrorRate },
    response_shape_failures: { actual: responseShapeFailures, max: 0, pass: responseShapeFailures === 0 },
  };
  for (const endpoint of endpoints) {
    const actual = perEndpoint[endpoint.id].latency_ms.p95;
    thresholds[`${endpoint.id}_p95_ms`] = {
      actual,
      max: endpoint.p95_budget_ms,
      pass: actual <= endpoint.p95_budget_ms,
    };
  }

  if (status !== "env_issue") {
    const passed = Object.values(thresholds).every((item) => item.pass);
    status = passed ? "pass" : "fail";
    reason = passed
      ? "Live control-plane API probe passed all thresholds."
      : "Live control-plane API probe breached shape, latency, or error-rate thresholds.";
  }

  const metrics = {
    probe: caseId,
    backend_url: backendUrl,
    total_requests: totalRequests,
    concurrency,
    timeout_ms: timeoutMs,
    ok_count: okResults.length,
    error_count: results.length - okResults.length,
    error_rate: errorRate,
    status_counts: statusCounts,
    response_shape_failures: responseShapeFailures,
    endpoints: perEndpoint,
  };

  await writeFile(metricsPath, `${JSON.stringify({ ...metrics, samples: results }, null, 2)}\n`, "utf8");
  await writeFile(endpointsPath, `${JSON.stringify(endpoints, null, 2)}\n`, "utf8");
  await writeFile(networkLogPath, results.map((item) => JSON.stringify(item)).join("\n") + (results.length > 0 ? "\n" : ""), "utf8");

  const finishedAt = new Date();
  const result = {
    source: "automation",
    case_id: caseId,
    run_id: runId,
    status,
    reason,
    started_at: startedAt.toISOString(),
    started_at_local: localIsoWithOffset(startedAt),
    finished_at: finishedAt.toISOString(),
    finished_at_local: localIsoWithOffset(finishedAt),
    duration_ms: finishedAt.getTime() - startedAt.getTime(),
    url: backendUrl,
    metrics_summary: {
      requests: metrics.total_requests,
      concurrency: metrics.concurrency,
      ok_count: metrics.ok_count,
      error_rate: metrics.error_rate,
      response_shape_failures: metrics.response_shape_failures,
      endpoints: Object.fromEntries(Object.entries(metrics.endpoints).map(([id, value]) => [
        id,
        {
          path: value.path,
          ok_count: value.ok_count,
          error_rate: value.error_rate,
          latency_p50_ms: value.latency_ms.p50,
          latency_p95_ms: value.latency_ms.p95,
        },
      ])),
      status_counts: metrics.status_counts,
    },
    thresholds_summary: thresholds,
    artifacts: {
      metrics_json: metricsPath,
      endpoints_json: endpointsPath,
      network_log: networkLogPath,
      automation_result_json: automationResultPath,
      result_json: resultPath,
    },
    evidence_collected: ["metrics", "network", "api_diagnostic", "filesystem"],
  };

  const resultText = `${JSON.stringify(result, null, 2)}\n`;
  await writeFile(automationResultPath, resultText, "utf8");
  await writeFile(resultPath, resultText, "utf8");
  console.log(JSON.stringify(result, null, 2));
  exit(status === "pass" ? 0 : status === "env_issue" ? 2 : 1);
}

await main();