LangBot/skills/skills/langbot-testing/probes/langbot-overhead-accounting-contract.mjs

#!/usr/bin/env node

import { mkdir, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { env, exit } from "node:process";

function pad(value, size = 2) {
  return String(value).padStart(size, "0");
}

function localIsoWithOffset(date = new Date()) {
  const offsetMinutes = -date.getTimezoneOffset();
  const sign = offsetMinutes >= 0 ? "+" : "-";
  const absolute = Math.abs(offsetMinutes);
  return [
    `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`,
    `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`,
    `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`,
  ].join("");
}

function timestampSlug(date = new Date()) {
  return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, "");
}

function percentile(values, percentileValue) {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);
  const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1);
  return Number(sorted[index].toFixed(3));
}

function stats(values) {
  return {
    min: Number(Math.min(...values).toFixed(3)),
    p50: percentile(values, 50),
    p95: percentile(values, 95),
    p99: percentile(values, 99),
    max: Number(Math.max(...values).toFixed(3)),
  };
}

function threshold(actual, limit, operator) {
  const pass = operator === "<=" ? actual <= limit : actual >= limit;
  return { actual, [operator === "<=" ? "max" : "min"]: limit, pass };
}

function makeSample(index) {
  const ingress = 1 + (index % 5) * 0.22;
  const pipeline = 2.8 + (index % 7) * 0.31;
  const persistence = 1.1 + (index % 4) * 0.2;
  const pluginIpc = 1.9 + (index % 6) * 0.27;
  const rag = index % 3 === 0 ? 4.4 : 0.8 + (index % 5) * 0.18;
  const streaming = 1.5 + (index % 8) * 0.24;
  const provider = 80 + (index % 13) * 11;
  const externalTool = index % 4 === 0 ? 25 + (index % 9) * 3 : 0;
  const network = 8 + (index % 10) * 1.7;
  const overhead = ingress + pipeline + persistence + pluginIpc + rag + streaming;
  const external = provider + externalTool + network;
  const total = overhead + external;
  return {
    index,
    segments_ms: {
      ingress,
      pipeline,
      persistence,
      plugin_ipc: pluginIpc,
      rag,
      streaming,
      provider,
      external_tool: externalTool,
      network,
    },
    langbot_overhead_ms: Number(overhead.toFixed(3)),
    external_latency_ms: Number(external.toFixed(3)),
    e2e_latency_ms: Number(total.toFixed(3)),
    accounting_gap_ms: Number((total - external - overhead).toFixed(6)),
  };
}

async function main() {
  const root = resolve(env.LBS_ROOT || process.cwd());
  const caseId = "langbot-overhead-accounting-contract";
  const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`;
  const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId));
  await mkdir(evidenceDir, { recursive: true });

  const startedAt = new Date();
  const sampleCount = Number(env.LANGBOT_PERF_CONTRACT_SAMPLES || "80");
  const overheadP95BudgetMs = Number(env.LANGBOT_PERF_OVERHEAD_P95_MS || "25");
  const samples = Array.from({ length: sampleCount }, (_, index) => makeSample(index));
  const overheads = samples.map((sample) => sample.langbot_overhead_ms);
  const e2e = samples.map((sample) => sample.e2e_latency_ms);
  const external = samples.map((sample) => sample.external_latency_ms);
  const gaps = samples.map((sample) => Math.abs(sample.accounting_gap_ms));
  const memory = process.memoryUsage();

  const metrics = {
    probe: caseId,
    sample_count: sampleCount,
    langbot_overhead_ms: stats(overheads),
    e2e_latency_ms: stats(e2e),
    external_latency_ms: stats(external),
    accounting_gap_max_ms: Number(Math.max(...gaps).toFixed(6)),
    samples,
  };
  const thresholds = {
    sample_count: threshold(sampleCount, 50, ">="),
    langbot_overhead_p95_ms: threshold(metrics.langbot_overhead_ms.p95, overheadP95BudgetMs, "<="),
    accounting_gap_max_ms: threshold(metrics.accounting_gap_max_ms, 0.001, "<="),
  };
  const status = Object.values(thresholds).every((item) => item.pass) ? "pass" : "fail";
  const metricsPath = join(evidenceDir, "metrics.json");
  const thresholdsPath = join(evidenceDir, "thresholds.json");
  const resourceLogPath = join(evidenceDir, "resource-log.json");
  const automationResultPath = join(evidenceDir, "automation-result.json");
  const resultPath = join(evidenceDir, "result.json");

  await writeFile(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`, "utf8");
  await writeFile(thresholdsPath, `${JSON.stringify(thresholds, null, 2)}\n`, "utf8");
  await writeFile(resourceLogPath, `${JSON.stringify({ memory, pid: process.pid }, null, 2)}\n`, "utf8");

  const finishedAt = new Date();
  const result = {
    source: "automation",
    case_id: caseId,
    run_id: runId,
    status,
    reason: status === "pass"
      ? "Overhead accounting contract passed all thresholds."
      : "Overhead accounting contract breached one or more thresholds.",
    started_at: startedAt.toISOString(),
    started_at_local: localIsoWithOffset(startedAt),
    finished_at: finishedAt.toISOString(),
    finished_at_local: localIsoWithOffset(finishedAt),
    duration_ms: finishedAt.getTime() - startedAt.getTime(),
    metrics_summary: {
      sample_count: metrics.sample_count,
      langbot_overhead_p95_ms: metrics.langbot_overhead_ms.p95,
      e2e_latency_p95_ms: metrics.e2e_latency_ms.p95,
      external_latency_p95_ms: metrics.external_latency_ms.p95,
      accounting_gap_max_ms: metrics.accounting_gap_max_ms,
    },
    thresholds_summary: thresholds,
    artifacts: {
      metrics_json: metricsPath,
      thresholds_json: thresholdsPath,
      resource_log_json: resourceLogPath,
      automation_result_json: automationResultPath,
      result_json: resultPath,
    },
    evidence_collected: ["metrics", "resource_log", "filesystem"],
  };

  const resultText = `${JSON.stringify(result, null, 2)}\n`;
  await writeFile(automationResultPath, resultText, "utf8");
  await writeFile(resultPath, resultText, "utf8");
  console.log(JSON.stringify(result, null, 2));
  exit(status === "pass" ? 0 : 1);
}

await main();