LangBot/.github/workflows/daily-endurance.yml

name: Daily Endurance

on:
  schedule:
    # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
    - cron: '23 19 * * *'
  workflow_dispatch:
    inputs:
      scenario_groups:
        description: 'Optional local groups, space separated; empty runs only base cases'
        type: string
        default: 'faults resources lifecycle plugins'
      extra_plugins:
        description: 'Additional source plugin paths; clear with the plugins group to run only base plugins'
        type: string
        default: 'misc/EssentialCommands misc/FAQManager'
      restart_services:
        description: 'Check owned Core and Runtime restarts once before resource sampling'
        type: boolean
        default: true

permissions:
  contents: read

concurrency:
  group: daily-endurance-${{ github.ref }}
  cancel-in-progress: false

jobs:
  endurance:
    name: Daily regression (30m)
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: astral-sh/setup-uv@v4
        with:
          enable-cache: true
      - name: Install locked dependencies
        run: uv sync --dev --frozen
      - name: Prepare CLI and sandbox image
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          mkdir -p .endurance-bin
          gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \
            --pattern lbctl_linux_amd64 --pattern checksums.txt
          cd .endurance-bin
          sha256sum --check --ignore-missing checksums.txt
          chmod +x lbctl_linux_amd64
          docker pull python:3.12-alpine
      - name: Run repeated scenarios and recovery checks
        env:
          SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle plugins' || inputs.scenario_groups }}
          EXTRA_PLUGINS: ${{ github.event_name == 'schedule' && 'misc/EssentialCommands misc/FAQManager' || inputs.extra_plugins }}
          RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }}
        run: |
          read -r -a groups <<< "$SCENARIO_GROUPS"
          read -r -a extras <<< "$EXTRA_PLUGINS"
          args=()
          if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi
          uv run --no-sync python tools/regression_ci.py \
            --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \
            --lbctl .endurance-bin/lbctl_linux_amd64 --plugins plugins --extra-plugins "${extras[@]}" --output .qa/daily
      - name: Publish observations
        if: always()
        run: |
          python - <<'PY_SUMMARY'
          import json, os
          from pathlib import Path

          root = Path('.qa/daily')
          def read(path):
              source = root / path
              return json.loads(source.read_text()) if source.exists() else {}

          gate = read('gate/gate.json')
          suite = read('gate/suite/summary.json')
          manifest = read('plugins/manifest.json')
          coverage = read('gate/suite/coverage.json')
          passed = gate.get('passed') is True and suite.get('passed') is True
          outer_failed = any(gate.get(key) for key in ('infrastructure_error', 'timed_out', 'resource_errors', 'coverage_errors'))
          outcome = 'PASS' if passed else 'INCOMPLETE' if suite.get('outcome') == 'INCOMPLETE' and not outer_failed else 'FAIL / incomplete'
          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
              out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n")
              out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n')
              out.write('Real Core/Runtime/source plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
              out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n")
              restart = gate.get('restart_checks')
              if restart:
                  out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n")
                  for error in restart.get('errors', []):
                      out.write(f"Restart check error: {error}\n")
              out.write(f"Plugin selection: `{manifest.get('selection', 'not reported')}`.\n\n")
              for plugin in manifest.get('plugins', []):
                  out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
              out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n")
              out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n")
              data = suite.get('test_data', {})
              out.write(f"Observed conversations: {data.get('observed_conversations', 'not reported')}; AgentRuns: {data.get('observed_agent_runs', 'not reported')}; group sessions: {data.get('group_sessions', 'not reported')}; owned API resources: {data.get('owned_api_resources', 'not reported')}.\n\n")
              out.write('One Workspace; each pipeline reuses one Box per CI run. This does not test cross-group Box file isolation or tenant isolation.\n\n')
              observations = suite.get('observations', {})
              if 'successful_requests_per_second' in observations:
                  out.write(f"Successful throughput: {observations['successful_requests_per_second']:.3f} RPS.\n\n")
              if observations.get('step_p95_seconds'):
                  out.write('| Case | Full-step P95 (s) |\n|---|---:|\n')
                  for name, p95 in observations['step_p95_seconds'].items():
                      out.write(f"| {name} | {p95} |\n")
              for kind, values in coverage.get('kinds', {}).items():
                  out.write(f"\n{kind} behavior evidence: {values['decided']} / {values['total']}; statuses: {values['status']}.\n")
              if suite.get('resources', {}).get('targets'):
                  out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
                  for target, metrics in suite['resources']['targets'].items():
                      for metric, values in metrics.items():
                          out.write(f"| {target} | {metric} | {values.get('growth', 'unavailable')} | {values.get('allowance', 'unavailable')} |\n")
              errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
              if gate.get('timed_out'):
                  errors.append('Suite exceeded its 1860-second outer process guard.')
              if gate.get('infrastructure_error'):
                  errors.append(gate['infrastructure_error'])
              if not gate or not suite:
                  errors.append('Missing verdict; inspect preparation and service logs.')
              for error in errors:
                  out.write(f"\n- {error}\n")
          PY_SUMMARY
      - name: Preserve verdicts and observations
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: daily-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            .qa/daily/gate/gate.json
            .qa/daily/gate/resources.jsonl
            .qa/daily/gate/suite.log
            .qa/daily/gate/suite/summary.json
            .qa/daily/gate/suite/cases.json
            .qa/daily/gate/suite/health.json
            .qa/daily/gate/suite/cleanup-checks.json
            .qa/daily/gate/suite/resources.jsonl
            .qa/daily/gate/suite/cycles.json
            .qa/daily/gate/suite/execution.json
            .qa/daily/gate/suite/coverage.json
            .qa/daily/gate/suite/coverage.md
            .qa/daily/plugins/manifest.json
            .qa/daily/plugins/*-build.log
            .qa/daily/restart-checks/profile.json
            .qa/daily/restart-checks/cases.json
            .qa/daily/restart-checks/summary.json
            .qa/daily/restart-checks/coverage.json
            .qa/daily/restart-checks/coverage.md
            .qa/daily/restart-checks/cleanup-checks.json
            .qa/daily/restart-checks/owned.json
            .qa/daily/*.log
          if-no-files-found: warn
          retention-days: 30

  api-contracts:
    name: Daily API contracts
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - uses: astral-sh/setup-uv@v4
        with:
          enable-cache: true
      - name: Install locked dependencies
        run: uv sync --dev --frozen
      - name: Prepare CLI and sandbox image
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          mkdir -p .api-bin
          gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .api-bin \
            --pattern lbctl_linux_amd64 --pattern checksums.txt
          cd .api-bin
          sha256sum --check --ignore-missing checksums.txt
          chmod +x lbctl_linux_amd64
          docker pull python:3.12-alpine
      - name: Run API contracts
        run: |
          uv run --no-sync python tools/regression_ci.py \
            --suite api --api-seed "$GITHUB_RUN_ID" \
            --lbctl .api-bin/lbctl_linux_amd64 --plugins plugins --output .qa/api
      - name: Publish API result
        if: always()
        run: |
          python - <<'PY_SUMMARY'
          import json, os
          from pathlib import Path

          root = Path('.qa/api')
          def read(path):
              source = root / path
              return json.loads(source.read_text()) if source.exists() else {}

          gate = read('gate/gate.json')
          suite = read('gate/suite/summary.json')
          coverage = read('gate/suite/coverage.json')
          manifest = read('plugins/manifest.json')
          passed = gate.get('passed') is True and suite.get('passed') is True
          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
              out.write(f"Daily API contracts: {'PASS' if passed else 'FAIL / incomplete'}\n\n")
              out.write(f"Core: `{os.environ['GITHUB_SHA']}`; shuffle seed: `{gate.get('api_seed', os.environ['GITHUB_RUN_ID'])}`.\n\n")
              out.write('Fixed order followed by a seeded chain shuffle, with sequential operations inside each chain. The execution guard is 600 seconds; dependency and service preparation is separate. This job is not a required PR check.\n\n')
              out.write(f"Cases: {suite.get('requests', 'not reported')}; statuses: {suite.get('status', 'not reported')}; suite execution: {suite.get('elapsed_seconds', 'not reported')}s.\n\n")
              out.write(f"API behavior contracts: {coverage.get('areas', {}).get('api', 'not reported')}.\n\n")
              out.write(f"Observed HTTP route templates (including preparation): {len(coverage['http_observations']['routes']) if 'http_observations' in coverage else 'not reported'}. A route hit does not establish all behavior behind it.\n\n")
              for plugin in manifest.get('plugins', []):
                  out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
              errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
              if gate.get('timed_out'):
                  errors.append('API suite exceeded its 600-second outer process guard.')
              if gate.get('infrastructure_error'):
                  errors.append(gate['infrastructure_error'])
              if not gate or not suite:
                  errors.append('Missing verdict; inspect preparation and service logs.')
              for error in errors:
                  out.write(f"- {error}\n")
          PY_SUMMARY
      - name: Preserve API verdicts and evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: daily-api-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            .qa/api/gate/gate.json
            .qa/api/gate/resources.jsonl
            .qa/api/gate/suite.log
            .qa/api/gate/suite/summary.json
            .qa/api/gate/suite/profile.json
            .qa/api/gate/suite/cases.json
            .qa/api/gate/suite/health.json
            .qa/api/gate/suite/cleanup-checks.json
            .qa/api/gate/suite/resource-case-*.json
            .qa/api/gate/suite/coverage.json
            .qa/api/gate/suite/coverage.md
            .qa/api/plugins/manifest.json
            .qa/api/plugins/*-build.log
            .qa/api/*.log
          if-no-files-found: warn
          retention-days: 30