LangBot/.github/workflows/run-tests.yml

name: Unit Tests

on:
  pull_request:
    types: [opened, ready_for_review, synchronize]
    paths:
      - 'src/langbot/**'
      - 'tests/**'
      - '.github/workflows/run-tests.yml'
      - 'pyproject.toml'
      - 'uv.lock'
      - 'run_tests.sh'
      - 'scripts/test-*.sh'
      - 'scripts/assert-required-tests.py'
  push:
    branches:
      - master
      - develop
      - 'feat/**'
    # No path filter on push: every push to the branches above runs the
    # full unit-test suite. feat/** branches in particular must be tested
    # on every push (they accumulate large changes before a PR exists).

jobs:
  test:
    name: Unit Tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.11', '3.12', '3.13']
      fail-fast: false

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        run: uv sync --dev

      - name: Run unit + smoke tests
        run: uv run pytest tests/unit_tests/ tests/smoke/ -q --tb=short

      - name: Run required security regressions
        run: |
          uv run pytest tests/test_cwe94_debug_exec.py \
            -q --tb=short -rs --junitxml=cwe94-results.xml
          python scripts/assert-required-tests.py cwe94-results.xml \
            --module tests.test_cwe94_debug_exec \
            --name-prefix test_no_ --expected-tests 2

      - name: Upload security regression results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: cwe94-results-${{ matrix.python-version }}
          path: cwe94-results.xml
          if-no-files-found: ignore

      - name: Test Summary
        if: always()
        run: |
          echo "## Unit Tests Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Python Version: ${{ matrix.python-version }}" >> $GITHUB_STEP_SUMMARY
          echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY

  integration:
    name: Fast Integration Tests
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        run: uv sync --dev

      - name: Run fast integration tests
        run: uv run pytest tests/integration/ -m "not slow" -q --tb=short

      - name: Integration Test Summary
        if: always()
        run: |
          echo "## Integration Tests Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY

  postgres-contracts:
    name: PostgreSQL Contract Tests
    runs-on: ubuntu-latest
    timeout-minutes: 15
    # Only this disposable, loopback-bound CI service uses trust authentication.
    # Admission fixtures create a separate unprivileged role with no password;
    # the literal postgres@ URL lets that fixture switch to the runtime role.
    env:
      TEST_POSTGRES_URL: postgresql+asyncpg://postgres@localhost:5432/langbot_test
      LANGBOT_ADMISSION_TEST_URL: postgresql+asyncpg://postgres@localhost:5432/langbot_admission_test
    services:
      postgres:
        image: pgvector/pgvector:pg16
        env:
          POSTGRES_USER: postgres
          POSTGRES_DB: langbot_test
          POSTGRES_HOST_AUTH_METHOD: trust
        ports:
          - '127.0.0.1:5432:5432'
        options: >-
          --health-cmd="pg_isready -U postgres -d langbot_test"
          --health-interval=5s
          --health-timeout=5s
          --health-retries=5

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        run: uv sync --dev --frozen

      - name: Create isolated admission test database
        run: >-
          docker exec "${{ job.services.postgres.id }}"
          psql -U postgres -d postgres -v ON_ERROR_STOP=1
          -c 'CREATE DATABASE langbot_admission_test;'

      - name: Run required PostgreSQL contracts
        run: |
          uv run pytest \
            tests/integration/persistence/test_pipeline_admission_postgres.py \
            tests/integration/persistence/test_rag_document_identity.py \
            tests/integration/persistence/test_rag_interrupted_ingestion.py \
            tests/integration/persistence/test_migration_branch_convergence.py \
            tests/integration/persistence/test_runner_timestamps_postgres.py \
            tests/unit_tests/api/service/test_monitoring_sessions.py \
            -k postgres -q --tb=short -rs --durations=10 \
            --junitxml=postgres-contract-results.xml
          python - <<'PY'
          from collections import Counter
          import xml.etree.ElementTree as ET

          cases = list(ET.parse('postgres-contract-results.xml').getroot().iter('testcase'))
          expected = {
              'test_pipeline_admission_postgres': 27,
              'test_rag_document_identity': 34,
              'test_rag_interrupted_ingestion': 22,
              'test_migration_branch_convergence': 7,
              'test_runner_timestamps_postgres': 32,
              'test_monitoring_sessions': 1,
          }
          counts = Counter(case.get('classname', '').rsplit('.', 1)[-1] for case in cases)
          for module, minimum in expected.items():
              assert counts[module] >= minimum, f'{module}: expected at least {minimum} cases, got {counts[module]}'
          unsuccessful = [
              case.get('name') for case in cases
              if any(case.find(tag) is not None for tag in ('skipped', 'failure', 'error'))
          ]
          assert not unsuccessful, f'PostgreSQL contracts must pass without skips: {unsuccessful}'
          print(f'PostgreSQL contracts: {len(cases)} passed, 0 skipped; {dict(counts)}')
          PY

      - name: Upload PostgreSQL contract results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: postgres-contract-results
          path: postgres-contract-results.xml
          if-no-files-found: ignore

  e2e:
    name: E2E Startup Tests
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Read official plugin source pin
        id: plugin-source
        run: |
          python3 - <<'PY'
          import json
          import os
          import re
          from pathlib import Path

          lock = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())
          assert lock['repository'] == 'langbot-app/langbot-plugins'
          assert re.fullmatch(r'[0-9a-f]{40}', lock['revision'])
          with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
              output.write(f"revision={lock['revision']}\n")
          PY

      - name: Checkout pinned official plugin sources
        uses: actions/checkout@v4
        with:
          repository: langbot-app/langbot-plugins
          ref: ${{ steps.plugin-source.outputs.revision }}
          path: .contract-sources/langbot-plugins
          persist-credentials: false

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        run: uv sync --dev

      - name: Run E2E startup tests
        run: uv run pytest tests/e2e --ignore=tests/e2e/test_local_runner_fake_provider.py -q --tb=short

      - name: Check Docker for the required LocalAgent Box contract
        run: docker info

      - name: Run required LocalAgent contracts
        env:
          LANGBOT_LEGACY_PLUGIN_ROOT: ${{ github.workspace }}/.contract-sources/langbot-plugins
        run: |
          uv run --no-sync pytest tests/e2e/test_local_runner_fake_provider.py \
            -q --tb=short -rs --junitxml=local-agent-results.xml
          python scripts/assert-required-tests.py local-agent-results.xml \
            --module tests.e2e.test_local_runner_fake_provider \
            --name-prefix test_local_runner_ --expected-tests 6

      - name: Upload LocalAgent contract results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: local-agent-contract-results
          path: local-agent-results.xml
          if-no-files-found: ignore

      - name: E2E Test Summary
        if: always()
        run: |
          echo "## E2E Startup Test Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY

  box-integration:
    name: Box Integration Tests
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        run: uv sync --dev

      - name: Check Docker runtime
        run: docker info

      - name: Run Box integration tests
        run: uv run pytest tests/integration_tests -q --tb=short

      - name: Box Integration Test Summary
        if: always()
        run: |
          echo "## Box Integration Test Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY

  coverage:
    name: Coverage Gate
    runs-on: ubuntu-latest
    needs: [test, integration, postgres-contracts]
    if: always()

    steps:
      - name: Verify required test jobs
        run: |
          test "${{ needs.test.result }}" = success
          test "${{ needs.integration.result }}" = success
          test "${{ needs.postgres-contracts.result }}" = success

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Install dependencies
        run: uv sync --dev

      - name: Run coverage (unit + smoke)
        run: |
          uv run pytest tests/unit_tests/ tests/smoke/ \
            --cov=langbot \
            --cov-report=xml \
            --cov-report=term-missing \
            --cov-fail-under=18 \
            -q --tb=short

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        with:
          files: ./coverage.xml
          flags: unit-tests
          name: coverage-report
          fail_ci_if_error: false
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

      - name: Coverage Summary
        if: always()
        run: |
          echo "## Coverage Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Threshold: 18%" >> $GITHUB_STEP_SUMMARY
          echo "Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY