diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7f5e173c3..1912f9c6e 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -48,6 +48,22 @@ jobs: - name: Run unit + smoke tests run: uv run pytest tests/unit_tests/ tests/smoke/ -q --tb=short + - name: Run required security regressions + run: | + uv run pytest tests/test_cwe94_debug_exec.py \ + -q --tb=short -rs --junitxml=cwe94-results.xml + python scripts/assert-required-tests.py cwe94-results.xml \ + --module tests.test_cwe94_debug_exec \ + --name-prefix test_no_ --expected-tests 2 + + - name: Upload security regression results + if: always() + uses: actions/upload-artifact@v4 + with: + name: cwe94-results-${{ matrix.python-version }} + path: cwe94-results.xml + if-no-files-found: ignore + - name: Test Summary if: always() run: | diff --git a/Makefile b/Makefile index c057a7687..3b54cf441 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,9 @@ test-integration-fast: test-coverage: bash scripts/test-coverage.sh -# Full local quality gate (quick + integration + coverage) +# Full local quality gate (lint + unit/smoke/fast integration coverage) test-all-local: - bash scripts/test-quick.sh - bash scripts/test-integration-fast.sh + uv run ruff check src/langbot/ tests/ --output-format=concise bash scripts/test-coverage.sh # Run linting only @@ -33,4 +32,4 @@ lint: # Fix linting issues lint-fix: ruff check --fix src/langbot/ tests/ - ruff format src/langbot/ tests/ \ No newline at end of file + ruff format src/langbot/ tests/ diff --git a/scripts/test-coverage.sh b/scripts/test-coverage.sh index 211ceae4d..d98c88063 100755 --- a/scripts/test-coverage.sh +++ b/scripts/test-coverage.sh @@ -1,7 +1,7 @@ #!/bin/bash # Coverage gate script -# Runs all tests with coverage, enforcing minimum coverage threshold +# Runs unit, smoke, and fast integration tests with a combined coverage threshold # Uses separate pytest invocations to avoid sys.modules pollution between test types set -euo pipefail @@ -9,57 +9,35 @@ set -euo pipefail echo "=== LangBot Coverage Gate ===" echo "" -# Coverage threshold (baseline from current coverage, conservative buffer) -# Current: ~22.14%, threshold: 18% +# Minimum combined coverage for unit, smoke, and fast integration tests COVERAGE_THRESHOLD=18 # Create temporary directory for coverage files COV_DIR=$(mktemp -d) -trap "rm -rf $COV_DIR" EXIT +trap 'rm -rf "$COV_DIR"' EXIT +export COVERAGE_FILE="$COV_DIR/.coverage" echo "[1/3] Running unit + smoke tests with coverage..." uv run pytest tests/unit_tests/ tests/smoke/ \ --cov=langbot \ - --cov-report=json:$COV_DIR/unit.json \ - --cov-report=term-missing \ + --cov-report= \ -q --tb=short echo "" echo "[2/3] Running fast integration tests with coverage..." uv run pytest tests/integration/ -m "not slow" \ --cov=langbot \ - --cov-report=json:$COV_DIR/integration.json \ - --cov-report=term-missing \ + --cov-append \ + --cov-report= \ -q --tb=short echo "" -echo "[3/3] Combining coverage reports..." -# Use coverage combine if available, otherwise just report total -if command -v coverage &> /dev/null; then - # Combine JSON reports - coverage combine --keep $COV_DIR/unit.json $COV_DIR/integration.json \ - --data-file=$COV_DIR/combined.data 2>/dev/null || true - - coverage report --data-file=$COV_DIR/combined.data || true -else - echo "Note: coverage combine not available, showing individual reports above" -fi - -# Generate final XML report for CI (from last run) -uv run pytest tests/unit_tests/ tests/smoke/ \ - --cov=langbot \ - --cov-report=xml:coverage.xml \ - --cov-report=term \ - --cov-fail-under=$COVERAGE_THRESHOLD \ - -q 2>/dev/null || { - # If threshold check fails on combined, check unit+smoke baseline - echo "" - echo "Coverage threshold: $COVERAGE_THRESHOLD%" - echo "Note: Full coverage requires running all test types separately" -} +echo "[3/3] Reporting combined coverage..." +uv run coverage xml -o coverage.xml +uv run coverage report --fail-under="$COVERAGE_THRESHOLD" echo "" echo "=== Coverage Gate Complete ===" echo "" echo "Coverage baseline: $COVERAGE_THRESHOLD%" -echo "Coverage report saved to coverage.xml" \ No newline at end of file +echo "Coverage report saved to coverage.xml"