test(ci): propagate coverage failures and require security regressions (#2669)

.github/workflows/run-tests.yml

@@ -48,6 +48,22 @@ jobs:
       - name: Run unit + smoke tests
         run: uv run pytest tests/unit_tests/ tests/smoke/ -q --tb=short
 
+      - name: Run required security regressions
+        run: |
+          uv run pytest tests/test_cwe94_debug_exec.py \
+            -q --tb=short -rs --junitxml=cwe94-results.xml
+          python scripts/assert-required-tests.py cwe94-results.xml \
+            --module tests.test_cwe94_debug_exec \
+            --name-prefix test_no_ --expected-tests 2
+
+      - name: Upload security regression results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cwe94-results-${{ matrix.python-version }}
+          path: cwe94-results.xml
+          if-no-files-found: ignore
+
       - name: Test Summary
         if: always()
         run: |

Makefile

@@ -19,10 +19,9 @@ test-integration-fast:
 test-coverage:
 	bash scripts/test-coverage.sh
 
-# Full local quality gate (quick + integration + coverage)
+# Full local quality gate (lint + unit/smoke/fast integration coverage)
 test-all-local:
-	bash scripts/test-quick.sh
-	bash scripts/test-integration-fast.sh
+	uv run ruff check src/langbot/ tests/ --output-format=concise
 	bash scripts/test-coverage.sh
 
 # Run linting only
@@ -33,4 +32,4 @@ lint:
 # Fix linting issues
 lint-fix:
 	ruff check --fix src/langbot/ tests/
-	ruff format src/langbot/ tests/
+	ruff format src/langbot/ tests/

scripts/test-coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Coverage gate script
-# Runs all tests with coverage, enforcing minimum coverage threshold
+# Runs unit, smoke, and fast integration tests with a combined coverage threshold
 # Uses separate pytest invocations to avoid sys.modules pollution between test types
 
 set -euo pipefail
@@ -9,57 +9,35 @@ set -euo pipefail
 echo "=== LangBot Coverage Gate ==="
 echo ""
 
-# Coverage threshold (baseline from current coverage, conservative buffer)
-# Current: ~22.14%, threshold: 18%
+# Minimum combined coverage for unit, smoke, and fast integration tests
 COVERAGE_THRESHOLD=18
 
 # Create temporary directory for coverage files
 COV_DIR=$(mktemp -d)
-trap "rm -rf $COV_DIR" EXIT
+trap 'rm -rf "$COV_DIR"' EXIT
+export COVERAGE_FILE="$COV_DIR/.coverage"
 
 echo "[1/3] Running unit + smoke tests with coverage..."
 uv run pytest tests/unit_tests/ tests/smoke/ \
     --cov=langbot \
-    --cov-report=json:$COV_DIR/unit.json \
-    --cov-report=term-missing \
+    --cov-report= \
     -q --tb=short
 echo ""
 
 echo "[2/3] Running fast integration tests with coverage..."
 uv run pytest tests/integration/ -m "not slow" \
     --cov=langbot \
-    --cov-report=json:$COV_DIR/integration.json \
-    --cov-report=term-missing \
+    --cov-append \
+    --cov-report= \
     -q --tb=short
 echo ""
 
-echo "[3/3] Combining coverage reports..."
-# Use coverage combine if available, otherwise just report total
-if command -v coverage &> /dev/null; then
-    # Combine JSON reports
-    coverage combine --keep $COV_DIR/unit.json $COV_DIR/integration.json \
-        --data-file=$COV_DIR/combined.data 2>/dev/null || true
-
-    coverage report --data-file=$COV_DIR/combined.data || true
-else
-    echo "Note: coverage combine not available, showing individual reports above"
-fi
-
-# Generate final XML report for CI (from last run)
-uv run pytest tests/unit_tests/ tests/smoke/ \
-    --cov=langbot \
-    --cov-report=xml:coverage.xml \
-    --cov-report=term \
-    --cov-fail-under=$COVERAGE_THRESHOLD \
-    -q 2>/dev/null || {
-    # If threshold check fails on combined, check unit+smoke baseline
-    echo ""
-    echo "Coverage threshold: $COVERAGE_THRESHOLD%"
-    echo "Note: Full coverage requires running all test types separately"
-}
+echo "[3/3] Reporting combined coverage..."
+uv run coverage xml -o coverage.xml
+uv run coverage report --fail-under="$COVERAGE_THRESHOLD"
 
 echo ""
 echo "=== Coverage Gate Complete ==="
 echo ""
 echo "Coverage baseline: $COVERAGE_THRESHOLD%"
-echo "Coverage report saved to coverage.xml"
+echo "Coverage report saved to coverage.xml"