mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-11-01 05:15:22 +08:00
fix(ci): keep infrastructure failures visible in smoke verdicts
This commit is contained in:
@@ -88,7 +88,9 @@ jobs:
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text())
|
||||
out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
|
||||
f"{data['elapsed_seconds']:.2f}/60 seconds; {data['requests']} requests.\n")
|
||||
f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n")
|
||||
if data.get('infrastructure_error'):
|
||||
out.write('Preparation or teardown failed; inspect service logs.\n')
|
||||
out.write('Service/dependency preparation is outside the 60-second execution budget.\n')
|
||||
else:
|
||||
out.write('Core smoke did not produce a verdict; inspect preparation failure.\n')
|
||||
|
||||
+11
-1
@@ -216,7 +216,17 @@ def main():
|
||||
parser.add_argument('--lbctl', type=Path, required=True)
|
||||
parser.add_argument('--plugins', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
result = run(args)
|
||||
if args.output.exists():
|
||||
parser.error('--output must be a new directory')
|
||||
try:
|
||||
result = run(args)
|
||||
except Exception as exc:
|
||||
verdict = args.output / 'gate' / 'gate.json'
|
||||
result = json.loads(verdict.read_text()) if verdict.exists() else {}
|
||||
result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}')
|
||||
verdict.parent.mkdir(parents=True, exist_ok=True)
|
||||
verdict.write_text(json.dumps(result, indent=2) + '\n')
|
||||
raise
|
||||
print(json.dumps(result, indent=2))
|
||||
raise SystemExit(0 if result['passed'] else 1)
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from regression_suite import validate
|
||||
from regression_suite import Suite, validate
|
||||
from regression_traffic import read_process_snapshot
|
||||
|
||||
|
||||
@@ -84,6 +84,10 @@ def main():
|
||||
validate(profile)
|
||||
if 'smoke' not in profile:
|
||||
parser.error('a smoke profile is required')
|
||||
suite = Suite(profile, args.profile.resolve(), args.output / 'suite')
|
||||
pids = dict(profile.get('observe', {}).get('pids', {}))
|
||||
for name, path in profile.get('observe', {}).get('pid_files', {}).items():
|
||||
pids[name] = int(Path(suite.path(path)).read_text().strip())
|
||||
result = supervise(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -96,7 +100,7 @@ def main():
|
||||
],
|
||||
args.output,
|
||||
60,
|
||||
profile.get('observe', {}).get('pids', {}),
|
||||
pids,
|
||||
)
|
||||
print(json.dumps(result, indent=2))
|
||||
raise SystemExit(0 if result['passed'] else 1)
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
"""A gate must fail closed on timeout, missing verdict, and inner failure."""
|
||||
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from regression_gate import supervise
|
||||
|
||||
@@ -30,6 +32,30 @@ class GateTest(unittest.TestCase):
|
||||
self.assertTrue(result['timed_out'])
|
||||
self.assertLess(result['elapsed_seconds'], 2)
|
||||
|
||||
def test_teardown_failure_overrides_successful_smoke_verdict(self):
|
||||
from regression_ci import main
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory) / 'ci'
|
||||
|
||||
def fail_teardown(args):
|
||||
verdict = args.output / 'gate' / 'gate.json'
|
||||
verdict.parent.mkdir(parents=True)
|
||||
verdict.write_text(json.dumps({'passed': True}))
|
||||
raise RuntimeError('sandbox teardown failed')
|
||||
|
||||
with (
|
||||
patch('regression_ci.run', side_effect=fail_teardown),
|
||||
patch.object(
|
||||
sys, 'argv', ['regression_ci', '--output', str(output), '--lbctl', 'unused', '--plugins', 'unused']
|
||||
),
|
||||
self.assertRaises(RuntimeError),
|
||||
):
|
||||
main()
|
||||
result = json.loads((output / 'gate/gate.json').read_text())
|
||||
self.assertFalse(result['passed'])
|
||||
self.assertIn('sandbox teardown failed', result['infrastructure_error'])
|
||||
|
||||
def test_zero_exit_without_business_verdict_is_failed(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = supervise([sys.executable, '-c', 'pass'], Path(directory) / 'gate', 5, {})
|
||||
|
||||
Reference in New Issue
Block a user