mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-11-01 05:15:22 +08:00
test(core): strengthen handler contracts and report source quality
This commit is contained in:
huanghuoguoguo
+127
-33
@@ -11,6 +11,7 @@ on:
|
||||
- 'uv.lock'
|
||||
- 'run_tests.sh'
|
||||
- 'scripts/test-*.sh'
|
||||
- 'scripts/test-*.py'
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -42,10 +43,63 @@ jobs:
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
run: uv sync --dev --frozen
|
||||
|
||||
- name: Run unit + smoke tests
|
||||
run: uv run pytest tests/unit_tests/ tests/smoke/ -q --tb=short
|
||||
if: matrix.python-version != '3.12'
|
||||
run: uv run pytest tests/unit_tests/ tests/smoke/ -q --tb=short -rs --durations=10
|
||||
|
||||
- name: Run unit + smoke tests with coverage
|
||||
if: matrix.python-version == '3.12'
|
||||
run: |
|
||||
uv run pytest tests/unit_tests/ tests/smoke/ \
|
||||
--cov=langbot --cov-branch --cov-report=xml --cov-report=json --cov-report=term-missing \
|
||||
--cov-fail-under=0 -q --tb=short -rs --durations=10
|
||||
# Preserve the existing line-coverage gate. With --cov-branch,
|
||||
# pytest-cov's aggregate total combines statements and branches.
|
||||
python - <<'PY'
|
||||
import json
|
||||
totals = json.load(open('coverage.json'))['totals']
|
||||
line_coverage = 100 * totals['covered_lines'] / totals['num_statements']
|
||||
assert line_coverage >= 18, f'Line coverage {line_coverage:.2f}% is below 18%'
|
||||
PY
|
||||
|
||||
- name: Report source coverage and complexity
|
||||
if: always() && matrix.python-version == '3.12' && hashFiles('coverage.json') != ''
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
|
||||
run: |
|
||||
args=()
|
||||
if [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ && "$BASE_SHA" != 0000000000000000000000000000000000000000 ]]; then
|
||||
git fetch --no-tags --depth=1 origin "$BASE_SHA"
|
||||
args+=(--base-ref "$BASE_SHA")
|
||||
fi
|
||||
uv run python scripts/test-complexity.py \
|
||||
--json complexity.json --summary complexity.md --coverage-json coverage.json "${args[@]}"
|
||||
cat complexity.md >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload source quality reports
|
||||
if: always() && matrix.python-version == '3.12'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: source-quality-reports
|
||||
path: |
|
||||
coverage.json
|
||||
coverage.xml
|
||||
complexity.json
|
||||
complexity.md
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
if: matrix.python-version == '3.12'
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
files: ./coverage.xml
|
||||
flags: unit-tests
|
||||
name: coverage-report
|
||||
fail_ci_if_error: false
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
|
||||
- name: Test Summary
|
||||
if: always()
|
||||
@@ -72,10 +126,10 @@ jobs:
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
run: uv sync --dev --frozen
|
||||
|
||||
- name: Run fast integration tests
|
||||
run: uv run pytest tests/integration/ -m "not slow" -q --tb=short
|
||||
run: uv run pytest tests/integration/ -m "not slow" -q --tb=short -rs --durations=10
|
||||
|
||||
- name: Integration Test Summary
|
||||
if: always()
|
||||
@@ -101,10 +155,10 @@ jobs:
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
run: uv sync --dev --frozen
|
||||
|
||||
- name: Run E2E startup tests
|
||||
run: uv run pytest tests/e2e -q --tb=short
|
||||
run: uv run pytest tests/e2e --ignore=tests/e2e/test_local_runner_fake_provider.py -q --tb=short -rs --durations=10
|
||||
|
||||
- name: E2E Test Summary
|
||||
if: always()
|
||||
@@ -130,13 +184,13 @@ jobs:
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
run: uv sync --dev --frozen
|
||||
|
||||
- name: Check Docker runtime
|
||||
run: docker info
|
||||
|
||||
- name: Run Box integration tests
|
||||
run: uv run pytest tests/integration_tests -q --tb=short
|
||||
run: uv run pytest tests/integration_tests -q --tb=short -rs --durations=10
|
||||
|
||||
- name: Box Integration Test Summary
|
||||
if: always()
|
||||
@@ -145,10 +199,13 @@ jobs:
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
coverage:
|
||||
name: Coverage Gate
|
||||
local-agent:
|
||||
name: Local Agent Contract E2E
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, integration]
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
LANGBOT_E2E_REQUIRE_LOCAL_AGENT: '1'
|
||||
LANGBOT_E2E_LOCAL_AGENT_PACKAGE: ${{ github.workspace }}/.pytest_cache/local-agent/LocalAgent.lbpkg
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -163,31 +220,68 @@ jobs:
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
run: uv sync --dev --frozen
|
||||
|
||||
- name: Run coverage (unit + smoke)
|
||||
run: |
|
||||
uv run pytest tests/unit_tests/ tests/smoke/ \
|
||||
--cov=langbot \
|
||||
--cov-report=xml \
|
||||
--cov-report=term-missing \
|
||||
--cov-fail-under=18 \
|
||||
-q --tb=short
|
||||
# Resolve every run, before consulting the archive cache. The latest
|
||||
# pointer is never cached or replaced with a stale successful version.
|
||||
- name: Resolve latest LocalAgent from Space
|
||||
id: local-agent
|
||||
run: uv run python scripts/test-local-agent-fixture.py resolve local-agent-resolution.json --github-output "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
- name: Cache resolved Space artifact
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
files: ./coverage.xml
|
||||
flags: unit-tests
|
||||
name: coverage-report
|
||||
fail_ci_if_error: false
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
path: ${{ github.workspace }}/.pytest_cache/local-agent
|
||||
key: local-agent-space-${{ steps.local-agent.outputs.cache_key }}
|
||||
|
||||
- name: Coverage Summary
|
||||
# Revalidate every cache hit against this run's metadata. Provisioning
|
||||
# failures are reported separately from Core/runtime contract failures.
|
||||
- name: Provision and verify resolved LocalAgent
|
||||
run: uv run python scripts/test-local-agent-fixture.py fetch local-agent-resolution.json "$LANGBOT_E2E_LOCAL_AGENT_PACKAGE"
|
||||
|
||||
- name: Check Docker runtime for LocalAgent Box probe
|
||||
run: docker info
|
||||
|
||||
- name: Run required LocalAgent contract tests
|
||||
run: |
|
||||
uv run pytest tests/e2e/test_local_runner_fake_provider.py \
|
||||
-q --tb=short -rs --durations=10 --junitxml=local-agent-results.xml
|
||||
python - <<'PY'
|
||||
import xml.etree.ElementTree as ET
|
||||
suites = ET.parse('local-agent-results.xml').getroot().iter('testsuite')
|
||||
counts = [suite.attrib for suite in suites]
|
||||
assert sum(int(suite['tests']) for suite in counts) > 0, 'No LocalAgent tests ran'
|
||||
assert sum(int(suite.get('skipped', 0)) for suite in counts) == 0, 'LocalAgent tests must not skip in CI'
|
||||
PY
|
||||
|
||||
- name: Upload LocalAgent test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: local-agent-test-results
|
||||
path: |
|
||||
local-agent-results.xml
|
||||
local-agent-resolution.json
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: LocalAgent Test Summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Coverage Results" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Threshold: 18%" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
|
||||
echo "## Local Agent Contract E2E" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Latest Space artifact: langbot-team/LocalAgent (version/checksum in resolution artifact)" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Missing fixtures fail this job; optional local source checkouts are not used." >> $GITHUB_STEP_SUMMARY
|
||||
echo "Test Status: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
# Preserve the existing check name for branch protection without running the
|
||||
# unit suite a fourth time. Python 3.12 above owns the unchanged 18% gate.
|
||||
coverage:
|
||||
name: Coverage Gate
|
||||
runs-on: ubuntu-latest
|
||||
needs: [test, integration]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Verify test and coverage results
|
||||
run: |
|
||||
test "${{ needs.test.result }}" = success
|
||||
test "${{ needs.integration.result }}" = success
|
||||
echo "Coverage threshold: 18%, enforced in Unit Tests (3.12)." >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Report Ruff's McCabe C901 scores, optionally against an explicit Git revision.
|
||||
|
||||
Scope: every Python file in src/langbot, including libs, migrations and templates.
|
||||
No complexity threshold is enforced. Ruff is the sole complexity calculator; AST
|
||||
parsing supplies qualified names, source ranges and change fingerprints only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
SOURCE = 'src/langbot'
|
||||
|
||||
|
||||
def command(args: list[str], root: Path) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(args, cwd=root, capture_output=True, check=False)
|
||||
|
||||
|
||||
def git(root: Path, *args: str) -> bytes:
|
||||
result = command(['git', *args], root)
|
||||
if result.returncode:
|
||||
raise ValueError(result.stderr.decode(errors='replace').strip())
|
||||
return result.stdout
|
||||
|
||||
|
||||
def index_functions(path: Path) -> dict[int, dict]:
|
||||
"""Identify functions without trying to reimplement Ruff's complexity metric."""
|
||||
functions = {}
|
||||
occurrences = Counter()
|
||||
|
||||
def visit(node: ast.AST, parents: tuple[str, ...] = ()) -> None:
|
||||
if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
parents = (*parents, node.name)
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
name = '.'.join(parents)
|
||||
occurrences[name] += 1
|
||||
functions[node.lineno] = {
|
||||
'name': name,
|
||||
'occurrence': occurrences[name],
|
||||
'line': node.lineno,
|
||||
'end_line': node.end_lineno,
|
||||
'fingerprint': hashlib.sha256(ast.dump(node).encode()).hexdigest(),
|
||||
}
|
||||
for child in ast.iter_child_nodes(node):
|
||||
visit(child, parents)
|
||||
|
||||
visit(ast.parse(path.read_bytes(), filename=str(path)))
|
||||
return functions
|
||||
|
||||
|
||||
def measure(root: Path) -> list[dict]:
|
||||
source = root / SOURCE
|
||||
if not source.is_dir():
|
||||
return []
|
||||
result = command(
|
||||
[
|
||||
sys.executable,
|
||||
'-m',
|
||||
'ruff',
|
||||
'check',
|
||||
'--isolated',
|
||||
'--no-cache',
|
||||
'--select',
|
||||
'C901',
|
||||
'--config',
|
||||
'lint.mccabe.max-complexity=0',
|
||||
'--config',
|
||||
'include=["*.py"]',
|
||||
'--target-version',
|
||||
'py312',
|
||||
'--ignore-noqa',
|
||||
'--no-respect-gitignore',
|
||||
'--exclude',
|
||||
'',
|
||||
'--output-format',
|
||||
'json',
|
||||
SOURCE,
|
||||
],
|
||||
root,
|
||||
)
|
||||
# Exit 1 is expected: max-complexity=0 intentionally reports every function.
|
||||
if result.returncode not in (0, 1):
|
||||
raise ValueError(f'Ruff failed: {result.stderr.decode(errors="replace").strip()}')
|
||||
diagnostics = json.loads(result.stdout)
|
||||
functions = []
|
||||
indexes = {}
|
||||
for diagnostic in diagnostics:
|
||||
match = re.fullmatch(r'`.+` is too complex \((\d+) > 0\)', diagnostic['message'])
|
||||
if diagnostic['code'] != 'C901' or not match:
|
||||
raise ValueError(f'Unexpected Ruff diagnostic: {diagnostic}')
|
||||
path = Path(diagnostic['filename'])
|
||||
if path not in indexes:
|
||||
indexes[path] = index_functions(path)
|
||||
line = diagnostic['location']['row']
|
||||
if line not in indexes[path]:
|
||||
raise ValueError(f'Cannot identify Ruff function at {path}:{line}')
|
||||
functions.append(
|
||||
{
|
||||
'path': path.relative_to(root).as_posix(),
|
||||
**indexes[path][line],
|
||||
'complexity': int(match[1]),
|
||||
}
|
||||
)
|
||||
return sorted(functions, key=lambda item: (item['path'], item['line']))
|
||||
|
||||
|
||||
def baseline(root: Path, ref: str) -> tuple[str, list[dict]]:
|
||||
revision = git(root, 'rev-parse', '--verify', '--end-of-options', f'{ref}^{{commit}}').decode().strip()
|
||||
# Snapshot source only. Never execute baseline code or load baseline config.
|
||||
entries = git(root, 'ls-tree', '--name-only', revision, '--', SOURCE)
|
||||
if not entries:
|
||||
return revision, []
|
||||
archive = git(root, 'archive', '--format=tar', revision, '--', SOURCE)
|
||||
with tempfile.TemporaryDirectory(prefix='langbot-complexity-') as directory:
|
||||
snapshot = Path(directory)
|
||||
with tarfile.open(fileobj=io.BytesIO(archive)) as contents:
|
||||
for member in contents:
|
||||
path = PurePosixPath(member.name)
|
||||
if not member.isfile() or path.suffix != '.py':
|
||||
continue
|
||||
if path.is_absolute() or '..' in path.parts or not path.is_relative_to(SOURCE):
|
||||
raise ValueError(f'Unsafe source archive path: {member.name}')
|
||||
destination = snapshot / path
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
with contents.extractfile(member) as handle:
|
||||
destination.write_bytes(handle.read())
|
||||
return revision, measure(snapshot)
|
||||
|
||||
|
||||
def file_renames(root: Path, revision: str) -> dict[str, str]:
|
||||
parts = git(root, 'diff', '--name-status', '-z', '--find-renames', revision, '--', SOURCE).split(b'\0')
|
||||
renames = {}
|
||||
index = 0
|
||||
while index < len(parts) and parts[index]:
|
||||
status = parts[index].decode()
|
||||
index += 1
|
||||
old_path = parts[index].decode()
|
||||
index += 1
|
||||
if status.startswith(('R', 'C')):
|
||||
new_path = parts[index].decode()
|
||||
index += 1
|
||||
if status.startswith('R'):
|
||||
renames[new_path] = old_path
|
||||
return renames
|
||||
|
||||
|
||||
def function_key(function: dict, path: str | None = None) -> tuple:
|
||||
return path or function['path'], function['name'], function['occurrence']
|
||||
|
||||
|
||||
def compare(before: list[dict], after: list[dict], renames: dict[str, str]) -> list[dict]:
|
||||
"""Do not infer function renames; show unmatched identities as added/deleted."""
|
||||
remaining = {function_key(item): item for item in before}
|
||||
changes = []
|
||||
for current in after:
|
||||
key = function_key(current, renames.get(current['path']))
|
||||
previous = remaining.pop(key, None)
|
||||
if previous is None:
|
||||
status = 'added'
|
||||
elif current['path'] != previous['path']:
|
||||
status = 'renamed_file'
|
||||
elif current['fingerprint'] != previous['fingerprint']:
|
||||
status = 'modified'
|
||||
else:
|
||||
continue
|
||||
changes.append(
|
||||
{
|
||||
'status': status,
|
||||
'before': previous,
|
||||
'after': current,
|
||||
'delta': current['complexity'] - previous['complexity'] if previous else None,
|
||||
}
|
||||
)
|
||||
changes.extend({'status': 'deleted', 'before': item, 'after': None, 'delta': None} for item in remaining.values())
|
||||
return changes
|
||||
|
||||
|
||||
def percentage(covered: int, total: int) -> float | None:
|
||||
return round(100 * covered / total, 2) if total else None
|
||||
|
||||
|
||||
def attach_coverage(functions: list[dict], coverage: dict, root: Path) -> dict:
|
||||
files = {}
|
||||
for filename, data in coverage['files'].items():
|
||||
path = Path(filename)
|
||||
if not path.is_absolute():
|
||||
path = root / path
|
||||
try:
|
||||
relative = path.relative_to(root).as_posix()
|
||||
except ValueError:
|
||||
continue
|
||||
if PurePosixPath(relative).is_relative_to(SOURCE):
|
||||
files[relative] = data
|
||||
branch_enabled = coverage.get('meta', {}).get('branch_coverage', False)
|
||||
for function in functions:
|
||||
data = files.get(function['path'])
|
||||
if data is None:
|
||||
function['coverage'] = None
|
||||
continue
|
||||
start, end = function['line'], function['end_line']
|
||||
excluded = set(data.get('excluded_lines', []))
|
||||
executed = {line for line in data['executed_lines'] if start <= line <= end} - excluded
|
||||
missing = {line for line in data['missing_lines'] if start <= line <= end} - excluded
|
||||
executed_branches = {tuple(arc) for arc in data.get('executed_branches', []) if start <= arc[0] <= end}
|
||||
missing_branches = {tuple(arc) for arc in data.get('missing_branches', []) if start <= arc[0] <= end}
|
||||
function['coverage'] = {
|
||||
'line_percent': percentage(len(executed), len(executed | missing)),
|
||||
'branch_percent': percentage(len(executed_branches), len(executed_branches | missing_branches))
|
||||
if branch_enabled
|
||||
else None,
|
||||
'covered_lines': len(executed),
|
||||
'missing_lines': len(missing),
|
||||
'covered_branches': len(executed_branches) if branch_enabled else None,
|
||||
'missing_branches': len(missing_branches) if branch_enabled else None,
|
||||
}
|
||||
summaries = [data['summary'] for data in files.values()]
|
||||
covered = sum(item['covered_lines'] for item in summaries)
|
||||
statements = sum(item['num_statements'] for item in summaries)
|
||||
covered_branches = sum(item.get('covered_branches', 0) for item in summaries)
|
||||
branches = sum(item.get('num_branches', 0) for item in summaries)
|
||||
return {
|
||||
'matched_source_files': len(files),
|
||||
'functions_without_coverage': sum(item.get('coverage') is None for item in functions),
|
||||
'line_percent': percentage(covered, statements),
|
||||
'branch_percent': percentage(covered_branches, branches) if branch_enabled else None,
|
||||
'covered_lines': covered,
|
||||
'statements': statements,
|
||||
'covered_branches': covered_branches if branch_enabled else None,
|
||||
'branches': branches if branch_enabled else None,
|
||||
'note': 'Coverage covers matching source files only. Function coverage uses inclusive def-to-end source '
|
||||
'ranges; nested functions overlap. Unmatched coverage is unknown, not zero. '
|
||||
'No-branch functions have null branch_percent. Coverage must come from this same checkout.',
|
||||
}
|
||||
|
||||
|
||||
def statistics(functions: list[dict]) -> dict:
|
||||
scores = [item['complexity'] for item in functions]
|
||||
return {
|
||||
'function_count': len(scores),
|
||||
'mean': round(sum(scores) / len(scores), 2) if scores else None,
|
||||
'maximum': max(scores, default=None),
|
||||
'distribution': {
|
||||
'1-5': sum(score <= 5 for score in scores),
|
||||
'6-10': sum(6 <= score <= 10 for score in scores),
|
||||
'11-20': sum(11 <= score <= 20 for score in scores),
|
||||
'21-50': sum(21 <= score <= 50 for score in scores),
|
||||
'51+': sum(score > 50 for score in scores),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def hotspot_key(function: dict) -> tuple:
|
||||
coverage = function.get('coverage') or {}
|
||||
line = coverage.get('line_percent')
|
||||
return -function['complexity'], line if line is not None else 101, function['path'], function['line']
|
||||
|
||||
|
||||
def format_percent(value: float | None) -> str:
|
||||
return f'{value:.2f}%' if value is not None else 'n/a'
|
||||
|
||||
|
||||
def describe(function: dict) -> str:
|
||||
text = f'`{function["path"]}:{function["line"]}` {function["name"]} CC={function["complexity"]}'
|
||||
coverage = function.get('coverage')
|
||||
if coverage:
|
||||
text += (
|
||||
f'; lines {format_percent(coverage["line_percent"])}, branches {format_percent(coverage["branch_percent"])}'
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def render_summary(report: dict, limit: int) -> str:
|
||||
stats = report['current']
|
||||
lines = [
|
||||
'# Source complexity report',
|
||||
'',
|
||||
f'Analyzer: {report["analyzer"]} (C901, report only; no complexity gate)',
|
||||
f'Scope: {SOURCE}/**/*.py, including libs, migrations and templates',
|
||||
'',
|
||||
f'Functions: {stats["function_count"]}; mean CC: {stats["mean"]}; maximum CC: {stats["maximum"]}',
|
||||
'Distribution: ' + ', '.join(f'{key}: {value}' for key, value in stats['distribution'].items()),
|
||||
'',
|
||||
]
|
||||
if 'coverage' in report:
|
||||
coverage = report['coverage']
|
||||
lines.extend(
|
||||
[
|
||||
f'Source line coverage: {format_percent(coverage["line_percent"])}; '
|
||||
f'source branch coverage: {format_percent(coverage["branch_percent"])}',
|
||||
f'Matched coverage files: {coverage["matched_source_files"]}; functions with unknown coverage: '
|
||||
f'{coverage["functions_without_coverage"]}',
|
||||
coverage['note'],
|
||||
'',
|
||||
]
|
||||
)
|
||||
if 'baseline' in report:
|
||||
base = report['baseline']
|
||||
lines.extend(
|
||||
[
|
||||
f'## Changes against {report["base_revision"]}',
|
||||
'',
|
||||
f'Baseline functions: {base["function_count"]}; mean CC: {base["mean"]}; maximum CC: {base["maximum"]}',
|
||||
'Baseline distribution: ' + ', '.join(f'{key}: {value}' for key, value in base['distribution'].items()),
|
||||
'Statuses: '
|
||||
+ ', '.join(
|
||||
f'{key}: {value}' for key, value in Counter(c['status'] for c in report['changes']).items()
|
||||
),
|
||||
'Matches use file path, qualified name and duplicate-definition occurrence. Git-detected file renames '
|
||||
'are matched; function renames appear as added/deleted. Changes are AST changes, excluding comments/formatting.',
|
||||
'',
|
||||
]
|
||||
)
|
||||
changes = sorted(report['changes'], key=lambda change: hotspot_key(change['after'] or change['before']))
|
||||
for change in changes[:limit]:
|
||||
delta = f'; delta {change["delta"]:+d}' if change['delta'] is not None else ''
|
||||
lines.append(f'- {change["status"]}: {describe(change["after"] or change["before"])}{delta}')
|
||||
if not changes:
|
||||
lines.append('No function changes in scope.')
|
||||
lines.append('')
|
||||
lines.extend(
|
||||
[f'## Top {limit} hotspots', '', 'Ordered by complexity, then lower known line coverage for ties.', '']
|
||||
)
|
||||
lines.extend(f'- {describe(function)}' for function in report['hotspots'])
|
||||
lines.extend(
|
||||
[
|
||||
'',
|
||||
'Scores are Ruff McCabe values, not a composite quality score. Nested functions are included '
|
||||
'as reported by Ruff. Full function records and changes are in the JSON artifact.',
|
||||
'',
|
||||
]
|
||||
)
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--root', type=Path, default=Path(__file__).resolve().parents[1])
|
||||
parser.add_argument('--base-ref', help='Explicit commit/ref to compare with the current working tree')
|
||||
parser.add_argument('--coverage-json', type=Path, help='coverage.py JSON generated from this checkout')
|
||||
parser.add_argument('--json', type=Path, default=Path('complexity.json'))
|
||||
parser.add_argument('--summary', type=Path, default=Path('complexity.md'))
|
||||
parser.add_argument('--top', type=int, default=20)
|
||||
args = parser.parse_args(argv)
|
||||
if args.top < 1:
|
||||
parser.error('--top must be positive')
|
||||
root = args.root.resolve()
|
||||
try:
|
||||
if not (root / SOURCE).is_dir():
|
||||
raise ValueError(f'Missing source directory: {root / SOURCE}')
|
||||
version = command([sys.executable, '-m', 'ruff', '--version'], root)
|
||||
if version.returncode:
|
||||
raise ValueError('Ruff is required; run uv sync --dev first')
|
||||
functions = measure(root)
|
||||
report = {
|
||||
'schema_version': 1,
|
||||
'analyzer': version.stdout.decode().strip(),
|
||||
'metric': 'Ruff C901 McCabe complexity',
|
||||
'scope': f'{SOURCE}/**/*.py (including libs)',
|
||||
'current': statistics(functions),
|
||||
'functions': functions,
|
||||
}
|
||||
if args.coverage_json:
|
||||
report['coverage'] = attach_coverage(functions, json.loads(args.coverage_json.read_text()), root)
|
||||
report['hotspots'] = sorted(functions, key=hotspot_key)[: args.top]
|
||||
if args.base_ref:
|
||||
revision, previous = baseline(root, args.base_ref)
|
||||
report.update(
|
||||
{
|
||||
'base_ref': args.base_ref,
|
||||
'base_revision': revision,
|
||||
'baseline': statistics(previous),
|
||||
'changes': compare(previous, functions, file_renames(root, revision)),
|
||||
}
|
||||
)
|
||||
summary = render_summary(report, args.top)
|
||||
for output in (args.json, args.summary):
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.json.write_text(json.dumps(report, indent=2) + '\n')
|
||||
args.summary.write_text(summary)
|
||||
print(summary)
|
||||
return 0
|
||||
except (OSError, ValueError, SyntaxError, KeyError, tarfile.TarError) as error:
|
||||
print(f'Complexity report failed: {error}', file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Resolve the latest official LocalAgent from Space and prepare its E2E fixture.
|
||||
|
||||
Resolution always contacts Space. Cache only the version/checksum-resolved
|
||||
archive, preserving its certification envelope for normal Runtime admission.
|
||||
The saved resolution JSON can also replay the exact package used by a CI run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import zipfile
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
|
||||
SPACE_URL = 'https://space.langbot.app'
|
||||
PLUGIN_AUTHOR = 'langbot-team'
|
||||
PLUGIN_NAME = 'LocalAgent'
|
||||
VERSIONS_URL = f'{SPACE_URL}/api/v1/marketplace/plugins/{PLUGIN_AUTHOR}/{PLUGIN_NAME}/versions'
|
||||
MAX_PACKAGE_BYTES = 16 * 1024 * 1024
|
||||
MAX_METADATA_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
def download_package(url: str, *, max_bytes: int = MAX_PACKAGE_BYTES) -> bytes:
|
||||
"""Use Core's HTTP client with bounded responses and network retries."""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with httpx.stream('GET', url, timeout=30, follow_redirects=True) as response:
|
||||
response.raise_for_status()
|
||||
body = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
body.extend(chunk)
|
||||
if len(body) > max_bytes:
|
||||
raise ValueError(f'Space response exceeds the {max_bytes}-byte limit')
|
||||
return bytes(body)
|
||||
except httpx.HTTPError as exc:
|
||||
if attempt == 2:
|
||||
raise RuntimeError(f'Cannot fetch LocalAgent fixture from Space: {url}: {exc}') from exc
|
||||
time.sleep(attempt + 1)
|
||||
raise AssertionError('Unreachable download retry state')
|
||||
|
||||
|
||||
def resolve_latest() -> dict:
|
||||
"""Select the first published version, matching Core's marketplace resolver."""
|
||||
payload = json.loads(download_package(VERSIONS_URL, max_bytes=MAX_METADATA_BYTES))
|
||||
if payload.get('code') != 0:
|
||||
raise ValueError('Space did not return a successful plugin version response')
|
||||
versions = payload.get('data', {}).get('versions', [])
|
||||
if not versions or not isinstance(versions, list) or not isinstance(versions[0], dict):
|
||||
raise ValueError('Space returned no LocalAgent versions')
|
||||
latest = versions[0]
|
||||
version = latest.get('version', '')
|
||||
checksum = latest.get('checksum', '')
|
||||
if not isinstance(version, str) or not re.fullmatch(r'[A-Za-z0-9._+-]+', version):
|
||||
raise ValueError('Space returned an invalid LocalAgent version')
|
||||
if not isinstance(checksum, str) or not re.fullmatch(r'[a-fA-F0-9]{64}', checksum):
|
||||
raise ValueError('Space did not provide a valid archive SHA256 checksum')
|
||||
if latest.get('plugin_id') != f'{PLUGIN_AUTHOR}/{PLUGIN_NAME}' or latest.get('status') != 'live':
|
||||
raise ValueError('Latest Space entry is not the published LocalAgent plugin')
|
||||
return {
|
||||
'author': PLUGIN_AUTHOR,
|
||||
'name': PLUGIN_NAME,
|
||||
'version': version,
|
||||
'url': f'{SPACE_URL}/api/v1/marketplace/plugins/download/{PLUGIN_AUTHOR}/{PLUGIN_NAME}/{version}',
|
||||
'sha256': checksum.lower(),
|
||||
'resolved_at': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'versions_url': VERSIONS_URL,
|
||||
}
|
||||
|
||||
|
||||
def validate_package(package: bytes, resolution: dict) -> None:
|
||||
"""Check the original archive bytes and manifest identity before installation."""
|
||||
actual = hashlib.sha256(package).hexdigest()
|
||||
if actual != resolution['sha256']:
|
||||
raise ValueError(f'LocalAgent SHA256 mismatch: expected {resolution["sha256"]}, got {actual}')
|
||||
with zipfile.ZipFile(io.BytesIO(package)) as archive:
|
||||
manifest = yaml.safe_load(archive.read('manifest.yaml'))
|
||||
metadata = manifest['metadata']
|
||||
for field in ('author', 'name', 'version'):
|
||||
if str(metadata[field]) != resolution[field]:
|
||||
raise ValueError(f'LocalAgent manifest {field} does not match the resolved artifact')
|
||||
|
||||
|
||||
def fetch_package(destination: Path, resolution: dict) -> Path:
|
||||
"""Reuse only a verified resolved archive or download it atomically."""
|
||||
if destination.is_file():
|
||||
try:
|
||||
validate_package(destination.read_bytes(), resolution)
|
||||
return destination
|
||||
except (ValueError, KeyError, zipfile.BadZipFile):
|
||||
print('Cached LocalAgent package does not match this run; downloading resolved artifact', file=sys.stderr)
|
||||
|
||||
package = download_package(resolution['url'])
|
||||
validate_package(package, resolution)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = destination.with_suffix('.tmp')
|
||||
temporary.write_bytes(package)
|
||||
temporary.replace(destination)
|
||||
return destination
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest='command', required=True)
|
||||
resolve = commands.add_parser('resolve', help='Query Space latest and save replay metadata; never cached')
|
||||
resolve.add_argument('resolution', type=Path)
|
||||
resolve.add_argument('--github-output', type=Path, help='Write the resolved artifact cache key for Actions')
|
||||
fetch = commands.add_parser('fetch', help='Download/verify the artifact described by saved resolution metadata')
|
||||
fetch.add_argument('resolution', type=Path)
|
||||
fetch.add_argument('destination', type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.command == 'resolve':
|
||||
resolution = resolve_latest()
|
||||
args.resolution.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.resolution.write_text(json.dumps(resolution, indent=2) + '\n', encoding='utf-8')
|
||||
if args.github_output:
|
||||
with args.github_output.open('a', encoding='utf-8') as output:
|
||||
output.write(f'cache_key={resolution["version"]}-{resolution["sha256"]}\n')
|
||||
print(json.dumps(resolution, indent=2))
|
||||
else:
|
||||
resolution = json.loads(args.resolution.read_text(encoding='utf-8'))
|
||||
package = fetch_package(args.destination.resolve(), resolution)
|
||||
print(f'{resolution["author"]}/{resolution["name"]}:{resolution["version"]} verified at {package}')
|
||||
print(f'Archive SHA256: {resolution["sha256"]}')
|
||||
except (OSError, ValueError, KeyError, RuntimeError, zipfile.BadZipFile) as exc:
|
||||
parser.exit(1, f'LocalAgent fixture provisioning failed: {exc}\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -21,6 +21,93 @@ LangBot uses a layered quality gate system for developers and CI:
|
||||
gates. They run in separate CI workflows. Frontend Playwright tests live under
|
||||
`web/tests/e2e` and are documented in `web/README.md`.
|
||||
|
||||
### Core handler contracts and the LocalAgent fixture
|
||||
|
||||
The handler unit tests execute Core's real `ChatMessageHandler` with SDK
|
||||
messages/queries and a fake orchestrator. They check plugin-altered input,
|
||||
stream output snapshots, completion, response IDs, and query isolation. They
|
||||
do not require LocalAgent to be installed and do not test its internal agent
|
||||
loop. Assertions on a hand-written UUID/list simulation are not a substitute
|
||||
for exercising the handler.
|
||||
|
||||
`test_local_runner_fake_provider.py` is a separate cross-repository contract
|
||||
test: real Core, real SDK Plugin Runtime, and the official LocalAgent plugin,
|
||||
with deterministic Host-side model/tool/RAG resources. It checks installation,
|
||||
Host action routing, ledger/transcript persistence, and selected plugin journeys
|
||||
without a paid model provider. LocalAgent's own unit tests still belong with the
|
||||
plugin. The Box/file-transfer probe additionally needs Docker.
|
||||
|
||||
CI's **Local Agent Contract E2E** job resolves the latest LocalAgent from the
|
||||
public Space version API on every run, matching Core's version selection.
|
||||
It saves `local-agent-resolution.json` with the resolved version, exact download
|
||||
URL, archive SHA256, and resolution time. Only the version/checksum-addressed
|
||||
archive is cached; latest metadata is always fetched anew. The script checks
|
||||
every cache hit and preserves the signed archive for normal Runtime admission.
|
||||
Space's `checksum` is the original archive SHA256; its normalized
|
||||
`artifact_digest` is different and must not be substituted. No Space token or
|
||||
model API key is needed.
|
||||
|
||||
Run the same path locally:
|
||||
|
||||
```bash
|
||||
export LANGBOT_E2E_LOCAL_AGENT_PACKAGE="$PWD/.pytest_cache/local-agent/LocalAgent.lbpkg"
|
||||
uv run python scripts/test-local-agent-fixture.py resolve .pytest_cache/local-agent-resolution.json
|
||||
uv run python scripts/test-local-agent-fixture.py fetch .pytest_cache/local-agent-resolution.json "$LANGBOT_E2E_LOCAL_AGENT_PACKAGE"
|
||||
LANGBOT_E2E_REQUIRE_LOCAL_AGENT=1 uv run pytest tests/e2e/test_local_runner_fake_provider.py -q -rs --durations=10
|
||||
```
|
||||
|
||||
A missing explicitly configured package always fails. Without the environment
|
||||
variables, local runs may package a sibling
|
||||
`langbot-plugin-demo/Runner/LocalAgent` checkout or skip if it is absent. CI
|
||||
requires the resolved package, checks Docker, and rejects any skipped tests; it
|
||||
never relies on that sibling checkout. A metadata/download/checksum failure
|
||||
appears in provisioning separately from a Core/Runtime test failure. Do not
|
||||
silently fall back to an older release when latest is unavailable or incompatible.
|
||||
|
||||
Because this tracks latest, an upstream plugin release can break an unchanged
|
||||
Core commit. Compare the saved resolution artifacts, and replay a failed run
|
||||
using its `local-agent-resolution.json` with the `fetch` command before assigning
|
||||
the regression to Core. Replay still validates the saved checksum and therefore
|
||||
fails clearly if Space replaces or removes that artifact. Plugin internals and
|
||||
new SDK compatibility requirements belong to their owning repositories.
|
||||
|
||||
The Python 3.12 unit/smoke matrix job also collects line and branch coverage and
|
||||
enforces the unchanged 18% **line** threshold. The pytest-cov total with
|
||||
`--cov-branch` combines lines and branches; it is not either individual metric.
|
||||
Python 3.11 and 3.13 retain their compatibility runs.
|
||||
The existing **Coverage Gate** check aggregates these results without rerunning
|
||||
the suite. The new LocalAgent job is mandatory within this workflow; making its
|
||||
check branch-protection-required remains a repository setting.
|
||||
|
||||
### Source complexity and coverage reports
|
||||
|
||||
The 3.12 job uploads `coverage.json`, `coverage.xml`, `complexity.json`, and
|
||||
`complexity.md`, and shows the summary in the Actions job summary. The complexity
|
||||
report uses the already-installed Ruff C901 McCabe rule with a threshold of zero
|
||||
to measure every function, without adding dependencies or a new complexity gate.
|
||||
It covers `src/langbot/**/*.py`, including vendored `libs`, migrations, and
|
||||
templates. It records the Ruff version so comparisons have an explicit analyzer.
|
||||
|
||||
```bash
|
||||
uv run pytest tests/unit_tests tests/smoke --cov=langbot --cov-branch --cov-report=json
|
||||
uv run python scripts/test-complexity.py --json complexity.json --summary complexity.md \
|
||||
--coverage-json coverage.json --base-ref origin/master
|
||||
```
|
||||
|
||||
The report includes function counts/distribution, hotspots, separate line/branch
|
||||
coverage, and function changes against the PR base (or previous push). It matches
|
||||
qualified names and Git-detected file renames; function renames appear as
|
||||
additions/deletions. Function-range coverage overlaps for nested functions and
|
||||
is a navigation aid, not a sum of independent coverage totals. Missing coverage
|
||||
is unknown, not zero.
|
||||
|
||||
Use these to identify changed, complex functions needing stronger tests. Coverage
|
||||
shows execution and complexity estimates branching, but neither proves assertions
|
||||
would detect defects. Keep real behavior assertions, targeted mutation probes,
|
||||
integration contracts, skip reasons, and timings alongside these reports. Do not
|
||||
combine the metrics into a single quality score or gate existing complexity debt
|
||||
with an arbitrary threshold.
|
||||
|
||||
### Developer Workflow
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""E2E coverage for the official Local Agent runner with fake Host resources.
|
||||
|
||||
These tests start the real LangBot application and the real SDK Plugin Runtime,
|
||||
load the consolidated ``langbot-plugin-demo/Runner/LocalAgent`` plugin, and verify Local Agent paths
|
||||
that must cross Host run-scoped APIs without calling any external provider.
|
||||
install the official LocalAgent package, and verify paths that cross Host
|
||||
run-scoped APIs without calling any external model provider. CI resolves the latest
|
||||
Space artifact on every run; a sibling source checkout remains optional for local development.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,10 +50,28 @@ def _local_agent_repo() -> Path:
|
||||
|
||||
|
||||
def _package_local_agent_plugin(tmpdir: Path) -> Path:
|
||||
"""Package the sibling Local Agent plugin for the real local-install flow."""
|
||||
"""Stage a Space artifact or package an optional local source checkout."""
|
||||
package_path = os.environ.get('LANGBOT_E2E_LOCAL_AGENT_PACKAGE')
|
||||
if package_path:
|
||||
package = Path(package_path).expanduser().resolve()
|
||||
if not package.is_file():
|
||||
pytest.fail(f'Explicit LocalAgent package does not exist: {package}')
|
||||
target = tmpdir / 'langbot-local-agent.zip'
|
||||
shutil.copyfile(package, target)
|
||||
return target
|
||||
|
||||
if os.environ.get('LANGBOT_E2E_REQUIRE_LOCAL_AGENT') == '1':
|
||||
pytest.fail(
|
||||
'LocalAgent E2E requires LANGBOT_E2E_LOCAL_AGENT_PACKAGE. '
|
||||
'Run scripts/test-local-agent-fixture.py to resolve and fetch the latest Space artifact.'
|
||||
)
|
||||
|
||||
local_agent_src = _local_agent_repo()
|
||||
if not (local_agent_src / 'manifest.yaml').exists():
|
||||
pytest.skip(f'local-agent repository not found at {local_agent_src}')
|
||||
pytest.skip(
|
||||
'Optional LocalAgent fixture unavailable; set LANGBOT_E2E_LOCAL_AGENT_PACKAGE '
|
||||
f'or check out source at {local_agent_src}'
|
||||
)
|
||||
|
||||
package_source = tmpdir / 'local-agent-package'
|
||||
ignore = shutil.ignore_patterns(
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Tests for ChatMessageHandler behavior with AgentRunOrchestrator.
|
||||
|
||||
Tests focus on:
|
||||
- Streaming mode behavior (single resp_message_id, pop/append pattern)
|
||||
- Streaming response text, finalization, card identity, and cross-query isolation
|
||||
- Non-streaming mode behavior (no pop)
|
||||
- Orchestrator invocation
|
||||
- Error handling for RunnerNotFoundError, RunnerExecutionError
|
||||
@@ -11,7 +11,6 @@ Avoids circular imports by using proper import structure.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
@@ -174,63 +173,28 @@ class MockApplication:
|
||||
self.sess_mgr.get_conversation = AsyncMock()
|
||||
|
||||
|
||||
class TestStreamingBehavior:
|
||||
"""Tests for streaming mode behavior."""
|
||||
def make_handler_query(text):
|
||||
"""Build SDK input entities; only the adapter and orchestration are fakes."""
|
||||
from tests.factories import text_query
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message
|
||||
from langbot_plugin.api.entities.builtin.provider.prompt import Prompt
|
||||
from langbot_plugin.api.entities.builtin.provider.session import Conversation, Session
|
||||
|
||||
def test_single_resp_message_id_for_streaming(self):
|
||||
"""Streaming mode should use single resp_message_id for entire response."""
|
||||
# Simulate the streaming logic: resp_message_id created outside loop
|
||||
resp_message_id = uuid.uuid4()
|
||||
|
||||
chunks = ['Hello', ' World', '!']
|
||||
resp_messages = []
|
||||
|
||||
for chunk in chunks:
|
||||
result = MockMessageChunk(chunk)
|
||||
result.resp_message_id = str(resp_message_id)
|
||||
|
||||
# Pop old chunk (streaming behavior)
|
||||
if resp_messages:
|
||||
resp_messages.pop()
|
||||
resp_messages.append(result)
|
||||
|
||||
# All chunks should have same resp_message_id
|
||||
assert len(resp_messages) == 1 # Only last chunk remains after pop/append
|
||||
assert resp_messages[0].resp_message_id == str(resp_message_id)
|
||||
|
||||
def test_pop_before_append_in_streaming(self):
|
||||
"""Streaming mode should pop old chunk before appending new."""
|
||||
resp_message_id = uuid.uuid4()
|
||||
resp_messages = []
|
||||
|
||||
# First chunk - no pop
|
||||
chunk1 = MockMessageChunk('Hello')
|
||||
chunk1.resp_message_id = str(resp_message_id)
|
||||
resp_messages.append(chunk1)
|
||||
assert len(resp_messages) == 1
|
||||
|
||||
# Second chunk - pop first, then append
|
||||
if resp_messages:
|
||||
resp_messages.pop()
|
||||
chunk2 = MockMessageChunk('Hello World')
|
||||
chunk2.resp_message_id = str(resp_message_id)
|
||||
resp_messages.append(chunk2)
|
||||
assert len(resp_messages) == 1
|
||||
assert resp_messages[0].content == 'Hello World'
|
||||
|
||||
def test_non_streaming_no_pop(self):
|
||||
"""Non-streaming mode should NOT pop previous responses."""
|
||||
resp_messages = []
|
||||
|
||||
# First message
|
||||
msg1 = MockMessageChunk('Response 1')
|
||||
resp_messages.append(msg1)
|
||||
assert len(resp_messages) == 1
|
||||
|
||||
# Second message - should NOT pop in non-streaming
|
||||
msg2 = MockMessageChunk('Response 2')
|
||||
resp_messages.append(msg2)
|
||||
assert len(resp_messages) == 2
|
||||
query = text_query(text)
|
||||
query.user_message = Message(role='user', content=text)
|
||||
query.adapter.is_stream_output_supported = AsyncMock(return_value=True)
|
||||
query.adapter.create_message_card = AsyncMock()
|
||||
query.session = Session(
|
||||
launcher_type=query.launcher_type,
|
||||
launcher_id=query.launcher_id,
|
||||
using_conversation=Conversation(
|
||||
prompt=Prompt(name='test', messages=[]),
|
||||
messages=[],
|
||||
pipeline_uuid=query.pipeline_uuid,
|
||||
bot_uuid=query.bot_uuid,
|
||||
),
|
||||
)
|
||||
return query
|
||||
|
||||
|
||||
class TestRunnerConfigResolverInChatHandler:
|
||||
@@ -333,139 +297,86 @@ class TestChatHandlerAsyncBehavior:
|
||||
"""Real async tests for ChatMessageHandler.handle() with mocked orchestrator."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_single_resp_message_id(self):
|
||||
"""Streaming mode: all chunks should have same resp_message_id."""
|
||||
async def test_interleaved_streams_keep_response_ids_and_messages_isolated(self):
|
||||
"""Two turns on one handler must never update each other's response card."""
|
||||
from langbot.pkg.pipeline.process.handlers.chat import ChatMessageHandler
|
||||
from langbot.pkg.pipeline import entities
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
|
||||
|
||||
# Create chunks for streaming
|
||||
chunks = [
|
||||
MockMessageChunk('Hello'),
|
||||
MockMessageChunk('Hello World'),
|
||||
MockMessageChunk('Hello World!'),
|
||||
class InterleavedOrchestrator(MockAgentRunOrchestrator):
|
||||
async def run_from_query(self, query):
|
||||
text = query.user_message.content
|
||||
yield MessageChunk(role='assistant', content=f'{text}: partial')
|
||||
yield Message(role='assistant', content=f'{text}: completed')
|
||||
|
||||
handler = ChatMessageHandler(MockApplication(orchestrator=InterleavedOrchestrator()))
|
||||
queries = [make_handler_query('first'), make_handler_query('second')]
|
||||
streams = [handler.handle(query) for query in queries]
|
||||
observed = [[], []]
|
||||
saved_messages = [None, None]
|
||||
|
||||
try:
|
||||
for index in [0, 1, 0, 1]:
|
||||
result = await anext(streams[index])
|
||||
assert result.result_type == entities.ResultType.CONTINUE
|
||||
assert result.new_query is queries[index]
|
||||
assert len(queries[index].resp_messages) == 1
|
||||
current = queries[index].resp_messages[0]
|
||||
assert isinstance(current, MessageChunk)
|
||||
observed[index].append((current.content, current.is_final, current.resp_message_id))
|
||||
saved_messages[index] = current.model_dump()
|
||||
|
||||
other = 1 - index
|
||||
if saved_messages[other] is not None:
|
||||
assert [message.model_dump() for message in queries[other].resp_messages] == [saved_messages[other]]
|
||||
|
||||
for stream in streams:
|
||||
with pytest.raises(StopAsyncIteration):
|
||||
await anext(stream)
|
||||
finally:
|
||||
for stream in streams:
|
||||
await stream.aclose()
|
||||
|
||||
first_id, second_id = (messages[0][2] for messages in observed)
|
||||
assert first_id and second_id and first_id != second_id
|
||||
assert observed == [
|
||||
[('first: partial', False, first_id), ('first: completed', True, first_id)],
|
||||
[('second: partial', False, second_id), ('second: completed', True, second_id)],
|
||||
]
|
||||
|
||||
orchestrator = MockAgentRunOrchestrator(chunks=chunks)
|
||||
mock_ap = MockApplication(orchestrator=orchestrator)
|
||||
|
||||
# Mock event context to not prevent default
|
||||
event_ctx = MockEventContext(prevented=False)
|
||||
mock_ap.plugin_connector.emit_event = AsyncMock(return_value=event_ctx)
|
||||
|
||||
query = MockQuery()
|
||||
query.adapter.is_stream = True # Enable streaming mode
|
||||
|
||||
handler = ChatMessageHandler(mock_ap)
|
||||
|
||||
# Mock event creation and StageProcessResult to bypass pydantic validation
|
||||
mock_event = MagicMock()
|
||||
mock_event.return_value = MagicMock()
|
||||
|
||||
def make_result(*args, **kwargs):
|
||||
return MagicMock(result_type=kwargs.get('result_type', entities.ResultType.CONTINUE))
|
||||
|
||||
with (
|
||||
patch('langbot.pkg.pipeline.process.handlers.chat.events') as mock_events_module,
|
||||
patch('langbot.pkg.pipeline.entities.StageProcessResult', side_effect=make_result),
|
||||
):
|
||||
mock_events_module.PersonNormalMessageReceived = mock_event
|
||||
mock_events_module.GroupNormalMessageReceived = mock_event
|
||||
|
||||
results = []
|
||||
async for result in handler.handle(query):
|
||||
results.append(result)
|
||||
|
||||
# Verify single resp_message_id
|
||||
resp_ids = [msg.resp_message_id for msg in query.resp_messages if hasattr(msg, 'resp_message_id')]
|
||||
assert len(set(resp_ids)) == 1 # All same ID
|
||||
|
||||
# Verify pop/append pattern: only last chunk remains
|
||||
assert len(query.resp_messages) == 1
|
||||
assert query.resp_messages[0].content == 'Hello World!'
|
||||
for query, response_id in zip(queries, [first_id, second_id]):
|
||||
query.adapter.create_message_card.assert_awaited_once_with(response_id, query.message_event)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_renders_accumulated_deltas_and_completes_same_message(self):
|
||||
"""Incremental runner deltas and message.completed update one response stream."""
|
||||
async def test_non_streaming_keeps_prior_and_runner_messages_and_yields_once(self):
|
||||
"""Non-streaming delivery preserves earlier messages and exposes only the completed turn."""
|
||||
from langbot.pkg.pipeline.process.handlers.chat import ChatMessageHandler
|
||||
from langbot.pkg.pipeline import entities
|
||||
from langbot_plugin.api.entities.builtin.provider import message as provider_message
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
|
||||
|
||||
prior = Message(role='assistant', content='earlier reply', resp_message_id='earlier-id')
|
||||
prior_snapshot = prior.model_dump()
|
||||
chunks = [
|
||||
provider_message.MessageChunk(role='assistant', content='Hel', all_content='Hel'),
|
||||
provider_message.MessageChunk(role='assistant', content='lo', all_content='Hello'),
|
||||
provider_message.Message(role='assistant', content='Hello'),
|
||||
MessageChunk(role='assistant', content='working'),
|
||||
Message(role='assistant', content='completed'),
|
||||
]
|
||||
orchestrator = MockAgentRunOrchestrator(chunks=chunks)
|
||||
mock_ap = MockApplication(orchestrator=orchestrator)
|
||||
mock_ap.plugin_connector.emit_event = AsyncMock(return_value=MockEventContext(prevented=False))
|
||||
query = MockQuery()
|
||||
query.adapter.is_stream = True
|
||||
handler = ChatMessageHandler(mock_ap)
|
||||
mock_event = MagicMock()
|
||||
mock_event.return_value = MagicMock()
|
||||
|
||||
def make_result(*args, **kwargs):
|
||||
return MagicMock(result_type=kwargs.get('result_type', entities.ResultType.CONTINUE))
|
||||
|
||||
handler = ChatMessageHandler(MockApplication(orchestrator=MockAgentRunOrchestrator(chunks=chunks)))
|
||||
query = make_handler_query('question')
|
||||
query.adapter.is_stream_output_supported.return_value = False
|
||||
query.resp_messages = [prior]
|
||||
observed = []
|
||||
with (
|
||||
patch('langbot.pkg.pipeline.process.handlers.chat.events') as mock_events_module,
|
||||
patch('langbot.pkg.pipeline.entities.StageProcessResult', side_effect=make_result),
|
||||
):
|
||||
mock_events_module.PersonNormalMessageReceived = mock_event
|
||||
mock_events_module.GroupNormalMessageReceived = mock_event
|
||||
|
||||
async for _ in handler.handle(query):
|
||||
current = query.resp_messages[-1]
|
||||
observed.append((current.content, current.is_final, current.resp_message_id))
|
||||
async for result in handler.handle(query):
|
||||
assert result.result_type == entities.ResultType.CONTINUE
|
||||
assert result.new_query is query
|
||||
observed.append([message.content for message in result.new_query.resp_messages])
|
||||
|
||||
assert [content for content, _, _ in observed] == ['Hel', 'Hello', 'Hello']
|
||||
assert [is_final for _, is_final, _ in observed] == [False, False, True]
|
||||
assert len({response_id for _, _, response_id in observed}) == 1
|
||||
assert all(isinstance(message, provider_message.MessageChunk) for message in query.resp_messages)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_streaming_keeps_results_and_yields_once(self):
|
||||
"""Non-streaming mode keeps runner results but runs downstream stages once."""
|
||||
from langbot.pkg.pipeline.process.handlers.chat import ChatMessageHandler
|
||||
from langbot.pkg.pipeline import entities
|
||||
|
||||
chunks = [
|
||||
MockMessageChunk('Response 1'),
|
||||
MockMessageChunk('Response 2'),
|
||||
]
|
||||
|
||||
orchestrator = MockAgentRunOrchestrator(chunks=chunks)
|
||||
mock_ap = MockApplication(orchestrator=orchestrator)
|
||||
mock_ap.plugin_connector.emit_event = AsyncMock(return_value=MockEventContext(prevented=False))
|
||||
|
||||
query = MockQuery()
|
||||
query.adapter.is_stream = False # Disable streaming mode
|
||||
|
||||
handler = ChatMessageHandler(mock_ap)
|
||||
|
||||
mock_event = MagicMock()
|
||||
mock_event.return_value = MagicMock()
|
||||
|
||||
def make_result(*args, **kwargs):
|
||||
return MagicMock(result_type=kwargs.get('result_type', entities.ResultType.CONTINUE))
|
||||
|
||||
with (
|
||||
patch('langbot.pkg.pipeline.process.handlers.chat.events') as mock_events_module,
|
||||
patch('langbot.pkg.pipeline.entities.StageProcessResult', side_effect=make_result),
|
||||
):
|
||||
mock_events_module.PersonNormalMessageReceived = mock_event
|
||||
mock_events_module.GroupNormalMessageReceived = mock_event
|
||||
|
||||
results = []
|
||||
async for result in handler.handle(query):
|
||||
results.append(result)
|
||||
|
||||
# No pop: all chunks should remain
|
||||
assert len(query.resp_messages) == 2
|
||||
assert query.resp_messages[0].content == 'Response 1'
|
||||
assert query.resp_messages[1].content == 'Response 2'
|
||||
assert len(results) == 1
|
||||
assert observed == [['earlier reply', 'working', 'completed']]
|
||||
assert query.resp_messages == [prior, *chunks]
|
||||
assert prior.model_dump() == prior_snapshot
|
||||
assert chunks[0].resp_message_id
|
||||
assert chunks[0].resp_message_id == chunks[1].resp_message_id
|
||||
assert chunks[0].resp_message_id != prior.resp_message_id
|
||||
query.adapter.create_message_card.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_turn_recreates_conversation_if_tool_resets_it(self):
|
||||
|
||||
@@ -89,6 +89,29 @@ def set_runner(fake_app):
|
||||
return _set_runner
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def handler_query():
|
||||
"""A prepared SDK query with history available for a successful handler turn."""
|
||||
from tests.factories import text_query
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message
|
||||
from langbot_plugin.api.entities.builtin.provider.prompt import Prompt
|
||||
from langbot_plugin.api.entities.builtin.provider.session import Conversation, Session
|
||||
|
||||
query = text_query('original')
|
||||
query.user_message = Message(role='user', content='original')
|
||||
query.session = Session(
|
||||
launcher_type=query.launcher_type,
|
||||
launcher_id=query.launcher_id,
|
||||
using_conversation=Conversation(
|
||||
prompt=Prompt(name='test', messages=[]),
|
||||
messages=[],
|
||||
pipeline_uuid=query.pipeline_uuid,
|
||||
bot_uuid=query.bot_uuid,
|
||||
),
|
||||
)
|
||||
return query
|
||||
|
||||
|
||||
# ============== CACHED LAZY IMPORTS ==============
|
||||
|
||||
_chat_handler_module = None
|
||||
@@ -186,77 +209,66 @@ class TestChatMessageHandlerReal:
|
||||
assert query.resp_messages[0] == reply_chain
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_message_alter_string(self, fake_app, mock_event_ctx, set_runner):
|
||||
"""user_message_alter as string updates query.user_message."""
|
||||
from tests.factories import text_query
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message
|
||||
@pytest.mark.parametrize('alteration_kind', ['string', 'element', 'list'])
|
||||
async def test_user_message_alter_reaches_orchestrator(
|
||||
self, fake_app, mock_event_ctx, handler_query, alteration_kind
|
||||
):
|
||||
"""The runner receives the exact plugin alteration before it starts."""
|
||||
from langbot_plugin.api.entities.builtin.provider.message import ContentElement, Message
|
||||
|
||||
chat = get_chat_handler()
|
||||
|
||||
mock_event_ctx.is_prevented_default.return_value = False
|
||||
mock_event_ctx.event.user_message_alter = 'altered text'
|
||||
expected = [ContentElement.from_text('altered text')]
|
||||
if alteration_kind == 'string':
|
||||
alteration = 'altered text'
|
||||
elif alteration_kind == 'element':
|
||||
alteration = expected[0]
|
||||
else:
|
||||
expected.append(ContentElement.from_image_url('https://example.com/input.png'))
|
||||
alteration = expected
|
||||
mock_event_ctx.event.user_message_alter = alteration
|
||||
fake_app.plugin_connector.emit_event = AsyncMock(return_value=mock_event_ctx)
|
||||
received = []
|
||||
|
||||
query = text_query('original')
|
||||
query.adapter = Mock()
|
||||
query.adapter.is_stream_output_supported = AsyncMock(return_value=False)
|
||||
query.user_message = Message(role='user', content=[])
|
||||
async def run_from_query(query):
|
||||
received.append(query.user_message.model_copy(deep=True))
|
||||
yield Message(role='assistant', content='ok')
|
||||
|
||||
class QuickRunner:
|
||||
name = 'local-agent'
|
||||
fake_app.agent_run_orchestrator.run_from_query = run_from_query
|
||||
handler = get_chat_handler().ChatMessageHandler(fake_app)
|
||||
|
||||
def __init__(self, app, config):
|
||||
self.app = app
|
||||
self.config = config
|
||||
results = [result async for result in handler.handle(handler_query)]
|
||||
|
||||
async def run(self, query):
|
||||
yield Message(role='assistant', content='ok')
|
||||
|
||||
set_runner(QuickRunner)
|
||||
|
||||
handler = chat.ChatMessageHandler(fake_app)
|
||||
|
||||
results = []
|
||||
async for result in handler.handle(query):
|
||||
results.append(result)
|
||||
|
||||
assert query.user_message.content is not None
|
||||
assert received == [Message(role='user', content=expected)]
|
||||
assert handler_query.user_message.content == expected
|
||||
assert len(results) == 1
|
||||
assert results[0].result_type == get_entities().ResultType.CONTINUE
|
||||
assert results[0].new_query is handler_query
|
||||
assert [message.content for message in handler_query.resp_messages] == ['ok']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adapter_without_stream_method_defaults_non_stream(self, fake_app, mock_event_ctx, set_runner):
|
||||
"""Adapter without is_stream_output_supported defaults to non-stream."""
|
||||
from tests.factories import text_query
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, ContentElement
|
||||
async def test_adapter_without_stream_method_defaults_non_stream(self, fake_app, mock_event_ctx, handler_query):
|
||||
"""Missing stream support still delivers the completed turn exactly once."""
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
|
||||
|
||||
chat = get_chat_handler()
|
||||
|
||||
mock_event_ctx.is_prevented_default.return_value = False
|
||||
mock_event_ctx.event.user_message_alter = None
|
||||
fake_app.plugin_connector.emit_event = AsyncMock(return_value=mock_event_ctx)
|
||||
handler_query.adapter = Mock(spec=[])
|
||||
partial = MessageChunk(role='assistant', content='working')
|
||||
completed = Message(role='assistant', content='response')
|
||||
|
||||
query = text_query('test')
|
||||
query.adapter = Mock(spec=[])
|
||||
query.user_message = Message(role='user', content=[ContentElement.from_text('test')])
|
||||
async def run_from_query(query):
|
||||
yield partial
|
||||
yield completed
|
||||
|
||||
class SingleRunner:
|
||||
name = 'local-agent'
|
||||
fake_app.agent_run_orchestrator.run_from_query = run_from_query
|
||||
handler = get_chat_handler().ChatMessageHandler(fake_app)
|
||||
observed = []
|
||||
|
||||
def __init__(self, app, config):
|
||||
self.app = app
|
||||
self.config = config
|
||||
async for result in handler.handle(handler_query):
|
||||
assert result.result_type == get_entities().ResultType.CONTINUE
|
||||
assert result.new_query is handler_query
|
||||
observed.append([message.content for message in result.new_query.resp_messages])
|
||||
|
||||
async def run(self, query):
|
||||
yield Message(role='assistant', content='response')
|
||||
|
||||
set_runner(SingleRunner)
|
||||
|
||||
handler = chat.ChatMessageHandler(fake_app)
|
||||
|
||||
results = []
|
||||
async for result in handler.handle(query):
|
||||
results.append(result)
|
||||
|
||||
assert len(results) >= 1
|
||||
assert observed == [['working', 'response']]
|
||||
assert handler_query.resp_messages == [partial, completed]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures('mock_circular_import_chain')
|
||||
@@ -264,42 +276,46 @@ class TestChatHandlerStreaming:
|
||||
"""Tests for streaming behavior."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_chunks_collected(self, fake_app, mock_event_ctx, set_runner):
|
||||
"""Streaming produces multiple results."""
|
||||
from tests.factories import text_query
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, ContentElement, MessageChunk
|
||||
@pytest.mark.parametrize('final_kind', ['message', 'chunk'])
|
||||
async def test_streaming_delivers_accumulated_text_and_one_final(
|
||||
self, fake_app, mock_event_ctx, handler_query, final_kind
|
||||
):
|
||||
"""Each yield updates the same card, preserving the full text and final flag."""
|
||||
from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk
|
||||
|
||||
chat = get_chat_handler()
|
||||
|
||||
mock_event_ctx.is_prevented_default.return_value = False
|
||||
fake_app.plugin_connector.emit_event = AsyncMock(return_value=mock_event_ctx)
|
||||
handler_query.adapter.is_stream_output_supported = AsyncMock(return_value=True)
|
||||
handler_query.adapter.create_message_card = AsyncMock()
|
||||
completed = (
|
||||
Message(role='assistant', content='Hello!')
|
||||
if final_kind == 'message'
|
||||
else MessageChunk(role='assistant', content='!', all_content='Hello!', is_final=True)
|
||||
)
|
||||
|
||||
query = text_query('stream test')
|
||||
query.adapter = Mock()
|
||||
query.adapter.is_stream_output_supported = AsyncMock(return_value=True)
|
||||
query.adapter.create_message_card = AsyncMock()
|
||||
query.user_message = Message(role='user', content=[ContentElement.from_text('test')])
|
||||
async def run_from_query(query):
|
||||
yield MessageChunk(role='assistant', content='Hel', all_content='Hel')
|
||||
yield MessageChunk(role='assistant', content='lo', all_content='Hello')
|
||||
yield completed
|
||||
|
||||
class StreamRunner:
|
||||
name = 'local-agent'
|
||||
fake_app.agent_run_orchestrator.run_from_query = run_from_query
|
||||
handler = get_chat_handler().ChatMessageHandler(fake_app)
|
||||
observed = []
|
||||
response_ids = []
|
||||
|
||||
def __init__(self, app, config):
|
||||
self.app = app
|
||||
self.config = config
|
||||
async for result in handler.handle(handler_query):
|
||||
assert result.result_type == get_entities().ResultType.CONTINUE
|
||||
assert result.new_query is handler_query
|
||||
assert len(result.new_query.resp_messages) == 1
|
||||
current = result.new_query.resp_messages[0]
|
||||
assert isinstance(current, MessageChunk)
|
||||
# Snapshot each yield: StageProcessResult points at the mutable query.
|
||||
observed.append((current.content, current.is_final))
|
||||
response_ids.append(current.resp_message_id)
|
||||
|
||||
async def run(self, query):
|
||||
yield MessageChunk(role='assistant', content='Hello', is_final=False)
|
||||
yield MessageChunk(role='assistant', content=' World', is_final=True)
|
||||
|
||||
set_runner(StreamRunner)
|
||||
|
||||
handler = chat.ChatMessageHandler(fake_app)
|
||||
|
||||
results = []
|
||||
async for result in handler.handle(query):
|
||||
results.append(result)
|
||||
|
||||
assert len(results) >= 1
|
||||
assert observed == [('Hel', False), ('Hello', False), ('Hello!', True)]
|
||||
assert response_ids[0]
|
||||
assert response_ids == [response_ids[0]] * 3
|
||||
handler_query.adapter.create_message_card.assert_awaited_once_with(response_ids[0], handler_query.message_event)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures('mock_circular_import_chain')
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Keep complexity reporting honest about metric values and baseline changes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reporter():
|
||||
path = Path(__file__).resolve().parents[2] / 'scripts/test-complexity.py'
|
||||
spec = importlib.util.spec_from_file_location('complexity_report', path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def source(root, text, name='sample.py'):
|
||||
path = root / 'src/langbot' / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text)
|
||||
return path
|
||||
|
||||
|
||||
def commit(root):
|
||||
def git(*args):
|
||||
return subprocess.run(['git', *args], cwd=root, check=True, capture_output=True).stdout.decode().strip()
|
||||
|
||||
git('init', '-q')
|
||||
git('add', 'src')
|
||||
git('-c', 'user.name=Test', '-c', '[email protected]', 'commit', '-qm', 'baseline')
|
||||
return git('rev-parse', 'HEAD')
|
||||
|
||||
|
||||
def test_real_ruff_counts_branches_nested_functions_and_ignores_noqa(reporter, tmp_path):
|
||||
source(
|
||||
tmp_path,
|
||||
"""\
|
||||
class Worker:
|
||||
def run(self, value): # noqa: C901
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
async def outer(value):
|
||||
def inner():
|
||||
return 1
|
||||
if value:
|
||||
return inner()
|
||||
return None
|
||||
""",
|
||||
)
|
||||
functions = {item['name']: item for item in reporter.measure(tmp_path)}
|
||||
assert functions['Worker.run']['complexity'] == 2
|
||||
assert functions['outer']['complexity'] == 3 # Ruff also includes the nested definition.
|
||||
assert functions['outer.inner']['complexity'] == 1
|
||||
assert functions['Worker.run']['path'] == 'src/langbot/sample.py'
|
||||
|
||||
|
||||
def test_all_source_includes_libs_and_overrides_repo_lint_exclusions(reporter, tmp_path):
|
||||
source(tmp_path, 'def vendored():\n pass\n', 'libs/helper.py')
|
||||
(tmp_path / 'pyproject.toml').write_text('[tool.ruff]\nexclude = ["src"]\n')
|
||||
(tmp_path / '.gitignore').write_text('src/\n')
|
||||
assert [item['name'] for item in reporter.measure(tmp_path)] == ['vendored']
|
||||
|
||||
|
||||
def test_invalid_source_is_an_analysis_failure(reporter, tmp_path):
|
||||
source(tmp_path, 'def broken(:\n')
|
||||
with pytest.raises(ValueError, match='Unexpected Ruff diagnostic'):
|
||||
reporter.measure(tmp_path)
|
||||
|
||||
|
||||
def test_duplicate_definitions_have_distinct_stable_occurrences(reporter, tmp_path):
|
||||
path = source(
|
||||
tmp_path,
|
||||
"""\
|
||||
class Worker:
|
||||
@property
|
||||
def value(self):
|
||||
return 1
|
||||
@value.setter
|
||||
def value(self, value):
|
||||
pass
|
||||
""",
|
||||
)
|
||||
definitions = list(reporter.index_functions(path).values())
|
||||
assert [item['name'] for item in definitions] == ['Worker.value', 'Worker.value']
|
||||
assert [item['occurrence'] for item in definitions] == [1, 2]
|
||||
|
||||
|
||||
def test_changed_function_deltas_additions_deletions_and_formatting(reporter, tmp_path):
|
||||
path = source(
|
||||
tmp_path,
|
||||
"""\
|
||||
def changed(value):
|
||||
return value
|
||||
|
||||
def removed():
|
||||
pass
|
||||
|
||||
def unchanged():
|
||||
return 1
|
||||
""",
|
||||
)
|
||||
before = reporter.measure(tmp_path)
|
||||
path.write_text("""\
|
||||
# Moving lines and adding comments does not change a function.
|
||||
def unchanged():
|
||||
return 1
|
||||
|
||||
def changed(value):
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
def added():
|
||||
pass
|
||||
""")
|
||||
changes = reporter.compare(before, reporter.measure(tmp_path), {})
|
||||
assert {item['status'] for item in changes} == {'modified', 'added', 'deleted'}
|
||||
modified = next(item for item in changes if item['status'] == 'modified')
|
||||
assert modified['after']['name'] == 'changed'
|
||||
assert modified['delta'] == 1
|
||||
assert all(item['delta'] is None for item in changes if item['status'] != 'modified')
|
||||
|
||||
|
||||
def test_function_renames_are_added_deleted_not_false_improvements(reporter, tmp_path):
|
||||
path = source(tmp_path, 'def old_name():\n pass\n')
|
||||
before = reporter.measure(tmp_path)
|
||||
path.write_text('def new_name():\n pass\n')
|
||||
changes = reporter.compare(before, reporter.measure(tmp_path), {})
|
||||
assert {item['status'] for item in changes} == {'added', 'deleted'}
|
||||
assert all(item['delta'] is None for item in changes)
|
||||
|
||||
|
||||
def test_git_baseline_and_file_rename_preserve_identity(reporter, tmp_path):
|
||||
path = source(tmp_path, 'def sample():\n return 1\n')
|
||||
revision = commit(tmp_path)
|
||||
path.rename(path.with_name('renamed.py'))
|
||||
# Stage the destination so git can detect its similarity to the deleted file.
|
||||
subprocess.run(['git', 'add', 'src'], cwd=tmp_path, check=True)
|
||||
resolved, before = reporter.baseline(tmp_path, revision)
|
||||
assert resolved == revision
|
||||
assert before[0]['path'] == 'src/langbot/sample.py'
|
||||
renames = reporter.file_renames(tmp_path, revision)
|
||||
assert renames == {'src/langbot/renamed.py': 'src/langbot/sample.py'}
|
||||
changes = reporter.compare(before, reporter.measure(tmp_path), renames)
|
||||
assert len(changes) == 1
|
||||
assert changes[0]['status'] == 'renamed_file'
|
||||
assert changes[0]['delta'] == 0
|
||||
|
||||
|
||||
def test_invalid_base_ref_fails_rather_than_using_head(reporter, tmp_path):
|
||||
source(tmp_path, 'def sample():\n pass\n')
|
||||
commit(tmp_path)
|
||||
with pytest.raises(ValueError):
|
||||
reporter.baseline(tmp_path, 'does-not-exist')
|
||||
|
||||
|
||||
def coverage_document(branch=True):
|
||||
return {
|
||||
'meta': {'branch_coverage': branch},
|
||||
'files': {
|
||||
'src/langbot/sample.py': {
|
||||
'executed_lines': [1, 2, 3],
|
||||
'missing_lines': [4],
|
||||
'executed_branches': [[2, 3]],
|
||||
'missing_branches': [[2, 4]],
|
||||
'summary': {
|
||||
'covered_lines': 3,
|
||||
'num_statements': 4,
|
||||
'covered_branches': 1,
|
||||
'num_branches': 2,
|
||||
'percent_covered': 66.666,
|
||||
},
|
||||
},
|
||||
'tests/unrelated.py': {
|
||||
'executed_lines': [1],
|
||||
'missing_lines': [],
|
||||
'summary': {'covered_lines': 1, 'num_statements': 1},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_line_and_branch_coverage_are_separate_not_combined(reporter, tmp_path):
|
||||
source(tmp_path, 'def sample(value):\n if value:\n return 1\n return 0\n')
|
||||
functions = reporter.measure(tmp_path)
|
||||
report = reporter.attach_coverage(functions, coverage_document(), tmp_path)
|
||||
assert report['line_percent'] == 75
|
||||
assert report['branch_percent'] == 50
|
||||
assert report['matched_source_files'] == 1
|
||||
assert functions[0]['coverage']['line_percent'] == 75
|
||||
assert functions[0]['coverage']['branch_percent'] == 50
|
||||
|
||||
|
||||
def test_unknown_coverage_is_not_reported_as_zero(reporter, tmp_path):
|
||||
source(tmp_path, 'def unknown():\n pass\n', 'unknown.py')
|
||||
functions = reporter.measure(tmp_path)
|
||||
report = reporter.attach_coverage(functions, coverage_document(branch=False), tmp_path)
|
||||
assert functions[0]['coverage'] is None
|
||||
assert report['functions_without_coverage'] == 1
|
||||
assert report['branch_percent'] is None
|
||||
|
||||
|
||||
def test_no_branch_function_has_no_artificial_branch_percent(reporter, tmp_path):
|
||||
source(tmp_path, '\n\n\n\ndef sample():\n return 1\n')
|
||||
functions = reporter.measure(tmp_path)
|
||||
reporter.attach_coverage(functions, coverage_document(), tmp_path)
|
||||
assert functions[0]['coverage']['branch_percent'] is None
|
||||
assert functions[0]['coverage']['line_percent'] is None
|
||||
|
||||
|
||||
def test_empty_scope_statistics_have_no_artificial_averages(reporter):
|
||||
stats = reporter.statistics([])
|
||||
assert stats['function_count'] == 0
|
||||
assert stats['maximum'] is None
|
||||
assert stats['mean'] is None
|
||||
assert sum(stats['distribution'].values()) == 0
|
||||
|
||||
|
||||
def test_cli_writes_reports_and_high_complexity_is_not_failure(reporter, tmp_path):
|
||||
# 60 independent decisions produce CC 61, intentionally above ordinary limits.
|
||||
source(tmp_path, 'def high(value):\n' + ' if value:\n value -= 1\n' * 60)
|
||||
json_path = tmp_path / 'reports/complexity.json'
|
||||
summary_path = tmp_path / 'reports/complexity.md'
|
||||
assert reporter.main(['--root', str(tmp_path), '--json', str(json_path), '--summary', str(summary_path)]) == 0
|
||||
document = json.loads(json_path.read_text())
|
||||
assert document['current']['maximum'] == 61
|
||||
assert document['current']['distribution']['51+'] == 1
|
||||
assert 'CC=61' in summary_path.read_text()
|
||||
assert 'report only; no complexity gate' in summary_path.read_text()
|
||||
|
||||
|
||||
def test_missing_source_fails_without_publishing_misleading_empty_report(reporter, tmp_path):
|
||||
destination = tmp_path / 'complexity.json'
|
||||
assert reporter.main(['--root', str(tmp_path), '--json', str(destination)]) == 1
|
||||
assert not destination.exists()
|
||||
|
||||
|
||||
def test_notebooks_and_stubs_are_outside_python_source_scope(reporter, tmp_path):
|
||||
source(tmp_path, 'def sample():\n pass\n')
|
||||
source(tmp_path, 'def annotation_only(): ...\n', 'types.pyi')
|
||||
source(tmp_path, 'invalid notebook content', 'notebook.ipynb')
|
||||
assert [item['name'] for item in reporter.measure(tmp_path)] == ['sample']
|
||||
|
||||
|
||||
def test_excluded_executed_lines_do_not_inflate_function_coverage(reporter, tmp_path):
|
||||
source(tmp_path, 'def sample(value):\n if value:\n return 1\n return 0\n')
|
||||
document = coverage_document()
|
||||
document['files']['src/langbot/sample.py']['excluded_lines'] = [3]
|
||||
functions = reporter.measure(tmp_path)
|
||||
reporter.attach_coverage(functions, document, tmp_path)
|
||||
assert functions[0]['coverage']['line_percent'] == 66.67
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Provisioning failures must not turn the required LocalAgent CI job green."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
from pathlib import Path
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
import zipfile
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.e2e import test_local_runner_fake_provider as local_agent_e2e
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provisioner():
|
||||
script = Path(__file__).resolve().parents[2] / 'scripts/test-local-agent-fixture.py'
|
||||
spec = importlib.util.spec_from_file_location('local_agent_fixture_provisioner', script)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def package_and_lock():
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, 'w') as archive:
|
||||
archive.writestr('manifest.yaml', 'metadata:\n author: langbot-team\n name: LocalAgent\n version: 0.2.1\n')
|
||||
archive.comment = b'preserve-certification-envelope'
|
||||
package = buffer.getvalue()
|
||||
return package, {
|
||||
'author': 'langbot-team',
|
||||
'name': 'LocalAgent',
|
||||
'version': '0.2.1',
|
||||
'url': 'https://space.langbot.app/test-fixture',
|
||||
'sha256': hashlib.sha256(package).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def test_verified_cached_package_needs_no_network(provisioner, package_and_lock, tmp_path, monkeypatch):
|
||||
package, lock = package_and_lock
|
||||
destination = tmp_path / 'LocalAgent.lbpkg'
|
||||
destination.write_bytes(package)
|
||||
download = Mock(side_effect=AssertionError('Verified cache should not access Space'))
|
||||
monkeypatch.setattr(provisioner, 'download_package', download)
|
||||
|
||||
assert provisioner.fetch_package(destination, lock) == destination
|
||||
assert destination.read_bytes() == package
|
||||
download.assert_not_called()
|
||||
|
||||
|
||||
def test_invalid_cache_is_replaced_without_repacking(provisioner, package_and_lock, tmp_path, monkeypatch):
|
||||
package, lock = package_and_lock
|
||||
destination = tmp_path / 'LocalAgent.lbpkg'
|
||||
destination.write_bytes(b'corrupt cache')
|
||||
monkeypatch.setattr(provisioner, 'download_package', lambda url: package)
|
||||
|
||||
provisioner.fetch_package(destination, lock)
|
||||
|
||||
assert destination.read_bytes() == package
|
||||
assert not destination.with_suffix('.tmp').exists()
|
||||
|
||||
|
||||
def test_download_checksum_mismatch_does_not_install(provisioner, package_and_lock, tmp_path, monkeypatch):
|
||||
_, lock = package_and_lock
|
||||
destination = tmp_path / 'LocalAgent.lbpkg'
|
||||
monkeypatch.setattr(provisioner, 'download_package', lambda url: b'wrong artifact')
|
||||
|
||||
with pytest.raises(ValueError, match='SHA256 mismatch'):
|
||||
provisioner.fetch_package(destination, lock)
|
||||
|
||||
assert not destination.exists()
|
||||
|
||||
|
||||
def test_manifest_identity_must_match_lock(provisioner, package_and_lock):
|
||||
package, lock = package_and_lock
|
||||
lock['version'] = 'different-version'
|
||||
|
||||
with pytest.raises(ValueError, match='manifest version'):
|
||||
provisioner.validate_package(package, lock)
|
||||
|
||||
|
||||
def test_space_unavailable_fails_after_bounded_retries(provisioner, package_and_lock, tmp_path, monkeypatch):
|
||||
_, lock = package_and_lock
|
||||
download = Mock(side_effect=httpx.ConnectError('unavailable'))
|
||||
monkeypatch.setattr(provisioner.httpx, 'stream', download)
|
||||
monkeypatch.setattr(provisioner.time, 'sleep', lambda seconds: None)
|
||||
|
||||
with pytest.raises(RuntimeError, match='Cannot fetch LocalAgent fixture from Space'):
|
||||
provisioner.fetch_package(tmp_path / 'LocalAgent.lbpkg', lock)
|
||||
|
||||
assert download.call_count == 3
|
||||
|
||||
|
||||
def test_explicit_package_preserves_signed_archive(package_and_lock, tmp_path, monkeypatch):
|
||||
package, _ = package_and_lock
|
||||
source = tmp_path / 'download.lbpkg'
|
||||
source.write_bytes(package)
|
||||
monkeypatch.setenv('LANGBOT_E2E_LOCAL_AGENT_PACKAGE', str(source))
|
||||
|
||||
staged = local_agent_e2e._package_local_agent_plugin(tmp_path)
|
||||
|
||||
assert staged.read_bytes() == package
|
||||
|
||||
|
||||
def test_required_job_cannot_fall_back_to_sibling_or_skip(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv('LANGBOT_E2E_LOCAL_AGENT_PACKAGE', raising=False)
|
||||
monkeypatch.setenv('LANGBOT_E2E_REQUIRE_LOCAL_AGENT', '1')
|
||||
|
||||
with pytest.raises(pytest.fail.Exception, match='requires LANGBOT_E2E_LOCAL_AGENT_PACKAGE'):
|
||||
local_agent_e2e._package_local_agent_plugin(tmp_path)
|
||||
|
||||
|
||||
def test_explicit_missing_package_fails_even_for_optional_run(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv('LANGBOT_E2E_LOCAL_AGENT_PACKAGE', str(tmp_path / 'missing.lbpkg'))
|
||||
monkeypatch.delenv('LANGBOT_E2E_REQUIRE_LOCAL_AGENT', raising=False)
|
||||
|
||||
with pytest.raises(pytest.fail.Exception, match='Explicit LocalAgent package does not exist'):
|
||||
local_agent_e2e._package_local_agent_plugin(tmp_path)
|
||||
|
||||
|
||||
def test_optional_local_run_can_skip_missing_fixture(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv('LANGBOT_E2E_LOCAL_AGENT_PACKAGE', raising=False)
|
||||
monkeypatch.delenv('LANGBOT_E2E_REQUIRE_LOCAL_AGENT', raising=False)
|
||||
monkeypatch.setattr(local_agent_e2e, '_local_agent_repo', lambda: tmp_path / 'missing-source')
|
||||
|
||||
with pytest.raises(pytest.skip.Exception, match='Optional LocalAgent fixture unavailable'):
|
||||
local_agent_e2e._package_local_agent_plugin(tmp_path)
|
||||
|
||||
|
||||
def test_resolution_uses_space_order_and_refreshes_every_time(provisioner, monkeypatch):
|
||||
versions = [
|
||||
{'version': '0.3.0', 'checksum': 'a' * 64, 'plugin_id': 'langbot-team/LocalAgent', 'status': 'live'},
|
||||
{'version': '0.2.1', 'checksum': 'b' * 64, 'plugin_id': 'langbot-team/LocalAgent', 'status': 'live'},
|
||||
]
|
||||
payload = {'code': 0, 'data': {'versions': versions}}
|
||||
download = Mock(return_value=json.dumps(payload).encode())
|
||||
monkeypatch.setattr(provisioner, 'download_package', download)
|
||||
|
||||
first = provisioner.resolve_latest()
|
||||
versions[0]['version'] = '0.4.0'
|
||||
versions[0]['checksum'] = 'c' * 64
|
||||
download.return_value = json.dumps(payload).encode()
|
||||
second = provisioner.resolve_latest()
|
||||
|
||||
assert first['version'] == '0.3.0'
|
||||
assert first['sha256'] == 'a' * 64
|
||||
assert second['version'] == '0.4.0'
|
||||
assert second['sha256'] == 'c' * 64
|
||||
assert second['url'].endswith('/langbot-team/LocalAgent/0.4.0')
|
||||
assert download.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'field,value',
|
||||
[
|
||||
('version', '../bad'),
|
||||
('checksum', ''),
|
||||
('plugin_id', 'other/LocalAgent'),
|
||||
('status', 'removed'),
|
||||
],
|
||||
)
|
||||
def test_resolution_rejects_unusable_latest_instead_of_using_old_release(provisioner, monkeypatch, field, value):
|
||||
latest = {'version': '0.3.0', 'checksum': 'a' * 64, 'plugin_id': 'langbot-team/LocalAgent', 'status': 'live'}
|
||||
latest[field] = value
|
||||
payload = json.dumps({'code': 0, 'data': {'versions': [latest]}}).encode()
|
||||
monkeypatch.setattr(provisioner, 'download_package', lambda *args, **kwargs: payload)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
provisioner.resolve_latest()
|
||||
|
||||
|
||||
def test_download_rejects_oversized_body(provisioner, monkeypatch):
|
||||
response = httpx.Response(200, content=b'oversized', request=httpx.Request('GET', 'https://space.langbot.app'))
|
||||
from contextlib import nullcontext
|
||||
|
||||
monkeypatch.setattr(provisioner.httpx, 'stream', lambda *args, **kwargs: nullcontext(response))
|
||||
|
||||
with pytest.raises(ValueError, match='exceeds'):
|
||||
provisioner.download_package('https://space.langbot.app', max_bytes=3)
|
||||
Reference in New Issue
Block a user