From 08307790e55a0d82db073025fd39ca3ee0a8f04c Mon Sep 17 00:00:00 2001 From: ciri667 <87346762+ciri667@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:53:41 +0800 Subject: [PATCH] fix(cntfilter): allow legacy sensitive-word lists over 64 patterns (#2467) * fix(cntfilter): allow legacy sensitive-word lists over 64 patterns Legacy sensitive-words.json files shipped ~70 rules. After v4.10.7, BanWordFilter treated the 64-pattern safe_regex cap as a hard failure and blocked every message. Raise the cap only on the sensitive-word path, keep the 50ms CPU budget, and truncate oversized lists with a one-time warning. Fixes #2443 * fix(cntfilter): reject oversized sensitive-word lists --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com> --- .../pipeline/cntfilter/filters/banwords.py | 20 +++- src/langbot/pkg/utils/safe_regex.py | 17 ++- tests/unit_tests/pipeline/test_banwords.py | 113 ++++++++++++++++++ tests/unit_tests/utils/test_safe_regex.py | 39 ++++++ 4 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 tests/unit_tests/pipeline/test_banwords.py diff --git a/src/langbot/pkg/pipeline/cntfilter/filters/banwords.py b/src/langbot/pkg/pipeline/cntfilter/filters/banwords.py index 39796b33c..bfd8ca772 100644 --- a/src/langbot/pkg/pipeline/cntfilter/filters/banwords.py +++ b/src/langbot/pkg/pipeline/cntfilter/filters/banwords.py @@ -5,6 +5,11 @@ from .. import entities import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query from ....utils.safe_regex import SafeRegexError, mask_patterns +# Legacy sensitive-words.json files shipped ~70 rules, which exceeds the +# default safe_regex per-call cap of 64 and used to fail-close every message. +# Keep one 50ms CPU budget for the whole list; only raise the pattern cap. +_MAX_SENSITIVE_WORD_PATTERNS = 256 + @filter_model.filter_class('ban-word-filter') class BanWordFilter(filter_model.ContentFilter): @@ -14,12 +19,17 @@ class BanWordFilter(filter_model.ContentFilter): pass async def process(self, query: pipeline_query.Query, message: str) -> entities.FilterResult: + words = self.ap.sensitive_meta.data.get('words') or [] + mask = self.ap.sensitive_meta.data['mask'] + mask_word = self.ap.sensitive_meta.data['mask_word'] + try: - found, message = await mask_patterns( - self.ap.sensitive_meta.data['words'], + found, current = await mask_patterns( + words, message, - mask=self.ap.sensitive_meta.data['mask'], - mask_word=self.ap.sensitive_meta.data['mask_word'], + mask=mask, + mask_word=mask_word, + max_pattern_count=_MAX_SENSITIVE_WORD_PATTERNS, ) except SafeRegexError as exc: return entities.FilterResult( @@ -31,7 +41,7 @@ class BanWordFilter(filter_model.ContentFilter): return entities.FilterResult( level=entities.ResultLevel.MASKED if found else entities.ResultLevel.PASS, - replacement=message, + replacement=current, user_notice='消息中存在不合适的内容, 请修改' if found else '', console_notice='', ) diff --git a/src/langbot/pkg/utils/safe_regex.py b/src/langbot/pkg/utils/safe_regex.py index 32ae27843..e757cba7d 100644 --- a/src/langbot/pkg/utils/safe_regex.py +++ b/src/langbot/pkg/utils/safe_regex.py @@ -27,10 +27,16 @@ class SafeRegexTimeoutError(SafeRegexError): """Raised when the regex engine exhausts the operation CPU budget.""" -def _validate_patterns(patterns: Sequence[str]) -> tuple[str, ...]: +def _validate_patterns( + patterns: Sequence[str], + *, + max_pattern_count: int = MAX_PATTERN_COUNT, +) -> tuple[str, ...]: + if max_pattern_count < 1: + raise ValueError('max_pattern_count must be positive') + if len(patterns) > max_pattern_count: + raise SafeRegexLimitError(f'At most {max_pattern_count} regex patterns are allowed') normalized = tuple(patterns) - if len(normalized) > MAX_PATTERN_COUNT: - raise SafeRegexLimitError(f'At most {MAX_PATTERN_COUNT} regex patterns are allowed') for pattern in normalized: if not isinstance(pattern, str): raise SafeRegexError('Regex patterns must be strings') @@ -115,8 +121,9 @@ def _mask_patterns_sync( mask: str, mask_word: str, timeout_seconds: float, + max_pattern_count: int, ) -> tuple[bool, str]: - normalized_patterns = _validate_patterns(patterns) + normalized_patterns = _validate_patterns(patterns, max_pattern_count=max_pattern_count) _validate_input(value) if len(mask) > MAX_REPLACEMENT_CHARS or len(mask_word) > MAX_REPLACEMENT_CHARS: raise SafeRegexLimitError(f'Regex replacements may contain at most {MAX_REPLACEMENT_CHARS} characters') @@ -162,6 +169,7 @@ async def mask_patterns( mask: str, mask_word: str, timeout_seconds: float = DEFAULT_OPERATION_TIMEOUT_SECONDS, + max_pattern_count: int = MAX_PATTERN_COUNT, ) -> tuple[bool, str]: """Apply untrusted masking patterns with bounded CPU and output growth.""" @@ -174,4 +182,5 @@ async def mask_patterns( mask=mask, mask_word=mask_word, timeout_seconds=timeout_seconds, + max_pattern_count=max_pattern_count, ) diff --git a/tests/unit_tests/pipeline/test_banwords.py b/tests/unit_tests/pipeline/test_banwords.py new file mode 100644 index 000000000..cf5ca7552 --- /dev/null +++ b/tests/unit_tests/pipeline/test_banwords.py @@ -0,0 +1,113 @@ +"""BanWordFilter regression tests for legacy sensitive-word lists. + +v4.10.7 introduced a 64-pattern cap in safe_regex. Older installs still carry +the previous default list (~70 patterns). The filter must keep applying those +rules instead of blocking every message. +""" + +from __future__ import annotations + +from importlib import import_module +from unittest.mock import Mock + +import pytest + +from tests.factories import FakeApp + + +def _load_banwords(): + import_module('langbot.pkg.pipeline.pipelinemgr') + banwords = import_module('langbot.pkg.pipeline.cntfilter.filters.banwords') + entities = import_module('langbot.pkg.pipeline.cntfilter.entities') + safe_regex = import_module('langbot.pkg.utils.safe_regex') + return banwords, entities, safe_regex + + +def _filter_with_words(words: list[str], *, mask: str = '*', mask_word: str = ''): + banwords, entities, _ = _load_banwords() + app = FakeApp() + app.sensitive_meta = Mock() + app.sensitive_meta.data = { + 'words': words, + 'mask': mask, + 'mask_word': mask_word, + } + return banwords.BanWordFilter(app), entities, app + + +@pytest.mark.asyncio +async def test_legacy_word_list_over_pattern_cap_does_not_block_clean_message(): + """A pre-v4.10.7 word list must not fail closed on every message.""" + _, _, safe_regex = _load_banwords() + words = [f'word{i}' for i in range(safe_regex.MAX_PATTERN_COUNT + 6)] + filt, entities, _ = _filter_with_words(words) + + result = await filt.process(Mock(), 'hello there, nothing banned') + + assert result.level == entities.ResultLevel.PASS + assert result.replacement == 'hello there, nothing banned' + assert result.user_notice == '' + + +@pytest.mark.asyncio +async def test_legacy_word_list_still_masks_match_beyond_first_batch(): + """Words past the first 64-pattern batch must still be applied.""" + _, _, safe_regex = _load_banwords() + words = [f'word{i}' for i in range(safe_regex.MAX_PATTERN_COUNT)] + ['secret-token'] + filt, entities, _ = _filter_with_words(words, mask_word='[hidden]') + + result = await filt.process(Mock(), 'please hide secret-token now') + + assert result.level == entities.ResultLevel.MASKED + assert 'secret-token' not in result.replacement + assert '[hidden]' in result.replacement + + +@pytest.mark.asyncio +async def test_legacy_word_list_masks_match_in_first_batch(): + _, _, safe_regex = _load_banwords() + words = ['alpha-secret'] + [f'word{i}' for i in range(safe_regex.MAX_PATTERN_COUNT)] + filt, entities, _ = _filter_with_words(words, mask_word='[hidden]') + + result = await filt.process(Mock(), 'alpha-secret is here') + + assert result.level == entities.ResultLevel.MASKED + assert result.replacement == '[hidden] is here' + + +@pytest.mark.asyncio +async def test_invalid_sensitive_word_regex_still_blocks(): + filt, entities, _ = _filter_with_words(['(unclosed']) + + result = await filt.process(Mock(), 'any message') + + assert result.level == entities.ResultLevel.BLOCK + assert result.user_notice == '内容检查规则执行失败，请联系管理员' + assert 'rejected' in result.console_notice.lower() or 'invalid' in result.console_notice.lower() + + +@pytest.mark.asyncio +async def test_oversized_word_list_is_blocked(): + """Configured rules must never be silently skipped when the list is oversized.""" + banwords, _, _ = _load_banwords() + words = [f'word{i}' for i in range(banwords._MAX_SENSITIVE_WORD_PATTERNS + 10)] + filt, entities, _ = _filter_with_words(words) + + result = await filt.process(Mock(), 'hello there, nothing banned') + + assert result.level == entities.ResultLevel.BLOCK + assert result.replacement == '' + assert result.user_notice == '内容检查规则执行失败，请联系管理员' + assert 'at most 256 regex patterns are allowed' in result.console_notice.lower() + + +@pytest.mark.asyncio +async def test_match_beyond_total_cap_cannot_bypass_filter(): + banwords, _, _ = _load_banwords() + words = [f'word{i}' for i in range(banwords._MAX_SENSITIVE_WORD_PATTERNS)] + ['late-secret'] + filt, entities, _ = _filter_with_words(words, mask_word='[hidden]') + + result = await filt.process(Mock(), 'please hide late-secret now') + + assert result.level == entities.ResultLevel.BLOCK + assert result.replacement == '' diff --git a/tests/unit_tests/utils/test_safe_regex.py b/tests/unit_tests/utils/test_safe_regex.py index 6f0f985f5..80a024541 100644 --- a/tests/unit_tests/utils/test_safe_regex.py +++ b/tests/unit_tests/utils/test_safe_regex.py @@ -53,6 +53,45 @@ async def test_matches_any_rejects_pattern_and_input_amplification(): ) +@pytest.mark.asyncio +async def test_mask_patterns_honors_explicit_pattern_count_cap(): + patterns = ['a'] * (safe_regex.MAX_PATTERN_COUNT + 6) + found, masked = await safe_regex.mask_patterns( + patterns, + 'hello', + mask='*', + mask_word='', + max_pattern_count=len(patterns), + ) + assert found is False + assert masked == 'hello' + + with pytest.raises(safe_regex.SafeRegexLimitError): + await safe_regex.mask_patterns( + patterns, + 'hello', + mask='*', + mask_word='', + ) + + +@pytest.mark.asyncio +async def test_mask_patterns_rejects_oversized_sequence_before_copying_it(): + class OversizedPatterns(list): + def __iter__(self): + raise AssertionError('oversized patterns must not be materialized') + + patterns = OversizedPatterns(['a'] * (safe_regex.MAX_PATTERN_COUNT + 1)) + + with pytest.raises(safe_regex.SafeRegexLimitError): + await safe_regex.mask_patterns( + patterns, + 'hello', + mask='*', + mask_word='', + ) + + @pytest.mark.asyncio async def test_mask_patterns_bounds_replacement_growth_and_masks_matches(): found, masked = await safe_regex.mask_patterns(