fix(cntfilter): allow legacy sensitive-word lists over 64 patterns (#2467)

* fix(cntfilter): allow legacy sensitive-word lists over 64 patterns

Legacy sensitive-words.json files shipped ~70 rules. After v4.10.7,
BanWordFilter treated the 64-pattern safe_regex cap as a hard failure
and blocked every message. Raise the cap only on the sensitive-word
path, keep the 50ms CPU budget, and truncate oversized lists with a
one-time warning.

Fixes #2443

* fix(cntfilter): reject oversized sensitive-word lists

---------

Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>

src/langbot/pkg/pipeline/cntfilter/filters/banwords.py

@@ -5,6 +5,11 @@ from .. import entities
 import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
 from ....utils.safe_regex import SafeRegexError, mask_patterns
 
+# Legacy sensitive-words.json files shipped ~70 rules, which exceeds the
+# default safe_regex per-call cap of 64 and used to fail-close every message.
+# Keep one 50ms CPU budget for the whole list; only raise the pattern cap.
+_MAX_SENSITIVE_WORD_PATTERNS = 256
+
 
 @filter_model.filter_class('ban-word-filter')
 class BanWordFilter(filter_model.ContentFilter):
@@ -14,12 +19,17 @@ class BanWordFilter(filter_model.ContentFilter):
         pass
 
     async def process(self, query: pipeline_query.Query, message: str) -> entities.FilterResult:
+        words = self.ap.sensitive_meta.data.get('words') or []
+        mask = self.ap.sensitive_meta.data['mask']
+        mask_word = self.ap.sensitive_meta.data['mask_word']
+
         try:
-            found, message = await mask_patterns(
+            found, current = await mask_patterns(
-                self.ap.sensitive_meta.data['words'],
+                words,
                 message,
-                mask=self.ap.sensitive_meta.data['mask'],
+                mask=mask,
-                mask_word=self.ap.sensitive_meta.data['mask_word'],
+                mask_word=mask_word,
+                max_pattern_count=_MAX_SENSITIVE_WORD_PATTERNS,
             )
         except SafeRegexError as exc:
             return entities.FilterResult(
@@ -31,7 +41,7 @@ class BanWordFilter(filter_model.ContentFilter):
 
         return entities.FilterResult(
             level=entities.ResultLevel.MASKED if found else entities.ResultLevel.PASS,
-            replacement=message,
+            replacement=current,
             user_notice='消息中存在不合适的内容, 请修改' if found else '',
             console_notice='',
         )

src/langbot/pkg/utils/safe_regex.py

@@ -27,10 +27,16 @@ class SafeRegexTimeoutError(SafeRegexError):
     """Raised when the regex engine exhausts the operation CPU budget."""
 
 
-def _validate_patterns(patterns: Sequence[str]) -> tuple[str, ...]:
+def _validate_patterns(
+    patterns: Sequence[str],
+    *,
+    max_pattern_count: int = MAX_PATTERN_COUNT,
+) -> tuple[str, ...]:
+    if max_pattern_count < 1:
+        raise ValueError('max_pattern_count must be positive')
+    if len(patterns) > max_pattern_count:
+        raise SafeRegexLimitError(f'At most {max_pattern_count} regex patterns are allowed')
     normalized = tuple(patterns)
-    if len(normalized) > MAX_PATTERN_COUNT:
-        raise SafeRegexLimitError(f'At most {MAX_PATTERN_COUNT} regex patterns are allowed')
     for pattern in normalized:
         if not isinstance(pattern, str):
             raise SafeRegexError('Regex patterns must be strings')
@@ -115,8 +121,9 @@ def _mask_patterns_sync(
     mask: str,
     mask_word: str,
     timeout_seconds: float,
+    max_pattern_count: int,
 ) -> tuple[bool, str]:
-    normalized_patterns = _validate_patterns(patterns)
+    normalized_patterns = _validate_patterns(patterns, max_pattern_count=max_pattern_count)
     _validate_input(value)
     if len(mask) > MAX_REPLACEMENT_CHARS or len(mask_word) > MAX_REPLACEMENT_CHARS:
         raise SafeRegexLimitError(f'Regex replacements may contain at most {MAX_REPLACEMENT_CHARS} characters')
@@ -162,6 +169,7 @@ async def mask_patterns(
     mask: str,
     mask_word: str,
     timeout_seconds: float = DEFAULT_OPERATION_TIMEOUT_SECONDS,
+    max_pattern_count: int = MAX_PATTERN_COUNT,
 ) -> tuple[bool, str]:
     """Apply untrusted masking patterns with bounded CPU and output growth."""
 
@@ -174,4 +182,5 @@ async def mask_patterns(
         mask=mask,
         mask_word=mask_word,
         timeout_seconds=timeout_seconds,
+        max_pattern_count=max_pattern_count,
     )

tests/unit_tests/pipeline/test_banwords.py

@@ -0,0 +1,113 @@
+"""BanWordFilter regression tests for legacy sensitive-word lists.
+
+v4.10.7 introduced a 64-pattern cap in safe_regex. Older installs still carry
+the previous default list (~70 patterns). The filter must keep applying those
+rules instead of blocking every message.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+from unittest.mock import Mock
+
+import pytest
+
+from tests.factories import FakeApp
+
+
+def _load_banwords():
+    import_module('langbot.pkg.pipeline.pipelinemgr')
+    banwords = import_module('langbot.pkg.pipeline.cntfilter.filters.banwords')
+    entities = import_module('langbot.pkg.pipeline.cntfilter.entities')
+    safe_regex = import_module('langbot.pkg.utils.safe_regex')
+    return banwords, entities, safe_regex
+
+
+def _filter_with_words(words: list[str], *, mask: str = '*', mask_word: str = ''):
+    banwords, entities, _ = _load_banwords()
+    app = FakeApp()
+    app.sensitive_meta = Mock()
+    app.sensitive_meta.data = {
+        'words': words,
+        'mask': mask,
+        'mask_word': mask_word,
+    }
+    return banwords.BanWordFilter(app), entities, app
+
+
+@pytest.mark.asyncio
+async def test_legacy_word_list_over_pattern_cap_does_not_block_clean_message():
+    """A pre-v4.10.7 word list must not fail closed on every message."""
+    _, _, safe_regex = _load_banwords()
+    words = [f'word{i}' for i in range(safe_regex.MAX_PATTERN_COUNT + 6)]
+    filt, entities, _ = _filter_with_words(words)
+
+    result = await filt.process(Mock(), 'hello there, nothing banned')
+
+    assert result.level == entities.ResultLevel.PASS
+    assert result.replacement == 'hello there, nothing banned'
+    assert result.user_notice == ''
+
+
+@pytest.mark.asyncio
+async def test_legacy_word_list_still_masks_match_beyond_first_batch():
+    """Words past the first 64-pattern batch must still be applied."""
+    _, _, safe_regex = _load_banwords()
+    words = [f'word{i}' for i in range(safe_regex.MAX_PATTERN_COUNT)] + ['secret-token']
+    filt, entities, _ = _filter_with_words(words, mask_word='[hidden]')
+
+    result = await filt.process(Mock(), 'please hide secret-token now')
+
+    assert result.level == entities.ResultLevel.MASKED
+    assert 'secret-token' not in result.replacement
+    assert '[hidden]' in result.replacement
+
+
+@pytest.mark.asyncio
+async def test_legacy_word_list_masks_match_in_first_batch():
+    _, _, safe_regex = _load_banwords()
+    words = ['alpha-secret'] + [f'word{i}' for i in range(safe_regex.MAX_PATTERN_COUNT)]
+    filt, entities, _ = _filter_with_words(words, mask_word='[hidden]')
+
+    result = await filt.process(Mock(), 'alpha-secret is here')
+
+    assert result.level == entities.ResultLevel.MASKED
+    assert result.replacement == '[hidden] is here'
+
+
+@pytest.mark.asyncio
+async def test_invalid_sensitive_word_regex_still_blocks():
+    filt, entities, _ = _filter_with_words(['(unclosed'])
+
+    result = await filt.process(Mock(), 'any message')
+
+    assert result.level == entities.ResultLevel.BLOCK
+    assert result.user_notice == '内容检查规则执行失败，请联系管理员'
+    assert 'rejected' in result.console_notice.lower() or 'invalid' in result.console_notice.lower()
+
+
+@pytest.mark.asyncio
+async def test_oversized_word_list_is_blocked():
+    """Configured rules must never be silently skipped when the list is oversized."""
+    banwords, _, _ = _load_banwords()
+    words = [f'word{i}' for i in range(banwords._MAX_SENSITIVE_WORD_PATTERNS + 10)]
+    filt, entities, _ = _filter_with_words(words)
+
+    result = await filt.process(Mock(), 'hello there, nothing banned')
+
+    assert result.level == entities.ResultLevel.BLOCK
+    assert result.replacement == ''
+    assert result.user_notice == '内容检查规则执行失败，请联系管理员'
+    assert 'at most 256 regex patterns are allowed' in result.console_notice.lower()
+
+
+@pytest.mark.asyncio
+async def test_match_beyond_total_cap_cannot_bypass_filter():
+    banwords, _, _ = _load_banwords()
+    words = [f'word{i}' for i in range(banwords._MAX_SENSITIVE_WORD_PATTERNS)] + ['late-secret']
+    filt, entities, _ = _filter_with_words(words, mask_word='[hidden]')
+
+    result = await filt.process(Mock(), 'please hide late-secret now')
+
+    assert result.level == entities.ResultLevel.BLOCK
+    assert result.replacement == ''

tests/unit_tests/utils/test_safe_regex.py

@@ -53,6 +53,45 @@ async def test_matches_any_rejects_pattern_and_input_amplification():
         )
 
 
+@pytest.mark.asyncio
+async def test_mask_patterns_honors_explicit_pattern_count_cap():
+    patterns = ['a'] * (safe_regex.MAX_PATTERN_COUNT + 6)
+    found, masked = await safe_regex.mask_patterns(
+        patterns,
+        'hello',
+        mask='*',
+        mask_word='',
+        max_pattern_count=len(patterns),
+    )
+    assert found is False
+    assert masked == 'hello'
+
+    with pytest.raises(safe_regex.SafeRegexLimitError):
+        await safe_regex.mask_patterns(
+            patterns,
+            'hello',
+            mask='*',
+            mask_word='',
+        )
+
+
+@pytest.mark.asyncio
+async def test_mask_patterns_rejects_oversized_sequence_before_copying_it():
+    class OversizedPatterns(list):
+        def __iter__(self):
+            raise AssertionError('oversized patterns must not be materialized')
+
+    patterns = OversizedPatterns(['a'] * (safe_regex.MAX_PATTERN_COUNT + 1))
+
+    with pytest.raises(safe_regex.SafeRegexLimitError):
+        await safe_regex.mask_patterns(
+            patterns,
+            'hello',
+            mask='*',
+            mask_word='',
+        )
+
+
 @pytest.mark.asyncio
 async def test_mask_patterns_bounds_replacement_growth_and_masks_matches():
     found, masked = await safe_regex.mask_patterns(