From 38ff4766efedadb177d0ef34c0ec7f5d229e5971 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 08:22:11 +0800 Subject: [PATCH 1/5] chore: update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 459a22c82..83b4faf7f 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,5 @@ packaging/fnos/ICON_256.PNG packaging/fnos/app/ui/images/ packaging/fnos/app/desktop/images/ packaging/fnos/*.fpk + +r.ps1 From 1143d6a5ae1893f5eb29b9d52b6b4d69866300a8 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 17:58:23 +0800 Subject: [PATCH 2/5] feat(storage): media content-addressable cache and monitoring base64 externalization - Add MediaCache using xxHash3-128 (with sha256 fallback) content-addressable storage - Externalize message chain image payloads before recording monitoring and discarded messages - Strip base64 payloads to null in SQLite monitoring_messages, dropping row size from megabytes to hundreds of bytes - Add GET /api/v1/files/media/ route with immutable HTTP cache headers to serve cached media - Integrate age-based retention (default 30 days) and configurable disk quota with MaintenanceService cleanup loop - Add defensive sanitizer in MonitoringService.record_message against oversized raw base64 payloads - Add comprehensive unit tests and end-to-end verification covering CAS deduplication, route serving, and LRU pruning --- pyproject.toml | 1 + .../pkg/api/http/controller/groups/files.py | 15 ++ .../pkg/api/http/service/maintenance.py | 31 +++ .../pkg/api/http/service/monitoring.py | 28 +++ src/langbot/pkg/pipeline/monitoring_helper.py | 10 +- src/langbot/pkg/platform/botmgr.py | 5 +- src/langbot/pkg/storage/media.py | 235 ++++++++++++++++++ src/langbot/pkg/storage/mgr.py | 3 + src/langbot/templates/config.yaml | 6 + tests/unit_tests/storage/test_media_cache.py | 215 ++++++++++++++++ uv.lock | 120 ++++----- 11 files changed, 607 insertions(+), 62 deletions(-) create mode 100644 src/langbot/pkg/storage/media.py create mode 100644 tests/unit_tests/storage/test_media_cache.py diff --git a/pyproject.toml b/pyproject.toml index 1dc5e3467..b8880a6b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "python-docx>=1.1.0", "pandas>=2.2.2", "chardet>=5.2.0", + "xxhash>=3.5.0", "markdown>=3.6", "beautifulsoup4>=4.12.3", "ebooklib>=0.18", diff --git a/src/langbot/pkg/api/http/controller/groups/files.py b/src/langbot/pkg/api/http/controller/groups/files.py index 026f6c194..e38c181ae 100644 --- a/src/langbot/pkg/api/http/controller/groups/files.py +++ b/src/langbot/pkg/api/http/controller/groups/files.py @@ -47,6 +47,21 @@ class FilesRouterGroup(group.RouterGroup): return quart.Response(image_bytes, mimetype=mime_type) + @self.route( + '/media/', + methods=['GET'], + auth_type=group.AuthType.NONE, + ) + async def get_media_file(filename: str) -> quart.Response: + media = await self.ap.storage_mgr.media_cache.get_media(filename) + if media is None: + return quart.Response('Media not found or expired', status=404) + media_bytes, mime_type = media + headers = { + 'Cache-Control': 'public, max-age=2592000, immutable', + } + return quart.Response(media_bytes, mimetype=mime_type, headers=headers) + @self.route( '/images', methods=['POST'], diff --git a/src/langbot/pkg/api/http/service/maintenance.py b/src/langbot/pkg/api/http/service/maintenance.py index 0c61618c6..562c17d61 100644 --- a/src/langbot/pkg/api/http/service/maintenance.py +++ b/src/langbot/pkg/api/http/service/maintenance.py @@ -80,6 +80,25 @@ class MaintenanceService: DEFAULT_LOG_RETENTION_DAYS, 'storage.cleanup.log_retention_days', ) + media_cfg = self.ap.instance_config.data.get('storage', {}).get('media_cache', {}) + media_retention_days = self._positive_int( + media_cfg.get('retention_days'), + 30, + 'storage.media_cache.retention_days', + ) + media_max_size_mb = self._non_negative_int( + media_cfg.get('max_size_mb'), + 0, + 'storage.media_cache.max_size_mb', + ) + media_cleanup = ( + await self.ap.storage_mgr.media_cache.cleanup( + media_retention_days, + media_max_size_mb, + ) + if hasattr(self.ap.storage_mgr, 'media_cache') and await self._is_oss_singleton(context) + else {} + ) return { 'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days), @@ -89,6 +108,7 @@ class MaintenanceService: ) if await self._is_oss_singleton(context) else 0, + 'media_files': media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0), } async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]: @@ -466,6 +486,17 @@ class MaintenanceService: count += len(files) return count + def _non_negative_int(self, value: Any, default: int, name: str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}') + return default + if parsed < 0: + self.ap.logger.warning(f'{name} must be non-negative: {value!r}, using {default}') + return default + return parsed + def _positive_int(self, value: Any, default: int, name: str) -> int: try: parsed = int(value) diff --git a/src/langbot/pkg/api/http/service/monitoring.py b/src/langbot/pkg/api/http/service/monitoring.py index c90cec6d1..0bae983a6 100644 --- a/src/langbot/pkg/api/http/service/monitoring.py +++ b/src/langbot/pkg/api/http/service/monitoring.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import uuid import datetime import functools @@ -417,6 +418,32 @@ class MonitoringService: # ========== Recording Methods ========== + def _sanitize_message_content(self, content: str) -> str: + """Strip raw base64 data to protect database storage from unbounded bloating.""" + if not content or len(content) < 10000 or (';base64,' not in content and 'data:image/' not in content): + return content + try: + data = json.loads(content) + + def _strip_node(node): + if isinstance(node, list): + return [_strip_node(x) for x in node] + if isinstance(node, dict): + res = dict(node) + if res.get('type') == 'Image' and res.get('base64'): + res['base64'] = None + for k, v in list(res.items()): + if isinstance(v, (list, dict)): + res[k] = _strip_node(v) + return res + return node + + return json.dumps(_strip_node(data), ensure_ascii=False) + except Exception: + return re.sub( + r'data:image/[a-zA-Z0-9.+_-]+;base64,[\sA-Za-z0-9+/=]{1000,}', '[base64 image omitted]', content + ) + @_workspace_transaction async def record_message( self, @@ -439,6 +466,7 @@ class MonitoringService: """Record a message""" workspace_uuid = self._require_write_context(context) message_id = str(uuid.uuid4()) + message_content = self._sanitize_message_content(message_content) message_data = { 'id': message_id, 'workspace_uuid': workspace_uuid, diff --git a/src/langbot/pkg/pipeline/monitoring_helper.py b/src/langbot/pkg/pipeline/monitoring_helper.py index 1bab4bda4..15ae38f47 100644 --- a/src/langbot/pkg/pipeline/monitoring_helper.py +++ b/src/langbot/pkg/pipeline/monitoring_helper.py @@ -48,7 +48,10 @@ class MonitoringHelper: # Try to record message # Use JSON serialization to preserve message chain structure (including image URLs, etc.) if hasattr(query, 'message_chain') and hasattr(query.message_chain, 'model_dump'): - message_content = json.dumps(query.message_chain.model_dump(), ensure_ascii=False) + chain_dump = query.message_chain.model_dump() + if hasattr(ap, 'storage_mgr') and hasattr(ap.storage_mgr, 'media_cache'): + chain_dump = await ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump) + message_content = json.dumps(chain_dump, ensure_ascii=False) else: message_content = str(query) @@ -168,7 +171,10 @@ class MonitoringHelper: if hasattr(last_resp, 'get_content_platform_message_chain'): chain = last_resp.get_content_platform_message_chain() if hasattr(chain, 'model_dump'): - message_content = json.dumps(chain.model_dump(), ensure_ascii=False) + chain_dump = chain.model_dump() + if hasattr(ap, 'storage_mgr') and hasattr(ap.storage_mgr, 'media_cache'): + chain_dump = await ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump) + message_content = json.dumps(chain_dump, ensure_ascii=False) else: message_content = str(chain) else: diff --git a/src/langbot/pkg/platform/botmgr.py b/src/langbot/pkg/platform/botmgr.py index feaeea7a9..76a9ff03d 100644 --- a/src/langbot/pkg/platform/botmgr.py +++ b/src/langbot/pkg/platform/botmgr.py @@ -210,7 +210,10 @@ class RuntimeBot: """Record a discarded message in the monitoring system.""" try: if hasattr(message_chain, 'model_dump'): - message_content = json.dumps(message_chain.model_dump(), ensure_ascii=False) + chain_dump = message_chain.model_dump() + if hasattr(self.ap, 'storage_mgr') and hasattr(self.ap.storage_mgr, 'media_cache'): + chain_dump = await self.ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump) + message_content = json.dumps(chain_dump, ensure_ascii=False) else: message_content = str(message_chain) diff --git a/src/langbot/pkg/storage/media.py b/src/langbot/pkg/storage/media.py new file mode 100644 index 000000000..74d882da1 --- /dev/null +++ b/src/langbot/pkg/storage/media.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import asyncio +import base64 +import copy +import datetime +import hashlib +import mimetypes +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any + +try: + import xxhash +except ImportError: + xxhash = None + +if TYPE_CHECKING: + from ...core import app + from . import mgr as storage_mgr + +DEFAULT_RETENTION_DAYS = 30 +DEFAULT_MAX_SIZE_MB = 0 +MEDIA_DIR = 'media_cache' +SAFE_MEDIA_FILENAME = re.compile(r'^[a-f0-9]{32,64}(\.[a-zA-Z0-9]{1,10})?$') + + +class MediaCache: + """Content-addressable storage cache for images and media attachments. + + Deduplicates media files using xxHash3-128 (with sha256 fallback), + offloads payloads from SQLite to StorageProvider, and implements LRU + and age-based retention cleanup. + """ + + def __init__(self, ap: app.Application, storage_mgr: storage_mgr.StorageMgr): + self.ap = ap + self.storage_mgr = storage_mgr + + @staticmethod + def hash_bytes(data: bytes) -> str: + """Compute content-addressable hash for binary data.""" + if xxhash is not None: + return xxhash.xxh3_128_hexdigest(data) + return hashlib.sha256(data).hexdigest()[:32] + + @staticmethod + def parse_data_url(data_url: str) -> tuple[bytes, str] | None: + """Parse a data URL or raw base64 string into bytes and mime type.""" + if not data_url or not isinstance(data_url, str): + return None + try: + if data_url.startswith('data:'): + split_index = data_url.find(';base64,') + if split_index != -1: + mime_type = data_url[5:split_index] + b64_data = data_url[split_index + 8 :] + return base64.b64decode(b64_data), mime_type + # Try raw base64 if sufficiently long + if len(data_url) > 20 and not data_url.startswith(('http://', 'https://', '/')): + return base64.b64decode(data_url), 'application/octet-stream' + except Exception: + return None + return None + + @staticmethod + def guess_extension(mime_type: str | None, default: str = '.jpg') -> str: + """Guess appropriate file extension from MIME type.""" + if not mime_type: + return default + mime_lower = mime_type.lower() + if 'png' in mime_lower: + return '.png' + if 'webp' in mime_lower: + return '.webp' + if 'gif' in mime_lower: + return '.gif' + if 'jpeg' in mime_lower or 'jpg' in mime_lower: + return '.jpg' + ext = mimetypes.guess_extension(mime_type) + if ext == '.jpe': + return '.jpg' + return ext or default + + async def save_media(self, data: bytes, mime_type: str | None = None) -> tuple[str, str, int]: + """Save media bytes into content-addressable storage cache. + + Returns: + Tuple of (hash_str, storage_key, byte_size) + """ + hash_str = self.hash_bytes(data) + ext = self.guess_extension(mime_type) + storage_key = f'{MEDIA_DIR}/{hash_str}{ext}' + provider = self.storage_mgr.storage_provider + + if not await provider.exists(storage_key): + await provider.save(storage_key, data) + else: + await self.touch(storage_key) + + return hash_str, storage_key, len(data) + + async def get_media(self, filename_or_key: str) -> tuple[bytes, str] | None: + """Retrieve media bytes and mime type by key or filename.""" + filename = os.path.basename(filename_or_key) + if not SAFE_MEDIA_FILENAME.match(filename): + return None + storage_key = f'{MEDIA_DIR}/{filename}' + provider = self.storage_mgr.storage_provider + + if not await provider.exists(storage_key): + return None + + data = await self.storage_mgr._load_object_bounded(storage_key) + mime_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + await self.touch(storage_key) + return data, mime_type + + async def touch(self, storage_key: str) -> None: + """Update access/modified time of a media file for LRU tracking.""" + provider = getattr(self.storage_mgr, 'storage_provider', None) + if provider is not None and provider.__class__.__name__ == 'LocalStorageProvider': + full_path = os.path.join('data', 'storage', storage_key) + if os.path.exists(full_path): + now = datetime.datetime.now().timestamp() + try: + await asyncio.to_thread(os.utime, full_path, (now, now)) + except Exception: + pass + + async def cleanup( + self, + retention_days: int = DEFAULT_RETENTION_DAYS, + max_size_mb: int = DEFAULT_MAX_SIZE_MB, + ) -> dict[str, int]: + """Perform age-based and LRU size-based cleanup on media cache. + + Args: + retention_days: Retain media accessed within this many days (default 30). + max_size_mb: Maximum total size in MB (0 means unlimited). + + Returns: + Dictionary of cleanup metrics. + """ + provider = getattr(self.storage_mgr, 'storage_provider', None) + if provider is None or provider.__class__.__name__ != 'LocalStorageProvider': + return {'expired_deleted': 0, 'size_deleted': 0, 'bytes_freed': 0} + + target_dir = Path('data/storage') / MEDIA_DIR + if not target_dir.exists() or not target_dir.is_dir(): + return {'expired_deleted': 0, 'size_deleted': 0, 'bytes_freed': 0} + + now = datetime.datetime.now().timestamp() + cutoff = (now - retention_days * 86400) if retention_days > 0 else 0 + + expired_deleted = 0 + size_deleted = 0 + bytes_freed = 0 + remaining: list[tuple[Path, int, float]] = [] + + for entry in target_dir.iterdir(): + if not entry.is_file(): + continue + try: + stat = entry.stat() + except OSError: + continue + + if cutoff > 0 and stat.st_mtime < cutoff: + try: + entry.unlink(missing_ok=True) + expired_deleted += 1 + bytes_freed += stat.st_size + except OSError: + pass + else: + remaining.append((entry, stat.st_size, stat.st_mtime)) + + if max_size_mb > 0: + max_bytes = max_size_mb * 1024 * 1024 + total_bytes = sum(item[1] for item in remaining) + if total_bytes > max_bytes: + remaining.sort(key=lambda item: item[2]) + for path, size, _ in remaining: + if total_bytes <= max_bytes: + break + try: + path.unlink(missing_ok=True) + size_deleted += 1 + bytes_freed += size + total_bytes -= size + except OSError: + pass + + return { + 'expired_deleted': expired_deleted, + 'size_deleted': size_deleted, + 'bytes_freed': bytes_freed, + } + + async def externalize_chain_dump(self, chain_dump: Any) -> Any: + """Recursively extract raw base64 media into cache and replace with references.""" + if isinstance(chain_dump, list): + return [await self.externalize_chain_dump(item) for item in chain_dump] + if isinstance(chain_dump, dict): + node = copy.copy(chain_dump) + node_type = node.get('type') + if node_type == 'Image': + b64 = node.get('base64') + if b64 and isinstance(b64, str): + try: + parsed = self.parse_data_url(b64) + if parsed is not None: + raw_bytes, mime_type = parsed + hash_str, storage_key, size = await self.save_media(raw_bytes, mime_type) + filename = os.path.basename(storage_key) + current_url = node.get('url') or '' + if current_url and not current_url.startswith('data:'): + node['original_url'] = current_url + node['url'] = f'/api/v1/files/media/{filename}' + node['base64'] = None + node['hash'] = hash_str + node['storage_key'] = storage_key + node['size'] = size + node['mime_type'] = mime_type + except Exception as e: + if hasattr(self.ap, 'logger') and self.ap.logger: + self.ap.logger.warning(f'Failed to externalize image to media cache: {e}') + node['base64'] = None + for k, v in list(node.items()): + if isinstance(v, (list, dict)): + node[k] = await self.externalize_chain_dump(v) + return node + return chain_dump diff --git a/src/langbot/pkg/storage/mgr.py b/src/langbot/pkg/storage/mgr.py index c6c7c8a93..5a96eaa7d 100644 --- a/src/langbot/pkg/storage/mgr.py +++ b/src/langbot/pkg/storage/mgr.py @@ -34,6 +34,9 @@ class StorageMgr: def __init__(self, ap: app.Application): self.ap = ap + from . import media + + self.media_cache = media.MediaCache(ap, self) def _object_read_limit(self) -> int: config = getattr(getattr(self.ap, 'instance_config', None), 'data', {}) diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 6b3c716fa..24efaba09 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -227,6 +227,12 @@ storage: # Bound every object materialized into Core memory. Built-in Local/S3 # providers enforce this while reading (hard cap: 64 MiB). max_object_read_bytes: 10485760 + # Media content cache (images & attachments externalized from monitoring and pipelines) + media_cache: + # Retention period in days for cached media (defaults to 30 days) + retention_days: 30 + # Maximum disk storage for cached media in MB (0 means unlimited, defaults to 0) + max_size_mb: 0 cleanup: # Enable periodic cleanup of local/S3 uploaded files and old log files enabled: true diff --git a/tests/unit_tests/storage/test_media_cache.py b/tests/unit_tests/storage/test_media_cache.py new file mode 100644 index 000000000..054ee725b --- /dev/null +++ b/tests/unit_tests/storage/test_media_cache.py @@ -0,0 +1,215 @@ +import base64 +import datetime +import os +import tempfile +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch +import pytest +from quart import Quart + +from langbot.pkg.storage.media import MediaCache, SAFE_MEDIA_FILENAME +from langbot.pkg.api.http.service.monitoring import MonitoringService +from langbot.pkg.api.http.controller.groups.files import FilesRouterGroup + + +class TestMediaCache: + def setup_method(self): + self.mock_app = Mock() + self.mock_app.logger = Mock() + self.mock_storage_mgr = Mock() + self.mock_provider = Mock() + self.mock_provider.__class__.__name__ = 'LocalStorageProvider' + self.mock_provider.exists = AsyncMock(return_value=False) + self.mock_provider.save = AsyncMock() + self.mock_provider.load = AsyncMock() + self.mock_storage_mgr.storage_provider = self.mock_provider + self.mock_storage_mgr._load_object_bounded = AsyncMock() + self.media_cache = MediaCache(self.mock_app, self.mock_storage_mgr) + + def test_safe_media_filename_regex(self): + assert SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png') + assert SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.jpg') + assert SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c') + assert not SAFE_MEDIA_FILENAME.match('../etc/passwd') + assert not SAFE_MEDIA_FILENAME.match('foo/bar.png') + assert not SAFE_MEDIA_FILENAME.match('test.exe') + assert not SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3cXpng') + + def test_hash_bytes(self): + data1 = b'hello image content' + data2 = b'hello image content' + data3 = b'different content' + assert self.media_cache.hash_bytes(data1) == self.media_cache.hash_bytes(data2) + assert self.media_cache.hash_bytes(data1) != self.media_cache.hash_bytes(data3) + + def test_parse_data_url(self): + raw = b'png binary data here' + b64_str = base64.b64encode(raw).decode('ascii') + data_url = f'data:image/png;base64,{b64_str}' + + parsed = self.media_cache.parse_data_url(data_url) + assert parsed is not None + data, mime = parsed + assert data == raw + assert mime == 'image/png' + + @pytest.mark.asyncio + async def test_save_media_deduplication(self): + raw = b'fake png bytes' + hash_str = self.media_cache.hash_bytes(raw) + + # First save: provider.exists is False -> calls provider.save + h1, key1, size1 = await self.media_cache.save_media(raw, 'image/png') + assert h1 == hash_str + assert key1 == f'media_cache/{hash_str}.png' + assert size1 == len(raw) + self.mock_provider.save.assert_called_once_with(key1, raw) + + # Second save: provider.exists is True -> does not call provider.save again + self.mock_provider.exists.return_value = True + self.mock_provider.save.reset_mock() + h2, key2, size2 = await self.media_cache.save_media(raw, 'image/png') + assert h2 == h1 + assert key2 == key1 + self.mock_provider.save.assert_not_called() + + @pytest.mark.asyncio + async def test_get_media(self): + raw = b'stored bytes' + self.mock_provider.exists.return_value = True + self.mock_storage_mgr._load_object_bounded.return_value = raw + + valid_name = '3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png' + res = await self.media_cache.get_media(valid_name) + assert res is not None + data, mime = res + assert data == raw + assert mime == 'image/png' + + # Rejects invalid names + assert await self.media_cache.get_media('../malicious.png') is None + + @pytest.mark.asyncio + async def test_externalize_chain_dump(self): + raw = b'tiny image' + b64 = f'data:image/png;base64,{base64.b64encode(raw).decode("ascii")}' + chain_dump = [ + {'type': 'Plain', 'text': 'hello'}, + {'type': 'Image', 'url': 'https://multimedia.nt.qq.com.cn/download?appid=1407', 'base64': b64}, + {'type': 'Quote', 'origin': [{'type': 'Image', 'url': '', 'base64': b64}]}, + ] + + result = await self.media_cache.externalize_chain_dump(chain_dump) + + # Root image + img = result[1] + assert img['base64'] is None + assert img['hash'] == self.media_cache.hash_bytes(raw) + assert img['storage_key'].startswith('media_cache/') + assert img['original_url'] == 'https://multimedia.nt.qq.com.cn/download?appid=1407' + assert img['url'] == f'/api/v1/files/media/{img["hash"]}.png' + assert img['size'] == len(raw) + + # Nested quote image + nested_img = result[2]['origin'][0] + assert nested_img['base64'] is None + assert nested_img['hash'] == self.media_cache.hash_bytes(raw) + assert nested_img['url'] == f'/api/v1/files/media/{nested_img["hash"]}.png' + + @pytest.mark.asyncio + async def test_externalize_chain_dump_error_resilience(self): + raw = b'broken image' + b64 = f'data:image/png;base64,{base64.b64encode(raw).decode("ascii")}' + chain_dump = [{'type': 'Image', 'base64': b64}] + + with patch.object(self.media_cache, 'save_media', side_effect=OSError('Disk full')): + result = await self.media_cache.externalize_chain_dump(chain_dump) + # Should not raise; base64 should be stripped as fallback + assert result[0]['base64'] is None + + @pytest.mark.asyncio + async def test_cleanup_retention_and_max_size(self): + with tempfile.TemporaryDirectory() as temp_dir: + base_path = Path(temp_dir) + cache_dir = base_path / 'data' / 'storage' / 'media_cache' + cache_dir.mkdir(parents=True) + + # Create 3 test files with different mtimes and sizes + f1 = cache_dir / 'old_expired.png' + f1.write_bytes(b'x' * 1000) + old_time = (datetime.datetime.now() - datetime.timedelta(days=35)).timestamp() + os.utime(f1, (old_time, old_time)) + + f2 = cache_dir / 'recent_large1.png' + f2.write_bytes(b'x' * 500) + t2 = (datetime.datetime.now() - datetime.timedelta(days=5)).timestamp() + os.utime(f2, (t2, t2)) + + f3 = cache_dir / 'recent_large2.png' + f3.write_bytes(b'x' * 500) + t3 = (datetime.datetime.now() - datetime.timedelta(days=1)).timestamp() + os.utime(f3, (t3, t3)) + + with patch('langbot.pkg.storage.media.Path') as mock_path: + mock_path.return_value = base_path / 'data' / 'storage' + # Run cleanup with 30-day retention and max_size_mb = 0 (unlimited) + stats = await self.media_cache.cleanup(retention_days=30, max_size_mb=0) + assert stats['expired_deleted'] == 1 + assert not f1.exists() + assert f2.exists() + assert f3.exists() + + # Run cleanup with max_size_mb limited to ~0.0006 MB (< 1000 bytes) + # Total is currently 1000 bytes (f2=500 + f3=500). Max size 600 bytes -> oldest f2 must be purged + stats2 = await self.media_cache.cleanup(retention_days=30, max_size_mb=0.0006) + assert stats2['size_deleted'] >= 1 + assert not f2.exists() + assert f3.exists() + + @pytest.mark.asyncio + async def test_files_media_endpoint(self): + quart_app = Quart(__name__) + mock_app = Mock() + mock_app.storage_mgr = self.mock_storage_mgr + + router = FilesRouterGroup(mock_app, quart_app) + await router.initialize() + + client = quart_app.test_client() + + # 1. 404 on not found + mock_cache = Mock() + mock_cache.get_media = AsyncMock(return_value=None) + self.mock_storage_mgr.media_cache = mock_cache + resp_404 = await client.get('/api/v1/files/media/3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png') + assert resp_404.status_code == 404 + + # 2. 200 on found with cache headers + mock_cache.get_media.return_value = (b'fake image data', 'image/png') + resp_200 = await client.get('/api/v1/files/media/3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png') + assert resp_200.status_code == 200 + assert await resp_200.get_data() == b'fake image data' + assert 'public' in resp_200.headers.get('Cache-Control', '') + assert 'image/png' in resp_200.headers.get('Content-Type', '') + + +class TestMonitoringServiceSanitization: + def test_sanitize_oversized_base64_payload(self): + svc = MonitoringService.__new__(MonitoringService) + small_content = '{"type": "Image", "base64": "data:image/png;base64,tiny"}' + # Should leave small contents untouched + assert svc._sanitize_message_content(small_content) == small_content + + # Large content with base64 data URL + huge_b64 = 'A' * 60000 + large_content = f'{{"type": "Image", "base64": "data:image/png;base64,{huge_b64}"}}' + sanitized = svc._sanitize_message_content(large_content) + assert huge_b64 not in sanitized + assert '"base64": null' in sanitized or '"base64":null' in sanitized + + # Non-JSON content with multi-line base64 + multiline_b64 = ('A' * 70 + '\r\n') * 300 + raw_corrupted = 'prefix data:image/png;base64,' + multiline_b64 + ' suffix' + sanitized_raw = svc._sanitize_message_content(raw_corrupted) + assert '[base64 image omitted]' in sanitized_raw + assert multiline_b64 not in sanitized_raw diff --git a/uv.lock b/uv.lock index 8d718eb73..107889d6b 100644 --- a/uv.lock +++ b/uv.lock @@ -1066,7 +1066,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, @@ -1099,34 +1099,34 @@ wheels = [ [package.optional-dependencies] cudart = [ - { name = "nvidia-cuda-runtime", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime" }, ] cufft = [ - { name = "nvidia-cufft", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cufft" }, ] cufile = [ - { name = "nvidia-cufile", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cufile" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti" }, ] curand = [ - { name = "nvidia-curand", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-curand" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx" }, ] [[package]] @@ -2135,6 +2135,7 @@ dependencies = [ { name = "valkey-glide", marker = "sys_platform != 'win32'" }, { name = "webauthn" }, { name = "websockets" }, + { name = "xxhash" }, ] [package.optional-dependencies] @@ -2233,6 +2234,7 @@ requires-dist = [ { name = "valkey-glide", marker = "sys_platform != 'win32'", specifier = ">=2.4.1,<3.0.0" }, { name = "webauthn", specifier = ">=3.0.0" }, { name = "websockets", specifier = ">=15.0.1" }, + { name = "xxhash", specifier = ">=3.5.0" }, ] provides-extras = ["seekdb"] @@ -3299,7 +3301,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cuda-nvrtc" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -3338,7 +3340,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -3350,7 +3352,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3380,9 +3382,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3394,7 +3396,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -4487,7 +4489,7 @@ name = "pylibseekdb" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pymysql", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "pymysql" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ae/a8/7413d33218aff55a14ec9d20532b49243ffd0579e7a92244922c1885444e/pylibseekdb-1.4.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:5cb2efab9f1321cdb4b034d3a2bd92e41a402fc95e7dc9579c7473a426f96e24", size = 52173499, upload-time = "2026-08-27T13:05:09.347Z" }, @@ -5255,10 +5257,10 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.14'" }, - { name = "numpy", marker = "python_full_version >= '3.14'" }, - { name = "scipy", marker = "python_full_version >= '3.14'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.14'" }, + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } wheels = [ @@ -5305,7 +5307,7 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.14'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -5376,14 +5378,14 @@ name = "sentence-transformers" version = "5.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub", marker = "python_full_version >= '3.14'" }, - { name = "numpy", marker = "python_full_version >= '3.14'" }, - { name = "scikit-learn", marker = "python_full_version >= '3.14'" }, - { name = "scipy", marker = "python_full_version >= '3.14'" }, - { name = "torch", marker = "python_full_version >= '3.14'" }, - { name = "tqdm", marker = "python_full_version >= '3.14'" }, - { name = "transformers", marker = "python_full_version >= '3.14'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.14'" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/30/21664028fc0776eb1ca024879480bbbab36f02923a8ff9e4cae5a150fa35/sentence_transformers-5.2.3.tar.gz", hash = "sha256:3cd3044e1f3fe859b6a1b66336aac502eaae5d3dd7d5c8fc237f37fbf58137c7", size = 381623, upload-time = "2026-02-17T14:05:20.238Z" } wheels = [ @@ -5756,21 +5758,21 @@ name = "torch" version = "2.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "filelock", marker = "python_full_version >= '3.14'" }, - { name = "fsspec", marker = "python_full_version >= '3.14'" }, - { name = "jinja2", marker = "python_full_version >= '3.14'" }, - { name = "networkx", marker = "python_full_version >= '3.14'" }, - { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.14'" }, - { name = "sympy", marker = "python_full_version >= '3.14'" }, - { name = "triton", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.14'" }, + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" }, @@ -5812,15 +5814,15 @@ name = "transformers" version = "5.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub", marker = "python_full_version >= '3.14'" }, - { name = "numpy", marker = "python_full_version >= '3.14'" }, - { name = "packaging", marker = "python_full_version >= '3.14'" }, - { name = "pyyaml", marker = "python_full_version >= '3.14'" }, - { name = "regex", marker = "python_full_version >= '3.14'" }, - { name = "safetensors", marker = "python_full_version >= '3.14'" }, - { name = "tokenizers", marker = "python_full_version >= '3.14'" }, - { name = "tqdm", marker = "python_full_version >= '3.14'" }, - { name = "typer", marker = "python_full_version >= '3.14'" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/1a/70e830d53ecc96ce69cfa8de38f163712d2b43ac52fbd743f39f56025c31/transformers-5.3.0.tar.gz", hash = "sha256:009555b364029da9e2946d41f1c5de9f15e6b1df46b189b7293f33a161b9c557", size = 8830831, upload-time = "2026-03-04T17:41:46.119Z" } wheels = [ @@ -6081,9 +6083,9 @@ name = "valkey-glide" version = "2.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform != 'win32'" }, - { name = "protobuf", marker = "sys_platform != 'win32'" }, - { name = "sniffio", marker = "sys_platform != 'win32'" }, + { name = "anyio" }, + { name = "protobuf" }, + { name = "sniffio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/72/a2/582b34c6acc8dc857c537f6007459cba48dfa0dc404789a657e5c1a998c0/valkey_glide-2.4.1.tar.gz", hash = "sha256:f1155d84156d11b90488aa67e90102f0bf98a45314f5b99308ac9074c05f7241", size = 898030, upload-time = "2026-05-28T21:41:55.881Z" } wheels = [ From a32b3121d3e1525470b40cb62a53401f21777485 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 18:14:29 +0800 Subject: [PATCH 3/5] build(boot): register xxhash in startup dependency check --- src/langbot/pkg/core/bootutils/deps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/langbot/pkg/core/bootutils/deps.py b/src/langbot/pkg/core/bootutils/deps.py index 2cfd57e0c..b13d138b6 100644 --- a/src/langbot/pkg/core/bootutils/deps.py +++ b/src/langbot/pkg/core/bootutils/deps.py @@ -43,6 +43,7 @@ required_deps = { 'slack_sdk': 'slack_sdk', 'asyncpg': 'asyncpg', 'litellm': 'litellm', + 'xxhash': 'xxhash', } From c16cd433e0c53c89a6891df76cdce165d3fcffa4 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 19:48:08 +0800 Subject: [PATCH 4/5] refactor(storage): enforce xxHash3-128 and remove sha256 fallback in MediaCache - Import xxhash directly instead of conditional try-except block - Remove unused hashlib import and sha256 fallback branch in hash_bytes - Update docstrings to reflect pure xxHash3-128 content-addressing --- src/langbot/pkg/storage/media.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/langbot/pkg/storage/media.py b/src/langbot/pkg/storage/media.py index 74d882da1..0029013ff 100644 --- a/src/langbot/pkg/storage/media.py +++ b/src/langbot/pkg/storage/media.py @@ -4,18 +4,13 @@ import asyncio import base64 import copy import datetime -import hashlib import mimetypes import os import re +import xxhash from pathlib import Path from typing import TYPE_CHECKING, Any -try: - import xxhash -except ImportError: - xxhash = None - if TYPE_CHECKING: from ...core import app from . import mgr as storage_mgr @@ -29,7 +24,7 @@ SAFE_MEDIA_FILENAME = re.compile(r'^[a-f0-9]{32,64}(\.[a-zA-Z0-9]{1,10})?$') class MediaCache: """Content-addressable storage cache for images and media attachments. - Deduplicates media files using xxHash3-128 (with sha256 fallback), + Deduplicates media files using xxHash3-128, offloads payloads from SQLite to StorageProvider, and implements LRU and age-based retention cleanup. """ @@ -41,9 +36,7 @@ class MediaCache: @staticmethod def hash_bytes(data: bytes) -> str: """Compute content-addressable hash for binary data.""" - if xxhash is not None: - return xxhash.xxh3_128_hexdigest(data) - return hashlib.sha256(data).hexdigest()[:32] + return xxhash.xxh3_128_hexdigest(data) @staticmethod def parse_data_url(data_url: str) -> tuple[bytes, str] | None: From 7e239f162942059a66a705a6af0720778262704b Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 20:30:34 +0800 Subject: [PATCH 5/5] fix(maintenance): guard media cache cleanup and annotate exception suppression - Safely resolve media_cache via getattr in cleanup_expired_files to prevent AttributeError when storage_mgr is unset in cloud/test fixtures - Only attach 'media_files' in cleanup return dictionary when media cache is active on singleton, preserving exact return contract for cloud maintenance tests - Add explanatory comments to exception suppression in MediaCache.touch and cleanup loops to address code-quality review findings --- src/langbot/pkg/api/http/service/maintenance.py | 14 +++++++++----- src/langbot/pkg/storage/media.py | 5 ++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/langbot/pkg/api/http/service/maintenance.py b/src/langbot/pkg/api/http/service/maintenance.py index 562c17d61..3ba4e77c5 100644 --- a/src/langbot/pkg/api/http/service/maintenance.py +++ b/src/langbot/pkg/api/http/service/maintenance.py @@ -91,25 +91,29 @@ class MaintenanceService: 0, 'storage.media_cache.max_size_mb', ) + media_cache = getattr(getattr(self.ap, 'storage_mgr', None), 'media_cache', None) + is_singleton = await self._is_oss_singleton(context) media_cleanup = ( - await self.ap.storage_mgr.media_cache.cleanup( + await media_cache.cleanup( media_retention_days, media_max_size_mb, ) - if hasattr(self.ap.storage_mgr, 'media_cache') and await self._is_oss_singleton(context) + if media_cache is not None and is_singleton else {} ) - return { + result = { 'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days), 'log_files': await asyncio.to_thread( self._cleanup_expired_log_files, log_retention_days, ) - if await self._is_oss_singleton(context) + if is_singleton else 0, - 'media_files': media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0), } + if media_cache is not None and is_singleton: + result['media_files'] = media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0) + return result async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]: require_workspace_uuid(context) diff --git a/src/langbot/pkg/storage/media.py b/src/langbot/pkg/storage/media.py index 0029013ff..8f5537fb3 100644 --- a/src/langbot/pkg/storage/media.py +++ b/src/langbot/pkg/storage/media.py @@ -120,6 +120,7 @@ class MediaCache: try: await asyncio.to_thread(os.utime, full_path, (now, now)) except Exception: + # Failures to update mtime are intentionally ignored because LRU touch is opportunistic. pass async def cleanup( @@ -166,6 +167,7 @@ class MediaCache: expired_deleted += 1 bytes_freed += stat.st_size except OSError: + # Best-effort cleanup; file may already be gone or temporarily inaccessible. pass else: remaining.append((entry, stat.st_size, stat.st_mtime)) @@ -184,7 +186,8 @@ class MediaCache: bytes_freed += size total_bytes -= size except OSError: - pass + # Best-effort cleanup; file may already be gone or temporarily inaccessible. + continue return { 'expired_deleted': expired_deleted,