xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-09-16 14:57:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(storage): media content-addressable cache and monitoring base64 externalization

Browse Source 
- Add MediaCache using xxHash3-128 (with sha256 fallback) content-addressable storage
- Externalize message chain image payloads before recording monitoring and discarded messages
- Strip base64 payloads to null in SQLite monitoring_messages, dropping row size from megabytes to hundreds of bytes
- Add GET /api/v1/files/media/<filename> route with immutable HTTP cache headers to serve cached media
- Integrate age-based retention (default 30 days) and configurable disk quota with MaintenanceService cleanup loop
- Add defensive sanitizer in MonitoringService.record_message against oversized raw base64 payloads
- Add comprehensive unit tests and end-to-end verification covering CAS deduplication, route serving, and LRU pruning
This commit is contained in:
BiFangKNT
2026-09-15 17:58:23 +08:00
parent 38ff4766ef
commit 1143d6a5ae
11 changed files with 607 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
uv.lock
Generated
+61 -59
View File
@@ -1066,7 +1066,7 @@ name = "cuda-bindings"
version = "13.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-pathfinder", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "cuda-pathfinder" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" },
@@ -1099,34 +1099,34 @@ wheels = [
[package.optional-dependencies]
cudart = [
{ name = "nvidia-cuda-runtime", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cuda-runtime" },
]
cufft = [
{ name = "nvidia-cufft", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cufft" },
]
cufile = [
{ name = "nvidia-cufile", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cufile" },
]
cupti = [
{ name = "nvidia-cuda-cupti", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cuda-cupti" },
]
curand = [
{ name = "nvidia-curand", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-curand" },
]
cusolver = [
{ name = "nvidia-cusolver", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cusolver" },
]
cusparse = [
{ name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cusparse" },
]
nvjitlink = [
{ name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-nvjitlink" },
]
nvrtc = [
{ name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc" },
]
nvtx = [
{ name = "nvidia-nvtx", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-nvtx" },
]
[[package]]
@@ -2135,6 +2135,7 @@ dependencies = [
{ name = "valkey-glide", marker = "sys_platform != 'win32'" },
{ name = "webauthn" },
{ name = "websockets" },
{ name = "xxhash" },
]
[package.optional-dependencies]
@@ -2233,6 +2234,7 @@ requires-dist = [
{ name = "valkey-glide", marker = "sys_platform != 'win32'", specifier = ">=2.4.1,<3.0.0" },
{ name = "webauthn", specifier = ">=3.0.0" },
{ name = "websockets", specifier = ">=15.0.1" },
{ name = "xxhash", specifier = ">=3.5.0" },
]
provides-extras = ["seekdb"]
@@ -3299,7 +3301,7 @@ name = "nvidia-cublas"
version = "13.1.1.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-cuda-nvrtc" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" },
@@ -3338,7 +3340,7 @@ name = "nvidia-cudnn-cu13"
version = "9.20.0.48"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-cublas" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" },
@@ -3350,7 +3352,7 @@ name = "nvidia-cufft"
version = "12.0.0.61"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
@@ -3380,9 +3382,9 @@ name = "nvidia-cusolver"
version = "12.0.4.66"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-cublas" },
{ name = "nvidia-cusparse" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
@@ -3394,7 +3396,7 @@ name = "nvidia-cusparse"
version = "12.6.3.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
@@ -4487,7 +4489,7 @@ name = "pylibseekdb"
version = "1.4.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pymysql", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "pymysql" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/ae/a8/7413d33218aff55a14ec9d20532b49243ffd0579e7a92244922c1885444e/pylibseekdb-1.4.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:5cb2efab9f1321cdb4b034d3a2bd92e41a402fc95e7dc9579c7473a426f96e24", size = 52173499, upload-time = "2026-08-27T13:05:09.347Z" },
@@ -5255,10 +5257,10 @@ name = "scikit-learn"
version = "1.8.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "joblib", marker = "python_full_version >= '3.14'" },
{ name = "numpy", marker = "python_full_version >= '3.14'" },
{ name = "scipy", marker = "python_full_version >= '3.14'" },
{ name = "threadpoolctl", marker = "python_full_version >= '3.14'" },
{ name = "joblib" },
{ name = "numpy" },
{ name = "scipy" },
{ name = "threadpoolctl" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" }
wheels = [
@@ -5305,7 +5307,7 @@ name = "scipy"
version = "1.17.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy", marker = "python_full_version >= '3.14'" },
{ name = "numpy" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [
@@ -5376,14 +5378,14 @@ name = "sentence-transformers"
version = "5.2.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "huggingface-hub", marker = "python_full_version >= '3.14'" },
{ name = "numpy", marker = "python_full_version >= '3.14'" },
{ name = "scikit-learn", marker = "python_full_version >= '3.14'" },
{ name = "scipy", marker = "python_full_version >= '3.14'" },
{ name = "torch", marker = "python_full_version >= '3.14'" },
{ name = "tqdm", marker = "python_full_version >= '3.14'" },
{ name = "transformers", marker = "python_full_version >= '3.14'" },
{ name = "typing-extensions", marker = "python_full_version >= '3.14'" },
{ name = "huggingface-hub" },
{ name = "numpy" },
{ name = "scikit-learn" },
{ name = "scipy" },
{ name = "torch" },
{ name = "tqdm" },
{ name = "transformers" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5b/30/21664028fc0776eb1ca024879480bbbab36f02923a8ff9e4cae5a150fa35/sentence_transformers-5.2.3.tar.gz", hash = "sha256:3cd3044e1f3fe859b6a1b66336aac502eaae5d3dd7d5c8fc237f37fbf58137c7", size = 381623, upload-time = "2026-02-17T14:05:20.238Z" }
wheels = [
@@ -5756,21 +5758,21 @@ name = "torch"
version = "2.12.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-bindings", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "filelock", marker = "python_full_version >= '3.14'" },
{ name = "fsspec", marker = "python_full_version >= '3.14'" },
{ name = "jinja2", marker = "python_full_version >= '3.14'" },
{ name = "networkx", marker = "python_full_version >= '3.14'" },
{ name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cudnn-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-cusparselt-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-nccl-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "nvidia-nvshmem-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "setuptools", marker = "python_full_version >= '3.14'" },
{ name = "sympy", marker = "python_full_version >= '3.14'" },
{ name = "triton", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "python_full_version >= '3.14'" },
{ name = "cuda-bindings", marker = "sys_platform == 'linux'" },
{ name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" },
{ name = "filelock" },
{ name = "fsspec" },
{ name = "jinja2" },
{ name = "networkx" },
{ name = "nvidia-cublas", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" },
{ name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" },
{ name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" },
{ name = "setuptools" },
{ name = "sympy" },
{ name = "triton", marker = "sys_platform == 'linux'" },
{ name = "typing-extensions" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" },
@@ -5812,15 +5814,15 @@ name = "transformers"
version = "5.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "huggingface-hub", marker = "python_full_version >= '3.14'" },
{ name = "numpy", marker = "python_full_version >= '3.14'" },
{ name = "packaging", marker = "python_full_version >= '3.14'" },
{ name = "pyyaml", marker = "python_full_version >= '3.14'" },
{ name = "regex", marker = "python_full_version >= '3.14'" },
{ name = "safetensors", marker = "python_full_version >= '3.14'" },
{ name = "tokenizers", marker = "python_full_version >= '3.14'" },
{ name = "tqdm", marker = "python_full_version >= '3.14'" },
{ name = "typer", marker = "python_full_version >= '3.14'" },
{ name = "huggingface-hub" },
{ name = "numpy" },
{ name = "packaging" },
{ name = "pyyaml" },
{ name = "regex" },
{ name = "safetensors" },
{ name = "tokenizers" },
{ name = "tqdm" },
{ name = "typer" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fc/1a/70e830d53ecc96ce69cfa8de38f163712d2b43ac52fbd743f39f56025c31/transformers-5.3.0.tar.gz", hash = "sha256:009555b364029da9e2946d41f1c5de9f15e6b1df46b189b7293f33a161b9c557", size = 8830831, upload-time = "2026-03-04T17:41:46.119Z" }
wheels = [
@@ -6081,9 +6083,9 @@ name = "valkey-glide"
version = "2.4.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "anyio", marker = "sys_platform != 'win32'" },
{ name = "protobuf", marker = "sys_platform != 'win32'" },
{ name = "sniffio", marker = "sys_platform != 'win32'" },
{ name = "anyio" },
{ name = "protobuf" },
{ name = "sniffio" },
]
sdist = { url = "https://files.pythonhosted.org/packages/72/a2/582b34c6acc8dc857c537f6007459cba48dfa0dc404789a657e5c1a998c0/valkey_glide-2.4.1.tar.gz", hash = "sha256:f1155d84156d11b90488aa67e90102f0bf98a45314f5b99308ac9074c05f7241", size = 898030, upload-time = "2026-05-28T21:41:55.881Z" }
wheels = [