mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-07-14 16:36:07 +00:00
0c405901d2
* feat(vector): add Valkey Search vector database backend Add a new opt-in VectorDatabase backend backed by the Valkey Search module (valkey/valkey-bundle), accessed via the official valkey-glide client's native ft command namespace. - Implements the full VectorDatabase ABC: VECTOR, FULL_TEXT and HYBRID search, all 8 metadata filter operators, and pagination with exact totals. - HYBRID uses filter-then-KNN (no app-side weighted fusion); vector_weight is accepted for interface parity but NOT honored (docstring + one-time warning + docs caveat). - Lazy connect so a down Valkey never blocks boot; mandatory client_name=langbot_vector_client; optional auth + TLS (never logged). - Registered via a single elif branch in vector/mgr.py; disabled by default (vdb.use stays chroma) for toC compatibility. - Adds valkey-glide>=2.4.1,<3.0.0; no protobuf/pydantic downgrade; no ORM change so no Alembic migration. - Unit tests (fast lane, no server) + slow-gated integration tests (TEST_VALKEY_URL, valkey/valkey-bundle:9.1.0) + integration doc. * fix(vector): paginate Valkey Search deletes and guard delete_by_filter Address self-review follow-ups for the Valkey Search VDB backend: - _search_keys now paginates through the full result set in batches of _DELETE_SCAN_BATCH instead of capping at a single hard-coded 10000-key page, so delete_by_file_id / delete_by_filter fully remove files and filters that match more than one page of chunks (no orphaned vectors). - Add unit regression tests for the delete_by_filter mass-deletion guard: a filter referencing only non-indexed fields must skip and return 0 (never fall back to match-all), and a supported filter still deletes matching keys. * refactor(vector): harden Valkey Search backend and add adversarial tests Address the self-review NICE-TO-HAVE items for the Valkey Search VDB backend: - Guard the username-without-password credential edge (skip auth + warn instead of building ServerCredentials(password=None, ...), which glide rejects). - Add an async close() teardown that closes the glide client and resets cached state (re-init is safe via the existing None guard). - Hoist 'import json' to module top (was imported inside three methods). - Document the FT TAG literal-brace limitation in _escape_tag (fails closed, never widens). Tests: - Add an adversarial-input integration test proving crafted file_id / query_text cannot break out of or widen a query (fail-closed on braces). - Add unit tests for close() and the credential-build guard. Signed-off-by: Daria Korenieva <daric2612@gmail.com> * fix(vector): make Valkey Search file_id TAG support arbitrary characters Valkey Search's FT TAG query parser cannot handle '{', '}' or '*' even when backslash-escaped, so a file_id containing those characters previously produced an unparseable query (it failed closed / raised). Percent-encode exactly those FT-unsafe characters (plus '%' for reversibility) in the file_id TAG value, applied identically at write time and query time, so an arbitrary file_id round-trips. For normal UUID/hash ids this is a no-op and the stored value is unchanged; the original file_id is always preserved verbatim in metadata_json. Strengthen the adversarial integration test to assert a brace/star-bearing file_id matches and deletes exactly its own row (no widening, no raise), and add unit tests for _encode_file_id and the filter encoding. Signed-off-by: Daria Korenieva <daric2612@gmail.com> * refactor(vector): address Valkey Search review feedback - Add configurable request_timeout (default 5000ms; glide default 250ms is too low for KNN); expose in config.yaml + docs table - Validate embedding dimension consistency in add_embeddings (fail fast on mixed lengths to avoid silent KNN corruption) - Use ft.info (O(1)) instead of ft.list (O(n)) for index existence checks in the query hot path; also closes the check-then-create TOCTOU window - Pipeline HSETs via a non-atomic Batch instead of N sequential awaits - Extract shared _iter_reply_docs to deduplicate reply parsing between _reply_to_chroma and list_by_filter - Parenthesize multi-condition pre-filters before the => KNN clause - Fail closed when a username is configured without a password - Catch only RequestError on ft.dropindex (let connection/auth errors surface) - Bound the delete_collection SCAN loop with a safety cap - Add VectorDatabase.close() (no-op default) + VectorDBManager.shutdown() - Simplify _MATCH_ALL literal; normalize typing to builtin generics * fix(vector/valkey_search): address round-2 review feedback - Serialize lazy client creation with an asyncio.Lock (double-checked) so concurrent first-use callers don't construct and leak duplicate clients. - Make the filter operator chain exhaustive: raise on an unhandled op rather than silently dropping the condition (which could widen delete_by_filter). - Cast numeric range (///) values to float, failing closed on non-numeric input and pre-empting a future NUMERIC-field injection surface. * refactor(vector): remove shutdown/close from base ABC per maintainer feedback Per maintainer request, interface changes to VectorDatabase ABC and VectorDBManager should be in a separate PR with implementation across all backends. The ValkeySearchVectorDatabase.close() method remains but does not override an ABC method. Signed-off-by: Daria Korenieva <daric2612@gmail.com> * docs(test): list valkey_search in vdb coverage exclusions Add valkey_search to the documented vector/vdbs/ coverage-exclusion list, matching the existing chroma/milvus/pgvector/qdrant/seekdb entries. These adapters require a live database instance and are covered by env-gated integration tests instead of unit tests. Signed-off-by: Daria Korenieva <daric2612@gmail.com> --------- Signed-off-by: Daria Korenieva <daric2612@gmail.com>
228 lines
5.7 KiB
TOML
228 lines
5.7 KiB
TOML
[project]
|
|
name = "langbot"
|
|
version = "4.10.5"
|
|
description = "Production-grade platform for building agentic IM bots"
|
|
readme = "README.md"
|
|
license-files = ["LICENSE"]
|
|
requires-python = ">=3.11,<4.0"
|
|
dependencies = [
|
|
"aiocqhttp>=1.4.4",
|
|
"aiofiles>=24.1.0",
|
|
"aiohttp>=3.14.1",
|
|
"aioshutil>=1.5",
|
|
"aiosqlite>=0.21.0",
|
|
"anthropic>=0.51.0",
|
|
"argon2-cffi>=23.1.0",
|
|
"async-lru>=2.0.5",
|
|
"certifi>=2025.4.26",
|
|
"colorlog~=6.6.0",
|
|
"cryptography>=48.0.1",
|
|
"dashscope>=1.25.10",
|
|
"dingtalk-stream>=0.24.0",
|
|
"discord-py>=2.5.2",
|
|
"pynacl>=1.5.0", # Required for Discord voice support
|
|
"gewechat-client>=0.1.5",
|
|
"lark-oapi>=1.5.5",
|
|
"mcp>=1.25.0",
|
|
"nakuru-project-idk>=0.0.2.1",
|
|
"ollama>=0.4.8",
|
|
"openai>1.0.0",
|
|
"pillow>=12.2.0",
|
|
"psutil>=7.0.0",
|
|
"pycryptodome>=3.22.0",
|
|
"pydantic>2.0",
|
|
"pyjwt>=2.12.0",
|
|
"python-telegram-bot>=22.0",
|
|
"pyyaml>=6.0.2",
|
|
"qq-botpy-rc>=1.2.1.6",
|
|
"qrcode>=7.4",
|
|
"quart>=0.20.0",
|
|
"quart-cors>=0.8.0",
|
|
"requests>=2.33.0",
|
|
"slack-sdk>=3.35.0",
|
|
"alembic>=1.15.0",
|
|
"sqlalchemy[asyncio]>=2.0.40",
|
|
"sqlmodel>=0.0.24",
|
|
"telegramify-markdown>=0.5.1",
|
|
"tiktoken>=0.9.0",
|
|
"urllib3>=2.7.0",
|
|
"websockets>=15.0.1",
|
|
"python-socks>=2.7.1", # dingtalk missing dependency
|
|
"pip>=26.1",
|
|
"ruff>=0.11.9",
|
|
"pre-commit>=4.2.0",
|
|
"uv>=0.11.15",
|
|
"mypy>=1.16.0",
|
|
"PyPDF2>=3.0.1",
|
|
"python-docx>=1.1.0",
|
|
"pandas>=2.2.2",
|
|
"chardet>=5.2.0",
|
|
"markdown>=3.6",
|
|
"beautifulsoup4>=4.12.3",
|
|
"ebooklib>=0.18",
|
|
"html2text>=2024.2.26",
|
|
"langchain>=1.3.9",
|
|
"langchain-core>=1.3.3",
|
|
"langsmith>=0.8.18",
|
|
"python-multipart>=0.0.27",
|
|
"Mako>=1.3.12",
|
|
"langchain-text-splitters>=1.1.2",
|
|
"chromadb>=1.0.0,<2.0.0",
|
|
"qdrant-client (>=1.15.1,<2.0.0)",
|
|
"pyseekdb==1.1.0.post3",
|
|
"langbot-plugin==0.4.13",
|
|
"asyncpg>=0.30.0",
|
|
"line-bot-sdk>=3.19.0",
|
|
"matrix-nio>=0.25.2",
|
|
"tboxsdk>=0.0.10",
|
|
"boto3>=1.35.0",
|
|
"pymilvus>=2.6.4",
|
|
"pgvector>=0.4.1",
|
|
"botocore>=1.42.39",
|
|
"litellm>=1.0.0",
|
|
"valkey-glide>=2.4.1,<3.0.0",
|
|
]
|
|
keywords = [
|
|
"bot",
|
|
"agent",
|
|
"telegram",
|
|
"plugins",
|
|
"openai",
|
|
"instant-messaging",
|
|
"wechat",
|
|
"qq",
|
|
"dify",
|
|
"llm",
|
|
"chatgpt",
|
|
"deepseek",
|
|
"onebot",
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 5 - Production/Stable",
|
|
"Framework :: AsyncIO",
|
|
"Framework :: Robot Framework",
|
|
"Framework :: Robot Framework :: Library",
|
|
"Operating System :: OS Independent",
|
|
"Programming Language :: Python :: 3",
|
|
"Topic :: Communications :: Chat",
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://langbot.app"
|
|
Documentation = "https://docs.langbot.app"
|
|
Repository = "https://github.com/langbot-app/LangBot"
|
|
|
|
[project.scripts]
|
|
langbot = "langbot.__main__:main"
|
|
|
|
[build-system]
|
|
requires = ["setuptools>=61.0", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.setuptools]
|
|
package-data = { "langbot" = ["templates/**", "pkg/provider/modelmgr/requesters/*", "pkg/platform/sources/*", "web/dist/**", "pkg/persistence/alembic/**"] }
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"moto>=5.2.1",
|
|
"pre-commit>=4.2.0",
|
|
"pytest>=9.0.3",
|
|
"pytest-asyncio>=1.0.0",
|
|
"pytest-cov>=7.0.0",
|
|
"ruff>=0.11.9",
|
|
]
|
|
|
|
[tool.ruff]
|
|
# Exclude a variety of commonly ignored directories.
|
|
exclude = [
|
|
".bzr",
|
|
".direnv",
|
|
".eggs",
|
|
".git",
|
|
".git-rewrite",
|
|
".hg",
|
|
".ipynb_checkpoints",
|
|
".mypy_cache",
|
|
".nox",
|
|
".pants.d",
|
|
".pyenv",
|
|
".pytest_cache",
|
|
".pytype",
|
|
".ruff_cache",
|
|
".svn",
|
|
".tox",
|
|
".venv",
|
|
".vscode",
|
|
"__pypackages__",
|
|
"_build",
|
|
"buck-out",
|
|
"build",
|
|
"dist",
|
|
"node_modules",
|
|
"site-packages",
|
|
"venv",
|
|
]
|
|
|
|
line-length = 120
|
|
indent-width = 4
|
|
|
|
# Assume Python 3.12
|
|
target-version = "py312"
|
|
|
|
[tool.ruff.lint]
|
|
# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default.
|
|
select = ["E4", "E7", "E9", "F"]
|
|
ignore = [
|
|
"E712", # Comparison to true should be 'if cond is true:' or 'if cond:' (E712)
|
|
"F402", # Import `loader` from line 8 shadowed by loop variable
|
|
"F403", # * used, unable to detect undefined names
|
|
"F405", # may be undefined, or defined from star imports
|
|
"E741", # Ambiguous variable name: `l`
|
|
"E722", # bare-except
|
|
"E721", # type-comparison
|
|
"F821", # undefined-all
|
|
"FURB113", # repeated-append
|
|
"FURB152", # math-constant
|
|
"UP007", # non-pep604-annotation
|
|
"UP032", # f-string
|
|
"UP045", # non-pep604-annotation-optional
|
|
"B005", # strip-with-multi-characters
|
|
"B006", # mutable-argument-default
|
|
"B007", # unused-loop-control-variable
|
|
"B026", # star-arg-unpacking-after-keyword-arg
|
|
"B903", # class-as-data-structure
|
|
"B904", # raise-without-from-inside-except
|
|
"B905", # zip-without-explicit-strict
|
|
"N806", # non-lowercase-variable-in-function
|
|
"N815", # mixed-case-variable-in-class-scope
|
|
"PT011", # pytest-raises-too-broad
|
|
"SIM102", # collapsible-if
|
|
"SIM103", # needless-bool
|
|
"SIM105", # suppressible-exception
|
|
"SIM107", # return-in-try-except-finally
|
|
"SIM108", # if-else-block-instead-of-if-exp
|
|
"SIM113", # enumerate-for-loop
|
|
"SIM117", # multiple-with-statements
|
|
"SIM210", # if-expr-with-true-false
|
|
]
|
|
|
|
# Allow fix for all enabled rules (when `--fix`) is provided.
|
|
fixable = ["ALL"]
|
|
unfixable = []
|
|
|
|
# Allow unused variables when underscore-prefixed.
|
|
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
|
|
|
|
[tool.ruff.format]
|
|
# Like Black, use double quotes for strings.
|
|
quote-style = "single"
|
|
|
|
# Like Black, indent with spaces, rather than tabs.
|
|
indent-style = "space"
|
|
|
|
# Like Black, respect magic trailing commas.
|
|
skip-magic-trailing-comma = false
|
|
|
|
# Like Black, automatically detect the appropriate line ending.
|
|
line-ending = "auto"
|