xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-10-23 17:05:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(llm): harden request policy and optimize connections, reasoning and prompt caching (#2661)

Browse Source 
* fix(llm): centralize request policy and preserve stream diagnostics

* test(llm): dispose SQLite engine after migration regression

* perf(llm): reuse connections and unify reasoning and prompt cache policy

* fix(llm): preserve explicit native prompt cache options

* perf(llm): fix measured TLS setup and streaming reuse overhead

* fix(llm): resolve plugin request-chain regressions found by E2E

---------

Co-authored-by: Hyu <[email protected]>
This commit is contained in:
RockChinQ
2026-10-15 19:48:35 +08:00
committed by GitHub
co-authored by Hyu
parent 502b38130f
commit 616981b0cf
30 changed files with 4556 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/langbot/pkg/plugin/handler.py
+13 -3
View File
@@ -443,13 +443,19 @@ def _resolve_remove_think(data: dict[str, Any], query: Any | None) -> bool:
return False
def _merge_model_extra_args(model: Any, call_extra_args: Any) -> dict[str, Any]:
def _merge_model_extra_args(model: Any, call_extra_args: Any, *, llm_policy: bool = False) -> dict[str, Any]:
"""Merge persisted model extra_args with action-level overrides."""
merged: dict[str, Any] = {}
model_extra_args = getattr(getattr(model, 'model_entity', None), 'extra_args', None)
if isinstance(model_extra_args, dict):
merged.update(model_extra_args)
if llm_policy:
# RuntimeProvider resolves model and call policy layers independently.
# Flattening them here makes a model request_policy.retries override a
# caller's legacy num_retries=0 (and similarly hides timeout overrides).
for key in ('request_policy', 'num_retries', 'max_retries', 'timeout'):
merged.pop(key, None)
if isinstance(call_extra_args, dict):
merged.update(call_extra_args)
@@ -1484,7 +1490,9 @@ class RuntimeConnectionHandler(handler.Handler):
funcs_obj = [resource_tool.LLMTool.model_validate({**func, 'func': _placeholder_func}) for func in funcs]
query = _resolve_action_query(data, session, self.ap, action_context)
effective_extra_args = _merge_model_extra_args(llm_model, extra_args)
if query is None:
query = _build_plugin_action_query(action_context)
effective_extra_args = _merge_model_extra_args(llm_model, extra_args, llm_policy=True)
remove_think = _resolve_remove_think(data, query)
effective_funcs = funcs_obj if 'func_call' in (llm_model.model_entity.abilities or []) else []
@@ -1580,7 +1588,9 @@ class RuntimeConnectionHandler(handler.Handler):
funcs_obj = [resource_tool.LLMTool.model_validate({**func, 'func': _placeholder_func}) for func in funcs]
query = _resolve_action_query(data, session, self.ap, action_context)
effective_extra_args = _merge_model_extra_args(llm_model, extra_args)
if query is None:
query = _build_plugin_action_query(action_context)
effective_extra_args = _merge_model_extra_args(llm_model, extra_args, llm_policy=True)
remove_think = _resolve_remove_think(data, query)
effective_funcs = funcs_obj if 'func_call' in (llm_model.model_entity.abilities or []) else []