From 601c6975ea1937d7069ee8d720bb1ad70e960f19 Mon Sep 17 00:00:00 2001
From: CWT <127104378+zx90316@users.noreply.github.com>
Date: Tue, 1 Sep 2026 22:36:35 +0800
Subject: [PATCH] fix(ollama): use litellm's ollama_chat provider for native
 tool-calling (#2494)

The Ollama requester declared litellm_provider: ollama, which routes
every request through litellm's legacy /api/generate-based
OllamaConfig. That config's get_supported_openai_params() does not
include "tools"/"tool_choice" at all, so an Ollama-hosted model in a
local-agent pipeline could never receive a structured tool definition
or return a structured tool_calls response - it could only try to
express a tool call as free text (typically inside its own <think>
reasoning), which LangBot then has no way to execute.

litellm's "ollama_chat" provider targets Ollama's modern /api/chat
endpoint instead, which correctly forwards tools/tool_choice and
correctly surfaces the model's native message.tool_calls field.
Verified against a real local Ollama 0.33.2 instance with the exact
system prompt, RAG-augmented user message, and tool set a live
pipeline sends.

Two follow-on fixes needed because the Ollama requester definition is
shared by LLM and text-embedding models:

- get_reasoning_capabilities: match family in ('ollama', 'ollama_chat')
  so the reasoning-level UI still works for this provider.
- scan_models: retry {base_url}/v1/models on a 404 from {base_url}/models,
  since Ollama's base_url is a bare host (must not include /v1 - that
  would break OllamaChatConfig.get_complete_url, which appends /api/chat
  to it directly), unlike most other OpenAI-compatible providers whose
  base_url already ends in /v1.
- invoke_embedding: litellm's embedding routing has no "ollama_chat"
  case, only "ollama". Build the embedding model name with an explicit
  custom_llm_provider="ollama" override when the requester is configured
  for ollama_chat, so embedding models (e.g. bge-m3) keep working.

Co-authored-by: zx90316 <zx90316@users.noreply.github.com>
---
 .../provider/modelmgr/requesters/litellmchat.py | 17 +++++++++++++++--
 .../modelmgr/requesters/ollamachat.yaml         |  2 +-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py
index e33f7fd03..138199072 100644
--- a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py
+++ b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py
@@ -573,7 +573,7 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
             levels = ['provider_default', 'disabled', 'enabled']
         elif family == 'doubao':
             levels = ['provider_default', 'disabled', 'low', 'medium', 'high']
-        elif family == 'ollama':
+        elif family in ('ollama', 'ollama_chat'):
             levels = ['provider_default']
             levels.append('disabled')
             if normalized_name.startswith('gpt-oss') or '/gpt-oss' in normalized_name:
@@ -1345,7 +1345,14 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
         extra_args: dict[str, typing.Any] = {},
     ) -> tuple[list[list[float]], dict]:
         """Invoke embedding and return vectors with usage info."""
-        model_name = self._build_litellm_model_name(model.model_entity.name)
+        # litellm's embedding routing has no "ollama_chat" branch (that provider
+        # exists only for /api/chat completions) — embeddings still go through
+        # the plain "ollama" provider. Requesters configured for ollama_chat
+        # (to get native tool-calling on the chat path) must fall back to
+        # "ollama" here specifically, or embedding calls raise "Unmapped LLM
+        # provider for this endpoint".
+        embedding_provider = 'ollama' if self._get_custom_llm_provider() == 'ollama_chat' else None
+        model_name = self._build_litellm_model_name(model.model_entity.name, embedding_provider)
         api_key = model.provider.token_mgr.get_token()
 
         args = {
@@ -1541,6 +1548,12 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
                 event_hooks=httpclient.httpx_response_limit_hooks(),
             ) as client:
                 response = await client.get(models_url, headers=headers)
+                if response.status_code == 404 and not base_url.rstrip('/').endswith('/v1'):
+                    # Some OpenAI-compatible servers (notably a bare Ollama host,
+                    # e.g. http://host:11434) expose the model list under /v1/models
+                    # rather than /models. Providers whose configured base_url
+                    # already ends in /v1 keep their original (working) URL.
+                    response = await client.get(f'{base_url}/v1/models', headers=headers)
                 response.raise_for_status()
                 payload = await httpclient.parse_json_response(response)
 
diff --git a/src/langbot/pkg/provider/modelmgr/requesters/ollamachat.yaml b/src/langbot/pkg/provider/modelmgr/requesters/ollamachat.yaml
index 83e116c8f..cf9441af9 100644
--- a/src/langbot/pkg/provider/modelmgr/requesters/ollamachat.yaml
+++ b/src/langbot/pkg/provider/modelmgr/requesters/ollamachat.yaml
@@ -7,7 +7,7 @@ metadata:
     zh_Hans: Ollama
   icon: ollama.svg
 spec:
-  litellm_provider: ollama
+  litellm_provider: ollama_chat
   config:
   - name: base_url
     label: