feat: add supports for testing llm models (#1454)

* feat: add supports for testing llm models

* fix: linter error

pkg/api/http/controller/groups/provider/models.py

@@ -36,3 +36,11 @@ class LLMModelsRouterGroup(group.RouterGroup):
                 await self.ap.model_service.delete_llm_model(model_uuid)
 
                 return self.success()
+
+        @self.route('/<model_uuid>/test', methods=['POST'])
+        async def _(model_uuid: str) -> str:
+            json_data = await quart.request.json
+
+            await self.ap.model_service.test_llm_model(model_uuid, json_data)
+
+            return self.success()

pkg/api/http/service/model.py

@@ -6,6 +6,8 @@ import sqlalchemy
 from ....core import app
 from ....entity.persistence import model as persistence_model
 from ....entity.persistence import pipeline as persistence_pipeline
+from ....provider.modelmgr import requester as model_requester
+from ....provider import entities as llm_entities
 
 
 class ModelsService:
@@ -78,3 +80,26 @@ class ModelsService:
         )
 
         await self.ap.model_mgr.remove_llm_model(model_uuid)
+
+    async def test_llm_model(self, model_uuid: str, model_data: dict) -> None:
+        runtime_llm_model: model_requester.RuntimeLLMModel | None = None
+
+        if model_uuid != '_':
+            for model in self.ap.model_mgr.llm_models:
+                if model.model_entity.uuid == model_uuid:
+                    runtime_llm_model = model
+                    break
+
+            if runtime_llm_model is None:
+                raise Exception('model not found')
+
+        else:
+            runtime_llm_model = await self.ap.model_mgr.init_runtime_llm_model(model_data)
+
+        await runtime_llm_model.requester.invoke_llm(
+            query=None,
+            model=runtime_llm_model,
+            messages=[llm_entities.Message(role='user', content='Hello, world!')],
+            funcs=[],
+            extra_args={},
+        )

pkg/provider/modelmgr/modelmgr.py

@@ -4,9 +4,6 @@ import sqlalchemy
 
 from . import entities, requester
 from ...core import app
-from ...core import entities as core_entities
-from .. import entities as llm_entities
-from ..tools import entities as tools_entities
 from ...discover import engine
 from . import token
 from ...entity.persistence import model as persistence_model
@@ -69,12 +66,11 @@ class ModelManager:
         for llm_model in llm_models:
             await self.load_llm_model(llm_model)
 
-    async def load_llm_model(
+    async def init_runtime_llm_model(
         self,
         model_info: persistence_model.LLMModel | sqlalchemy.Row[persistence_model.LLMModel] | dict,
     ):
-        """加载模型"""
+        """初始化运行时模型"""
-
         if isinstance(model_info, sqlalchemy.Row):
             model_info = persistence_model.LLMModel(**model_info._mapping)
         elif isinstance(model_info, dict):
@@ -92,6 +88,15 @@ class ModelManager:
             ),
             requester=requester_inst,
         )
+
+        return runtime_llm_model
+
+    async def load_llm_model(
+        self,
+        model_info: persistence_model.LLMModel | sqlalchemy.Row[persistence_model.LLMModel] | dict,
+    ):
+        """加载模型"""
+        runtime_llm_model = await self.init_runtime_llm_model(model_info)
         self.llm_models.append(runtime_llm_model)
 
     async def get_model_by_name(self, name: str) -> entities.LLMModelInfo:  # deprecated
@@ -132,12 +137,3 @@ class ModelManager:
             if component.metadata.name == name:
                 return component
         return None
-
-    async def invoke_llm(
-        self,
-        query: core_entities.Query,
-        model_uuid: str,
-        messages: list[llm_entities.Message],
-        funcs: list[tools_entities.LLMFunction] = None,
-    ) -> llm_entities.Message:
-        pass

web/src/app/home/models/component/llm-form/LLMForm.tsx

@@ -130,6 +130,7 @@ export default function LLMForm({
   const [requesterDefaultURLList, setRequesterDefaultURLList] = useState<
     string[]
   >([]);
+  const [modelTesting, setModelTesting] = useState(false);
 
   useEffect(() => {
     initLLMModelFormComponent().then(() => {
@@ -308,6 +309,34 @@ export default function LLMForm({
     }
   }
 
+  function testLLMModelInForm() {
+    setModelTesting(true);
+    httpClient
+      .testLLMModel('_', {
+        uuid: '',
+        name: form.getValues('name'),
+        description: '',
+        requester: form.getValues('model_provider'),
+        requester_config: {
+          base_url: form.getValues('url'),
+          timeout: 120,
+        },
+        api_keys: [form.getValues('api_key')],
+        abilities: form.getValues('abilities'),
+        extra_args: form.getValues('extra_args'),
+      })
+      .then((res) => {
+        console.log(res);
+        toast.success(t('models.testSuccess'));
+      })
+      .catch(() => {
+        toast.error(t('models.testError'));
+      })
+      .finally(() => {
+        setModelTesting(false);
+      });
+  }
+
   return (
     <div>
       <Dialog
@@ -579,6 +608,15 @@ export default function LLMForm({
               {editMode ? t('common.save') : t('common.submit')}
             </Button>
 
+            <Button
+              type="button"
+              variant="outline"
+              onClick={() => testLLMModelInForm()}
+              disabled={modelTesting}
+            >
+              {t('common.test')}
+            </Button>
+
             <Button
               type="button"
               variant="outline"

web/src/app/infra/http/HttpClient.ts

@@ -271,6 +271,10 @@ class HttpClient {
     return this.put(`/api/v1/provider/models/llm/${uuid}`, model);
   }
 
+  public testLLMModel(uuid: string, model: LLMModel): Promise<object> {
+    return this.post(`/api/v1/provider/models/llm/${uuid}/test`, model);
+  }
+
   // ============ Pipeline API ============
   public getGeneralPipelineMetadata(): Promise<GetPipelineMetadataResponseData> {
     // as designed, this method will be deprecated, and only for developer to check the prefered config schema

web/src/i18n/locales/en-US.ts

@@ -37,6 +37,7 @@ const enUS = {
     deleteSuccess: 'Deleted successfully',
     deleteError: 'Delete failed: ',
     addRound: 'Add Round',
+    test: 'Test',
   },
   notFound: {
     title: 'Page not found',
@@ -89,6 +90,8 @@ const enUS = {
     modelProviderDescription:
       'Please fill in the model name provided by the supplier',
     selectModel: 'Select Model',
+    testSuccess: 'Test successful',
+    testError: 'Test failed, please check your model configuration',
   },
   bots: {
     title: 'Bots',

web/src/i18n/locales/zh-Hans.ts

@@ -37,6 +37,7 @@ const zhHans = {
     deleteSuccess: '删除成功',
     deleteError: '删除失败：',
     addRound: '添加回合',
+    test: '测试',
   },
   notFound: {
     title: '页面不存在',
@@ -89,6 +90,8 @@ const zhHans = {
     selectModelProvider: '选择模型供应商',
     modelProviderDescription: '请填写供应商向您提供的模型名称',
     selectModel: '请选择模型',
+    testSuccess: '测试成功',
+    testError: '测试失败，请检查模型配置',
   },
   bots: {
     title: '机器人',