refactor: 重构插件系统

pkg/provider/requester/tokenizer.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import abc
+import typing
+
+from ...core import app
+from .. import entities as llm_entities
+from . import entities
+
+
+class LLMTokenizer(metaclass=abc.ABCMeta):
+
+    ap: app.Application
+
+    def __init__(self, ap: app.Application):
+        self.ap = ap
+
+    async def initialize(self):
+        """初始化分词器
+        """
+        pass
+
+    @abc.abstractmethod
+    async def count_token(
+        self,
+        messages: list[llm_entities.Message],
+        model: entities.LLMModelInfo
+    ) -> int:
+        pass