xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-06-02 03:55:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: remove telemetry

Browse Source
This commit is contained in:
Junyan Qin
2025-05-10 16:17:01 +08:00
parent d1f7b93d77
commit 425681ea09
18 changed files with 17 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.pre-commit-config.yaml
View File

@@ -3,7 +3,11 @@ repos:
# Ruff version.
rev: v0.11.7
hooks:
# Run the linter.
# Run the linter of backend.
- id: ruff
# Run the formatter.
# Run the formatter of backend.
- id: ruff-format
# Run the linter of frontend.
- id: eslint
# Run the formatter of frontend.
- id: eslint-format

3
pkg/audit/__init__.py
View File

@@ -1,3 +0,0 @@
"""
审计相关操作
"""

0
pkg/audit/center/__init__.py
View File

86
pkg/audit/center/apigroup.py
View File

@@ -1,86 +0,0 @@
from __future__ import annotations
import abc
import uuid
import json
import asyncio
import aiohttp
from ...core import app, entities as core_entities
class APIGroup(metaclass=abc.ABCMeta):
"""API 组抽象类"""
_basic_info: dict = None
_runtime_info: dict = None
prefix = None
ap: app.Application
def __init__(self, prefix: str, ap: app.Application):
self.prefix = prefix
self.ap = ap
async def _do(
self,
method: str,
path: str,
data: dict = None,
params: dict = None,
headers: dict = {},
**kwargs,
):
"""
执行请求
"""
self._runtime_info['account_id'] = '-1'
url = self.prefix + path
data = json.dumps(data)
headers['Content-Type'] = 'application/json'
try:
async with aiohttp.ClientSession() as session:
async with session.request(
method, url, data=data, params=params, headers=headers, **kwargs
) as resp:
self.ap.logger.debug('data: %s', data)
self.ap.logger.debug('ret: %s', await resp.text())
except Exception as e:
self.ap.logger.debug(f'上报失败: {e}')
async def do(
self,
method: str,
path: str,
data: dict = None,
params: dict = None,
headers: dict = {},
**kwargs,
) -> asyncio.Task:
"""执行请求"""
return self.ap.task_mgr.create_task(
self._do(method, path, data, params, headers, **kwargs),
kind='telemetry-operation',
name=f'{method} {path}',
scopes=[core_entities.LifecycleControlScope.APPLICATION],
).task
def gen_rid(self):
"""生成一个请求 ID"""
return str(uuid.uuid4())
def basic_info(self):
"""获取基本信息"""
basic_info = APIGroup._basic_info.copy()
basic_info['rid'] = self.gen_rid()
return basic_info
def runtime_info(self):
"""获取运行时信息"""
return APIGroup._runtime_info

0
pkg/audit/center/groups/__init__.py
View File

55
pkg/audit/center/groups/main.py
View File

@@ -1,55 +0,0 @@
from __future__ import annotations
from .. import apigroup
from ....core import app
class V2MainDataAPI(apigroup.APIGroup):
"""主程序相关 数据API"""
def __init__(self, prefix: str, ap: app.Application):
self.ap = ap
super().__init__(prefix + '/main', ap)
async def do(self, *args, **kwargs):
if not self.ap.instance_config.data['telemetry']['report']:
return None
return await super().do(*args, **kwargs)
async def post_update_record(
self,
spent_seconds: int,
infer_reason: str,
old_version: str,
new_version: str,
):
"""提交更新记录"""
return await self.do(
'POST',
'/update',
data={
'basic': self.basic_info(),
'update_info': {
'spent_seconds': spent_seconds,
'infer_reason': infer_reason,
'old_version': old_version,
'new_version': new_version,
},
},
)
async def post_announcement_showed(
self,
ids: list[int],
):
"""提交公告已阅"""
return await self.do(
'POST',
'/announcement',
data={
'basic': self.basic_info(),
'announcement_info': {
'ids': ids,
},
},
)

59
pkg/audit/center/groups/plugin.py
View File

@@ -1,59 +0,0 @@
from __future__ import annotations
from ....core import app
from .. import apigroup
class V2PluginDataAPI(apigroup.APIGroup):
"""插件数据相关 API"""
def __init__(self, prefix: str, ap: app.Application):
self.ap = ap
super().__init__(prefix + '/plugin', ap)
async def do(self, *args, **kwargs):
if not self.ap.instance_config.data['telemetry']['report']:
return None
return await super().do(*args, **kwargs)
async def post_install_record(self, plugin: dict):
"""提交插件安装记录"""
return await self.do(
'POST',
'/install',
data={
'basic': self.basic_info(),
'plugin': plugin,
},
)
async def post_remove_record(self, plugin: dict):
"""提交插件卸载记录"""
return await self.do(
'POST',
'/remove',
data={
'basic': self.basic_info(),
'plugin': plugin,
},
)
async def post_update_record(
self,
plugin: dict,
old_version: str,
new_version: str,
):
"""提交插件更新记录"""
return await self.do(
'POST',
'/update',
data={
'basic': self.basic_info(),
'plugin': plugin,
'update_info': {
'old_version': old_version,
'new_version': new_version,
},
},
)

87
pkg/audit/center/groups/usage.py
View File

@@ -1,87 +0,0 @@
from __future__ import annotations
from .. import apigroup
from ....core import app
class V2UsageDataAPI(apigroup.APIGroup):
"""使用量数据相关 API"""
def __init__(self, prefix: str, ap: app.Application):
self.ap = ap
super().__init__(prefix + '/usage', ap)
async def do(self, *args, **kwargs):
if not self.ap.instance_config.data['telemetry']['report']:
return None
return await super().do(*args, **kwargs)
async def post_query_record(
self,
session_type: str,
session_id: str,
query_ability_provider: str,
usage: int,
model_name: str,
response_seconds: int,
retry_times: int,
):
"""提交请求记录"""
return await self.do(
'POST',
'/query',
data={
'basic': self.basic_info(),
'runtime': self.runtime_info(),
'session_info': {
'type': session_type,
'id': session_id,
},
'query_info': {
'ability_provider': query_ability_provider,
'usage': usage,
'model_name': model_name,
'response_seconds': response_seconds,
'retry_times': retry_times,
},
},
)
async def post_event_record(
self,
plugins: list[dict],
event_name: str,
):
"""提交事件触发记录"""
return await self.do(
'POST',
'/event',
data={
'basic': self.basic_info(),
'runtime': self.runtime_info(),
'plugins': plugins,
'event_info': {
'name': event_name,
},
},
)
async def post_function_record(
self,
plugin: dict,
function_name: str,
function_description: str,
):
"""提交内容函数使用记录"""
return await self.do(
'POST',
'/function',
data={
'basic': self.basic_info(),
'plugin': plugin,
'function_info': {
'name': function_name,
'description': function_description,
},
},
)

40
pkg/audit/center/v2.py
View File

@@ -1,40 +0,0 @@
from __future__ import annotations
import logging
from . import apigroup
from .groups import main
from .groups import usage
from .groups import plugin
from ...core import app
class V2CenterAPI:
"""中央服务器 v2 API 交互类"""
main: main.V2MainDataAPI = None
"""主 API 组"""
usage: usage.V2UsageDataAPI = None
"""使用量 API 组"""
plugin: plugin.V2PluginDataAPI = None
"""插件 API 组"""
def __init__(
self,
ap: app.Application,
backend_url: str,
basic_info: dict = None,
runtime_info: dict = None,
):
"""初始化"""
logging.debug('basic_info: %s, runtime_info: %s', basic_info, runtime_info)
apigroup.APIGroup._basic_info = basic_info
apigroup.APIGroup._runtime_info = runtime_info
self.main = main.V2MainDataAPI(backend_url, ap)
self.usage = usage.V2UsageDataAPI(backend_url, ap)
self.plugin = plugin.V2PluginDataAPI(backend_url, ap)

89
pkg/audit/identifier.py
View File

@@ -1,89 +0,0 @@
# 实例 识别码 控制
import os
import uuid
import json
import time
identifier = {
'host_id': '',
'instance_id': '',
'host_create_ts': 0,
'instance_create_ts': 0,
}
HOST_ID_FILE = os.path.expanduser('~/.langbot/host_id.json')
INSTANCE_ID_FILE = 'data/labels/instance_id.json'
def init():
global identifier
if not os.path.exists(os.path.expanduser('~/.langbot')):
os.mkdir(os.path.expanduser('~/.langbot'))
if not os.path.exists(HOST_ID_FILE):
new_host_id = 'host_' + str(uuid.uuid4())
new_host_create_ts = int(time.time())
with open(HOST_ID_FILE, 'w') as f:
json.dump({'host_id': new_host_id, 'host_create_ts': new_host_create_ts}, f)
identifier['host_id'] = new_host_id
identifier['host_create_ts'] = new_host_create_ts
else:
loaded_host_id = ''
loaded_host_create_ts = 0
with open(HOST_ID_FILE, 'r') as f:
file_content = json.load(f)
loaded_host_id = file_content['host_id']
loaded_host_create_ts = file_content['host_create_ts']
identifier['host_id'] = loaded_host_id
identifier['host_create_ts'] = loaded_host_create_ts
# 检查实例 id
if os.path.exists(INSTANCE_ID_FILE):
instance_id = {}
with open(INSTANCE_ID_FILE, 'r') as f:
instance_id = json.load(f)
if (
instance_id['host_id'] != identifier['host_id']
): # 如果实例 id 不是当前主机的，删除
os.remove(INSTANCE_ID_FILE)
if not os.path.exists(INSTANCE_ID_FILE):
new_instance_id = 'instance_' + str(uuid.uuid4())
new_instance_create_ts = int(time.time())
with open(INSTANCE_ID_FILE, 'w') as f:
json.dump(
{
'host_id': identifier['host_id'],
'instance_id': new_instance_id,
'instance_create_ts': new_instance_create_ts,
},
f,
)
identifier['instance_id'] = new_instance_id
identifier['instance_create_ts'] = new_instance_create_ts
else:
loaded_instance_id = ''
loaded_instance_create_ts = 0
with open(INSTANCE_ID_FILE, 'r') as f:
file_content = json.load(f)
loaded_instance_id = file_content['instance_id']
loaded_instance_create_ts = file_content['instance_create_ts']
identifier['instance_id'] = loaded_instance_id
identifier['instance_create_ts'] = loaded_instance_create_ts
def print_out():
global identifier
print(identifier)

3
pkg/core/app.py
View File

@@ -11,7 +11,6 @@ from ..provider.session import sessionmgr as llm_session_mgr
from ..provider.modelmgr import modelmgr as llm_model_mgr
from ..provider.tools import toolmgr as llm_tool_mgr
from ..config import manager as config_mgr
from ..audit.center import v2 as center_mgr
from ..command import cmdmgr
from ..plugin import manager as plugin_mgr
from ..pipeline import pool
@@ -77,8 +76,6 @@ class Application:
# =========================
ctr_mgr: center_mgr.V2CenterAPI = None
plugin_mgr: plugin_mgr.PluginManager = None
query_pool: pool.QueryPool = None

3
pkg/core/boot.py
View File

@@ -5,7 +5,6 @@ import asyncio
import os
from . import app
from ..audit import identifier
from . import stage
from ..utils import constants, importutil
@@ -26,8 +25,6 @@ stage_order = [
async def make_app(loop: asyncio.AbstractEventLoop) -> app.Application:
# 生成标识符
identifier.init()
# 确定是否为调试模式
if 'DEBUG' in os.environ and os.environ['DEBUG'] in ['true', '1']:

20
pkg/core/stages/build_app.py
View File

@@ -2,9 +2,7 @@ from __future__ import annotations
from .. import stage, app
from ...utils import version, proxy, announce, platform
from ...audit.center import v2 as center_v2
from ...audit import identifier
from ...utils import version, proxy, announce
from ...pipeline import pool, controller, pipelinemgr
from ...plugin import manager as plugin_mgr
from ...command import cmdmgr
@@ -43,22 +41,6 @@ class BuildAppStage(stage.BootingStage):
await ver_mgr.initialize()
ap.ver_mgr = ver_mgr
center_v2_api = center_v2.V2CenterAPI(
ap,
backend_url=ap.instance_config.data['telemetry']['url'],
basic_info={
'host_id': identifier.identifier['host_id'],
'instance_id': identifier.identifier['instance_id'],
'semantic_version': ver_mgr.get_current_version(),
'platform': platform.get_platform(),
},
runtime_info={
'admin_id': '{}'.format(ap.instance_config.data['admins']),
'msg_source': str([]),
},
)
ap.ctr_mgr = center_v2_api
# 发送公告
ann_mgr = announce.AnnouncementManager(ap)
ap.ann_mgr = ann_mgr

14
pkg/pipeline/process/handlers/chat.py
View File

@@ -1,7 +1,6 @@
from __future__ import annotations
import typing
import time
import traceback
@@ -64,8 +63,6 @@ class ChatMessageHandler(handler.MessageHandler):
text_length = 0
start_time = time.time()
try:
for r in runner_module.preregistered_runners:
if r.name == query.pipeline_config['ai']['runner']['runner']:
@@ -109,12 +106,5 @@ class ChatMessageHandler(handler.MessageHandler):
debug_notice=traceback.format_exc(),
)
finally:
await self.ap.ctr_mgr.usage.post_query_record(
session_type=query.session.launcher_type.value,
session_id=str(query.session.launcher_id),
query_ability_provider='LangBot.Chat',
usage=text_length,
model_name=query.use_model.name,
response_seconds=int(time.time() - start_time),
retry_times=-1,
)
# TODO statistics
pass

46
pkg/plugin/manager.py
View File

@@ -194,14 +194,7 @@ class PluginManager:
"""安装插件"""
await self.installer.install_plugin(plugin_source, task_context)
await self.ap.ctr_mgr.plugin.post_install_record(
{
'name': 'unknown',
'remote': plugin_source,
'author': 'unknown',
'version': 'HEAD',
}
)
# TODO statistics
task_context.trace('重载插件..', 'reload-plugin')
await self.ap.reload(scope='plugin')
@@ -221,14 +214,7 @@ class PluginManager:
await self.destroy_plugin(plugin_container)
await self.installer.uninstall_plugin(plugin_name, task_context)
await self.ap.ctr_mgr.plugin.post_remove_record(
{
'name': plugin_name,
'remote': plugin_container.plugin_repository,
'author': plugin_container.plugin_author,
'version': plugin_container.plugin_version,
}
)
# TODO statistics
task_context.trace('重载插件..', 'reload-plugin')
await self.ap.reload(scope='plugin')
@@ -242,18 +228,7 @@ class PluginManager:
"""更新插件"""
await self.installer.update_plugin(plugin_name, plugin_source, task_context)
plugin_container = self.get_plugin_by_name(plugin_name)
await self.ap.ctr_mgr.plugin.post_update_record(
plugin={
'name': plugin_name,
'remote': plugin_container.plugin_repository,
'author': plugin_container.plugin_author,
'version': plugin_container.plugin_version,
},
old_version=plugin_container.plugin_version,
new_version='HEAD',
)
# TODO statistics
task_context.trace('重载插件..', 'reload-plugin')
await self.ap.reload(scope='plugin')
@@ -313,20 +288,7 @@ class PluginManager:
f'事件 {event.__class__.__name__}({ctx.eid}) 处理完成，返回值 {ctx.__return_value__}'
)
if emitted_plugins:
plugins_info: list[dict] = [
{
'name': plugin.plugin_name,
'remote': plugin.plugin_repository,
'version': plugin.plugin_version,
'author': plugin.plugin_author,
}
for plugin in emitted_plugins
]
await self.ap.ctr_mgr.usage.post_event_record(
plugins=plugins_info, event_name=event.__class__.__name__
)
# TODO statistics
return ctx

12
pkg/provider/tools/loaders/plugin.py
View File

@@ -73,17 +73,7 @@ class PluginToolLoader(loader.ToolLoader):
plugin = p
break
if plugin is not None:
await self.ap.ctr_mgr.usage.post_function_record(
plugin={
'name': plugin.plugin_name,
'remote': plugin.plugin_repository,
'version': plugin.plugin_version,
'author': plugin.plugin_author,
},
function_name=function.name,
function_description=function.description,
)
# TODO statistics
async def shutdown(self):
"""关闭工具"""

5
pkg/utils/announce.py
View File

@@ -108,10 +108,7 @@ class AnnouncementManager:
for ann in announcements:
ann_text += f'[公告] {ann.time}: {ann.content}\n'
if announcements:
await self.ap.ctr_mgr.main.post_announcement_showed(
ids=[item.id for item in announcements]
)
# TODO statistics
return ann_text, logging.INFO
except Exception as e:

10
pkg/utils/version.py
View File

@@ -3,7 +3,6 @@ from __future__ import annotations
import os
import typing
import logging
import time
import requests
@@ -41,10 +40,8 @@ class VersionManager:
async def update_all(self):
"""检查更新并下载源码"""
start_time = time.time()
current_tag = self.get_current_version()
old_tag = current_tag
rls_list = await self.get_release_list()
@@ -141,12 +138,7 @@ class VersionManager:
with open('current_tag', 'w') as f:
f.write(current_tag)
await self.ap.ctr_mgr.main.post_update_record(
spent_seconds=int(time.time() - start_time),
infer_reason='update',
old_version=old_tag,
new_version=current_tag,
)
# TODO statistics
async def is_new_version_available(self) -> bool:
"""检查是否有新版本"""