mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-06-09 07:16:04 +00:00
chore: Add PyPI package support for uvx/pip installation (#1764)
* Initial plan * Add package structure and resource path utilities - Created langbot/ package with __init__.py and __main__.py entry point - Added paths utility to find frontend and resource files from package installation - Updated config loading to use resource paths - Updated frontend serving to use resource paths - Added MANIFEST.in for package data inclusion - Updated pyproject.toml with build system and entry points Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> * Add PyPI publishing workflow and update license - Created GitHub Actions workflow to build frontend and publish to PyPI - Added license field to pyproject.toml to fix deprecation warning - Updated .gitignore to exclude build artifacts - Tested package building successfully Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> * Add PyPI installation documentation - Created PYPI_INSTALLATION.md with detailed installation and usage instructions - Updated README.md to feature uvx/pip installation as recommended method - Updated README_EN.md with same changes for English documentation Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> * Address code review feedback - Made package-data configuration more specific to langbot package only - Improved path detection with caching to avoid repeated file I/O - Removed sys.path searching which was incorrect for package data - Removed interactive input() call for non-interactive environment compatibility - Simplified error messages for version check Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> * Fix code review issues - Use specific exception types instead of bare except - Fix misleading comments about directory levels - Remove redundant existence check before makedirs with exist_ok=True - Use context manager for file opening to ensure proper cleanup Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> * Simplify package configuration and document behavioral differences - Removed redundant package-data configuration, relying on MANIFEST.in - Added documentation about behavioral differences between package and source installation - Clarified that include-package-data=true uses MANIFEST.in for data files Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> * chore: update pyproject.toml * chore: try pack templates in langbot/ * chore: update * chore: update * chore: update * chore: update * chore: update * chore: adjust dir structure * chore: fix imports * fix: read default-pipeline-config.json * fix: read default-pipeline-config.json * fix: tests * ci: publish pypi * chore: bump version 4.6.0-beta.1 for testing * chore: add templates/** * fix: send adapters and requesters icons * chore: bump version 4.6.0b2 for testing * chore: add platform field for docker-compose.yaml --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com> Co-authored-by: Junyan Qin <rockchinq@gmail.com>
This commit is contained in:
Copilot
0
src/langbot/pkg/persistence/__init__.py
Normal file
0
src/langbot/pkg/persistence/__init__.py
Normal file
41
src/langbot/pkg/persistence/database.py
Normal file
41
src/langbot/pkg/persistence/database.py
Normal file
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from ..core import app
|
||||
|
||||
|
||||
preregistered_managers: list[type[BaseDatabaseManager]] = []
|
||||
|
||||
|
||||
def manager_class(name: str) -> None:
|
||||
"""Register a database manager class"""
|
||||
|
||||
def decorator(cls: type[BaseDatabaseManager]) -> type[BaseDatabaseManager]:
|
||||
cls.name = name
|
||||
preregistered_managers.append(cls)
|
||||
return cls
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
class BaseDatabaseManager(abc.ABC):
|
||||
"""Base database manager class"""
|
||||
|
||||
name: str
|
||||
|
||||
ap: app.Application
|
||||
|
||||
engine: sqlalchemy_asyncio.AsyncEngine
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
@abc.abstractmethod
|
||||
async def initialize(self) -> None:
|
||||
pass
|
||||
|
||||
def get_engine(self) -> sqlalchemy_asyncio.AsyncEngine:
|
||||
return self.engine
|
||||
0
src/langbot/pkg/persistence/databases/__init__.py
Normal file
0
src/langbot/pkg/persistence/databases/__init__.py
Normal file
21
src/langbot/pkg/persistence/databases/postgresql.py
Normal file
21
src/langbot/pkg/persistence/databases/postgresql.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from .. import database
|
||||
|
||||
|
||||
@database.manager_class('postgresql')
|
||||
class PostgreSQLDatabaseManager(database.BaseDatabaseManager):
|
||||
"""PostgreSQL database manager"""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
|
||||
|
||||
host = postgresql_config.get('host', '127.0.0.1')
|
||||
port = postgresql_config.get('port', 5432)
|
||||
user = postgresql_config.get('user', 'postgres')
|
||||
password = postgresql_config.get('password', 'postgres')
|
||||
database = postgresql_config.get('database', 'postgres')
|
||||
engine_url = f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{database}'
|
||||
self.engine = sqlalchemy_asyncio.create_async_engine(engine_url)
|
||||
15
src/langbot/pkg/persistence/databases/sqlite.py
Normal file
15
src/langbot/pkg/persistence/databases/sqlite.py
Normal file
@@ -0,0 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from .. import database
|
||||
|
||||
|
||||
@database.manager_class('sqlite')
|
||||
class SQLiteDatabaseManager(database.BaseDatabaseManager):
|
||||
"""SQLite database manager"""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
db_file_path = self.ap.instance_config.data.get('database', {}).get('sqlite', {}).get('path', 'data/langbot.db')
|
||||
engine_url = f'sqlite+aiosqlite:///{db_file_path}'
|
||||
self.engine = sqlalchemy_asyncio.create_async_engine(engine_url)
|
||||
146
src/langbot/pkg/persistence/mgr.py
Normal file
146
src/langbot/pkg/persistence/mgr.py
Normal file
@@ -0,0 +1,146 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import typing
|
||||
import json
|
||||
import uuid
|
||||
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
import sqlalchemy
|
||||
|
||||
from . import database, migration
|
||||
from ..entity.persistence import base, pipeline, metadata
|
||||
from ..entity import persistence
|
||||
from ..core import app
|
||||
from ..utils import constants, importutil
|
||||
from ..api.http.service import pipeline as pipeline_service
|
||||
from . import databases, migrations
|
||||
|
||||
importutil.import_modules_in_pkg(databases)
|
||||
importutil.import_modules_in_pkg(migrations)
|
||||
importutil.import_modules_in_pkg(persistence)
|
||||
|
||||
|
||||
class PersistenceManager:
|
||||
"""Persistence module manager"""
|
||||
|
||||
ap: app.Application
|
||||
|
||||
db: database.BaseDatabaseManager
|
||||
"""Database manager"""
|
||||
|
||||
meta: sqlalchemy.MetaData
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
self.meta = base.Base.metadata
|
||||
|
||||
async def initialize(self):
|
||||
database_type = self.ap.instance_config.data.get('database', {}).get('use', 'sqlite')
|
||||
self.ap.logger.info(f'Initializing database type: {database_type}...')
|
||||
for manager in database.preregistered_managers:
|
||||
if manager.name == database_type:
|
||||
self.db = manager(self.ap)
|
||||
await self.db.initialize()
|
||||
break
|
||||
|
||||
await self.create_tables()
|
||||
|
||||
# run migrations
|
||||
database_version = await self.execute_async(
|
||||
sqlalchemy.select(metadata.Metadata).where(metadata.Metadata.key == 'database_version')
|
||||
)
|
||||
|
||||
database_version = int(database_version.fetchone()[1])
|
||||
required_database_version = constants.required_database_version
|
||||
|
||||
if database_version < required_database_version:
|
||||
migrations = migration.preregistered_db_migrations
|
||||
migrations.sort(key=lambda x: x.number)
|
||||
|
||||
last_migration_number = database_version
|
||||
|
||||
for migration_cls in migrations:
|
||||
migration_instance = migration_cls(self.ap)
|
||||
|
||||
if (
|
||||
migration_instance.number > database_version
|
||||
and migration_instance.number <= required_database_version
|
||||
):
|
||||
await migration_instance.upgrade()
|
||||
await self.execute_async(
|
||||
sqlalchemy.update(metadata.Metadata)
|
||||
.where(metadata.Metadata.key == 'database_version')
|
||||
.values({'value': str(migration_instance.number)})
|
||||
)
|
||||
last_migration_number = migration_instance.number
|
||||
self.ap.logger.info(f'Migration {migration_instance.number} completed.')
|
||||
|
||||
self.ap.logger.info(f'Successfully upgraded database to version {last_migration_number}.')
|
||||
|
||||
await self.write_default_pipeline()
|
||||
|
||||
async def create_tables(self):
|
||||
# create tables
|
||||
async with self.get_db_engine().connect() as conn:
|
||||
await conn.run_sync(self.meta.create_all)
|
||||
|
||||
await conn.commit()
|
||||
|
||||
# ======= write initial data =======
|
||||
|
||||
# write initial metadata
|
||||
self.ap.logger.info('Creating initial metadata...')
|
||||
for item in metadata.initial_metadata:
|
||||
# check if the item exists
|
||||
result = await self.execute_async(
|
||||
sqlalchemy.select(metadata.Metadata).where(metadata.Metadata.key == item['key'])
|
||||
)
|
||||
row = result.first()
|
||||
if row is None:
|
||||
await self.execute_async(sqlalchemy.insert(metadata.Metadata).values(item))
|
||||
|
||||
async def write_default_pipeline(self):
|
||||
# write default pipeline
|
||||
result = await self.execute_async(sqlalchemy.select(pipeline.LegacyPipeline))
|
||||
default_pipeline_uuid = None
|
||||
if result.first() is None:
|
||||
self.ap.logger.info('Creating default pipeline...')
|
||||
|
||||
pipeline_config = json.loads(importutil.read_resource_file('templates/default-pipeline-config.json'))
|
||||
|
||||
default_pipeline_uuid = str(uuid.uuid4())
|
||||
pipeline_data = {
|
||||
'uuid': default_pipeline_uuid,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
'stages': pipeline_service.default_stage_order,
|
||||
'is_default': True,
|
||||
'name': 'ChatPipeline',
|
||||
'description': 'Default pipeline, new bots will be bound to this pipeline | 默认提供的流水线，您配置的机器人将自动绑定到此流水线',
|
||||
'config': pipeline_config,
|
||||
'extensions_preferences': {},
|
||||
}
|
||||
|
||||
await self.execute_async(sqlalchemy.insert(pipeline.LegacyPipeline).values(pipeline_data))
|
||||
|
||||
# =================================
|
||||
|
||||
async def execute_async(self, *args, **kwargs) -> sqlalchemy.engine.cursor.CursorResult:
|
||||
async with self.get_db_engine().connect() as conn:
|
||||
result = await conn.execute(*args, **kwargs)
|
||||
await conn.commit()
|
||||
return result
|
||||
|
||||
def get_db_engine(self) -> sqlalchemy_asyncio.AsyncEngine:
|
||||
return self.db.get_engine()
|
||||
|
||||
def serialize_model(
|
||||
self, model: typing.Type[sqlalchemy.Base], data: sqlalchemy.Base, masked_columns: list[str] = []
|
||||
) -> dict:
|
||||
return {
|
||||
column.name: getattr(data, column.name)
|
||||
if not isinstance(getattr(data, column.name), (datetime.datetime))
|
||||
else getattr(data, column.name).isoformat()
|
||||
for column in model.__table__.columns
|
||||
if column.name not in masked_columns
|
||||
}
|
||||
40
src/langbot/pkg/persistence/migration.py
Normal file
40
src/langbot/pkg/persistence/migration.py
Normal file
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
import abc
|
||||
|
||||
from ..core import app
|
||||
|
||||
|
||||
preregistered_db_migrations: list[typing.Type[DBMigration]] = []
|
||||
|
||||
|
||||
def migration_class(number: int):
|
||||
"""Migration class decorator"""
|
||||
|
||||
def wrapper(cls: typing.Type[DBMigration]) -> typing.Type[DBMigration]:
|
||||
cls.number = number
|
||||
preregistered_db_migrations.append(cls)
|
||||
return cls
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class DBMigration(abc.ABC):
|
||||
"""Database migration"""
|
||||
|
||||
number: int
|
||||
"""Migration number"""
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
|
||||
@abc.abstractmethod
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
0
src/langbot/pkg/persistence/migrations/__init__.py
Normal file
0
src/langbot/pkg/persistence/migrations/__init__.py
Normal file
@@ -0,0 +1,252 @@
|
||||
from .. import migration
|
||||
from copy import deepcopy
|
||||
import uuid
|
||||
import os
|
||||
import sqlalchemy
|
||||
import shutil
|
||||
|
||||
from ...config import manager as config_manager
|
||||
from ...entity.persistence import (
|
||||
model as persistence_model,
|
||||
pipeline as persistence_pipeline,
|
||||
bot as persistence_bot,
|
||||
)
|
||||
|
||||
|
||||
@migration.migration_class(1)
|
||||
class DBMigrateV3Config(migration.DBMigration):
|
||||
"""Migrate v3 config to v4 database"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
"""
|
||||
Migrate all config files under data/config.
|
||||
After migration, all previous config files are saved under data/legacy/config.
|
||||
After migration, all config files under data/metadata/ are saved under data/legacy/metadata.
|
||||
"""
|
||||
|
||||
if self.ap.provider_cfg is None:
|
||||
return
|
||||
|
||||
# ======= Migrate model =======
|
||||
# Only migrate the currently selected model
|
||||
model_name = self.ap.provider_cfg.data.get('model', 'gpt-4o')
|
||||
|
||||
model_requester = 'openai-chat-completions'
|
||||
model_requester_config = {}
|
||||
model_api_keys = ['sk-proj-1234567890']
|
||||
model_abilities = []
|
||||
model_extra_args = {}
|
||||
|
||||
if os.path.exists('data/metadata/llm-models.json'):
|
||||
_llm_model_meta = await config_manager.load_json_config('data/metadata/llm-models.json', completion=False)
|
||||
|
||||
for item in _llm_model_meta.data.get('list', []):
|
||||
if item.get('name') == model_name:
|
||||
if 'model_name' in item:
|
||||
model_name = item['model_name']
|
||||
if 'requester' in item:
|
||||
model_requester = item['requester']
|
||||
if 'token_mgr' in item:
|
||||
_token_mgr = item['token_mgr']
|
||||
|
||||
if _token_mgr in self.ap.provider_cfg.data.get('keys', {}):
|
||||
model_api_keys = self.ap.provider_cfg.data.get('keys', {})[_token_mgr]
|
||||
|
||||
if 'tool_call_supported' in item and item['tool_call_supported']:
|
||||
model_abilities.append('func_call')
|
||||
|
||||
if 'vision_supported' in item and item['vision_supported']:
|
||||
model_abilities.append('vision')
|
||||
|
||||
if (
|
||||
model_requester in self.ap.provider_cfg.data.get('requester', {})
|
||||
and 'args' in self.ap.provider_cfg.data.get('requester', {})[model_requester]
|
||||
):
|
||||
model_extra_args = self.ap.provider_cfg.data.get('requester', {})[model_requester]['args']
|
||||
|
||||
if model_requester in self.ap.provider_cfg.data.get('requester', {}):
|
||||
model_requester_config = self.ap.provider_cfg.data.get('requester', {})[model_requester]
|
||||
model_requester_config = {
|
||||
'base_url': model_requester_config['base-url'],
|
||||
'timeout': model_requester_config['timeout'],
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
model_uuid = str(uuid.uuid4())
|
||||
|
||||
llm_model_data = {
|
||||
'uuid': model_uuid,
|
||||
'name': model_name,
|
||||
'description': '由 LangBot v3 迁移而来',
|
||||
'requester': model_requester,
|
||||
'requester_config': model_requester_config,
|
||||
'api_keys': model_api_keys,
|
||||
'abilities': model_abilities,
|
||||
'extra_args': model_extra_args,
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.LLMModel).values(**llm_model_data)
|
||||
)
|
||||
|
||||
# ======= Migrate pipeline config =======
|
||||
# Modify to default pipeline
|
||||
default_pipeline = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
for pipeline in (
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
)
|
||||
)
|
||||
).all()
|
||||
][0]
|
||||
|
||||
pipeline_uuid = str(uuid.uuid4())
|
||||
pipeline_name = 'ChatPipeline'
|
||||
|
||||
if default_pipeline:
|
||||
pipeline_name = default_pipeline['name']
|
||||
pipeline_uuid = default_pipeline['uuid']
|
||||
|
||||
pipeline_config = default_pipeline['config']
|
||||
|
||||
# ai
|
||||
pipeline_config['ai']['runner'] = {
|
||||
'runner': self.ap.provider_cfg.data['runner'],
|
||||
}
|
||||
pipeline_config['ai']['local-agent']['model'] = model_uuid
|
||||
pipeline_config['ai']['local-agent']['max-round'] = self.ap.pipeline_cfg.data['msg-truncate']['round'][
|
||||
'max-round'
|
||||
]
|
||||
|
||||
pipeline_config['ai']['local-agent']['prompt'] = [
|
||||
{
|
||||
'role': 'system',
|
||||
'content': self.ap.provider_cfg.data['prompt']['default'],
|
||||
}
|
||||
]
|
||||
pipeline_config['ai']['dify-service-api'] = {
|
||||
'base-url': self.ap.provider_cfg.data['dify-service-api']['base-url'],
|
||||
'app-type': self.ap.provider_cfg.data['dify-service-api']['app-type'],
|
||||
'api-key': self.ap.provider_cfg.data['dify-service-api'][
|
||||
self.ap.provider_cfg.data['dify-service-api']['app-type']
|
||||
]['api-key'],
|
||||
'thinking-convert': self.ap.provider_cfg.data['dify-service-api']['options']['convert-thinking-tips'],
|
||||
'timeout': self.ap.provider_cfg.data['dify-service-api'][
|
||||
self.ap.provider_cfg.data['dify-service-api']['app-type']
|
||||
]['timeout'],
|
||||
}
|
||||
pipeline_config['ai']['dashscope-app-api'] = {
|
||||
'app-type': self.ap.provider_cfg.data['dashscope-app-api']['app-type'],
|
||||
'api-key': self.ap.provider_cfg.data['dashscope-app-api']['api-key'],
|
||||
'references_quote': self.ap.provider_cfg.data['dashscope-app-api'][
|
||||
self.ap.provider_cfg.data['dashscope-app-api']['app-type']
|
||||
]['references_quote'],
|
||||
}
|
||||
|
||||
# trigger
|
||||
pipeline_config['trigger']['group-respond-rules'] = self.ap.pipeline_cfg.data['respond-rules']['default']
|
||||
pipeline_config['trigger']['access-control'] = self.ap.pipeline_cfg.data['access-control']
|
||||
pipeline_config['trigger']['ignore-rules'] = self.ap.pipeline_cfg.data['ignore-rules']
|
||||
|
||||
# safety
|
||||
pipeline_config['safety']['content-filter'] = {
|
||||
'scope': 'all',
|
||||
'check-sensitive-words': self.ap.pipeline_cfg.data['check-sensitive-words'],
|
||||
}
|
||||
pipeline_config['safety']['rate-limit'] = {
|
||||
'window-length': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['window-size'],
|
||||
'limitation': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['limit'],
|
||||
'strategy': self.ap.pipeline_cfg.data['rate-limit']['strategy'],
|
||||
}
|
||||
|
||||
# output
|
||||
pipeline_config['output']['long-text-processing'] = self.ap.platform_cfg.data['long-text-process']
|
||||
pipeline_config['output']['force-delay'] = self.ap.platform_cfg.data['force-delay']
|
||||
pipeline_config['output']['misc'] = {
|
||||
'hide-exception': self.ap.platform_cfg.data['hide-exception-info'],
|
||||
'quote-origin': self.ap.platform_cfg.data['quote-origin'],
|
||||
'at-sender': self.ap.platform_cfg.data['at-sender'],
|
||||
'track-function-calls': self.ap.platform_cfg.data['track-function-calls'],
|
||||
}
|
||||
|
||||
default_pipeline['description'] = default_pipeline['description'] + ' [已迁移 LangBot v3 配置]'
|
||||
default_pipeline['config'] = pipeline_config
|
||||
default_pipeline.pop('created_at')
|
||||
default_pipeline.pop('updated_at')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.values(default_pipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == default_pipeline['uuid'])
|
||||
)
|
||||
|
||||
# ======= Migrate bot =======
|
||||
# Only migrate enabled bots
|
||||
for adapter in self.ap.platform_cfg.data.get('platform-adapters', []):
|
||||
if not adapter.get('enable'):
|
||||
continue
|
||||
|
||||
args = deepcopy(adapter)
|
||||
args.pop('adapter')
|
||||
args.pop('enable')
|
||||
|
||||
bot_data = {
|
||||
'uuid': str(uuid.uuid4()),
|
||||
'name': adapter.get('adapter'),
|
||||
'description': '由 LangBot v3 迁移而来',
|
||||
'adapter': adapter.get('adapter'),
|
||||
'adapter_config': args,
|
||||
'enable': True,
|
||||
'use_pipeline_uuid': pipeline_uuid,
|
||||
'use_pipeline_name': pipeline_name,
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(**bot_data))
|
||||
|
||||
# ======= Migrate system settings =======
|
||||
self.ap.instance_config.data['admins'] = self.ap.system_cfg.data['admin-sessions']
|
||||
self.ap.instance_config.data['api']['port'] = self.ap.system_cfg.data['http-api']['port']
|
||||
self.ap.instance_config.data['command'] = {
|
||||
'prefix': self.ap.command_cfg.data['command-prefix'],
|
||||
'enable': self.ap.command_cfg.data['command-enable']
|
||||
if 'command-enable' in self.ap.command_cfg.data
|
||||
else True,
|
||||
'privilege': self.ap.command_cfg.data['privilege'],
|
||||
}
|
||||
self.ap.instance_config.data['concurrency']['pipeline'] = self.ap.system_cfg.data['pipeline-concurrency']
|
||||
self.ap.instance_config.data['concurrency']['session'] = self.ap.system_cfg.data['session-concurrency'][
|
||||
'default'
|
||||
]
|
||||
self.ap.instance_config.data['mcp'] = self.ap.provider_cfg.data['mcp']
|
||||
self.ap.instance_config.data['proxy'] = self.ap.system_cfg.data['network-proxies']
|
||||
await self.ap.instance_config.dump_config()
|
||||
|
||||
# ======= move files =======
|
||||
# Migrate all config files under data/config
|
||||
all_legacy_dir_name = [
|
||||
'config',
|
||||
# 'metadata',
|
||||
'prompts',
|
||||
'scenario',
|
||||
]
|
||||
|
||||
def move_legacy_files(dir_name: str):
|
||||
if not os.path.exists(f'data/legacy/{dir_name}'):
|
||||
os.makedirs(f'data/legacy/{dir_name}')
|
||||
|
||||
if os.path.exists(f'data/{dir_name}'):
|
||||
for file in os.listdir(f'data/{dir_name}'):
|
||||
if file.endswith('.json'):
|
||||
shutil.move(f'data/{dir_name}/{file}', f'data/legacy/{dir_name}/{file}')
|
||||
|
||||
os.rmdir(f'data/{dir_name}')
|
||||
|
||||
for dir_name in all_legacy_dir_name:
|
||||
move_legacy_files(dir_name)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
@@ -0,0 +1,41 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(2)
|
||||
class DBMigrateCombineQuoteMsgConfig(migration.DBMigration):
|
||||
"""Combine quote message config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
if 'misc' not in config['trigger']:
|
||||
config['trigger']['misc'] = {}
|
||||
|
||||
if 'combine-quote-message' not in config['trigger']['misc']:
|
||||
config['trigger']['misc']['combine-quote-message'] = False
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
49
src/langbot/pkg/persistence/migrations/dbm003_n8n_config.py
Normal file
49
src/langbot/pkg/persistence/migrations/dbm003_n8n_config.py
Normal file
@@ -0,0 +1,49 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(3)
|
||||
class DBMigrateN8nConfig(migration.DBMigration):
|
||||
"""N8n config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
if 'n8n-service-api' not in config['ai']:
|
||||
config['ai']['n8n-service-api'] = {
|
||||
'webhook-url': 'http://your-n8n-webhook-url',
|
||||
'auth-type': 'none',
|
||||
'basic-username': '',
|
||||
'basic-password': '',
|
||||
'jwt-secret': '',
|
||||
'jwt-algorithm': 'HS256',
|
||||
'header-name': '',
|
||||
'header-value': '',
|
||||
'timeout': 120,
|
||||
'output-key': 'response',
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
38
src/langbot/pkg/persistence/migrations/dbm004_rag_kb_uuid.py
Normal file
38
src/langbot/pkg/persistence/migrations/dbm004_rag_kb_uuid.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(4)
|
||||
class DBMigrateRAGKBUUID(migration.DBMigration):
|
||||
"""RAG知识库UUID"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
if 'knowledge-base' not in config['ai']['local-agent']:
|
||||
config['ai']['local-agent']['knowledge-base'] = ''
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -0,0 +1,38 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(5)
|
||||
class DBMigratePipelineRemoveCotConfig(migration.DBMigration):
|
||||
"""Pipeline remove cot config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
if 'remove-think' not in config['output']['misc']:
|
||||
config['output']['misc']['remove-think'] = False
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -0,0 +1,45 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(6)
|
||||
class DBMigrateLangflowApiConfig(migration.DBMigration):
|
||||
"""Langflow API config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
if 'langflow-api' not in config['ai']:
|
||||
config['ai']['langflow-api'] = {
|
||||
'base-url': 'http://localhost:7860',
|
||||
'api-key': 'your-api-key',
|
||||
'flow-id': 'your-flow-id',
|
||||
'input-type': 'chat',
|
||||
'output-type': 'chat',
|
||||
'tweaks': '{}',
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -0,0 +1,44 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(7)
|
||||
class DBMigratePluginInstallSource(migration.DBMigration):
|
||||
"""插件安装来源"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
# 查询表结构获取所有列名（异步执行 SQL）
|
||||
|
||||
columns = []
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name = 'plugin_settings';"
|
||||
)
|
||||
)
|
||||
all_result = result.fetchall()
|
||||
columns = [row[0] for row in all_result]
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(plugin_settings);'))
|
||||
all_result = result.fetchall()
|
||||
columns = [row[1] for row in all_result]
|
||||
|
||||
# 检查并添加 install_source 列
|
||||
if 'install_source' not in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"ALTER TABLE plugin_settings ADD COLUMN install_source VARCHAR(255) NOT NULL DEFAULT 'github'"
|
||||
)
|
||||
)
|
||||
|
||||
# 检查并添加 install_info 列
|
||||
if 'install_info' not in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("ALTER TABLE plugin_settings ADD COLUMN install_info JSON NOT NULL DEFAULT '{}'")
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -0,0 +1,22 @@
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(8)
|
||||
class DBMigratePluginConfig(migration.DBMigration):
|
||||
"""插件配置"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
|
||||
if 'plugin' not in self.ap.instance_config.data:
|
||||
self.ap.instance_config.data['plugin'] = {
|
||||
'runtime_ws_url': 'ws://langbot_plugin_runtime:5400/control/ws',
|
||||
'enable_marketplace': True,
|
||||
'cloud_service_url': 'https://space.langbot.app',
|
||||
}
|
||||
|
||||
await self.ap.instance_config.dump_config()
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -0,0 +1,20 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(9)
|
||||
class DBMigratePipelineExtensionPreferences(migration.DBMigration):
|
||||
"""Pipeline extension preferences"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
|
||||
sql_text = sqlalchemy.text(
|
||||
"ALTER TABLE legacy_pipelines ADD COLUMN extensions_preferences JSON NOT NULL DEFAULT '{}'"
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
sql_text = sqlalchemy.text('ALTER TABLE legacy_pipelines DROP COLUMN extensions_preferences')
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
@@ -0,0 +1,88 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(10)
|
||||
class DBMigratePipelineMultiKnowledgeBase(migration.DBMigration):
|
||||
"""Pipeline support multiple knowledge base binding"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
# Convert knowledge-base from string to array
|
||||
if 'local-agent' in config['ai']:
|
||||
current_kb = config['ai']['local-agent'].get('knowledge-base', '')
|
||||
|
||||
# If it's already a list, skip
|
||||
if isinstance(current_kb, list):
|
||||
continue
|
||||
|
||||
# Convert string to list
|
||||
if current_kb and current_kb != '__none__':
|
||||
config['ai']['local-agent']['knowledge-bases'] = [current_kb]
|
||||
else:
|
||||
config['ai']['local-agent']['knowledge-bases'] = []
|
||||
|
||||
# Remove old field
|
||||
if 'knowledge-base' in config['ai']['local-agent']:
|
||||
del config['ai']['local-agent']['knowledge-base']
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
# Convert knowledge-bases from array back to string
|
||||
if 'local-agent' in config['ai']:
|
||||
current_kbs = config['ai']['local-agent'].get('knowledge-bases', [])
|
||||
|
||||
# If it's already a string, skip
|
||||
if isinstance(current_kbs, str):
|
||||
continue
|
||||
|
||||
# Convert list to string (take first one or empty)
|
||||
if current_kbs and len(current_kbs) > 0:
|
||||
config['ai']['local-agent']['knowledge-base'] = current_kbs[0]
|
||||
else:
|
||||
config['ai']['local-agent']['knowledge-base'] = ''
|
||||
|
||||
# Remove new field
|
||||
if 'knowledge-bases' in config['ai']['local-agent']:
|
||||
del config['ai']['local-agent']['knowledge-bases']
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,40 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
|
||||
|
||||
@migration.migration_class(11)
|
||||
class DBMigrateDifyApiConfig(migration.DBMigration):
|
||||
"""Langflow API config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# read all pipelines
|
||||
pipelines = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
for pipeline in pipelines:
|
||||
serialized_pipeline = self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
config = serialized_pipeline['config']
|
||||
|
||||
if 'base-prompt' not in config['ai']['dify-service-api']:
|
||||
config['ai']['dify-service-api']['base-prompt'] = (
|
||||
'When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image.',
|
||||
)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == serialized_pipeline['uuid'])
|
||||
.values(
|
||||
{
|
||||
'config': config,
|
||||
'for_version': self.ap.ver_mgr.get_current_version(),
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
Reference in New Issue
Block a user