mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-06-02 03:55:55 +00:00
feat(rag): add knowledge base migration from v4.9.0 to plugin architecture
Rewrite dbm020 to backup old knowledge_bases data and preserve external_knowledge_bases table. Add migration API endpoints and frontend dialog so users can opt-in to auto-install LangRAG plugin and restore their knowledge bases with original UUIDs preserved. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,290 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ... import group
|
||||
from ......core import taskmgr
|
||||
from ......entity.persistence import metadata as persistence_metadata
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
LANGRAG_PLUGIN_AUTHOR = 'langbot-team'
|
||||
LANGRAG_PLUGIN_NAME = 'LangRAG'
|
||||
LANGRAG_PLUGIN_ID = f'{LANGRAG_PLUGIN_AUTHOR}/{LANGRAG_PLUGIN_NAME}'
|
||||
|
||||
|
||||
@group.group_class('knowledge/migration', '/api/v1/knowledge/migration')
|
||||
class KnowledgeMigrationRouterGroup(group.RouterGroup):
|
||||
async def _get_migration_flag(self) -> bool:
|
||||
"""Check if rag_plugin_migration_needed flag is set."""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_metadata.Metadata).where(
|
||||
persistence_metadata.Metadata.key == 'rag_plugin_migration_needed'
|
||||
)
|
||||
)
|
||||
row = result.first()
|
||||
return row is not None and row.value == 'true'
|
||||
|
||||
async def _set_migration_flag(self, value: str):
|
||||
"""Set rag_plugin_migration_needed flag."""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_metadata.Metadata)
|
||||
.where(persistence_metadata.Metadata.key == 'rag_plugin_migration_needed')
|
||||
.values(value=value)
|
||||
)
|
||||
|
||||
async def _table_exists(self, table_name: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return result.scalar()
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;"
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _execute_rag_migration(self, task_context: taskmgr.TaskContext):
|
||||
"""Execute RAG migration: install langrag plugin and restore backup data."""
|
||||
warnings = []
|
||||
|
||||
# Step 1: Install langrag plugin from marketplace
|
||||
task_context.trace('Installing LangRAG plugin from marketplace...', action='install-plugin')
|
||||
try:
|
||||
install_info = {
|
||||
'plugin_author': LANGRAG_PLUGIN_AUTHOR,
|
||||
'plugin_name': LANGRAG_PLUGIN_NAME,
|
||||
'plugin_version': 'latest',
|
||||
}
|
||||
await self.ap.plugin_connector.install_plugin(
|
||||
PluginInstallSource.MARKETPLACE, install_info, task_context=task_context
|
||||
)
|
||||
except Exception as e:
|
||||
# Plugin may already be installed
|
||||
self.ap.logger.warning(f'LangRAG plugin install returned: {e}')
|
||||
task_context.trace(f'Plugin install note: {e}')
|
||||
|
||||
# Step 2: Wait for the plugin to be available
|
||||
task_context.trace('Waiting for LangRAG plugin to become available...', action='wait-plugin')
|
||||
max_retries = 30
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_ids = [e.get('plugin_id') for e in engines]
|
||||
if LANGRAG_PLUGIN_ID in engine_ids:
|
||||
task_context.trace('LangRAG plugin is ready.')
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if i == max_retries - 1:
|
||||
raise Exception(
|
||||
f'LangRAG plugin ({LANGRAG_PLUGIN_ID}) did not become available after {max_retries} retries'
|
||||
)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Step 3: Restore internal knowledge bases from backup
|
||||
task_context.trace('Restoring internal knowledge bases...', action='restore-internal')
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM knowledge_bases_backup;')
|
||||
)
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
for row in rows:
|
||||
row_dict = dict(zip(columns, row))
|
||||
kb_uuid = row_dict.get('uuid')
|
||||
name = row_dict.get('name', 'Untitled')
|
||||
description = row_dict.get('description', '')
|
||||
emoji = row_dict.get('emoji', '\U0001f4da')
|
||||
embedding_model_uuid = row_dict.get('embedding_model_uuid', '')
|
||||
top_k = row_dict.get('top_k', 5)
|
||||
created_at = row_dict.get('created_at')
|
||||
updated_at = row_dict.get('updated_at')
|
||||
|
||||
creation_settings = json.dumps({'embedding_model_uuid': embedding_model_uuid})
|
||||
retrieval_settings = json.dumps({'top_k': top_k})
|
||||
|
||||
# Insert into knowledge_bases with the same UUID
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
uuid=kb_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
plugin_id=LANGRAG_PLUGIN_ID,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=retrieval_settings,
|
||||
)
|
||||
)
|
||||
|
||||
# Notify langrag plugin to recognize this KB
|
||||
try:
|
||||
config = {'embedding_model_uuid': embedding_model_uuid}
|
||||
await self.ap.plugin_connector.rag_on_kb_create(LANGRAG_PLUGIN_ID, kb_uuid, config)
|
||||
task_context.trace(f'Restored internal KB: {name} ({kb_uuid})')
|
||||
except Exception as e:
|
||||
warning = f'Failed to notify plugin for KB {name} ({kb_uuid}): {e}'
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
|
||||
# Reload all knowledge bases into runtime
|
||||
await self.ap.rag_mgr.load_knowledge_bases_from_db()
|
||||
|
||||
# Step 4: Restore external knowledge bases (read from preserved original table)
|
||||
task_context.trace('Restoring external knowledge bases...', action='restore-external')
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
|
||||
)
|
||||
rows = result.fetchall()
|
||||
columns = result.keys()
|
||||
|
||||
# Get current available engines for matching
|
||||
try:
|
||||
engines = await self.ap.plugin_connector.list_knowledge_engines()
|
||||
engine_id_set = {e.get('plugin_id') for e in engines}
|
||||
except Exception:
|
||||
engine_id_set = set()
|
||||
|
||||
for row in rows:
|
||||
row_dict = dict(zip(columns, row))
|
||||
kb_uuid = row_dict.get('uuid')
|
||||
name = row_dict.get('name', 'Untitled')
|
||||
description = row_dict.get('description', '')
|
||||
emoji = row_dict.get('emoji', '\U0001f517')
|
||||
plugin_author = row_dict.get('plugin_author', '')
|
||||
plugin_name = row_dict.get('plugin_name', '')
|
||||
retriever_config = row_dict.get('retriever_config', {})
|
||||
created_at = row_dict.get('created_at')
|
||||
|
||||
external_plugin_id = f'{plugin_author}/{plugin_name}'
|
||||
|
||||
if external_plugin_id not in engine_id_set:
|
||||
warning = (
|
||||
f'External KB "{name}" ({kb_uuid}) uses plugin {external_plugin_id} '
|
||||
f'which is not available as a Knowledge Engine. Skipped.'
|
||||
)
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
continue
|
||||
|
||||
# Parse retriever_config
|
||||
if isinstance(retriever_config, str):
|
||||
try:
|
||||
retriever_config = json.loads(retriever_config)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
retriever_config = {}
|
||||
|
||||
creation_settings = json.dumps(retriever_config)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'INSERT INTO knowledge_bases '
|
||||
'(uuid, name, description, emoji, created_at, updated_at, '
|
||||
'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
|
||||
'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
|
||||
':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
|
||||
).bindparams(
|
||||
uuid=kb_uuid,
|
||||
name=name,
|
||||
description=description,
|
||||
emoji=emoji,
|
||||
created_at=created_at,
|
||||
updated_at=created_at,
|
||||
plugin_id=external_plugin_id,
|
||||
collection_id=kb_uuid,
|
||||
creation_settings=creation_settings,
|
||||
retrieval_settings=json.dumps({}),
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
await self.ap.plugin_connector.rag_on_kb_create(
|
||||
external_plugin_id, kb_uuid, retriever_config
|
||||
)
|
||||
task_context.trace(f'Restored external KB: {name} ({kb_uuid})')
|
||||
except Exception as e:
|
||||
warning = f'Failed to notify plugin for external KB {name} ({kb_uuid}): {e}'
|
||||
warnings.append(warning)
|
||||
task_context.trace(warning)
|
||||
|
||||
# Reload again after external KBs
|
||||
await self.ap.rag_mgr.load_knowledge_bases_from_db()
|
||||
|
||||
# Step 5: Clear migration flag
|
||||
await self._set_migration_flag('false')
|
||||
task_context.trace('RAG migration completed.', action='done')
|
||||
|
||||
if warnings:
|
||||
task_context.trace(f'Completed with {len(warnings)} warning(s).')
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
|
||||
internal_kb_count = 0
|
||||
external_kb_count = 0
|
||||
|
||||
if needed:
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases_backup;')
|
||||
)
|
||||
internal_kb_count = result.scalar() or 0
|
||||
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
)
|
||||
external_kb_count = result.scalar() or 0
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'needed': needed,
|
||||
'internal_kb_count': internal_kb_count,
|
||||
'external_kb_count': external_kb_count,
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._execute_rag_migration(task_context=ctx),
|
||||
kind='rag-migration',
|
||||
name='rag-migration-execute',
|
||||
label='Migrating knowledge bases to plugin architecture',
|
||||
context=ctx,
|
||||
)
|
||||
|
||||
return self.success(data={'task_id': wrapper.id})
|
||||
|
||||
@self.route('/dismiss', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
needed = await self._get_migration_flag()
|
||||
if not needed:
|
||||
return self.http_status(400, -1, 'RAG migration is not needed')
|
||||
|
||||
await self._set_migration_flag('false')
|
||||
return self.success()
|
||||
@@ -9,20 +9,22 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
"""Migrate to unified Knowledge Engine plugin architecture.
|
||||
|
||||
Changes:
|
||||
- Add knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings columns to knowledge_bases
|
||||
- Migrate existing top_k values into retrieval_settings JSON
|
||||
- Migrate existing embedding_model_uuid into creation_settings JSON
|
||||
- Drop embedding_model_uuid and top_k columns (PostgreSQL only; SQLite leaves them unmapped)
|
||||
- Drop external_knowledge_bases table (no longer needed; external KB data is not migrated)
|
||||
- Backup existing knowledge_bases data to knowledge_bases_backup
|
||||
- Clear knowledge_bases table and add new plugin architecture columns
|
||||
- Drop old columns (PostgreSQL only; SQLite leaves them unmapped)
|
||||
- Preserve external_knowledge_bases table as-is for future migration
|
||||
- Set rag_plugin_migration_needed flag in metadata if old data exists
|
||||
"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
has_internal_data = await self._backup_knowledge_bases()
|
||||
has_external_data = await self._check_external_knowledge_bases()
|
||||
await self._clear_knowledge_bases()
|
||||
await self._add_columns_to_knowledge_bases()
|
||||
await self._migrate_top_k_to_retrieval_settings()
|
||||
await self._migrate_embedding_model_uuid_to_creation_settings()
|
||||
await self._drop_old_columns()
|
||||
await self._drop_external_knowledge_bases_table()
|
||||
if has_internal_data or has_external_data:
|
||||
await self._set_migration_flag()
|
||||
|
||||
async def _get_table_columns(self, table_name: str) -> list[str]:
|
||||
"""Get column names from a table (works for both SQLite and PostgreSQL)."""
|
||||
@@ -57,6 +59,56 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _backup_knowledge_bases(self) -> bool:
|
||||
"""Backup knowledge_bases data. Returns True if data was backed up."""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases;')
|
||||
)
|
||||
count = result.scalar()
|
||||
if count == 0:
|
||||
return False
|
||||
|
||||
# Drop backup table if it already exists (from a previous failed migration)
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('DROP TABLE knowledge_bases_backup;')
|
||||
)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('CREATE TABLE knowledge_bases_backup AS SELECT * FROM knowledge_bases;')
|
||||
)
|
||||
self.ap.logger.info(
|
||||
'Backed up %d knowledge base(s) to knowledge_bases_backup table.',
|
||||
count,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _check_external_knowledge_bases(self) -> bool:
|
||||
"""Check if external_knowledge_bases table exists and has data.
|
||||
|
||||
The table is preserved as-is (not dropped) for future migration.
|
||||
"""
|
||||
if not await self._table_exists('external_knowledge_bases'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
)
|
||||
count = result.scalar()
|
||||
if count > 0:
|
||||
self.ap.logger.info(
|
||||
'Found %d external knowledge base(s) in external_knowledge_bases table. '
|
||||
'Table preserved for future migration.',
|
||||
count,
|
||||
)
|
||||
return count > 0
|
||||
|
||||
async def _clear_knowledge_bases(self):
|
||||
"""Clear all rows from knowledge_bases table (preserve table structure)."""
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('DELETE FROM knowledge_bases;')
|
||||
)
|
||||
|
||||
async def _add_columns_to_knowledge_bases(self):
|
||||
"""Add new RAG plugin architecture columns to knowledge_bases table."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
@@ -74,73 +126,6 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
sqlalchemy.text(f'ALTER TABLE knowledge_bases ADD COLUMN {col_name} {col_type};')
|
||||
)
|
||||
|
||||
# For existing knowledge bases without knowledge_engine_plugin_id,
|
||||
# set collection_id = uuid (same default as new KBs)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE knowledge_bases SET collection_id = uuid WHERE collection_id IS NULL;')
|
||||
)
|
||||
|
||||
async def _migrate_top_k_to_retrieval_settings(self):
|
||||
"""Migrate existing top_k values into retrieval_settings JSON."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
if 'top_k' not in columns:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT uuid, top_k FROM knowledge_bases WHERE top_k IS NOT NULL AND retrieval_settings IS NULL;'
|
||||
)
|
||||
)
|
||||
rows = result.fetchall()
|
||||
|
||||
for row in rows:
|
||||
kb_uuid = row[0]
|
||||
top_k = row[1]
|
||||
retrieval_settings = json.dumps({'top_k': top_k})
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE knowledge_bases SET retrieval_settings = :rs WHERE uuid = :uuid;').bindparams(
|
||||
rs=retrieval_settings, uuid=kb_uuid
|
||||
)
|
||||
)
|
||||
|
||||
async def _migrate_embedding_model_uuid_to_creation_settings(self):
|
||||
"""Migrate existing embedding_model_uuid into creation_settings JSON."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
if 'embedding_model_uuid' not in columns:
|
||||
return
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT uuid, embedding_model_uuid, creation_settings FROM knowledge_bases '
|
||||
"WHERE embedding_model_uuid IS NOT NULL AND embedding_model_uuid != '';"
|
||||
)
|
||||
)
|
||||
rows = result.fetchall()
|
||||
|
||||
for row in rows:
|
||||
kb_uuid = row[0]
|
||||
emb_uuid = row[1]
|
||||
existing_settings = row[2]
|
||||
|
||||
if existing_settings and isinstance(existing_settings, str):
|
||||
try:
|
||||
settings = json.loads(existing_settings)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
settings = {}
|
||||
elif isinstance(existing_settings, dict):
|
||||
settings = existing_settings
|
||||
else:
|
||||
settings = {}
|
||||
|
||||
if 'embedding_model_uuid' not in settings:
|
||||
settings['embedding_model_uuid'] = emb_uuid
|
||||
new_settings = json.dumps(settings)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE knowledge_bases SET creation_settings = :cs WHERE uuid = :uuid;'
|
||||
).bindparams(cs=new_settings, uuid=kb_uuid)
|
||||
)
|
||||
|
||||
async def _drop_old_columns(self):
|
||||
"""Drop embedding_model_uuid and top_k columns (PostgreSQL only).
|
||||
|
||||
@@ -162,22 +147,26 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN top_k;')
|
||||
)
|
||||
|
||||
async def _drop_external_knowledge_bases_table(self):
|
||||
"""Drop the external_knowledge_bases table if it exists."""
|
||||
if await self._table_exists('external_knowledge_bases'):
|
||||
# Log existing external KBs before dropping, so users are aware of data loss
|
||||
rows = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
|
||||
)
|
||||
existing = rows.fetchall()
|
||||
if existing:
|
||||
self.ap.logger.warning(
|
||||
'Dropping external_knowledge_bases table with %d existing record(s). '
|
||||
'These external KB configurations will be removed: %s',
|
||||
len(existing),
|
||||
[dict(row._mapping) for row in existing],
|
||||
async def _set_migration_flag(self):
|
||||
"""Set rag_plugin_migration_needed flag in metadata table."""
|
||||
# Check if the key already exists
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT value FROM metadata WHERE key = 'rag_plugin_migration_needed';")
|
||||
)
|
||||
row = result.first()
|
||||
if row is not None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"UPDATE metadata SET value = 'true' WHERE key = 'rag_plugin_migration_needed';"
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE external_knowledge_bases;'))
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"INSERT INTO metadata (key, value) VALUES ('rag_plugin_migration_needed', 'true');"
|
||||
)
|
||||
)
|
||||
self.ap.logger.info('Set rag_plugin_migration_needed=true in metadata.')
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
|
||||
@@ -2,7 +2,7 @@ import langbot
|
||||
|
||||
semantic_version = f'v{langbot.__version__}'
|
||||
|
||||
required_database_version = 19
|
||||
required_database_version = 20
|
||||
"""Tag the version of the database schema, used to check if the database needs to be migrated"""
|
||||
|
||||
debug_mode = False
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
'use client';
|
||||
|
||||
import { useState } from 'react';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { httpClient } from '@/app/infra/http/HttpClient';
|
||||
import { useAsyncTask, AsyncTaskStatus } from '@/hooks/useAsyncTask';
|
||||
import { toast } from 'sonner';
|
||||
import { Loader2 } from 'lucide-react';
|
||||
|
||||
interface KBMigrationDialogProps {
|
||||
open: boolean;
|
||||
onOpenChange: (open: boolean) => void;
|
||||
internalKbCount: number;
|
||||
externalKbCount: number;
|
||||
onMigrationComplete: () => void;
|
||||
}
|
||||
|
||||
export default function KBMigrationDialog({
|
||||
open,
|
||||
onOpenChange,
|
||||
internalKbCount,
|
||||
externalKbCount,
|
||||
onMigrationComplete,
|
||||
}: KBMigrationDialogProps) {
|
||||
const { t } = useTranslation();
|
||||
const [dismissing, setDismissing] = useState(false);
|
||||
|
||||
const asyncTask = useAsyncTask({
|
||||
onSuccess: () => {
|
||||
toast.success(t('knowledge.migration.success'));
|
||||
onOpenChange(false);
|
||||
onMigrationComplete();
|
||||
},
|
||||
onError: (error) => {
|
||||
toast.error(`${t('knowledge.migration.error')}${error}`);
|
||||
},
|
||||
});
|
||||
|
||||
const handleStartMigration = async () => {
|
||||
try {
|
||||
const resp = await httpClient.executeRagMigration();
|
||||
asyncTask.startTask(resp.task_id);
|
||||
} catch {
|
||||
toast.error(t('knowledge.migration.error'));
|
||||
}
|
||||
};
|
||||
|
||||
const handleDismiss = async () => {
|
||||
setDismissing(true);
|
||||
try {
|
||||
await httpClient.dismissRagMigration();
|
||||
onOpenChange(false);
|
||||
} catch {
|
||||
toast.error(t('knowledge.migration.dismissError'));
|
||||
} finally {
|
||||
setDismissing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const isRunning = asyncTask.status === AsyncTaskStatus.RUNNING;
|
||||
const isError = asyncTask.status === AsyncTaskStatus.ERROR;
|
||||
const totalCount = internalKbCount + externalKbCount;
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
open={open}
|
||||
onOpenChange={(v) => {
|
||||
if (!isRunning) onOpenChange(v);
|
||||
}}
|
||||
>
|
||||
<DialogContent className="sm:max-w-[500px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{t('knowledge.migration.title')}</DialogTitle>
|
||||
<DialogDescription>
|
||||
{t('knowledge.migration.description')}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="py-4 space-y-3">
|
||||
{!isRunning && !isError && (
|
||||
<>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{t('knowledge.migration.detected', {
|
||||
total: totalCount,
|
||||
internal: internalKbCount,
|
||||
external: externalKbCount,
|
||||
})}
|
||||
</p>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{t('knowledge.migration.installHint')}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
|
||||
{isRunning && (
|
||||
<div className="flex items-center gap-3">
|
||||
<Loader2 className="h-5 w-5 animate-spin text-primary" />
|
||||
<p className="text-sm">{t('knowledge.migration.running')}</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{isError && (
|
||||
<div className="space-y-2">
|
||||
<p className="text-sm text-destructive">
|
||||
{t('knowledge.migration.error')}
|
||||
</p>
|
||||
{asyncTask.error && (
|
||||
<p className="text-xs text-muted-foreground bg-muted p-2 rounded">
|
||||
{asyncTask.error}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
{!isRunning && (
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={handleDismiss}
|
||||
disabled={dismissing}
|
||||
>
|
||||
{t('knowledge.migration.dismiss')}
|
||||
</Button>
|
||||
)}
|
||||
{!isRunning && !isError && (
|
||||
<Button onClick={handleStartMigration}>
|
||||
{t('knowledge.migration.start')}
|
||||
</Button>
|
||||
)}
|
||||
{isError && (
|
||||
<Button onClick={handleStartMigration}>
|
||||
{t('knowledge.migration.retry')}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { useEffect, useState } from 'react';
|
||||
import { KnowledgeBaseVO } from '@/app/home/knowledge/components/kb-card/KBCardVO';
|
||||
import KBCard from '@/app/home/knowledge/components/kb-card/KBCard';
|
||||
import KBDetailDialog from '@/app/home/knowledge/KBDetailDialog';
|
||||
import KBMigrationDialog from '@/app/home/knowledge/components/kb-migration-dialog/KBMigrationDialog';
|
||||
import { httpClient } from '@/app/infra/http/HttpClient';
|
||||
import { KnowledgeBase } from '@/app/infra/entities/api';
|
||||
|
||||
@@ -18,10 +19,29 @@ export default function KnowledgePage() {
|
||||
const [selectedKbId, setSelectedKbId] = useState<string>('');
|
||||
const [detailDialogOpen, setDetailDialogOpen] = useState(false);
|
||||
|
||||
// Migration dialog state
|
||||
const [migrationDialogOpen, setMigrationDialogOpen] = useState(false);
|
||||
const [migrationInternalCount, setMigrationInternalCount] = useState(0);
|
||||
const [migrationExternalCount, setMigrationExternalCount] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
getKnowledgeBaseList();
|
||||
checkMigrationStatus();
|
||||
}, []);
|
||||
|
||||
async function checkMigrationStatus() {
|
||||
try {
|
||||
const resp = await httpClient.getRagMigrationStatus();
|
||||
if (resp.needed) {
|
||||
setMigrationInternalCount(resp.internal_kb_count);
|
||||
setMigrationExternalCount(resp.external_kb_count);
|
||||
setMigrationDialogOpen(true);
|
||||
}
|
||||
} catch {
|
||||
// Silently ignore - migration check is non-critical
|
||||
}
|
||||
}
|
||||
|
||||
async function getKnowledgeBaseList() {
|
||||
const resp = await httpClient.getKnowledgeBases();
|
||||
|
||||
@@ -85,8 +105,20 @@ export default function KnowledgePage() {
|
||||
getKnowledgeBaseList();
|
||||
};
|
||||
|
||||
const handleMigrationComplete = () => {
|
||||
getKnowledgeBaseList();
|
||||
};
|
||||
|
||||
return (
|
||||
<div>
|
||||
<KBMigrationDialog
|
||||
open={migrationDialogOpen}
|
||||
onOpenChange={setMigrationDialogOpen}
|
||||
internalKbCount={migrationInternalCount}
|
||||
externalKbCount={migrationExternalCount}
|
||||
onMigrationComplete={handleMigrationComplete}
|
||||
/>
|
||||
|
||||
<KBDetailDialog
|
||||
open={detailDialogOpen}
|
||||
onOpenChange={setDetailDialogOpen}
|
||||
|
||||
@@ -262,6 +262,12 @@ export interface ApiRespSystemInfo {
|
||||
limitation: SystemLimitation;
|
||||
}
|
||||
|
||||
export interface RagMigrationStatusResp {
|
||||
needed: boolean;
|
||||
internal_kb_count: number;
|
||||
external_kb_count: number;
|
||||
}
|
||||
|
||||
export interface ApiRespPluginSystemStatus {
|
||||
is_enable: boolean;
|
||||
is_connected: boolean;
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
ModelProvider,
|
||||
ApiRespKnowledgeEngines,
|
||||
ApiRespParsers,
|
||||
RagMigrationStatusResp,
|
||||
} from '@/app/infra/entities/api';
|
||||
import { Plugin } from '@/app/infra/entities/plugin';
|
||||
import { GetBotLogsRequest } from '@/app/infra/http/requestParam/bots/GetBotLogsRequest';
|
||||
@@ -710,6 +711,19 @@ export class BackendClient extends BaseHttpClient {
|
||||
return this.get('/api/v1/system/status/plugin-system');
|
||||
}
|
||||
|
||||
// ============ RAG Migration API ============
|
||||
public getRagMigrationStatus(): Promise<RagMigrationStatusResp> {
|
||||
return this.get('/api/v1/knowledge/migration/status');
|
||||
}
|
||||
|
||||
public executeRagMigration(): Promise<AsyncTaskCreatedResp> {
|
||||
return this.post('/api/v1/knowledge/migration/execute');
|
||||
}
|
||||
|
||||
public dismissRagMigration(): Promise<object> {
|
||||
return this.post('/api/v1/knowledge/migration/dismiss');
|
||||
}
|
||||
|
||||
public getPluginDebugInfo(): Promise<{
|
||||
debug_url: string;
|
||||
plugin_debug_key: string;
|
||||
|
||||
@@ -773,6 +773,22 @@ const enUS = {
|
||||
retrieverConfiguration: 'Retriever Configuration',
|
||||
retrieverInstallInfo: 'You can install Knowledge Retriever plugins from',
|
||||
retrieverMarketLink: 'here',
|
||||
migration: {
|
||||
title: 'Knowledge Base Migration',
|
||||
description:
|
||||
'Legacy knowledge base data detected. Migration to the new plugin-based RAG architecture is required.',
|
||||
detected:
|
||||
'Found {{total}} knowledge base(s) to migrate ({{internal}} internal, {{external}} external).',
|
||||
installHint:
|
||||
'Migration will automatically install the LangRAG plugin and restore your knowledge base data. Documents and vector data will be preserved.',
|
||||
start: 'Migrate Now',
|
||||
dismiss: 'Skip for Now',
|
||||
running: 'Migrating knowledge bases, please wait...',
|
||||
success: 'Knowledge base migration completed',
|
||||
error: 'Knowledge base migration failed: ',
|
||||
dismissError: 'Operation failed',
|
||||
retry: 'Retry',
|
||||
},
|
||||
},
|
||||
register: {
|
||||
title: 'Initialize LangBot 👋',
|
||||
|
||||
@@ -762,6 +762,22 @@ const jaJP = {
|
||||
retrieverConfiguration: '検索器設定',
|
||||
retrieverInstallInfo: 'ナレッジ検索器プラグインは',
|
||||
retrieverMarketLink: 'こちらからインストールできます',
|
||||
migration: {
|
||||
title: 'ナレッジベースの移行',
|
||||
description:
|
||||
'旧バージョンのナレッジベースデータが検出されました。新しいプラグインベースのRAGアーキテクチャへの移行が必要です。',
|
||||
detected:
|
||||
'移行が必要なナレッジベースが{{total}}件見つかりました(内部{{internal}}件、外部{{external}}件)。',
|
||||
installHint:
|
||||
'移行により LangRAG プラグインが自動的にインストールされ、ナレッジベースデータが復元されます。ドキュメントとベクトルデータは保持されます。',
|
||||
start: '今すぐ移行',
|
||||
dismiss: '後で',
|
||||
running: 'ナレッジベースを移行中です。しばらくお待ちください...',
|
||||
success: 'ナレッジベースの移行が完了しました',
|
||||
error: 'ナレッジベースの移行に失敗しました:',
|
||||
dismissError: '操作に失敗しました',
|
||||
retry: 'リトライ',
|
||||
},
|
||||
},
|
||||
register: {
|
||||
title: 'LangBot を初期化 👋',
|
||||
|
||||
@@ -742,6 +742,21 @@ const zhHans = {
|
||||
retrieverConfiguration: '检索器配置',
|
||||
retrieverInstallInfo: '您可以从',
|
||||
retrieverMarketLink: '此处安装知识检索器插件',
|
||||
migration: {
|
||||
title: '知识库迁移',
|
||||
description: '检测到旧版知识库数据,需要迁移到新的插件化 RAG 架构。',
|
||||
detected:
|
||||
'共检测到 {{total}} 个知识库需要迁移({{internal}} 个内置知识库,{{external}} 个外部知识库)。',
|
||||
installHint:
|
||||
'迁移将自动安装 LangRAG 插件并恢复您的知识库数据,文档和向量数据将被保留。',
|
||||
start: '立即迁移',
|
||||
dismiss: '暂不迁移',
|
||||
running: '正在迁移知识库,请稍候...',
|
||||
success: '知识库迁移完成',
|
||||
error: '知识库迁移失败:',
|
||||
dismissError: '操作失败',
|
||||
retry: '重试',
|
||||
},
|
||||
},
|
||||
register: {
|
||||
title: '初始化 LangBot 👋',
|
||||
|
||||
@@ -722,6 +722,21 @@ const zhHant = {
|
||||
retrieverConfiguration: '檢索器配置',
|
||||
retrieverInstallInfo: '您可以從',
|
||||
retrieverMarketLink: '此處安裝知識檢索器插件',
|
||||
migration: {
|
||||
title: '知識庫遷移',
|
||||
description: '檢測到舊版知識庫資料,需要遷移到新的插件化 RAG 架構。',
|
||||
detected:
|
||||
'共檢測到 {{total}} 個知識庫需要遷移({{internal}} 個內建知識庫,{{external}} 個外部知識庫)。',
|
||||
installHint:
|
||||
'遷移將自動安裝 LangRAG 插件並恢復您的知識庫資料,文件和向量資料將被保留。',
|
||||
start: '立即遷移',
|
||||
dismiss: '暫不遷移',
|
||||
running: '正在遷移知識庫,請稍候...',
|
||||
success: '知識庫遷移完成',
|
||||
error: '知識庫遷移失敗:',
|
||||
dismissError: '操作失敗',
|
||||
retry: '重試',
|
||||
},
|
||||
},
|
||||
register: {
|
||||
title: '初始化 LangBot 👋',
|
||||
|
||||
Reference in New Issue
Block a user