feat: update version to 4.9.0 in pyproject.toml, __init__.py, and uv.lock

Merge branch 'master' into feat/dbm20-rag

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langbot"
-version = "4.8.7"
+version = "4.9.0"
 description = "Production-grade platform for building agentic IM bots"
 readme = "README.md"
 license-files = ["LICENSE"]

src/langbot/__init__.py

@@ -1,3 +1,3 @@
 """LangBot - Production-grade platform for building agentic IM bots"""
 
-__version__ = '4.8.7'
+__version__ = '4.9.0'

src/langbot/pkg/api/http/controller/groups/knowledge/migration.py

@@ -0,0 +1,372 @@
+import asyncio
+import json
+
+import httpx
+import quart
+import sqlalchemy
+
+from ... import group
+from ......core import taskmgr
+from ......entity.persistence import metadata as persistence_metadata
+from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
+
+LANGRAG_PLUGIN_AUTHOR = 'langbot-team'
+LANGRAG_PLUGIN_NAME = 'LangRAG'
+LANGRAG_PLUGIN_ID = f'{LANGRAG_PLUGIN_AUTHOR}/{LANGRAG_PLUGIN_NAME}'
+DEFAULT_SPACE_URL = 'https://space.langbot.app'
+
+# Old Retriever plugin_name -> New Connector plugin_name
+EXTERNAL_PLUGIN_NAME_MAPPING = {
+    'DifyDatasetsRetriever': 'DifyDatasetsConnector',
+    'RAGFlowRetriever': 'RAGFlowConnector',
+    'FastGPTRetriever': 'FastGPTConnector',
+}
+
+# Per-plugin: which old retriever_config fields belong to creation_settings.
+# Remaining fields go to retrieval_settings.
+# None means ALL fields go to creation_settings (no retrieval_schema).
+EXTERNAL_PLUGIN_CREATION_FIELDS: dict[str, set[str] | None] = {
+    'langbot-team/DifyDatasetsConnector': {'api_base_url', 'dify_apikey', 'dataset_id'},
+    'langbot-team/RAGFlowConnector': {'api_base_url', 'api_key', 'dataset_ids'},
+    'langbot-team/FastGPTConnector': None,  # all fields -> creation_settings
+}
+
+
+@group.group_class('knowledge/migration', '/api/v1/knowledge/migration')
+class KnowledgeMigrationRouterGroup(group.RouterGroup):
+    async def _get_migration_flag(self) -> bool:
+        """Check if rag_plugin_migration_needed flag is set."""
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.select(persistence_metadata.Metadata).where(
+                persistence_metadata.Metadata.key == 'rag_plugin_migration_needed'
+            )
+        )
+        row = result.first()
+        return row is not None and row.value == 'true'
+
+    async def _set_migration_flag(self, value: str):
+        """Set rag_plugin_migration_needed flag."""
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.update(persistence_metadata.Metadata)
+            .where(persistence_metadata.Metadata.key == 'rag_plugin_migration_needed')
+            .values(value=value)
+        )
+
+    async def _table_exists(self, table_name: str) -> bool:
+        """Check if a table exists."""
+        if self.ap.persistence_mgr.db.name == 'postgresql':
+            result = await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text(
+                    'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
+                ).bindparams(table_name=table_name)
+            )
+            return result.scalar()
+        else:
+            result = await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
+                    table_name=table_name
+                )
+            )
+            return result.first() is not None
+
+    async def _install_plugin_from_marketplace(
+        self, plugin_id: str, task_context: taskmgr.TaskContext, space_url: str
+    ) -> None:
+        """Install a single plugin from the marketplace."""
+        p_author, p_name = plugin_id.split('/', 1)
+        self.ap.logger.info(f'RAG migration: installing plugin {plugin_id} from marketplace...')
+        task_context.trace(f'Installing plugin {plugin_id} from marketplace...')
+
+        async with httpx.AsyncClient(trust_env=True, timeout=15) as client:
+            resp = await client.get(f'{space_url}/api/v1/marketplace/plugins/{p_author}/{p_name}')
+            resp.raise_for_status()
+            p_data = resp.json().get('data', {}).get('plugin', {})
+            p_version = p_data.get('latest_version')
+            if not p_version:
+                raise Exception(f'Could not determine latest version for {plugin_id}')
+
+        await self.ap.plugin_connector.install_plugin(
+            PluginInstallSource.MARKETPLACE,
+            {
+                'plugin_author': p_author,
+                'plugin_name': p_name,
+                'plugin_version': p_version,
+            },
+            task_context=task_context,
+        )
+        self.ap.logger.info(f'RAG migration: plugin {plugin_id} install request sent.')
+
+    async def _execute_rag_migration(self, task_context: taskmgr.TaskContext, install_plugin: bool = True):
+        """Execute RAG migration: install required plugins and restore backup data."""
+        warnings = []
+
+        # Collect all plugins we need: LangRAG (always) + connector plugins (from external KBs)
+        needed_plugins: dict[str, str] = {
+            LANGRAG_PLUGIN_ID: LANGRAG_PLUGIN_NAME,
+        }
+
+        has_external = await self._table_exists('external_knowledge_bases')
+        if has_external:
+            result = await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text('SELECT DISTINCT plugin_author, plugin_name FROM external_knowledge_bases;')
+            )
+            for row in result.fetchall():
+                plugin_author = row[0] or ''
+                plugin_name = row[1] or ''
+                mapped_name = EXTERNAL_PLUGIN_NAME_MAPPING.get(plugin_name, plugin_name)
+                plugin_id = f'{plugin_author}/{mapped_name}'
+                if plugin_id not in needed_plugins:
+                    needed_plugins[plugin_id] = mapped_name
+
+        self.ap.logger.info(f'RAG migration: plugins needed: {list(needed_plugins.keys())}')
+
+        if install_plugin:
+            # Step 1: Install all required plugins from marketplace
+            task_context.trace('Installing required plugins...', action='install-plugin')
+            space_url = self.ap.instance_config.data.get('space', {}).get('url', DEFAULT_SPACE_URL).rstrip('/')
+
+            for plugin_id in needed_plugins:
+                try:
+                    await self._install_plugin_from_marketplace(plugin_id, task_context, space_url)
+                except Exception as e:
+                    self.ap.logger.warning(f'RAG migration: plugin {plugin_id} install returned: {e}')
+                    task_context.trace(f'Plugin install note ({plugin_id}): {e}')
+
+            # Step 2: Wait for all plugins to become available as knowledge engines
+            task_context.trace(
+                f'Waiting for plugins to become available: {list(needed_plugins.keys())}...',
+                action='wait-plugin',
+            )
+            max_retries = 30
+            engine_id_set: set[str] = set()
+            for i in range(max_retries):
+                try:
+                    engines = await self.ap.plugin_connector.list_knowledge_engines()
+                    engine_id_set = {e.get('plugin_id') for e in engines}
+                except Exception:
+                    pass
+                if all(pid in engine_id_set for pid in needed_plugins):
+                    self.ap.logger.info(f'RAG migration: all plugins ready: {engine_id_set}')
+                    task_context.trace('All required plugins are ready.')
+                    break
+                if i == max_retries - 1:
+                    still_missing = [pid for pid in needed_plugins if pid not in engine_id_set]
+                    warning = f'Plugin(s) {still_missing} did not become available after {max_retries} retries'
+                    self.ap.logger.warning(f'RAG migration: {warning}')
+                    warnings.append(warning)
+                    task_context.trace(warning)
+                await asyncio.sleep(2)
+        else:
+            try:
+                engines = await self.ap.plugin_connector.list_knowledge_engines()
+                engine_id_set = {e.get('plugin_id') for e in engines}
+            except Exception:
+                engine_id_set = set()
+
+        # Step 3: Restore internal knowledge bases from backup
+        task_context.trace('Restoring internal knowledge bases...', action='restore-internal')
+        if await self._table_exists('knowledge_bases_backup'):
+            result = await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text('SELECT * FROM knowledge_bases_backup;')
+            )
+            rows = result.fetchall()
+            columns = result.keys()
+
+            for row in rows:
+                row_dict = dict(zip(columns, row))
+                kb_uuid = row_dict.get('uuid')
+                name = row_dict.get('name', 'Untitled')
+                description = row_dict.get('description', '')
+                emoji = row_dict.get('emoji', '\U0001f4da')
+                embedding_model_uuid = row_dict.get('embedding_model_uuid', '')
+                top_k = row_dict.get('top_k', 5)
+                created_at = row_dict.get('created_at')
+                updated_at = row_dict.get('updated_at')
+
+                creation_settings = json.dumps({'embedding_model_uuid': embedding_model_uuid})
+                retrieval_settings = json.dumps({'top_k': top_k})
+
+                await self.ap.persistence_mgr.execute_async(
+                    sqlalchemy.text(
+                        'INSERT INTO knowledge_bases '
+                        '(uuid, name, description, emoji, created_at, updated_at, '
+                        'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
+                        'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
+                        ':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
+                    ).bindparams(
+                        uuid=kb_uuid,
+                        name=name,
+                        description=description,
+                        emoji=emoji,
+                        created_at=created_at,
+                        updated_at=updated_at,
+                        plugin_id=LANGRAG_PLUGIN_ID,
+                        collection_id=kb_uuid,
+                        creation_settings=creation_settings,
+                        retrieval_settings=retrieval_settings,
+                    )
+                )
+
+                try:
+                    config = {'embedding_model_uuid': embedding_model_uuid}
+                    await self.ap.plugin_connector.rag_on_kb_create(LANGRAG_PLUGIN_ID, kb_uuid, config)
+                    task_context.trace(f'Restored internal KB: {name} ({kb_uuid})')
+                except Exception as e:
+                    warning = f'Failed to notify plugin for KB {name} ({kb_uuid}): {e}'
+                    warnings.append(warning)
+                    task_context.trace(warning)
+
+            await self.ap.rag_mgr.load_knowledge_bases_from_db()
+
+        # Step 4: Restore external knowledge bases
+        task_context.trace('Restoring external knowledge bases...', action='restore-external')
+        if has_external:
+            result = await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
+            )
+            rows = result.fetchall()
+            columns = result.keys()
+
+            self.ap.logger.info(
+                f'RAG migration: {len(rows)} external KB(s) to restore. Available engines: {engine_id_set}'
+            )
+            task_context.trace(f'Found {len(rows)} external KB(s). Available engines: {engine_id_set}')
+
+            for row in rows:
+                row_dict = dict(zip(columns, row))
+                kb_uuid = row_dict.get('uuid')
+                name = row_dict.get('name', 'Untitled')
+                description = row_dict.get('description', '')
+                emoji = row_dict.get('emoji', '\U0001f517')
+                plugin_author = row_dict.get('plugin_author', '')
+                plugin_name = row_dict.get('plugin_name', '')
+                retriever_config = row_dict.get('retriever_config', {})
+                created_at = row_dict.get('created_at')
+
+                mapped_plugin_name = EXTERNAL_PLUGIN_NAME_MAPPING.get(plugin_name, plugin_name)
+                external_plugin_id = f'{plugin_author}/{mapped_plugin_name}'
+
+                self.ap.logger.info(
+                    f'RAG migration: processing external KB "{name}" ({kb_uuid}), '
+                    f'plugin: {plugin_author}/{plugin_name} -> {external_plugin_id}'
+                )
+
+                if isinstance(retriever_config, str):
+                    try:
+                        retriever_config = json.loads(retriever_config)
+                    except (json.JSONDecodeError, TypeError):
+                        retriever_config = {}
+
+                creation_fields = EXTERNAL_PLUGIN_CREATION_FIELDS.get(external_plugin_id)
+                if creation_fields is None:
+                    creation_settings_dict = retriever_config
+                    retrieval_settings_dict = {}
+                else:
+                    creation_settings_dict = {k: v for k, v in retriever_config.items() if k in creation_fields}
+                    retrieval_settings_dict = {k: v for k, v in retriever_config.items() if k not in creation_fields}
+
+                await self.ap.persistence_mgr.execute_async(
+                    sqlalchemy.text(
+                        'INSERT INTO knowledge_bases '
+                        '(uuid, name, description, emoji, created_at, updated_at, '
+                        'knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings) '
+                        'VALUES (:uuid, :name, :description, :emoji, :created_at, :updated_at, '
+                        ':plugin_id, :collection_id, :creation_settings, :retrieval_settings);'
+                    ).bindparams(
+                        uuid=kb_uuid,
+                        name=name,
+                        description=description,
+                        emoji=emoji,
+                        created_at=created_at,
+                        updated_at=created_at,
+                        plugin_id=external_plugin_id,
+                        collection_id=kb_uuid,
+                        creation_settings=json.dumps(creation_settings_dict),
+                        retrieval_settings=json.dumps(retrieval_settings_dict),
+                    )
+                )
+
+                if external_plugin_id not in engine_id_set:
+                    warning = (
+                        f'External KB "{name}" ({kb_uuid}) record saved, but plugin {external_plugin_id} '
+                        f'is not installed yet. Install the connector plugin to use it.'
+                    )
+                    warnings.append(warning)
+                    task_context.trace(warning)
+                else:
+                    try:
+                        await self.ap.plugin_connector.rag_on_kb_create(
+                            external_plugin_id, kb_uuid, creation_settings_dict
+                        )
+                        task_context.trace(f'Restored external KB: {name} ({kb_uuid})')
+                    except Exception as e:
+                        warning = f'Failed to notify plugin for external KB {name} ({kb_uuid}): {e}'
+                        warnings.append(warning)
+                        task_context.trace(warning)
+
+            await self.ap.rag_mgr.load_knowledge_bases_from_db()
+
+        # Step 5: Clear migration flag
+        await self._set_migration_flag('false')
+        task_context.trace('RAG migration completed.', action='done')
+
+        if warnings:
+            task_context.trace(f'Completed with {len(warnings)} warning(s).')
+
+    async def initialize(self) -> None:
+        @self.route('/status', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
+        async def _() -> str:
+            needed = await self._get_migration_flag()
+
+            internal_kb_count = 0
+            external_kb_count = 0
+
+            if needed:
+                if await self._table_exists('knowledge_bases_backup'):
+                    result = await self.ap.persistence_mgr.execute_async(
+                        sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases_backup;')
+                    )
+                    internal_kb_count = result.scalar() or 0
+
+                if await self._table_exists('external_knowledge_bases'):
+                    result = await self.ap.persistence_mgr.execute_async(
+                        sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
+                    )
+                    external_kb_count = result.scalar() or 0
+
+            return self.success(
+                data={
+                    'needed': needed,
+                    'internal_kb_count': internal_kb_count,
+                    'external_kb_count': external_kb_count,
+                }
+            )
+
+        @self.route('/execute', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
+        async def _() -> str:
+            needed = await self._get_migration_flag()
+            if not needed:
+                return self.http_status(400, -1, 'RAG migration is not needed')
+
+            data = await quart.request.get_json(silent=True) or {}
+            install_plugin = data.get('install_plugin', True)
+
+            ctx = taskmgr.TaskContext.new()
+            wrapper = self.ap.task_mgr.create_user_task(
+                self._execute_rag_migration(task_context=ctx, install_plugin=install_plugin),
+                kind='rag-migration',
+                name='rag-migration-execute',
+                label='Migrating knowledge bases to plugin architecture',
+                context=ctx,
+            )
+
+            return self.success(data={'task_id': wrapper.id})
+
+        @self.route('/dismiss', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
+        async def _() -> str:
+            needed = await self._get_migration_flag()
+            if not needed:
+                return self.http_status(400, -1, 'RAG migration is not needed')
+
+            await self._set_migration_flag('false')
+            return self.success()

src/langbot/pkg/persistence/migrations/dbm020_knowledge_engine_plugin_architecture.py

@@ -1,5 +1,3 @@
-import json
-
 import sqlalchemy
 from .. import migration
 
@@ -9,20 +7,22 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
     """Migrate to unified Knowledge Engine plugin architecture.
 
     Changes:
-    - Add knowledge_engine_plugin_id, collection_id, creation_settings, retrieval_settings columns to knowledge_bases
-    - Migrate existing top_k values into retrieval_settings JSON
-    - Migrate existing embedding_model_uuid into creation_settings JSON
-    - Drop embedding_model_uuid and top_k columns (PostgreSQL only; SQLite leaves them unmapped)
-    - Drop external_knowledge_bases table (no longer needed; external KB data is not migrated)
+    - Backup existing knowledge_bases data to knowledge_bases_backup
+    - Clear knowledge_bases table and add new plugin architecture columns
+    - Drop old columns (PostgreSQL only; SQLite leaves them unmapped)
+    - Preserve external_knowledge_bases table as-is for future migration
+    - Set rag_plugin_migration_needed flag in metadata if old data exists
     """
 
     async def upgrade(self):
         """Upgrade"""
+        has_internal_data = await self._backup_knowledge_bases()
+        has_external_data = await self._check_external_knowledge_bases()
+        await self._clear_knowledge_bases()
         await self._add_columns_to_knowledge_bases()
-        await self._migrate_top_k_to_retrieval_settings()
-        await self._migrate_embedding_model_uuid_to_creation_settings()
         await self._drop_old_columns()
-        await self._drop_external_knowledge_bases_table()
+        if has_internal_data or has_external_data:
+            await self._set_migration_flag()
 
     async def _get_table_columns(self, table_name: str) -> list[str]:
         """Get column names from a table (works for both SQLite and PostgreSQL)."""
@@ -57,6 +57,50 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
             )
             return result.first() is not None
 
+    async def _backup_knowledge_bases(self) -> bool:
+        """Backup knowledge_bases data. Returns True if data was backed up."""
+        result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases;'))
+        count = result.scalar()
+        if count == 0:
+            return False
+
+        # Drop backup table if it already exists (from a previous failed migration)
+        if await self._table_exists('knowledge_bases_backup'):
+            await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE knowledge_bases_backup;'))
+
+        await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.text('CREATE TABLE knowledge_bases_backup AS SELECT * FROM knowledge_bases;')
+        )
+        self.ap.logger.info(
+            'Backed up %d knowledge base(s) to knowledge_bases_backup table.',
+            count,
+        )
+        return True
+
+    async def _check_external_knowledge_bases(self) -> bool:
+        """Check if external_knowledge_bases table exists and has data.
+
+        The table is preserved as-is (not dropped) for future migration.
+        """
+        if not await self._table_exists('external_knowledge_bases'):
+            return False
+
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
+        )
+        count = result.scalar()
+        if count > 0:
+            self.ap.logger.info(
+                'Found %d external knowledge base(s) in external_knowledge_bases table. '
+                'Table preserved for future migration.',
+                count,
+            )
+        return count > 0
+
+    async def _clear_knowledge_bases(self):
+        """Clear all rows from knowledge_bases table (preserve table structure)."""
+        await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DELETE FROM knowledge_bases;'))
+
     async def _add_columns_to_knowledge_bases(self):
         """Add new RAG plugin architecture columns to knowledge_bases table."""
         columns = await self._get_table_columns('knowledge_bases')
@@ -74,73 +118,6 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
                     sqlalchemy.text(f'ALTER TABLE knowledge_bases ADD COLUMN {col_name} {col_type};')
                 )
 
-        # For existing knowledge bases without knowledge_engine_plugin_id,
-        # set collection_id = uuid (same default as new KBs)
-        await self.ap.persistence_mgr.execute_async(
-            sqlalchemy.text('UPDATE knowledge_bases SET collection_id = uuid WHERE collection_id IS NULL;')
-        )
-
-    async def _migrate_top_k_to_retrieval_settings(self):
-        """Migrate existing top_k values into retrieval_settings JSON."""
-        columns = await self._get_table_columns('knowledge_bases')
-        if 'top_k' not in columns:
-            return
-
-        result = await self.ap.persistence_mgr.execute_async(
-            sqlalchemy.text(
-                'SELECT uuid, top_k FROM knowledge_bases WHERE top_k IS NOT NULL AND retrieval_settings IS NULL;'
-            )
-        )
-        rows = result.fetchall()
-
-        for row in rows:
-            kb_uuid = row[0]
-            top_k = row[1]
-            retrieval_settings = json.dumps({'top_k': top_k})
-            await self.ap.persistence_mgr.execute_async(
-                sqlalchemy.text('UPDATE knowledge_bases SET retrieval_settings = :rs WHERE uuid = :uuid;').bindparams(
-                    rs=retrieval_settings, uuid=kb_uuid
-                )
-            )
-
-    async def _migrate_embedding_model_uuid_to_creation_settings(self):
-        """Migrate existing embedding_model_uuid into creation_settings JSON."""
-        columns = await self._get_table_columns('knowledge_bases')
-        if 'embedding_model_uuid' not in columns:
-            return
-
-        result = await self.ap.persistence_mgr.execute_async(
-            sqlalchemy.text(
-                'SELECT uuid, embedding_model_uuid, creation_settings FROM knowledge_bases '
-                "WHERE embedding_model_uuid IS NOT NULL AND embedding_model_uuid != '';"
-            )
-        )
-        rows = result.fetchall()
-
-        for row in rows:
-            kb_uuid = row[0]
-            emb_uuid = row[1]
-            existing_settings = row[2]
-
-            if existing_settings and isinstance(existing_settings, str):
-                try:
-                    settings = json.loads(existing_settings)
-                except (json.JSONDecodeError, TypeError):
-                    settings = {}
-            elif isinstance(existing_settings, dict):
-                settings = existing_settings
-            else:
-                settings = {}
-
-            if 'embedding_model_uuid' not in settings:
-                settings['embedding_model_uuid'] = emb_uuid
-                new_settings = json.dumps(settings)
-                await self.ap.persistence_mgr.execute_async(
-                    sqlalchemy.text(
-                        'UPDATE knowledge_bases SET creation_settings = :cs WHERE uuid = :uuid;'
-                    ).bindparams(cs=new_settings, uuid=kb_uuid)
-                )
-
     async def _drop_old_columns(self):
         """Drop embedding_model_uuid and top_k columns (PostgreSQL only).
 
@@ -162,22 +139,22 @@ class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
                 sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN top_k;')
             )
 
-    async def _drop_external_knowledge_bases_table(self):
-        """Drop the external_knowledge_bases table if it exists."""
-        if await self._table_exists('external_knowledge_bases'):
-            # Log existing external KBs before dropping, so users are aware of data loss
-            rows = await self.ap.persistence_mgr.execute_async(
-                sqlalchemy.text('SELECT * FROM external_knowledge_bases;')
+    async def _set_migration_flag(self):
+        """Set rag_plugin_migration_needed flag in metadata table."""
+        # Check if the key already exists
+        result = await self.ap.persistence_mgr.execute_async(
+            sqlalchemy.text("SELECT value FROM metadata WHERE key = 'rag_plugin_migration_needed';")
+        )
+        row = result.first()
+        if row is not None:
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text("UPDATE metadata SET value = 'true' WHERE key = 'rag_plugin_migration_needed';")
             )
-            existing = rows.fetchall()
-            if existing:
-                self.ap.logger.warning(
-                    'Dropping external_knowledge_bases table with %d existing record(s). '
-                    'These external KB configurations will be removed: %s',
-                    len(existing),
-                    [dict(row._mapping) for row in existing],
-                )
-            await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE external_knowledge_bases;'))
+        else:
+            await self.ap.persistence_mgr.execute_async(
+                sqlalchemy.text("INSERT INTO metadata (key, value) VALUES ('rag_plugin_migration_needed', 'true');")
+            )
+        self.ap.logger.info('Set rag_plugin_migration_needed=true in metadata.')
 
     async def downgrade(self):
         """Downgrade"""

src/langbot/pkg/utils/constants.py

@@ -2,7 +2,7 @@ import langbot
 
 semantic_version = f'v{langbot.__version__}'
 
-required_database_version = 19
+required_database_version = 20
 """Tag the version of the database schema, used to check if the database needs to be migrated"""
 
 debug_mode = False

uv.lock

@@ -1832,7 +1832,7 @@ wheels = [
 
 [[package]]
 name = "langbot"
-version = "4.8.7"
+version = "4.9.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiocqhttp" },

web/src/app/home/knowledge/components/kb-migration-dialog/KBMigrationDialog.tsx

@@ -0,0 +1,157 @@
+'use client';
+
+import { useState } from 'react';
+import {
+  Dialog,
+  DialogContent,
+  DialogHeader,
+  DialogTitle,
+  DialogDescription,
+  DialogFooter,
+} from '@/components/ui/dialog';
+import { Button } from '@/components/ui/button';
+import { useTranslation } from 'react-i18next';
+import { httpClient } from '@/app/infra/http/HttpClient';
+import { useAsyncTask, AsyncTaskStatus } from '@/hooks/useAsyncTask';
+import { toast } from 'sonner';
+import { Loader2 } from 'lucide-react';
+
+interface KBMigrationDialogProps {
+  open: boolean;
+  onOpenChange: (open: boolean) => void;
+  internalKbCount: number;
+  externalKbCount: number;
+  onMigrationComplete: () => void;
+}
+
+export default function KBMigrationDialog({
+  open,
+  onOpenChange,
+  internalKbCount,
+  externalKbCount,
+  onMigrationComplete,
+}: KBMigrationDialogProps) {
+  const { t } = useTranslation();
+  const [dismissing, setDismissing] = useState(false);
+
+  const asyncTask = useAsyncTask({
+    onSuccess: () => {
+      toast.success(t('knowledge.migration.success'));
+      onOpenChange(false);
+      onMigrationComplete();
+    },
+    onError: (error) => {
+      toast.error(`${t('knowledge.migration.error')}${error}`);
+    },
+  });
+
+  const handleMigration = async (installPlugin: boolean) => {
+    try {
+      const resp = await httpClient.executeRagMigration(installPlugin);
+      asyncTask.startTask(resp.task_id);
+    } catch {
+      toast.error(t('knowledge.migration.error'));
+    }
+  };
+
+  const handleDismiss = async () => {
+    setDismissing(true);
+    try {
+      await httpClient.dismissRagMigration();
+      onOpenChange(false);
+    } catch {
+      toast.error(t('knowledge.migration.dismissError'));
+    } finally {
+      setDismissing(false);
+    }
+  };
+
+  const isRunning = asyncTask.status === AsyncTaskStatus.RUNNING;
+  const isError = asyncTask.status === AsyncTaskStatus.ERROR;
+  const totalCount = internalKbCount + externalKbCount;
+
+  return (
+    <Dialog
+      open={open}
+      onOpenChange={(v) => {
+        if (!isRunning) onOpenChange(v);
+      }}
+    >
+      <DialogContent className="sm:max-w-md">
+        <DialogHeader>
+          <DialogTitle>{t('knowledge.migration.title')}</DialogTitle>
+          <DialogDescription>
+            {t('knowledge.migration.description')}
+          </DialogDescription>
+        </DialogHeader>
+
+        <div className="py-4 space-y-3">
+          {!isRunning && !isError && (
+            <p className="text-sm text-muted-foreground">
+              {t('knowledge.migration.detected', {
+                total: totalCount,
+                internal: internalKbCount,
+                external: externalKbCount,
+              })}
+            </p>
+          )}
+
+          {isRunning && (
+            <div className="flex items-center gap-3">
+              <Loader2 className="h-5 w-5 animate-spin text-primary" />
+              <p className="text-sm">{t('knowledge.migration.running')}</p>
+            </div>
+          )}
+
+          {isError && (
+            <div className="space-y-2">
+              <p className="text-sm text-destructive">
+                {t('knowledge.migration.error')}
+              </p>
+              {asyncTask.error && (
+                <p className="text-xs text-muted-foreground bg-muted p-2 rounded">
+                  {asyncTask.error}
+                </p>
+              )}
+            </div>
+          )}
+        </div>
+
+        <DialogFooter className="flex flex-col gap-2 sm:flex-col">
+          {!isRunning && !isError && (
+            <>
+              <Button onClick={() => handleMigration(true)} className="w-full">
+                {t('knowledge.migration.startWithInstall')}
+              </Button>
+              <Button
+                variant="outline"
+                onClick={() => handleMigration(false)}
+                className="w-full"
+              >
+                {t('knowledge.migration.startDataOnly')}
+              </Button>
+              <p className="text-xs text-muted-foreground text-center">
+                {t('knowledge.migration.dataOnlyHint')}
+              </p>
+            </>
+          )}
+          {isError && (
+            <Button onClick={() => handleMigration(true)} className="w-full">
+              {t('knowledge.migration.retry')}
+            </Button>
+          )}
+          {!isRunning && (
+            <Button
+              variant="ghost"
+              onClick={handleDismiss}
+              disabled={dismissing}
+              className="w-full text-destructive hover:text-destructive"
+            >
+              {t('knowledge.migration.dismiss')}
+            </Button>
+          )}
+        </DialogFooter>
+      </DialogContent>
+    </Dialog>
+  );
+}

web/src/app/home/knowledge/page.tsx

@@ -7,6 +7,7 @@ import { useEffect, useState } from 'react';
 import { KnowledgeBaseVO } from '@/app/home/knowledge/components/kb-card/KBCardVO';
 import KBCard from '@/app/home/knowledge/components/kb-card/KBCard';
 import KBDetailDialog from '@/app/home/knowledge/KBDetailDialog';
+import KBMigrationDialog from '@/app/home/knowledge/components/kb-migration-dialog/KBMigrationDialog';
 import { httpClient } from '@/app/infra/http/HttpClient';
 import { KnowledgeBase } from '@/app/infra/entities/api';
 
@@ -18,10 +19,29 @@ export default function KnowledgePage() {
   const [selectedKbId, setSelectedKbId] = useState<string>('');
   const [detailDialogOpen, setDetailDialogOpen] = useState(false);
 
+  // Migration dialog state
+  const [migrationDialogOpen, setMigrationDialogOpen] = useState(false);
+  const [migrationInternalCount, setMigrationInternalCount] = useState(0);
+  const [migrationExternalCount, setMigrationExternalCount] = useState(0);
+
   useEffect(() => {
     getKnowledgeBaseList();
+    checkMigrationStatus();
   }, []);
 
+  async function checkMigrationStatus() {
+    try {
+      const resp = await httpClient.getRagMigrationStatus();
+      if (resp.needed) {
+        setMigrationInternalCount(resp.internal_kb_count);
+        setMigrationExternalCount(resp.external_kb_count);
+        setMigrationDialogOpen(true);
+      }
+    } catch {
+      // Silently ignore - migration check is non-critical
+    }
+  }
+
   async function getKnowledgeBaseList() {
     const resp = await httpClient.getKnowledgeBases();
 
@@ -85,8 +105,20 @@ export default function KnowledgePage() {
     getKnowledgeBaseList();
   };
 
+  const handleMigrationComplete = () => {
+    getKnowledgeBaseList();
+  };
+
   return (
     <div>
+      <KBMigrationDialog
+        open={migrationDialogOpen}
+        onOpenChange={setMigrationDialogOpen}
+        internalKbCount={migrationInternalCount}
+        externalKbCount={migrationExternalCount}
+        onMigrationComplete={handleMigrationComplete}
+      />
+
       <KBDetailDialog
         open={detailDialogOpen}
         onOpenChange={setDetailDialogOpen}

web/src/app/infra/entities/api/index.ts

@@ -262,6 +262,12 @@ export interface ApiRespSystemInfo {
   limitation: SystemLimitation;
 }
 
+export interface RagMigrationStatusResp {
+  needed: boolean;
+  internal_kb_count: number;
+  external_kb_count: number;
+}
+
 export interface ApiRespPluginSystemStatus {
   is_enable: boolean;
   is_connected: boolean;

web/src/app/infra/http/BackendClient.ts

@@ -40,6 +40,7 @@ import {
   ModelProvider,
   ApiRespKnowledgeEngines,
   ApiRespParsers,
+  RagMigrationStatusResp,
 } from '@/app/infra/entities/api';
 import { Plugin } from '@/app/infra/entities/plugin';
 import { GetBotLogsRequest } from '@/app/infra/http/requestParam/bots/GetBotLogsRequest';
@@ -710,6 +711,23 @@ export class BackendClient extends BaseHttpClient {
     return this.get('/api/v1/system/status/plugin-system');
   }
 
+  // ============ RAG Migration API ============
+  public getRagMigrationStatus(): Promise<RagMigrationStatusResp> {
+    return this.get('/api/v1/knowledge/migration/status');
+  }
+
+  public executeRagMigration(
+    installPlugin: boolean = true,
+  ): Promise<AsyncTaskCreatedResp> {
+    return this.post('/api/v1/knowledge/migration/execute', {
+      install_plugin: installPlugin,
+    });
+  }
+
+  public dismissRagMigration(): Promise<object> {
+    return this.post('/api/v1/knowledge/migration/dismiss');
+  }
+
   public getPluginDebugInfo(): Promise<{
     debug_url: string;
     plugin_debug_key: string;

web/src/i18n/locales/en-US.ts

@@ -773,6 +773,23 @@ const enUS = {
     retrieverConfiguration: 'Retriever Configuration',
     retrieverInstallInfo: 'You can install Knowledge Retriever plugins from',
     retrieverMarketLink: 'here',
+    migration: {
+      title: 'Knowledge Base Migration',
+      description:
+        'The new version has refactored the knowledge base into a plugin-based architecture, unifying built-in and external knowledge bases as "Knowledge Engine" plugins. Migration of legacy knowledge base data is required. Your old data has been automatically backed up in the database.',
+      detected:
+        'Found {{total}} knowledge base(s) to migrate ({{internal}} internal, {{external}} external).',
+      startWithInstall: 'Auto-install Plugin & Migrate',
+      startDataOnly: 'Migrate Data Only',
+      dataOnlyHint:
+        '"Migrate Data Only" is for offline/intranet environments. Please install the corresponding plugin manually after migration.',
+      dismiss: 'Discard Original Data',
+      running: 'Migrating knowledge bases, please wait...',
+      success: 'Knowledge base migration completed',
+      error: 'Knowledge base migration failed: ',
+      dismissError: 'Operation failed',
+      retry: 'Retry',
+    },
   },
   register: {
     title: 'Initialize LangBot 👋',

web/src/i18n/locales/ja-JP.ts

@@ -762,6 +762,23 @@ const jaJP = {
     retrieverConfiguration: '検索器設定',
     retrieverInstallInfo: 'ナレッジ検索器プラグインは',
     retrieverMarketLink: 'こちらからインストールできます',
+    migration: {
+      title: 'ナレッジベースの移行',
+      description:
+        '新バージョンではナレッジベースをプラグインベースのアーキテクチャに再構築し、内蔵ナレッジベースと外部ナレッジベースを「ナレッジエンジン」プラグインとして統合しました。旧ナレッジベースデータの移行が必要です。旧データはデータベースに自動的にバックアップされています。',
+      detected:
+        '移行が必要なナレッジベースが{{total}}件見つかりました（内部{{internal}}件、外部{{external}}件）。',
+      startWithInstall: 'プラグインを自動インストールして移行',
+      startDataOnly: 'データのみ移行',
+      dataOnlyHint:
+        '「データのみ移行」はオフライン環境向けです。移行完了後に対応するプラグインを手動でインストールしてください。',
+      dismiss: '元データを破棄',
+      running: 'ナレッジベースを移行中です。しばらくお待ちください...',
+      success: 'ナレッジベースの移行が完了しました',
+      error: 'ナレッジベースの移行に失敗しました：',
+      dismissError: '操作に失敗しました',
+      retry: 'リトライ',
+    },
   },
   register: {
     title: 'LangBot を初期化 👋',

web/src/i18n/locales/zh-Hans.ts

@@ -742,6 +742,23 @@ const zhHans = {
     retrieverConfiguration: '检索器配置',
     retrieverInstallInfo: '您可以从',
     retrieverMarketLink: '此处安装知识检索器插件',
+    migration: {
+      title: '知识库迁移',
+      description:
+        '新版本已将知识库重构为插件化架构，并统一内置知识库和外部知识库为「知识引擎」插件，需要对旧知识库数据进行迁移。您的旧数据已自动备份在数据库中。',
+      detected:
+        '共检测到 {{total}} 个知识库需要迁移（{{internal}} 个内置知识库，{{external}} 个外部知识库）。',
+      startWithInstall: '自动安装插件并迁移',
+      startDataOnly: '仅迁移数据',
+      dataOnlyHint:
+        '「仅迁移数据」适合内网环境使用，请在迁移完成后自行安装对应插件',
+      dismiss: '丢弃原数据',
+      running: '正在迁移知识库，请稍候...',
+      success: '知识库迁移完成',
+      error: '知识库迁移失败：',
+      dismissError: '操作失败',
+      retry: '重试',
+    },
   },
   register: {
     title: '初始化 LangBot 👋',

web/src/i18n/locales/zh-Hant.ts

@@ -722,6 +722,23 @@ const zhHant = {
     retrieverConfiguration: '檢索器配置',
     retrieverInstallInfo: '您可以從',
     retrieverMarketLink: '此處安裝知識檢索器插件',
+    migration: {
+      title: '知識庫遷移',
+      description:
+        '新版本已將知識庫重構為插件化架構，並統一內建知識庫和外部知識庫為「知識引擎」插件，需要對舊知識庫資料進行遷移。您的舊資料已自動備份在資料庫中。',
+      detected:
+        '共檢測到 {{total}} 個知識庫需要遷移（{{internal}} 個內建知識庫，{{external}} 個外部知識庫）。',
+      startWithInstall: '自動安裝插件並遷移',
+      startDataOnly: '僅遷移資料',
+      dataOnlyHint:
+        '「僅遷移資料」適合內網環境使用，請在遷移完成後自行安裝對應插件',
+      dismiss: '丟棄原數據',
+      running: '正在遷移知識庫，請稍候...',
+      success: '知識庫遷移完成',
+      error: '知識庫遷移失敗：',
+      dismissError: '操作失敗',
+      retry: '重試',
+    },
   },
   register: {
     title: '初始化 LangBot 👋',