mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-21 16:05:19 +08:00
fix(rag): retain interrupted ingestion state and engine identity
This commit is contained in:
@@ -294,6 +294,11 @@ class KnowledgeService:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
if isinstance(context, (RequestContext, ExecutionContext)):
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is not None:
|
||||
await runtime_kb.reconcile_interrupted_ingestions(execution_context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
@@ -332,11 +337,20 @@ class KnowledgeService:
|
||||
kb_uuid: str,
|
||||
) -> None:
|
||||
"""删除知识库"""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
require_workspace_uuid(context)
|
||||
if await self.get_knowledge_base(context, kb_uuid) is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
|
||||
# delete files
|
||||
execution_context = self._execution_context(context)
|
||||
runtime_kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(execution_context, kb_uuid)
|
||||
if runtime_kb is not None:
|
||||
async with runtime_kb.ingestion_admission_lock:
|
||||
await self._delete_knowledge_base_records(context, kb_uuid)
|
||||
else:
|
||||
await self._delete_knowledge_base_records(context, kb_uuid)
|
||||
|
||||
async def _delete_knowledge_base_records(self, context: RequestContext | ExecutionContext, kb_uuid: str) -> None:
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
# NOTE: Chunk cleanup is for legacy (pre-plugin) KBs that stored chunks locally.
|
||||
# For plugin-based Knowledge Engines, the Chunk table is not populated, so this is a no-op.
|
||||
files = await self.ap.persistence_mgr.execute_async(
|
||||
@@ -344,6 +358,12 @@ class KnowledgeService:
|
||||
.where(persistence_rag.File.workspace_uuid == workspace_uuid)
|
||||
.where(persistence_rag.File.kb_id == kb_uuid)
|
||||
)
|
||||
files = files.all()
|
||||
if any(file.status in {'pending', 'processing', 'interrupted'} for file in files):
|
||||
raise RuntimeError(
|
||||
'Knowledge base has active or interrupted ingestion; retain its files and reconcile '
|
||||
'plugin/upstream state before deleting the knowledge base.'
|
||||
)
|
||||
for file in files:
|
||||
# delete chunks
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
|
||||
@@ -13,6 +13,7 @@ import sqlalchemy
|
||||
from ....core import app
|
||||
from ....entity.persistence import bstorage as persistence_bstorage
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
from ....entity.persistence import rag as persistence_rag
|
||||
from ..authz import WorkspaceRequiredError
|
||||
from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
@@ -223,6 +224,7 @@ class MaintenanceService:
|
||||
retention_days,
|
||||
True,
|
||||
)
|
||||
candidates = await self._exclude_ingestion_uploads(context, candidates)
|
||||
return await asyncio.to_thread(
|
||||
self._delete_local_candidates,
|
||||
candidates,
|
||||
@@ -233,6 +235,22 @@ class MaintenanceService:
|
||||
|
||||
return 0
|
||||
|
||||
async def _exclude_ingestion_uploads(
|
||||
self, context: TenantContext, candidates: list[dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Keep source material for active or unacknowledged remote ingestion."""
|
||||
protected = set()
|
||||
for offset in range(0, len(candidates), 500):
|
||||
keys = [item['key'] for item in candidates[offset : offset + 500]]
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.File.file_name)
|
||||
.where(persistence_rag.File.workspace_uuid == require_workspace_uuid(context))
|
||||
.where(persistence_rag.File.status.in_(['pending', 'processing', 'interrupted']))
|
||||
.where(persistence_rag.File.file_name.in_(keys))
|
||||
)
|
||||
protected.update(result.scalars().all())
|
||||
return [item for item in candidates if item['key'] not in protected]
|
||||
|
||||
async def _expired_uploaded_candidates(
|
||||
self,
|
||||
context: TenantContext,
|
||||
@@ -256,6 +274,7 @@ class MaintenanceService:
|
||||
) -> int:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
candidates = await self._expired_s3_upload_candidates(context, retention_days)
|
||||
candidates = await self._exclude_ingestion_uploads(context, candidates)
|
||||
deleted = 0
|
||||
for item in candidates:
|
||||
await provider.delete(item['key'])
|
||||
|
||||
Reference in New Issue
Block a user