feat(plugins): consolidate official plugins with independent releases (LANG-180) (#2797)

Co-authored-by: multica-agent <[email protected]>

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/__init__.py

@@ -0,0 +1,17 @@
+"""Knowledge engine package exports.
+
+Keep the LangRAG import lazy so importing helper modules such as
+``components.knowledge_engine.query_rewrite`` does not require optional parser
+dependencies.
+"""
+
+
+def __getattr__(name: str):
+    if name == "LangRAG":
+        from .langrag import LangRAG
+
+        return LangRAG
+    raise AttributeError(name)
+
+
+__all__ = ["LangRAG"]

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/chunker.py

@@ -0,0 +1,400 @@
+"""Text chunking utilities using recursive character splitting.
+
+Extracted from langrag.py to allow reuse across different index strategies.
+Also provides section-aware chunking that preserves document structure.
+"""
+
+from __future__ import annotations
+
+import re
+import logging
+from dataclasses import dataclass, field
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CHUNK_SIZE = 512
+DEFAULT_CHUNK_OVERLAP = 50
+
+# Separators in order of semantic significance (paragraph → sentence → word → char)
+_SEPARATORS = [
+    "\n\n",  # Paragraphs
+    "\n",    # Lines
+    ". ",    # Sentences
+    "。",    # Chinese/Japanese sentence end
+    "! ",    # Exclamations
+    "? ",    # Questions
+    "; ",    # Semicolons
+    "；",    # Chinese semicolon
+    "，",    # Chinese comma
+    ", ",    # Commas
+    " ",     # Spaces (words)
+    "",      # Characters (fallback)
+]
+
+
+def chunk_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+    """Split text into overlapping chunks using recursive character splitting.
+
+    Attempts to split at natural semantic boundaries (paragraphs, sentences,
+    words) before falling back to character-level splitting. This preserves
+    readability and context compared to a naive sliding-window approach.
+
+    Args:
+        text: The text to split.
+        chunk_size: Maximum characters per chunk.
+        chunk_overlap: Number of overlapping characters between consecutive chunks.
+
+    Returns:
+        List of text chunks.
+    """
+    if not text:
+        return []
+
+    if chunk_overlap >= chunk_size:
+        logger.warning(
+            f"chunk_overlap ({chunk_overlap}) >= chunk_size ({chunk_size}), "
+            "clamping overlap to chunk_size - 1"
+        )
+        chunk_overlap = chunk_size - 1
+
+    return _split_recursive(text, _SEPARATORS, chunk_size, chunk_overlap)
+
+
+def _split_recursive(
+    text: str,
+    separators: list[str],
+    chunk_size: int,
+    chunk_overlap: int,
+) -> list[str]:
+    """Recursively split *text* trying each separator in priority order."""
+    final_chunks: list[str] = []
+    separator = separators[0] if separators else ""
+    remaining_separators = separators[1:] if len(separators) > 1 else []
+
+    splits = _split_by_separator(text, separator)
+
+    current_parts: list[str] = []
+    current_len = 0
+
+    for part in splits:
+        part_len = len(part)
+
+        # Single piece already exceeds limit → recurse with finer separator
+        if part_len > chunk_size:
+            # Flush accumulated parts first
+            if current_parts:
+                final_chunks.append("".join(current_parts))
+                current_parts = _overlap_parts(current_parts, chunk_overlap)
+                current_len = sum(len(p) for p in current_parts)
+
+            if remaining_separators:
+                final_chunks.extend(
+                    _split_recursive(part, remaining_separators, chunk_size, chunk_overlap)
+                )
+            else:
+                final_chunks.extend(_split_by_char(part, chunk_size, chunk_overlap))
+            continue
+
+        # Would adding this part exceed the limit?
+        if current_parts and current_len + part_len > chunk_size:
+            final_chunks.append("".join(current_parts))
+            current_parts = _overlap_parts(current_parts, chunk_overlap)
+            current_len = sum(len(p) for p in current_parts)
+
+        current_parts.append(part)
+        current_len += part_len
+
+    if current_parts:
+        final_chunks.append("".join(current_parts))
+
+    return final_chunks
+
+
+def _split_by_separator(text: str, separator: str) -> list[str]:
+    """Split *text* by *separator*, keeping the separator attached to the left piece."""
+    if separator == "":
+        return list(text)
+
+    parts = text.split(separator)
+    result: list[str] = []
+    for piece in parts[:-1]:
+        if piece or separator.strip():
+            result.append(piece + separator)
+    if parts[-1]:
+        result.append(parts[-1])
+    return result
+
+
+def _overlap_parts(parts: list[str], overlap: int) -> list[str]:
+    """Return trailing *parts* whose combined length fits within *overlap*."""
+    if overlap <= 0:
+        return []
+
+    kept: list[str] = []
+    total = 0
+    for part in reversed(parts):
+        if total + len(part) > overlap:
+            break
+        kept.insert(0, part)
+        total += len(part)
+    return kept
+
+
+def _split_by_char(text: str, chunk_size: int, overlap: int) -> list[str]:
+    """Last-resort character-level split."""
+    chunks: list[str] = []
+    start = 0
+    while start < len(text):
+        chunks.append(text[start : start + chunk_size])
+        start += chunk_size - overlap
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# Section-aware chunking
+# ---------------------------------------------------------------------------
+
+# Regex matching a contiguous block of Markdown table rows (|...|)
+_TABLE_LINE_RE = re.compile(r"^(\|.+\|)\s*$")
+
+
+@dataclass
+class SectionChunk:
+    """A chunk derived from a structured section, carrying structural metadata."""
+
+    text: str
+    heading: str | None = None
+    level: int = 0
+    page: int | None = None
+    heading_path: str = ""
+    has_table: bool = False
+    section_index: int = 0
+
+
+def chunk_sections(
+    sections: list,
+    chunk_size: int,
+    chunk_overlap: int,
+) -> list[SectionChunk]:
+    """Split structured sections into chunks while preserving heading/page metadata.
+
+    Each section is chunked independently.  Short sections (≤ *chunk_size*)
+    become a single chunk; long sections are split internally using
+    table-aware logic that avoids breaking Markdown tables.
+
+    Args:
+        sections: List of TextSection objects (from ParseResult).
+        chunk_size: Maximum characters per chunk.
+        chunk_overlap: Overlap between consecutive chunks within a section.
+
+    Returns:
+        List of :class:`SectionChunk` instances.
+    """
+    if not sections:
+        return []
+
+    heading_paths = _compute_heading_paths(sections)
+    result: list[SectionChunk] = []
+
+    for sec_idx, section in enumerate(sections):
+        content = section.content
+        if not content or not content.strip():
+            continue
+
+        heading = getattr(section, "heading", None)
+        level = getattr(section, "level", 0)
+        page = getattr(section, "page", None)
+        h_path = heading_paths[sec_idx] if sec_idx < len(heading_paths) else ""
+
+        sub_chunks = _split_section_content(content, chunk_size, chunk_overlap)
+
+        for sc_text, sc_has_table in sub_chunks:
+            result.append(
+                SectionChunk(
+                    text=sc_text,
+                    heading=heading,
+                    level=level,
+                    page=page,
+                    heading_path=h_path,
+                    has_table=sc_has_table,
+                    section_index=sec_idx,
+                )
+            )
+
+    return result
+
+
+def _compute_heading_paths(sections: list) -> list[str]:
+    """Compute hierarchical heading paths for each section using a stack.
+
+    For example, if sections have headings at levels:
+        [("Chapter 1", 1), ("1.1 Overview", 2), ("1.2 Details", 2), ("Chapter 2", 1)]
+    the paths would be:
+        ["Chapter 1", "Chapter 1 > 1.1 Overview", "Chapter 1 > 1.2 Details", "Chapter 2"]
+    """
+    stack: list[tuple[int, str]] = []  # (level, heading)
+    paths: list[str] = []
+
+    for section in sections:
+        heading = getattr(section, "heading", None) or ""
+        level = getattr(section, "level", 0)
+
+        # Pop entries at same or deeper level
+        while stack and stack[-1][0] >= level:
+            stack.pop()
+
+        stack.append((level, heading))
+        paths.append(" > ".join(h for _, h in stack if h))
+
+    return paths
+
+
+def _segment_text_and_tables(content: str) -> list[tuple[str, bool]]:
+    """Split content into alternating (text_block, is_table) segments.
+
+    Contiguous lines matching ``|...|`` are grouped as a single table segment.
+    Everything else is grouped as a text segment.
+
+    Returns:
+        List of ``(segment_text, is_table)`` tuples.
+    """
+    lines = content.split("\n")
+    segments: list[tuple[str, bool]] = []
+    current_lines: list[str] = []
+    in_table = False
+
+    for line in lines:
+        is_table_line = bool(_TABLE_LINE_RE.match(line))
+
+        if is_table_line != in_table:
+            # Flush current segment
+            if current_lines:
+                text = "\n".join(current_lines)
+                if text.strip():
+                    segments.append((text, in_table))
+                current_lines = []
+            in_table = is_table_line
+
+        current_lines.append(line)
+
+    # Flush last segment
+    if current_lines:
+        text = "\n".join(current_lines)
+        if text.strip():
+            segments.append((text, in_table))
+
+    return segments
+
+
+def _split_section_content(
+    content: str, chunk_size: int, overlap: int
+) -> list[tuple[str, bool]]:
+    """Split a single section's content into chunks, keeping tables intact.
+
+    Returns:
+        List of ``(chunk_text, has_table)`` tuples.
+    """
+    # If the whole section fits, return it directly
+    if len(content) <= chunk_size:
+        has_table = bool(_TABLE_LINE_RE.search(content))
+        return [(content, has_table)]
+
+    segments = _segment_text_and_tables(content)
+
+    # If only one segment (no tables or all-table), use simpler logic
+    if len(segments) == 1:
+        seg_text, is_table = segments[0]
+        if is_table:
+            return [(t, True) for t in _split_table_by_rows(seg_text, chunk_size)]
+        else:
+            return [(t, False) for t in chunk_text(seg_text, chunk_size, overlap)]
+
+    # Multiple segments: try to pack segments into chunks respecting chunk_size
+    result: list[tuple[str, bool]] = []
+    current_parts: list[str] = []
+    current_len = 0
+    current_has_table = False
+
+    for seg_text, is_table in segments:
+        seg_len = len(seg_text)
+
+        if is_table and seg_len > chunk_size:
+            # Flush accumulated text first
+            if current_parts:
+                result.append(("\n".join(current_parts), current_has_table))
+                current_parts = []
+                current_len = 0
+                current_has_table = False
+            # Split the oversized table
+            for t in _split_table_by_rows(seg_text, chunk_size):
+                result.append((t, True))
+            continue
+
+        if not is_table and seg_len > chunk_size:
+            # Flush accumulated text first
+            if current_parts:
+                result.append(("\n".join(current_parts), current_has_table))
+                current_parts = []
+                current_len = 0
+                current_has_table = False
+            # Split the oversized text
+            for t in chunk_text(seg_text, chunk_size, overlap):
+                result.append((t, False))
+            continue
+
+        # Would adding this segment exceed chunk_size?
+        added_len = seg_len + (1 if current_parts else 0)  # newline joiner
+        if current_parts and current_len + added_len > chunk_size:
+            result.append(("\n".join(current_parts), current_has_table))
+            current_parts = []
+            current_len = 0
+            current_has_table = False
+
+        current_parts.append(seg_text)
+        current_len += added_len
+        if is_table:
+            current_has_table = True
+
+    if current_parts:
+        result.append(("\n".join(current_parts), current_has_table))
+
+    return result
+
+
+def _split_table_by_rows(table_text: str, chunk_size: int) -> list[str]:
+    """Split an oversized Markdown table by rows, preserving the header.
+
+    Keeps the first row (header) and the second row (separator ``|---|``)
+    at the top of every chunk so that each chunk is a valid table fragment.
+    """
+    lines = table_text.split("\n")
+    if len(lines) <= 2:
+        return [table_text]
+
+    # Identify header: first two lines (header + separator)
+    header_lines = lines[:2]
+    header_text = "\n".join(header_lines)
+    header_len = len(header_text)
+
+    data_lines = lines[2:]
+    if not data_lines:
+        return [table_text]
+
+    chunks: list[str] = []
+    current_rows: list[str] = []
+    current_len = header_len
+
+    for row in data_lines:
+        row_len = len(row) + 1  # +1 for newline
+        if current_rows and current_len + row_len > chunk_size:
+            chunks.append(header_text + "\n" + "\n".join(current_rows))
+            current_rows = []
+            current_len = header_len
+
+        current_rows.append(row)
+        current_len += row_len
+
+    if current_rows:
+        chunks.append(header_text + "\n" + "\n".join(current_rows))
+
+    return chunks if chunks else [table_text]

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/langrag.py

@@ -0,0 +1,964 @@
+import logging
+from collections.abc import Awaitable, Iterable
+from components.shared_state import SerialState, serialized
+
+from langbot_plugin.api.definition.components.knowledge_engine import (
+    KnowledgeEngine,
+    KnowledgeEngineCapability,
+)
+from langbot_plugin.api.entities.builtin.rag import (
+    IngestionContext,
+    IngestionResult,
+    RetrievalContext,
+    RetrievalResponse,
+    RetrievalResultEntry,
+    DocumentStatus,
+    SearchType,
+)
+
+from .parser import FileParser
+from .query_rewrite import retrieve_with_rewrite
+from .rerank import llm_rerank, model_rerank
+from .strategies import get_strategy
+from components.observability.telemetry import add_stage_duration, hash_text, telemetry
+
+logger = logging.getLogger(__name__)
+
+# Batch size for embedding API calls.
+# Larger batches = fewer round-trips.  Keep under ~64 to avoid IPC response
+# timeouts.
+EMBEDDING_BATCH_SIZE = 32
+PARSED_TEXT_LIMIT_BYTES = 4 * 1024 * 1024
+
+
+def _validate_parsed_text_budget(parts: Iterable[str]) -> None:
+    """Bound aggregate UTF-8 content before strategy work, without joining it."""
+    total_bytes = 0
+    for text in parts:
+        # Encode bounded slices so even a rejected input cannot require a
+        # second, arbitrarily large allocation just to measure its byte size.
+        for offset in range(0, len(text), 64 * 1024):
+            total_bytes += len(text[offset:offset + 64 * 1024].encode("utf-8"))
+            if total_bytes > PARSED_TEXT_LIMIT_BYTES:
+                raise ValueError("Parsed text exceeds the 4 MiB UTF-8 byte limit")
+
+
+def _query_log_ref(query: str | None) -> str:
+    """Return a privacy-preserving query reference for logs."""
+    return f"hash={hash_text(query)}, length={len(query or '')}"
+
+
+def _trace_spans(trace_id: str, stage_durations: dict[str, float]) -> list[dict]:
+    """Build LangRAG-local spans from recorded stage timings."""
+    return [
+        {
+            "trace_id": trace_id,
+            "span_id": f"{trace_id}:{stage}",
+            "name": stage,
+            "kind": "langrag.stage",
+            "status": "completed",
+            "duration_ms": duration,
+        }
+        for stage, duration in stage_durations.items()
+    ]
+
+
+class LangRAG(KnowledgeEngine):
+    """Simple Knowledge Engine implementation using Plugin IPC.
+
+    Provides:
+    - Document ingestion with parsing, chunking, embedding, and vector storage
+    - Vector-based retrieval
+    - Full integration with Host's embedding models and vector database
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._state = SerialState()
+
+    async def initialize(self) -> None:
+        await super().initialize()
+        # The runtime dispatches installation revocation to the plugin instance,
+        # which holds no back-reference to its components; hand it this
+        # component's state so the revocation hook can drop per-binding locks and
+        # fences. Without the hook (langbot-plugin 0.7.4) this attribute is simply
+        # never read.
+        plugin = getattr(self, 'plugin', None)
+        if plugin is not None:
+            plugin.knowledge_engine_serial_state = self._state
+
+    @classmethod
+    def get_capabilities(cls) -> list[str]:
+        """Declare supported capabilities."""
+        return [
+            KnowledgeEngineCapability.DOC_INGESTION,
+            KnowledgeEngineCapability.DOC_PARSING,
+        ]
+
+    # ========== Lifecycle Hooks ==========
+
+    async def on_knowledge_base_create(self, kb_id: str, config: dict) -> None:
+        logger.info(f"Knowledge base created: {kb_id}")
+
+    async def on_knowledge_base_delete(self, kb_id: str) -> None:
+        logger.info(f"Knowledge base deleted: {kb_id}")
+        # Deleting a knowledge base resolves its pending ambiguity, so this is the
+        # operator path out of a fence; a fresh knowledge base starts unfenced.
+        await self._state.clear_fence(self.plugin, kb_id)
+
+    # ========== Helpers ==========
+
+    async def _embed_and_upsert(
+        self,
+        collection_id: str,
+        embedding_model_uuid: str,
+        texts: list[str],
+        ids: list[str],
+        metas: list[dict],
+        trace_id: str | None = None,
+        parent_stage_durations: dict[str, float] | None = None,
+    ) -> int:
+        """Embed a batch of texts and upsert into the vector store.
+
+        Returns the number of vectors stored.
+        """
+        telemetry = self.plugin.telemetry
+        started_at = telemetry.start_timer()
+        status = "failed"
+        error = None
+        vectors_stored = 0
+        stage_durations: dict[str, float] = {}
+        try:
+            stage_started = telemetry.start_timer()
+            vectors = await self.plugin.invoke_embedding(embedding_model_uuid, texts)
+            add_stage_duration(
+                stage_durations,
+                "embedding",
+                telemetry.elapsed_ms(stage_started),
+            )
+            if parent_stage_durations is not None:
+                add_stage_duration(
+                    parent_stage_durations,
+                    "embedding",
+                    stage_durations["embedding"],
+                )
+            stage_started = telemetry.start_timer()
+            await self._state.mutate(
+                self.plugin,
+                collection_id,
+                lambda: self.plugin.vector_upsert(
+                    collection_id=collection_id,
+                    vectors=vectors,
+                    ids=ids,
+                    metadata=metas,
+                    documents=texts,
+                ),
+                'Host vector_upsert outcome unknown',
+            )
+            add_stage_duration(
+                stage_durations,
+                "vector_upsert",
+                telemetry.elapsed_ms(stage_started),
+            )
+            if parent_stage_durations is not None:
+                add_stage_duration(
+                    parent_stage_durations,
+                    "vector_upsert",
+                    stage_durations["vector_upsert"],
+                )
+            vectors_stored = len(texts)
+            status = "completed"
+            return vectors_stored
+        except Exception as e:
+            error = e
+            raise
+        finally:
+            await self._record_diagnostic(
+                "record_embedding_batch",
+                telemetry.record_embedding_batch(
+                    collection_id=collection_id,
+                    ids=ids,
+                    metas=metas,
+                    texts=texts,
+                    status=status,
+                    duration_ms=telemetry.elapsed_ms(started_at),
+                    vectors_stored=vectors_stored,
+                    stage_durations_ms=stage_durations,
+                    trace_id=trace_id,
+                    error=error,
+                ),
+            )
+
+    @staticmethod
+    async def _record_diagnostic(label: str, awaitable: Awaitable[None]) -> None:
+        """Record one diagnostic best effort.
+
+        Telemetry is observability, never a precondition: a fenced or unreachable
+        telemetry state must not fail the operation it describes, and it must not
+        replace the error the caller is about to see.
+        """
+        try:
+            await awaitable
+        except Exception:
+            logger.warning("LangRAG telemetry %s failed", label, exc_info=True)
+
+    @staticmethod
+    def _metadata_int(value) -> int | None:
+        if isinstance(value, bool):
+            return None
+        if isinstance(value, int):
+            return value
+        if isinstance(value, str) and value.isdigit():
+            return int(value)
+        return None
+
+    @staticmethod
+    def _neighbor_id(meta: dict, offset: int) -> str | None:
+        """Build the vector ID for an adjacent retrieval unit.
+
+        The three index strategies use different vector ID schemes.  Context
+        expansion only needs one representative vector per adjacent unit
+        because the stored metadata ``text`` field carries the display context.
+        """
+        doc_id = meta.get("document_id", "")
+        index_type = meta.get("index_type", "chunk")
+
+        if index_type == "parent_child":
+            parent_idx = LangRAG._metadata_int(meta.get("parent_index"))
+            if doc_id and parent_idx is not None and parent_idx + offset >= 0:
+                return f"{doc_id}_p{parent_idx + offset}_c0"
+            return None
+
+        if index_type == "qa":
+            chunk_idx = LangRAG._metadata_int(meta.get("chunk_index"))
+            if doc_id and chunk_idx is not None and chunk_idx + offset >= 0:
+                return f"{doc_id}_{chunk_idx + offset}_qa0"
+            return None
+
+        chunk_idx = LangRAG._metadata_int(meta.get("chunk_index"))
+        if doc_id and chunk_idx is not None and chunk_idx + offset >= 0:
+            return f"{doc_id}_{chunk_idx + offset}"
+        return None
+
+    async def _expand_context(
+        self,
+        results: list[dict],
+        collection_id: str,
+        window: int,
+    ) -> None:
+        """Expand each result with adjacent chunks from the same document.
+
+        For each hit, looks up adjacent retrieval units from the same document
+        and appends their text to the result's metadata as
+        ``context_before`` and ``context_after``.
+
+        Requires the vector store to support ``vector_get_by_ids``.  If the
+        method is unavailable or fails, this is a no-op.
+        """
+        get_by_ids = getattr(self.plugin, "vector_get_by_ids", None)
+        if not callable(get_by_ids):
+            return
+
+        ids_to_fetch: set[str] = set()
+        # Map result → adjacent IDs needed
+        for res in results:
+            meta = res.get("metadata", {})
+            for offset in range(1, window + 1):
+                before_id = self._neighbor_id(meta, -offset)
+                after_id = self._neighbor_id(meta, offset)
+                if before_id:
+                    ids_to_fetch.add(before_id)
+                if after_id:
+                    ids_to_fetch.add(after_id)
+
+        if not ids_to_fetch:
+            return
+
+        adjacent = await get_by_ids(
+            collection_id=collection_id, ids=list(ids_to_fetch)
+        )
+        # Build lookup: id → text
+        adj_map: dict[str, str] = {}
+        for item in adjacent:
+            item_id = item.get("id", "")
+            item_text = item.get("metadata", {}).get("text", "")
+            if item_id and item_text:
+                adj_map[item_id] = item_text
+
+        # Attach context to results
+        for res in results:
+            meta = res.get("metadata", {})
+            before_parts = []
+            after_parts = []
+            for offset in range(1, window + 1):
+                before_id = self._neighbor_id(meta, -offset)
+                after_id = self._neighbor_id(meta, offset)
+                if before_id in adj_map:
+                    before_parts.insert(0, adj_map[before_id])
+                if after_id in adj_map:
+                    after_parts.append(adj_map[after_id])
+            if before_parts:
+                meta["context_before"] = "\n".join(before_parts)
+            if after_parts:
+                meta["context_after"] = "\n".join(after_parts)
+
+    @staticmethod
+    async def _timed_batches(generator, stage_durations: dict[str, float]):
+        """Yield strategy batches while accumulating generator runtime."""
+        iterator = generator.__aiter__()
+        while True:
+            stage_started = telemetry.start_timer()
+            try:
+                batch = await iterator.__anext__()
+            except StopAsyncIteration:
+                break
+            else:
+                add_stage_duration(
+                    stage_durations,
+                    "chunking",
+                    telemetry.elapsed_ms(stage_started),
+                )
+                yield batch
+
+    # ========== Core Methods ==========
+
+    @serialized
+    async def ingest(self, context: IngestionContext) -> IngestionResult:
+        """Handle document ingestion: Read -> Parse -> Chunk -> Embed -> Store.
+
+        Uses an async generator pipeline: the strategy yields batches
+        incrementally, and each batch is embedded and upserted as soon as it
+        is ready.  This ensures partial results are persisted early.
+        """
+        telemetry = self.plugin.telemetry
+        started_at = telemetry.start_timer()
+        trace_id = telemetry.new_trace_id("ingest")
+        stage_durations: dict[str, float] = {}
+        doc_id = context.file_object.metadata.document_id
+        filename = context.file_object.metadata.filename
+        file_size = getattr(context.file_object.metadata, "file_size", None)
+        metadata_extra = getattr(context.file_object.metadata, "extra", {}) or {}
+        collection_id = context.get_collection_id()
+        index_type = context.creation_settings.get("index_type") or "chunk"
+        telemetry_status = DocumentStatus.FAILED
+        telemetry_error = None
+        telemetry_chunks_created = 0
+        telemetry_text_length = None
+        telemetry_content_hash = None
+        telemetry_sections_count = None
+        parser_source = None
+        correlation = telemetry.correlation_summary(
+            context.creation_settings,
+            metadata_extra,
+            trace_id=trace_id,
+        )
+
+        logger.info(
+            f"Ingesting file: {filename} (doc={doc_id}) into collection: {collection_id}"
+        )
+
+        try:
+            # 1. Parse file content (prefer pre-parsed content from external Parser plugin)
+            sections = None
+            doc_metadata = None
+            if context.parsed_content and context.parsed_content.text:
+                stage_started = telemetry.start_timer()
+                text_content = context.parsed_content.text
+                parser_source = "external"
+                logger.info(
+                    f"Using pre-parsed content from external parser for {filename}"
+                )
+                # Extract structured sections and metadata if available
+                if context.parsed_content.sections:
+                    sections = context.parsed_content.sections
+                    telemetry_sections_count = len(sections)
+                    logger.info(
+                        f"Found {len(sections)} structured sections from parser"
+                    )
+                if context.parsed_content.metadata:
+                    doc_metadata = context.parsed_content.metadata
+                    correlation.update(
+                        telemetry.correlation_summary(doc_metadata, trace_id=trace_id)
+                    )
+                    logger.info(
+                        f"Found document metadata from parser: "
+                        f"{[k for k in doc_metadata if k != 'images']}"
+                    )
+                add_stage_duration(
+                    stage_durations,
+                    "parser_external",
+                    telemetry.elapsed_ms(stage_started),
+                )
+            else:
+                parser_source = "internal"
+                logger.warning(
+                    f"No external parser content for {filename}; "
+                    "falling back to internal FileParser. Consider configuring an "
+                    "external parser (e.g. GeneralParsers) for better results."
+                )
+                try:
+                    stage_started = telemetry.start_timer()
+                    content_bytes = await self.plugin.get_knowledge_file_stream(
+                        context.file_object.storage_path
+                    )
+                    add_stage_duration(
+                        stage_durations,
+                        "file_read",
+                        telemetry.elapsed_ms(stage_started),
+                    )
+                except Exception as e:
+                    telemetry_error = e
+                    telemetry_status = DocumentStatus.FAILED
+                    logger.error(f"Failed to get file content: {e}")
+                    return IngestionResult(
+                        document_id=doc_id,
+                        status=DocumentStatus.FAILED,
+                        error_message=f"Could not read file: {e}",
+                    )
+                parser = FileParser(self.plugin.offload)
+                stage_started = telemetry.start_timer()
+                text_content = await parser.parse(content_bytes, filename)
+                add_stage_duration(
+                    stage_durations,
+                    "parser_internal",
+                    telemetry.elapsed_ms(stage_started),
+                )
+
+            # Bound both representations independently: strategies consume the
+            # sections when present, not necessarily the parser's flat text.
+            _validate_parsed_text_budget((text_content or "",))
+            if sections:
+                _validate_parsed_text_budget(section.content for section in sections)
+            telemetry_text_length = len(text_content) if text_content else 0
+            telemetry_content_hash = hash_text(text_content)
+
+            if not text_content:
+                telemetry_status = DocumentStatus.COMPLETED
+                logger.warning(f"No text content extracted from file: {filename}")
+                return IngestionResult(
+                    document_id=doc_id,
+                    status=DocumentStatus.COMPLETED,
+                    chunks_created=0,
+                )
+
+            # 3. Build chunks via strategy (async generator)
+            strategy = get_strategy(index_type)
+            embedding_model_uuid = context.creation_settings.get(
+                "embedding_model_uuid", ""
+            )
+            logger.info(f"Strategy: {index_type} ({strategy.__class__.__name__})")
+
+            # 4. Progressive ingest: consume generator → accumulate → embed+upsert
+            #    when a full batch is ready.  Embedding happens between generator
+            #    yields (sequential, not concurrent) to avoid IPC timeout issues.
+            pending_texts: list[str] = []
+            pending_ids: list[str] = []
+            pending_metas: list[dict] = []
+            total_stored = 0
+
+            async for (
+                batch_texts,
+                batch_ids,
+                batch_metas,
+            ) in self._timed_batches(
+                strategy.build_chunks_and_metadata(
+                    text_content,
+                    doc_id,
+                    filename,
+                    context.creation_settings,
+                    plugin=self.plugin,
+                    sections=sections,
+                    doc_metadata=doc_metadata,
+                ),
+                stage_durations,
+            ):
+                pending_texts.extend(batch_texts)
+                pending_ids.extend(batch_ids)
+                pending_metas.extend(batch_metas)
+
+                # Flush full batches immediately
+                while len(pending_texts) >= EMBEDDING_BATCH_SIZE:
+                    t = pending_texts[:EMBEDDING_BATCH_SIZE]
+                    i = pending_ids[:EMBEDDING_BATCH_SIZE]
+                    m = pending_metas[:EMBEDDING_BATCH_SIZE]
+                    pending_texts = pending_texts[EMBEDDING_BATCH_SIZE:]
+                    pending_ids = pending_ids[EMBEDDING_BATCH_SIZE:]
+                    pending_metas = pending_metas[EMBEDDING_BATCH_SIZE:]
+                    total_stored += await self._embed_and_upsert(
+                        collection_id,
+                        embedding_model_uuid,
+                        t,
+                        i,
+                        m,
+                        trace_id=trace_id,
+                        parent_stage_durations=stage_durations,
+                    )
+                    telemetry_chunks_created = total_stored
+
+            # Flush remaining
+            if pending_texts:
+                total_stored += await self._embed_and_upsert(
+                    collection_id,
+                    embedding_model_uuid,
+                    pending_texts,
+                    pending_ids,
+                    pending_metas,
+                    trace_id=trace_id,
+                    parent_stage_durations=stage_durations,
+                )
+                telemetry_chunks_created = total_stored
+
+            if total_stored:
+                unit = "Q&A pairs" if index_type == "qa" else "chunks"
+                logger.info(f"Ingestion complete: {total_stored} {unit} stored")
+
+            telemetry_status = DocumentStatus.COMPLETED
+            telemetry_chunks_created = total_stored
+            return IngestionResult(
+                document_id=doc_id,
+                status=DocumentStatus.COMPLETED,
+                chunks_created=total_stored,
+            )
+
+        except Exception as e:
+            telemetry_error = e
+            telemetry_status = DocumentStatus.FAILED
+            logger.error(f"Ingestion failed for {filename}: {e}")
+            return IngestionResult(
+                document_id=doc_id,
+                status=DocumentStatus.FAILED,
+                error_message=str(e),
+            )
+        finally:
+            await self._record_diagnostic(
+                "record_ingest",
+                telemetry.record_ingest(
+                    document_id=doc_id,
+                    filename=filename,
+                    collection_id=collection_id,
+                    status=telemetry_status,
+                    duration_ms=telemetry.elapsed_ms(started_at),
+                    index_type=index_type,
+                    chunks_created=telemetry_chunks_created,
+                    file_size=file_size,
+                    text_length=telemetry_text_length,
+                    content_hash=telemetry_content_hash,
+                    sections_count=telemetry_sections_count,
+                    settings=context.creation_settings,
+                    stage_durations_ms=stage_durations,
+                    parser_source=parser_source,
+                    knowledge_base_id=context.knowledge_base_id,
+                    trace_id=trace_id,
+                    correlation=correlation,
+                    error=telemetry_error,
+                ),
+            )
+
+    async def retrieve(self, context: RetrievalContext) -> RetrievalResponse:
+        """Retrieve relevant content with support for vector, full-text, and hybrid search."""
+        telemetry = self.plugin.telemetry
+        started_at = telemetry.start_timer()
+        trace_id = telemetry.new_trace_id("retrieval")
+        stage_durations: dict[str, float] = {}
+        query = context.query
+        top_k = context.retrieval_settings.get("top_k", 5)
+        collection_id = context.get_collection_id()
+        search_type = context.retrieval_settings.get("search_type", SearchType.VECTOR)
+        raw_count = None
+        result_count = None
+        fetch_k = None
+        reranked = False
+        telemetry_status = "failed"
+        telemetry_error = None
+        reference_count = None
+        heading_count = None
+        context_expanded_count = None
+        distance_min = None
+        distance_avg = None
+        distance_max = None
+        correlation = telemetry.correlation_summary(
+            context.filters,
+            context.creation_settings,
+            context.retrieval_settings,
+            trace_id=trace_id,
+        )
+
+        try:
+            # Read hybrid fusion weight (YAML default 0.7; vdb fallback handles None)
+            vector_weight = float(context.retrieval_settings.get("vector_weight", 0.7))
+
+            # Determine strategy for post-processing
+            index_type = context.creation_settings.get("index_type") or "chunk"
+            strategy = get_strategy(index_type)
+            logger.info(
+                f"Retrieve: strategy={index_type}, top_k={top_k}, "
+                f"query_rewrite={context.retrieval_settings.get('query_rewrite', 'off')}, "
+                f"query_ref=({_query_log_ref(query)})"
+            )
+            logger.info(
+                f"Retrieve search config: search_type={search_type}, "
+                f"vector_weight={vector_weight}"
+            )
+
+            rerank_mode = context.retrieval_settings.get("rerank", "off")
+            rerank_llm = context.retrieval_settings.get("rerank_llm_model_uuid", "")
+            rerank_model = context.retrieval_settings.get("rerank_model_uuid", "")
+            rerank_enabled = (
+                rerank_mode == "llm"
+                and bool(rerank_llm)
+            ) or (
+                rerank_mode in ("rerank_model", "model")
+                and bool(rerank_model)
+            )
+
+            # For parent_child/qa, over-fetch to allow dedup to still yield top_k.
+            # When reranking is enabled, keep a larger candidate pool for rerankers.
+            if index_type in ("parent_child", "qa") or rerank_enabled:
+                fetch_k = top_k * 3
+            else:
+                fetch_k = top_k
+
+            # Check query rewrite settings
+            query_rewrite = context.retrieval_settings.get("query_rewrite", "off")
+            rewrite_llm = context.retrieval_settings.get("rewrite_llm_model_uuid", "")
+
+            if query_rewrite != "off" and rewrite_llm:
+                logger.info(f"Query rewrite enabled: strategy={query_rewrite}")
+                stage_started = telemetry.start_timer()
+                results = await retrieve_with_rewrite(
+                    plugin=self.plugin,
+                    query=query,
+                    query_rewrite=query_rewrite,
+                    rewrite_llm=rewrite_llm,
+                    collection_id=collection_id,
+                    embedding_model_uuid=context.creation_settings.get(
+                        "embedding_model_uuid", ""
+                    ),
+                    fetch_k=fetch_k,
+                    filters=context.filters or None,
+                    search_type=search_type,
+                    vector_weight=vector_weight,
+                    stage_durations=stage_durations,
+                )
+                add_stage_duration(
+                    stage_durations,
+                    f"rewrite_{query_rewrite}",
+                    telemetry.elapsed_ms(stage_started),
+                )
+            else:
+                # Original logic: embed query → vector_search
+                query_vector: list[float] = []
+                if search_type != SearchType.FULL_TEXT:
+                    embedding_model_uuid = context.creation_settings.get(
+                        "embedding_model_uuid", ""
+                    )
+                    stage_started = telemetry.start_timer()
+                    query_vectors = await self.plugin.invoke_embedding(
+                        embedding_model_uuid, [query]
+                    )
+                    query_vector = query_vectors[0]
+                    add_stage_duration(
+                        stage_durations,
+                        "query_embedding",
+                        telemetry.elapsed_ms(stage_started),
+                    )
+
+                stage_started = telemetry.start_timer()
+                results = await self.plugin.vector_search(
+                    collection_id=collection_id,
+                    query_vector=query_vector,
+                    top_k=fetch_k,
+                    filters=context.filters or None,
+                    search_type=search_type,
+                    query_text=query,
+                    vector_weight=vector_weight,
+                )
+                add_stage_duration(
+                    stage_durations,
+                    "vector_search",
+                    telemetry.elapsed_ms(stage_started),
+                )
+
+            # Post-process (strategy may deduplicate). If reranking is enabled,
+            # preserve the over-fetched pool and let the reranker apply top_k.
+            raw_count = len(results)
+            postprocess_k = fetch_k if rerank_enabled else top_k
+            stage_started = telemetry.start_timer()
+            results = strategy.postprocess_results(results, postprocess_k)
+            add_stage_duration(
+                stage_durations,
+                "postprocess",
+                telemetry.elapsed_ms(stage_started),
+            )
+            logger.info(
+                f"Retrieve post-process: {raw_count} raw → {len(results)} after dedup "
+                f"(fetch_k={fetch_k}, postprocess_k={postprocess_k})"
+            )
+
+            if rerank_mode in ("rerank_model", "model") and rerank_model:
+                logger.info("[Rerank] Host rerank model enabled")
+                stage_started = telemetry.start_timer()
+                results = await model_rerank(
+                    plugin=self.plugin,
+                    rerank_model_uuid=rerank_model,
+                    query=query,
+                    results=results,
+                    top_k=top_k,
+                )
+                add_stage_duration(
+                    stage_durations,
+                    "rerank_model",
+                    telemetry.elapsed_ms(stage_started),
+                )
+                reranked = True
+            elif rerank_mode == "llm" and rerank_llm:
+                logger.info("[Rerank] LLM reranking enabled")
+                stage_started = telemetry.start_timer()
+                results = await llm_rerank(
+                    plugin=self.plugin,
+                    llm_uuid=rerank_llm,
+                    query=query,
+                    results=results,
+                    top_k=top_k,
+                )
+                add_stage_duration(
+                    stage_durations,
+                    "rerank_llm",
+                    telemetry.elapsed_ms(stage_started),
+                )
+                reranked = True
+
+            # C1: Heading hit weighting — boost results whose heading_path
+            # contains query keywords.  Skipped when LLM reranking is active
+            # because the LLM already understands heading relevance.
+            if not reranked:
+                stage_started = telemetry.start_timer()
+                query_keywords = [w for w in query.lower().split() if len(w) >= 2]
+                if query_keywords:
+                    for res in results:
+                        heading_path = (
+                            res.get("metadata", {}).get("heading_path", "") or ""
+                        ).lower()
+                        if not heading_path:
+                            continue
+                        distance = res.get("distance")
+                        if distance is not None and isinstance(distance, (int, float)):
+                            for kw in query_keywords:
+                                if kw in heading_path:
+                                    distance *= 0.9
+                            res["distance"] = distance
+                    # Re-sort by distance (lower is better) after weighting
+                    results.sort(
+                        key=lambda r: (
+                            r.get("distance")
+                            if isinstance(r.get("distance"), (int, float))
+                            else float("inf")
+                        )
+                    )
+                add_stage_duration(
+                    stage_durations,
+                    "heading_weight",
+                    telemetry.elapsed_ms(stage_started),
+                )
+
+            # C2: Context window — attempt to fetch adjacent chunks from the same
+            # document to provide surrounding context.  Only works when the vector
+            # store supports ``vector_get_by_ids`` (gracefully skipped otherwise).
+            context_window = context.retrieval_settings.get("context_window", 0)
+            if context_window and context_window > 0:
+                try:
+                    stage_started = telemetry.start_timer()
+                    await self._expand_context(results, collection_id, context_window)
+                    add_stage_duration(
+                        stage_durations,
+                        "context_expansion",
+                        telemetry.elapsed_ms(stage_started),
+                    )
+                except Exception as e:
+                    logger.debug(
+                        f"Context window expansion skipped (vector_get_by_ids "
+                        f"not supported or failed): {e}"
+                    )
+
+            # Format results
+            stage_started = telemetry.start_timer()
+            distances = [
+                float(res["distance"])
+                for res in results
+                if isinstance(res.get("distance"), (int, float))
+            ]
+            if distances:
+                distance_min = round(min(distances), 6)
+                distance_avg = round(sum(distances) / len(distances), 6)
+                distance_max = round(max(distances), 6)
+            entries: list[RetrievalResultEntry] = []
+            for res in results:
+                meta = res.get("metadata", {})
+                content_text = meta.get("text", "")
+                raw_score = res.get("score")
+                distance = res.get("distance")
+                if distance is None and raw_score is not None:
+                    # Compatibility with older hosts that incorrectly returned
+                    # distance under the score field.
+                    distance = raw_score
+
+                doc_name = meta.get("document_name", "")
+                page = meta.get("page")
+                heading_path = meta.get("heading_path", "")
+
+                # Build structured reference string
+                ref_parts = [doc_name]
+                if page is not None:
+                    ref_parts.append(f"p.{page}")
+                if heading_path:
+                    ref_parts.append(f'"{heading_path}"')
+                reference = "[" + ", ".join(ref_parts) + "]" if ref_parts else ""
+
+                content_entry: dict = {
+                    "type": "text",
+                    "text": content_text,
+                    "file_name": doc_name,
+                }
+                if page is not None:
+                    content_entry["page"] = page
+                if heading_path:
+                    content_entry["heading_path"] = heading_path
+                if reference:
+                    content_entry["reference"] = reference
+
+                entries.append(
+                    RetrievalResultEntry(
+                        id=res["id"],
+                        content=[content_entry],
+                        metadata=meta,
+                        score=raw_score,
+                        distance=distance,
+                    )
+                )
+
+            result_count = len(entries)
+            reference_count = sum(
+                1
+                for entry in entries
+                for item in entry.content
+                if isinstance(item, dict) and item.get("reference")
+            )
+            heading_count = sum(
+                1
+                for entry in entries
+                for item in entry.content
+                if isinstance(item, dict) and item.get("heading_path")
+            )
+            context_expanded_count = sum(
+                1
+                for entry in entries
+                if entry.metadata.get("context_before")
+                or entry.metadata.get("context_after")
+            )
+            add_stage_duration(
+                stage_durations,
+                "format_response",
+                telemetry.elapsed_ms(stage_started),
+            )
+            telemetry_status = "completed"
+            trace_spans = _trace_spans(trace_id, stage_durations)
+            return RetrievalResponse(
+                results=entries,
+                total_found=len(entries),
+                metadata={
+                    "trace_id": trace_id,
+                    "raw_count": raw_count,
+                    "stage_durations_ms": stage_durations,
+                    "trace_spans": trace_spans,
+                },
+            )
+        except Exception as e:
+            telemetry_error = e
+            telemetry_status = "failed"
+            raise
+        finally:
+            await self._record_diagnostic(
+                "record_retrieval",
+                telemetry.record_retrieval(
+                    query=query,
+                    collection_id=collection_id,
+                    status=telemetry_status,
+                    duration_ms=telemetry.elapsed_ms(started_at),
+                    index_type=context.creation_settings.get("index_type") or "chunk",
+                    search_type=search_type,
+                    top_k=top_k,
+                    fetch_k=fetch_k,
+                    raw_count=raw_count,
+                    result_count=result_count,
+                    reference_count=reference_count,
+                    heading_count=heading_count,
+                    context_expanded_count=context_expanded_count,
+                    distance_min=distance_min,
+                    distance_avg=distance_avg,
+                    distance_max=distance_max,
+                    filters=context.filters,
+                    creation_settings=context.creation_settings,
+                    retrieval_settings=context.retrieval_settings,
+                    stage_durations_ms=stage_durations,
+                    trace_spans=_trace_spans(trace_id, stage_durations),
+                    reranked=reranked,
+                    knowledge_base_id=context.knowledge_base_id,
+                    trace_id=trace_id,
+                    correlation=correlation,
+                    error=telemetry_error,
+                ),
+            )
+
+    @serialized
+    async def delete_document(self, kb_id: str, document_id: str) -> bool:
+        """Delete a document's vectors by file_id."""
+        telemetry = self.plugin.telemetry
+        started_at = telemetry.start_timer()
+        trace_id = telemetry.new_trace_id("delete")
+        status = "failed"
+        error = None
+        deleted = None
+        count = None
+        try:
+            result_count = await self._state.mutate(
+                self.plugin,
+                kb_id,
+                lambda: self.plugin.vector_delete(
+                    collection_id=kb_id,
+                    file_ids=[document_id],
+                ),
+                'Host vector_delete outcome unknown',
+            )
+            # The Host answered, but a malformed count does not say what happened
+            # to the rows: the vector store's state is unknown, so this knowledge
+            # base has to be reconciled before it accepts another mutation.
+            if type(result_count) is not int or result_count < 0:
+                await self._state.fence(
+                    self.plugin,
+                    kb_id,
+                    'Host vector_delete returned a malformed count',
+                )
+                raise ValueError("Host vector_delete must return a nonnegative integer count")
+            count = result_count
+            # An authoritative no-op also confirms absence (empty ingest or retry).
+            deleted = True
+            status = "completed"
+            return deleted
+        except Exception as e:
+            error = e
+            raise
+        finally:
+            await self._record_diagnostic(
+                "record_delete",
+                telemetry.record_delete(
+                    collection_id=kb_id,
+                    document_id=document_id,
+                    status=status,
+                    duration_ms=telemetry.elapsed_ms(started_at),
+                    deleted=deleted,
+                    vectors_deleted=count,
+                    knowledge_base_id=kb_id,
+                    trace_id=trace_id,
+                    error=error,
+                ),
+            )

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/langrag.yaml

@@ -0,0 +1,252 @@
+apiVersion: v1
+kind: KnowledgeEngine
+metadata:
+  name: langrag
+  label:
+    en_US: LangRAG
+    zh_Hans: LangRAG
+  description:
+    en_US: Simple Knowledge Engine with document chunking, vector, full-text, and hybrid retrieval
+    zh_Hans: 简单 RAG 引擎，支持文档分块及向量、全文、混合检索
+spec:
+  creation_schema:
+    - name: embedding_model_uuid
+      label:
+        en_US: Embedding Model
+        zh_Hans: 嵌入模型
+      description:
+        en_US: Select embedding model for text vectorization
+        zh_Hans: 选择用于文本向量化的嵌入模型
+      type: embedding-model-selector
+      required: true
+      default: ''
+    - name: index_type
+      label:
+        en_US: Index Type
+        zh_Hans: 索引类型
+      description:
+        en_US: 'Chunk: flat chunking; Parent-Child: two-level chunking for richer context; QA: LLM-generated question-answer pairs for better query matching'
+        zh_Hans: 'Chunk：平坦分块；Parent-Child：双层分块，检索返回更丰富上下文；QA：LLM 生成问答对，提升查询匹配质量'
+      type: select
+      required: false
+      default: chunk
+      options:
+        - name: chunk
+          label:
+            en_US: Chunk
+            zh_Hans: 分块
+        - name: parent_child
+          label:
+            en_US: Parent-Child
+            zh_Hans: 父子索引
+        - name: qa
+          label:
+            en_US: QA Index
+            zh_Hans: 问答索引
+    - name: chunk_size
+      label:
+        en_US: Chunk Size
+        zh_Hans: 分块大小
+      type: integer
+      required: false
+      default: 512
+    - name: overlap
+      label:
+        en_US: Chunk Overlap
+        zh_Hans: 分块重叠
+      type: integer
+      required: false
+      default: 50
+    - name: parent_chunk_size
+      label:
+        en_US: Parent Chunk Size
+        zh_Hans: 父块大小
+      description:
+        en_US: Size of parent chunks (only used with Parent-Child index type)
+        zh_Hans: 父块大小（仅在 Parent-Child 索引类型下生效）
+      type: integer
+      required: false
+      default: 2048
+      show_if:
+        field: index_type
+        operator: eq
+        value: parent_child
+    - name: child_chunk_size
+      label:
+        en_US: Child Chunk Size
+        zh_Hans: 子块大小
+      description:
+        en_US: Size of child chunks (only used with Parent-Child index type)
+        zh_Hans: 子块大小（仅在 Parent-Child 索引类型下生效）
+      type: integer
+      required: false
+      default: 256
+      show_if:
+        field: index_type
+        operator: eq
+        value: parent_child
+    - name: qa_llm_model_uuid
+      label:
+        en_US: QA Generation LLM
+        zh_Hans: QA 生成用 LLM
+      description:
+        en_US: LLM model for generating Q&A pairs (required when index type is QA)
+        zh_Hans: 用于生成问答对的 LLM 模型（索引类型为 QA 时需要）
+      type: llm-model-selector
+      required: false
+      default: ''
+      show_if:
+        field: index_type
+        operator: eq
+        value: qa
+    - name: questions_per_chunk
+      label:
+        en_US: Questions Per Chunk
+        zh_Hans: 每块生成问题数
+      description:
+        en_US: Number of Q&A pairs to generate per chunk (only used with QA index type)
+        zh_Hans: 每个文本块生成的问答对数量（仅在 QA 索引类型下生效）
+      type: integer
+      required: false
+      default: 1
+      show_if:
+        field: index_type
+        operator: eq
+        value: qa
+  retrieval_schema:
+    - name: top_k
+      label:
+        en_US: Top K
+        zh_Hans: 召回数量
+      type: integer
+      required: false
+      default: 5
+    - name: search_type
+      label:
+        en_US: Search Type
+        zh_Hans: 检索类型
+      description:
+        en_US: 'Vector: semantic similarity; Full-text: keyword matching; Hybrid: combined'
+        zh_Hans: 'Vector：语义相似度；Full-text：关键词匹配；Hybrid：混合检索'
+      type: select
+      required: false
+      default: vector
+      options:
+        - name: vector
+          label:
+            en_US: Vector
+            zh_Hans: 向量检索
+        - name: full_text
+          label:
+            en_US: Full-text
+            zh_Hans: 全文检索
+        - name: hybrid
+          label:
+            en_US: Hybrid
+            zh_Hans: 混合检索
+    - name: vector_weight
+      label:
+        en_US: Vector Search Weight
+        zh_Hans: 向量搜索权重
+      description:
+        en_US: 'Balance between vector and keyword search in hybrid mode (0.0 = keyword only, 1.0 = vector only)'
+        zh_Hans: '混合检索中向量搜索的权重（0.0 = 仅关键词，1.0 = 仅向量）'
+      type: float
+      required: false
+      default: 0.7
+      show_if:
+        field: search_type
+        operator: eq
+        value: hybrid
+    - name: query_rewrite
+      label:
+        en_US: Query Rewriting
+        zh_Hans: 查询重写
+      description:
+        en_US: 'Off: direct search; HyDE: hypothetical document; Multi-Query: query expansion; Step-Back: abstract generalization'
+        zh_Hans: 'Off：直接检索；HyDE：假设文档嵌入；Multi-Query：多查询扩展；Step-Back：回退抽象提问'
+      type: select
+      required: false
+      default: "off"
+      options:
+        - name: "off"
+          label: { en_US: "Off", zh_Hans: "关闭" }
+        - name: hyde
+          label: { en_US: "HyDE", zh_Hans: "HyDE（假设文档）" }
+        - name: multi_query
+          label: { en_US: "Multi-Query", zh_Hans: "多查询扩展" }
+        - name: step_back
+          label: { en_US: "Step-Back", zh_Hans: "回退提问" }
+    - name: rewrite_llm_model_uuid
+      label:
+        en_US: Query Rewrite LLM
+        zh_Hans: 查询重写 LLM
+      description:
+        en_US: LLM model for query rewriting (required when query rewriting is enabled)
+        zh_Hans: 用于查询重写的 LLM 模型（启用查询重写时需要）
+      type: llm-model-selector
+      required: false
+      default: ''
+      show_if:
+        field: query_rewrite
+        operator: neq
+        value: "off"
+    - name: rerank
+      label:
+        en_US: Reranking
+        zh_Hans: 重排序
+      description:
+        en_US: 'Off: no reranking; Rerank Model: use a LangBot rerank model; LLM: use a language model to rerank candidates by relevance'
+        zh_Hans: 'Off：不重排序；Rerank Model：使用 LangBot 重排序模型；LLM：使用大语言模型按相关性重排候选结果'
+      type: select
+      required: false
+      default: "off"
+      options:
+        - name: "off"
+          label: { en_US: "Off", zh_Hans: "关闭" }
+        - name: "rerank_model"
+          label: { en_US: "Rerank Model", zh_Hans: "重排序模型" }
+        - name: "llm"
+          label: { en_US: "LLM", zh_Hans: "LLM 重排序" }
+    - name: rerank_model_uuid
+      label:
+        en_US: Rerank Model
+        zh_Hans: 重排序模型
+      description:
+        en_US: LangBot rerank model for reranking retrieved candidates
+        zh_Hans: 用于对检索候选结果重排序的 LangBot 重排序模型
+      type: rerank-model-selector
+      required: false
+      default: ''
+      show_if:
+        field: rerank
+        operator: eq
+        value: "rerank_model"
+    - name: rerank_llm_model_uuid
+      label:
+        en_US: Rerank LLM
+        zh_Hans: 重排序 LLM
+      description:
+        en_US: LLM model for reranking retrieved candidates
+        zh_Hans: 用于对检索候选结果重排序的 LLM 模型
+      type: llm-model-selector
+      required: false
+      default: ''
+      show_if:
+        field: rerank
+        operator: eq
+        value: "llm"
+    - name: context_window
+      label:
+        en_US: Context Window
+        zh_Hans: 上下文窗口
+      description:
+        en_US: 'Number of adjacent chunks to include as context for each hit (0 = disabled). Requires vector store ID-based lookup support.'
+        zh_Hans: '为每个命中 chunk 附加的相邻 chunk 数量（0 = 禁用）。需要向量库支持按 ID 查询。'
+      type: integer
+      required: false
+      default: 0
+execution:
+  python:
+    path: langrag.py
+    attr: LangRAG

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/parser.py

@@ -0,0 +1,230 @@
+"""DEPRECATED: Internal file parser — fallback only.
+
+This parser provides basic text extraction for cases where no external Parser
+plugin (e.g. GeneralParsers) is configured.  It lacks section splitting,
+metadata extraction, table-aware chunking, and many other features available
+in GeneralParsers.
+
+New features should NOT be added here.  Instead, invest in GeneralParsers
+or a dedicated parser plugin.  This module will be removed in a future
+version once external parser adoption is widespread.
+"""
+
+from __future__ import annotations
+
+import io
+import re
+from components.offload import BoundedOffload
+import logging
+from typing import Union, Callable, Any
+
+import chardet
+
+
+logger = logging.getLogger(__name__)
+
+
+class FileParser:
+    """
+    A robust file parser class to extract text content from various document formats.
+    It supports TXT, PDF, DOCX, Markdown, HTML files.
+
+    This version is designed to work in the Plugin environment, accepting file bytes
+    directly rather than relying on Host's storage manager.
+    """
+
+    def __init__(self, offload=None):
+        self.offload = offload or BoundedOffload()
+
+    async def _run_sync(self, sync_func: Callable, *args: Any, **kwargs: Any) -> Any:
+        """
+        Runs a synchronous function in a separate thread to prevent blocking the event loop.
+        """
+        try:
+            return await self.offload.run(sync_func, *args, **kwargs)
+        except Exception as e:
+            logger.error(f'Error running synchronous function {sync_func.__name__}: {e}')
+            raise
+
+    async def parse(self, file_bytes: bytes, filename: str) -> Union[str, None]:
+        """
+        Parses the file based on its extension and returns the extracted text content.
+
+        Args:
+            file_bytes: The raw bytes of the file content.
+            filename: The name of the file (used to determine extension).
+
+        Returns:
+            Union[str, None]: The extracted text content as a single string, or None if parsing fails.
+        """
+        if len(file_bytes) > 16 * 1024 * 1024:
+            raise ValueError("File exceeds the 16 MiB internal parser limit")
+        # Extract extension from filename
+        if '.' in filename:
+            file_extension = filename.rsplit('.', 1)[-1].lower()
+        else:
+            file_extension = ''
+
+        parser_method = getattr(self, f'_parse_{file_extension}', None)
+
+        if parser_method is None:
+            logger.warning(f'Unsupported file format: {file_extension} for file {filename}, trying as text')
+            # Fallback: try to decode as text
+            return await self._run_sync(self._decode_text, file_bytes)
+
+        try:
+            return await parser_method(file_bytes, filename)
+        except Exception as e:
+            logger.error(f'Failed to parse {file_extension} file {filename}: {e}')
+            return None
+
+    def _decode_text(self, file_bytes: bytes) -> str:
+        """Decode bytes to text with encoding detection."""
+        detected = chardet.detect(file_bytes)
+        encoding = detected['encoding'] or 'utf-8'
+        return file_bytes.decode(encoding, errors='ignore')
+
+    # --- Specific Parser Methods ---
+
+    async def _parse_txt(self, file_bytes: bytes, filename: str) -> str:
+        """Parses a TXT file and returns its content."""
+        logger.info(f'Parsing TXT file: {filename}')
+        return await self._run_sync(self._decode_text, file_bytes)
+
+    async def _parse_pdf(self, file_bytes: bytes, filename: str) -> str:
+        """Parses a PDF file and returns its text content."""
+        logger.info(f'Parsing PDF file: {filename}')
+
+        def _parse_pdf_sync():
+            import PyPDF2
+
+            pdf_reader = PyPDF2.PdfReader(io.BytesIO(file_bytes))
+            text_content = []
+            for page in pdf_reader.pages:
+                text = page.extract_text()
+                if text:
+                    text_content.append(text)
+            return '\n'.join(text_content)
+
+        return await self._run_sync(_parse_pdf_sync)
+
+    async def _parse_docx(self, file_bytes: bytes, filename: str) -> str:
+        """Parses a DOCX file and returns its text content."""
+        logger.info(f'Parsing DOCX file: {filename}')
+
+        def _parse_docx_sync():
+            from docx import Document
+
+            doc = Document(io.BytesIO(file_bytes))
+            text_content = [paragraph.text for paragraph in doc.paragraphs if paragraph.text.strip()]
+            return '\n'.join(text_content)
+
+        return await self._run_sync(_parse_docx_sync)
+
+    async def _parse_doc(self, file_bytes: bytes, filename: str) -> str:
+        """Handles .doc files, explicitly stating lack of direct support."""
+        logger.warning(f'Direct .doc parsing is not supported for {filename}. Please convert to .docx first.')
+        raise NotImplementedError('Direct .doc parsing not supported. Please convert to .docx first.')
+
+    async def _parse_md(self, file_bytes: bytes, filename: str) -> str:
+        """Parses a Markdown file, converting it to structured plain text."""
+        logger.info(f'Parsing Markdown file: {filename}')
+
+        def _parse_markdown_sync():
+            import markdown
+            from bs4 import BeautifulSoup
+
+            md_content = file_bytes.decode('utf-8', errors='ignore')
+            html_content = markdown.markdown(
+                md_content, extensions=['extra', 'codehilite', 'tables', 'toc', 'fenced_code']
+            )
+            soup = BeautifulSoup(html_content, 'html.parser')
+            text_parts = []
+            for element in soup.children:
+                if element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                    level = int(element.name[1])
+                    text_parts.append('#' * level + ' ' + element.get_text().strip())
+                elif element.name == 'p':
+                    text = element.get_text().strip()
+                    if text:
+                        text_parts.append(text)
+                elif element.name in ['ul', 'ol']:
+                    for li in element.find_all('li'):
+                        text_parts.append(f'* {li.get_text().strip()}')
+                elif element.name == 'pre':
+                    code_block = element.get_text().strip()
+                    if code_block:
+                        text_parts.append(f'```\n{code_block}\n```')
+                elif element.name == 'table':
+                    table_str = self._extract_table_to_markdown_sync(element)
+                    if table_str:
+                        text_parts.append(table_str)
+                elif element.name:
+                    text = element.get_text(separator=' ', strip=True)
+                    if text:
+                        text_parts.append(text)
+            cleaned_text = re.sub(r'\n\s*\n', '\n\n', '\n'.join(text_parts))
+            return cleaned_text.strip()
+
+        return await self._run_sync(_parse_markdown_sync)
+
+    async def _parse_html(self, file_bytes: bytes, filename: str) -> str:
+        """Parses an HTML file, extracting structured plain text."""
+        logger.info(f'Parsing HTML file: {filename}')
+
+        def _parse_html_sync():
+            from bs4 import BeautifulSoup
+
+            html_content = file_bytes.decode('utf-8', errors='ignore')
+            soup = BeautifulSoup(html_content, 'html.parser')
+            for script_or_style in soup(['script', 'style']):
+                script_or_style.decompose()
+            text_parts = []
+            for element in soup.body.children if soup.body else soup.children:
+                if element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                    level = int(element.name[1])
+                    text_parts.append('#' * level + ' ' + element.get_text().strip())
+                elif element.name == 'p':
+                    text = element.get_text().strip()
+                    if text:
+                        text_parts.append(text)
+                elif element.name in ['ul', 'ol']:
+                    for li in element.find_all('li'):
+                        text = li.get_text().strip()
+                        if text:
+                            text_parts.append(f'* {text}')
+                elif element.name == 'table':
+                    table_str = self._extract_table_to_markdown_sync(element)
+                    if table_str:
+                        text_parts.append(table_str)
+                elif element.name:
+                    text = element.get_text(separator=' ', strip=True)
+                    if text:
+                        text_parts.append(text)
+            cleaned_text = re.sub(r'\n\s*\n', '\n\n', '\n'.join(text_parts))
+            return cleaned_text.strip()
+
+        return await self._run_sync(_parse_html_sync)
+
+    def _extract_table_to_markdown_sync(self, table_element) -> str:
+        """Helper to convert a BeautifulSoup table element into a Markdown table string."""
+        headers = [th.get_text().strip() for th in table_element.find_all('th')]
+        rows = []
+        for tr in table_element.find_all('tr'):
+            cells = [td.get_text().strip() for td in tr.find_all('td')]
+            if cells:
+                rows.append(cells)
+
+        if not headers and not rows:
+            return ''
+
+        table_lines = []
+        if headers:
+            table_lines.append(' | '.join(headers))
+            table_lines.append(' | '.join(['---'] * len(headers)))
+
+        for row_cells in rows:
+            padded_cells = row_cells + [''] * (len(headers) - len(row_cells)) if headers else row_cells
+            table_lines.append(' | '.join(padded_cells))
+
+        return '\n'.join(table_lines)

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/query_rewrite.py

@@ -0,0 +1,372 @@
+"""Query rewriting strategies for retrieval augmentation.
+
+Provides HyDE, Multi-Query, and Step-Back rewriting to improve recall quality.
+Each strategy rewrites/expands the user query before embedding and searching.
+"""
+
+import logging
+
+from langbot_plugin.api.entities.builtin.provider.message import Message
+from components.observability.telemetry import add_stage_duration, hash_text, telemetry
+
+logger = logging.getLogger(__name__)
+
+MULTI_QUERY_COUNT = 3
+
+HYDE_PROMPT = """Please write a short passage that would answer the following question.
+Write as if you are writing a paragraph from a reference document.
+Do not say "I don't know" — just write a plausible answer.
+
+Question: {query}
+
+Passage:"""
+
+MULTI_QUERY_PROMPT = """Given the following user question, generate {n} different search queries
+that could help find relevant information. Each query should approach the question from a
+different angle or use different terminology.
+
+User question: {query}
+
+Output exactly {n} queries, one per line (no numbering, no bullets):"""
+
+STEP_BACK_PROMPT = """Given the following specific question, generate a more general/abstract
+question that would help retrieve broader background context.
+
+Specific question: {query}
+
+General question:"""
+
+
+def _extract_text(msg: Message) -> str:
+    """Extract plain text from an LLM response Message."""
+    if isinstance(msg.content, str):
+        return msg.content.strip()
+    if isinstance(msg.content, list):
+        return "".join(e.text for e in msg.content if e.type == "text").strip()
+    return ""
+
+
+def _vector_result_sort_key(result: dict) -> tuple[int, float]:
+    """Prefer lower distance; fall back to legacy score-as-distance."""
+    distance = result.get("distance")
+    if isinstance(distance, (int, float)):
+        return (0, float(distance))
+
+    score = result.get("score")
+    if isinstance(score, (int, float)):
+        return (1, float(score))
+
+    return (2, float("inf"))
+
+
+def _uses_vector(search_type) -> bool:
+    """Return whether this search mode needs an embedding vector."""
+    return getattr(search_type, "value", search_type) != "full_text"
+
+
+def _text_log_ref(text: str | None) -> str:
+    """Return a privacy-preserving text reference for logs."""
+    return f"hash={hash_text(text)}, length={len(text or '')}"
+
+
+async def _timed_call(stage_durations: dict[str, float] | None, stage: str, call):
+    started_at = telemetry.start_timer()
+    try:
+        return await call()
+    finally:
+        if stage_durations is not None:
+            add_stage_duration(stage_durations, stage, telemetry.elapsed_ms(started_at))
+
+
+async def retrieve_with_rewrite(
+    plugin,
+    query: str,
+    query_rewrite: str,
+    rewrite_llm: str,
+    collection_id: str,
+    embedding_model_uuid: str,
+    fetch_k: int,
+    filters,
+    search_type,
+    vector_weight=None,
+    stage_durations: dict[str, float] | None = None,
+) -> list[dict]:
+    """Route to the appropriate rewrite strategy and return raw search results."""
+    if query_rewrite == "hyde":
+        return await _retrieve_hyde(
+            plugin,
+            query,
+            rewrite_llm,
+            collection_id,
+            embedding_model_uuid,
+            fetch_k,
+            filters,
+            search_type,
+            vector_weight,
+            stage_durations,
+        )
+    elif query_rewrite == "multi_query":
+        return await _retrieve_multi_query(
+            plugin,
+            query,
+            rewrite_llm,
+            collection_id,
+            embedding_model_uuid,
+            fetch_k,
+            filters,
+            search_type,
+            vector_weight,
+            stage_durations,
+        )
+    elif query_rewrite == "step_back":
+        return await _retrieve_step_back(
+            plugin,
+            query,
+            rewrite_llm,
+            collection_id,
+            embedding_model_uuid,
+            fetch_k,
+            filters,
+            search_type,
+            vector_weight,
+            stage_durations,
+        )
+    else:
+        logger.warning(
+            f"Unknown query_rewrite strategy: {query_rewrite}, falling back to direct search"
+        )
+        query_vector: list[float] = []
+        if _uses_vector(search_type):
+            query_vectors = await _timed_call(
+                stage_durations,
+                "rewrite_embedding",
+                lambda: plugin.invoke_embedding(embedding_model_uuid, [query]),
+            )
+            query_vector = query_vectors[0]
+        return await _timed_call(
+            stage_durations,
+            "rewrite_vector_search",
+            lambda: plugin.vector_search(
+                collection_id=collection_id,
+                query_vector=query_vector,
+                top_k=fetch_k,
+                filters=filters,
+                search_type=search_type,
+                query_text=query,
+                vector_weight=vector_weight,
+            ),
+        )
+
+
+async def _retrieve_hyde(
+    plugin,
+    query,
+    rewrite_llm,
+    collection_id,
+    embedding_model_uuid,
+    fetch_k,
+    filters,
+    search_type,
+    vector_weight=None,
+    stage_durations: dict[str, float] | None = None,
+) -> list[dict]:
+    """HyDE: generate a hypothetical document, embed it, and search with that vector."""
+    logger.info(
+        f"[HyDE] Generating hypothetical document "
+        f"for query_ref=({_text_log_ref(query)})"
+    )
+    prompt = HYDE_PROMPT.format(query=query)
+    resp = await _timed_call(
+        stage_durations,
+        "rewrite_llm",
+        lambda: plugin.invoke_llm(rewrite_llm, [Message(role="user", content=prompt)]),
+    )
+    hypothetical_doc = _extract_text(resp) or query
+    logger.info(f"[HyDE] Hypothetical document ref: {_text_log_ref(hypothetical_doc)}")
+
+    query_vector: list[float] = []
+    if _uses_vector(search_type):
+        logger.info("[HyDE] Embedding hypothetical document and searching...")
+        hyde_vectors = await _timed_call(
+            stage_durations,
+            "rewrite_embedding",
+            lambda: plugin.invoke_embedding(embedding_model_uuid, [hypothetical_doc]),
+        )
+        query_vector = hyde_vectors[0]
+    else:
+        logger.info("[HyDE] Searching with hypothetical document text...")
+    results = await _timed_call(
+        stage_durations,
+        "rewrite_vector_search",
+        lambda: plugin.vector_search(
+            collection_id=collection_id,
+            query_vector=query_vector,
+            top_k=fetch_k,
+            filters=filters,
+            search_type=search_type,
+            query_text=hypothetical_doc,
+            vector_weight=vector_weight,
+        ),
+    )
+    logger.info(f"[HyDE] Search returned {len(results)} results")
+    return results
+
+
+async def _retrieve_multi_query(
+    plugin,
+    query,
+    rewrite_llm,
+    collection_id,
+    embedding_model_uuid,
+    fetch_k,
+    filters,
+    search_type,
+    vector_weight=None,
+    stage_durations: dict[str, float] | None = None,
+) -> list[dict]:
+    """Multi-Query: generate N query variants, search with each, merge and deduplicate."""
+    logger.info(
+        f"[Multi-Query] Generating {MULTI_QUERY_COUNT} sub-queries "
+        f"for query_ref=({_text_log_ref(query)})"
+    )
+    prompt = MULTI_QUERY_PROMPT.format(query=query, n=MULTI_QUERY_COUNT)
+    resp = await _timed_call(
+        stage_durations,
+        "rewrite_llm",
+        lambda: plugin.invoke_llm(rewrite_llm, [Message(role="user", content=prompt)]),
+    )
+    raw_text = _extract_text(resp)
+    sub_queries = [line.strip() for line in raw_text.splitlines() if line.strip()]
+    sub_queries = sub_queries[:MULTI_QUERY_COUNT]
+    logger.info(
+        "[Multi-Query] Generated sub-query refs: "
+        + ", ".join(_text_log_ref(sq) for sq in sub_queries)
+    )
+
+    # Search original query + sub-queries
+    all_queries = [query] + sub_queries
+    logger.info(
+        f"[Multi-Query] Searching {len(all_queries)} queries "
+        f"(1 original + {len(sub_queries)} generated)"
+    )
+    if _uses_vector(search_type):
+        all_vectors = await _timed_call(
+            stage_durations,
+            "rewrite_embedding",
+            lambda: plugin.invoke_embedding(embedding_model_uuid, all_queries),
+        )
+    else:
+        all_vectors = [[] for _ in all_queries]
+
+    # Search with each vector and merge results
+    seen_ids: set[str] = set()
+    merged: list[dict] = []
+    for i, (search_query, vec) in enumerate(zip(all_queries, all_vectors)):
+        results = await _timed_call(
+            stage_durations,
+            "rewrite_vector_search",
+            lambda search_query=search_query, vec=vec: plugin.vector_search(
+                collection_id=collection_id,
+                query_vector=vec,
+                top_k=fetch_k,
+                filters=filters,
+                search_type=search_type,
+                query_text=search_query,
+                vector_weight=vector_weight,
+            ),
+        )
+        new_count = sum(1 for r in results if r["id"] not in seen_ids)
+        logger.info(
+            f"[Multi-Query] Query [{i}] returned {len(results)} results ({new_count} new)"
+        )
+        for r in results:
+            rid = r["id"]
+            if rid not in seen_ids:
+                seen_ids.add(rid)
+                merged.append(r)
+
+    # Prefer lower distance. Older hosts incorrectly exposed distance in score,
+    # so score is only used as a compatibility fallback here.
+    merged.sort(key=_vector_result_sort_key)
+    merged = merged[:fetch_k]
+    logger.info(
+        f"[Multi-Query] Merged: {len(seen_ids)} unique results, returning {len(merged)}"
+    )
+    return merged
+
+
+async def _retrieve_step_back(
+    plugin,
+    query,
+    rewrite_llm,
+    collection_id,
+    embedding_model_uuid,
+    fetch_k,
+    filters,
+    search_type,
+    vector_weight=None,
+    stage_durations: dict[str, float] | None = None,
+) -> list[dict]:
+    """Step-Back: generate a broader question, search with both original and abstract queries."""
+    logger.info(
+        f"[Step-Back] Generating abstract question "
+        f"for query_ref=({_text_log_ref(query)})"
+    )
+    prompt = STEP_BACK_PROMPT.format(query=query)
+    resp = await _timed_call(
+        stage_durations,
+        "rewrite_llm",
+        lambda: plugin.invoke_llm(rewrite_llm, [Message(role="user", content=prompt)]),
+    )
+    abstract_query = _extract_text(resp) or query
+    logger.info(f"[Step-Back] Abstract query ref: {_text_log_ref(abstract_query)}")
+
+    # Search with both original and abstract queries
+    both_queries = [query, abstract_query]
+    logger.info("[Step-Back] Searching original + abstract queries...")
+    if _uses_vector(search_type):
+        both_vectors = await _timed_call(
+            stage_durations,
+            "rewrite_embedding",
+            lambda: plugin.invoke_embedding(
+                embedding_model_uuid,
+                both_queries,
+            ),
+        )
+    else:
+        both_vectors = [[] for _ in both_queries]
+
+    # Search with both vectors and merge
+    seen_ids: set[str] = set()
+    merged: list[dict] = []
+    for i, (search_query, vec) in enumerate(zip(both_queries, both_vectors)):
+        label = "original" if i == 0 else "abstract"
+        results = await _timed_call(
+            stage_durations,
+            "rewrite_vector_search",
+            lambda search_query=search_query, vec=vec: plugin.vector_search(
+                collection_id=collection_id,
+                query_vector=vec,
+                top_k=fetch_k,
+                filters=filters,
+                search_type=search_type,
+                query_text=search_query,
+                vector_weight=vector_weight,
+            ),
+        )
+        new_count = sum(1 for r in results if r["id"] not in seen_ids)
+        logger.info(
+            f"[Step-Back] {label} query returned {len(results)} results ({new_count} new)"
+        )
+        for r in results:
+            rid = r["id"]
+            if rid not in seen_ids:
+                seen_ids.add(rid)
+                merged.append(r)
+
+    merged.sort(key=_vector_result_sort_key)
+    merged = merged[:fetch_k]
+    logger.info(
+        f"[Step-Back] Merged: {len(seen_ids)} unique results, returning {len(merged)}"
+    )
+    return merged

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/rerank.py

@@ -0,0 +1,215 @@
+"""Reranking helpers for retrieval results.
+
+Supports Host-provided rerank models and the legacy LLM listwise reranker.
+"""
+
+import logging
+import re
+
+from langbot_plugin.api.entities.builtin.provider.message import Message
+from components.observability.telemetry import hash_text
+
+logger = logging.getLogger(__name__)
+
+# Maximum characters per candidate passage sent to the LLM.
+_PASSAGE_TRUNCATE = 300
+
+RERANK_PROMPT = """\
+Given a query and candidate passages, rank the passages by relevance to the query.
+Return ONLY the passage numbers in order from most relevant to least relevant.
+Format: comma-separated numbers, e.g. "3,1,0,2,4"
+
+Query: {query}
+
+Candidates:
+{candidates}
+
+Ranking:"""
+
+
+def _extract_text(msg: Message) -> str:
+    """Extract plain text from an LLM response Message."""
+    if isinstance(msg.content, str):
+        return msg.content.strip()
+    if isinstance(msg.content, list):
+        return "".join(e.text for e in msg.content if e.type == "text").strip()
+    return ""
+
+
+def _parse_ranking(text: str, n: int) -> list[int] | None:
+    """Parse a comma/space separated list of integers from LLM output.
+
+    Returns a deduplicated list of valid indices (0..n-1), or *None* if
+    parsing fails entirely (no valid indices found).
+    """
+    nums = re.findall(r"\d+", text)
+    if not nums:
+        return None
+
+    seen: set[int] = set()
+    indices: list[int] = []
+    for s in nums:
+        idx = int(s)
+        if 0 <= idx < n and idx not in seen:
+            seen.add(idx)
+            indices.append(idx)
+    return indices if indices else None
+
+
+def _candidate_text(res: dict) -> str:
+    """Extract the display text sent to rerankers."""
+    return res.get("metadata", {}).get("text", "") or ""
+
+
+def _text_log_ref(text: str | None) -> str:
+    """Return a privacy-preserving text reference for logs."""
+    return f"hash={hash_text(text)}, length={len(text or '')}"
+
+
+def _apply_ranking(
+    results: list[dict],
+    ranking: list[int],
+    top_k: int,
+    scores_by_index: dict[int, float] | None = None,
+) -> list[dict]:
+    """Apply a candidate index ranking to result dicts."""
+    if len(ranking) < len(results):
+        remaining = [i for i in range(len(results)) if i not in set(ranking)]
+        ranking.extend(remaining)
+
+    reranked = [results[i] for i in ranking[:top_k]]
+
+    # Rewrite distance so downstream sorting stays consistent with reranking.
+    for rank, res in enumerate(reranked):
+        res["distance"] = 0.01 * (rank + 1)
+        if scores_by_index and ranking[rank] in scores_by_index:
+            res["score"] = scores_by_index[ranking[rank]]
+
+    return reranked
+
+
+async def model_rerank(
+    plugin,
+    rerank_model_uuid: str,
+    query: str,
+    results: list[dict],
+    top_k: int,
+) -> list[dict]:
+    """Rerank *results* using a Host rerank model.
+
+    The Host returns score entries with candidate indices. On failure or an
+    unusable response, retrieval falls back to the original order.
+    """
+    if not results:
+        return results
+
+    n = len(results)
+    logger.info(
+        f"[Rerank] Model reranking {n} candidates "
+        f"for query_ref=({_text_log_ref(query)})"
+    )
+    documents = [_candidate_text(res) for res in results]
+
+    try:
+        scores = await plugin.invoke_rerank(
+            rerank_model_uuid,
+            query,
+            documents,
+            top_k=top_k,
+        )
+    except Exception as e:
+        logger.warning(
+            f"[Rerank] Model rerank call failed, falling back to original order: {e}"
+        )
+        return results[:top_k]
+
+    ranking: list[int] = []
+    scores_by_index: dict[int, float] = {}
+    seen: set[int] = set()
+
+    sorted_scores = sorted(
+        scores,
+        key=lambda item: item.get("relevance_score", 0),
+        reverse=True,
+    )
+    for item in sorted_scores:
+        try:
+            idx = int(item.get("index"))
+        except (TypeError, ValueError):
+            continue
+        if 0 <= idx < n and idx not in seen:
+            seen.add(idx)
+            ranking.append(idx)
+            score = item.get("relevance_score")
+            if isinstance(score, (int, float)):
+                scores_by_index[idx] = float(score)
+
+    if not ranking:
+        logger.warning(
+            "[Rerank] Model rerank returned no valid indices, "
+            "falling back to original order"
+        )
+        return results[:top_k]
+
+    reranked = _apply_ranking(results, ranking, top_k, scores_by_index)
+    logger.info(
+        f"[Rerank] Done: {n} candidates -> top {len(reranked)} "
+        f"(order: {ranking[:top_k]})"
+    )
+    return reranked
+
+
+async def llm_rerank(
+    plugin,
+    llm_uuid: str,
+    query: str,
+    results: list[dict],
+    top_k: int,
+) -> list[dict]:
+    """Rerank *results* using an LLM and return the top *top_k* entries.
+
+    On any failure (LLM error, unparseable response) the function falls back
+    to returning ``results[:top_k]`` so that retrieval is never blocked.
+    """
+    if not results:
+        return results
+
+    n = len(results)
+    logger.info(
+        f"[Rerank] LLM reranking {n} candidates "
+        f"for query_ref=({_text_log_ref(query)})"
+    )
+
+    # Build numbered candidate list
+    lines: list[str] = []
+    for i, res in enumerate(results):
+        text = _candidate_text(res)[:_PASSAGE_TRUNCATE]
+        lines.append(f"[{i}] {text}")
+    candidates_block = "\n".join(lines)
+
+    prompt = RERANK_PROMPT.format(query=query, candidates=candidates_block)
+
+    try:
+        resp = await plugin.invoke_llm(
+            llm_uuid, [Message(role="user", content=prompt)]
+        )
+        raw = _extract_text(resp)
+        logger.info(f"[Rerank] LLM response ref: {_text_log_ref(raw)}")
+    except Exception as e:
+        logger.warning(f"[Rerank] LLM call failed, falling back to original order: {e}")
+        return results[:top_k]
+
+    ranking = _parse_ranking(raw, n)
+    if ranking is None:
+        logger.warning(
+            "[Rerank] Failed to parse ranking from LLM response, "
+            "falling back to original order"
+        )
+        return results[:top_k]
+
+    reranked = _apply_ranking(results, ranking, top_k)
+    logger.info(
+        f"[Rerank] Done: {n} candidates -> top {len(reranked)} "
+        f"(order: {ranking[:top_k]})"
+    )
+    return reranked

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/strategies/__init__.py

@@ -0,0 +1,30 @@
+"""Index strategy registry and factory."""
+
+import logging
+
+from .base import IndexStrategy
+from .chunk import ChunkStrategy
+from .parent_child import ParentChildStrategy
+from .qa import QAStrategy
+
+logger = logging.getLogger(__name__)
+
+_STRATEGIES: dict[str, type[IndexStrategy]] = {
+    "chunk": ChunkStrategy,
+    "parent_child": ParentChildStrategy,
+    "qa": QAStrategy,
+}
+
+
+def get_strategy(index_type: str) -> IndexStrategy:
+    """Return an IndexStrategy instance for the given *index_type*.
+
+    Falls back to :class:`ChunkStrategy` for unknown values.
+    """
+    cls = _STRATEGIES.get(index_type)
+    if cls is None:
+        logger.warning(
+            f"Unknown index_type '{index_type}', falling back to 'chunk' strategy"
+        )
+        cls = ChunkStrategy
+    return cls()

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/strategies/base.py

@@ -0,0 +1,97 @@
+"""Abstract base class for index strategies."""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from collections.abc import AsyncGenerator
+
+
+class IndexStrategy(ABC):
+    """Base class for RAG index strategies.
+
+    Each strategy defines how documents are chunked/indexed during ingestion
+    and how search results are post-processed during retrieval.
+    """
+
+    @abstractmethod
+    async def build_chunks_and_metadata(
+        self,
+        text: str,
+        doc_id: str,
+        filename: str,
+        creation_settings: dict,
+        plugin=None,
+        *,
+        sections: list | None = None,
+        doc_metadata: dict | None = None,
+    ) -> AsyncGenerator[tuple[list[str], list[str], list[dict]], None]:
+        """Build chunks, IDs, and metadata for a parsed document.
+
+        Yields batches of ``(texts_to_embed, ids, metadatas)``.  Most
+        strategies yield a single batch; streaming strategies (e.g. QA)
+        may yield incrementally to enable pipelined embedding.
+
+
+        Args:
+            text: Full parsed text of the document.
+            doc_id: Unique document identifier.
+            filename: Original filename.
+            creation_settings: Knowledge base creation settings dict.
+            plugin: Optional plugin reference for strategies that need
+                Host API access (e.g. LLM calls during ingestion).
+            sections: Optional list of TextSection objects from an external
+                parser.  When provided, strategies should use section-aware
+                chunking instead of flat text splitting.
+            doc_metadata: Optional document-level metadata from the parser
+                (e.g. page_count, has_tables).  Fields (except ``images``)
+                are merged into each chunk's metadata with a ``doc_`` prefix.
+
+        Yields:
+            A tuple of (texts_to_embed, ids, metadatas).
+            - texts_to_embed: text chunks to be embedded.
+            - ids: unique ID for each chunk.
+            - metadatas: metadata dict for each chunk.
+        """
+        yield  # abstract – subclasses must override
+
+    @staticmethod
+    def _build_doc_meta_fields(doc_metadata: dict | None) -> dict:
+        """Extract document-level metadata fields with ``doc_`` prefix.
+
+        Excludes heavy fields like ``images`` to keep chunk metadata lean.
+
+        TODO: Chroma 1.5.0+ supports array metadata types (list of str/int/float/bool)
+        with $contains/$not_contains operators. LangBot currently uses Chroma 1.4.1
+        which only accepts scalar types (str, int, float, bool, None). This function
+        converts non-scalar values to JSON strings for compatibility. Once LangBot
+        upgrades to Chroma 1.5.0+, this conversion layer can be removed to enable
+        native array filtering. See: https://www.trychroma.com/changelog/metadata-arrays
+        """
+        if not doc_metadata:
+            return {}
+        skip = {"images"}
+        result = {}
+        for k, v in doc_metadata.items():
+            if k in skip:
+                continue
+            # Chroma 1.4.1 only accepts scalar types; convert list/dict to JSON string
+            if isinstance(v, (str, int, float, bool)) or v is None:
+                result[f"doc_{k}"] = v
+            else:
+                result[f"doc_{k}"] = json.dumps(v, ensure_ascii=False)
+        return result
+
+    def postprocess_results(self, results: list[dict], top_k: int) -> list[dict]:
+        """Post-process search results before returning to the caller.
+
+        Default implementation is a no-op passthrough.
+
+        Args:
+            results: Raw search results from the vector store.
+            top_k: Desired number of results.
+
+        Returns:
+            Processed results list.
+        """
+        return results[:top_k]

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/strategies/chunk.py

@@ -0,0 +1,84 @@
+"""Default flat chunking strategy — preserves the original LangRAG behavior."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+
+from .base import IndexStrategy
+from components.offload import BoundedOffload
+from ..chunker import chunk_text, chunk_sections, DEFAULT_CHUNK_SIZE, DEFAULT_CHUNK_OVERLAP
+
+
+class ChunkStrategy(IndexStrategy):
+    """Simple fixed-size chunking with overlap.
+
+    Each chunk is independently embedded and stored.  This is the original
+    behaviour of LangRAG prior to strategy modularisation.
+
+    When *sections* are provided (from an external parser), uses
+    section-aware chunking that preserves heading/page metadata.
+    """
+
+    async def build_chunks_and_metadata(
+        self,
+        text: str,
+        doc_id: str,
+        filename: str,
+        creation_settings: dict,
+        plugin=None,
+        *,
+        sections: list | None = None,
+        doc_metadata: dict | None = None,
+    ) -> AsyncGenerator[tuple[list[str], list[str], list[dict]], None]:
+        chunk_size = creation_settings.get("chunk_size") or DEFAULT_CHUNK_SIZE
+        overlap = creation_settings.get("overlap") or DEFAULT_CHUNK_OVERLAP
+        doc_fields = self._build_doc_meta_fields(doc_metadata)
+        offload = getattr(plugin, "offload", None) or BoundedOffload()
+
+        if sections:
+            # Section-aware path
+            s_chunks = await offload.run(chunk_sections, sections, chunk_size, overlap)
+            chunks_text: list[str] = []
+            ids: list[str] = []
+            metadatas: list[dict] = []
+            for i, sc in enumerate(s_chunks):
+                chunks_text.append(sc.text)
+                ids.append(f"{doc_id}_{i}")
+                meta = {
+                    "file_id": doc_id,
+                    "document_id": doc_id,
+                    "document_name": filename,
+                    "chunk_index": i,
+                    "text": sc.text,
+                    "index_type": "chunk",
+                    "heading": sc.heading,
+                    "level": sc.level,
+                    "page": sc.page,
+                    "heading_path": sc.heading_path,
+                    "has_table": sc.has_table,
+                    "section_index": sc.section_index,
+                }
+                meta.update(doc_fields)
+                metadatas.append(meta)
+
+            yield chunks_text, ids, metadatas
+        else:
+            # Fallback: flat text chunking (original behaviour)
+            chunks = await offload.run(chunk_text, text, chunk_size, overlap)
+
+            ids = []
+            metadatas = []
+            for i, chunk in enumerate(chunks):
+                ids.append(f"{doc_id}_{i}")
+                meta = {
+                    "file_id": doc_id,
+                    "document_id": doc_id,
+                    "document_name": filename,
+                    "chunk_index": i,
+                    "text": chunk,
+                    "index_type": "chunk",
+                }
+                meta.update(doc_fields)
+                metadatas.append(meta)
+
+            yield chunks, ids, metadatas

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/strategies/parent_child.py

@@ -0,0 +1,152 @@
+"""Parent-Child index strategy.
+
+Uses two-level chunking: large parent chunks for context richness, small child
+chunks for precise embedding matches.  Child chunks are embedded, but the
+metadata ``text`` field stores the parent chunk so that retrieval automatically
+returns full context.
+
+When *sections* are provided, each section naturally maps to a parent chunk.
+Sections shorter than *parent_chunk_size* become a single parent; longer ones
+are split.  Child chunks are then created from each parent as usual.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncGenerator
+
+from .base import IndexStrategy
+from components.offload import BoundedOffload
+from ..chunker import (
+    chunk_text,
+    chunk_sections,
+    DEFAULT_CHUNK_OVERLAP,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_PARENT_CHUNK_SIZE = 2048
+DEFAULT_CHILD_CHUNK_SIZE = 256
+
+
+class ParentChildStrategy(IndexStrategy):
+    """Two-level parent / child chunking strategy.
+
+    - Parent chunks are large (default 2048 chars), split with no overlap.
+    - Each parent is further split into small child chunks (default 256 chars)
+      with configurable overlap.
+    - Child chunks are the units that get embedded.
+    - The ``text`` metadata field on each child stores the **parent** chunk,
+      so search results automatically carry richer context.
+    - ``postprocess_results`` deduplicates by parent, keeping only the
+      highest-scoring child per parent.
+
+    When *sections* are provided, sections are used as natural parent
+    boundaries via section-aware chunking at parent_chunk_size, then each
+    parent is further split into children.
+    """
+
+    async def build_chunks_and_metadata(
+        self,
+        text: str,
+        doc_id: str,
+        filename: str,
+        creation_settings: dict,
+        plugin=None,
+        *,
+        sections: list | None = None,
+        doc_metadata: dict | None = None,
+    ) -> AsyncGenerator[tuple[list[str], list[str], list[dict]], None]:
+        parent_size = (
+            creation_settings.get("parent_chunk_size") or DEFAULT_PARENT_CHUNK_SIZE
+        )
+        child_size = (
+            creation_settings.get("child_chunk_size") or DEFAULT_CHILD_CHUNK_SIZE
+        )
+        overlap = creation_settings.get("overlap") or DEFAULT_CHUNK_OVERLAP
+        doc_fields = self._build_doc_meta_fields(doc_metadata)
+        offload = getattr(plugin, "offload", None) or BoundedOffload()
+
+        texts_to_embed: list[str] = []
+        ids: list[str] = []
+        metadatas: list[dict] = []
+
+        if sections:
+            # Section-aware: use sections as parent boundaries
+            parent_chunks = await offload.run(chunk_sections, sections, parent_size, 0)
+
+            for p_idx, parent_sc in enumerate(parent_chunks):
+                parent_text = parent_sc.text
+                child_chunks = await offload.run(chunk_text, parent_text, child_size, overlap)
+                for c_idx, child in enumerate(child_chunks):
+                    texts_to_embed.append(child)
+                    ids.append(f"{doc_id}_p{p_idx}_c{c_idx}")
+                    meta = {
+                        "file_id": doc_id,
+                        "document_id": doc_id,
+                        "document_name": filename,
+                        "parent_index": p_idx,
+                        "child_text": child,
+                        "text": parent_text,
+                        "index_type": "parent_child",
+                        "heading": parent_sc.heading,
+                        "level": parent_sc.level,
+                        "page": parent_sc.page,
+                        "heading_path": parent_sc.heading_path,
+                        "has_table": parent_sc.has_table,
+                        "section_index": parent_sc.section_index,
+                    }
+                    meta.update(doc_fields)
+                    metadatas.append(meta)
+        else:
+            # Fallback: flat text chunking (original behaviour)
+            parent_chunks_text = await offload.run(chunk_text, text, parent_size, 0)
+
+            for p_idx, parent in enumerate(parent_chunks_text):
+                child_chunks = await offload.run(chunk_text, parent, child_size, overlap)
+                for c_idx, child in enumerate(child_chunks):
+                    texts_to_embed.append(child)
+                    ids.append(f"{doc_id}_p{p_idx}_c{c_idx}")
+                    meta = {
+                        "file_id": doc_id,
+                        "document_id": doc_id,
+                        "document_name": filename,
+                        "parent_index": p_idx,
+                        "child_text": child,
+                        "text": parent,
+                        "index_type": "parent_child",
+                    }
+                    meta.update(doc_fields)
+                    metadatas.append(meta)
+
+        yield texts_to_embed, ids, metadatas
+
+    def postprocess_results(self, results: list[dict], top_k: int) -> list[dict]:
+        """Deduplicate by parent chunk, keeping the highest-scoring child."""
+
+        def _metric(res: dict) -> tuple[int, float]:
+            distance = res.get("distance")
+            if isinstance(distance, (int, float)):
+                return (0, float(distance))
+
+            score = res.get("score")
+            if isinstance(score, (int, float)):
+                return (1, float(score))
+
+            return (2, float("inf"))
+
+        seen: dict[str, dict] = {}  # key → best result
+        for res in results:
+            meta = res.get("metadata", {})
+            doc_id = meta.get("document_id", "")
+            parent_idx = meta.get("parent_index", "")
+            key = f"{doc_id}:{parent_idx}"
+
+            if key not in seen:
+                seen[key] = res
+            else:
+                if _metric(res) < _metric(seen[key]):
+                    seen[key] = res
+
+        deduped = list(seen.values())
+        return deduped[:top_k]

plugins/KnowledgeEngine/LangRAG/components/knowledge_engine/strategies/qa.py

@@ -0,0 +1,260 @@
+"""QA Index strategy — generates question-answer pairs from document chunks.
+
+Instead of embedding raw chunk text, this strategy uses an LLM to generate
+question-answer pairs from each chunk.  The *questions* are embedded and stored
+as vectors, while the *answers* are stored in metadata.  At retrieval time the
+user's query (naturally phrased as a question) matches against generated
+questions, yielding better semantic alignment than raw-chunk matching.
+
+This strategy yields Q&A pairs incrementally (per chunk) so that the ingest
+pipeline can embed and upsert concurrently with ongoing LLM generation.
+
+When *sections* are provided, section-aware chunks are used instead of flat
+text chunks, preserving heading/page metadata in each Q&A pair's metadata.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from collections.abc import AsyncGenerator
+
+from langbot_plugin.api.entities.builtin.provider.message import Message
+
+from .base import IndexStrategy
+from components.offload import BoundedOffload
+from ..chunker import (
+    chunk_text,
+    chunk_sections,
+    DEFAULT_CHUNK_SIZE,
+    DEFAULT_CHUNK_OVERLAP,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_QUESTIONS_PER_CHUNK = 1
+
+QA_GENERATION_PROMPT = """\
+Based on the following text, generate {n} question-answer pairs.
+Each question should be directly answerable from the text.
+Each answer should be concise and grounded in the text.
+
+Text:
+{chunk_text}
+
+Output exactly {n} pairs in this JSON format (no other text):
+[
+  {{"q": "question", "a": "answer"}},
+  ...
+]"""
+
+
+def _parse_qa_pairs(text: str) -> list[tuple[str, str]]:
+    """Parse Q&A pairs from LLM response.
+
+    Tries JSON first, falls back to ``Q:/A:`` text format.
+    """
+    # --- Try JSON ---
+    try:
+        # Extract JSON array even if surrounded by markdown fences
+        json_match = re.search(r"\[.*?\]", text, re.DOTALL)
+        if json_match:
+            items = json.loads(json_match.group())
+            pairs = []
+            for item in items:
+                q = str(item.get("q", "")).strip()
+                a = str(item.get("a", "")).strip()
+                if q and a:
+                    pairs.append((q, a))
+            if pairs:
+                return pairs
+    except (json.JSONDecodeError, AttributeError):
+        pass
+
+    # --- Fallback: Q:/A: text format ---
+    pairs: list[tuple[str, str]] = []
+    blocks = re.split(r"\nQ:\s*", "\n" + text)
+    for block in blocks:
+        block = block.strip()
+        if not block:
+            continue
+        parts = re.split(r"\nA:\s*", block, maxsplit=1)
+        if len(parts) == 2:
+            q, a = parts[0].strip(), parts[1].strip()
+            if q and a:
+                pairs.append((q, a))
+    return pairs
+
+
+def _extract_text(msg: Message) -> str:
+    """Extract plain text from an LLM response Message."""
+    if isinstance(msg.content, str):
+        return msg.content.strip()
+    if isinstance(msg.content, list):
+        return "".join(e.text for e in msg.content if e.type == "text").strip()
+    return ""
+
+
+class QAStrategy(IndexStrategy):
+    """QA Index strategy.
+
+    During ingestion each chunk is sent to an LLM which generates N
+    question-answer pairs.  The **questions** become the texts that get
+    embedded.  The **answers** are stored in the ``text`` metadata field so
+    they are returned as retrieval results.  The original chunk is kept in
+    ``source_chunk`` for additional context.
+
+    Yields Q&A pairs per chunk so the ingest pipeline can start embedding
+    while the next chunk's LLM call is in flight.
+
+    When *sections* are provided, section-aware chunks are used, and each
+    Q&A pair inherits the chunk's heading/page/heading_path metadata.
+    """
+
+    async def build_chunks_and_metadata(
+        self,
+        text: str,
+        doc_id: str,
+        filename: str,
+        creation_settings: dict,
+        plugin=None,
+        *,
+        sections: list | None = None,
+        doc_metadata: dict | None = None,
+    ) -> AsyncGenerator[tuple[list[str], list[str], list[dict]], None]:
+        if plugin is None:
+            raise RuntimeError("QAStrategy requires a plugin reference for LLM calls")
+
+        chunk_size = creation_settings.get("chunk_size") or DEFAULT_CHUNK_SIZE
+        overlap = creation_settings.get("overlap") or DEFAULT_CHUNK_OVERLAP
+        qa_llm_uuid = creation_settings.get("qa_llm_model_uuid", "")
+        n_questions = (
+            creation_settings.get("questions_per_chunk") or DEFAULT_QUESTIONS_PER_CHUNK
+        )
+        doc_fields = self._build_doc_meta_fields(doc_metadata)
+        offload = getattr(plugin, "offload", None) or BoundedOffload()
+
+        if not qa_llm_uuid:
+            raise ValueError(
+                "QA Index requires 'qa_llm_model_uuid' in creation settings"
+            )
+
+        # Build chunk list — section-aware or flat
+        if sections:
+            s_chunks = await offload.run(chunk_sections, sections, chunk_size, overlap)
+            # Wrap into a uniform structure for the loop below
+            chunk_items = [
+                {
+                    "text": sc.text,
+                    "heading": sc.heading,
+                    "level": sc.level,
+                    "page": sc.page,
+                    "heading_path": sc.heading_path,
+                    "has_table": sc.has_table,
+                    "section_index": sc.section_index,
+                }
+                for sc in s_chunks
+            ]
+        else:
+            flat_chunks = await offload.run(chunk_text, text, chunk_size, overlap)
+            chunk_items = [{"text": c} for c in flat_chunks]
+
+        logger.info(
+            f"[QA] Generating Q&A pairs: {len(chunk_items)} chunks × "
+            f"{n_questions} questions, LLM={qa_llm_uuid}"
+        )
+
+        total_qa = 0
+
+        for chunk_idx, chunk_info in enumerate(chunk_items):
+            chunk = chunk_info["text"]
+            prompt = QA_GENERATION_PROMPT.format(n=n_questions, chunk_text=chunk)
+            messages = [Message(role="user", content=prompt)]
+
+            try:
+                response = await plugin.invoke_llm(qa_llm_uuid, messages)
+                response_text = _extract_text(response)
+            except Exception as e:
+                logger.warning(
+                    f"[QA] LLM call failed for chunk {chunk_idx}: {e}, skipping"
+                )
+                continue
+
+            qa_pairs = _parse_qa_pairs(response_text)
+            if not qa_pairs:
+                logger.warning(
+                    f"[QA] No Q&A pairs parsed from chunk {chunk_idx}, "
+                    f"raw response: {response_text[:200]!r}"
+                )
+                continue
+
+            logger.info(
+                f"[QA] Chunk {chunk_idx}/{len(chunk_items) - 1}: "
+                f"generated {len(qa_pairs)} Q&A pairs"
+            )
+
+            questions: list[str] = []
+            ids: list[str] = []
+            metadatas: list[dict] = []
+
+            for qa_idx, (question, answer) in enumerate(qa_pairs):
+                vec_id = f"{doc_id}_{chunk_idx}_qa{qa_idx}"
+                questions.append(question)
+                ids.append(vec_id)
+                meta = {
+                    "file_id": doc_id,
+                    "document_id": doc_id,
+                    "document_name": filename,
+                    "chunk_index": chunk_idx,
+                    "qa_index": qa_idx,
+                    "question": question,
+                    "answer": answer,
+                    "text": answer,
+                    "source_chunk": chunk,
+                    "index_type": "qa",
+                }
+                # Add section metadata if available
+                if "heading" in chunk_info:
+                    meta["heading"] = chunk_info["heading"]
+                    meta["level"] = chunk_info["level"]
+                    meta["page"] = chunk_info["page"]
+                    meta["heading_path"] = chunk_info["heading_path"]
+                    meta["has_table"] = chunk_info["has_table"]
+                    meta["section_index"] = chunk_info["section_index"]
+                meta.update(doc_fields)
+                metadatas.append(meta)
+
+            total_qa += len(questions)
+            yield questions, ids, metadatas
+
+        logger.info(f"[QA] Total: {total_qa} Q&A pairs from {len(chunk_items)} chunks")
+
+    def postprocess_results(self, results: list[dict], top_k: int) -> list[dict]:
+        """Deduplicate by source chunk, keeping the highest-scoring QA per chunk."""
+
+        def _metric(res: dict) -> tuple[int, float]:
+            distance = res.get("distance")
+            if isinstance(distance, (int, float)):
+                return (0, float(distance))
+
+            score = res.get("score")
+            if isinstance(score, (int, float)):
+                return (1, float(score))
+
+            return (2, float("inf"))
+
+        seen: dict[str, dict] = {}
+        for res in results:
+            meta = res.get("metadata", {})
+            doc_id = meta.get("document_id", "")
+            chunk_idx = meta.get("chunk_index", "")
+            key = f"{doc_id}:{chunk_idx}"
+
+            if key not in seen:
+                seen[key] = res
+            else:
+                if _metric(res) < _metric(seen[key]):
+                    seen[key] = res
+
+        return list(seen.values())[:top_k]