chore: Add PyPI package support for uvx/pip installation (#1764)

* Initial plan

* Add package structure and resource path utilities

- Created langbot/ package with __init__.py and __main__.py entry point
- Added paths utility to find frontend and resource files from package installation
- Updated config loading to use resource paths
- Updated frontend serving to use resource paths
- Added MANIFEST.in for package data inclusion
- Updated pyproject.toml with build system and entry points

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add PyPI publishing workflow and update license

- Created GitHub Actions workflow to build frontend and publish to PyPI
- Added license field to pyproject.toml to fix deprecation warning
- Updated .gitignore to exclude build artifacts
- Tested package building successfully

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add PyPI installation documentation

- Created PYPI_INSTALLATION.md with detailed installation and usage instructions
- Updated README.md to feature uvx/pip installation as recommended method
- Updated README_EN.md with same changes for English documentation

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Address code review feedback

- Made package-data configuration more specific to langbot package only
- Improved path detection with caching to avoid repeated file I/O
- Removed sys.path searching which was incorrect for package data
- Removed interactive input() call for non-interactive environment compatibility
- Simplified error messages for version check

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Fix code review issues

- Use specific exception types instead of bare except
- Fix misleading comments about directory levels
- Remove redundant existence check before makedirs with exist_ok=True
- Use context manager for file opening to ensure proper cleanup

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Simplify package configuration and document behavioral differences

- Removed redundant package-data configuration, relying on MANIFEST.in
- Added documentation about behavioral differences between package and source installation
- Clarified that include-package-data=true uses MANIFEST.in for data files

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* chore: update pyproject.toml

* chore: try pack templates in langbot/

* chore: update

* chore: update

* chore: update

* chore: update

* chore: update

* chore: adjust dir structure

* chore: fix imports

* fix: read default-pipeline-config.json

* fix: read default-pipeline-config.json

* fix: tests

* ci: publish pypi

* chore: bump version 4.6.0-beta.1 for testing

* chore: add templates/**

* fix: send adapters and requesters icons

* chore: bump version 4.6.0b2 for testing

* chore: add platform field for docker-compose.yaml

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>
Co-authored-by: Junyan Qin <rockchinq@gmail.com>

src/langbot/pkg/rag/knowledge/services/base_service.py

@@ -0,0 +1,15 @@
+# 封装异步操作
+import asyncio
+
+
+class BaseService:
+    def __init__(self):
+        pass
+
+    async def _run_sync(self, func, *args, **kwargs):
+        """
+        在单独的线程中运行同步函数。
+        如果第一个参数是 session，则在 to_thread 中获取新的 session。
+        """
+
+        return await asyncio.to_thread(func, *args, **kwargs)

src/langbot/pkg/rag/knowledge/services/chunker.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import json
+from typing import List
+from langbot.pkg.rag.knowledge.services import base_service
+from langbot.pkg.core import app
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+
+class Chunker(base_service.BaseService):
+    """
+    A class for splitting long texts into smaller, overlapping chunks.
+    """
+
+    def __init__(self, ap: app.Application, chunk_size: int = 500, chunk_overlap: int = 50):
+        self.ap = ap
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        if self.chunk_overlap >= self.chunk_size:
+            self.ap.logger.warning(
+                'Chunk overlap is greater than or equal to chunk size. This may lead to empty or malformed chunks.'
+            )
+
+    def _split_text_sync(self, text: str) -> List[str]:
+        """
+        Synchronously splits a long text into chunks with specified overlap.
+        This is a CPU-bound operation, intended to be run in a separate thread.
+        """
+        if not text:
+            return []
+
+        text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function=len,
+            is_separator_regex=False,
+        )
+        return text_splitter.split_text(text)
+
+    async def chunk(self, text: str) -> List[str]:
+        """
+        Asynchronously chunks a given text into smaller pieces.
+        """
+        self.ap.logger.info(f'Chunking text (length: {len(text)})...')
+        # Run the synchronous splitting logic in a separate thread
+        chunks = await self._run_sync(self._split_text_sync, text)
+        self.ap.logger.info(f'Text chunked into {len(chunks)} pieces.')
+        self.ap.logger.debug(f'Chunks: {json.dumps(chunks, indent=4, ensure_ascii=False)}')
+        return chunks

src/langbot/pkg/rag/knowledge/services/embedder.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+import uuid
+from typing import List
+from langbot.pkg.rag.knowledge.services.base_service import BaseService
+from langbot.pkg.entity.persistence import rag as persistence_rag
+from langbot.pkg.core import app
+from langbot.pkg.provider.modelmgr.requester import RuntimeEmbeddingModel
+import sqlalchemy
+
+
+class Embedder(BaseService):
+    def __init__(self, ap: app.Application) -> None:
+        super().__init__()
+        self.ap = ap
+
+    async def embed_and_store(
+        self, kb_id: str, file_id: str, chunks: List[str], embedding_model: RuntimeEmbeddingModel
+    ) -> list[persistence_rag.Chunk]:
+        # save chunk to db
+        chunk_entities: list[persistence_rag.Chunk] = []
+        chunk_ids: list[str] = []
+
+        for chunk_text in chunks:
+            chunk_uuid = str(uuid.uuid4())
+            chunk_ids.append(chunk_uuid)
+            chunk_entity = persistence_rag.Chunk(uuid=chunk_uuid, file_id=file_id, text=chunk_text)
+            chunk_entities.append(chunk_entity)
+
+        chunk_dicts = [
+            self.ap.persistence_mgr.serialize_model(persistence_rag.Chunk, chunk) for chunk in chunk_entities
+        ]
+
+        await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_rag.Chunk).values(chunk_dicts))
+
+        # get embeddings
+        embeddings_list: list[list[float]] = await embedding_model.requester.invoke_embedding(
+            model=embedding_model,
+            input_text=chunks,
+            extra_args={},  # TODO: add extra args
+        )
+
+        # save embeddings to vdb
+        await self.ap.vector_db_mgr.vector_db.add_embeddings(kb_id, chunk_ids, embeddings_list, chunk_dicts)
+
+        self.ap.logger.info(f'Successfully saved {len(chunk_entities)} embeddings to Knowledge Base.')
+
+        return chunk_entities

src/langbot/pkg/rag/knowledge/services/parser.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import PyPDF2
+import io
+from docx import Document
+import chardet
+from typing import Union, Callable, Any
+import markdown
+from bs4 import BeautifulSoup
+import re
+import asyncio  # Import asyncio for async operations
+from langbot.pkg.core import app
+
+
+class FileParser:
+    """
+    A robust file parser class to extract text content from various document formats.
+    It supports TXT, PDF, DOCX, XLSX, CSV, Markdown, HTML, and EPUB files.
+    All core file reading operations are designed to be run synchronously in a thread pool
+    to avoid blocking the asyncio event loop.
+    """
+
+    def __init__(self, ap: app.Application):
+        self.ap = ap
+
+    async def _run_sync(self, sync_func: Callable, *args: Any, **kwargs: Any) -> Any:
+        """
+        Runs a synchronous function in a separate thread to prevent blocking the event loop.
+        This is a general utility method for wrapping blocking I/O operations.
+        """
+        try:
+            return await asyncio.to_thread(sync_func, *args, **kwargs)
+        except Exception as e:
+            self.ap.logger.error(f'Error running synchronous function {sync_func.__name__}: {e}')
+            raise
+
+    async def parse(self, file_name: str, extension: str) -> Union[str, None]:
+        """
+        Parses the file based on its extension and returns the extracted text content.
+        This is the main asynchronous entry point for parsing.
+
+        Args:
+            file_name (str): The name of the file to be parsed, get from ap.storage_mgr
+
+        Returns:
+            Union[str, None]: The extracted text content as a single string, or None if parsing fails.
+        """
+
+        file_extension = extension.lower()
+        parser_method = getattr(self, f'_parse_{file_extension}', None)
+
+        if parser_method is None:
+            self.ap.logger.error(f'Unsupported file format: {file_extension} for file {file_name}')
+            return None
+
+        try:
+            # Pass file_path to the specific parser methods
+            return await parser_method(file_name)
+        except Exception as e:
+            self.ap.logger.error(f'Failed to parse {file_extension} file {file_name}: {e}')
+            return None
+
+    # --- Helper for reading files with encoding detection ---
+    async def _read_file_content(self, file_name: str) -> Union[str, bytes]:
+        """
+        Reads a file with automatic encoding detection, ensuring the synchronous
+        file read operation runs in a separate thread.
+        """
+
+        # def _read_sync():
+        #     with open(file_path, 'rb') as file:
+        #         raw_data = file.read()
+        #         detected = chardet.detect(raw_data)
+        #         encoding = detected['encoding'] or 'utf-8'
+
+        #     if mode == 'r':
+        #         return raw_data.decode(encoding, errors='ignore')
+        #     return raw_data  # For binary mode
+
+        # return await self._run_sync(_read_sync)
+        file_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+        detected = chardet.detect(file_bytes)
+        encoding = detected['encoding'] or 'utf-8'
+
+        return file_bytes.decode(encoding, errors='ignore')
+
+    # --- Specific Parser Methods ---
+
+    async def _parse_txt(self, file_name: str) -> str:
+        """Parses a TXT file and returns its content."""
+        self.ap.logger.info(f'Parsing TXT file: {file_name}')
+        return await self._read_file_content(file_name)
+
+    async def _parse_pdf(self, file_name: str) -> str:
+        """Parses a PDF file and returns its text content."""
+        self.ap.logger.info(f'Parsing PDF file: {file_name}')
+
+        # def _parse_pdf_sync():
+        #     text_content = []
+        #     with open(file_name, 'rb') as file:
+        #         pdf_reader = PyPDF2.PdfReader(file)
+        #         for page in pdf_reader.pages:
+        #             text = page.extract_text()
+        #             if text:
+        #                 text_content.append(text)
+        #     return '\n'.join(text_content)
+
+        # return await self._run_sync(_parse_pdf_sync)
+
+        pdf_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+        def _parse_pdf_sync():
+            pdf_reader = PyPDF2.PdfReader(io.BytesIO(pdf_bytes))
+            text_content = []
+            for page in pdf_reader.pages:
+                text = page.extract_text()
+                if text:
+                    text_content.append(text)
+            return '\n'.join(text_content)
+
+        return await self._run_sync(_parse_pdf_sync)
+
+    async def _parse_docx(self, file_name: str) -> str:
+        """Parses a DOCX file and returns its text content."""
+        self.ap.logger.info(f'Parsing DOCX file: {file_name}')
+
+        docx_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+        def _parse_docx_sync():
+            doc = Document(io.BytesIO(docx_bytes))
+            text_content = [paragraph.text for paragraph in doc.paragraphs if paragraph.text.strip()]
+            return '\n'.join(text_content)
+
+        return await self._run_sync(_parse_docx_sync)
+
+    async def _parse_doc(self, file_name: str) -> str:
+        """Handles .doc files, explicitly stating lack of direct support."""
+        self.ap.logger.warning(f'Direct .doc parsing is not supported for {file_name}. Please convert to .docx first.')
+        raise NotImplementedError('Direct .doc parsing not supported. Please convert to .docx first.')
+
+    # async def _parse_xlsx(self, file_name: str) -> str:
+    #     """Parses an XLSX file, returning text from all sheets."""
+    #     self.ap.logger.info(f'Parsing XLSX file: {file_name}')
+
+    #     xlsx_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+    #     def _parse_xlsx_sync():
+    #         excel_file = pd.ExcelFile(io.BytesIO(xlsx_bytes))
+    #         all_sheet_content = []
+    #         for sheet_name in excel_file.sheet_names:
+    #             df = pd.read_excel(io.BytesIO(xlsx_bytes), sheet_name=sheet_name)
+    #             sheet_text = f'--- Sheet: {sheet_name} ---\n{df.to_string(index=False)}\n'
+    #             all_sheet_content.append(sheet_text)
+    #         return '\n'.join(all_sheet_content)
+
+    #     return await self._run_sync(_parse_xlsx_sync)
+
+    # async def _parse_csv(self, file_name: str) -> str:
+    #     """Parses a CSV file and returns its content as a string."""
+    #     self.ap.logger.info(f'Parsing CSV file: {file_name}')
+
+    #     csv_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+    #     def _parse_csv_sync():
+    #         # pd.read_csv can often detect encoding, but explicit detection is safer
+    #         # raw_data = self._read_file_content(
+    #         #     file_name, mode='rb'
+    #         # )  # Note: this will need to be await outside this sync function
+    #         # _ = raw_data
+    #         # For simplicity, we'll let pandas handle encoding internally after a raw read.
+    #         # A more robust solution might pass encoding directly to pd.read_csv after detection.
+    #         detected = chardet.detect(io.BytesIO(csv_bytes))
+    #         encoding = detected['encoding'] or 'utf-8'
+    #         df = pd.read_csv(io.BytesIO(csv_bytes), encoding=encoding)
+    #         return df.to_string(index=False)
+
+    #     return await self._run_sync(_parse_csv_sync)
+
+    async def _parse_md(self, file_name: str) -> str:
+        """Parses a Markdown file, converting it to structured plain text."""
+        self.ap.logger.info(f'Parsing Markdown file: {file_name}')
+
+        md_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+        def _parse_markdown_sync():
+            md_content = io.BytesIO(md_bytes).read().decode('utf-8', errors='ignore')
+            html_content = markdown.markdown(
+                md_content, extensions=['extra', 'codehilite', 'tables', 'toc', 'fenced_code']
+            )
+            soup = BeautifulSoup(html_content, 'html.parser')
+            text_parts = []
+            for element in soup.children:
+                if element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                    level = int(element.name[1])
+                    text_parts.append('#' * level + ' ' + element.get_text().strip())
+                elif element.name == 'p':
+                    text = element.get_text().strip()
+                    if text:
+                        text_parts.append(text)
+                elif element.name in ['ul', 'ol']:
+                    for li in element.find_all('li'):
+                        text_parts.append(f'* {li.get_text().strip()}')
+                elif element.name == 'pre':
+                    code_block = element.get_text().strip()
+                    if code_block:
+                        text_parts.append(f'```\n{code_block}\n```')
+                elif element.name == 'table':
+                    table_str = self._extract_table_to_markdown_sync(element)  # Call sync helper
+                    if table_str:
+                        text_parts.append(table_str)
+                elif element.name:
+                    text = element.get_text(separator=' ', strip=True)
+                    if text:
+                        text_parts.append(text)
+            cleaned_text = re.sub(r'\n\s*\n', '\n\n', '\n'.join(text_parts))
+            return cleaned_text.strip()
+
+        return await self._run_sync(_parse_markdown_sync)
+
+    async def _parse_html(self, file_name: str) -> str:
+        """Parses an HTML file, extracting structured plain text."""
+        self.ap.logger.info(f'Parsing HTML file: {file_name}')
+
+        html_bytes = await self.ap.storage_mgr.storage_provider.load(file_name)
+
+        def _parse_html_sync():
+            html_content = io.BytesIO(html_bytes).read().decode('utf-8', errors='ignore')
+            soup = BeautifulSoup(html_content, 'html.parser')
+            for script_or_style in soup(['script', 'style']):
+                script_or_style.decompose()
+            text_parts = []
+            for element in soup.body.children if soup.body else soup.children:
+                if element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                    level = int(element.name[1])
+                    text_parts.append('#' * level + ' ' + element.get_text().strip())
+                elif element.name == 'p':
+                    text = element.get_text().strip()
+                    if text:
+                        text_parts.append(text)
+                elif element.name in ['ul', 'ol']:
+                    for li in element.find_all('li'):
+                        text = li.get_text().strip()
+                        if text:
+                            text_parts.append(f'* {text}')
+                elif element.name == 'table':
+                    table_str = self._extract_table_to_markdown_sync(element)  # Call sync helper
+                    if table_str:
+                        text_parts.append(table_str)
+                elif element.name:
+                    text = element.get_text(separator=' ', strip=True)
+                    if text:
+                        text_parts.append(text)
+            cleaned_text = re.sub(r'\n\s*\n', '\n\n', '\n'.join(text_parts))
+            return cleaned_text.strip()
+
+        return await self._run_sync(_parse_html_sync)
+
+    def _add_toc_items_sync(self, toc_list: list, text_content: list, level: int):
+        """Recursively adds TOC items to text_content (synchronous helper)."""
+        indent = '  ' * level
+        for item in toc_list:
+            if isinstance(item, tuple):
+                chapter, subchapters = item
+                text_content.append(f'{indent}- {chapter.title}')
+                self._add_toc_items_sync(subchapters, text_content, level + 1)
+            else:
+                text_content.append(f'{indent}- {item.title}')
+
+    def _extract_table_to_markdown_sync(self, table_element: BeautifulSoup) -> str:
+        """Helper to convert a BeautifulSoup table element into a Markdown table string (synchronous)."""
+        headers = [th.get_text().strip() for th in table_element.find_all('th')]
+        rows = []
+        for tr in table_element.find_all('tr'):
+            cells = [td.get_text().strip() for td in tr.find_all('td')]
+            if cells:
+                rows.append(cells)
+
+        if not headers and not rows:
+            return ''
+
+        table_lines = []
+        if headers:
+            table_lines.append(' | '.join(headers))
+            table_lines.append(' | '.join(['---'] * len(headers)))
+
+        for row_cells in rows:
+            padded_cells = row_cells + [''] * (len(headers) - len(row_cells)) if headers else row_cells
+            table_lines.append(' | '.join(padded_cells))
+
+        return '\n'.join(table_lines)

src/langbot/pkg/rag/knowledge/services/retriever.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from . import base_service
+from ....core import app
+from ....provider.modelmgr.requester import RuntimeEmbeddingModel
+from ....entity.rag import retriever as retriever_entities
+
+
+class Retriever(base_service.BaseService):
+    def __init__(self, ap: app.Application):
+        super().__init__()
+        self.ap = ap
+
+    async def retrieve(
+        self, kb_id: str, query: str, embedding_model: RuntimeEmbeddingModel, k: int = 5
+    ) -> list[retriever_entities.RetrieveResultEntry]:
+        self.ap.logger.info(
+            f"Retrieving for query: '{query[:10]}' with k={k} using {embedding_model.model_entity.uuid}"
+        )
+
+        query_embedding: list[float] = await embedding_model.requester.invoke_embedding(
+            model=embedding_model,
+            input_text=[query],
+            extra_args={},  # TODO: add extra args
+        )
+
+        vector_results = await self.ap.vector_db_mgr.vector_db.search(kb_id, query_embedding[0], k)
+
+        # 'ids' shape mirrors the Chroma-style response contract for compatibility
+        matched_vector_ids = vector_results.get('ids', [[]])[0]
+        distances = vector_results.get('distances', [[]])[0]
+        vector_metadatas = vector_results.get('metadatas', [[]])[0]
+
+        if not matched_vector_ids:
+            self.ap.logger.info('No relevant chunks found in vector database.')
+            return []
+
+        result: list[retriever_entities.RetrieveResultEntry] = []
+
+        for i, id in enumerate(matched_vector_ids):
+            entry = retriever_entities.RetrieveResultEntry(
+                id=id,
+                metadata=vector_metadatas[i],
+                distance=distances[i],
+            )
+            result.append(entry)
+
+        return result