chore: Add PyPI package support for uvx/pip installation (#1764)

* Initial plan

* Add package structure and resource path utilities

- Created langbot/ package with __init__.py and __main__.py entry point
- Added paths utility to find frontend and resource files from package installation
- Updated config loading to use resource paths
- Updated frontend serving to use resource paths
- Added MANIFEST.in for package data inclusion
- Updated pyproject.toml with build system and entry points

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add PyPI publishing workflow and update license

- Created GitHub Actions workflow to build frontend and publish to PyPI
- Added license field to pyproject.toml to fix deprecation warning
- Updated .gitignore to exclude build artifacts
- Tested package building successfully

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Add PyPI installation documentation

- Created PYPI_INSTALLATION.md with detailed installation and usage instructions
- Updated README.md to feature uvx/pip installation as recommended method
- Updated README_EN.md with same changes for English documentation

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Address code review feedback

- Made package-data configuration more specific to langbot package only
- Improved path detection with caching to avoid repeated file I/O
- Removed sys.path searching which was incorrect for package data
- Removed interactive input() call for non-interactive environment compatibility
- Simplified error messages for version check

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Fix code review issues

- Use specific exception types instead of bare except
- Fix misleading comments about directory levels
- Remove redundant existence check before makedirs with exist_ok=True
- Use context manager for file opening to ensure proper cleanup

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* Simplify package configuration and document behavioral differences

- Removed redundant package-data configuration, relying on MANIFEST.in
- Added documentation about behavioral differences between package and source installation
- Clarified that include-package-data=true uses MANIFEST.in for data files

Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>

* chore: update pyproject.toml

* chore: try pack templates in langbot/

* chore: update

* chore: update

* chore: update

* chore: update

* chore: update

* chore: adjust dir structure

* chore: fix imports

* fix: read default-pipeline-config.json

* fix: read default-pipeline-config.json

* fix: tests

* ci: publish pypi

* chore: bump version 4.6.0-beta.1 for testing

* chore: add templates/**

* fix: send adapters and requesters icons

* chore: bump version 4.6.0b2 for testing

* chore: add platform field for docker-compose.yaml

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: RockChinQ <45992437+RockChinQ@users.noreply.github.com>
Co-authored-by: Junyan Qin <rockchinq@gmail.com>

src/langbot/pkg/entity/errors/platform.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+
+class AdapterNotFoundError(Exception):
+    def __init__(self, adapter_name: str):
+        self.adapter_name = adapter_name
+
+    def __str__(self):
+        return f'Adapter {self.adapter_name} not found'

src/langbot/pkg/entity/errors/provider.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+
+class RequesterNotFoundError(Exception):
+    def __init__(self, requester_name: str):
+        self.requester_name = requester_name
+
+    def __str__(self):
+        return f'Requester {self.requester_name} not found'

src/langbot/pkg/entity/persistence/apikey.py

@@ -0,0 +1,21 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class ApiKey(Base):
+    """API Key for external service authentication"""
+
+    __tablename__ = 'api_keys'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    key = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, unique=True)
+    description = sqlalchemy.Column(sqlalchemy.String(512), nullable=True, default='')
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/base.py

@@ -0,0 +1,5 @@
+import sqlalchemy.orm
+
+
+class Base(sqlalchemy.orm.DeclarativeBase):
+    pass

src/langbot/pkg/entity/persistence/bot.py

@@ -0,0 +1,25 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class Bot(Base):
+    """Bot"""
+
+    __tablename__ = 'bots'
+
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    description = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    adapter = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    adapter_config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
+    enable = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False)
+    use_pipeline_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
+    use_pipeline_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/bstorage.py

@@ -0,0 +1,22 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class BinaryStorage(Base):
+    """Current for plugin use only"""
+
+    __tablename__ = 'binary_storages'
+
+    unique_key = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
+    key = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    owner_type = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    owner = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    value = sqlalchemy.Column(sqlalchemy.LargeBinary, nullable=False)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/mcp.py

@@ -0,0 +1,20 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class MCPServer(Base):
+    __tablename__ = 'mcp_servers'
+
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    enable = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False)
+    mode = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)  # stdio, sse
+    extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/metadata.py

@@ -0,0 +1,21 @@
+import sqlalchemy
+
+from .base import Base
+from ...utils import constants
+
+
+initial_metadata = [
+    {
+        'key': 'database_version',
+        'value': str(constants.required_database_version),
+    },
+]
+
+
+class Metadata(Base):
+    """Database metadata"""
+
+    __tablename__ = 'metadata'
+
+    key = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
+    value = sqlalchemy.Column(sqlalchemy.String(255))

src/langbot/pkg/entity/persistence/model.py

@@ -0,0 +1,46 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class LLMModel(Base):
+    """LLM model"""
+
+    __tablename__ = 'llm_models'
+
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    description = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    requester = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    requester_config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
+    api_keys = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
+    abilities = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=[])
+    extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )
+
+
+class EmbeddingModel(Base):
+    """Embedding 模型"""
+
+    __tablename__ = 'embedding_models'
+
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    description = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    requester = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    requester_config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
+    api_keys = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
+    extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/pipeline.py

@@ -0,0 +1,46 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class LegacyPipeline(Base):
+    """Legacy pipeline"""
+
+    __tablename__ = 'legacy_pipelines'
+
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    description = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )
+    for_version = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    is_default = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False)
+    stages = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
+    config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
+    extensions_preferences = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={})
+
+
+class PipelineRunRecord(Base):
+    """Pipeline run record"""
+
+    __tablename__ = 'pipeline_run_records'
+
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    pipeline_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    status = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )
+    started_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False)
+    finished_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False)
+    result = sqlalchemy.Column(sqlalchemy.JSON, nullable=False)
+    knowledge_base_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)

src/langbot/pkg/entity/persistence/plugin.py

@@ -0,0 +1,24 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class PluginSetting(Base):
+    """Plugin setting"""
+
+    __tablename__ = 'plugin_settings'
+
+    plugin_author = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
+    plugin_name = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
+    enabled = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True)
+    priority = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
+    config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=dict)
+    install_source = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, default='github')
+    install_info = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=dict)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/rag.py

@@ -0,0 +1,50 @@
+import sqlalchemy
+from .base import Base
+
+# Base = declarative_base()
+# DATABASE_URL = os.getenv('DATABASE_URL', 'sqlite:///./rag_knowledge.db')
+# print("Using database URL:", DATABASE_URL)
+
+
+# engine = create_engine(DATABASE_URL, connect_args={'check_same_thread': False})
+
+# SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+# def create_db_and_tables():
+#     """Creates all database tables defined in the Base."""
+#     Base.metadata.create_all(bind=engine)
+#     print('Database tables created or already exist.')
+
+
+class KnowledgeBase(Base):
+    __tablename__ = 'knowledge_bases'
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    name = sqlalchemy.Column(sqlalchemy.String, index=True)
+    description = sqlalchemy.Column(sqlalchemy.Text)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.func.now())
+    embedding_model_uuid = sqlalchemy.Column(sqlalchemy.String, default='')
+    top_k = sqlalchemy.Column(sqlalchemy.Integer, default=5)
+
+
+class File(Base):
+    __tablename__ = 'knowledge_base_files'
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    kb_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
+    file_name = sqlalchemy.Column(sqlalchemy.String)
+    extension = sqlalchemy.Column(sqlalchemy.String)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, default=sqlalchemy.func.now())
+    status = sqlalchemy.Column(sqlalchemy.String, default='pending')  # pending, processing, completed, failed
+
+
+class Chunk(Base):
+    __tablename__ = 'knowledge_base_chunks'
+    uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+    file_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
+    text = sqlalchemy.Column(sqlalchemy.Text)
+
+
+# class Vector(Base):
+#     __tablename__ = 'knowledge_base_vectors'
+#     uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True)
+#     chunk_id = sqlalchemy.Column(sqlalchemy.String, nullable=True)
+#     embedding = sqlalchemy.Column(sqlalchemy.LargeBinary)

src/langbot/pkg/entity/persistence/user.py

@@ -0,0 +1,18 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class User(Base):
+    __tablename__ = 'users'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    user = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    password = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/persistence/vector.py

@@ -0,0 +1,13 @@
+from sqlalchemy import Column, Integer, ForeignKey, LargeBinary
+from sqlalchemy.orm import declarative_base, relationship
+
+Base = declarative_base()
+
+
+class Vector(Base):
+    __tablename__ = 'vectors'
+    id = Column(Integer, primary_key=True, index=True)
+    chunk_id = Column(Integer, ForeignKey('chunks.id'), unique=True)
+    embedding = Column(LargeBinary)  # Store embeddings as binary
+
+    chunk = relationship('Chunk', back_populates='vector')

src/langbot/pkg/entity/persistence/webhook.py

@@ -0,0 +1,22 @@
+import sqlalchemy
+
+from .base import Base
+
+
+class Webhook(Base):
+    """Webhook for pushing bot events to external systems"""
+
+    __tablename__ = 'webhooks'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
+    name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
+    url = sqlalchemy.Column(sqlalchemy.String(1024), nullable=False)
+    description = sqlalchemy.Column(sqlalchemy.String(512), nullable=True, default='')
+    enabled = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True)
+    created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
+    updated_at = sqlalchemy.Column(
+        sqlalchemy.DateTime,
+        nullable=False,
+        server_default=sqlalchemy.func.now(),
+        onupdate=sqlalchemy.func.now(),
+    )

src/langbot/pkg/entity/rag/retriever.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import pydantic
+
+from typing import Any
+
+
+class RetrieveResultEntry(pydantic.BaseModel):
+    id: str
+
+    metadata: dict[str, Any]
+
+    distance: float