mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-07-22 04:16:07 +00:00
feat(tenancy): harden shared cloud runtime boundaries
This commit is contained in:
+183
-29
@@ -64,9 +64,26 @@ class VectorDBManager:
|
||||
|
||||
# Get pgvector configuration
|
||||
pgvector_config = kb_config.get('pgvector', {})
|
||||
use_business_database = pgvector_config.get('use_business_database', False)
|
||||
allowed_dimensions = pgvector_config.get(
|
||||
'allowed_dimensions',
|
||||
[384, 512, 768, 1024, 1536],
|
||||
)
|
||||
common_options = {
|
||||
'use_business_database': use_business_database,
|
||||
'allowed_dimensions': allowed_dimensions,
|
||||
}
|
||||
if use_business_database:
|
||||
self.vector_db = PgVectorDatabase(self.ap, **common_options)
|
||||
self.ap.logger.info('Initialized pgvector on the shared business PostgreSQL database.')
|
||||
return
|
||||
connection_string = pgvector_config.get('connection_string')
|
||||
if connection_string:
|
||||
self.vector_db = PgVectorDatabase(self.ap, connection_string=connection_string)
|
||||
self.vector_db = PgVectorDatabase(
|
||||
self.ap,
|
||||
connection_string=connection_string,
|
||||
**common_options,
|
||||
)
|
||||
else:
|
||||
# Use individual parameters
|
||||
host = pgvector_config.get('host', 'localhost')
|
||||
@@ -75,7 +92,13 @@ class VectorDBManager:
|
||||
user = pgvector_config.get('user', 'postgres')
|
||||
password = pgvector_config.get('password', 'postgres')
|
||||
self.vector_db = PgVectorDatabase(
|
||||
self.ap, host=host, port=port, database=database, user=user, password=password
|
||||
self.ap,
|
||||
host=host,
|
||||
port=port,
|
||||
database=database,
|
||||
user=user,
|
||||
password=password,
|
||||
**common_options,
|
||||
)
|
||||
self.ap.logger.info('Initialized pgvector database backend.')
|
||||
|
||||
@@ -156,23 +179,24 @@ class VectorDBManager:
|
||||
"""
|
||||
|
||||
await self._validate_execution_context(execution_context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_rag.KnowledgeBase.collection_id,
|
||||
persistence_rag.KnowledgeBase.legacy_vector_collection,
|
||||
persistence_workspace.Workspace.source,
|
||||
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_rag.KnowledgeBase.collection_id,
|
||||
persistence_rag.KnowledgeBase.legacy_vector_collection,
|
||||
persistence_workspace.Workspace.source,
|
||||
)
|
||||
.join(
|
||||
persistence_workspace.Workspace,
|
||||
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
|
||||
)
|
||||
.where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
.join(
|
||||
persistence_workspace.Workspace,
|
||||
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
|
||||
)
|
||||
.where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.first()
|
||||
if row is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
@@ -194,6 +218,58 @@ class VectorDBManager:
|
||||
|
||||
return self.physical_collection_name(execution_context, knowledge_base_uuid)
|
||||
|
||||
def _pgvector_database(self):
|
||||
from .vdbs.pgvector_db import PgVectorDatabase
|
||||
|
||||
return self.vector_db if isinstance(self.vector_db, PgVectorDatabase) else None
|
||||
|
||||
async def _resolve_pgvector_scope(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
knowledge_base_uuid: str,
|
||||
*,
|
||||
expected_dimension: int | None,
|
||||
initialize_dimension: bool,
|
||||
):
|
||||
"""Bind and verify the server-owned knowledge-base vector dimension."""
|
||||
|
||||
from .vdbs.pgvector_db import PgVectorScope
|
||||
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is None: # pragma: no cover - private call invariant
|
||||
raise RuntimeError('pgvector scope requested for another vector backend')
|
||||
if expected_dimension is not None and expected_dimension not in pgvector.allowed_dimensions:
|
||||
raise ValueError(f'Embedding dimension {expected_dimension} is not enabled for this deployment')
|
||||
|
||||
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
|
||||
query = sqlalchemy.select(persistence_rag.KnowledgeBase.embedding_dimension).where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
)
|
||||
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
|
||||
if current_dimension is None and expected_dimension is not None and initialize_dimension:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
persistence_rag.KnowledgeBase.embedding_dimension.is_(None),
|
||||
)
|
||||
.values(embedding_dimension=expected_dimension)
|
||||
)
|
||||
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
|
||||
|
||||
if expected_dimension is not None and current_dimension != expected_dimension:
|
||||
if current_dimension is None:
|
||||
raise ValueError('Knowledge base has no selected pgvector embedding dimension')
|
||||
raise ValueError(f'Knowledge base embedding dimension is {current_dimension}, not {expected_dimension}')
|
||||
|
||||
return PgVectorScope(
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
knowledge_base_uuid=knowledge_base_uuid,
|
||||
embedding_dimension=current_dimension,
|
||||
)
|
||||
|
||||
async def upsert(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
@@ -219,6 +295,25 @@ class VectorDBManager:
|
||||
}
|
||||
for item in source_metadata
|
||||
]
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
if not vectors:
|
||||
return
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=len(vectors[0]),
|
||||
initialize_dimension=True,
|
||||
)
|
||||
await pgvector.add_embeddings(
|
||||
collection=collection_name,
|
||||
ids=ids,
|
||||
embeddings_list=vectors,
|
||||
metadatas=scoped_metadata,
|
||||
documents=documents,
|
||||
scope=scope,
|
||||
)
|
||||
return
|
||||
await self.vector_db.add_embeddings(
|
||||
collection=collection_name,
|
||||
ids=ids,
|
||||
@@ -248,15 +343,34 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
results = await self.vector_db.search(
|
||||
collection=collection_name,
|
||||
query_embedding=query_vector,
|
||||
k=limit,
|
||||
search_type=search_type,
|
||||
query_text=query_text,
|
||||
filter=filter,
|
||||
vector_weight=vector_weight,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=len(query_vector),
|
||||
initialize_dimension=False,
|
||||
)
|
||||
results = await pgvector.search(
|
||||
collection=collection_name,
|
||||
query_embedding=query_vector,
|
||||
k=limit,
|
||||
search_type=search_type,
|
||||
query_text=query_text,
|
||||
filter=filter,
|
||||
vector_weight=vector_weight,
|
||||
scope=scope,
|
||||
)
|
||||
else:
|
||||
results = await self.vector_db.search(
|
||||
collection=collection_name,
|
||||
query_embedding=query_vector,
|
||||
k=limit,
|
||||
search_type=search_type,
|
||||
query_text=query_text,
|
||||
filter=filter,
|
||||
vector_weight=vector_weight,
|
||||
)
|
||||
|
||||
if not results or 'ids' not in results or not results['ids']:
|
||||
return []
|
||||
@@ -297,8 +411,20 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
scope = None
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
for file_id in file_ids:
|
||||
await self.vector_db.delete_by_file_id(collection_name, file_id)
|
||||
if pgvector is not None:
|
||||
await pgvector.delete_by_file_id(collection_name, file_id, scope=scope)
|
||||
else:
|
||||
await self.vector_db.delete_by_file_id(collection_name, file_id)
|
||||
|
||||
async def delete_collection(
|
||||
self,
|
||||
@@ -311,7 +437,17 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
await self.vector_db.delete_collection(collection_name)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
await pgvector.delete_collection(collection_name, scope=scope)
|
||||
else:
|
||||
await self.vector_db.delete_collection(collection_name)
|
||||
|
||||
async def delete_by_filter(
|
||||
self,
|
||||
@@ -328,6 +464,15 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
return await pgvector.delete_by_filter(collection_name, filter, scope=scope)
|
||||
return await self.vector_db.delete_by_filter(collection_name, filter)
|
||||
|
||||
async def list_by_filter(
|
||||
@@ -347,4 +492,13 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
return await pgvector.list_by_filter(collection_name, filter, limit, offset, scope=scope)
|
||||
return await self.vector_db.list_by_filter(collection_name, filter, limit, offset)
|
||||
|
||||
@@ -1,22 +1,31 @@
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict
|
||||
from sqlalchemy import create_engine, text, Column, String, Text
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
|
||||
import contextlib
|
||||
import dataclasses
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from langbot.pkg.vector.vdb import VectorDatabase
|
||||
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
|
||||
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
from langbot.pkg.core import app
|
||||
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
|
||||
from langbot.pkg.vector.vdb import VectorDatabase
|
||||
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
DEFAULT_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
|
||||
|
||||
# pgvector schema only stores these metadata fields.
|
||||
_PG_SUPPORTED_FIELDS = {'text', 'file_id', 'chunk_uuid'}
|
||||
|
||||
# Callers use canonical metadata key 'uuid' but pgvector stores it as 'chunk_uuid'.
|
||||
_PG_FIELD_ALIASES = {'uuid': 'chunk_uuid'}
|
||||
|
||||
# Map schema field names to SQLAlchemy columns (resolved lazily from PgVectorEntry).
|
||||
_PG_COLUMN_MAP = {
|
||||
'text': 'text',
|
||||
'file_id': 'file_id',
|
||||
@@ -24,21 +33,50 @@ _PG_COLUMN_MAP = {
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class PgVectorScope:
|
||||
"""Trusted relational tenant key for one knowledge-base operation."""
|
||||
|
||||
workspace_uuid: str
|
||||
knowledge_base_uuid: str
|
||||
embedding_dimension: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for field_name in ('workspace_uuid', 'knowledge_base_uuid'):
|
||||
value = getattr(self, field_name)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError(f'{field_name} must not be empty')
|
||||
object.__setattr__(self, field_name, value.strip())
|
||||
dimension = self.embedding_dimension
|
||||
if dimension is not None and (isinstance(dimension, bool) or not isinstance(dimension, int) or dimension <= 0):
|
||||
raise ValueError('embedding_dimension must be a positive integer')
|
||||
|
||||
|
||||
class PgVectorEntry(Base):
|
||||
"""SQLAlchemy model for pgvector entries"""
|
||||
"""Tenant-scoped pgvector row created only by release/OSS migrations."""
|
||||
|
||||
__tablename__ = 'langbot_vectors'
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
collection = Column(String, index=True, nullable=False)
|
||||
embedding = Column(Vector(1536)) # Default dimension, will be created dynamically
|
||||
text = Column(Text)
|
||||
file_id = Column(String, index=True)
|
||||
chunk_uuid = Column(String)
|
||||
workspace_uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True)
|
||||
knowledge_base_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
vector_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
|
||||
embedding = sqlalchemy.Column(Vector(), nullable=False)
|
||||
text = sqlalchemy.Column(sqlalchemy.Text)
|
||||
file_id = sqlalchemy.Column(sqlalchemy.String(255), index=True)
|
||||
chunk_uuid = sqlalchemy.Column(sqlalchemy.String(255))
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.CheckConstraint(
|
||||
'vector_dims(embedding) = embedding_dimension',
|
||||
name='ck_langbot_vectors_embedding_dimension',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
|
||||
"""Translate canonical filter dict into a list of SQLAlchemy conditions."""
|
||||
"""Translate canonical filter dict into SQLAlchemy conditions."""
|
||||
|
||||
triples = normalize_filter(filter_dict)
|
||||
triples = strip_unsupported_fields(triples, _PG_SUPPORTED_FIELDS, _PG_FIELD_ALIASES)
|
||||
|
||||
@@ -65,83 +103,139 @@ def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
|
||||
|
||||
|
||||
class PgVectorDatabase(VectorDatabase):
|
||||
"""PostgreSQL with pgvector extension database implementation"""
|
||||
"""PostgreSQL vector adapter with explicit Workspace/RLS scope.
|
||||
|
||||
Cloud reuses the business database engine and never performs DDL. OSS can
|
||||
still opt into a standalone pgvector database; that compatibility mode may
|
||||
create a fresh schema, but it uses the same explicit tenant keys.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: app.Application,
|
||||
connection_string: str = None,
|
||||
connection_string: str | None = None,
|
||||
host: str = 'localhost',
|
||||
port: int = 5432,
|
||||
database: str = 'langbot',
|
||||
user: str = 'postgres',
|
||||
password: str = 'postgres',
|
||||
):
|
||||
"""Initialize pgvector database
|
||||
|
||||
Args:
|
||||
ap: Application instance
|
||||
connection_string: Full PostgreSQL connection string (overrides other params)
|
||||
host: PostgreSQL host
|
||||
port: PostgreSQL port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
"""
|
||||
*,
|
||||
use_business_database: bool = False,
|
||||
allowed_dimensions: list[int] | tuple[int, ...] = DEFAULT_ALLOWED_DIMENSIONS,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self.use_business_database = use_business_database
|
||||
self.allowed_dimensions = self._normalize_allowed_dimensions(allowed_dimensions)
|
||||
self.engine = None
|
||||
self.async_engine = None
|
||||
self.AsyncSessionLocal: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
if use_business_database:
|
||||
persistence_mgr = getattr(ap, 'persistence_mgr', None)
|
||||
if persistence_mgr is None:
|
||||
raise RuntimeError('Shared pgvector requires the initialized business persistence manager')
|
||||
business_engine = persistence_mgr.get_db_engine()
|
||||
if business_engine.dialect.name != 'postgresql':
|
||||
raise RuntimeError('Shared pgvector requires the PostgreSQL business database')
|
||||
self.async_engine = business_engine
|
||||
self.ap.logger.info('Connected pgvector adapter to the shared PostgreSQL business database')
|
||||
return
|
||||
|
||||
# Build connection string if not provided
|
||||
if connection_string:
|
||||
self.connection_string = connection_string
|
||||
else:
|
||||
self.connection_string = f'postgresql+psycopg://{user}:{password}@{host}:{port}/{database}'
|
||||
|
||||
self.async_connection_string = self.connection_string.replace('postgresql://', 'postgresql+asyncpg://').replace(
|
||||
'postgresql+psycopg://', 'postgresql+asyncpg://'
|
||||
)
|
||||
self._initialize_standalone_db()
|
||||
|
||||
self.engine = None
|
||||
self.async_engine = None
|
||||
self.SessionLocal = None
|
||||
self.AsyncSessionLocal = None
|
||||
self._collections = set()
|
||||
self._initialize_db()
|
||||
@staticmethod
|
||||
def _normalize_allowed_dimensions(dimensions: list[int] | tuple[int, ...]) -> frozenset[int]:
|
||||
if not isinstance(dimensions, (list, tuple)) or not dimensions:
|
||||
raise ValueError('pgvector allowed_dimensions must be a non-empty list')
|
||||
if any(isinstance(item, bool) or not isinstance(item, int) or item <= 0 for item in dimensions):
|
||||
raise ValueError('pgvector allowed_dimensions must contain positive integers')
|
||||
unsupported = set(dimensions) - set(DEFAULT_ALLOWED_DIMENSIONS)
|
||||
if unsupported:
|
||||
raise ValueError(f'pgvector dimensions do not have release-created ANN indexes: {sorted(unsupported)}')
|
||||
return frozenset(dimensions)
|
||||
|
||||
def _initialize_db(self):
|
||||
"""Initialize database connection and create tables"""
|
||||
try:
|
||||
# Create async engine for async operations
|
||||
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
|
||||
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
|
||||
def _initialize_standalone_db(self) -> None:
|
||||
"""Initialize the explicit OSS external database compatibility path."""
|
||||
|
||||
# Create sync engine for table creation
|
||||
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
|
||||
self.engine = create_engine(sync_connection_string, echo=False)
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
# Create pgvector extension and tables
|
||||
with self.engine.connect() as conn:
|
||||
# Enable pgvector extension
|
||||
conn.execute(text('CREATE EXTENSION IF NOT EXISTS vector'))
|
||||
conn.commit()
|
||||
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
|
||||
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
|
||||
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
|
||||
self.engine = create_engine(sync_connection_string, echo=False)
|
||||
|
||||
# Create tables
|
||||
Base.metadata.create_all(self.engine)
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(sqlalchemy.text('CREATE EXTENSION IF NOT EXISTS vector'))
|
||||
existing_tables = set(sqlalchemy.inspect(conn).get_table_names())
|
||||
if PgVectorEntry.__tablename__ in existing_tables:
|
||||
columns = {
|
||||
column['name'] for column in sqlalchemy.inspect(conn).get_columns(PgVectorEntry.__tablename__)
|
||||
}
|
||||
required = {
|
||||
'workspace_uuid',
|
||||
'knowledge_base_uuid',
|
||||
'vector_id',
|
||||
'embedding_dimension',
|
||||
'embedding',
|
||||
}
|
||||
if not required.issubset(columns):
|
||||
raise RuntimeError(
|
||||
'The external pgvector database uses the legacy unscoped schema; '
|
||||
'migrate it before enabling multi-tenant vector access'
|
||||
)
|
||||
Base.metadata.create_all(conn)
|
||||
|
||||
self.ap.logger.info('Connected to PostgreSQL with pgvector')
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Failed to connect to PostgreSQL: {e}')
|
||||
raise
|
||||
self.ap.logger.info('Connected to standalone PostgreSQL pgvector database')
|
||||
|
||||
def _require_scope(self, scope: PgVectorScope | None, *, require_dimension: bool) -> PgVectorScope:
|
||||
if not isinstance(scope, PgVectorScope):
|
||||
raise ValueError('pgvector operations require a trusted PgVectorScope')
|
||||
dimension = scope.embedding_dimension
|
||||
if require_dimension and dimension is None:
|
||||
raise ValueError('pgvector operation requires an embedding dimension')
|
||||
if dimension is not None and dimension not in self.allowed_dimensions:
|
||||
raise ValueError(f'Embedding dimension {dimension} is not enabled for this pgvector deployment')
|
||||
return scope
|
||||
|
||||
@staticmethod
|
||||
def _scope_conditions(scope: PgVectorScope) -> tuple[Any, Any]:
|
||||
return (
|
||||
PgVectorEntry.workspace_uuid == scope.workspace_uuid,
|
||||
PgVectorEntry.knowledge_base_uuid == scope.knowledge_base_uuid,
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _session(self, scope: PgVectorScope) -> AsyncIterator[AsyncSession]:
|
||||
admission = getattr(self.ap, 'deployment_admission', None)
|
||||
if admission is not None:
|
||||
admission.require_active()
|
||||
|
||||
if self.use_business_database:
|
||||
async with self.ap.persistence_mgr.tenant_uow(scope.workspace_uuid) as uow:
|
||||
yield uow.session
|
||||
if admission is not None:
|
||||
admission.require_active()
|
||||
return
|
||||
|
||||
if self.AsyncSessionLocal is None: # pragma: no cover - constructor invariant
|
||||
raise RuntimeError('Standalone pgvector session factory is unavailable')
|
||||
async with self.AsyncSessionLocal() as session, session.begin():
|
||||
yield session
|
||||
if admission is not None:
|
||||
admission.require_active()
|
||||
|
||||
async def get_or_create_collection(self, collection: str):
|
||||
"""Get or create a collection (logical grouping in pgvector)
|
||||
"""Retain the common adapter API; relational rows need no collection DDL."""
|
||||
|
||||
Args:
|
||||
collection: Collection name (knowledge base UUID)
|
||||
"""
|
||||
# In pgvector, collections are logical - we just track them
|
||||
if collection not in self._collections:
|
||||
self._collections.add(collection)
|
||||
self.ap.logger.info(f"Registered pgvector collection '{collection}'")
|
||||
if not isinstance(collection, str) or not collection.strip():
|
||||
raise ValueError('collection must not be empty')
|
||||
return collection
|
||||
|
||||
async def add_embeddings(
|
||||
@@ -151,38 +245,59 @@ class PgVectorDatabase(VectorDatabase):
|
||||
embeddings_list: list[list[float]],
|
||||
metadatas: list[dict[str, Any]],
|
||||
documents: list[str] | None = None,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> None:
|
||||
"""Add vector embeddings to pgvector
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
ids: List of unique IDs for each vector
|
||||
embeddings_list: List of embedding vectors
|
||||
metadatas: List of metadata dictionaries
|
||||
"""
|
||||
scope = self._require_scope(scope, require_dimension=True)
|
||||
await self.get_or_create_collection(collection)
|
||||
if not ids:
|
||||
return
|
||||
if len(ids) != len(embeddings_list) or len(metadatas) != len(ids):
|
||||
raise ValueError('pgvector ids, embeddings and metadata lengths must match')
|
||||
if documents is not None and len(documents) != len(ids):
|
||||
raise ValueError('pgvector documents length must match ids')
|
||||
if len(set(ids)) != len(ids) or any(not isinstance(item, str) or not item.strip() for item in ids):
|
||||
raise ValueError('pgvector vector IDs must be unique non-empty strings per upsert')
|
||||
expected_dimension = scope.embedding_dimension
|
||||
if any(len(embedding) != expected_dimension for embedding in embeddings_list):
|
||||
raise ValueError(f'All embeddings must have the selected dimension {expected_dimension}')
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
for i, vector_id in enumerate(ids):
|
||||
metadata = metadatas[i] if i < len(metadatas) else {}
|
||||
values = []
|
||||
for index, vector_id in enumerate(ids):
|
||||
metadata = metadatas[index]
|
||||
document = documents[index] if documents is not None else None
|
||||
values.append(
|
||||
{
|
||||
'workspace_uuid': scope.workspace_uuid,
|
||||
'knowledge_base_uuid': scope.knowledge_base_uuid,
|
||||
'vector_id': vector_id.strip(),
|
||||
'embedding_dimension': expected_dimension,
|
||||
'embedding': embeddings_list[index],
|
||||
'text': metadata.get('text', document or ''),
|
||||
'file_id': metadata.get('file_id', ''),
|
||||
'chunk_uuid': metadata.get('uuid', metadata.get('chunk_uuid', '')),
|
||||
}
|
||||
)
|
||||
|
||||
entry = PgVectorEntry(
|
||||
id=vector_id,
|
||||
collection=collection,
|
||||
embedding=embeddings_list[i],
|
||||
text=metadata.get('text', ''),
|
||||
file_id=metadata.get('file_id', ''),
|
||||
chunk_uuid=metadata.get('uuid', ''),
|
||||
)
|
||||
session.add(entry)
|
||||
|
||||
await session.commit()
|
||||
self.ap.logger.info(f"Added {len(ids)} embeddings to pgvector collection '{collection}'")
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error adding embeddings to pgvector: {e}')
|
||||
raise
|
||||
statement = postgresql_insert(PgVectorEntry).values(values)
|
||||
excluded = statement.excluded
|
||||
statement = statement.on_conflict_do_update(
|
||||
index_elements=[
|
||||
PgVectorEntry.workspace_uuid,
|
||||
PgVectorEntry.knowledge_base_uuid,
|
||||
PgVectorEntry.vector_id,
|
||||
],
|
||||
set_={
|
||||
'embedding_dimension': excluded.embedding_dimension,
|
||||
'embedding': excluded.embedding,
|
||||
'text': excluded.text,
|
||||
'file_id': excluded.file_id,
|
||||
'chunk_uuid': excluded.chunk_uuid,
|
||||
},
|
||||
)
|
||||
async with self._session(scope) as session:
|
||||
await session.execute(statement)
|
||||
self.ap.logger.info(f'Upserted {len(ids)} pgvector embeddings for knowledge base {scope.knowledge_base_uuid}')
|
||||
|
||||
async def search(
|
||||
self,
|
||||
@@ -193,125 +308,79 @@ class PgVectorDatabase(VectorDatabase):
|
||||
query_text: str = '',
|
||||
filter: dict[str, Any] | None = None,
|
||||
vector_weight: float | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Search for similar vectors using cosine distance
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
query_embedding: Query vector
|
||||
k: Number of top results to return
|
||||
|
||||
Returns:
|
||||
Dictionary with search results in Chroma-compatible format
|
||||
"""
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del query_text, vector_weight
|
||||
scope = self._require_scope(scope, require_dimension=True)
|
||||
await self.get_or_create_collection(collection)
|
||||
if search_type != 'vector':
|
||||
raise ValueError('pgvector currently supports vector search only')
|
||||
if k <= 0:
|
||||
raise ValueError('pgvector search limit must be positive')
|
||||
if len(query_embedding) != scope.embedding_dimension:
|
||||
raise ValueError(f'Query embedding must have the selected dimension {scope.embedding_dimension}')
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Use cosine distance for similarity search
|
||||
from sqlalchemy import select
|
||||
typed_embedding = sqlalchemy.cast(PgVectorEntry.embedding, Vector(scope.embedding_dimension))
|
||||
distance = typed_embedding.cosine_distance(query_embedding)
|
||||
statement = (
|
||||
sqlalchemy.select(
|
||||
PgVectorEntry.vector_id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
distance.label('distance'),
|
||||
)
|
||||
.where(*self._scope_conditions(scope), PgVectorEntry.embedding_dimension == scope.embedding_dimension)
|
||||
.order_by(distance)
|
||||
.limit(k)
|
||||
)
|
||||
for condition in _build_pg_conditions(filter or {}):
|
||||
statement = statement.where(condition)
|
||||
|
||||
# Query for similar vectors
|
||||
stmt = (
|
||||
select(
|
||||
PgVectorEntry.id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
PgVectorEntry.embedding.cosine_distance(query_embedding).label('distance'),
|
||||
)
|
||||
.filter(PgVectorEntry.collection == collection)
|
||||
.order_by(PgVectorEntry.embedding.cosine_distance(query_embedding))
|
||||
.limit(k)
|
||||
)
|
||||
async with self._session(scope) as session:
|
||||
rows = (await session.execute(statement)).all()
|
||||
|
||||
if filter:
|
||||
for cond in _build_pg_conditions(filter):
|
||||
stmt = stmt.filter(cond)
|
||||
ids = [row.vector_id for row in rows]
|
||||
distances = [float(row.distance) for row in rows]
|
||||
metadatas = [
|
||||
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''} for row in rows
|
||||
]
|
||||
return {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
|
||||
|
||||
result = await session.execute(stmt)
|
||||
rows = result.fetchall()
|
||||
|
||||
# Convert to Chroma-compatible format
|
||||
ids = []
|
||||
distances = []
|
||||
metadatas = []
|
||||
|
||||
for row in rows:
|
||||
ids.append(row.id)
|
||||
distances.append(float(row.distance))
|
||||
metadatas.append(
|
||||
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''}
|
||||
)
|
||||
|
||||
result_dict = {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
|
||||
|
||||
self.ap.logger.info(f"pgvector search in '{collection}' returned {len(ids)} results")
|
||||
return result_dict
|
||||
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Error searching pgvector: {e}')
|
||||
raise
|
||||
|
||||
async def delete_by_file_id(self, collection: str, file_id: str) -> None:
|
||||
"""Delete vectors by file_id
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
file_id: File ID to filter deletion
|
||||
"""
|
||||
async def delete_by_file_id(
|
||||
self,
|
||||
collection: str,
|
||||
file_id: str,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> None:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
statement = sqlalchemy.delete(PgVectorEntry).where(
|
||||
*self._scope_conditions(scope),
|
||||
PgVectorEntry.file_id == file_id,
|
||||
)
|
||||
async with self._session(scope) as session:
|
||||
await session.execute(statement)
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import delete
|
||||
|
||||
stmt = delete(PgVectorEntry).where(
|
||||
PgVectorEntry.collection == collection, PgVectorEntry.file_id == file_id
|
||||
)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
self.ap.logger.info(
|
||||
f"Deleted embeddings from pgvector collection '{collection}' with file_id: {file_id}"
|
||||
)
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error deleting from pgvector: {e}')
|
||||
raise
|
||||
|
||||
async def delete_by_filter(self, collection: str, filter: dict[str, Any]) -> int:
|
||||
"""Delete vectors matching a metadata filter.
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
filter: Canonical metadata filter dict
|
||||
"""
|
||||
async def delete_by_filter(
|
||||
self,
|
||||
collection: str,
|
||||
filter: dict[str, Any],
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> int:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
conditions = _build_pg_conditions(filter)
|
||||
if not conditions:
|
||||
self.ap.logger.warning(
|
||||
f"pgvector delete_by_filter on '{collection}': filter produced no conditions, skipping"
|
||||
)
|
||||
self.ap.logger.warning('pgvector delete_by_filter produced no supported conditions; skipping')
|
||||
return 0
|
||||
|
||||
await self.get_or_create_collection(collection)
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import delete
|
||||
|
||||
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
|
||||
for cond in conditions:
|
||||
stmt = stmt.where(cond)
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
deleted = result.rowcount
|
||||
self.ap.logger.info(f"Deleted {deleted} embeddings from pgvector collection '{collection}' by filter")
|
||||
return deleted
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error deleting from pgvector by filter: {e}')
|
||||
raise
|
||||
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope), *conditions)
|
||||
async with self._session(scope) as session:
|
||||
result = await session.execute(statement)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def list_by_filter(
|
||||
self,
|
||||
@@ -319,85 +388,62 @@ class PgVectorDatabase(VectorDatabase):
|
||||
filter: dict[str, Any] | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
if limit <= 0 or offset < 0:
|
||||
raise ValueError('pgvector pagination requires limit > 0 and offset >= 0')
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import select, func
|
||||
conditions = [*self._scope_conditions(scope), *_build_pg_conditions(filter or {})]
|
||||
statement = (
|
||||
sqlalchemy.select(
|
||||
PgVectorEntry.vector_id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
)
|
||||
.where(*conditions)
|
||||
.order_by(PgVectorEntry.vector_id)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
count_statement = sqlalchemy.select(sqlalchemy.func.count()).select_from(PgVectorEntry).where(*conditions)
|
||||
async with self._session(scope) as session:
|
||||
rows = (await session.execute(statement)).all()
|
||||
total = int((await session.execute(count_statement)).scalar_one())
|
||||
|
||||
stmt = (
|
||||
select(
|
||||
PgVectorEntry.id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
)
|
||||
.filter(PgVectorEntry.collection == collection)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
return (
|
||||
[
|
||||
{
|
||||
'id': row.vector_id,
|
||||
'document': row.text or '',
|
||||
'metadata': {
|
||||
'text': row.text or '',
|
||||
'file_id': row.file_id or '',
|
||||
'uuid': row.chunk_uuid or '',
|
||||
},
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
total,
|
||||
)
|
||||
|
||||
count_stmt = (
|
||||
select(func.count()).select_from(PgVectorEntry).filter(PgVectorEntry.collection == collection)
|
||||
)
|
||||
async def delete_collection(
|
||||
self,
|
||||
collection: str,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> None:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope))
|
||||
async with self._session(scope) as session:
|
||||
await session.execute(statement)
|
||||
|
||||
if filter:
|
||||
for cond in _build_pg_conditions(filter):
|
||||
stmt = stmt.filter(cond)
|
||||
count_stmt = count_stmt.filter(cond)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
rows = result.fetchall()
|
||||
|
||||
count_result = await session.execute(count_stmt)
|
||||
total = count_result.scalar() or 0
|
||||
|
||||
items = []
|
||||
for row in rows:
|
||||
items.append(
|
||||
{
|
||||
'id': row.id,
|
||||
'document': row.text or '',
|
||||
'metadata': {
|
||||
'text': row.text or '',
|
||||
'file_id': row.file_id or '',
|
||||
'uuid': row.chunk_uuid or '',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return items, total
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Error listing from pgvector: {e}')
|
||||
raise
|
||||
|
||||
async def delete_collection(self, collection: str):
|
||||
"""Delete all vectors in a collection
|
||||
|
||||
Args:
|
||||
collection: Collection name to delete
|
||||
"""
|
||||
if collection in self._collections:
|
||||
self._collections.remove(collection)
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import delete
|
||||
|
||||
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
self.ap.logger.info(f"Deleted pgvector collection '{collection}'")
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error deleting pgvector collection: {e}')
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""Close database connections"""
|
||||
if self.async_engine:
|
||||
async def close(self) -> None:
|
||||
if not self.use_business_database and self.async_engine is not None:
|
||||
await self.async_engine.dispose()
|
||||
if self.engine:
|
||||
if self.engine is not None:
|
||||
self.engine.dispose()
|
||||
|
||||
Reference in New Issue
Block a user