feat(tenancy): harden shared cloud runtime boundaries

This commit is contained in:
Junyan Qin
2026-07-20 01:47:42 +08:00
parent 41772920ef
commit a47bfe8167
121 changed files with 18152 additions and 5588 deletions
+183 -29
View File
@@ -64,9 +64,26 @@ class VectorDBManager:
# Get pgvector configuration
pgvector_config = kb_config.get('pgvector', {})
use_business_database = pgvector_config.get('use_business_database', False)
allowed_dimensions = pgvector_config.get(
'allowed_dimensions',
[384, 512, 768, 1024, 1536],
)
common_options = {
'use_business_database': use_business_database,
'allowed_dimensions': allowed_dimensions,
}
if use_business_database:
self.vector_db = PgVectorDatabase(self.ap, **common_options)
self.ap.logger.info('Initialized pgvector on the shared business PostgreSQL database.')
return
connection_string = pgvector_config.get('connection_string')
if connection_string:
self.vector_db = PgVectorDatabase(self.ap, connection_string=connection_string)
self.vector_db = PgVectorDatabase(
self.ap,
connection_string=connection_string,
**common_options,
)
else:
# Use individual parameters
host = pgvector_config.get('host', 'localhost')
@@ -75,7 +92,13 @@ class VectorDBManager:
user = pgvector_config.get('user', 'postgres')
password = pgvector_config.get('password', 'postgres')
self.vector_db = PgVectorDatabase(
self.ap, host=host, port=port, database=database, user=user, password=password
self.ap,
host=host,
port=port,
database=database,
user=user,
password=password,
**common_options,
)
self.ap.logger.info('Initialized pgvector database backend.')
@@ -156,23 +179,24 @@ class VectorDBManager:
"""
await self._validate_execution_context(execution_context)
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_rag.KnowledgeBase.collection_id,
persistence_rag.KnowledgeBase.legacy_vector_collection,
persistence_workspace.Workspace.source,
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_rag.KnowledgeBase.collection_id,
persistence_rag.KnowledgeBase.legacy_vector_collection,
persistence_workspace.Workspace.source,
)
.join(
persistence_workspace.Workspace,
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
)
.where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
)
.limit(1)
)
.join(
persistence_workspace.Workspace,
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
)
.where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
)
.limit(1)
)
row = result.first()
if row is None:
raise WorkspaceNotFoundError('Knowledge base not found')
@@ -194,6 +218,58 @@ class VectorDBManager:
return self.physical_collection_name(execution_context, knowledge_base_uuid)
def _pgvector_database(self):
from .vdbs.pgvector_db import PgVectorDatabase
return self.vector_db if isinstance(self.vector_db, PgVectorDatabase) else None
async def _resolve_pgvector_scope(
self,
execution_context: ExecutionContext,
knowledge_base_uuid: str,
*,
expected_dimension: int | None,
initialize_dimension: bool,
):
"""Bind and verify the server-owned knowledge-base vector dimension."""
from .vdbs.pgvector_db import PgVectorScope
pgvector = self._pgvector_database()
if pgvector is None: # pragma: no cover - private call invariant
raise RuntimeError('pgvector scope requested for another vector backend')
if expected_dimension is not None and expected_dimension not in pgvector.allowed_dimensions:
raise ValueError(f'Embedding dimension {expected_dimension} is not enabled for this deployment')
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
query = sqlalchemy.select(persistence_rag.KnowledgeBase.embedding_dimension).where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
)
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
if current_dimension is None and expected_dimension is not None and initialize_dimension:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.KnowledgeBase)
.where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
persistence_rag.KnowledgeBase.embedding_dimension.is_(None),
)
.values(embedding_dimension=expected_dimension)
)
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
if expected_dimension is not None and current_dimension != expected_dimension:
if current_dimension is None:
raise ValueError('Knowledge base has no selected pgvector embedding dimension')
raise ValueError(f'Knowledge base embedding dimension is {current_dimension}, not {expected_dimension}')
return PgVectorScope(
workspace_uuid=execution_context.workspace_uuid,
knowledge_base_uuid=knowledge_base_uuid,
embedding_dimension=current_dimension,
)
async def upsert(
self,
execution_context: ExecutionContext,
@@ -219,6 +295,25 @@ class VectorDBManager:
}
for item in source_metadata
]
pgvector = self._pgvector_database()
if pgvector is not None:
if not vectors:
return
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=len(vectors[0]),
initialize_dimension=True,
)
await pgvector.add_embeddings(
collection=collection_name,
ids=ids,
embeddings_list=vectors,
metadatas=scoped_metadata,
documents=documents,
scope=scope,
)
return
await self.vector_db.add_embeddings(
collection=collection_name,
ids=ids,
@@ -248,15 +343,34 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
results = await self.vector_db.search(
collection=collection_name,
query_embedding=query_vector,
k=limit,
search_type=search_type,
query_text=query_text,
filter=filter,
vector_weight=vector_weight,
)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=len(query_vector),
initialize_dimension=False,
)
results = await pgvector.search(
collection=collection_name,
query_embedding=query_vector,
k=limit,
search_type=search_type,
query_text=query_text,
filter=filter,
vector_weight=vector_weight,
scope=scope,
)
else:
results = await self.vector_db.search(
collection=collection_name,
query_embedding=query_vector,
k=limit,
search_type=search_type,
query_text=query_text,
filter=filter,
vector_weight=vector_weight,
)
if not results or 'ids' not in results or not results['ids']:
return []
@@ -297,8 +411,20 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
pgvector = self._pgvector_database()
scope = None
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
for file_id in file_ids:
await self.vector_db.delete_by_file_id(collection_name, file_id)
if pgvector is not None:
await pgvector.delete_by_file_id(collection_name, file_id, scope=scope)
else:
await self.vector_db.delete_by_file_id(collection_name, file_id)
async def delete_collection(
self,
@@ -311,7 +437,17 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
await self.vector_db.delete_collection(collection_name)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
await pgvector.delete_collection(collection_name, scope=scope)
else:
await self.vector_db.delete_collection(collection_name)
async def delete_by_filter(
self,
@@ -328,6 +464,15 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
return await pgvector.delete_by_filter(collection_name, filter, scope=scope)
return await self.vector_db.delete_by_filter(collection_name, filter)
async def list_by_filter(
@@ -347,4 +492,13 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
return await pgvector.list_by_filter(collection_name, filter, limit, offset, scope=scope)
return await self.vector_db.list_by_filter(collection_name, filter, limit, offset)
+322 -276
View File
@@ -1,22 +1,31 @@
from __future__ import annotations
from typing import Any, Dict
from sqlalchemy import create_engine, text, Column, String, Text
from sqlalchemy.orm import declarative_base
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
import contextlib
import dataclasses
from collections.abc import AsyncIterator
from typing import Any
import sqlalchemy
from pgvector.sqlalchemy import Vector
from langbot.pkg.vector.vdb import VectorDatabase
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import declarative_base
from langbot.pkg.core import app
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
from langbot.pkg.vector.vdb import VectorDatabase
Base = declarative_base()
DEFAULT_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
# pgvector schema only stores these metadata fields.
_PG_SUPPORTED_FIELDS = {'text', 'file_id', 'chunk_uuid'}
# Callers use canonical metadata key 'uuid' but pgvector stores it as 'chunk_uuid'.
_PG_FIELD_ALIASES = {'uuid': 'chunk_uuid'}
# Map schema field names to SQLAlchemy columns (resolved lazily from PgVectorEntry).
_PG_COLUMN_MAP = {
'text': 'text',
'file_id': 'file_id',
@@ -24,21 +33,50 @@ _PG_COLUMN_MAP = {
}
@dataclasses.dataclass(frozen=True, slots=True)
class PgVectorScope:
"""Trusted relational tenant key for one knowledge-base operation."""
workspace_uuid: str
knowledge_base_uuid: str
embedding_dimension: int | None = None
def __post_init__(self) -> None:
for field_name in ('workspace_uuid', 'knowledge_base_uuid'):
value = getattr(self, field_name)
if not isinstance(value, str) or not value.strip():
raise ValueError(f'{field_name} must not be empty')
object.__setattr__(self, field_name, value.strip())
dimension = self.embedding_dimension
if dimension is not None and (isinstance(dimension, bool) or not isinstance(dimension, int) or dimension <= 0):
raise ValueError('embedding_dimension must be a positive integer')
class PgVectorEntry(Base):
"""SQLAlchemy model for pgvector entries"""
"""Tenant-scoped pgvector row created only by release/OSS migrations."""
__tablename__ = 'langbot_vectors'
id = Column(String, primary_key=True)
collection = Column(String, index=True, nullable=False)
embedding = Column(Vector(1536)) # Default dimension, will be created dynamically
text = Column(Text)
file_id = Column(String, index=True)
chunk_uuid = Column(String)
workspace_uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True)
knowledge_base_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
vector_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
embedding = sqlalchemy.Column(Vector(), nullable=False)
text = sqlalchemy.Column(sqlalchemy.Text)
file_id = sqlalchemy.Column(sqlalchemy.String(255), index=True)
chunk_uuid = sqlalchemy.Column(sqlalchemy.String(255))
__table_args__ = (
sqlalchemy.CheckConstraint(
'vector_dims(embedding) = embedding_dimension',
name='ck_langbot_vectors_embedding_dimension',
),
)
def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
"""Translate canonical filter dict into a list of SQLAlchemy conditions."""
"""Translate canonical filter dict into SQLAlchemy conditions."""
triples = normalize_filter(filter_dict)
triples = strip_unsupported_fields(triples, _PG_SUPPORTED_FIELDS, _PG_FIELD_ALIASES)
@@ -65,83 +103,139 @@ def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
class PgVectorDatabase(VectorDatabase):
"""PostgreSQL with pgvector extension database implementation"""
"""PostgreSQL vector adapter with explicit Workspace/RLS scope.
Cloud reuses the business database engine and never performs DDL. OSS can
still opt into a standalone pgvector database; that compatibility mode may
create a fresh schema, but it uses the same explicit tenant keys.
"""
def __init__(
self,
ap: app.Application,
connection_string: str = None,
connection_string: str | None = None,
host: str = 'localhost',
port: int = 5432,
database: str = 'langbot',
user: str = 'postgres',
password: str = 'postgres',
):
"""Initialize pgvector database
Args:
ap: Application instance
connection_string: Full PostgreSQL connection string (overrides other params)
host: PostgreSQL host
port: PostgreSQL port
database: Database name
user: Database user
password: Database password
"""
*,
use_business_database: bool = False,
allowed_dimensions: list[int] | tuple[int, ...] = DEFAULT_ALLOWED_DIMENSIONS,
) -> None:
self.ap = ap
self.use_business_database = use_business_database
self.allowed_dimensions = self._normalize_allowed_dimensions(allowed_dimensions)
self.engine = None
self.async_engine = None
self.AsyncSessionLocal: async_sessionmaker[AsyncSession] | None = None
if use_business_database:
persistence_mgr = getattr(ap, 'persistence_mgr', None)
if persistence_mgr is None:
raise RuntimeError('Shared pgvector requires the initialized business persistence manager')
business_engine = persistence_mgr.get_db_engine()
if business_engine.dialect.name != 'postgresql':
raise RuntimeError('Shared pgvector requires the PostgreSQL business database')
self.async_engine = business_engine
self.ap.logger.info('Connected pgvector adapter to the shared PostgreSQL business database')
return
# Build connection string if not provided
if connection_string:
self.connection_string = connection_string
else:
self.connection_string = f'postgresql+psycopg://{user}:{password}@{host}:{port}/{database}'
self.async_connection_string = self.connection_string.replace('postgresql://', 'postgresql+asyncpg://').replace(
'postgresql+psycopg://', 'postgresql+asyncpg://'
)
self._initialize_standalone_db()
self.engine = None
self.async_engine = None
self.SessionLocal = None
self.AsyncSessionLocal = None
self._collections = set()
self._initialize_db()
@staticmethod
def _normalize_allowed_dimensions(dimensions: list[int] | tuple[int, ...]) -> frozenset[int]:
if not isinstance(dimensions, (list, tuple)) or not dimensions:
raise ValueError('pgvector allowed_dimensions must be a non-empty list')
if any(isinstance(item, bool) or not isinstance(item, int) or item <= 0 for item in dimensions):
raise ValueError('pgvector allowed_dimensions must contain positive integers')
unsupported = set(dimensions) - set(DEFAULT_ALLOWED_DIMENSIONS)
if unsupported:
raise ValueError(f'pgvector dimensions do not have release-created ANN indexes: {sorted(unsupported)}')
return frozenset(dimensions)
def _initialize_db(self):
"""Initialize database connection and create tables"""
try:
# Create async engine for async operations
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
def _initialize_standalone_db(self) -> None:
"""Initialize the explicit OSS external database compatibility path."""
# Create sync engine for table creation
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
self.engine = create_engine(sync_connection_string, echo=False)
from sqlalchemy import create_engine
# Create pgvector extension and tables
with self.engine.connect() as conn:
# Enable pgvector extension
conn.execute(text('CREATE EXTENSION IF NOT EXISTS vector'))
conn.commit()
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
self.engine = create_engine(sync_connection_string, echo=False)
# Create tables
Base.metadata.create_all(self.engine)
with self.engine.begin() as conn:
conn.execute(sqlalchemy.text('CREATE EXTENSION IF NOT EXISTS vector'))
existing_tables = set(sqlalchemy.inspect(conn).get_table_names())
if PgVectorEntry.__tablename__ in existing_tables:
columns = {
column['name'] for column in sqlalchemy.inspect(conn).get_columns(PgVectorEntry.__tablename__)
}
required = {
'workspace_uuid',
'knowledge_base_uuid',
'vector_id',
'embedding_dimension',
'embedding',
}
if not required.issubset(columns):
raise RuntimeError(
'The external pgvector database uses the legacy unscoped schema; '
'migrate it before enabling multi-tenant vector access'
)
Base.metadata.create_all(conn)
self.ap.logger.info('Connected to PostgreSQL with pgvector')
except Exception as e:
self.ap.logger.error(f'Failed to connect to PostgreSQL: {e}')
raise
self.ap.logger.info('Connected to standalone PostgreSQL pgvector database')
def _require_scope(self, scope: PgVectorScope | None, *, require_dimension: bool) -> PgVectorScope:
if not isinstance(scope, PgVectorScope):
raise ValueError('pgvector operations require a trusted PgVectorScope')
dimension = scope.embedding_dimension
if require_dimension and dimension is None:
raise ValueError('pgvector operation requires an embedding dimension')
if dimension is not None and dimension not in self.allowed_dimensions:
raise ValueError(f'Embedding dimension {dimension} is not enabled for this pgvector deployment')
return scope
@staticmethod
def _scope_conditions(scope: PgVectorScope) -> tuple[Any, Any]:
return (
PgVectorEntry.workspace_uuid == scope.workspace_uuid,
PgVectorEntry.knowledge_base_uuid == scope.knowledge_base_uuid,
)
@contextlib.asynccontextmanager
async def _session(self, scope: PgVectorScope) -> AsyncIterator[AsyncSession]:
admission = getattr(self.ap, 'deployment_admission', None)
if admission is not None:
admission.require_active()
if self.use_business_database:
async with self.ap.persistence_mgr.tenant_uow(scope.workspace_uuid) as uow:
yield uow.session
if admission is not None:
admission.require_active()
return
if self.AsyncSessionLocal is None: # pragma: no cover - constructor invariant
raise RuntimeError('Standalone pgvector session factory is unavailable')
async with self.AsyncSessionLocal() as session, session.begin():
yield session
if admission is not None:
admission.require_active()
async def get_or_create_collection(self, collection: str):
"""Get or create a collection (logical grouping in pgvector)
"""Retain the common adapter API; relational rows need no collection DDL."""
Args:
collection: Collection name (knowledge base UUID)
"""
# In pgvector, collections are logical - we just track them
if collection not in self._collections:
self._collections.add(collection)
self.ap.logger.info(f"Registered pgvector collection '{collection}'")
if not isinstance(collection, str) or not collection.strip():
raise ValueError('collection must not be empty')
return collection
async def add_embeddings(
@@ -151,38 +245,59 @@ class PgVectorDatabase(VectorDatabase):
embeddings_list: list[list[float]],
metadatas: list[dict[str, Any]],
documents: list[str] | None = None,
*,
scope: PgVectorScope | None = None,
) -> None:
"""Add vector embeddings to pgvector
Args:
collection: Collection name
ids: List of unique IDs for each vector
embeddings_list: List of embedding vectors
metadatas: List of metadata dictionaries
"""
scope = self._require_scope(scope, require_dimension=True)
await self.get_or_create_collection(collection)
if not ids:
return
if len(ids) != len(embeddings_list) or len(metadatas) != len(ids):
raise ValueError('pgvector ids, embeddings and metadata lengths must match')
if documents is not None and len(documents) != len(ids):
raise ValueError('pgvector documents length must match ids')
if len(set(ids)) != len(ids) or any(not isinstance(item, str) or not item.strip() for item in ids):
raise ValueError('pgvector vector IDs must be unique non-empty strings per upsert')
expected_dimension = scope.embedding_dimension
if any(len(embedding) != expected_dimension for embedding in embeddings_list):
raise ValueError(f'All embeddings must have the selected dimension {expected_dimension}')
async with self.AsyncSessionLocal() as session:
try:
for i, vector_id in enumerate(ids):
metadata = metadatas[i] if i < len(metadatas) else {}
values = []
for index, vector_id in enumerate(ids):
metadata = metadatas[index]
document = documents[index] if documents is not None else None
values.append(
{
'workspace_uuid': scope.workspace_uuid,
'knowledge_base_uuid': scope.knowledge_base_uuid,
'vector_id': vector_id.strip(),
'embedding_dimension': expected_dimension,
'embedding': embeddings_list[index],
'text': metadata.get('text', document or ''),
'file_id': metadata.get('file_id', ''),
'chunk_uuid': metadata.get('uuid', metadata.get('chunk_uuid', '')),
}
)
entry = PgVectorEntry(
id=vector_id,
collection=collection,
embedding=embeddings_list[i],
text=metadata.get('text', ''),
file_id=metadata.get('file_id', ''),
chunk_uuid=metadata.get('uuid', ''),
)
session.add(entry)
await session.commit()
self.ap.logger.info(f"Added {len(ids)} embeddings to pgvector collection '{collection}'")
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error adding embeddings to pgvector: {e}')
raise
statement = postgresql_insert(PgVectorEntry).values(values)
excluded = statement.excluded
statement = statement.on_conflict_do_update(
index_elements=[
PgVectorEntry.workspace_uuid,
PgVectorEntry.knowledge_base_uuid,
PgVectorEntry.vector_id,
],
set_={
'embedding_dimension': excluded.embedding_dimension,
'embedding': excluded.embedding,
'text': excluded.text,
'file_id': excluded.file_id,
'chunk_uuid': excluded.chunk_uuid,
},
)
async with self._session(scope) as session:
await session.execute(statement)
self.ap.logger.info(f'Upserted {len(ids)} pgvector embeddings for knowledge base {scope.knowledge_base_uuid}')
async def search(
self,
@@ -193,125 +308,79 @@ class PgVectorDatabase(VectorDatabase):
query_text: str = '',
filter: dict[str, Any] | None = None,
vector_weight: float | None = None,
) -> Dict[str, Any]:
"""Search for similar vectors using cosine distance
Args:
collection: Collection name
query_embedding: Query vector
k: Number of top results to return
Returns:
Dictionary with search results in Chroma-compatible format
"""
*,
scope: PgVectorScope | None = None,
) -> dict[str, Any]:
del query_text, vector_weight
scope = self._require_scope(scope, require_dimension=True)
await self.get_or_create_collection(collection)
if search_type != 'vector':
raise ValueError('pgvector currently supports vector search only')
if k <= 0:
raise ValueError('pgvector search limit must be positive')
if len(query_embedding) != scope.embedding_dimension:
raise ValueError(f'Query embedding must have the selected dimension {scope.embedding_dimension}')
async with self.AsyncSessionLocal() as session:
try:
# Use cosine distance for similarity search
from sqlalchemy import select
typed_embedding = sqlalchemy.cast(PgVectorEntry.embedding, Vector(scope.embedding_dimension))
distance = typed_embedding.cosine_distance(query_embedding)
statement = (
sqlalchemy.select(
PgVectorEntry.vector_id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
distance.label('distance'),
)
.where(*self._scope_conditions(scope), PgVectorEntry.embedding_dimension == scope.embedding_dimension)
.order_by(distance)
.limit(k)
)
for condition in _build_pg_conditions(filter or {}):
statement = statement.where(condition)
# Query for similar vectors
stmt = (
select(
PgVectorEntry.id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
PgVectorEntry.embedding.cosine_distance(query_embedding).label('distance'),
)
.filter(PgVectorEntry.collection == collection)
.order_by(PgVectorEntry.embedding.cosine_distance(query_embedding))
.limit(k)
)
async with self._session(scope) as session:
rows = (await session.execute(statement)).all()
if filter:
for cond in _build_pg_conditions(filter):
stmt = stmt.filter(cond)
ids = [row.vector_id for row in rows]
distances = [float(row.distance) for row in rows]
metadatas = [
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''} for row in rows
]
return {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
result = await session.execute(stmt)
rows = result.fetchall()
# Convert to Chroma-compatible format
ids = []
distances = []
metadatas = []
for row in rows:
ids.append(row.id)
distances.append(float(row.distance))
metadatas.append(
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''}
)
result_dict = {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
self.ap.logger.info(f"pgvector search in '{collection}' returned {len(ids)} results")
return result_dict
except Exception as e:
self.ap.logger.error(f'Error searching pgvector: {e}')
raise
async def delete_by_file_id(self, collection: str, file_id: str) -> None:
"""Delete vectors by file_id
Args:
collection: Collection name
file_id: File ID to filter deletion
"""
async def delete_by_file_id(
self,
collection: str,
file_id: str,
*,
scope: PgVectorScope | None = None,
) -> None:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
statement = sqlalchemy.delete(PgVectorEntry).where(
*self._scope_conditions(scope),
PgVectorEntry.file_id == file_id,
)
async with self._session(scope) as session:
await session.execute(statement)
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import delete
stmt = delete(PgVectorEntry).where(
PgVectorEntry.collection == collection, PgVectorEntry.file_id == file_id
)
await session.execute(stmt)
await session.commit()
self.ap.logger.info(
f"Deleted embeddings from pgvector collection '{collection}' with file_id: {file_id}"
)
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error deleting from pgvector: {e}')
raise
async def delete_by_filter(self, collection: str, filter: dict[str, Any]) -> int:
"""Delete vectors matching a metadata filter.
Args:
collection: Collection name
filter: Canonical metadata filter dict
"""
async def delete_by_filter(
self,
collection: str,
filter: dict[str, Any],
*,
scope: PgVectorScope | None = None,
) -> int:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
conditions = _build_pg_conditions(filter)
if not conditions:
self.ap.logger.warning(
f"pgvector delete_by_filter on '{collection}': filter produced no conditions, skipping"
)
self.ap.logger.warning('pgvector delete_by_filter produced no supported conditions; skipping')
return 0
await self.get_or_create_collection(collection)
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import delete
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
for cond in conditions:
stmt = stmt.where(cond)
result = await session.execute(stmt)
await session.commit()
deleted = result.rowcount
self.ap.logger.info(f"Deleted {deleted} embeddings from pgvector collection '{collection}' by filter")
return deleted
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error deleting from pgvector by filter: {e}')
raise
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope), *conditions)
async with self._session(scope) as session:
result = await session.execute(statement)
return int(result.rowcount or 0)
async def list_by_filter(
self,
@@ -319,85 +388,62 @@ class PgVectorDatabase(VectorDatabase):
filter: dict[str, Any] | None = None,
limit: int = 20,
offset: int = 0,
*,
scope: PgVectorScope | None = None,
) -> tuple[list[dict[str, Any]], int]:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
if limit <= 0 or offset < 0:
raise ValueError('pgvector pagination requires limit > 0 and offset >= 0')
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import select, func
conditions = [*self._scope_conditions(scope), *_build_pg_conditions(filter or {})]
statement = (
sqlalchemy.select(
PgVectorEntry.vector_id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
)
.where(*conditions)
.order_by(PgVectorEntry.vector_id)
.offset(offset)
.limit(limit)
)
count_statement = sqlalchemy.select(sqlalchemy.func.count()).select_from(PgVectorEntry).where(*conditions)
async with self._session(scope) as session:
rows = (await session.execute(statement)).all()
total = int((await session.execute(count_statement)).scalar_one())
stmt = (
select(
PgVectorEntry.id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
)
.filter(PgVectorEntry.collection == collection)
.offset(offset)
.limit(limit)
)
return (
[
{
'id': row.vector_id,
'document': row.text or '',
'metadata': {
'text': row.text or '',
'file_id': row.file_id or '',
'uuid': row.chunk_uuid or '',
},
}
for row in rows
],
total,
)
count_stmt = (
select(func.count()).select_from(PgVectorEntry).filter(PgVectorEntry.collection == collection)
)
async def delete_collection(
self,
collection: str,
*,
scope: PgVectorScope | None = None,
) -> None:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope))
async with self._session(scope) as session:
await session.execute(statement)
if filter:
for cond in _build_pg_conditions(filter):
stmt = stmt.filter(cond)
count_stmt = count_stmt.filter(cond)
result = await session.execute(stmt)
rows = result.fetchall()
count_result = await session.execute(count_stmt)
total = count_result.scalar() or 0
items = []
for row in rows:
items.append(
{
'id': row.id,
'document': row.text or '',
'metadata': {
'text': row.text or '',
'file_id': row.file_id or '',
'uuid': row.chunk_uuid or '',
},
}
)
return items, total
except Exception as e:
self.ap.logger.error(f'Error listing from pgvector: {e}')
raise
async def delete_collection(self, collection: str):
"""Delete all vectors in a collection
Args:
collection: Collection name to delete
"""
if collection in self._collections:
self._collections.remove(collection)
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import delete
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
await session.execute(stmt)
await session.commit()
self.ap.logger.info(f"Deleted pgvector collection '{collection}'")
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error deleting pgvector collection: {e}')
raise
async def close(self):
"""Close database connections"""
if self.async_engine:
async def close(self) -> None:
if not self.use_business_database and self.async_engine is not None:
await self.async_engine.dispose()
if self.engine:
if self.engine is not None:
self.engine.dispose()