Merge origin/master into feat/marketplace-installed-state-and-search

Resolved one conflict in src/langbot/pkg/plugin/connector.py. Master added
archive admission (`_admit_plugin_archive` + the certification digest check
before Runtime apply) at the same points this branch added install-stage
reporting, so both sides are kept:

- the archive is admitted first, then the 'inspecting plugin package' stage is
  reported
- the certification digest is verified before Runtime apply, then the
  'installing or starting plugin' stage is reported

Verified after merge: py_compile + ruff, tsc --noEmit, 98/98 unit tests.

src/langbot/pkg/api/http/controller/groups/plugins.py

@@ -923,10 +923,13 @@ class PluginsRouterGroup(group.RouterGroup):
                 return self.http_status(400, -1, 'file is required')
 
             file_bytes = file.read()
+            form = await quart.request.form
+            administrator_force = form.get('administrator_force', '').strip().lower() == 'true'
             execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
 
             data = {
                 'plugin_file': file_bytes,
+                'administrator_force': administrator_force,
             }
 
             ctx = taskmgr.TaskContext.new()

src/langbot/pkg/core/stages/load_config.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import os
 import copy
+import json
 from typing import Any
 from langbot.pkg.utils import bounded_executor, constants
 import yaml
@@ -42,6 +43,7 @@ _RUNTIME_POLICY_DEFAULTS = {
     },
     'plugin': {
         'connect_timeout_seconds': 180.0,
+        'certification': {'trusted_public_keys': {}},
         'worker': {
             'max_cpus': 1.0,
             'max_memory_mb': 512,
@@ -214,6 +216,30 @@ def _apply_env_overrides_to_config(cfg: dict) -> dict:
     return cfg
 
 
+def _apply_certification_key_ring_env(cfg: dict) -> dict:
+    """Load the public certification key ring from one strict JSON env value.
+
+    The generic environment override intentionally skips dictionaries. This
+    narrow exception keeps trusted issuer keys deployable without relying on a
+    mutable persisted config file, while rejecting malformed input instead of
+    silently running with an empty trust ring.
+    """
+    raw = os.getenv('PLUGIN__CERTIFICATION__TRUSTED_PUBLIC_KEYS_JSON')
+    if raw is None:
+        return cfg
+    try:
+        key_ring = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ValueError('PLUGIN__CERTIFICATION__TRUSTED_PUBLIC_KEYS_JSON must be valid JSON') from exc
+    if not isinstance(key_ring, dict) or any(
+        not isinstance(key_id, str) or not key_id.strip() or not isinstance(key, str) or not key.strip()
+        for key_id, key in key_ring.items()
+    ):
+        raise ValueError('PLUGIN__CERTIFICATION__TRUSTED_PUBLIC_KEYS_JSON must be a non-empty string-to-string mapping')
+    cfg['plugin']['certification']['trusted_public_keys'] = key_ring
+    return cfg
+
+
 @stage.stage_class('LoadConfigStage')
 class LoadConfigStage(stage.BootingStage):
     """Load config file stage"""
@@ -267,6 +293,7 @@ class LoadConfigStage(stage.BootingStage):
 
         # Apply environment variable overrides to data/config.yaml
         ap.instance_config.data = _apply_env_overrides_to_config(ap.instance_config.data)
+        ap.instance_config.data = _apply_certification_key_ring_env(ap.instance_config.data)
 
         blocking_config = ap.instance_config.data['system']['blocking_executor']
         ap.blocking_executor = bounded_executor.configure_bounded_default_executor(

src/langbot/pkg/plugin/archive.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import hashlib
 import io
 import zipfile
+from dataclasses import dataclass
+from enum import Enum
 
 import yaml
 
@@ -14,6 +17,34 @@ _PLUGIN_METADATA_MAX_BYTES = 1024 * 1024
 _PLUGIN_REQUIREMENTS_MAX_ENTRIES = 1000
 
 
+class ArchiveCertificateState(str, Enum):
+    """Syntactic certificate declaration state; this is not verification."""
+
+    ABSENT = 'absent'
+    MALFORMED = 'malformed'
+    DECLARED = 'declared'
+
+
+@dataclass(frozen=True)
+class ArchiveCertificateDeclaration:
+    """Bounded, verifier-facing certificate declaration from ``manifest.yaml``."""
+
+    state: ArchiveCertificateState
+    runtime_profile: str | None = None
+    payload: dict[str, object] | None = None
+
+
+@dataclass(frozen=True)
+class PluginArchiveInspection:
+    """Validated archive metadata plus unverified certificate declaration facts."""
+
+    manifest: dict
+    requirements: list[str]
+    names: list[str]
+    artifact_digest: str
+    certificate: ArchiveCertificateDeclaration
+
+
 def _read_plugin_archive_member(
     archive: zipfile.ZipFile,
     member: zipfile.ZipInfo,
@@ -29,12 +60,30 @@ def _read_plugin_archive_member(
     return content
 
 
-def inspect_plugin_archive_metadata(
-    file_bytes: bytes,
-    *,
-    require_manifest: bool = True,
-) -> tuple[dict, list[str], list[str]]:
-    """Validate archive size metadata and read only bounded preview fields."""
+def _inspect_certificate_declaration(manifest: dict) -> ArchiveCertificateDeclaration:
+    declaration = manifest.get('certification')
+    if declaration is None:
+        return ArchiveCertificateDeclaration(ArchiveCertificateState.ABSENT)
+    if not isinstance(declaration, dict):
+        return ArchiveCertificateDeclaration(ArchiveCertificateState.MALFORMED)
+
+    runtime_profile = declaration.get('runtime_profile')
+    payload = declaration.get('certificate')
+    if not isinstance(runtime_profile, str) or not runtime_profile.strip() or not isinstance(payload, dict):
+        return ArchiveCertificateDeclaration(ArchiveCertificateState.MALFORMED)
+    return ArchiveCertificateDeclaration(
+        ArchiveCertificateState.DECLARED,
+        runtime_profile=runtime_profile,
+        payload=payload,
+    )
+
+
+def inspect_plugin_archive(file_bytes: bytes, *, require_manifest: bool = True) -> PluginArchiveInspection:
+    """Validate an archive and expose certificate declaration facts for a verifier.
+
+    Certificate signatures and issuer trust are deliberately not evaluated here;
+    callers must pass the declaration and artifact digest to an SDK verifier.
+    """
 
     with zipfile.ZipFile(io.BytesIO(file_bytes)) as archive:
         members = archive.infolist()
@@ -95,4 +144,25 @@ def inspect_plugin_archive_metadata(
                 for line in content.splitlines()
                 if line.strip() and not line.strip().startswith('#')
             ][:_PLUGIN_REQUIREMENTS_MAX_ENTRIES]
-        return manifest, requirements, names
+
+        return PluginArchiveInspection(
+            manifest=manifest,
+            requirements=requirements,
+            names=names,
+            artifact_digest=hashlib.sha256(file_bytes).hexdigest(),
+            certificate=_inspect_certificate_declaration(manifest),
+        )
+
+
+def inspect_plugin_archive_metadata(
+    file_bytes: bytes,
+    *,
+    require_manifest: bool = True,
+) -> tuple[dict, list[str], list[str]]:
+    """Legacy tuple API for archive metadata callers.
+
+    Use ``inspect_plugin_archive`` when certificate declaration facts are needed.
+    """
+
+    inspection = inspect_plugin_archive(file_bytes, require_manifest=require_manifest)
+    return inspection.manifest, inspection.requirements, inspection.names

src/langbot/pkg/plugin/certification.py

@@ -0,0 +1,234 @@
+"""Pure certified-plugin facts and admission policies.
+
+This module intentionally does not verify signatures.  An SDK-backed verifier
+must produce ``CertificateFacts`` from an inspected archive before admission.
+"""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from enum import Enum
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
+from langbot_plugin.certification import normalized_zip_digest, verify_archive
+
+
+SHARED_RUNTIME_V1 = 'shared-runtime-v1'
+DEDICATED_RUNTIME = 'dedicated'
+
+
+class CertificateVerification(str, Enum):
+    ABSENT = 'absent'
+    MALFORMED = 'malformed'
+    INVALID = 'invalid'
+    VALID = 'valid'
+
+
+class DeploymentMode(str, Enum):
+    CLOUD = 'cloud'
+    OSS = 'oss'
+
+
+class AdmissionDisposition(str, Enum):
+    SHARED_ELIGIBLE = 'shared_eligible'
+    DEDICATED_ALLOWED = 'dedicated_allowed'
+    REJECTED = 'rejected'
+    ADMINISTRATOR_FORCE_REQUIRED = 'administrator_force_required'
+
+
+class AdmissionCode(str, Enum):
+    SHARED_ELIGIBLE = 'CERTIFIED_PLUGIN_SHARED_ELIGIBLE'
+    CLOUD_CERTIFICATE_REQUIRED = 'CERTIFIED_PLUGIN_CLOUD_CERTIFICATE_REQUIRED'
+    CLOUD_CERTIFICATE_INVALID = 'CERTIFIED_PLUGIN_CLOUD_CERTIFICATE_INVALID'
+    OSS_LEGACY_DEDICATED = 'CERTIFIED_PLUGIN_OSS_LEGACY_DEDICATED'
+    OSS_FORCE_REQUIRED = 'CERTIFIED_PLUGIN_OSS_FORCE_REQUIRED'
+    OSS_FORCED_DEDICATED = 'CERTIFIED_PLUGIN_OSS_FORCED_DEDICATED'
+    OSS_CERTIFIED_DEDICATED = 'CERTIFIED_PLUGIN_OSS_CERTIFIED_DEDICATED'
+
+
+class PluginLogVisibility(str, Enum):
+    TENANT_SCOPED = 'tenant_scoped'
+    DETAILED_PROCESS = 'detailed_process'
+
+
+@dataclass(frozen=True)
+class CertificateFacts:
+    """Certificate result supplied by an archive verifier.
+
+    ``VALID`` means the verifier has validated both the certificate and its
+    binding to the immutable artifact digest in ``PluginCertificationFacts``.
+    """
+
+    verification: CertificateVerification
+    runtime_profile: str | None = None
+    certificate_id: str | None = None
+
+    @property
+    def is_valid_shared_runtime(self) -> bool:
+        return self.verification is CertificateVerification.VALID and self.runtime_profile == SHARED_RUNTIME_V1
+
+    @property
+    def is_declared(self) -> bool:
+        return self.verification is not CertificateVerification.ABSENT
+
+
+@dataclass(frozen=True)
+class PluginCertificationFacts:
+    """Immutable Core-side facts for one plugin installation artifact."""
+
+    installation_uuid: str
+    artifact_digest: str
+    certificate: CertificateFacts
+
+    def __post_init__(self) -> None:
+        if len(self.artifact_digest) != 64 or any(
+            character not in '0123456789abcdef' for character in self.artifact_digest.lower()
+        ):
+            raise ValueError('artifact_digest must be a lowercase-or-uppercase SHA-256 hex digest')
+
+
+@dataclass(frozen=True)
+class VerifiedArchiveCertificate:
+    """SDK verification facts bound to the comment-normalized ZIP digest."""
+
+    normalized_digest: str
+    certificate: CertificateFacts
+
+    def for_installation(self, installation_uuid: str) -> PluginCertificationFacts:
+        return PluginCertificationFacts(
+            installation_uuid=installation_uuid,
+            artifact_digest=self.normalized_digest,
+            certificate=self.certificate,
+        )
+
+
+def trusted_public_key_ring(config: object) -> dict[str, Callable[[bytes, bytes], bool]]:
+    """Build the non-secret Ed25519 verifier ring from instance configuration.
+
+    ``plugin.certification.trusted_public_keys`` is a mapping of key IDs to
+    standard base64-encoded 32-byte Ed25519 public keys.  Configuration errors
+    are explicit so an operator never silently gets a weaker trust policy.
+    """
+
+    if config is None:
+        return {}
+    if not isinstance(config, Mapping):
+        raise ValueError('plugin.certification.trusted_public_keys must be a mapping')
+
+    ring: dict[str, Callable[[bytes, bytes], bool]] = {}
+    for raw_key_id, raw_public_key in config.items():
+        key_id = str(raw_key_id).strip()
+        if not key_id or not isinstance(raw_public_key, str):
+            raise ValueError('plugin.certification.trusted_public_keys entries must have string IDs and values')
+        try:
+            public_key_bytes = base64.b64decode(raw_public_key.encode('ascii'), validate=True)
+            public_key = Ed25519PublicKey.from_public_bytes(public_key_bytes)
+        except (UnicodeEncodeError, ValueError) as exc:
+            raise ValueError(f'plugin.certification trusted public key {key_id!r} is invalid') from exc
+
+        def verify(payload: bytes, signature: bytes, *, verifier: Ed25519PublicKey = public_key) -> bool:
+            try:
+                verifier.verify(signature, payload)
+            except (InvalidSignature, TypeError, ValueError):
+                return False
+            return True
+
+        ring[key_id] = verify
+    return ring
+
+
+def verify_plugin_archive_certificate(
+    archive: bytes,
+    *,
+    trusted_public_keys: object,
+) -> VerifiedArchiveCertificate:
+    """Use the SDK ZIP-comment API and retain its normalized-digest binding."""
+
+    verification = verify_archive(archive, trusted_public_key_ring(trusted_public_keys).get)
+    envelope = verification.envelope
+    runtime_profile = envelope.shared_runtime if envelope is not None else None
+    certificate_id = envelope.key_id if envelope is not None else None
+    state = {
+        'absent': CertificateVerification.ABSENT,
+        'malformed': CertificateVerification.MALFORMED,
+        'valid': CertificateVerification.VALID,
+    }.get(verification.status, CertificateVerification.INVALID)
+    return VerifiedArchiveCertificate(
+        normalized_digest=normalized_zip_digest(archive),
+        certificate=CertificateFacts(
+            verification=state,
+            runtime_profile=runtime_profile,
+            certificate_id=certificate_id,
+        ),
+    )
+
+
+@dataclass(frozen=True)
+class PluginAdmissionDecision:
+    disposition: AdmissionDisposition
+    code: AdmissionCode
+    runtime_profile: str
+
+
+def decide_plugin_admission(
+    *,
+    deployment: DeploymentMode | str,
+    facts: PluginCertificationFacts,
+    administrator_force: bool = False,
+) -> PluginAdmissionDecision:
+    """Apply Cloud fail-closed and OSS administrator-force admission rules."""
+
+    mode = DeploymentMode(deployment)
+    certificate = facts.certificate
+    if certificate.is_valid_shared_runtime:
+        return PluginAdmissionDecision(
+            AdmissionDisposition.SHARED_ELIGIBLE,
+            AdmissionCode.SHARED_ELIGIBLE,
+            SHARED_RUNTIME_V1,
+        )
+
+    if mode is DeploymentMode.CLOUD:
+        code = (
+            AdmissionCode.CLOUD_CERTIFICATE_REQUIRED
+            if certificate.verification is CertificateVerification.ABSENT
+            else AdmissionCode.CLOUD_CERTIFICATE_INVALID
+        )
+        return PluginAdmissionDecision(AdmissionDisposition.REJECTED, code, DEDICATED_RUNTIME)
+
+    if certificate.verification is CertificateVerification.ABSENT:
+        return PluginAdmissionDecision(
+            AdmissionDisposition.DEDICATED_ALLOWED,
+            AdmissionCode.OSS_LEGACY_DEDICATED,
+            DEDICATED_RUNTIME,
+        )
+
+    if certificate.verification is CertificateVerification.VALID:
+        return PluginAdmissionDecision(
+            AdmissionDisposition.DEDICATED_ALLOWED,
+            AdmissionCode.OSS_CERTIFIED_DEDICATED,
+            DEDICATED_RUNTIME,
+        )
+
+    if administrator_force:
+        return PluginAdmissionDecision(
+            AdmissionDisposition.DEDICATED_ALLOWED,
+            AdmissionCode.OSS_FORCED_DEDICATED,
+            DEDICATED_RUNTIME,
+        )
+
+    return PluginAdmissionDecision(
+        AdmissionDisposition.ADMINISTRATOR_FORCE_REQUIRED,
+        AdmissionCode.OSS_FORCE_REQUIRED,
+        DEDICATED_RUNTIME,
+    )
+
+
+def decide_plugin_log_visibility(facts: PluginCertificationFacts) -> PluginLogVisibility:
+    """Select the minimum log visibility compatible with a verified shared runtime."""
+
+    if facts.certificate.is_valid_shared_runtime:
+        return PluginLogVisibility.TENANT_SCOPED
+    return PluginLogVisibility.DETAILED_PROCESS

src/langbot/pkg/plugin/connector.py

@@ -22,6 +22,13 @@ from langbot_plugin.api.entities.builtin.pipeline.query import provider_session
 from ..core import app
 from . import handler
 from .archive import inspect_plugin_archive_metadata
+from .certification import (
+    AdmissionDisposition,
+    PluginCertificationFacts,
+    VerifiedArchiveCertificate,
+    decide_plugin_admission,
+    verify_plugin_archive_certificate,
+)
 from .github import (
     validate_github_plugin_install_info,
     validate_github_release_asset_url,
@@ -150,6 +157,30 @@ def _decode_json_object(body: bytes, *, subject: str) -> dict[str, Any]:
     return payload
 
 
+def _select_marketplace_plugin_version(
+    versions: Any,
+    *,
+    requested_version: str | None,
+    plugin_author: str,
+    plugin_name: str,
+) -> str:
+    if not isinstance(versions, list) or not versions:
+        raise ValueError(f'Plugin {plugin_author}/{plugin_name} has no versions')
+
+    if requested_version is None:
+        candidate = versions[0]
+        if not isinstance(candidate, dict) or not candidate.get('version'):
+            raise ValueError(f'Plugin {plugin_author}/{plugin_name} has no versions')
+        return str(candidate['version'])
+
+    for candidate in versions:
+        if isinstance(candidate, dict) and str(candidate.get('version') or '') == requested_version:
+            return requested_version
+    raise ValueError(
+        f'Plugin {plugin_author}/{plugin_name} version {requested_version} is not available in marketplace'
+    )
+
+
 class PluginRuntimeNotConnectedError(RuntimeError):
     """Raised when plugin runtime operations are requested before connection."""
 
@@ -1622,6 +1653,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
         execution_context: ExecutionContext,
         plugin_author: str,
         plugin_name: str,
+        plugin_version: str | None,
         task_context: taskmgr.TaskContext | None,
     ) -> tuple[bytes | None, str | None]:
         """Return a plugin package, or install an MCP/skill and return none."""
@@ -1686,21 +1718,59 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
                 subject='Marketplace plugin versions',
             )
             versions = versions_payload.get('data', {}).get('versions', [])
-            if (
-                not isinstance(versions, list)
-                or not versions
-                or not isinstance(versions[0], dict)
-                or not versions[0].get('version')
-            ):
-                raise ValueError(f'Plugin {plugin_author}/{plugin_name} has no versions')
-            latest_version = str(versions[0]['version'])
+            requested_version = str(plugin_version or '').strip()
+            version = _select_marketplace_plugin_version(
+                versions,
+                requested_version=requested_version or None,
+                plugin_author=plugin_author,
+                plugin_name=plugin_name,
+            )
             _download_status, plugin_package = await _marketplace_get(
                 client,
-                f'{space_url}/api/v1/marketplace/plugins/download/{plugin_author}/{plugin_name}/{latest_version}',
+                f'{space_url}/api/v1/marketplace/plugins/download/{plugin_author}/{plugin_name}/{version}',
                 max_bytes=_MARKETPLACE_PLUGIN_DOWNLOAD_MAX_BYTES,
                 task_context=task_context,
             )
-            return plugin_package, latest_version
+            return plugin_package, version
+
+    def _admit_plugin_archive(
+        self,
+        file_bytes: bytes,
+        install_info: dict[str, Any],
+    ) -> tuple[dict[str, Any], VerifiedArchiveCertificate]:
+        """Verify and admit one archive before it can reach durable storage or Runtime."""
+
+        certification_config = self.ap.instance_config.data.get('plugin', {}).get('certification', {})
+        if not isinstance(certification_config, dict):
+            raise ValueError('plugin.certification must be a mapping')
+        verified = verify_plugin_archive_certificate(
+            file_bytes,
+            trusted_public_keys=certification_config.get('trusted_public_keys', {}),
+        )
+        facts = PluginCertificationFacts(
+            installation_uuid='pending-installation',
+            artifact_digest=verified.normalized_digest,
+            certificate=verified.certificate,
+        )
+        decision = decide_plugin_admission(
+            deployment=getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss'),
+            facts=facts,
+            administrator_force=install_info.get('administrator_force') is True,
+        )
+        if decision.disposition not in {
+            AdmissionDisposition.DEDICATED_ALLOWED,
+            AdmissionDisposition.SHARED_ELIGIBLE,
+        }:
+            raise ValueError(decision.code.value)
+        certification_info = {
+            'normalized_digest': facts.artifact_digest,
+            'verification': facts.certificate.verification.value,
+            'certificate_runtime_profile': facts.certificate.runtime_profile,
+            'certificate_id': facts.certificate.certificate_id,
+            'runtime_profile': decision.runtime_profile,
+            'admission_code': decision.code.value,
+        }
+        return {**install_info, '_certification': certification_info}, verified
 
     async def install_plugin(
         self,
@@ -1733,6 +1803,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
                 execution_context,
                 plugin_author,
                 plugin_name,
+                str(install_info.get('plugin_version') or '') or None,
                 task_context,
             )
             if file_bytes is None:
@@ -1752,6 +1823,7 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
         else:
             raise ValueError(f'Unsupported plugin install source: {install_source.value}')
 
+        install_info, verified_certificate = self._admit_plugin_archive(file_bytes, install_info)
         if task_context is not None:
             task_context.set_current_action('inspecting plugin package')
         manifest_author, manifest_name = self._inspect_plugin_package(file_bytes, task_context)
@@ -1788,6 +1860,9 @@ class PluginRuntimeConnector(ManagedRuntimeConnector):
             plugin_author=plugin_author,
             plugin_name=plugin_name,
         )
+        certification_facts = verified_certificate.for_installation(binding.installation_uuid)
+        if certification_facts.artifact_digest != install_info['_certification']['normalized_digest']:
+            raise RuntimeError('Plugin certification digest changed before Runtime apply')
         if task_context is not None:
             # The runtime installs the plugin's dependencies and starts it
             # inside apply_plugin_installation. It does not stream

src/langbot/templates/config.yaml

@@ -261,6 +261,12 @@ plugin:
     runtime_ws_url: 'ws://langbot_plugin_runtime:5400/control/ws'
     enable_marketplace: true
     display_plugin_debug_url: 'ws://localhost:5401/plugin/debug/ws'
+    certification:
+        # Non-secret Ed25519 issuer key ring used to verify the SDK ZIP-comment
+        # certification envelope. Values are standard base64-encoded raw public
+        # keys; add keys during issuer rotation and remove retired IDs only after
+        # every affected archive has been upgraded.
+        trusted_public_keys: {}
     worker:
         # Instance-wide maximum for every plugin installation. Plugin
         # manifests cannot raise or override these limits.