ops(cloud): deploy exact production stack to jp09

.github/workflows/deploy-prod.yml

@@ -0,0 +1,77 @@
+name: Build and deploy production
+
+on:
+  push:
+    branches: [deploy/prod]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: langbot-production
+  cancel-in-progress: false
+
+env:
+  CORE_IMAGE: ${{ secrets.DOCKER_USERNAME }}/langbot
+  CLOUD_IMAGE: ${{ secrets.DOCKER_USERNAME }}/langbot-cloud-core
+  SPACE_REF: e1b261dac45e886efc667b1096a4ec493c6a6111
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    environment: production
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build exact Core image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: |
+            ${{ env.CORE_IMAGE }}:prod-${{ github.sha }}
+            ${{ env.CORE_IMAGE }}:deploy-prod
+          cache-from: type=gha,scope=core-prod
+          cache-to: type=gha,mode=max,scope=core-prod
+      - name: Checkout production Cloud adapter
+        uses: actions/checkout@v4
+        with:
+          repository: langbot-app/langbot-space
+          ref: ${{ env.SPACE_REF }}
+          path: .space
+      - name: Build exact Cloud Core image
+        uses: docker/build-push-action@v6
+        with:
+          context: .space
+          file: .space/Dockerfile.cloud
+          push: true
+          build-args: LANGBOT_CORE_IMAGE=${{ env.CORE_IMAGE }}:prod-${{ github.sha }}
+          tags: |
+            ${{ env.CLOUD_IMAGE }}:prod-${{ github.sha }}
+            ${{ env.CLOUD_IMAGE }}:deploy-prod
+          cache-from: type=gha,scope=cloud-core-prod
+          cache-to: type=gha,mode=max,scope=cloud-core-prod
+      - name: Configure SSH
+        env:
+          SSH_KEY: ${{ secrets.JP09_SSH_KEY }}
+          KNOWN_HOSTS: ${{ secrets.JP09_KNOWN_HOSTS }}
+        run: |
+          install -m 700 -d ~/.ssh
+          install -m 600 /dev/null ~/.ssh/id_ed25519
+          printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519
+          printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts
+      - name: Upload release manifest and deploy
+        env:
+          HOST: ${{ secrets.JP09_HOST }}
+          USER: ${{ secrets.JP09_USER }}
+          PORT: ${{ secrets.JP09_PORT }}
+        run: |
+          remote="$USER@$HOST"
+          ssh -p "$PORT" "$remote" 'install -d -m 700 /opt/langbot-cloud-prod'
+          scp -P "$PORT" deploy/prod/docker-compose.yml deploy/prod/deploy.sh "$remote:/opt/langbot-cloud-prod/"
+          ssh -p "$PORT" "$remote" "chmod 700 /opt/langbot-cloud-prod/deploy.sh && /opt/langbot-cloud-prod/deploy.sh prod-${GITHUB_SHA}"

deploy/prod/deploy.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+cd /opt/langbot-cloud-prod
+TAG=${1:?usage: deploy.sh prod-<40-char-sha>}
+[[ "$TAG" =~ ^prod-[0-9a-f]{40}$ ]] || { echo 'invalid immutable image tag' >&2; exit 2; }
+[[ -s .env ]] || { echo '/opt/langbot-cloud-prod/.env is missing' >&2; exit 3; }
+
+update_env() {
+  local key=$1 value=$2
+  python3 - "$key" "$value" <<'PY'
+from pathlib import Path
+import os
+import sys
+
+path = Path('.env')
+key, value = sys.argv[1:]
+lines = path.read_text().splitlines()
+updated = False
+for index, line in enumerate(lines):
+    if line.startswith(f'{key}='):
+        lines[index] = f'{key}={value}'
+        updated = True
+        break
+if not updated:
+    lines.append(f'{key}={value}')
+temporary = Path('.env.tmp')
+temporary.write_text('\n'.join(lines) + '\n')
+os.chmod(temporary, 0o600)
+temporary.replace(path)
+PY
+}
+update_env LANGBOT_IMAGE_TAG "$TAG"
+set -a
+. ./.env
+set +a
+
+docker compose pull postgres redis migrate plugin-runtime box core
+docker compose up -d postgres redis
+for _ in $(seq 1 60); do
+  if docker compose exec -T postgres pg_isready -U langbot_operator -d langbot >/dev/null 2>&1; then break; fi
+  sleep 2
+done
+docker compose exec -T postgres pg_isready -U langbot_operator -d langbot >/dev/null
+
+docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -U langbot_operator -d langbot \
+  -v runtime_password="$POSTGRES_RUNTIME_PASSWORD" <<'SQL'
+SELECT format('CREATE ROLE langbot_runtime LOGIN PASSWORD %L', :'runtime_password')
+WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'langbot_runtime')\gexec
+ALTER ROLE langbot_runtime PASSWORD :'runtime_password';
+GRANT CONNECT ON DATABASE langbot TO langbot_runtime;
+GRANT USAGE, CREATE ON SCHEMA public TO langbot_runtime;
+SQL
+
+docker compose --profile tools run --rm migrate
+
+docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -U langbot_operator -d langbot <<'SQL'
+GRANT USAGE ON SCHEMA public TO langbot_runtime;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO langbot_runtime;
+GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA public TO langbot_runtime;
+ALTER DEFAULT PRIVILEGES FOR ROLE langbot_operator IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO langbot_runtime;
+ALTER DEFAULT PRIVILEGES FOR ROLE langbot_operator IN SCHEMA public GRANT USAGE, SELECT, UPDATE ON SEQUENCES TO langbot_runtime;
+SQL
+
+docker compose up -d --remove-orphans plugin-runtime box core
+for _ in $(seq 1 90); do
+  if docker compose exec -T core python -c 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:5300/healthz", timeout=3)' >/dev/null 2>&1; then
+    docker compose ps
+    exit 0
+  fi
+  sleep 2
+done
+docker compose logs --tail=200 core plugin-runtime box >&2
+exit 1

deploy/prod/docker-compose.yml

@@ -0,0 +1,157 @@
+services:
+  postgres:
+    image: pgvector/pgvector:pg17
+    container_name: langbot-cloud-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: langbot
+      POSTGRES_USER: langbot_operator
+      POSTGRES_PASSWORD: ${POSTGRES_OPERATOR_PASSWORD}
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: [CMD-SHELL, "pg_isready -U langbot_operator -d langbot"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+    networks: [internal]
+
+  redis:
+    image: redis:7.4-alpine
+    container_name: langbot-cloud-redis
+    restart: unless-stopped
+    command: [redis-server, --appendonly, "yes", --requirepass, "${REDIS_PASSWORD}"]
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      test: [CMD-SHELL, "redis-cli -a '$${REDIS_PASSWORD}' ping | grep PONG"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    environment:
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    networks: [internal]
+
+  migrate:
+    image: rockchin/langbot-cloud-core:${LANGBOT_IMAGE_TAG}
+    profiles: [tools]
+    command: [uv, run, langbot, migrate, --cloud]
+    environment: &core-env
+      TZ: Asia/Shanghai
+      SYSTEM__INSTANCE_ID: ${CLOUD_V2_INSTANCE_UUID}
+      SYSTEM__EDITION: cloud
+      SYSTEM__RECOVERY_KEY: ${SYSTEM_RECOVERY_KEY}
+      SYSTEM__JWT__SECRET: ${JWT_SECRET}
+      SYSTEM__LIMITATION__MAX_BOTS: "2"
+      SYSTEM__LIMITATION__MAX_PIPELINES: "3"
+      SYSTEM__LIMITATION__MAX_EXTENSIONS: "3"
+      SYSTEM__LIMITATION__MAX_KNOWLEDGE_BASES: "2"
+      API__WEBHOOK_PREFIX: https://cloud.langbot.app
+      API__WEBUI_URL: https://cloud.langbot.app
+      WORKSPACE__INVITATIONS__PUBLIC_WEB_URL: https://cloud.langbot.app
+      DATABASE__USE: postgresql
+      DATABASE__POSTGRESQL__URL: postgresql+asyncpg://langbot_runtime:${POSTGRES_RUNTIME_PASSWORD}@postgres:5432/langbot
+      DATABASE__CLOUD_MIGRATION__OPERATOR_DSN_ENV: LANGBOT_CLOUD_MIGRATION_DSN
+      LANGBOT_CLOUD_MIGRATION_DSN: postgresql://langbot_operator:${POSTGRES_OPERATOR_PASSWORD}@postgres:5432/langbot
+      VDB__USE: pgvector
+      VDB__PGVECTOR__USE_BUSINESS_DATABASE: "true"
+      VDB__PGVECTOR__ALLOWED_DIMENSIONS: "384,512,768,1024,1536"
+      PLUGIN__ENABLE: "true"
+      PLUGIN__RUNTIME_WS_URL: ws://plugin-runtime:5400/control/ws
+      PLUGIN__DISPLAY_PLUGIN_DEBUG_URL: wss://cloud.langbot.app/plugin/debug/ws
+      PLUGIN__WORKER__MAX_CPUS: "0.25"
+      PLUGIN__WORKER__MAX_MEMORY_MB: "256"
+      PLUGIN__WORKER__MAX_PIDS: "128"
+      PLUGIN__WORKER__MAX_WORKERS: "16"
+      PLUGIN__WORKER__MAX_TOTAL_CPUS: "4.0"
+      PLUGIN__WORKER__MAX_TOTAL_MEMORY_MB: "4096"
+      PLUGIN__WORKER__REQUIRE_HARD_LIMITS: "true"
+      LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN: ${PLUGIN_RUNTIME_CONTROL_TOKEN}
+      BOX__ENABLED: "true"
+      BOX__BACKEND: nsjail
+      BOX__RUNTIME__ENDPOINT: ws://box:5410
+      BOX__ADMISSION__REQUIRED: "true"
+      BOX__ADMISSION__LOGICAL_SESSION_ID: global
+      BOX__ADMISSION__REQUIRED_BACKEND: nsjail
+      BOX__ADMISSION__MAX_SESSIONS: "1"
+      BOX__ADMISSION__MAX_MANAGED_PROCESSES: "0"
+      BOX__ADMISSION__CPUS: "0.25"
+      BOX__ADMISSION__MEMORY_MB: "256"
+      BOX__ADMISSION__WORKSPACE_QUOTA_MB: "256"
+      BOX__LOCAL__HOST_ROOT: /app/data/box
+      BOX__LOCAL__ALLOWED_MOUNT_ROOTS: /app/data/box
+      LANGBOT_BOX_CONTROL_TOKEN: ${BOX_CONTROL_TOKEN}
+      MCP__STDIO__ENABLED: "false"
+      LANGBOT_SPACE_CONTROL_PLANE_URL: https://space.langbot.app
+      LANGBOT_SPACE_CONTROL_PLANE_TOKEN: ${CLOUD_V2_CONTROL_PLANE_TOKEN}
+      LANGBOT_SPACE_CONTROL_PLANE_PUBLIC_KEY: ${CLOUD_V2_MANIFEST_PUBLIC_KEY}
+      LANGBOT_SPACE_CONTROL_PLANE_KEY_ID: ${CLOUD_V2_MANIFEST_KEY_ID}
+      SPACE__URL: https://space.langbot.app
+    depends_on:
+      postgres: {condition: service_healthy}
+    networks: [internal]
+
+  plugin-runtime:
+    image: rockchin/langbot:${LANGBOT_IMAGE_TAG}
+    container_name: langbot-cloud-plugin-runtime
+    restart: unless-stopped
+    command: [uv, run, python, -m, langbot_plugin.cli.__init__, rt]
+    environment:
+      LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN: ${PLUGIN_RUNTIME_CONTROL_TOKEN}
+    volumes:
+      - plugin-data:/app/data
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    cgroup: host
+    privileged: true
+    expose: ["5400"]
+    networks: [internal]
+
+  box:
+    image: rockchin/langbot:${LANGBOT_IMAGE_TAG}
+    container_name: langbot-cloud-box
+    restart: unless-stopped
+    command: [uv, run, python, -m, langbot_box.cli, runtime, --host, 0.0.0.0, --port, "5410"]
+    environment:
+      LANGBOT_BOX_CONTROL_TOKEN: ${BOX_CONTROL_TOKEN}
+      LANGBOT_BOX_ROOT: /app/data/box
+    volumes:
+      - box-data:/app/data/box
+      - /sys/fs/cgroup:/sys/fs/cgroup:rw
+    cgroup: host
+    privileged: true
+    expose: ["5410"]
+    networks: [internal]
+
+  core:
+    image: rockchin/langbot-cloud-core:${LANGBOT_IMAGE_TAG}
+    container_name: langbot-cloud-core
+    restart: unless-stopped
+    environment: *core-env
+    volumes:
+      - core-data:/app/data
+      - box-data:/app/data/box
+    depends_on:
+      postgres: {condition: service_healthy}
+      redis: {condition: service_healthy}
+      plugin-runtime: {condition: service_started}
+      box: {condition: service_started}
+    expose: ["5300"]
+    healthcheck:
+      test: [CMD-SHELL, "python -c 'import urllib.request; urllib.request.urlopen(\"http://127.0.0.1:5300/healthz\", timeout=3)'" ]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+      start_period: 30s
+    networks: [internal, shared-network]
+
+networks:
+  internal:
+  shared-network:
+    external: true
+
+volumes:
+  postgres-data:
+  redis-data:
+  plugin-data:
+  box-data:
+  core-data: