feat(checkpoint): remote-checkpoint indexing → load → deploy pipeline

.changeset/ensdb-sdk-schema-metadata-helpers.md

@@ -0,0 +1,5 @@
+---
+"@ensnode/ensdb-sdk": minor
+---
+
+Add public schema and ENSNode metadata helpers. `EnsDbReader` now exposes `schemaExists(schemaName)` and a public, typed `getEnsNodeMetadata({ key })` that returns the full `{ key, value }` record. `EnsDbWriter` now exposes `dropSchema(schemaName)`, `renameSchema(from, to)`, and a public `writeEnsNodeMetadata(metadata)` that re-keys a `SerializedEnsNodeMetadata` record to the writer's ENSIndexer schema. `SerializedEnsNodeMetadata` is now re-exported from the package entrypoint.

.github/actions/deploy_railway_services/action.yml

@@ -0,0 +1,99 @@
+name: Deploy Railway services
+description: >-
+  Update + redeploy a set of Railway service instances to one Docker image, in series.
+  Services are processed in the order given; ENSRainbow-type services should be listed first.
+
+inputs:
+  railway_token:
+    description: "Railway API token (Authorization: Bearer)"
+    required: true
+
+  environment_id:
+    description: "Railway environment id the service instances belong to"
+    required: true
+
+  image:
+    description: "Full ghcr image reference including tag (e.g. ghcr.io/namehash/ensnode/ensindexer:1.2.3)"
+    required: true
+
+  service_ids:
+    description: "Newline-separated Railway service ids, applied IN ORDER (ENSRainbow-type first)"
+    required: true
+
+  project_id:
+    description: "Railway project id (optional; unused by the update/deploy mutations but accepted for parity)"
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Update + redeploy service instances in order
+      shell: bash
+      env:
+        RAILWAY_TOKEN: ${{ inputs.railway_token }}
+        ENVIRONMENT_ID: ${{ inputs.environment_id }}
+        DOCKER_IMAGE: ${{ inputs.image }}
+        SERVICE_IDS: ${{ inputs.service_ids }}
+      run: |
+        set -euo pipefail
+
+        # Railway's GraphQL endpoint returns HTTP 200 even when a mutation fails (the error lives in
+        # the response body's top-level `errors` array), so --fail alone misses it. Fail loudly when
+        # the body carries a GraphQL error.
+        fail_on_graphql_error() {
+          local resp=$1
+          if printf '%s' "$resp" | jq -e '.errors' >/dev/null 2>&1; then
+            echo "Railway GraphQL error: $resp" >&2
+            exit 1
+          fi
+        }
+
+        update_service_image() {
+          local environment_id=$1
+          local service_id=$2
+          local docker_image=$3
+          echo "Updating service $service_id"
+          local resp
+          resp="$(curl --request POST \
+            --silent \
+            --fail \
+            --show-error \
+            --url https://backboard.railway.app/graphql/v2 \
+            --header 'Authorization: Bearer '"$RAILWAY_TOKEN" \
+            --header 'Content-Type: application/json' \
+            --data '{
+              "query": "mutation serviceInstanceUpdate($environmentId: String, $input: ServiceInstanceUpdateInput!, $serviceId: String!) { serviceInstanceUpdate(environmentId: $environmentId input: $input serviceId: $serviceId) }",
+              "variables": {
+                "environmentId": "'"$environment_id"'",
+                "input": {"source": {"image": "'"$docker_image"'"}},
+                "serviceId": "'"$service_id"'"
+              }
+            }')"
+          fail_on_graphql_error "$resp"
+          echo "Finished updating $service_id"
+        }
+
+        redeploy_service() {
+          local environment_id=$1
+          local service_id=$2
+          echo "Redeploying $service_id"
+          local resp
+          resp="$(curl --request POST \
+            --silent \
+            --fail \
+            --show-error \
+            --url https://backboard.railway.app/graphql/v2 \
+            --header 'Authorization: Bearer '"$RAILWAY_TOKEN" \
+            --header 'Content-Type: application/json' \
+            --data "{\"query\":\"mutation serviceInstanceDeploy(\$serviceId: String!, \$environmentId: String!) { serviceInstanceDeploy(serviceId: \$serviceId, environmentId: \$environmentId) }\",\"variables\":{\"environmentId\":\"${environment_id}\",\"serviceId\":\"${service_id}\"}}")"
+          fail_on_graphql_error "$resp"
+          echo "Finished redeploying $service_id"
+        }
+
+        # Process each service id in order: update its image, then redeploy it.
+        while IFS= read -r service_id; do
+          [ -n "$service_id" ] || continue
+          update_service_image "$ENVIRONMENT_ID" "$service_id" "$DOCKER_IMAGE"
+          redeploy_service "$ENVIRONMENT_ID" "$service_id"
+        done <<< "$SERVICE_IDS"

.github/workflows/checkpoint.yml

@@ -0,0 +1,89 @@
+name: "[next] Checkpoint: produce a point-in-time dev checkpoint on Cherry"
+
+# Produce a resumable, sha-identified checkpoint of a specific commit/config at a specific chain-time,
+# uploaded to R2 for a developer to restore locally (`ensdb-cli load`) for debugging. No deploy, no
+# target DB, no images. Manual trigger only.
+
+on:
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: "Commit to index"
+        required: true
+        type: string
+      config:
+        description: "Named config to index"
+        required: true
+        type: choice
+        options:
+          - alpha
+          - mainnet
+      timestamp:
+        description: "Chain-time to index up to (unix seconds)"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: checkpoint-dev-${{ inputs.config }}-${{ inputs.commit }}-${{ inputs.timestamp }}
+  cancel-in-progress: false
+
+jobs:
+  checkpoint:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    timeout-minutes: 600
+    env:
+      CHERRY_API_TOKEN: ${{ secrets.CHERRY_API_TOKEN }}
+      CHERRY_PROJECT_ID: ${{ secrets.CHERRY_PROJECT_ID }}
+      CHERRY_SSH_KEY_ID: ${{ secrets.CHERRY_SSH_KEY_ID }}
+      CHERRY_SSH_KEY: ${{ secrets.CHERRY_SSH_KEY }}
+      R2_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+      R2_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+      ALCHEMY_API_KEY: ${{ secrets.ALCHEMY_API_KEY }}
+      R2_CHECKPOINTS_BUCKET: ensindexer-checkpoints
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+
+      - name: Write SSH key
+        run: |
+          install -m600 /dev/null /tmp/cherry_key
+          printf '%s\n' "$CHERRY_SSH_KEY" > /tmp/cherry_key
+          echo "SSH_KEY=/tmp/cherry_key" >> "$GITHUB_ENV"
+
+      - name: Index config at commit to the requested chain-time
+        env:
+          CONFIGS: ${{ inputs.config }}
+          SHA: ${{ inputs.commit }}
+          MODE: end-block
+          TIMESTAMP: ${{ inputs.timestamp }}
+        run: bash packages/ensindexer-checkpoint/scripts/checkpoint.sh
+
+      - name: Report checkpoint R2 location
+        env:
+          CONFIG: ${{ inputs.config }}
+          COMMIT: ${{ inputs.commit }}
+          TIMESTAMP: ${{ inputs.timestamp }}
+        run: |
+          OBJ="${CONFIG}-${COMMIT}-t${TIMESTAMP}.dump"
+          {
+            echo "### Checkpoint ready"
+            echo ""
+            echo "- R2: \`${R2_CHECKPOINTS_BUCKET}/checkpoints/${OBJ}\`"
+            echo "- metadata: \`${R2_CHECKPOINTS_BUCKET}/checkpoints/${OBJ}.metadata.json\`"
+            echo ""
+            echo "Restore locally (the metadata sidecar must sit next to the dump; load reads it automatically):"
+            echo '```'
+            echo "rclone copy r2:${R2_CHECKPOINTS_BUCKET}/checkpoints/${OBJ} ."
+            echo "rclone copy r2:${R2_CHECKPOINTS_BUCKET}/checkpoints/${OBJ}.metadata.json ."
+            echo "ensdb-cli load ${OBJ} --into \$LOCAL_ENSDB_URL --schema <your-ensIndexerSchemaName>"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Tear down the Cherry box (always)
+        if: always()
+        run: bash packages/ensindexer-checkpoint/scripts/cherry-down.sh || true

.github/workflows/checkpoint_gc.yml

@@ -0,0 +1,54 @@
+name: "[next] Garbage Collector: terminate orphaned ENSIndexer checkpoint Cherry boxes"
+
+# Safety net (the third box-teardown guard, alongside the workflow's always() teardown and the on-box
+# self-destruct watchdog): terminate any Cherry server named `ensindexer-checkpoint-*` older than the
+# TTL, in case a runner died before tearing its box down. Only ever touches checkpoint boxes — never
+# Railway/production.
+
+on:
+  schedule:
+    - cron: "17 * * * *" # hourly
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+concurrency:
+  group: checkpoint-gc
+  cancel-in-progress: false
+
+jobs:
+  collect:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    timeout-minutes: 15
+    env:
+      CHERRY_API_TOKEN: ${{ secrets.CHERRY_API_TOKEN }}
+      CHERRY_PROJECT_ID: ${{ secrets.CHERRY_PROJECT_ID }}
+      BOX_HOSTNAME_PREFIX: ensindexer-checkpoint
+      MAX_AGE_HOURS: "14"
+    steps:
+      - name: Terminate stale checkpoint boxes
+        run: |
+          set -euo pipefail
+          API=https://api.cherryservers.com/v1
+          now=$(date +%s)
+          servers="$(curl -fsS -H "Authorization: Bearer $CHERRY_API_TOKEN" "$API/projects/$CHERRY_PROJECT_ID/servers")"
+          echo "$servers" | jq -c '.[]?' | while read -r s; do
+            id="$(echo "$s" | jq -r '.id')"
+            host="$(echo "$s" | jq -r '.hostname // ""')"
+            # Cherry response shape varies; try common created-time fields.
+            created="$(echo "$s" | jq -r '.created // .created_at // .deployed_at // ""')"
+            case "$host" in
+              "$BOX_HOSTNAME_PREFIX"*) ;;
+              *) continue ;;
+            esac
+            cts="$(date -d "$created" +%s 2>/dev/null || echo 0)"
+            [ "$cts" -gt 0 ] || { echo "skip $id ($host): unparseable created '$created'"; continue; }
+            age=$(( (now - cts) / 3600 ))
+            if [ "$age" -ge "$MAX_AGE_HOURS" ]; then
+              echo "collecting server $id ($host, ${age}h old)"
+              curl -fsS -H "Authorization: Bearer $CHERRY_API_TOKEN" -X DELETE "$API/servers/$id" || true
+            else
+              echo "keep server $id ($host, ${age}h old)"
+            fi
+          done

.github/workflows/deploy_ensapi_hotfix.yml

@@ -0,0 +1,96 @@
+name: "[next] Deploy: ENSApi-only hotfix to Blue/Green Environment on Railway platform"
+
+# ENSApi-only hotfixes need no reindex: merge the hotfix, let GitHub build the image, then bump only
+# the ENSApi service images on Railway (no schema change, no indexer/rainbow/admin redeploy). This is
+# the blue/green deploy restricted to the five ENSApi instances. Yellow (Render) is intentionally
+# out of scope.
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: "Target environment (green or blue)"
+        required: true
+        type: choice
+        options:
+          - green
+          - blue
+      tag:
+        description: "ENSApi Docker Image Tag to deploy"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy-ensapi-hotfix-${{ inputs.target }}
+  cancel-in-progress: false
+
+jobs:
+  deploy-ensapi:
+    runs-on: blacksmith-4vcpu-ubuntu-2204
+    env:
+      TARGET_ENVIRONMENT: ${{ inputs.target }}
+      TAG: ${{ inputs.tag }}
+      ENSAPI_DOCKER_IMAGE: "ghcr.io/namehash/ensnode/ensapi:${{ inputs.tag }}"
+      RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
+      RAILWAY_PROJECT_ID: ${{ secrets.RAILWAY_PROJECT_ID }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+
+      - name: Check if ENSApi Docker image exists
+        run: |
+          docker manifest inspect ${{ env.ENSAPI_DOCKER_IMAGE }} || { echo "Given docker image does not exist: ${{ env.ENSAPI_DOCKER_IMAGE }}"; exit 1; }
+
+      - name: Resolve target environment + ENSApi service ids
+        run: |
+          case "$TARGET_ENVIRONMENT" in
+            "green")
+              echo "RAILWAY_ENVIRONMENT_ID=${{ secrets.GREEN_RAILWAY_ENVIRONMENT_ID }}" >> "$GITHUB_ENV"
+              echo "ALPHA_API_SVC_ID=${{ secrets.GREEN_ALPHA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "MAINNET_API_SVC_ID=${{ secrets.GREEN_MAINNET_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "ALPHA_SEPOLIA_API_SVC_ID=${{ secrets.GREEN_ALPHA_SEPOLIA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "V2_SEPOLIA_API_SVC_ID=${{ secrets.GREEN_V2_SEPOLIA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "SEPOLIA_API_SVC_ID=${{ secrets.GREEN_SEPOLIA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "SLACK_TITLE=':large_green_circle: GREEN ENSApi hotfix - '"${{ env.TAG }} >> "$GITHUB_ENV"
+              ;;
+            "blue")
+              echo "RAILWAY_ENVIRONMENT_ID=${{ secrets.BLUE_RAILWAY_ENVIRONMENT_ID }}" >> "$GITHUB_ENV"
+              echo "ALPHA_API_SVC_ID=${{ secrets.BLUE_ALPHA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "MAINNET_API_SVC_ID=${{ secrets.BLUE_MAINNET_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "ALPHA_SEPOLIA_API_SVC_ID=${{ secrets.BLUE_ALPHA_SEPOLIA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "V2_SEPOLIA_API_SVC_ID=${{ secrets.BLUE_V2_SEPOLIA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "SEPOLIA_API_SVC_ID=${{ secrets.BLUE_SEPOLIA_API_SVC_ID }}" >> "$GITHUB_ENV"
+              echo "SLACK_TITLE=':large_blue_circle: BLUE ENSApi hotfix - '"${{ env.TAG }} >> "$GITHUB_ENV"
+              ;;
+            *)
+              echo "Environment not recognized, skipping workflow"
+              exit 1
+              ;;
+          esac
+
+      - name: Update + redeploy the five ENSApi instances
+        uses: ./.github/actions/deploy_railway_services
+        with:
+          railway_token: ${{ env.RAILWAY_TOKEN }}
+          environment_id: ${{ env.RAILWAY_ENVIRONMENT_ID }}
+          image: ${{ env.ENSAPI_DOCKER_IMAGE }}
+          project_id: ${{ env.RAILWAY_PROJECT_ID }}
+          # No ENSRainbow services here, so ordering is cosmetic; keep the original order.
+          service_ids: |
+            ${{ env.ALPHA_API_SVC_ID }}
+            ${{ env.MAINNET_API_SVC_ID }}
+            ${{ env.ALPHA_SEPOLIA_API_SVC_ID }}
+            ${{ env.V2_SEPOLIA_API_SVC_ID }}
+            ${{ env.SEPOLIA_API_SVC_ID }}
+
+      - uses: ./.github/actions/send_slack_notification
+        with:
+          slack_webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          slack_title: ${{ env.SLACK_TITLE }}
+          slack_message: "✅ ENSApi hotfix deploy completed"