Harden Testnet BENS deployment startup

.github/scripts/tests/test_validate_testnet_bens.py

@@ -1,17 +1,20 @@
 import importlib.util
+import http.server
 import json
 import os
 import re
 import shutil
 import subprocess
 import tempfile
+import threading
 import unittest
 from pathlib import Path
 
 
 ROOT = Path(__file__).resolve().parents[3]
 SCRIPT_PATH = ROOT / "scripts" / "validate-testnet-bens.py"
 RPC_RETRY_SCRIPT = ROOT / ".github" / "scripts" / "retry-testnet-rpc.sh"
+SUBGRAPH_DEPLOY_SCRIPT = ROOT / "docker-compose" / "bens" / "deploy-subgraph.sh"
 
 
 def load_module():
@@ -85,15 +88,194 @@ def test_deployer_invokes_graph_cli_without_executable_shims(self):
         compose = (
             ROOT / "docker-compose" / "docker-compose-testnet.yml"
         ).read_text(encoding="utf-8")
-        graph_cli = "node node_modules/@graphprotocol/graph-cli/bin/run"
-
-        self.assertIn(f"{graph_cli} codegen --output-dir src/types/", compose)
-        self.assertIn(f"{graph_cli} build", compose)
-        self.assertIn(f"{graph_cli} create dos-names", compose)
-        self.assertIn(f"{graph_cli} deploy dos-names", compose)
-        self.assertNotIn("npm run codegen", compose)
-        self.assertNotIn("npm run build", compose)
-        self.assertNotIn("npx graph", compose)
+        deploy_script = SUBGRAPH_DEPLOY_SCRIPT.read_text(encoding="utf-8")
+
+        self.assertIn("exec /bin/sh /runtime/deploy-subgraph.sh", compose)
+        self.assertIn("run_graph_cli codegen --output-dir src/types/", deploy_script)
+        self.assertIn("run_graph_cli build", deploy_script)
+        self.assertIn("run_graph_cli create dos-names", deploy_script)
+        self.assertIn("run_graph_cli deploy dos-names", deploy_script)
+        self.assertNotIn("npm run codegen", deploy_script)
+        self.assertNotIn("npm run build", deploy_script)
+        self.assertNotIn("npx graph", deploy_script)
+
+    def test_deployer_forwards_the_unique_subgraph_version(self):
+        compose = (
+            ROOT / "docker-compose" / "docker-compose-testnet.yml"
+        ).read_text(encoding="utf-8")
+        deployer = compose.split("  bens-deployer:", 1)[1].split("\n  backend:", 1)[0]
+
+        self.assertIn(
+            "BENS_SUBGRAPH_VERSION: ${BENS_SUBGRAPH_VERSION:-testnet}", deployer
+        )
+        self.assertIn(
+            'BENS_SUBGRAPH_VERSION="github-${DEPLOY_ID}"',
+            (ROOT / ".github" / "workflows" / "deploy-config.yml").read_text(
+                encoding="utf-8"
+            ),
+        )
+
+    def test_subgraph_retry_uses_manifest_cid_and_rejects_unready_states(self):
+        manifest_cid = "Qm" + "B" * 44
+        asset_cid = "Qm" + "A" * 44
+        old_cid = "Qm" + "C" * 44
+        ready = {
+            "data": {
+                "_meta": {
+                    "deployment": manifest_cid,
+                    "hasIndexingErrors": False,
+                }
+            }
+        }
+        rejected_states = [
+            {
+                "data": {
+                    "_meta": {
+                        "deployment": old_cid,
+                        "hasIndexingErrors": False,
+                    }
+                }
+            },
+            {
+                "errors": [{"message": "indexing unavailable"}],
+                "data": {
+                    "_meta": {
+                        "deployment": manifest_cid,
+                        "hasIndexingErrors": False,
+                    }
+                },
+            },
+            {"data": {}},
+            {
+                "data": {
+                    "_meta": {
+                        "deployment": manifest_cid,
+                        "hasIndexingErrors": True,
+                    }
+                }
+            },
+        ]
+
+        for rejected in rejected_states:
+            with self.subTest(rejected=rejected):
+                result, calls = self._run_subgraph_deployer(
+                    [rejected, ready], manifest_cid, asset_cid
+                )
+                self.assertEqual(0, result.returncode, result.stderr)
+                deploy_calls = [call for call in calls if call.startswith("deploy ")]
+                self.assertEqual(2, len(deploy_calls), calls)
+                self.assertNotIn("--ipfs-hash", deploy_calls[0])
+                self.assertIn(f"--ipfs-hash {manifest_cid}", deploy_calls[1])
+                self.assertNotIn(asset_cid, deploy_calls[1])
+                self.assertEqual(1, calls.count("build"), calls)
+
+        result, calls = self._run_subgraph_deployer(
+            [rejected_states[0]] * 3, manifest_cid, asset_cid
+        )
+        self.assertNotEqual(0, result.returncode)
+        self.assertEqual(3, len([call for call in calls if call.startswith("deploy ")]))
+        self.assertEqual(1, calls.count("build"), calls)
+
+    def _run_subgraph_deployer(self, responses, manifest_cid, asset_cid):
+        class ResponseHandler(http.server.BaseHTTPRequestHandler):
+            queue = list(responses)
+
+            def do_POST(self):
+                length = int(self.headers.get("content-length", "0"))
+                self.rfile.read(length)
+                body = self.queue.pop(0) if self.queue else responses[-1]
+                payload = json.dumps(body).encode("utf-8")
+                self.send_response(200)
+                self.send_header("content-type", "application/json")
+                self.send_header("content-length", str(len(payload)))
+                self.end_headers()
+                self.wfile.write(payload)
+
+            def log_message(self, _format, *_args):
+                return
+
+        bash = "bash"
+        if os.name == "nt":
+            bash = r"C:\Program Files\Git\bin\bash.exe"
+
+        server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), ResponseHandler)
+        thread = threading.Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                source = root / "source"
+                work = root / "work"
+                source.mkdir()
+                (source / "package.json").write_text("{}", encoding="utf-8")
+                calls_path = root / "graph-calls"
+                graph_runner = root / "fake-graph-cli.sh"
+                graph_runner.write_text(
+                    "#!/bin/sh\n"
+                    'printf "%s\\n" "$*" >> "$DOSCAN_GRAPH_CALLS"\n'
+                    'if [ "$1" != "deploy" ]; then exit 0; fi\n'
+                    'case " $* " in\n'
+                    '  *" --ipfs-hash "*) exit 1 ;;\n'
+                    "esac\n"
+                    f'printf "%s\\n" "Add file to IPFS .. {asset_cid}"\n'
+                    f'printf "%s\\n" "Build completed: {manifest_cid}"\n'
+                    'printf "%s\\n" "HTTP error deploying the subgraph ECONNRESET"\n'
+                    "exit 1\n",
+                    encoding="utf-8",
+                )
+                graph_runner.chmod(0o755)
+                npm_runner = root / "fake-npm.sh"
+                npm_runner.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+                npm_runner.chmod(0o755)
+                environment = os.environ.copy()
+                environment.update(
+                    {
+                        "DOSCAN_GRAPH_ADMIN_URL": "http://127.0.0.1:1",
+                        "DOSCAN_GRAPH_QUERY_URL": (
+                            f"http://127.0.0.1:{server.server_port}/graphql"
+                        ),
+                        "DOSCAN_GRAPH_IPFS_URL": "http://127.0.0.1:2",
+                        "DOSCAN_GRAPH_CLI_RUNNER": graph_runner.as_posix(),
+                        "DOSCAN_GRAPH_CALLS": calls_path.as_posix(),
+                        "DOSCAN_NPM_RUNNER": npm_runner.as_posix(),
+                        "DOSCAN_SUBGRAPH_SOURCE_DIR": source.as_posix(),
+                        "DOSCAN_SUBGRAPH_WORK_DIR": work.as_posix(),
+                        "DOSCAN_SUBGRAPH_DEPLOY_LOG": (root / "deploy.log").as_posix(),
+                        "DOSCAN_SUBGRAPH_READINESS_ATTEMPTS": "1",
+                        "DOSCAN_SUBGRAPH_RETRY_DELAY_SECONDS": "0",
+                    }
+                )
+                result = subprocess.run(
+                    [bash, SUBGRAPH_DEPLOY_SCRIPT.as_posix()],
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                    env=environment,
+                )
+                calls = (
+                    calls_path.read_text(encoding="utf-8").splitlines()
+                    if calls_path.exists()
+                    else []
+                )
+                return result, calls
+        finally:
+            server.shutdown()
+            server.server_close()
+            thread.join(timeout=5)
+
+    def test_bens_runtime_config_is_readable_by_the_non_root_image_user(self):
+        workflow = (
+            ROOT / ".github" / "workflows" / "deploy-config.yml"
+        ).read_text(encoding="utf-8")
+        self.assertIn('sudo rm -rf "${DEPLOY_PATH}/bens"', workflow)
+        install_block = workflow.rsplit(
+            'sudo rm -rf "${DEPLOY_PATH}/bens"', 1
+        )[1].split('cd "${DEPLOY_PATH}"', 1)[0]
+
+        self.assertIn('sudo chmod 0755 "${DEPLOY_PATH}/bens"', install_block)
+        self.assertIn(
+            'sudo chmod 0644 "${DEPLOY_PATH}/bens/config.json"', install_block
+        )
 
     def test_caddy_validation_retries_the_pinned_image_pull(self):
         workflow = (

.github/workflows/deploy-config.yml

@@ -1218,6 +1218,8 @@ jobs:
               sudo install -m 0644 "${SRC}/docker-compose/envs/common-visualizer.env" "${DEPLOY_PATH}/envs/common-visualizer.env"
               sudo rm -rf "${DEPLOY_PATH}/bens"
               sudo cp -a "${SRC}/docker-compose/bens" "${DEPLOY_PATH}/bens"
+              sudo chmod 0755 "${DEPLOY_PATH}/bens"
+              sudo chmod 0644 "${DEPLOY_PATH}/bens/config.json"
 
               cd "${DEPLOY_PATH}"
               sudo env DOSCAN_BLOCKSCOUT_SECRETS_ENV="${SECRETS_ENV}" docker compose config -q

docker-compose/bens/deploy-subgraph.sh

@@ -0,0 +1,98 @@
+#!/bin/sh
+
+set -eu
+
+GRAPH_ADMIN_URL="${DOSCAN_GRAPH_ADMIN_URL:-http://bens-graph-node:8020}"
+GRAPH_QUERY_URL="${DOSCAN_GRAPH_QUERY_URL:-http://bens-graph-node:8000/subgraphs/name/dos-names}"
+GRAPH_IPFS_URL="${DOSCAN_GRAPH_IPFS_URL:-http://bens-ipfs:5001}"
+READINESS_ATTEMPTS="${DOSCAN_SUBGRAPH_READINESS_ATTEMPTS:-6}"
+RETRY_DELAY_SECONDS="${DOSCAN_SUBGRAPH_RETRY_DELAY_SECONDS:-5}"
+SOURCE_DIR="${DOSCAN_SUBGRAPH_SOURCE_DIR:-/source}"
+WORK_DIR="${DOSCAN_SUBGRAPH_WORK_DIR:-/work}"
+deploy_log="${DOSCAN_SUBGRAPH_DEPLOY_LOG:-/tmp/dos-names-deploy.log}"
+
+run_graph_cli() {
+  if [ -n "${DOSCAN_GRAPH_CLI_RUNNER:-}" ]; then
+    "${DOSCAN_GRAPH_CLI_RUNNER}" "$@"
+  else
+    node node_modules/@graphprotocol/graph-cli/bin/run "$@"
+  fi
+}
+
+run_npm() {
+  if [ -n "${DOSCAN_NPM_RUNNER:-}" ]; then
+    "${DOSCAN_NPM_RUNNER}" "$@"
+  else
+    npm "$@"
+  fi
+}
+
+subgraph_ready() {
+  node -e 'const expectedDeployment = process.argv[1]; const queryUrl = process.argv[2]; fetch(queryUrl, {method: "POST", headers: {"content-type": "application/json"}, body: JSON.stringify({query: "{ _meta { deployment hasIndexingErrors } }"})}).then(async (response) => { const body = await response.json(); if (!response.ok || body.errors || !body.data || !body.data._meta || body.data._meta.deployment !== expectedDeployment || body.data._meta.hasIndexingErrors !== false) process.exit(1); }).catch(() => process.exit(1));' \
+    "${subgraph_ipfs_hash}" "${GRAPH_QUERY_URL}"
+}
+
+wait_for_subgraph() {
+  readiness_attempt=1
+  while [ "${readiness_attempt}" -le "${READINESS_ATTEMPTS}" ]; do
+    if subgraph_ready; then
+      return 0
+    fi
+    sleep "${RETRY_DELAY_SECONDS}"
+    readiness_attempt=$((readiness_attempt + 1))
+  done
+  return 1
+}
+
+mkdir -p "${WORK_DIR}"
+cp -a "${SOURCE_DIR}/." "${WORK_DIR}/"
+cd "${WORK_DIR}"
+run_npm ci
+run_graph_cli codegen --output-dir src/types/
+run_graph_cli build
+run_graph_cli create dos-names --node "${GRAPH_ADMIN_URL}" || true
+
+set +e
+run_graph_cli deploy dos-names \
+  --ipfs "${GRAPH_IPFS_URL}" \
+  --node "${GRAPH_ADMIN_URL}" \
+  --version-label "${BENS_SUBGRAPH_VERSION:-testnet}" \
+  >"${deploy_log}" 2>&1
+deploy_rc="${?}"
+set -e
+cat "${deploy_log}"
+
+subgraph_ipfs_hash="$(grep -Eo '(Build completed|Subgraph IPFS hash): Qm[1-9A-HJ-NP-Za-km-z]{44}' "${deploy_log}" | grep -Eo 'Qm[1-9A-HJ-NP-Za-km-z]{44}' | tail -n 1)"
+if [ -z "${subgraph_ipfs_hash}" ]; then
+  echo "Graph CLI did not report the uploaded subgraph IPFS hash" >&2
+  exit 1
+fi
+
+subgraph_deployed=0
+if wait_for_subgraph; then
+  subgraph_deployed=1
+else
+  echo "Initial Graph deploy exited ${deploy_rc} without an active subgraph; retrying the uploaded IPFS hash" >&2
+  for deploy_attempt in 2 3; do
+    set +e
+    run_graph_cli deploy dos-names \
+      --ipfs "${GRAPH_IPFS_URL}" \
+      --ipfs-hash "${subgraph_ipfs_hash}" \
+      --node "${GRAPH_ADMIN_URL}" \
+      --version-label "${BENS_SUBGRAPH_VERSION:-testnet}" \
+      >"${deploy_log}" 2>&1
+    deploy_rc="${?}"
+    set -e
+    cat "${deploy_log}"
+    if wait_for_subgraph; then
+      subgraph_deployed=1
+      break
+    fi
+    echo "Graph deploy attempt ${deploy_attempt} exited ${deploy_rc} without an active subgraph" >&2
+  done
+fi
+
+if [ "${subgraph_deployed}" -ne 1 ]; then
+  echo "DOS Names subgraph was not activated after three deploy attempts" >&2
+  exit 1
+fi

docker-compose/docker-compose-testnet.yml

@@ -120,6 +120,8 @@ services:
   bens-deployer:
     image: node:22.18.0-alpine3.22@sha256:1b2479dd35a99687d6638f5976fd235e26c5b37e8122f786fcd5fe231d63de5b
     profiles: ["bens-deploy"]
+    environment:
+      BENS_SUBGRAPH_VERSION: ${BENS_SUBGRAPH_VERSION:-testnet}
     depends_on:
       bens-graph-node:
         condition: service_started
@@ -128,22 +130,13 @@ services:
     working_dir: /work
     volumes:
       - ./bens/subgraph:/source:ro
+      - ./bens/deploy-subgraph.sh:/runtime/deploy-subgraph.sh:ro
     tmpfs:
       - /work
     command:
       - /bin/sh
       - -ec
-      - |
-          cp -a /source/. /work/
-          npm ci
-          node node_modules/@graphprotocol/graph-cli/bin/run codegen --output-dir src/types/
-          node node_modules/@graphprotocol/graph-cli/bin/run build
-          node node_modules/@graphprotocol/graph-cli/bin/run create dos-names \
-            --node http://bens-graph-node:8020 || true
-          node node_modules/@graphprotocol/graph-cli/bin/run deploy dos-names \
-            --ipfs http://bens-ipfs:5001 \
-            --node http://bens-graph-node:8020 \
-            --version-label "$${BENS_SUBGRAPH_VERSION:-testnet}"
+      - exec /bin/sh /runtime/deploy-subgraph.sh
 
   backend:
     image: ghcr.io/dos/doscan:11.2.6.commit.99eb6e8a@sha256:60b6655e9c02028a8ce6a6eb5a7a1db6d4c23ab616e68a26319e00787089e084