[#2353] Docker container does not shutdown cleanly on SIGTERM

nix/docker/README.md

@@ -0,0 +1,101 @@
+
+## Build Cardano Node + Image
+
+https://docs.cardano.org/projects/cardano-node/en/latest/getting-started/building-the-node-using-nix.html
+
+```
+# Build + Install the cardano node
+nix-build -A scripts.mainnet.node -o ~/bin/cardano-node
+
+# Build + Install the cardano Docker image
+docker load -i $(nix-build -A dockerImage --no-out-link) \
+  && GITHASH=`git log -n1 --pretty='%H'` \
+  && docker tag inputoutput/cardano-node:$GITHASH inputoutput/cardano-node:dev \
+  && docker rmi inputoutput/cardano-node:$GITHASH
+
+GITTAG=`git describe --exact-match --tags $GITHASH`
+if [ $? -eq 0 ]; then
+  echo "Current tag: $GITTAG"
+  docker tag inputoutput/cardano-node:dev inputoutput/cardano-node:$GITTAG
+fi
+
+# Bash into the node to look around
+docker run --rm -it --entrypoint=bash \
+  -v node-data:/opt/cardano/data \
+  inputoutput/cardano-node:dev
+
+cardano-node run \
+  --config /opt/cardano/config/mainnet-config.json \
+  --topology /opt/cardano/config/mainnet-topology.json \
+  --socket-path /opt/cardano/ipc/socket \
+  --database-path /opt/cardano/data \
+  --host-addr 0.0.0.0 \
+  --port 3001
+```
+
+## Manual Testing
+
+1. Run -e NETWORK=mainnet and check graceful shutdown SIGINT with -it
+2. Run -e NETWORK=mainnet and check graceful shutdown SIGTERM with --detach
+3. Run without -e NETWORK and check graceful shutdown SIGINT with -it
+4. Run without -e NETWORK and check graceful shutdown SIGTERM with --detach
+5. Check cardano-cli access
+
+### Run with NETWORK
+
+Run -e NETWORK=mainnet and check graceful shutdown SIGINT with -it
+
+```
+docker run --rm -it \
+  -p 3001:3001 \
+  -e NETWORK=mainnet \
+  -v node-data:/data/db \
+  inputoutput/cardano-node:dev
+```
+
+Run -e NETWORK=mainnet and check graceful shutdown SIGTERM with --detach
+
+```
+docker rm -f relay
+docker run --detach \
+  --name=relay \
+  -p 3001:3001 \
+  -e NETWORK=mainnet \
+  -v node-data:/data/db \
+  inputoutput/cardano-node:dev
+
+docker logs -f relay
+```
+
+### Run without NETWORK
+
+Check graceful shutdown SIGINT with -it
+
+```
+docker run --rm -it \
+  -p 3001:3001 \
+  -v node-data:/opt/cardano/data \
+  inputoutput/cardano-node:dev run
+```
+
+Check graceful shutdown SIGTERM with --detach
+
+```
+docker rm -f relay
+docker run --detach \
+  --name=relay \
+  -p 3001:3001 \
+  -v node-data:/opt/cardano/data \
+  -v node-ipc:/opt/cardano/ipc \
+  inputoutput/cardano-node:dev run
+
+docker logs -f relay
+```
+
+### Check cardano-cli
+
+```
+docker run --rm -it \
+  -v node-ipc:/opt/cardano/ipc \
+  inputoutput/cardano-node:dev cli query tip --mainnet
+```

nix/docker/context/bin/entrypoint

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ -n $NETWORK ]]; then
+
+  exec /usr/local/bin/run-network
+
+elif [[ $1 == "run" ]]; then
+
+  exec /usr/local/bin/run-node $@
+
+elif [[ $1 == "cli" ]]; then
+
+  exec /usr/local/bin/run-client $@
+
+else
+
+  echo "Nothing to do! Perhaps try [run|cli]"
+  exit 1
+
+fi

nix/docker/context/bin/proc-sigterm

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+#####################################################################
+#
+# Trap the SIGTERM signal
+#
+trapSIGTERM () {
+
+  PID="$!"
+  trap "sigtermHandler $PID" SIGTERM
+
+  # Wait for the child process to terminate
+  wait $PID
+}
+
+# Signal the cardano-node with SIGINT for graceful shutdown
+#
+# The wait may return before the gracefull shutdown is done
+# Wait a little longer before we will kill everything
+
+sigtermHandler () {
+  echo "Signalling for shutdown ..."
+  kill -SIGINT $1
+  wait $1
+  sleep 2
+}

nix/docker/context/bin/run-client

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Shift the first option by one index
+shift
+
+if [[ -z $CARDANO_NODE_SOCKET_PATH ]]; then
+  export CARDANO_NODE_SOCKET_PATH="/opt/cardano/ipc/socket"
+fi
+
+/usr/local/bin/cardano-cli $@

nix/docker/context/bin/run-node

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+source /usr/local/bin/proc-sigterm
+
+echo "Running the cardano node ..."
+
+# Shift the first option by one index
+shift
+
+# Define a few defaults
+CARDANO_CONFIG_BASE="/opt/cardano/config"
+
+if [[ -z $CARDANO_CONFIG ]]; then
+  CARDANO_CONFIG="$CARDANO_CONFIG_BASE/mainnet-config.json"
+fi
+
+if [[ -z $CARDANO_TOPOLOGY ]]; then
+  CARDANO_TOPOLOGY="$CARDANO_CONFIG_BASE/mainnet-topology.json"
+fi
+
+if [[ -z $CARDANO_DATABASE_PATH ]]; then
+  CARDANO_DATABASE_PATH="/opt/cardano/data"
+fi
+
+if [[ -z $CARDANO_SOCKET_PATH ]]; then
+  CARDANO_SOCKET_PATH="/opt/cardano/ipc/socket"
+fi
+
+if [[ -z $CARDANO_BIND_ADDR ]]; then
+  CARDANO_BIND_ADDR="0.0.0.0"
+fi
+
+if [[ -z $CARDANO_PORT ]]; then
+  CARDANO_PORT=3001
+fi
+
+if [ -v $CARDANO_BLOCK_PRODUCER ]; then
+  CARDANO_BLOCK_PRODUCER=false
+fi
+
+#####################################################################
+#
+# Print run environment
+#
+printRunEnv () {
+
+  echo "CARDANO_BIND_ADDR=$CARDANO_BIND_ADDR"
+  echo "CARDANO_BLOCK_PRODUCER=$CARDANO_BLOCK_PRODUCER"
+  echo "CARDANO_CONFIG=$CARDANO_CONFIG"
+  echo "CARDANO_DATABASE_PATH=$CARDANO_DATABASE_PATH"
+  echo "CARDANO_PORT=$CARDANO_PORT"
+  echo "CARDANO_SOCKET_PATH=$CARDANO_SOCKET_PATH"
+  echo "CARDANO_TOPOLOGY=$CARDANO_TOPOLOGY"
+
+  if [ "$CARDANO_BLOCK_PRODUCER" = true ]; then
+
+    if [ -v $CARDANO_SHELLEY_KES_KEY ]; then
+      CARDANO_SHELLEY_KES_KEY="$CARDANO_CONFIG_BASE/keys/kes.skey"
+    fi
+
+    if [ -v $CARDANO_SHELLEY_VRF_KEY ]; then
+      CARDANO_SHELLEY_VRF_KEY="$CARDANO_CONFIG_BASE/keys/vrf.skey"
+    fi
+
+    if [ -v $CARDANO_SHELLEY_OPERATIONAL_CERTIFICATE ]; then
+      CARDANO_SHELLEY_OPERATIONAL_CERTIFICATE="$CARDANO_CONFIG_BASE/keys/node.cert"
+    fi
+
+    echo "CARDANO_SHELLEY_KES_KEY=$CARDANO_SHELLEY_KES_KEY"
+    echo "CARDANO_SHELLEY_VRF_KEY=$CARDANO_SHELLEY_VRF_KEY"
+    echo "CARDANO_SHELLEY_OPERATIONAL_CERTIFICATE=$CARDANO_SHELLEY_OPERATIONAL_CERTIFICATE"
+  fi
+}
+
+# Override default values with explicit options
+options=($@)
+for i in "${!options[@]}"
+do
+    j=$((i + 1))
+    key=${options[i]}
+    val=${options[j]}
+    found=false
+
+    # echo "$i/$j: ${key} ${val}"
+
+    case ${key} in
+      --config) CARDANO_CONFIG=${val}; found=true;;
+      --topology) CARDANO_TOPOLOGY=${val}; found=true;;
+      --database-path) CARDANO_DATABASE_PATH=${val}; found=true;;
+      --socket-path) CARDANO_SOCKET_PATH=${val}; found=true;;
+      --host-addr) CARDANO_BIND_ADDR=${val}; found=true;;
+      --port) CARDANO_PORT=${val}; found=true;;
+    esac
+
+    if [[ $found == true ]]; then
+      options[i]="";
+      options[j]="";
+    fi
+done
+
+printRunEnv
+
+effopts=(--config $CARDANO_CONFIG \
+  --topology $CARDANO_TOPOLOGY \
+  --database-path $CARDANO_DATABASE_PATH \
+  --socket-path $CARDANO_SOCKET_PATH \
+  --host-addr $CARDANO_BIND_ADDR \
+  --port $CARDANO_PORT)
+
+effopts+=(${options[@]})
+
+echo "cardano-node run ${effopts[@]}"
+
+# The IPC socket dir is not created on demand
+mkdir -p `dirname $CARDANO_SOCKET_PATH`
+
+/usr/local/bin/cardano-node run ${effopts[@]} &
+
+trapSIGTERM

nix/docker.nix → nix/docker/default.nix

@@ -30,6 +30,7 @@
 
 # The main contents of the image.
 , cardano-cli
+, cardano-node
 , customConfig
 , scripts
 
@@ -85,36 +86,60 @@ let
   # To choose a network, use `-e NETWORK testnet`
   clusterStatements = lib.concatStringsSep "\n" (lib.mapAttrsToList (env: scripts: ''
     elif [[ "$NETWORK" == "${env}" ]]; then
-      exec ${scripts.${script}}
+      ${scripts.${script}} &
+      trapSIGTERM
   '') scripts);
 
-  entry-point = writeScriptBin "entry-point" ''
-    #!${runtimeShell}
+  runNetwork = pkgs.writeShellScriptBin "run-network" ''
+
+    source /usr/local/bin/proc-sigterm
+
+    # The IPC socket dir is not created on demand
+    mkdir -p `dirname ${customConfig.socketPath}`
+
     if [[ -z "$NETWORK" ]]; then
-      exec ${pkgs.${exe}}/bin/${exe} $@
+      echo "[Error] Cannot obtain NETWORK env variable"
+      exit 1
     ${clusterStatements}
     else
-      echo "Managed configuration for network "$NETWORK" does not exist"
+      echo "[Error] Managed configuration for network "$NETWORK" does not exist"
+      exit 1
     fi
   '';
 
-  # Remove the leading '/' and trailing basename
-  socketPath = customConfig.socketPath;
-  socketDir = lib.concatStringsSep "/" (builtins.filter (x: x != "")
-    (lib.splitString "/" (lib.removeSuffix (baseNameOf socketPath) socketPath)));
+  # The Docker context with static content
+  context = ./context;
 
-  nodeDockerImage = dockerTools.buildImage {
+  # Mainnet configuration files used by the 'run' option
+  mainnetConfigs = builtins.filterSource
+    (path: type: type == "regular" && lib.hasPrefix "mainnet-" (baseNameOf path))
+    ../../configuration/cardano;
+
+in
+  dockerTools.buildImage {
     name = "${repoName}";
-    fromImage = baseImage;
     tag = "${gitrev}";
+    fromImage = baseImage;
     created = "now";   # Set creation date to build time. Breaks reproducibility
-    contents = [ entry-point ];
+    contents = [
+    ];
+
+    # May require system-features = kvm in /etc/nix/nix.conf
+    # https://discourse.nixos.org/t/cannot-build-docker-image/7445
+    # runAsRoot = '' ln -s ${cardano-node} bin/cardano-node '';
+
     extraCommands = ''
-      mkdir -p ${socketDir}
+      mkdir -p opt/cardano/config
+      mkdir -p opt/cardano/data
+      mkdir -p opt/cardano/ipc
+      mkdir -p usr/local/bin
+      cp ${mainnetConfigs}/* opt/cardano/config
+      cp ${runNetwork}/bin/* usr/local/bin
+      cp ${context}/bin/* usr/local/bin
+      ln -s ${cardano-node}/bin/cardano-node usr/local/bin/cardano-node
+      ln -s ${cardano-cli}/bin/cardano-cli usr/local/bin/cardano-cli
     '';
     config = {
-      EntryPoint = [ "${entry-point}/bin/entry-point" ];
+      EntryPoint = [ "entrypoint" ];
     };
-  };
-
-in nodeDockerImage
+  }

nix/pkgs.nix

@@ -83,12 +83,12 @@ final: prev: with final;
 
   dockerImage = let
     defaultConfig = {
+      socketPath = "/ipc/node.socket";
       stateDir = "/data";
       dbPrefix = "db";
-      socketPath = "/ipc/node.socket";
     };
     customConfig' = defaultConfig // customConfig;
-  in callPackage ./docker.nix {
+  in callPackage ./docker {
     exe = "cardano-node";
     scripts = import ./scripts.nix {
       inherit pkgs;
@@ -103,12 +103,13 @@ final: prev: with final;
       socketPath = "/ipc/node.socket";
     };
     customConfig' = defaultConfig // customConfig;
-  in callPackage ./docker.nix {
+  in callPackage ./docker {
     exe = "cardano-submit-api";
     scripts = import ./scripts-submit-api.nix {
       inherit pkgs;
       customConfig = customConfig';
     };
+    customConfig = customConfig';
     script = "submit-api";
   };