faucet healthcheck attempt

input-output-hk · Nov 19, 2020 · 4f95889 · 4f95889
1 parent 5514f87
commit 4f95889
Show file tree

Hide file tree

Showing 3 changed files with 115 additions and 78 deletions.
diff --git a/docker/mantis.nix b/docker/mantis.nix
@@ -1,5 +1,5 @@
 { lib, mkEnv, buildLayeredImage, writeShellScript, mantis, mantis-faucet
-, coreutils, gnused, gnugrep, debugUtils }:
+, coreutils, gnused, gnugrep, curl, debugUtils }:
 let
   mantis-entrypoint = writeShellScript "mantis" ''
     set -exuo pipefail
@@ -24,22 +24,39 @@ let
   faucet-entrypoint = writeShellScript "mantis-faucet" ''
     set -exuo pipefail
 
-    mkdir -p /tmp
-    mkdir -p "$NOMAD_TASK_DIR/mantis"
-    mkdir -p "$NOMAD_SECRETS_DIR/keystore"
+    case $1 in
+      healthcheck)
+        test WalletAvailable = "$(
+          curl \
+            http://$NOMAD_ADDR_rpc \
+            -H 'Content-Type: application/json' \
+            -X POST \
+            -d '{"jsonrpc": "2.0", "method": "faucet_status", "params": [], "id": 1}' \
+          | jq -e -r .result.status
+        )"
+      ;;
+      *)
+        mkdir -p /tmp
+        mkdir -p "$NOMAD_TASK_DIR/mantis"
+        mkdir -p "$NOMAD_SECRETS_DIR/keystore"
 
-    cd "$NOMAD_TASK_DIR"
+        cd "$NOMAD_TASK_DIR"
 
-    cp faucet.conf running.conf
-    cp "$NOMAD_SECRETS_DIR/account" "$NOMAD_SECRETS_DIR/keystore/UTC--2020-10-16T14-48-29.47Z-$COINBASE"
+        cp faucet.conf running.conf
+        cp "$NOMAD_SECRETS_DIR/account" "$NOMAD_SECRETS_DIR/keystore/UTC--2020-10-16T14-48-29.47Z-$COINBASE"
 
-    chown --reference . --recursive . || true
-    ulimit -c unlimited
-    exec mantis faucet "-Duser.home=$NOMAD_TASK_DIR" "$@"
+        chown --reference . --recursive . || true
+        ulimit -c unlimited
+        exec mantis faucet "-Duser.home=$NOMAD_TASK_DIR" "$@"
+      ;;
+    esac
   '';
 in {
   mantis = buildLayeredImage {
     name = "docker.mantis.ws/mantis";
+
+    contents = debugUtils;
+
     config = {
       Entrypoint = [ mantis-entrypoint ];
 
@@ -57,7 +74,7 @@ in {
       Entrypoint = [ faucet-entrypoint ];
 
       Env = mkEnv {
-        PATH = lib.makeBinPath [ coreutils gnugrep gnused mantis-faucet ];
+        PATH = lib.makeBinPath [ coreutils gnugrep gnused mantis-faucet curl ];
       };
     };
   };

diff --git a/jobs/mantis.nix b/jobs/mantis.nix
@@ -211,11 +211,12 @@ let
     lib.nameValuePair name (mkMantis {
       resources = {
         # For c5.2xlarge in clusters/mantis/testnet/default.nix, the url ref below
-        # provides 3.4 GHz * 8 vCPU = 27.2 GHz max.  80% is 21760 MHz.
+        # provides 3.4 GHz * 8 vCPU = 27.2 GHz max.
+        # Mantis mainly uses only one core.
         # Allocating by vCPU or core quantity not yet available.
         # Ref: https://github.com/hashicorp/nomad/blob/master/client/fingerprint/env_aws.go
-        cpu = 21760;
-        memoryMB = 5 * 1024;
+        cpu = 3400;
+        memoryMB = 4 * 1024;
       };
 
       inherit name requiredPeerCount;
@@ -388,10 +389,18 @@ let
 
   faucetName = "${namespace}-mantis-faucet";
   faucet = {
+    networks = [{
+      ports = {
+        metrics.to = 7000;
+        rpc.to = 8000;
+      };
+    }];
+
     services = {
       "${faucetName}" = {
         addressMode = "host";
         portLabel = "rpc";
+        task = "faucet";
 
         tags =
           [ "ingress" namespace "faucet" faucetName mantis-faucet-source.rev ];
@@ -404,7 +413,25 @@ let
           ingressMode = "http";
           ingressServer = "_${faucetName}._tcp.service.consul";
         };
+
+        # FIXME: this always returns FaucetUnavailable
+        # checks = [{
+        #   taskName = "faucet";
+        #   type = "script";
+        #   name = "faucet_health";
+        #   command = "healthcheck";
+        #   interval = "60s";
+        #   timeout = "5s";
+        #   portLabel = "rpc";
+
+        #   checkRestart = {
+        #     limit = 5;
+        #     grace = "300s";
+        #     ignoreWarnings = false;
+        #   };
+        # }];
       };
+
       "${faucetName}-prometheus" = {
         addressMode = "host";
         portLabel = "metrics";
@@ -418,76 +445,15 @@ let
       };
     };
 
-    networks = [{
-      ports = {
-        metrics.to = 7000;
-        rpc.to = 8000;
-      };
-    }];
-
-    tasks.telegraf = {
-      driver = "docker";
-
-      vault.policies = [ "nomad-cluster" ];
-
-      resources = {
-        cpu = 100; # mhz
-        memoryMB = 128;
-      };
-
-      config = {
-        image = dockerImages.telegraf.id;
-        args = [ "-config" "local/telegraf.config" ];
-
-        labels = [{
-          inherit namespace;
-          name = "faucet";
-          imageTag = dockerImages.telegraf.image.imageTag;
-        }];
-
-        logging = {
-          type = "journald";
-          config = [{
-            tag = "faucet-telegraf";
-            labels = "name,namespace,imageTag";
-          }];
-        };
-      };
-
-      templates = [{
-        data = ''
-          [agent]
-          flush_interval = "10s"
-          interval = "10s"
-          omit_hostname = false
-
-          [global_tags]
-          client_id = "faucet"
-          namespace = "${namespace}"
-
-          [inputs.prometheus]
-          metric_version = 1
-
-          urls = [ "http://{{ env "NOMAD_ADDR_metrics" }}" ]
-
-          [outputs.influxdb]
-          database = "telegraf"
-          urls = ["http://{{with node "monitoring" }}{{ .Node.Address }}{{ end }}:8428"]
-        '';
-
-        destination = "local/telegraf.config";
-      }];
-    };
-
     tasks.faucet = {
       name = "faucet";
       driver = "docker";
 
       vault.policies = [ "nomad-cluster" ];
 
       resources = {
-        cpu = 21760;
-        memoryMB = 5 * 1024;
+        cpu = 100;
+        memoryMB = 1024;
       };
 
       config = {
@@ -631,6 +597,60 @@ let
         }
       ];
     };
+
+    tasks.telegraf = {
+      driver = "docker";
+
+      vault.policies = [ "nomad-cluster" ];
+
+      resources = {
+        cpu = 100; # mhz
+        memoryMB = 128;
+      };
+
+      config = {
+        image = dockerImages.telegraf.id;
+        args = [ "-config" "local/telegraf.config" ];
+
+        labels = [{
+          inherit namespace;
+          name = "faucet";
+          imageTag = dockerImages.telegraf.image.imageTag;
+        }];
+
+        logging = {
+          type = "journald";
+          config = [{
+            tag = "faucet-telegraf";
+            labels = "name,namespace,imageTag";
+          }];
+        };
+      };
+
+      templates = [{
+        data = ''
+          [agent]
+          flush_interval = "10s"
+          interval = "10s"
+          omit_hostname = false
+
+          [global_tags]
+          client_id = "faucet"
+          namespace = "${namespace}"
+
+          [inputs.prometheus]
+          metric_version = 1
+
+          urls = [ "http://{{ env "NOMAD_ADDR_metrics" }}" ]
+
+          [outputs.influxdb]
+          database = "telegraf"
+          urls = ["http://{{with node "monitoring" }}{{ .Node.Address }}{{ end }}:8428"]
+        '';
+
+        destination = "local/telegraf.config";
+      }];
+    };
   };
 in {
   "${namespace}-mantis" = mkNomadJob "mantis" {

diff --git a/overlay.nix b/overlay.nix
@@ -10,7 +10,7 @@ in {
   # The branch was `chore/update-sbt-add-nix`, for future reference.
   mantis-source = builtins.fetchGit {
     url = "https://github.com/input-output-hk/mantis";
-    rev = "dcb6d7b714bc1e23f75dd3e62365b316dab68ec3";
+    rev = "aaf3c852b4882369ac5a0f518344d9b02e63b8ef";
     ref = "develop";
     submodules = true;
   };