Morpho SRV topology discovery

input-output-hk · Jan 28, 2021 · 48f3a76 · 48f3a76
1 parent 7900e1e
commit 48f3a76
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 74 deletions.
diff --git a/docker/morpho.nix b/docker/morpho.nix
@@ -1,55 +1,32 @@
-{ lib, buildLayeredImage, mkEnv, morpho-node, coreutils, gnused, gnugrep, procps
-, writeShellScript, jq, diffutils }:
+{ lib, buildLayeredImage, mkEnv, morpho-node, coreutils, bashInteractive
+, writeShellScript, iana-etc, runCommandNoCC }:
 let
   run-morpho-node = writeShellScript "morpho-node" ''
-    set -exuo pipefail
-
-    cd "$NOMAD_TASK_DIR"
-    name="morpho-checkpoint-node"
-
-    cp morpho-topology.json running-morpho-topology.json
-
-    (
-      while true; do
-        while diff -u running-morpho-topology.json morpho-topology.json > /dev/stderr; do
-          sleep 300
-        done
-
-        if ! diff -u running-morpho-topology.json morpho-topology.json > /dev/stderr; then
-          cp morpho-topology.json running-morpho-topology.json
-          pkill "$name" || true
-        fi
-      done
-    ) &
-
-    starts=0
-    while true; do
-      starts="$((starts+1))"
-      echo "Start Number $starts" > /dev/stderr
-      morpho-checkpoint-node \
-        --topology "$NOMAD_TASK_DIR/running-morpho-topology.json" \
-        --database-path /local/db \
-        --port "$NOMAD_PORT_morpho" \
-        --config "$NOMAD_TASK_DIR/morpho-config.yaml" \
-        --socket-dir "$NOMAD_TASK_DIR/socket" \
-        "$@" || true
-      sleep 10
-    done
+    morpho-checkpoint-node \
+      --topology "$NOMAD_TASK_DIR/morpho-topology.json" \
+      --database-path /local/db \
+      --port "$NOMAD_PORT_morpho" \
+      --config "$NOMAD_TASK_DIR/morpho-config.yaml" \
+      --socket-dir "$NOMAD_TASK_DIR/socket" \
+      "$@"
+  '';
+  # Needed for DNS to work
+  etcRoot = runCommandNoCC "etc" {} ''
+    mkdir -p $out/etc
+    ln -s ${iana-etc}/etc/services $out/etc/services
   '';
 in {
   morpho = buildLayeredImage {
     name = "docker.mantis.ws/morpho-node";
+    contents = [ etcRoot ];
     config = {
       Entrypoint = [ run-morpho-node ];
       Env = mkEnv {
         PATH = lib.makeBinPath [
-          coreutils
-          gnugrep
-          gnused
           morpho-node
-          jq
-          procps
-          diffutils
+
+          bashInteractive
+          coreutils
         ];
       };
     };

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -11,7 +11,7 @@
     utils.url = "github:numtide/flake-utils";
     ops-lib.url = "github:input-output-hk/ops-lib/zfs-image?dir=zfs";
     inclusive.follows = "bitte/inclusive";
-    morpho-node.url = "github:input-output-hk/ECIP-Checkpointing/master";
+    morpho-node.url = "github:input-output-hk/ECIP-Checkpointing";
     mantis-explorer.url = "github:input-output-hk/mantis-explorer";
     mantis-faucet-web.url =
       "github:input-output-hk/mantis-faucet-web/nix-build";

diff --git a/jobs/mantis-staging.nix b/jobs/mantis-staging.nix
@@ -28,6 +28,7 @@ let
     inherit namespace morpho-source vault dockerImages;
     name = "obft-node-${toString n}";
     nodeNumber = n;
+    nodeCount = amountOfMorphoNodes;
   });
 
   mkMantis = { name, resources, count ? 1, templates, serviceName, tags ? [ ]
@@ -935,7 +936,7 @@ in {
       mkMorpho = import ./tasks/morpho.nix;
       generateMorphoTaskGroup = nbNodes: node:
         lib.nameValuePair node.name (lib.recursiveUpdate (mkPassive 1)
-          (mkMorpho (node // { inherit nbNodes; })));
+          (mkMorpho (node // { inherit lib nbNodes; })));
       morphoTaskGroups =
         map (generateMorphoTaskGroup (builtins.length morphoNodes)) morphoNodes;
     in lib.listToAttrs morphoTaskGroups;

diff --git a/jobs/tasks/morpho.nix b/jobs/tasks/morpho.nix
@@ -1,4 +1,4 @@
-{ namespace, name, nodeNumber, morpho-source, vault, dockerImages, nbNodes }: {
+{ lib, namespace, name, nodeNumber, nodeCount, morpho-source, vault, dockerImages, nbNodes }: {
   services = {
     "${namespace}-morpho-node" = {
       portLabel = "morpho";
@@ -19,6 +19,8 @@
 
   networks = [{
     mode = "bridge";
+    # From https://github.com/input-output-hk/bitte/blob/33cb20fa1cd7c6e4d3bc75253fc166bf048b500c/profiles/docker.nix#L16
+    dns.servers = [ "172.17.0.1" ];
     ports = {
       discovery.to = 6000;
       metrics.to = 6100;
@@ -105,33 +107,19 @@
         splay = "15m";
       }
       {
-        data = ''
-          [
-            {{- range $index1, $service1 := service "${namespace}-morpho-node" -}}
-            {{ if ne $index1 0 }},{{ end }}
-              {
-                "nodeAddress": {
-                "addr": "{{ .Address }}",
-                "port": {{ .Port }},
-                "valency": 1
-                },
-                "nodeId": {{- index (split "-" .ServiceMeta.Name) 2 -}},
-                "producers": [
-                {{- range $index2, $service2 := service "${namespace}-morpho-node" -}}
-                {{ if ne $index2 0 }},{{ end }}
-                  {
-                      "addr": "{{ .Address }}",
-                      "port": {{ .Port }},
-                      "valency": 1
-                  }
-                {{- end -}}
-                ]}
-            {{- end }}
-            ]
-        '';
+        data =
+          let
+            addressFor = n: {
+              addr = "_mantis-staging-morpho-node._obft-node-${toString n}.service.consul.";
+              # No port -> SRV query above address
+              valency = 1;
+            };
+            data = map (n: {
+              nodeId = n;
+              producers = map addressFor (lib.remove n (lib.range 1 nodeCount));
+            }) (lib.range 1 nodeCount);
+          in builtins.toJSON data;
         destination = "local/morpho-topology.json";
-        changeMode = "noop";
-        splay = "15m";
       }
     ];