add snapshot downloader

change ordering of init containers again to make sure they all download snapshots
oxheadalpha · Feb 24, 2021 · 5bc4c0f · nicolasochem · Feb 24, 2021 · 5bc4c0f
1 parent 24f724f
commit 5bc4c0f
Show file tree

Hide file tree

Showing 9 changed files with 104 additions and 67 deletions.
diff --git a/baker-endorser/Dockerfile b/baker-endorser/Dockerfile
@@ -1,4 +1,4 @@
-FROM tezos/tezos:master
+FROM tezos/tezos:v8-release
 RUN sudo apk add jq bash
 COPY entrypoint.sh /
 ENTRYPOINT ["/entrypoint.sh"]

diff --git a/charts/tezos/templates/baker.yaml b/charts/tezos/templates/baker.yaml
@@ -93,24 +93,6 @@ spec:
               name: config-volume
             - mountPath: /var/tezos
               name: var-volume
-        - image: "{{ .Values.tezos_k8s_images.baker_endorser }}"
-          imagePullPolicy: IfNotPresent
-          name: baker
-          volumeMounts:
-            - mountPath: /var/tezos
-              name: var-volume
-          envFrom:
-            - configMapRef:
-                name: tezos-config
-            - secretRef:
-                name: tezos-secret
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: DAEMON
-            value: baker
         - image: "{{ .Values.tezos_k8s_images.baker_endorser }}"
           imagePullPolicy: IfNotPresent
           name: endorser
@@ -150,6 +132,37 @@ spec:
               name: var-volume
 {{- end }}
       initContainers:
+        - image: {{ .Values.tezos_k8s_images.config_generator }}
+          imagePullPolicy: IfNotPresent
+          name: config-generator
+          args:
+            - "--generate-config-json"
+          envFrom:
+            - secretRef:
+                name: tezos-secret
+            - configMapRef:
+                name: tezos-config
+          env:
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          volumeMounts:
+            - mountPath: /etc/tezos
+              name: config-volume
+            - mountPath: /var/tezos
+              name: var-volume
+        - image: "{{ .Values.tezos_k8s_images.snapshot_downloader }}"
+          imagePullPolicy: IfNotPresent
+          name: snapshot-downloader
+          volumeMounts:
+            - mountPath: /var/tezos
+              name: var-volume
+            - mountPath: /etc/tezos
+              name: config-volume
+          envFrom:
+            - configMapRef:
+                name: tezos-config
 {{- if not .Values.is_invitation }}
         - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }}
           imagePullPolicy: IfNotPresent
@@ -183,26 +196,6 @@ spec:
               fieldRef:
                 fieldPath: metadata.name
 {{- end }}
-        - image: {{ .Values.tezos_k8s_images.config_generator }}
-          imagePullPolicy: IfNotPresent
-          name: config-generator
-          args:
-            - "--generate-config-json"
-          envFrom:
-            - secretRef:
-                name: tezos-secret
-            - configMapRef:
-                name: tezos-config
-          env:
-            - name: MY_POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-          volumeMounts:
-            - mountPath: /etc/tezos
-              name: config-volume
-            - mountPath: /var/tezos
-              name: var-volume
       securityContext:
         fsGroup: 100
       volumes:

diff --git a/charts/tezos/templates/configs.yaml b/charts/tezos/templates/configs.yaml
@@ -11,6 +11,7 @@ data:
       "timestamp": "{{ .Values.genesis.timestamp }}",
       "chain_type": "{{ .Values.chain_type }}",
       "network": "{{ .Values.network }}",
+      "snapshot_url": "{{ .Values.snapshot_url }}",
       "proto_command": "{{ .Values.protocol.command }}",
       "protocol_hash": "{{ .Values.protocol.hash }}"
     }

diff --git a/charts/tezos/templates/node.yaml b/charts/tezos/templates/node.yaml
@@ -63,19 +63,37 @@ spec:
               name: var-volume
 {{- end }}
       initContainers:
-        - args:
-            - "-c"
-            - "[ -f /var/tezos/node/identity.json ] || (mkdir -p /var/tezos/node && /usr/local/bin/tezos-node identity generate 0 --data-dir /var/tezos/node --config-file /etc/tezos/config.json)"
-          command:
-            - /bin/sh
-          image: {{ .Values.images.tezos | quote }}
-          name: identity-job
+        - image: {{ .Values.tezos_k8s_images.config_generator }}
           imagePullPolicy: IfNotPresent
+          name: config-generator
+          args:
+            - "--generate-config-json"
+          envFrom:
+            - secretRef:
+                name: tezos-secret
+            - configMapRef:
+                name: tezos-config
+          env:
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
           volumeMounts:
             - mountPath: /etc/tezos
               name: config-volume
             - mountPath: /var/tezos
               name: var-volume
+        - image: "{{ .Values.tezos_k8s_images.snapshot_downloader }}"
+          imagePullPolicy: IfNotPresent
+          name: snapshot-downloader
+          volumeMounts:
+            - mountPath: /var/tezos
+              name: var-volume
+            - mountPath: /etc/tezos
+              name: config-volume
+          envFrom:
+            - configMapRef:
+                name: tezos-config
 {{- if not .Values.is_invitation }}
         - image: {{ .Values.tezos_k8s_images.wait_for_bootstrap }}
           imagePullPolicy: IfNotPresent
@@ -109,26 +127,6 @@ spec:
               fieldRef:
                 fieldPath: metadata.name
 {{- end }}
-        - image: {{ .Values.tezos_k8s_images.config_generator }}
-          imagePullPolicy: IfNotPresent
-          name: config-generator
-          args:
-            - "--generate-config-json"
-          envFrom:
-            - secretRef:
-                name: tezos-secret
-            - configMapRef:
-                name: tezos-config
-          env:
-            - name: MY_POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-          volumeMounts:
-            - mountPath: /etc/tezos
-              name: config-volume
-            - mountPath: /var/tezos
-              name: var-volume
       securityContext:
         fsGroup: 100
       volumes:

diff --git a/charts/tezos/values.yaml b/charts/tezos/values.yaml
@@ -18,7 +18,9 @@ tezos_k8s_images:
   config_generator: tezos-k8s-config-generator:dev
   wait_for_bootstrap: tezos-k8s-wait-for-bootstrap:dev
   zerotier: tezos-k8s-zerotier:dev
+  snapshot_downloader: tezos-k8s-snapshot-downloader:dev
 
+snapshot_url: https://mainnet.xtz-shots.io/rolling
 protocol:
   bootstrap_mutez: "4000000000000"
   hash: PsDELPH1Kxsxt8f9eWbxQeRxkjfbxoqM52jvs5Y5fBxWWh4ifpo

diff --git a/config-generator/entrypoint.py b/config-generator/entrypoint.py
@@ -112,14 +112,17 @@ def get_node_config(
     net_addr=None,
 ):
 
-    node_config = { "data-dir": "/var/tezos/node",
+    node_config = { "data-dir": "/var/tezos/node/data",
             "rpc": {
                 "listen-addrs": [f"{os.getenv('MY_POD_IP')}:8732", "127.0.0.1:8732"],
                 },
             "p2p": {
                 "bootstrap-peers": bootstrap_peers,
                 "listen-addr": ( net_addr + ":9732" if net_addr else "[::]:9732" )
                 },
+            "shell": {
+                "history_mode": "rolling"
+                },
             #"log": { "level": "debug"},
         }
     if CHAIN_PARAMS["chain_type"] == "public":

diff --git a/devspace.yaml b/devspace.yaml
@@ -28,6 +28,10 @@ images:
     image: tezos-k8s-wait-for-bootstrap
     dockerfile: ./wait-for-bootstrap/Dockerfile
     context: ./wait-for-bootstrap
+  snapshot-downloader:
+    image: tezos-k8s-snapshot-downloader
+    dockerfile: ./snapshot-downloader/Dockerfile
+    context: ./snapshot-downloader
   config-generator:
     image: tezos-k8s-config-generator
     dockerfile: ./config-generator/Dockerfile

diff --git a/snapshot-downloader/Dockerfile b/snapshot-downloader/Dockerfile
@@ -0,0 +1,5 @@
+FROM tezos/tezos:master
+RUN sudo apk add curl lz4 xz jq
+COPY entrypoint.sh /
+ENTRYPOINT ["/entrypoint.sh"]
+CMD []
diff --git a/snapshot-downloader/entrypoint.sh b/snapshot-downloader/entrypoint.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -x
+
+bin_dir="/usr/local/bin"
+
+data_dir="/var/tezos"
+node_dir="$data_dir/node"
+node_data_dir="$node_dir/data"
+node="$bin_dir/tezos-node"
+
+tezos_network=$(echo $CHAIN_PARAMS | jq -r '.network')
+snapshot_url=$(echo $CHAIN_PARAMS | jq -r '.snapshot_url')
+if [ -d ${node_dir}/data/context ]; then
+    echo "Blockchain has already been imported, exiting"
+    exit 0
+elif [ -z "$snapshot_url" ]; then
+    echo "No snapshot was passed as parameter, exiting"
+    exit 0
+else
+    echo "Did not find pre-existing data, importing blockchain"
+    mkdir -p ${node_dir}/data
+    echo '{ "version": "0.0.4" }' > ${node_dir}/version.json
+    cp -v /usr/local/share/tezos/alphanet_version ${node_dir}
+    snapshot_file=${node_dir}/chain.snapshot
+    curl -L -o $snapshot_file $snapshot_url
+    exec "${node}" snapshot import ${snapshot_file} --data-dir ${node_data_dir} --network $tezos_network --config-file /etc/tezos/config.json
+    find ${node_dir}
+    rm -rvf ${snapshot_file}
+    echo ""
+fi