feat: implement backup uploader sidecar container (#148)

Signed-off-by: Lenin Mehedy <lenin.mehedy@swirldslabs.com>

charts/hedera-network/templates/configmaps.yaml

@@ -87,11 +87,8 @@ kind: ConfigMap
 metadata:
   name: backup-config
 data:
-  # add your data here
-  property1: value1
-  property2: value2
-
-{{- if $.Values.sidecars.otel.enabled }}
+  BACKUP_UPLOADER_BUCKET_1: {{ .Values.sidecars.backup.bucket1 }}
+  BACKUP_UPLOADER_BUCKET_2: {{ .Values.sidecars.backup.bucket2 }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -100,4 +97,3 @@ metadata:
 data:
   config.yaml:
   {{- .Files.Get "configs/otel-collector-config.yaml" | toYaml | nindent 4 }}
-{{ end }}

charts/hedera-network/templates/network-node-statefulset.yaml

@@ -9,7 +9,7 @@ metadata:
     app: network-{{ $node.name }}
 spec:
   replicas: 1
-  serviceName: "network-node-{{ $node.name }}"
+  serviceName: "network-{{ $node.name }}"
   selector:
     matchLabels:
       app: network-{{ $node.name }}
@@ -21,16 +21,18 @@ spec:
       volumes:
         - name: hgcapp-storage # change me
           emptyDir: {}
-        {{- if $.Values.sidecars.otel.enabled }}
+        {{- if $node.sidecars.otel }}
         - name: otel-collector-config
           configMap:
             name: otel-collector-config
         {{- end }}
       initContainers:
         - name: init-container
           image: {{ $.Values.infrastructure.docker.images.root }}
-          command: [ "/bin/sh", "-c", "mkdir /opt/hgcapp/accountbalance /opt/hgcapp/events /opt/hgcapp/recordstream /opt/hgcapp/sidecar /opt/hgcapp/recordstream/uploader-stats/"  ]
-          runAsUser: {{ $.Values.mirror.user }}
+          command: [ "/bin/sh", "-c", "mkdir /opt/hgcapp/hedera-backups /opt/hgcapp/accountbalance /opt/hgcapp/events /opt/hgcapp/recordstream /opt/hgcapp/sidecar /opt/hgcapp/recordstream/uploader-stats/"  ]
+          securityContext:
+            runAsUser: {{ $.Values.hedera.user }}
+            runAsGroup: {{ $.Values.hedera.group }}
           volumeMounts:
             - name: hgcapp-storage
               mountPath: /opt/hgcapp/
@@ -52,13 +54,13 @@ spec:
           - name: hgcapp-storage
             mountPath: /opt/hgcapp/
 
-  {{- if $node.uploaderSidecars }}
-
+  {{- if $node.sidecars.accountBalancesUploader }}
       # Account Balance Uploader
       - name: account-balances-uploader
         image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
         securityContext:
-          runAsUser: {{ $.Values.mirror.user }}
+          runAsUser: {{ $.Values.hedera.user }}
+          runAsGroup: {{ $.Values.hedera.group }}
         command:
           - /usr/bin/env
           - python3.7
@@ -74,12 +76,15 @@ spec:
                 name: account-balance-uploader-config
           - secretRef:
               name: uploader-secrets
+  {{- end }}
 
-        # Event Stream Uploader
+  {{- if $node.sidecars.eventStreamUploader }}
+      # Event Stream Uploader
       - name: event-stream-uploader
         image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
         securityContext:
-          runAsUser: {{ $.Values.mirror.user }}
+          runAsUser: {{ $.Values.hedera.user }}
+          runAsGroup: {{ $.Values.hedera.group }}
         command:
           - /usr/bin/env
           - python3.7
@@ -95,12 +100,15 @@ spec:
               name: event-stream-uploader-config
           - secretRef:
               name: uploader-secrets
+  {{- end }}
 
-        # Record Stream Uploader
+  {{- if $node.sidecars.recordStreamUploader }}
+      # Record Stream Uploader
       - name: record-stream-uploader
         image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
         securityContext:
-          runAsUser: {{ $.Values.mirror.user }}
+          runAsUser: {{ $.Values.hedera.user }}
+          runAsGroup: {{ $.Values.hedera.group }}
         command:
           - /usr/bin/env
           - python3.7
@@ -118,12 +126,15 @@ spec:
               name: record-stream-uploader-config
           - secretRef:
               name: uploader-secrets
+  {{- end }}
 
+  {{- if $node.sidecars.recordSidecarUploader }}
       # Record Stream Sidecar Uploader
       - name: record-sidecar-uploader
         image: {{ $.Values.infrastructure.docker.images.mirroruploader }}
         securityContext:
-          runAsUser: {{ $.Values.mirror.user }}
+          runAsUser: {{ $.Values.hedera.user }}
+          runAsGroup: {{ $.Values.hedera.group }}
         command:
           - /usr/bin/env
           - python3.7
@@ -139,12 +150,28 @@ spec:
               name: record-stream-sidecar-config
           - secretRef:
               name: uploader-secrets
-  {{ end }}
-
-  {{ if $.Values.sidecars.otel.enabled }}
+  {{- end }}
 
+  {{- if $node.sidecars.otel }}
       # OTel collector side car
       {{- include "sidecars.otel-collector" $ | nindent 6 }}
-  {{ end }}
+  {{- end }}
 
+  {{- if $node.sidecars.backupUploader }}
+      # Backup Uploader
+      - name: backup-uploader
+        image: {{ $.Values.infrastructure.docker.images.backupUploader }}
+        securityContext:
+          runAsUser: {{ $.Values.hedera.user }}
+          runAsGroup: {{ $.Values.hedera.user }}
+        volumeMounts:
+          - name: hgcapp-storage
+            mountPath: /opt/hgcapp/
+            readOnly: true
+        envFrom:
+          - configMapRef:
+              name: backup-config
+          - secretRef:
+              name: backup-secrets
+  {{- end }}
 {{ end }}

charts/hedera-network/values.yaml

@@ -1,10 +1,11 @@
 hedera:
+  user: 2000
+  group: 2000
   # TODO: do we need an easy way (to avoid repetition) to create multiple notes if they all have the same configuration?
   nodes:
     - name: node-1
       # TODO: Configuration like address ( and similar ), can be autogenerated by helm, do we need to think about this ?
       address: 0.0.1
-      cpu: 1
       requests:
         cpu: 1
         memory: 1G
@@ -13,7 +14,13 @@ hedera:
         memory: 1G
       latency: 10ms
       bandwidth: 1 Mbps
-      uploaderSidecars: true
+      sidecars:
+        otel: true
+        backupUploader: true
+        accountBalancesUploader: true
+        eventStreamUploader: true
+        recordStreamUploader: true
+        recordSidecarUploader: true
     - name: node-2
       address: 0.0.2
       requests:
@@ -24,7 +31,13 @@ hedera:
         memory: 1G
       latency: 10ms
       bandwidth: 2 Mbps
-      uploaderSidecars: true
+      sidecars:
+        otel: true
+        backupUploader: true
+        accountBalancesUploader: true
+        eventStreamUploader: true
+        recordStreamUploader: true
+        recordSidecarUploader: true
     - name: node-3
       address: 0.0.3
       requests:
@@ -35,7 +48,13 @@ hedera:
         memory: 1G
       latency: 10ms
       bandwidth: 3 Mbps
-      uploaderSidecars: true
+      sidecars:
+        otel: true
+        backupUploader: true
+        accountBalancesUploader: true
+        eventStreamUploader: true
+        recordStreamUploader: true
+        recordSidecarUploader: true
 
 infrastructure:
   docker:
@@ -46,8 +65,7 @@ infrastructure:
       haproxy: haproxy:lts-alpine3.18
       jsonrpcrelay: jsonrpcrelay
       mirroruploader: gcr.io/hedera-registry/uploader-mirror:1.3.0
-      streamuploader: ubuntu
-      backupuploader: ubuntu
+      backupUploader: gcr.io/hedera-registry/hedera-backups:0.6.0
 
 mirror:
   # TODO: Use accurate folder names
@@ -69,7 +87,6 @@ mirror:
     watchDir: /opt/hgcapp/events
     bucketPath: /events
     bucketName: "dev"
-  user: 2000
   debug: true
   reaper:
     enable: true
@@ -93,7 +110,6 @@ mirror:
 # sidecar configurations
 sidecars:
   otel:
-    enabled: true
     image:
       repository: "otel/opentelemetry-collector-contrib"
       tag: "0.72.0"
@@ -123,3 +139,6 @@ sidecars:
         mountPath: /etc/otel-collector-config.yaml
         subPath: config.yaml #key in the configmap
         readOnly: true
+  backup:
+    bucket1: backup-1
+    bucket2: backup-2