Migrate image to fluentd.image and sumologic.setup.job.image (#1148)

.travis.yml

@@ -7,10 +7,10 @@ before_install:
   - rm -f ./Gemfile.lock
   - gem install bundler
   # yq
-  - curl --retry 10 --retry-max-time 120 --retry-delay 5 -L --remote-name https://github.com/mikefarah/yq/releases/download/3.2.1/yq_linux_amd64
-  - sudo mv yq_linux_amd64 /usr/local/bin/yq-3.2.1
-  - sudo chmod +x /usr/local/bin/yq-3.2.1
-  - sudo ln -s /usr/local/bin/yq-3.2.1 /usr/local/bin/yq
+  - curl --retry 10 --retry-max-time 120 --retry-delay 5 -L --remote-name https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux_amd64
+  - sudo mv yq_linux_amd64 /usr/local/bin/yq-3.4.1
+  - sudo chmod +x /usr/local/bin/yq-3.4.1
+  - sudo ln -s /usr/local/bin/yq-3.4.1 /usr/local/bin/yq
   # shellcheck
   - curl --retry 10 --retry-max-time 120 --retry-delay 5 -Lo- https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz | tar -xJf -
   - sudo cp shellcheck-v0.7.1/shellcheck /usr/local/bin && rm -rf shellcheck-v0.7.1

ci/tests.sh

@@ -69,6 +69,10 @@ echo "Test helm templates generation"
 echo "Test upgrade script..."
 ./tests/upgrade_script/run.sh || (echo "Failed testing upgrade script" && exit 1)
 
+# Test upgrade v2 script
+echo "Test upgrade v2 script..."
+./tests/upgrade_v2_script/run.sh || (echo "Failed testing upgrade v2 script" && exit 1)
+
 # Test fluentd plugins
 test_fluentd_plugins "${VERSION}" || (echo "Failed testing fluentd plugins" && exit 1)
 

deploy/docs/v2_migration_doc.md

@@ -70,6 +70,9 @@ the exact steps for migration.
           - name: "config-reloader"
     ```
 
+- We've separated our fluentd image from setup job image, hence `image` was migrated
+  to `sumologic.setup.job.image` and to `fluentd.image`
+
 ## How to upgrade
 
 **Note: The below steps are using Helm 3. Helm 2 is not supported.**

deploy/helm/sumologic/README.md

@@ -11,9 +11,6 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 
 Parameter | Description | Default
 --- | --- | ---
-`image.repository` | Image repository for Sumo Logic docker container. | `sumologic/kubernetes-fluentd`
-`image.tag` | Image tag for Sumo Logic docker container. | `1.0.0-rc.2`
-`image.pullPolicy` | Image pullPolicy for Sumo Logic docker container. | `IfNotPresent`
 `nameOverride` | Used to override the Chart name. | `Nil`
 `sumologic.setupEnabled` | If enabled, a pre-install hook will create Collector and Sources in Sumo Logic. | `true`
 `sumologic.cleanUpEnabled` | If enabled, a pre-delete hook will destroy Kubernetes secret and Sumo Logic Collector. | `false`
@@ -39,7 +36,13 @@ Parameter | Description | Default
 `sumologic.setup.job.annotations` | Annotations for the Job. | `[{"helm.sh/hook":"pre-install,pre-upgrade","helm.sh/hook-delete-policy":"before-hook-creation,hook-succeeded","helm.sh/hook-weight":"3"}]`
 `sumologic.setup.job.podLabels` | Additional labels for the setup Job pod. | `{}`
 `sumologic.setup.job.podAnnotations` | Additional annotations for the setup Job pod. | `{}`
+`sumologic.setup.job.image.repository` | Image repository for Sumo Logic setup job docker container. | `sumologic/kubernetes-fluentd`
+`sumologic.setup.job.image.tag` | Image tag for Sumo Logic setup job docker container. | `1.3.0`
+`sumologic.setup.job.image.pullPolicy` | Image pullPolicy for Sumo Logic docker container. | `IfNotPresent`
 `sumologic.setup.serviceAccount.annotations` | Annotations for the ServiceAccount. | `[{"helm.sh/hook":"pre-install,pre-upgrade","helm.sh/hook-delete-policy":"before-hook-creation,hook-succeeded","helm.sh/hook-weight":"0"}]`
+`fluentd.image.repository` | Image repository for Sumo Logic docker container. | `sumologic/kubernetes-fluentd`
+`fluentd.image.tag` | Image tag for Sumo Logic docker container. | `1.3.0`
+`fluentd.image.pullPolicy` | Image pullPolicy for Sumo Logic docker container. | `IfNotPresent`
 `fluentd.additionalPlugins` | Additional Fluentd plugins to install from RubyGems. Please see our [documentation](./Additional_Fluentd_Plugins.md) for more information. | `[]`
 `fluentd.compression.enabled` | Flag to control if data is sent to Sumo Logic compressed or not | `true`
 `fluentd.compression.encoding` | Specifies which encoding should be used to compress data (either `gzip` or `deflate`) | `gzip`

deploy/helm/sumologic/templates/cleanup/cleanup-job.yaml

@@ -40,8 +40,8 @@ spec:
             mountPath: /etc/terraform
       containers:
         - name: cleanup
-          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          image: {{ .Values.sumologic.setup.job.image.repository }}:{{ .Values.sumologic.setup.job.image.tag }}
+          imagePullPolicy: {{ .Values.sumologic.setup.job.image.pullPolicy }}
           volumeMounts:
           - name: cleanup
             mountPath: /etc/terraform

deploy/helm/sumologic/templates/events-statefulset.yaml

@@ -61,8 +61,8 @@ spec:
       {{- end }}
       containers:
       - name: fluentd-events
-        image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ .Values.fluentd.image.repository }}:{{ .Values.fluentd.image.tag }}
+        imagePullPolicy: {{ .Values.fluentd.image.pullPolicy }}
         resources:
           {{- toYaml .Values.fluentd.events.statefulset.resources | nindent 10 }}
         volumeMounts:

deploy/helm/sumologic/templates/metrics-statefulset.yaml

@@ -101,8 +101,8 @@ spec:
       {{- end }}
       containers:
       - name: fluentd
-        image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ .Values.fluentd.image.repository }}:{{ .Values.fluentd.image.tag }}
+        imagePullPolicy: {{ .Values.fluentd.image.pullPolicy }}
         resources:
           {{- toYaml .Values.fluentd.metrics.statefulset.resources | nindent 10 }}
         ports:

deploy/helm/sumologic/templates/setup/setup-job.yaml

@@ -43,8 +43,8 @@ spec:
           defaultMode: 0777
       containers:
       - name: setup
-        image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ .Values.sumologic.setup.job.image.repository }}:{{ .Values.sumologic.setup.job.image.tag }}
+        imagePullPolicy: {{ .Values.sumologic.setup.job.image.pullPolicy }}
         command: ["/etc/terraform/setup.sh"]
         resources:
           {{- toYaml .Values.sumologic.setup.job.resources | nindent 10 }}

deploy/helm/sumologic/templates/statefulset.yaml

@@ -101,8 +101,8 @@ spec:
       {{- end }}
       containers:
       - name: fluentd
-        image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ .Values.fluentd.image.repository }}:{{ .Values.fluentd.image.tag }}
+        imagePullPolicy: {{ .Values.fluentd.image.pullPolicy }}
         resources:
           {{- toYaml .Values.fluentd.logs.statefulset.resources | nindent 10 }}
         ports:

deploy/helm/sumologic/upgrade-1.0.0.sh

@@ -11,7 +11,7 @@ readonly PREVIOUS_VERSION=0.17
 readonly TEMP_FILE=upgrade-1.0.0-temp-file
 
 readonly MIN_BASH_VERSION=4.0
-readonly MIN_YQ_VERSION=3.2.1
+readonly MIN_YQ_VERSION=3.4.1
 
 readonly KEY_MAPPINGS="
 eventsDeployment.nodeSelector:fluentd.events.statefulset.nodeSelector
@@ -435,7 +435,7 @@ prometheus-operator:
             - ${NAMESPACE}
         selector:
           matchLabels:
-            app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-events" | yq m -a -i "${TEMP_FILE}" -
+            app: ${HELM_RELEASE_NAME}-${NAMESPACE}-fluentd-events" | yq m -a=append -i "${TEMP_FILE}" -
   fi
 
   if [[ -n "$(yq r "${TEMP_FILE}" -- prometheus-operator.prometheus.prometheusSpec.containers)" ]]; then

deploy/helm/sumologic/upgrade-2.0.0.sh

@@ -18,6 +18,12 @@ prometheus-operator.prometheusOperator.tlsProxy.enabled:kube-prometheus-stack.pr
 readonly KEY_VALUE_MAPPINGS="
 "
 
+readonly KEY_MAPPINGS_MULTIPLE="
+image.repository:fluentd.image.repository:sumologic.setup.job.image.repository
+image.tag:fluentd.image.tag:sumologic.setup.job.image.tag
+image.pullPolicy:fluentd.image.pullPolicy:sumologic.setup.job.image.pullPolicy
+"
+
 readonly KEYS_TO_DELETE="
 prometheus-operator
 "
@@ -114,6 +120,11 @@ function create_temp_file() {
 }
 
 function migrate_prometheus_operator_to_kube_prometheus_stack() {
+  # Nothing to migrate, return
+  if [[ -z $(yq r "${TEMP_FILE}" prometheus-operator) ]] ; then
+    return
+  fi
+
   info "Migrating prometheus-config-reloader container to config-reloader in prometheusSpec"
   yq m -i --arrays append \
     "${TEMP_FILE}" \
@@ -152,6 +163,8 @@ function migrate_customer_keys() {
   readonly MAPPINGS
   IFS=$'\n' read -r -d ' ' -a MAPPINGS_KEY_VALUE <<< "${KEY_VALUE_MAPPINGS}"
   readonly MAPPINGS_KEY_VALUE
+  IFS=$'\n' read -r -d ' ' -a MAPPINGS_MULTIPLE <<< "${KEY_MAPPINGS_MULTIPLE}"
+  readonly MAPPINGS_MULTIPLE
   set -e
 
   readonly CUSTOMER_KEYS=$(yq --printMode p r "${OLD_VALUES_YAML}" -- '**')
@@ -167,6 +180,19 @@ function migrate_customer_keys() {
           yq d -i "${TEMP_FILE}" -- "${maps[0]}"
         fi
       done
+    elif [[ ${MAPPINGS_MULTIPLE[*]} =~ ${key}: ]]; then
+      # whatever you want to do when arr contains value
+      info "Mapping ${key} into:"
+      for i in "${MAPPINGS_MULTIPLE[@]}"; do
+        IFS=':' read -r -a maps <<< "${i}"
+        if [[ ${maps[0]} == "${key}" ]]; then
+          for element in "${maps[@]:1}"; do
+            info "- ${element}"
+            yq w -i "${TEMP_FILE}" -- "${element}" "$(yq r "${OLD_VALUES_YAML}" -- "${maps[0]}")"
+            yq d -i "${TEMP_FILE}" -- "${maps[0]}"
+          done
+        fi
+      done
     else
       yq w -i "${TEMP_FILE}" -- "${key}" "$(yq r "${OLD_VALUES_YAML}" -- "${key}")"
     fi
@@ -195,7 +221,6 @@ function migrate_pre_upgrade_hook() {
   fi
 }
 
-
 function get_regex() {
     # Get regex from old yaml file and strip `'` and `"` from beginning/end of it
     local write_index="${1}"
@@ -208,8 +233,10 @@ function check_user_image() {
   readonly USER_VERSION="$(yq r "${OLD_VALUES_YAML}" -- image.tag)"
   if [[ -n "${USER_VERSION}" ]]; then
     if [[ "${USER_VERSION}" =~ ^"${PREVIOUS_VERSION}"\.[[:digit:]]+$ ]]; then
-      yq w -i "${TEMP_FILE}" -- image.tag 2.0.0
-      info "Changing image.tag from '${USER_VERSION}' to '2.0.0'"
+      info "Migrating from image.tag '${USER_VERSION}' to sumologic.setup.job.image.tag '2.0.0'"
+      yq w -i "${TEMP_FILE}" -- sumologic.setup.job.image.tag 2.0.0
+      info "Migrating from image.tag '${USER_VERSION}' to fluentd.image.tag '2.0.0'"
+      yq w -i "${TEMP_FILE}" -- fluentd.image.tag 2.0.0
     else
       warning "You are using unsupported version: ${USER_VERSION}"
       warning "Please upgrade to '${PREVIOUS_VERSION}.x' or ensure that new_values.yaml is valid"

deploy/helm/sumologic/values.yaml

@@ -1,11 +1,6 @@
 ## Sumo Logic Kubernetes Collection configuration file
 ## All the comments start with two or more # characters
 
-image:
-  repository: sumologic/kubernetes-fluentd
-  tag: 1.3.0
-  pullPolicy: IfNotPresent
-
 nameOverride: ""
 
 sumologic:
@@ -91,6 +86,10 @@ sumologic:
 
   setup:
     job:
+      image:
+        repository: sumologic/kubernetes-fluentd
+        tag: 1.3.0
+        pullPolicy: IfNotPresent
       resources:
         limits:
           memory: 256Mi
@@ -195,6 +194,11 @@ sumologic:
     spans_per_request: 100
 
 fluentd:
+  image:
+    repository: sumologic/kubernetes-fluentd
+    tag: 1.3.0
+    pullPolicy: IfNotPresent
+
   ## Specifies whether a PodSecurityPolicy should be created
   podSecurityPolicy:
     create: false

tests/run.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 readonly SCRIPT_PATH="$( dirname "$(realpath "${0}")" )"
 readonly CONFIG_FILES=$(find "${SCRIPT_PATH}"/* -maxdepth 1 -name 'config.sh')

tests/upgrade_script/static/chart_envs.output.yaml

@@ -1,30 +1,30 @@
 fluent-bit:
   env:
-  - name: CHART
-    valueFrom:
-      configMapKeyRef:
-        name: sumologic-configmap
-        key: fluentdLogs
-  - name: NAMESPACE
-    valueFrom:
-      fieldRef:
-        fieldPath: metadata.namespace
+    - name: CHART
+      valueFrom:
+        configMapKeyRef:
+          name: sumologic-configmap
+          key: fluentdLogs
+    - name: NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
 prometheus-operator:
   prometheus:
     prometheusSpec:
       resources: {}
       thanos:
         version: v0.10.0
       containers:
-      - name: prometheus-config-reloader
-        env:
-        - name: CHART
-          valueFrom:
-            configMapKeyRef:
-              name: sumologic-configmap
-              key: fluentdMetrics
-        - name: NAMESPACE
-          valueFrom:
-            configMapKeyRef:
-              name: sumologic-configmap
-              key: fluentdNamespace
+        - name: prometheus-config-reloader
+          env:
+            - name: CHART
+              valueFrom:
+                configMapKeyRef:
+                  name: sumologic-configmap
+                  key: fluentdMetrics
+            - name: NAMESPACE
+              valueFrom:
+                configMapKeyRef:
+                  name: sumologic-configmap
+                  key: fluentdNamespace

tests/upgrade_script/static/key_mapping_one_to_multiple.output.yaml

@@ -17,8 +17,7 @@ fluentd:
       enabled: fluentd.logs.autoscaling.enabled, fluentd.metrics.autoscaling.enabled
       maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas
       minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas
-      targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage,
-        fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
+      targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
   metrics:
     statefulset:
       affinity: fluentd.logs.statefulset.affinity, fluentd.metrics.statefulset.affinity
@@ -37,5 +36,4 @@ fluentd:
       enabled: fluentd.logs.autoscaling.enabled, fluentd.metrics.autoscaling.enabled
       maxReplicas: fluentd.logs.autoscaling.maxReplicas, fluentd.metrics.autoscaling.maxReplicas
       minReplicas: fluentd.logs.autoscaling.minReplicas, fluentd.metrics.autoscaling.minReplicas
-      targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage,
-        fluentd.metrics.autoscaling.targetCPUUtilizationPercentage
+      targetCPUUtilizationPercentage: fluentd.logs.autoscaling.targetCPUUtilizationPercentage, fluentd.metrics.autoscaling.targetCPUUtilizationPercentage