From a961ad5b91633e80918e2332d2ce24905ba3219c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 11:51:02 -0500
Subject: [PATCH 01/21] add st2.packs.volumes section to volumes.yaml

---
 values.yaml | 50 ++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/values.yaml b/values.yaml
index f8fe2eb0..118d5bf0 100644
--- a/values.yaml
+++ b/values.yaml
@@ -73,9 +73,11 @@ st2:
 
   # Custom pack configs and image settings.
   #
-  # By default, system packs are available. However, since 'st2 pack install' cannot be run in the k8s cluster,
-  # you will need to bake additional packs into an 'st2packs' image. Please see github.com/stackstorm/stackstorm-ha/README.md
+  # By default, system packs are available. By default, however, `st2 pack install` cannot be run in the k8s cluster,
+  # so you will need to bake additional packs into an 'st2packs' image. Please see github.com/stackstorm/stackstorm-ha/README.md
   # for details on how to build this image.
+  # To change this default, and use persistent/shared/writable storage that is available in your cluster, you need to
+  # enable st2.packs.volumes below, adding volume definitions customized for use your cluster's storage provider.
   packs:
     # Custom StackStorm pack configs. Each record creates a file in '/opt/stackstorm/configs/'
     # https://docs.stackstorm.com/reference/pack_configs.html#configuration-file
@@ -85,9 +87,12 @@ st2:
         # example core pack config yaml
 
     # Custom packs images settings.
+    #
     # For each given st2packs container you can define repository, name, tag and pullPolicy for this image below.
     # Multiple pack images can help when dealing with frequent updates by only rebuilding smaller images for desired packs
     # E.g. having all desired StackStorm-Exchange packs in one image and several custom packs in additional images
+    #
+    # This must be empty if st2.packs.volumes is enabled.
     images:
       #- repository: index.docker.io/stackstorm
       #  name: st2packs
@@ -96,6 +101,47 @@ st2:
       # Optional name of the imagePullSecret if your custom packs image is hosted by a private Docker registry
       #  pullSecret: st2packs-auth
 
+    # Custom packs volumes definitions.
+    #
+    # Use this instead of st2.packs.images to have StackStorm use persistent/shared/writable storage configured
+    # previously in your cluster. The choice of storage solution is cluster-dependent (it changes besed on where the
+    # cluster is hosted and which storage solutions are available in your cluster).
+    #
+    # To use this, set enabled to true, and add cluster-specific volume definitions for at least packs and virtualenvs below.
+    # Please consult the documentation for your cluster's storage solution.
+    # Some generic examples are listed under st2.packs.volumes.packs below.
+    volumes:
+      # to enable st2.packs.volumes, st2.packs.images must not be empty
+      enabled: false
+
+      packs: # mounted to /opt/stackstorm/packs
+        # packs volume definition is required if st2.packs.volumes is enabled
+
+        # example using persistentVolumeClaim:
+        #persistentVolumeClaim:
+        #  claim-name: pvc-st2-packs
+
+        # example using NFS:
+        #nfs:
+        #  server: "10.12.34.56"
+        #  path: /var/nfsshare/packs
+
+        # example using a flexVolume + rook-ceph
+        #flexVolume:
+        #  driver: ceph.rook.io/rook
+        #  options:
+        #    fsName: fs1
+        #    clusterNamespace: rook-ceph
+        #    path: /st2/packs
+
+      virtualenvs: # mounted to /opt/stackstorm/virtualenvs
+        # virtualenvs volume definition is required if st2.packs.volumes is enabled
+        # see the examples under st2.packs.volumes.packs
+
+      configs: # mounted to /opt/stackstorm/configs
+        # configs volume definition is optional, but only used if st2.packs.volumes is enabled
+        # see the examples under st2.packs.volumes.packs
+
     # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer
     # It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance.
     # Each sensor node needs to be provided with proper partition information to share work with other sensor

From 51960b2012fca90eece5eef9bb6ffdd9fff7f414 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 12:06:08 -0500
Subject: [PATCH 02/21] consolidate packs-volume-mounts into templates

---
 templates/_helpers.tpl     | 19 ++++++++++++++
 templates/deployments.yaml | 52 ++++++--------------------------------
 templates/jobs.yaml        |  7 +----
 3 files changed, 28 insertions(+), 50 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index c4ea9c3a..21fbf5fa 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -129,6 +129,25 @@ Create the name of the stackstorm-ha service account to use
   emptyDir: {}
   {{- end }}
 {{- end -}}
+{{- define "packs-volume-mounts" -}}
+  {{- if .Values.st2.packs.images }}
+- name: st2-packs-vol
+  mountPath: /opt/stackstorm/packs
+  readOnly: true
+- name: st2-virtualenvs-vol
+  mountPath: /opt/stackstorm/virtualenvs
+  readOnly: true
+  {{- end }}
+{{- end -}}
+# define this here as well to simplify comparison with packs-volume-mounts
+{{- define "packs-volume-mounts-for-register-job" -}}
+  {{- if .Values.st2.packs.images }}
+- name: st2-packs-vol
+  mountPath: /opt/stackstorm/packs
+- name: st2-virtualenvs-vol
+  mountPath: /opt/stackstorm/virtualenvs
+  {{- end }}
+{{- end -}}
 
 # For custom st2packs-initContainers reduce duplicity by defining them here once
 # Merge packs and virtualenvs from st2 with those from st2packs images
diff --git a/templates/deployments.yaml b/templates/deployments.yaml
index 55d3354c..2013f76b 100644
--- a/templates/deployments.yaml
+++ b/templates/deployments.yaml
@@ -200,14 +200,7 @@ spec:
           mountPath: /etc/st2/keys
           readOnly: true
         {{- end }}
-        {{- if .Values.st2.packs.images }}
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs
-          readOnly: true
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs
-          readOnly: true
-        {{- end }}
+        {{- include "packs-volume-mounts" . | nindent 8 }}
         {{- if .Values.st2api.postStartScript }}
         - name: st2-post-start-script-vol
           mountPath: /post-start.sh
@@ -232,9 +225,7 @@ spec:
               path: datastore_key.json
         {{- end }}
         {{- include "st2-config-volume" . | nindent 8 }}
-        {{- if .Values.st2.packs.images }}
-{{- include "packs-volumes" . | indent 8 }}
-        {{- end }}
+        {{- include "packs-volumes" . | nindent 8 }}
         {{- if .Values.st2api.postStartScript }}
         - name: st2-post-start-script-vol
           configMap:
@@ -1064,14 +1055,7 @@ spec:
             name: {{ $.Release.Name }}-st2-urls
         volumeMounts:
         {{- include "st2-config-volume-mounts" $ | nindent 8 }}
-        {{- if $.Values.st2.packs.images }}
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs
-          readOnly: true
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs
-          readOnly: true
-        {{- end }}
+        {{- include "packs-volume-mounts" $ | nindent 8 }}
         {{- if $.Values.st2.datastore_crypto_key }}
         - name: st2-encryption-key-vol
           mountPath: /etc/st2/keys
@@ -1101,9 +1085,7 @@ spec:
               path: datastore_key.json
         {{- end }}
         {{- include "st2-config-volume" $ | nindent 8 }}
-        {{- if $.Values.st2.packs.images }}
-{{- include "packs-volumes" $ | indent 8 }}
-        {{- end }}
+        {{- include "packs-volumes" $ | nindent 8 }}
         {{- if $.Values.st2sensorcontainer.postStartScript }}
         - name: st2-post-start-script-vol
           configMap:
@@ -1203,14 +1185,7 @@ spec:
           mountPath: /etc/st2/keys
           readOnly: true
         {{- end }}
-        {{- if .Values.st2.packs.images }}
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs
-          readOnly: true
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs
-          readOnly: true
-        {{- end }}
+        {{- include "packs-volume-mounts" . | nindent 8 }}
         - name: st2-post-start-script-vol
           mountPath: /post-start.sh
           subPath: post-start.sh
@@ -1241,9 +1216,7 @@ spec:
               path: {{ tpl .Values.st2.system_user.ssh_key_file . | base }}
               # 0400 file permission
               mode: 256
-        {{- if .Values.st2.packs.images }}
-{{- include "packs-volumes" . | indent 8 }}
-        {{- end }}
+        {{- include "packs-volumes" . | nindent 8 }}
         - name: st2-post-start-script-vol
           configMap:
             name: {{ .Release.Name }}-st2actionrunner-post-start-script
@@ -1471,14 +1444,7 @@ spec:
           mountPath: /etc/st2/keys
           readOnly: true
         {{- end }}
-        {{- if .Values.st2.packs.images }}
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs
-          readOnly: true
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs
-          readOnly: true
-        {{- end }}
+        {{- include "packs-volume-mounts" . | nindent 8 }}
         - name: st2-post-start-script-vol
           mountPath: /post-start.sh
           subPath: post-start.sh
@@ -1529,9 +1495,7 @@ spec:
               path: {{ tpl .Values.st2.system_user.ssh_key_file . | base }}
               # 0400 file permission
               mode: 256
-        {{- if .Values.st2.packs.images }}
-{{- include "packs-volumes" . | indent 8 }}
-        {{- end }}
+        {{- include "packs-volumes" . | nindent 8 }}
         - name: st2-post-start-script-vol
           configMap:
             name: {{ .Release.Name }}-st2client-post-start-script
diff --git a/templates/jobs.yaml b/templates/jobs.yaml
index 6320cc64..7db568a4 100644
--- a/templates/jobs.yaml
+++ b/templates/jobs.yaml
@@ -408,12 +408,7 @@ spec:
         {{- include "st2-config-volume-mounts" . | nindent 8 }}
         - name: st2-pack-configs-vol
           mountPath: /opt/stackstorm/configs/
-        {{- if .Values.st2.packs.images }}
-        - name: st2-packs-vol
-          mountPath: /opt/stackstorm/packs/
-        - name: st2-virtualenvs-vol
-          mountPath: /opt/stackstorm/virtualenvs/
-        {{- end }}
+        {{- include "packs-volume-mounts-for-register-job" . | nindent 8 }}
         # TODO: Find out default resource limits for this specific service (#5)
         #resources:
       volumes:

From 7ac2ed5a12cc338f78772ed8babea8dcd15342ed Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 12:12:27 -0500
Subject: [PATCH 03/21] add st2.packs.volumes definitions to packs-volumes
 templates

---
 templates/_helpers.tpl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 21fbf5fa..0084438d 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -127,6 +127,11 @@ Create the name of the stackstorm-ha service account to use
   emptyDir: {}
 - name: st2-virtualenvs-vol
   emptyDir: {}
+  {{- else if .Values.st2.packs.volumes.enabled }}
+- name: st2-packs-vol
+{{ toYaml .Values.st2.packs.volumes.packs | indent 2 }}
+- name: st2-virtualenvs-vol
+{{ toYaml .Values.st2.packs.volumes.virtualenvs | indent 2 }}
   {{- end }}
 {{- end -}}
 {{- define "packs-volume-mounts" -}}
@@ -137,11 +142,16 @@ Create the name of the stackstorm-ha service account to use
 - name: st2-virtualenvs-vol
   mountPath: /opt/stackstorm/virtualenvs
   readOnly: true
+  {{- else if .Values.st2.packs.volumes.enabled }}
+- name: st2-packs-vol
+  mountPath: /opt/stackstorm/packs
+- name: st2-virtualenvs-vol
+  mountPath: /opt/stackstorm/virtualenvs
   {{- end }}
 {{- end -}}
 # define this here as well to simplify comparison with packs-volume-mounts
 {{- define "packs-volume-mounts-for-register-job" -}}
-  {{- if .Values.st2.packs.images }}
+  {{- if or .Values.st2.packs.images .Values.st2.packs.volumes.enabled }}
 - name: st2-packs-vol
   mountPath: /opt/stackstorm/packs
 - name: st2-virtualenvs-vol

From 0e603ad87c46cfd48db41c409028b46d34fb976f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 12:26:38 -0500
Subject: [PATCH 04/21] consolidate pack-configs-volume into templates

---
 templates/_helpers.tpl     | 11 +++++++++++
 templates/deployments.yaml |  7 ++-----
 templates/jobs.yaml        |  9 +++------
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 0084438d..9b8f4b97 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -120,6 +120,17 @@ Create the name of the stackstorm-ha service account to use
   {{- end }}
 {{- end -}}
 
+# consolidate pack-configs-volumes definitions
+{{- define "pack-configs-volume" -}}
+- name: st2-pack-configs-vol
+  configMap:
+    name: {{ .Release.Name }}-st2-pack-configs
+{{- end -}}
+{{- define "pack-configs-volume-mount" -}}
+- name: st2-pack-configs-vol
+  mountPath: /opt/stackstorm/configs/
+{{- end -}}
+
 # For custom st2packs-Container reduce duplicity by defining it here once
 {{- define "packs-volumes" -}}
   {{- if .Values.st2.packs.images }}
diff --git a/templates/deployments.yaml b/templates/deployments.yaml
index 2013f76b..8baa587c 100644
--- a/templates/deployments.yaml
+++ b/templates/deployments.yaml
@@ -1433,8 +1433,6 @@ spec:
         - name: st2-rbac-mappings-vol
           mountPath: /opt/stackstorm/rbac/mappings/
         {{- end }}
-        - name: st2-pack-configs-vol
-          mountPath: /opt/stackstorm/configs/
         - name: st2client-config-vol
           mountPath: /root/.st2/
         - name: st2-ssh-key-vol
@@ -1445,6 +1443,7 @@ spec:
           readOnly: true
         {{- end }}
         {{- include "packs-volume-mounts" . | nindent 8 }}
+        {{- include "pack-configs-volume-mount" . | nindent 8 }}
         - name: st2-post-start-script-vol
           mountPath: /post-start.sh
           subPath: post-start.sh
@@ -1481,9 +1480,6 @@ spec:
           configMap:
             name: {{ .Release.Name }}-st2-rbac-mappings
         {{- end }}
-        - name: st2-pack-configs-vol
-          configMap:
-            name: {{ .Release.Name }}-st2-pack-configs
         - name: st2client-config-vol
           emptyDir:
             medium: Memory
@@ -1496,6 +1492,7 @@ spec:
               # 0400 file permission
               mode: 256
         {{- include "packs-volumes" . | nindent 8 }}
+        {{- include "pack-configs-volume" . | nindent 8 }}
         - name: st2-post-start-script-vol
           configMap:
             name: {{ .Release.Name }}-st2client-post-start-script
diff --git a/templates/jobs.yaml b/templates/jobs.yaml
index 7db568a4..8a7d9391 100644
--- a/templates/jobs.yaml
+++ b/templates/jobs.yaml
@@ -406,17 +406,14 @@ spec:
           - --register-fail-on-failure
         volumeMounts:
         {{- include "st2-config-volume-mounts" . | nindent 8 }}
-        - name: st2-pack-configs-vol
-          mountPath: /opt/stackstorm/configs/
         {{- include "packs-volume-mounts-for-register-job" . | nindent 8 }}
+        {{- include "pack-configs-volume-mount" . | nindent 8 }}
         # TODO: Find out default resource limits for this specific service (#5)
         #resources:
       volumes:
         {{- include "st2-config-volume" . | nindent 8 }}
-        - name: st2-pack-configs-vol
-          configMap:
-            name: {{ .Release.Name }}-st2-pack-configs
-        {{- include "packs-volumes" $ | nindent 8 }}
+        {{- include "packs-volumes" . | nindent 8 }}
+        {{- include "pack-configs-volume" . | nindent 8 }}
       restartPolicy: OnFailure
     {{- if .Values.dnsPolicy }}
       dnsPolicy: {{ .Values.dnsPolicy }}

From c7f50cd64270dc6d76b3439958cdbc389fb8bade Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 13:48:18 -0500
Subject: [PATCH 05/21] add pack-configs-volume to more pods when
 st2.packs.volumes.enabled

---
 templates/_helpers.tpl     |  5 +++++
 templates/deployments.yaml | 12 ++++++++++++
 values.yaml                |  1 +
 3 files changed, 18 insertions(+)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 9b8f4b97..0f272540 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -122,9 +122,14 @@ Create the name of the stackstorm-ha service account to use
 
 # consolidate pack-configs-volumes definitions
 {{- define "pack-configs-volume" -}}
+  {{- if and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs }}
+- name: st2-pack-configs-vol
+{{ toYaml .Values.st2.packs.volumes.configs | indent 2 }}
+  {{- else }}
 - name: st2-pack-configs-vol
   configMap:
     name: {{ .Release.Name }}-st2-pack-configs
+  {{- end }}
 {{- end -}}
 {{- define "pack-configs-volume-mount" -}}
 - name: st2-pack-configs-vol
diff --git a/templates/deployments.yaml b/templates/deployments.yaml
index 8baa587c..1526e066 100644
--- a/templates/deployments.yaml
+++ b/templates/deployments.yaml
@@ -201,6 +201,9 @@ spec:
           readOnly: true
         {{- end }}
         {{- include "packs-volume-mounts" . | nindent 8 }}
+        {{- if .Values.st2.packs.volumes.enabled }}
+          {{- include "pack-configs-volume-mount" . | nindent 8 }}
+        {{- end }}
         {{- if .Values.st2api.postStartScript }}
         - name: st2-post-start-script-vol
           mountPath: /post-start.sh
@@ -226,6 +229,9 @@ spec:
         {{- end }}
         {{- include "st2-config-volume" . | nindent 8 }}
         {{- include "packs-volumes" . | nindent 8 }}
+        {{- if .Values.st2.packs.volumes.enabled }}
+          {{- include "pack-configs-volume" . | nindent 8 }}
+        {{- end }}
         {{- if .Values.st2api.postStartScript }}
         - name: st2-post-start-script-vol
           configMap:
@@ -1186,6 +1192,9 @@ spec:
           readOnly: true
         {{- end }}
         {{- include "packs-volume-mounts" . | nindent 8 }}
+        {{- if .Values.st2.packs.volumes.enabled }}
+          {{- include "pack-configs-volume-mount" . | nindent 8 }}
+        {{- end }}
         - name: st2-post-start-script-vol
           mountPath: /post-start.sh
           subPath: post-start.sh
@@ -1217,6 +1226,9 @@ spec:
               # 0400 file permission
               mode: 256
         {{- include "packs-volumes" . | nindent 8 }}
+        {{- if .Values.st2.packs.volumes.enabled }}
+          {{- include "pack-configs-volume" . | nindent 8 }}
+        {{- end }}
         - name: st2-post-start-script-vol
           configMap:
             name: {{ .Release.Name }}-st2actionrunner-post-start-script
diff --git a/values.yaml b/values.yaml
index 118d5bf0..7960a587 100644
--- a/values.yaml
+++ b/values.yaml
@@ -81,6 +81,7 @@ st2:
   packs:
     # Custom StackStorm pack configs. Each record creates a file in '/opt/stackstorm/configs/'
     # https://docs.stackstorm.com/reference/pack_configs.html#configuration-file
+    # NOTE: This is ignored if st2.packs.volumes.configs is defined and st2.packs.volumes is enabled
     configs:
       core.yaml: |
         ---

From e0ae60ae9235203b08e6334415eb2b6f45c22d55 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 13:48:46 -0500
Subject: [PATCH 06/21] add error messages if st2.packs.volumes config is
 incomplete

---
 templates/deployments.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/templates/deployments.yaml b/templates/deployments.yaml
index 1526e066..93a0fe1a 100644
--- a/templates/deployments.yaml
+++ b/templates/deployments.yaml
@@ -2,6 +2,14 @@
 {{- if and .Values.st2.packs.image }}
 {{- fail "Value st2.packs.image was renamed to st2.packs.images and is now a list of images" }}
 {{- end }}
+{{- if .Values.st2.packs.volumes.enabled }}
+  {{- if .Values.st2.packs.images }}
+{{- fail "st2.packs.images is not compatible with st2.packs.volumes.enabled. Please use only one method for setting up packs directories." }}
+  {{- end }}
+  {{- if not (and .Values.st2.packs.volumes.packs .Values.st2.packs.volumes.virtualenvs) }}
+{{- fail "Volume definition(s) missing! When st2.packs.volumes.enabled, you must define volumes for both packs and virtualenvs." }}
+  {{- end }}
+{{- end }}
 
 ---
 apiVersion: apps/v1

From 86d4793173adacae51a3f55f566f3a738c6dbe3c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 13:58:22 -0500
Subject: [PATCH 07/21] When st2.packs.volumes.enabled, make register-content
 packs-initContainer include system packs

---
 templates/_helpers.tpl | 2 ++
 templates/jobs.yaml    | 4 +---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 0f272540..98fd4728 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -195,6 +195,8 @@ Create the name of the stackstorm-ha service account to use
       /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
       /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
     {{- end }}
+  {{- end }}
+  {{- if or $.Values.st2.packs.images $.Values.st2.packs.volumes.enabled }}
 # System packs
 - name: st2-system-packs
   image: '{{ template "imageRepository" . }}/st2actionrunner:{{ tpl (.Values.st2actionrunner.image.tag | default .Values.image.tag) . }}'
diff --git a/templates/jobs.yaml b/templates/jobs.yaml
index 8a7d9391..c4250785 100644
--- a/templates/jobs.yaml
+++ b/templates/jobs.yaml
@@ -374,9 +374,7 @@ spec:
       {{- end }}
       initContainers:
       {{- include "init-containers-wait-for-db" . | nindent 6 }}
-      {{- if $.Values.st2.packs.images -}}
-        {{- include "packs-initContainers" . | nindent 6 }}
-      {{ end }}
+      {{- include "packs-initContainers" . | nindent 6 }}
       {{- if $.Values.jobs.preRegisterContentCommand }}
       - name: st2-register-content-custom-init
         image: '{{ template "imageRepository" . }}/st2actionrunner:{{ tpl (.Values.jobs.image.tag | default (.Values.st2actionrunner.image.tag | default .Values.image.tag)) . }}'

From bd72f5108b0be1f120b25885587a8cd2a917dd25 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 15:12:09 -0500
Subject: [PATCH 08/21] disable packs.configs ConfigMap if it is not
 needed/used

---
 templates/configmaps_packs.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/templates/configmaps_packs.yaml b/templates/configmaps_packs.yaml
index b87e9998..4802138b 100644
--- a/templates/configmaps_packs.yaml
+++ b/templates/configmaps_packs.yaml
@@ -1,3 +1,4 @@
+{{- if not (and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs) -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -14,3 +15,4 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
 {{ toYaml .Values.st2.packs.configs | indent 2 }}
+{{- end -}}

From 0f1c9c3801ac2761be29eb32830f0e96fe305cb2 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 15:23:44 -0500
Subject: [PATCH 09/21] add instructions for st2.packs.volumes values

---
 README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 77 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 29251c51..eea83cca 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,7 @@ st2:
       - name: circleci
         ref: circle_ci.CircleCIWebhookSensor
 ```
-	
+
 ### [st2actionrunner](https://docs.stackstorm.com/reference/ha.html#st2actionrunner)
 Stackstorm workers that actually execute actions.
 `5` replicas for K8s Deployment are configured by default to increase StackStorm ability to execute actions without excessive queuing.
@@ -181,15 +181,23 @@ StackStorm employs redis sentinel as a distributed coordination backend, require
 As any other Helm dependency, it's possible to further configure it for specific scaling needs via `values.yaml`.
 
 ## Install custom st2 packs in the cluster
-In distributed environment of the Kubernetes cluster `st2 pack install` won’t work.
+There are two ways to install st2 packs in the k8s cluster.
+
+1. The `st2packs` method is the default. This method will work for practically all clusters, but `st2 pack install` does not work. The packs are injected via `st2packs` images instead.
+
+2. The other method defines shared/writable `volumes`. This method allows `st2 pack install` to work, but requires a persistent storage backend to be available in the cluster. This chart will not configure a storage backend for you.
+
+### Method 1: st2packs images (the default)
+The `st2packs` method is the default. `st2 pack install` does not work because this chart uses read-only `emptyDir` volumes for `/opt/stackstorm/{packs,virtualenvs}`.
 Instead, you need to bake the packs into a custom docker image, push it to a private or public docker registry and reference that image in Helm values.
-Helm chart will take it from there, sharing `/opt/stackstorm/{packs,virtualenvs}` via a sidecar container in pods which require access to the packs.
+Helm chart will take it from there, sharing `/opt/stackstorm/{packs,virtualenvs}` via a sidecar container in pods which require access to the packs
+(the sidecar is the only place where the volumes are writable).
 
-### Building st2packs image
+#### Building st2packs image
 For your convenience, we created a new `st2-pack-install <pack1> <pack2> <pack3>` utility and included it in a container that will help to install custom packs during the Docker build process without relying on live DB and MQ connection.
 Please see https://github.com/StackStorm/st2packs-dockerfiles/ for instructions on how to build your custom `st2packs` image.
 
-### How to provide custom pack configs
+#### How to provide custom pack configs
 Update the `st2.packs.configs` section of Helm values:
 
 For example:
@@ -205,7 +213,7 @@ For example:
 ```
 Don't forget running Helm upgrade to apply new changes.
 
-### Pull st2packs from a private Docker registry
+#### Pull st2packs from a private Docker registry
 If you need to pull your custom packs Docker image from a private repository, create a Kubernetes Docker registry secret and pass it to Helm values.
 See [K8s documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) for more info.
 ```
@@ -214,6 +222,69 @@ kubectl create secret docker-registry st2packs-auth --docker-server=<your-regist
 ```
 Once secret created, reference its name in helm value: `st2.packs.images[].pullSecret`.
 
+### Method 2: Shared Volumes
+This method requires cluster-specific storage setup and configuration. As the storage volumes are both writable and shared, `st2 pack install` should work like it does for standalone StackStorm installations. The volumes get mounted at `/opt/stackstorm/{packs,virtualenvs}` in the containers that need read or write access to those directories. With this method, `/opt/stackstorm/configs` can also be mounted as a writable volume instead of using `st2.packs.configs`.
+
+#### Configure the storage volumes
+Enable the `st2.packs.voluems` section of Helm values and add volume definitions for both `packs` and `virtualenvs`.
+Each of the volume definitions should be customized for your cluster and storage solution.
+
+NOTE: Make sure that `st2.packs.images` is empty. This method does not use or work with the `st2packs` images.
+
+For example, to use persistentVolumeClaims:
+```
+  volumes:
+    enabled: true
+    packs:
+      persistentVolumeClaim:
+        claim-name: pvc-st2-packs
+    virtualenvs:
+      persistentVolumeClaim:
+        claim-name: pvc-st2-virtualenvs
+```
+
+Or, for example, to use NFS:
+```
+  volumes:
+    enabled: true
+    packs:
+      nfs:
+        server: nfs.example.com
+        path: /var/nfsshare/packs
+    virtualenvs:
+      nfs:
+        server: nfs.example.com
+        path: /var/nfsshare/virtualenvs
+```
+
+Please consult the documentation for your cluster's storage solution to see how to add the storage backend to your cluster and how to define volumes that use your storage backend.
+
+#### How to provide custom pack configs
+You may either use the `st2.packs.configs` section of Helm values (like Method 1, see above),
+or add another shared writable volume similar to `packs` and `virtualenvs`. This volume gets mounted
+to `/opt/stackstorm/configs` instead of the `st2.packs.config` values. NOTE: If you define a configs volume,
+anything in `st2.packs.configs` will NOT be visible to StackStorm.
+
+For example, to use persistentVolumeClaims:
+```
+  volumes:
+    enabled: true
+    ... # define packs and virtualenvs volumes as shown above
+    configs:
+      persistentVolumeClaim:
+        claim-name: pvc-st2-pack-configs
+```
+
+Or, for example, to use NFS:
+```
+  volumes:
+    enabled: true
+    ... # define packs and virtualenvs volumes as shown above
+    configs:
+      nfs:
+        server: nfs.example.com
+        path: /var/nfsshare/configs
+```
 
 ## Tips & Tricks
 Grab all logs for entire StackStorm cluster with dependent services in Helm release:

From 2e2b7e93b8f99283a77d32a95c1d6cd3c9b68d47 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 15:51:50 -0500
Subject: [PATCH 10/21] add st2.packs.volumes caveats similar to those
 documented in #118

---
 README.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/README.md b/README.md
index eea83cca..1ec99482 100644
--- a/README.md
+++ b/README.md
@@ -286,6 +286,23 @@ Or, for example, to use NFS:
         path: /var/nfsshare/configs
 ```
 
+#### Caveat: Mounting and copying packs
+If you use something like NFS where you can mount the shares outside of the StackStorm pods, there are a couple of things to keep in mind.
+
+Though you could manually copy packs into the `packs` shared volume, be aware that StackStorm does not automatically register any changed content.
+So, if you manually copy a pack into the `packs` shared volume, then you also need to trigger updating the virtualenv and registering the content,
+possibly using APIs like:
+[packs/install](https://api.stackstorm.com/api/v1/packs/#/packs_controller.install.post), and
+[packs/register](https://api.stackstorm.com/api/v1/packs/#/packs_controller.register.post)
+You will have to repeat the process each time the packs code is modified.
+
+#### Caveat: System packs
+After Helm installs, upgrades, or rolls back a StackStorm install, it runs an `st2-register-content` batch job.
+This job will copy and register system packs. If you have made any changes (like disabling default aliases), those changes will be overwritten.
+
+NOTE: Upgrades will not remove files (such as a renamed or removed action) if they were removed in newer StackStorm versions.
+This mirrors the how pack registration works. Make sure to review any upgrade notes and manually handle any removals.
+
 ## Tips & Tricks
 Grab all logs for entire StackStorm cluster with dependent services in Helm release:
 ```

From 808a7d6b009e4afa62854ee630826fcc6bd6f81d Mon Sep 17 00:00:00 2001
From: Eric Reeves <eric@alluvium.com>
Date: Sat, 19 Jun 2021 07:07:25 -0500
Subject: [PATCH 11/21] Populate default empty maps for st2.packs.images,
 st2.packs.volumes.(packs|virtualenvs|configmaps)

---
 values.yaml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/values.yaml b/values.yaml
index 7960a587..943aa4f7 100644
--- a/values.yaml
+++ b/values.yaml
@@ -94,7 +94,7 @@ st2:
     # E.g. having all desired StackStorm-Exchange packs in one image and several custom packs in additional images
     #
     # This must be empty if st2.packs.volumes is enabled.
-    images:
+    images: {}
       #- repository: index.docker.io/stackstorm
       #  name: st2packs
       #  tag: example
@@ -115,7 +115,8 @@ st2:
       # to enable st2.packs.volumes, st2.packs.images must not be empty
       enabled: false
 
-      packs: # mounted to /opt/stackstorm/packs
+      packs: {}
+        # mounted to /opt/stackstorm/packs
         # packs volume definition is required if st2.packs.volumes is enabled
 
         # example using persistentVolumeClaim:
@@ -135,11 +136,13 @@ st2:
         #    clusterNamespace: rook-ceph
         #    path: /st2/packs
 
-      virtualenvs: # mounted to /opt/stackstorm/virtualenvs
+      virtualenvs: {}
+        # mounted to /opt/stackstorm/virtualenvs
         # virtualenvs volume definition is required if st2.packs.volumes is enabled
         # see the examples under st2.packs.volumes.packs
 
-      configs: # mounted to /opt/stackstorm/configs
+      configs: {}
+        # mounted to /opt/stackstorm/configs
         # configs volume definition is optional, but only used if st2.packs.volumes is enabled
         # see the examples under st2.packs.volumes.packs
 

From 6e638d0907a1ffdfcef74515f0c282164f77a789 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 19 Jun 2021 22:03:25 -0500
Subject: [PATCH 12/21] st2.packs.images is an array

---
 values.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/values.yaml b/values.yaml
index 943aa4f7..e8b0c8bd 100644
--- a/values.yaml
+++ b/values.yaml
@@ -94,7 +94,7 @@ st2:
     # E.g. having all desired StackStorm-Exchange packs in one image and several custom packs in additional images
     #
     # This must be empty if st2.packs.volumes is enabled.
-    images: {}
+    images: []
       #- repository: index.docker.io/stackstorm
       #  name: st2packs
       #  tag: example

From b80a058c62ec74cd6c4a4f46d32d2182c2a72fa3 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Thu, 17 Jun 2021 17:38:08 -0500
Subject: [PATCH 13/21] add changelog entry

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1335db50..32505171 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@
 * Make system_user configurable when using custom st2actionrunner images that do not provide stanley (#220) (by @cognifloyd)
 * Allow providing scripts in values for use in lifecycle postStart hooks of all deployments. (#206) (by @cognifloyd)
 * Add preRegisterContentCommand in an initContainer for register-content job to run last-minute content customizations (#213) (by @cognifloyd)
+* New feature: Shared packs volumes `st2.packs.volumes` -- Instead of using `st2packs` images to install packs, allow using cluster-specific persistent volumes to store packs, virtualenvs, and (optionally) configs. This enables using `st2 pack install`. (#199) (by @cognifloyd)
 
 ## v0.60.0
 * Switch st2 version to `v3.5dev` as a new latest development version (#187)

From 48873610cc57517d54c4c8cd29d460fe78f02d31 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 10 Jul 2021 11:21:58 -0500
Subject: [PATCH 14/21] allow using st2.packs.volumes with st2.packs.images

---
 templates/deployments.yaml | 11 ++++-------
 values.yaml                |  9 +++------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/templates/deployments.yaml b/templates/deployments.yaml
index 93a0fe1a..62fe1469 100644
--- a/templates/deployments.yaml
+++ b/templates/deployments.yaml
@@ -3,9 +3,6 @@
 {{- fail "Value st2.packs.image was renamed to st2.packs.images and is now a list of images" }}
 {{- end }}
 {{- if .Values.st2.packs.volumes.enabled }}
-  {{- if .Values.st2.packs.images }}
-{{- fail "st2.packs.images is not compatible with st2.packs.volumes.enabled. Please use only one method for setting up packs directories." }}
-  {{- end }}
   {{- if not (and .Values.st2.packs.volumes.packs .Values.st2.packs.volumes.virtualenvs) }}
 {{- fail "Volume definition(s) missing! When st2.packs.volumes.enabled, you must define volumes for both packs and virtualenvs." }}
   {{- end }}
@@ -186,7 +183,7 @@ spec:
       initContainers:
       {{- include "init-containers-wait-for-db" . | nindent 6 }}
       {{- include "init-containers-wait-for-mq" . | nindent 6 }}
-      {{- if .Values.st2.packs.images }}
+      {{- if and .Values.st2.packs.images (not .Values.st2.packs.volumes.enabled) }}
         {{- include "packs-initContainers" . | nindent 6 }}
       {{- end }}
       containers:
@@ -1039,7 +1036,7 @@ spec:
       initContainers:
       {{- include "init-containers-wait-for-db" $ | nindent 6 }}
       {{- include "init-containers-wait-for-mq" $ | nindent 6 }}
-      {{- if $.Values.st2.packs.images }}
+      {{- if and $.Values.st2.packs.images (not $.Values.st2.packs.volumes.enabled) }}
         {{- include "packs-initContainers" $ | nindent 6 }}
       {{- end }}
       containers:
@@ -1177,7 +1174,7 @@ spec:
       initContainers:
       {{- include "init-containers-wait-for-db" . | nindent 6 }}
       {{- include "init-containers-wait-for-mq" . | nindent 6 }}
-      {{- if .Values.st2.packs.images }}
+      {{- if and .Values.st2.packs.images (not .Values.st2.packs.volumes.enabled) }}
         {{- include "packs-initContainers" . | nindent 6 }}
       {{- end }}
       containers:
@@ -1399,7 +1396,7 @@ spec:
       - name: {{ .Values.image.pullSecret }}
       {{- end }}
       initContainers:
-      {{- if .Values.st2.packs.images }}
+      {{- if and .Values.st2.packs.images (not .Values.st2.packs.volumes.enabled) }}
         {{- include "packs-initContainers" . | nindent 6 }}
       {{- end }}
       # Sidecar container for generating st2client config with st2 username & password pair and sharing produced file with the main container
diff --git a/values.yaml b/values.yaml
index e8b0c8bd..59cbb7a8 100644
--- a/values.yaml
+++ b/values.yaml
@@ -92,8 +92,6 @@ st2:
     # For each given st2packs container you can define repository, name, tag and pullPolicy for this image below.
     # Multiple pack images can help when dealing with frequent updates by only rebuilding smaller images for desired packs
     # E.g. having all desired StackStorm-Exchange packs in one image and several custom packs in additional images
-    #
-    # This must be empty if st2.packs.volumes is enabled.
     images: []
       #- repository: index.docker.io/stackstorm
       #  name: st2packs
@@ -104,15 +102,14 @@ st2:
 
     # Custom packs volumes definitions.
     #
-    # Use this instead of st2.packs.images to have StackStorm use persistent/shared/writable storage configured
-    # previously in your cluster. The choice of storage solution is cluster-dependent (it changes besed on where the
-    # cluster is hosted and which storage solutions are available in your cluster).
+    # Use this to have StackStorm use persistent/shared/writable storage configured previously in your cluster.
+    # The choice of storage solution is cluster-dependent (it changes besed on where the cluster is hosted
+    # and which storage solutions are available in your cluster).
     #
     # To use this, set enabled to true, and add cluster-specific volume definitions for at least packs and virtualenvs below.
     # Please consult the documentation for your cluster's storage solution.
     # Some generic examples are listed under st2.packs.volumes.packs below.
     volumes:
-      # to enable st2.packs.volumes, st2.packs.images must not be empty
       enabled: false
 
       packs: {}

From 1bf2ae9c1523a686678f5c5cec4a2849eb6ef7ed Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 10 Jul 2021 11:51:35 -0500
Subject: [PATCH 15/21] update changelog entry

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 32505171..1a96002c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,7 +18,7 @@
 * Make system_user configurable when using custom st2actionrunner images that do not provide stanley (#220) (by @cognifloyd)
 * Allow providing scripts in values for use in lifecycle postStart hooks of all deployments. (#206) (by @cognifloyd)
 * Add preRegisterContentCommand in an initContainer for register-content job to run last-minute content customizations (#213) (by @cognifloyd)
-* New feature: Shared packs volumes `st2.packs.volumes` -- Instead of using `st2packs` images to install packs, allow using cluster-specific persistent volumes to store packs, virtualenvs, and (optionally) configs. This enables using `st2 pack install`. (#199) (by @cognifloyd)
+* New feature: Shared packs volumes `st2.packs.volumes`. Allow using cluster-specific persistent volumes to store packs, virtualenvs, and (optionally) configs. This enables using `st2 pack install`. It even works with `st2packs` images in `st2.packs.images`. (#199) (by @cognifloyd)
 
 ## v0.60.0
 * Switch st2 version to `v3.5dev` as a new latest development version (#187)

From 7be8ebc17030c1e0b3100ea411cbba7e8d999752 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 10 Jul 2021 13:39:11 -0500
Subject: [PATCH 16/21] explain using volumes with st2packs images in README

---
 README.md | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 1ec99482..32b9d227 100644
--- a/README.md
+++ b/README.md
@@ -187,8 +187,10 @@ There are two ways to install st2 packs in the k8s cluster.
 
 2. The other method defines shared/writable `volumes`. This method allows `st2 pack install` to work, but requires a persistent storage backend to be available in the cluster. This chart will not configure a storage backend for you.
 
+NOTE: In general, we recommend using only one of these methods. See the NOTE under Method 2 below about how both methods can be used together with care.
+
 ### Method 1: st2packs images (the default)
-The `st2packs` method is the default. `st2 pack install` does not work because this chart uses read-only `emptyDir` volumes for `/opt/stackstorm/{packs,virtualenvs}`.
+The `st2packs` method is the default. `st2 pack install` does not work because this chart (by default) uses read-only `emptyDir` volumes for `/opt/stackstorm/{packs,virtualenvs}`.
 Instead, you need to bake the packs into a custom docker image, push it to a private or public docker registry and reference that image in Helm values.
 Helm chart will take it from there, sharing `/opt/stackstorm/{packs,virtualenvs}` via a sidecar container in pods which require access to the packs
 (the sidecar is the only place where the volumes are writable).
@@ -213,6 +215,8 @@ For example:
 ```
 Don't forget running Helm upgrade to apply new changes.
 
+NOTE: `st2.packs.configs` will be ignored if you use `st2packs` images with `volumes.configs` (optional part of Method 2, described below).
+
 #### Pull st2packs from a private Docker registry
 If you need to pull your custom packs Docker image from a private repository, create a Kubernetes Docker registry secret and pass it to Helm values.
 See [K8s documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) for more info.
@@ -225,12 +229,13 @@ Once secret created, reference its name in helm value: `st2.packs.images[].pullS
 ### Method 2: Shared Volumes
 This method requires cluster-specific storage setup and configuration. As the storage volumes are both writable and shared, `st2 pack install` should work like it does for standalone StackStorm installations. The volumes get mounted at `/opt/stackstorm/{packs,virtualenvs}` in the containers that need read or write access to those directories. With this method, `/opt/stackstorm/configs` can also be mounted as a writable volume instead of using `st2.packs.configs`.
 
+NOTE: With care, `st2packs` images can be used with `volumes`. Just make sure to keep the `st2packs` images up-to-date with any changes made via `st2 pack install`.
+If a pack is installed via an `st2packs` image and then it gets updated with `st2 pack install`, a subsequent `helm upgrade` will revert back to the version in the `st2packs` image.
+
 #### Configure the storage volumes
 Enable the `st2.packs.voluems` section of Helm values and add volume definitions for both `packs` and `virtualenvs`.
 Each of the volume definitions should be customized for your cluster and storage solution.
 
-NOTE: Make sure that `st2.packs.images` is empty. This method does not use or work with the `st2packs` images.
-
 For example, to use persistentVolumeClaims:
 ```
   volumes:
@@ -262,8 +267,9 @@ Please consult the documentation for your cluster's storage solution to see how
 #### How to provide custom pack configs
 You may either use the `st2.packs.configs` section of Helm values (like Method 1, see above),
 or add another shared writable volume similar to `packs` and `virtualenvs`. This volume gets mounted
-to `/opt/stackstorm/configs` instead of the `st2.packs.config` values. NOTE: If you define a configs volume,
-anything in `st2.packs.configs` will NOT be visible to StackStorm.
+to `/opt/stackstorm/configs` instead of the `st2.packs.config` values.
+
+NOTE: If you define a configs volume, anything in `st2.packs.configs` will NOT be visible to StackStorm.
 
 For example, to use persistentVolumeClaims:
 ```

From 938ee9e92beec38220093de48284ef9d1f032441 Mon Sep 17 00:00:00 2001
From: Eric Reeves <eric@alluvium.com>
Date: Sat, 10 Jul 2021 12:25:42 -0500
Subject: [PATCH 17/21] Check for volumes.enabled before checking for images

---
 templates/_helpers.tpl | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 98fd4728..cd56f6aa 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -138,31 +138,31 @@ Create the name of the stackstorm-ha service account to use
 
 # For custom st2packs-Container reduce duplicity by defining it here once
 {{- define "packs-volumes" -}}
-  {{- if .Values.st2.packs.images }}
-- name: st2-packs-vol
-  emptyDir: {}
-- name: st2-virtualenvs-vol
-  emptyDir: {}
-  {{- else if .Values.st2.packs.volumes.enabled }}
+  {{- if .Values.st2.packs.volumes.enabled }}
 - name: st2-packs-vol
 {{ toYaml .Values.st2.packs.volumes.packs | indent 2 }}
 - name: st2-virtualenvs-vol
 {{ toYaml .Values.st2.packs.volumes.virtualenvs | indent 2 }}
+  {{- else if .Values.st2.packs.images }}
+- name: st2-packs-vol
+  emptyDir: {}
+- name: st2-virtualenvs-vol
+  emptyDir: {}
   {{- end }}
 {{- end -}}
 {{- define "packs-volume-mounts" -}}
-  {{- if .Values.st2.packs.images }}
+  {{- if .Values.st2.packs.volumes.enabled }}
 - name: st2-packs-vol
   mountPath: /opt/stackstorm/packs
-  readOnly: true
 - name: st2-virtualenvs-vol
   mountPath: /opt/stackstorm/virtualenvs
-  readOnly: true
-  {{- else if .Values.st2.packs.volumes.enabled }}
+  {{- else if .Values.st2.packs.images }}
 - name: st2-packs-vol
   mountPath: /opt/stackstorm/packs
+  readOnly: true
 - name: st2-virtualenvs-vol
   mountPath: /opt/stackstorm/virtualenvs
+  readOnly: true
   {{- end }}
 {{- end -}}
 # define this here as well to simplify comparison with packs-volume-mounts

From a9e582f9520706c72524eae6e96c480ad814b55e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 10 Jul 2021 13:52:52 -0500
Subject: [PATCH 18/21] Allow st2.packs.configs to work with
 st2.packs.volumes.configs

---
 README.md                       |  6 +++---
 templates/_helpers.tpl          | 25 +++++++++++++++++++++++++
 templates/configmaps_packs.yaml |  4 +---
 values.yaml                     |  3 ++-
 4 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 32b9d227..63cbd550 100644
--- a/README.md
+++ b/README.md
@@ -215,7 +215,7 @@ For example:
 ```
 Don't forget running Helm upgrade to apply new changes.
 
-NOTE: `st2.packs.configs` will be ignored if you use `st2packs` images with `volumes.configs` (optional part of Method 2, described below).
+NOTE: On `helm upgrade` any configs in `st2.packs.configs` will overwrite the contents of `st2.packs.volumes.configs` (optional part of Method 2, described below).
 
 #### Pull st2packs from a private Docker registry
 If you need to pull your custom packs Docker image from a private repository, create a Kubernetes Docker registry secret and pass it to Helm values.
@@ -227,7 +227,7 @@ kubectl create secret docker-registry st2packs-auth --docker-server=<your-regist
 Once secret created, reference its name in helm value: `st2.packs.images[].pullSecret`.
 
 ### Method 2: Shared Volumes
-This method requires cluster-specific storage setup and configuration. As the storage volumes are both writable and shared, `st2 pack install` should work like it does for standalone StackStorm installations. The volumes get mounted at `/opt/stackstorm/{packs,virtualenvs}` in the containers that need read or write access to those directories. With this method, `/opt/stackstorm/configs` can also be mounted as a writable volume instead of using `st2.packs.configs`.
+This method requires cluster-specific storage setup and configuration. As the storage volumes are both writable and shared, `st2 pack install` should work like it does for standalone StackStorm installations. The volumes get mounted at `/opt/stackstorm/{packs,virtualenvs}` in the containers that need read or write access to those directories. With this method, `/opt/stackstorm/configs` can also be mounted as a writable volume (in which case the contents of `st2.packs.configs` takes precedence on `helm upgrade`).
 
 NOTE: With care, `st2packs` images can be used with `volumes`. Just make sure to keep the `st2packs` images up-to-date with any changes made via `st2 pack install`.
 If a pack is installed via an `st2packs` image and then it gets updated with `st2 pack install`, a subsequent `helm upgrade` will revert back to the version in the `st2packs` image.
@@ -269,7 +269,7 @@ You may either use the `st2.packs.configs` section of Helm values (like Method 1
 or add another shared writable volume similar to `packs` and `virtualenvs`. This volume gets mounted
 to `/opt/stackstorm/configs` instead of the `st2.packs.config` values.
 
-NOTE: If you define a configs volume, anything in `st2.packs.configs` will NOT be visible to StackStorm.
+NOTE: If you define a configs volume and specify `st2.packs.configs`, anything in `st2.packs.configs` takes precdence during `helm upgrade`, overwriting config files already in the volume.
 
 For example, to use persistentVolumeClaims:
 ```
diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index cd56f6aa..f0fccb38 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -125,6 +125,11 @@ Create the name of the stackstorm-ha service account to use
   {{- if and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs }}
 - name: st2-pack-configs-vol
 {{ toYaml .Values.st2.packs.volumes.configs | indent 2 }}
+    {{- if .Values.st2.packs.configs }}
+- name: st2-pack-configs-from-helm-vol
+  configMap:
+    name: {{ .Release.Name }}-st2-pack-configs
+    {{- end }}
   {{- else }}
 - name: st2-pack-configs-vol
   configMap:
@@ -134,6 +139,10 @@ Create the name of the stackstorm-ha service account to use
 {{- define "pack-configs-volume-mount" -}}
 - name: st2-pack-configs-vol
   mountPath: /opt/stackstorm/configs/
+  {{- if and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs .Values.st2.packs.configs }}
+- name: st2-pack-configs-from-helm-vol
+  mountPath: /opt/stackstorm/configs-helm/
+  {{- end }}
 {{- end -}}
 
 # For custom st2packs-Container reduce duplicity by defining it here once
@@ -213,6 +222,22 @@ Create the name of the stackstorm-ha service account to use
       /bin/cp -aR /opt/stackstorm/packs/. /opt/stackstorm/packs-shared &&
       /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared
   {{- end }}
+  {{- if and $.Values.st2.packs.configs $.Values.st2.packs.volumes.enabled }}
+# Pack configs defined in helm values
+- name: st2-pack-configs-from-helm
+  image: '{{ template "imageRepository" . }}/st2actionrunner:{{ tpl (.Values.st2actionrunner.image.tag | default .Values.image.tag) . }}'
+  imagePullPolicy: {{ .Values.image.pullPolicy }}
+  volumeMounts:
+  - name: st2-pack-configs-vol
+    mountPath: /opt/stackstorm/configs-shared
+  - name: st2-pack-configs-from-helm-vol
+    mountPath: /opt/stackstorm/configs
+  command:
+    - 'sh'
+    - '-ec'
+    - |
+      /bin/cp -aR /opt/stackstorm/configs/. /opt/stackstorm/configs-shared &&
+  {{- end }}
 {{- end -}}
 
 
diff --git a/templates/configmaps_packs.yaml b/templates/configmaps_packs.yaml
index 4802138b..c31dd487 100644
--- a/templates/configmaps_packs.yaml
+++ b/templates/configmaps_packs.yaml
@@ -1,11 +1,10 @@
-{{- if not (and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs) -}}
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Release.Name }}-st2-pack-configs
   annotations:
-    description: Custom StackStorm pack configs, shipped in '/opt/stackstorm/configs/'
+    description: StackStorm pack configs defined in helm values, shipped in (or copied to) '/opt/stackstorm/configs/'
   labels:
     app: st2
     tier: backend
@@ -15,4 +14,3 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
 {{ toYaml .Values.st2.packs.configs | indent 2 }}
-{{- end -}}
diff --git a/values.yaml b/values.yaml
index 59cbb7a8..aa05b2a1 100644
--- a/values.yaml
+++ b/values.yaml
@@ -81,7 +81,7 @@ st2:
   packs:
     # Custom StackStorm pack configs. Each record creates a file in '/opt/stackstorm/configs/'
     # https://docs.stackstorm.com/reference/pack_configs.html#configuration-file
-    # NOTE: This is ignored if st2.packs.volumes.configs is defined and st2.packs.volumes is enabled
+    # NOTE: This takes precedence over the contents of st2.packs.volumes.configs (if defined) on helm upgrade.
     configs:
       core.yaml: |
         ---
@@ -141,6 +141,7 @@ st2:
       configs: {}
         # mounted to /opt/stackstorm/configs
         # configs volume definition is optional, but only used if st2.packs.volumes is enabled
+        # Anything in `st2.packs.configs` will be added to this volume automatically on helm install/upgrade.
         # see the examples under st2.packs.volumes.packs
 
     # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer

From f1a39702578dbbc82e9411afe62dff43904684cc Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 10 Jul 2021 14:38:08 -0500
Subject: [PATCH 19/21] improve spacing in templates

---
 templates/_helpers.tpl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index f0fccb38..3622e306 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -124,12 +124,12 @@ Create the name of the stackstorm-ha service account to use
 {{- define "pack-configs-volume" -}}
   {{- if and .Values.st2.packs.volumes.enabled .Values.st2.packs.volumes.configs }}
 - name: st2-pack-configs-vol
-{{ toYaml .Values.st2.packs.volumes.configs | indent 2 }}
-    {{- if .Values.st2.packs.configs }}
+  {{- toYaml .Values.st2.packs.volumes.configs | nindent 2 }}
+  {{-   if .Values.st2.packs.configs }}
 - name: st2-pack-configs-from-helm-vol
   configMap:
     name: {{ .Release.Name }}-st2-pack-configs
-    {{- end }}
+  {{-   end }}
   {{- else }}
 - name: st2-pack-configs-vol
   configMap:
@@ -149,9 +149,9 @@ Create the name of the stackstorm-ha service account to use
 {{- define "packs-volumes" -}}
   {{- if .Values.st2.packs.volumes.enabled }}
 - name: st2-packs-vol
-{{ toYaml .Values.st2.packs.volumes.packs | indent 2 }}
+  {{- toYaml .Values.st2.packs.volumes.packs | nindent 2 }}
 - name: st2-virtualenvs-vol
-{{ toYaml .Values.st2.packs.volumes.virtualenvs | indent 2 }}
+  {{- toYaml .Values.st2.packs.volumes.virtualenvs | nindent 2 }}
   {{- else if .Values.st2.packs.images }}
 - name: st2-packs-vol
   emptyDir: {}

From 21108182a1475f8ef8c87e3a3e2cbe1fb79ce579 Mon Sep 17 00:00:00 2001
From: Eric Reeves <eric@alluvium.com>
Date: Tue, 13 Jul 2021 19:21:24 -0500
Subject: [PATCH 20/21] The trailing '&&' without a second command is bad
 syntax.

---
 templates/_helpers.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
index 3622e306..cc7a9eef 100644
--- a/templates/_helpers.tpl
+++ b/templates/_helpers.tpl
@@ -236,7 +236,7 @@ Create the name of the stackstorm-ha service account to use
     - 'sh'
     - '-ec'
     - |
-      /bin/cp -aR /opt/stackstorm/configs/. /opt/stackstorm/configs-shared &&
+      /bin/cp -aR /opt/stackstorm/configs/. /opt/stackstorm/configs-shared
   {{- end }}
 {{- end -}}
 

From aedf6cf1898c6dc533148997abbadf14863e5ae1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 23 Jul 2021 18:39:32 -0500
Subject: [PATCH 21/21] fix rook-ceph volume example

---
 values.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/values.yaml b/values.yaml
index aa05b2a1..b91f630b 100644
--- a/values.yaml
+++ b/values.yaml
@@ -128,6 +128,7 @@ st2:
         # example using a flexVolume + rook-ceph
         #flexVolume:
         #  driver: ceph.rook.io/rook
+        #  fsType: ceph
         #  options:
         #    fsName: fs1
         #    clusterNamespace: rook-ceph