Support PVC for worker tiered store in K8s

This resolves worker tiered storage using PVC, as point 4 in #10253 The PVC will be satisfied by either `hostPath` or `local` under the hood to provide local storage to the worker. pr-link: #11171 change-id: cid-413d3440e21a05ae2a33efbe573deeae9beafeba
Alluxio · Mar 17, 2020 · 988059f · 988059f
1 parent c19e218
commit 988059f
Show file tree

Hide file tree

Showing 3 changed files with 104 additions and 16 deletions.
diff --git a/docs/en/deploy/Running-Alluxio-On-Kubernetes.md b/docs/en/deploy/Running-Alluxio-On-Kubernetes.md
@@ -247,6 +247,9 @@ Alluxio manages local storage, including memory, on the worker Pods.
 [Multiple-Tier Storage]({{ '/en/core-services/Caching.html#multiple-tier-storage' | relativize_url }})
 can be configured using the following reference configurations.
 
+There 3 supported volume `type`: [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath), [emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir) 
+and [persistentVolumeClaim](https://kubernetes.io/docs/concepts/storage/volumes/#persistentvolumeclaim).
+
 **Memory Tier Only**
 
 ```properties
@@ -279,20 +282,58 @@ tieredstore:
     low: 0.7
 ```
 
-> There 2 supported volume `type`: [hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath) and [emptyDir](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir)
-`hostPath` volumes can only be used by the `root` user without resource limits.
-[Local persistent volumes](https://kubernetes.io/docs/concepts/storage/volumes/#local) instead of
-`hostPath` must be configured manually as of now.
+> Note: If a `hostPath` file or directory is created at runtime, it can only be used by the `root` user.
+`hostPath` volumes do not have resource limits. 
+You can either run Alluxio containers with `root` or make sure the local paths exist and are accessible to 
+the user `alluxio` with UID and GID 1000. 
+You can find more details [here](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath).
+
+**Memory and SSD Storage in Multiple-Tiers, using PVC**
+
+You can also use PVCs for each tier and provision [PersistentVolume](https://kubernetes.io/docs/concepts/storage/persistent-volumes/).
+For worker tiered storage please use either `hostPath` or `local` volume so that the worker will read and write 
+locally to achieve the best performance.
+
+```properties
+tieredstore:
+  levels:
+  - level: 0
+    mediumtype: MEM
+    path: /dev/shm
+    type: persistentVolumeClaim
+    name: alluxio-mem
+    quota: 1G
+    high: 0.95
+    low: 0.7
+  - level: 1
+    mediumtype: SSD
+    path: /ssd-disk
+    type: persistentVolumeClaim
+    name: alluxio-ssd
+    quota: 10G
+    high: 0.95
+    low: 0.7
+```
+
+> Note: There is one PVC per tier. 
+When the PVC is bound to a PV of type `hostPath` or `local`, each worker Pod will resolve to the local path on the Node.
+Please also note that a `local` volumes requires `nodeAffinity` and Pods using this volume can only run on the Nodes 
+specified in the `nodeAffinity` rule of this volume.
+You can find more details [here](https://kubernetes.io/docs/concepts/storage/volumes/#local).
 
 **Memory and SSD Storage in a Single-Tier**
 
+You can also have multiple volumes on the same tier.
+This configuration will create one `persistentVolumeClaim` for each volume.
+
 ```properties
 tieredstore:
   levels:
   - level: 0
     mediumtype: MEM,SSD
     path: /dev/shm,/alluxio-ssd
-    type: hostPath
+    type: persistentVolumeClaim
+    name: alluxio-mem,alluxio-ssd
     quota: 1GB,10GB
     high: 0.95
     low: 0.7

diff --git a/integration/kubernetes/helm-chart/alluxio/templates/_helpers.tpl b/integration/kubernetes/helm-chart/alluxio/templates/_helpers.tpl
@@ -160,17 +160,19 @@ resources:
 {{- define "alluxio.worker.tieredstoreVolumeMounts" -}}
   {{- if .Values.tieredstore.levels }}
     {{- range .Values.tieredstore.levels }}
+      {{- /* The mediumtype can have multiple parts like MEM,SSD */}}
       {{- if .mediumtype }}
+        {{- /* Mount each part */}}
         {{- if contains "," .mediumtype }}
-{{- $type := .type }}
-{{- $path := .path }}
-{{- $split := split "," .mediumtype }}
+          {{- $type := .type }}
+          {{- $path := .path }}
+          {{- $split := split "," .mediumtype }}
           {{- range $key, $val := $split }}
-            {{- if eq $type "hostPath"}}
-            - mountPath:  {{ index ($path | split ",") $key }}
-              name: {{ $val | lower }}-{{ $key | replace "_" "" }}
-            {{- end}}
+            {{- /* Example: For path="/tmp/mem,/tmp/ssd", mountPath resolves to /tmp/mem and /tmp/ssd */}}
+            - mountPath: {{ index ($path | split ",") $key }}
+              name: {{ $val | lower }}-{{ replace $key "_" "" }}
           {{- end}}
+        {{- /* The mediumtype is a single value. */}}
         {{- else}}
             - mountPath: {{ .path }}
               name: {{ .mediumtype | replace "," "-" | lower }}
@@ -191,33 +193,49 @@ resources:
   {{- if .Values.tieredstore.levels }}
     {{- range .Values.tieredstore.levels }}
       {{- if .mediumtype }}
+        {{- /* The mediumtype can have multiple parts like MEM,SSD */}}
         {{- if contains "," .mediumtype }}
           {{- $split := split "," .mediumtype }}
           {{- $type := .type }}
           {{- $path := .path }}
+          {{- $volumeName := .name }}
+          {{- /* A volume will be generated for each part */}}
           {{- range $key, $val := $split }}
+            {{- /* Example: For mediumtype="MEM,SSD", mediumName resolves to mem-0 and ssd-1 */}}
+            {{- $mediumName := printf "%v-%v" (lower $val) (replace $key "_" "") }}
             {{- if eq $type "hostPath"}}
         - hostPath:
             path: {{ index ($path | split ",") $key }}
             type: DirectoryOrCreate
-          name: {{ $val | lower }}-{{ $key | replace "_" "" }}
+          name: {{ $mediumName }}
+            {{- else if eq $type "persistentVolumeClaim" }}
+        - name: {{ $mediumName }}
+          persistentVolumeClaim:
+            {{- /* Example: For volumeName="/tmp/mem,/tmp/ssd", claimName resolves to /tmp/mem and /tmp/ssd */}}
+            claimName: {{ index ($volumeName | split ",") $key }}
             {{- else }}
-        - name: {{ $val | lower }}-{{ $key | replace "_" "" }}
+        - name: {{ $mediumName }}
           emptyDir:
             medium: "Memory"
               {{- if .quota }}
             sizeLimit: {{ .quota }}
               {{- end}}
             {{- end}}
           {{- end}}
+        {{- /* The mediumtype is a single value like MEM. */}}
         {{- else}}
+          {{- $mediumName := .mediumtype | lower }}
           {{- if eq .type "hostPath"}}
         - hostPath:
             path: {{ .path }}
             type: DirectoryOrCreate
-          name: {{ .mediumtype | replace "," "-" | lower }}
+          name: {{ $mediumName }}
+          {{- else if eq .type "persistentVolumeClaim" }}
+        - name: {{ $mediumName }}
+          persistentVolumeClaim:
+            claimName: {{ .name }}
           {{- else }}
-        - name: {{ .mediumtype | replace "," "-" | lower }}
+        - name: {{ $mediumName }}
           emptyDir:
             medium: "Memory"
             {{- if .quota }}

diff --git a/integration/kubernetes/helm-chart/alluxio/values.yaml b/integration/kubernetes/helm-chart/alluxio/values.yaml
@@ -134,6 +134,35 @@ journal:
 ## Worker ##
 
 # Tiered Storage
+# emptyDir example
+#  - level: 0
+#    mediumtype: MEM
+#    path: /dev/shm
+#    type: emptyDir
+#    quota: 1G
+#
+# hostPath example
+#  - level: 0
+#    mediumtype: MEM
+#    path: /dev/shm
+#    type: hostPath
+#    quota: 1G
+#
+# persistentVolumeClaim example
+#  - level: 1
+#    mediumtype: SSD
+#    type: persistentVolumeClaim
+#    name: alluxio-ssd
+#    path: /dev/ssd
+#    quota: 10G
+#
+# multi-part mediumtype example
+#  - level: 1
+#    mediumtype: SSD,HDD
+#    type: persistentVolumeClaim
+#    name: alluxio-ssd,alluxio-hdd
+#    path: /dev/ssd,/dev/hdd
+#    quota: 10G,10G
 tieredstore:
   levels:
   - level: 0