test minio multi tenant (#41)

.gitignore

@@ -6,3 +6,4 @@ __pycache__
 .nginxconfd
 .env
 mc
+drives.yaml

docs/Minio Multi-Tenant.md

@@ -0,0 +1,129 @@
+# Minio Multi-Tenant
+
+This document describes how to use Minio multi-tenant instead of the single-tenant we used previously.
+
+## Deploy DirectPV
+
+```
+kubectl krew install directpv
+kubectl directpv install --tolerations "cwmc-role=worker:NoSchedule"
+kubectl patch deployment controller -n directpv --type='json' -p='[{"op": "add", "path": "/spec/template/spec/tolerations", "value": [{"key": "cwmc-role", "operator": "Equal", "value": "worker", "effect": "NoSchedule"}]}]'
+```
+
+Wait for directpv pods to be ready
+
+Generate drives.yaml file:
+
+```
+kubectl directpv discover
+```
+
+Review the file and apply it - be careful, this will delete all the data on those drives:
+
+```
+kubectl directpv init drives.yaml --dangerous
+```
+
+## Deploy Minio Operator and example tenants
+
+Set env vars:
+
+```
+GLOBAL_DOMAIN_SUFFIX=".example.com"
+CLUSTER_NAME=
+WORKER_NODE_EXTERNAL_IP=
+CLUSTER_DOMAIN_SUFFIX=".${CLUSTER_NAME}-admin${GLOBAL_DOMAIN_SUFFIX}"
+SIMPLE_TENANT_ROOT_USER=
+SIMPLE_TENANT_ROOT_PASSWORD=
+SIMPLE_TENANT_USER_USER=
+SIMPLE_TENANT_USER_PASSWORD=
+COMPLEX_TENANT_ROOT_USER=
+COMPLEX_TENANT_ROOT_PASSWORD=
+COMPLEX_TENANT_USER_USER=
+COMPLEX_TENANT_USER_PASSWORD=
+```
+
+Set DNS:
+
+```
+cwm-worker-cluster route53 set-cloudwm-obj-subdomain-a-record minio-operator.${CLUSTER_NAME}-admin ${WORKER_NODE_EXTERNAL_IP}
+cwm-worker-cluster route53 set-cloudwm-obj-subdomain-a-record minio-tenant-simple.${CLUSTER_NAME}-admin ${WORKER_NODE_EXTERNAL_IP}
+cwm-worker-cluster route53 set-cloudwm-obj-subdomain-a-record minio-tenant-simple-console.${CLUSTER_NAME}-admin ${WORKER_NODE_EXTERNAL_IP}
+cwm-worker-cluster route53 set-cloudwm-obj-subdomain-a-record minio-tenant-complex.${CLUSTER_NAME}-admin ${WORKER_NODE_EXTERNAL_IP}
+cwm-worker-cluster route53 set-cloudwm-obj-subdomain-a-record minio-tenant-complex-console.${CLUSTER_NAME}-admin ${WORKER_NODE_EXTERNAL_IP}
+```
+
+Create tenants configuration:
+
+```
+echo "
+tenants:
+  - name: simple
+    root_user: $SIMPLE_TENANT_ROOT_USER
+    root_password: $SIMPLE_TENANT_ROOT_PASSWORD
+    storage_class_standard: 'EC:0'
+    domain_suffix: $CLUSTER_DOMAIN_SUFFIX
+    user_user: $SIMPLE_TENANT_USER_USER
+    user_password: $SIMPLE_TENANT_USER_PASSWORD
+  - name: complex
+    root_user: $COMPLEX_TENANT_ROOT_USER
+    root_password: $COMPLEX_TENANT_ROOT_PASSWORD
+    storage_class_standard: 'EC:2'
+    domain_suffix: $CLUSTER_DOMAIN_SUFFIX
+    user_user: $COMPLEX_TENANT_USER_USER
+    user_password: $COMPLEX_TENANT_USER_PASSWORD
+" > helm-minio-mt/values-tenants.yaml
+```
+
+Run the following while connected to the relevant cluster:
+
+```
+kubectl create ns simple
+kubectl create ns complex
+helm dependency update helm-minio-mt
+helm upgrade --install minio-mt helm-minio-mt --namespace minio-operator --create-namespace \
+    --set operator.console.ingress.host=minio-operator${CLUSTER_DOMAIN_SUFFIX} \
+    --set operator.console.ingress.tls[0].hosts[0]=minio-operator${CLUSTER_DOMAIN_SUFFIX} \
+    --set operator.console.ingress.tls[0].secretName=console-tls \
+    --values helm-minio-mt/values-tenants.yaml
+```
+
+## Login to Minio Operator Console
+
+Get the JWT Token:
+
+```
+kubectl -n minio-operator  get secret console-sa-secret -o jsonpath="{.data.token}" | base64 --decode
+```
+
+Login at https://minio-operator.CLUSTER_NAME-admin.cloudwm-obj.com
+
+## Login to the tenants
+
+```
+echo "
+-- Simple --
+
+https://minio-tenant-simple.${CLUSTER_NAME}-admin${GLOBAL_DOMAIN_SUFFIX}
+
+root user:
+${SIMPLE_TENANT_ROOT_USER}
+${SIMPLE_TENANT_ROOT_PASSWORD}
+
+tenant user:
+${SIMPLE_TENANT_USER_USER}
+${SIMPLE_TENANT_USER_PASSWORD}
+
+-- Complex --
+
+https://minio-tenant-complex.${CLUSTER_NAME}-admin${GLOBAL_DOMAIN_SUFFIX}
+
+root user:
+${COMPLEX_TENANT_ROOT_USER}
+${COMPLEX_TENANT_ROOT_PASSWORD}
+
+tenant user:
+${COMPLEX_TENANT_USER_USER}
+${COMPLEX_TENANT_USER_PASSWORD}
+"
+```

helm-minio-mt/.gitignore

@@ -0,0 +1,2 @@
+charts
+values-tenants.yaml

helm-minio-mt/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: operator
+  repository: https://operator.min.io/
+  version: 5.0.7
+digest: sha256:2fcedda1e95f8d42b7f2a8d7f9706cf86dd3915aeb552eb0ca697f562ae1fac4
+generated: "2023-08-31T09:29:41.551458599+03:00"

helm-minio-mt/Chart.yaml

@@ -0,0 +1,8 @@
+apiVersion: v2
+name: minio-mt
+version: 0.0.0
+
+dependencies:
+  - name: operator
+    version: 5.0.7
+    repository: https://operator.min.io/

helm-minio-mt/templates/tenant-env-configuration-secret.yaml

@@ -0,0 +1,18 @@
+{{ range $.Values.tenants }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .name }}-env-configuration
+  namespace: {{ .name }}
+stringData:
+  config.env: |
+    export MINIO_BROWSER="on"
+    export MINIO_ROOT_USER={{ .root_user | quote }}
+    export MINIO_ROOT_PASSWORD={{ .root_password | quote }}
+    export MINIO_STORAGE_CLASS_STANDARD={{ .storage_class_standard | quote }}
+    export MINIO_DOMAIN="minio-tenant-{{ .name }}{{ .domain_suffix }}"
+    export MINIO_BROWSER_REDIRECT_URL="https://minio-tenant-{{ .name }}-console{{ .domain_suffix }}"
+    export MINIO_SERVER_URL="https://minio-tenant-{{ .name }}{{ .domain_suffix }}"
+---
+{{ end }}

helm-minio-mt/templates/tenant-ingress.yaml

@@ -0,0 +1,39 @@
+{{ range $.Values.tenants }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ .name }}-ingress
+  namespace: {{ .name }}
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    nginx.ingress.kubernetes.io/proxy-body-size: 5t
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - minio-tenant-{{ .name }}{{ .domain_suffix }}
+        - minio-tenant-{{ .name }}-console{{ .domain_suffix }}
+      secretName: tenant-tls
+  rules:
+    - host: minio-tenant-{{ .name }}{{ .domain_suffix }}
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: minio
+                port:
+                  number: 80
+    - host: minio-tenant-{{ .name }}-console{{ .domain_suffix }}
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: {{ .name }}-console
+                port:
+                  number: 9090
+---
+{{ end }}

helm-minio-mt/templates/tenant-secret.yaml

@@ -0,0 +1,12 @@
+{{ range $.Values.tenants }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .name }}-secret
+  namespace: {{ .name }}
+stringData:
+  accesskey: {{ .root_user | quote }}
+  secretkey: {{ .root_password | quote }}
+---
+{{ end }}

helm-minio-mt/templates/tenant-storage-user-secret.yaml

@@ -0,0 +1,12 @@
+{{ range $.Values.tenants }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: storage-user
+  namespace: {{ .name }}
+stringData:
+  CONSOLE_ACCESS_KEY: {{ .user_user | quote }}
+  CONSOLE_SECRET_KEY: {{ .user_password | quote }}
+---
+{{ end }}

helm-minio-mt/templates/tenant.yaml

@@ -0,0 +1,74 @@
+{{ range $.Values.tenants }}
+apiVersion: minio.min.io/v2
+kind: Tenant
+metadata:
+  name: {{ .name }}
+  namespace: {{ .name }}
+spec:
+  configuration:
+    name: {{ .name }}-env-configuration
+  credsSecret:
+    name: {{ .name }}-secret
+  image: minio/minio:RELEASE.2023-08-29T23-07-35Z
+  mountPath: /export
+  pools:
+    {{ if eq .name "simple" }}
+    - name: pool-0
+      servers: 1
+      tolerations:
+        - effect: NoSchedule
+          key: cwmc-role
+          operator: Equal
+          value: worker
+      volumeClaimTemplate:
+        metadata:
+          name: data
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: "10Gi"
+          storageClassName: directpv-min-io
+      volumesPerServer: 1
+    {{ else }}
+    - name: pool-1
+      servers: 2
+      tolerations:
+        - effect: NoSchedule
+          key: cwmc-role
+          operator: Equal
+          value: worker
+      volumeClaimTemplate:
+        metadata:
+          name: data1
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: "10Gi"
+          storageClassName: directpv-min-io
+      volumesPerServer: 2
+    - name: pool-2
+      servers: 2
+      tolerations:
+        - effect: NoSchedule
+          key: cwmc-role
+          operator: Equal
+          value: worker
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: "10Gi"
+          storageClassName: directpv-min-io
+      volumesPerServer: 2
+    {{ end }}
+  requestAutoCert: false
+  users:
+    - name: storage-user
+---
+{{ end }}

helm-minio-mt/values.yaml

@@ -0,0 +1,15 @@
+operator:
+  operator:
+    tolerations:
+      - key: cwmc-role
+        value: worker
+        effect: NoSchedule
+  console:
+    ingress:
+      enabled: true
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt
+    tolerations:
+      - key: cwmc-role
+        value: worker
+        effect: NoSchedule