feat: added longhorn addon (#1204)

* added longhorn addon

* enabled longhorn in the tests

* fix: move longhorn to longhorn-system namespace

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: move longhorn to longhorn-system namespace

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: move longhorn to longhorn-system namespace

Signed-off-by: Devin Buhl <devin@buhl.casa>

* fix: move longhorn to longhorn-system namespace

Signed-off-by: Devin Buhl <devin@buhl.casa>

---------

Signed-off-by: Devin Buhl <devin@buhl.casa>
Co-authored-by: Devin Buhl <devin@buhl.casa>

.github/tests/addons.yaml

@@ -35,3 +35,6 @@ volsync:
 
 spegel:
   enabled: true
+
+longhorn:
+  enabled: true

bootstrap/templates/kubernetes/apps/longhorn-system/kustomization.yaml.j2

@@ -0,0 +1,8 @@
+#% if longhorn.enabled | default(false) %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./namespace.yaml
+  - ./longhorn/ks.yaml
+#% endif %#

bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/helmrelease.yaml.j2

@@ -0,0 +1,48 @@
+#% if longhorn.enabled | default(false) %#
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: longhorn
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: longhorn
+      version: 1.5.3
+      sourceRef:
+        kind: HelmRepository
+        name: longhorn
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    monitoring:
+      enabled: true
+      createPrometheusRules: true
+    defaultSettings:
+      defaultReplicaCount: 3
+      backupstorePollInterval: 0
+      createDefaultDiskLabeledNodes: true
+      restoreVolumeRecurringJobs: true
+      storageOverProvisioningPercentage: 100
+      storageMinimalAvailablePercentage: 1
+      guaranteedEngineManagerCPU: 20
+      guaranteedReplicaManagerCPU: 20
+      orphanAutoDeletion: true
+      concurrentAutomaticEngineUpgradePerNodeLimit: 3
+      defaultLonghornStaticStorageClass: longhorn
+      nodeDownPodDeletionPolicy: delete-both-statefulset-and-deployment-pod
+    ingress:
+      enabled: true
+      ingressClassName: internal
+      host: longhorn.${SECRET_DOMAIN}
+      tls: true
+#% endif %#

bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/kustomization.yaml.j2

@@ -0,0 +1,8 @@
+#% if longhorn.enabled | default(false) %#
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
+  - ./snapshot.yaml
+#% endif %#

bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/app/snapshot.yaml.j2

@@ -0,0 +1,16 @@
+#% if longhorn.enabled | default(false) %#
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+reclaimPolicy: Delete
+provisioner: driver.longhorn.io
+parameters:
+  dataLocality: disabled
+  numberOfReplicas: "1" # Faster restores from the snapshotclass
+  replicaAutoBalance: best-effort
+  staleReplicaTimeout: "30"
+allowVolumeExpansion: true
+volumeBindingMode: Immediate
+metadata:
+  name: longhorn-snapshot
+#% endif %#

bootstrap/templates/kubernetes/apps/longhorn-system/longhorn/ks.yaml.j2

@@ -0,0 +1,22 @@
+#% if longhorn.enabled | default(false) %#
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app longhorn
+  namespace: flux-system
+spec:
+  targetNamespace: longhorn-system
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/longhorn-system/longhorn/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+#% endif %#

bootstrap/templates/kubernetes/apps/longhorn-system/namespace.yaml.j2

@@ -0,0 +1,9 @@
+#% if longhorn.enabled | default(false) %#
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+#% endif %#

bootstrap/templates/kubernetes/apps/storage/volsync/ks.yaml.j2

@@ -10,6 +10,10 @@ spec:
   commonMetadata:
     labels:
       app.kubernetes.io/name: *app
+  #% if longhorn.enabled | default(false) %#
+  dependsOn:
+    - name: longhorn
+  #% endif %#
   path: ./kubernetes/apps/storage/volsync/snapshot-controller
   prune: true
   sourceRef:

bootstrap/templates/kubernetes/apps/storage/volsync/snapshot-controller/helmrelease.yaml.j2

@@ -27,6 +27,17 @@ spec:
     keepHistory: false
   values:
     controller:
+      #% if longhorn.enabled | default(false) %#
+      volumeSnapshotClasses:
+        - name: longhorn-snapclass
+          annotations:
+            snapshot.storage.kubernetes.io/is-default-class: "true"
+          driver: driver.longhorn.io
+          deletionPolicy: Delete
+          # Ref: https://github.com/longhorn/longhorn/issues/2534#issuecomment-1010508714
+          parameters:
+            type: snap
+      #% endif %#
       serviceMonitor:
         create: true
     webhook:

bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2

@@ -15,6 +15,7 @@ resources:
   - ./jetstack.yaml
   - ./k8s-gateway.yaml
   - ./kubernetes-dashboard.yaml
+  - ./longhorn.yaml
   - ./metrics-server.yaml
   - ./openebs.yaml
   - ./piraeus.yaml

bootstrap/templates/kubernetes/flux/repositories/helm/longhorn.yaml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: longhorn
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://charts.longhorn.io

bootstrap/vars/addons.sample.yaml

@@ -38,7 +38,7 @@ csi_driver_nfs:
 
 # https://github.com/rancher/system-upgrade-controller
 system_upgrade_controller:
-  # WARNING: Only enable this if you also track the version of k3s in the
+  # IMPORTANT: Only enable this if you also track the version of k3s in the
   # ansible configuration files. Running ansible against an already provisioned
   # cluster with this enabled might cause your cluster to be downgraded.
   # Note: If bootstrap_distribution is set to k0s or talos this will be ignored.
@@ -59,3 +59,11 @@ volsync:
 spegel:
   # Note: This only applies to k0s at the moment
   enabled: false
+
+# https://github.com/longhorn/longhorn
+longhorn:
+  # IMPORTANT: Only enable this if you have 3 or more nodes in your cluster
+  # and you have enough IOPS on disk to support the number of replicas.
+  # Note: Consider enabling volsync if you would like to configure snapshots
+  # and backups to durable storage.
+  enabled: false