helmrelease.yaml

---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: rook-ceph-cluster
spec:
  interval: 30m
  timeout: 15m
  chart:
    spec:
      chart: rook-ceph-cluster
      version: v1.14.5
      sourceRef:
        kind: HelmRepository
        name: rook-ceph-charts
        namespace: flux-system
  install:
    remediation:
      retries: 3
  upgrade:
    cleanupOnFail: true
    remediation:
      strategy: rollback
      retries: 3
  values:
    monitoring:
      enabled: true
      createPrometheusRules: true
    ingress:
      dashboard:
        ingressClassName: internal
        annotations:
          hajimari.io/appName: Rook
          hajimari.io/icon: mdi:chess-rook
        host:
          name: &host rook.${PUBLIC_DOMAIN}
          path: /
        tls:
          - hosts:
              - *host
    toolbox:
      enabled: true
    configOverride: |
      [global]
      bdev_enable_discard = true
      bdev_async_discard = true
      osd_class_update_on_start = false
    cephClusterSpec:
      crashCollector:
        disable: false
      csi:
        readAffinity:
          enabled: true
      dashboard:
        enabled: true
        urlPrefix: /
        ssl: false
        prometheusEndpoint: http://thanos-query-frontend.monitoring.svc.cluster.local:10902
      mgr:
        modules:
          - name: pg_autoscaler
            enabled: true
      network:
        provider: host
        addressRanges:
          public: ["${NET_SERVERS_CIDR}"]
          cluster: ["${NET_THUNDERBOLT_CIDR}"]
        connections:
          requireMsgr2: true
      placement:
        mgr: &placement
          nodeAffinity:
            requiredDuringSchedulingIgnoredDuringExecution:
              nodeSelectorTerms:
                - matchExpressions:
                    - key: node-role.kubernetes.io/control-plane
                      operator: Exists
        mon:
          <<: *placement
      resources:
        mgr:
          requests:
            cpu: 100m
            memory: 512Mi
          limits:
            memory: 2Gi
        mon:
          requests:
            cpu: 50m
            memory: 512Mi
          limits:
            memory: 1Gi
        osd:
          requests:
            cpu: 500m
            memory: 2Gi
          limits:
            memory: 8Gi
        mgr-sidecar:
          requests:
            cpu: 50m
            memory: 128Mi
          limits:
            memory: 256Mi
      storage:
        useAllNodes: false
        useAllDevices: false
        config:
          osdsPerDevice: "1"
        nodes:
          - name: w0
            devices:
              - name: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_S6S2NS0TC14873N
              - name: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_S6S2NS0TC14865L
          - name: w1
            devices:
              - name: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_S6S2NS0TC14871H
              - name: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_S6S2NS0W122087H
          - name: w2
            devices:
              - name: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_S6S2NS0W120772T
              - name: /dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_2TB_S6S2NS0TC14864B
    cephBlockPools:
      - name: ceph-blockpool
        spec:
          failureDomain: host
          replicated:
            size: 3
        storageClass:
          enabled: true
          name: ${CLUSTER_STORAGE_BLOCK}
          isDefault: true
          reclaimPolicy: Delete
          allowVolumeExpansion: true
          parameters:
            imageFormat: "2"
            imageFeatures: layering
            csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
            csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
            csi.storage.k8s.io/fstype: ext4
    cephBlockPoolsVolumeSnapshotClass:
      enabled: true
      name: ${CLUSTER_SNAPSHOT_BLOCK}
      isDefault: true
      deletionPolicy: Delete
    cephFileSystems:
      - name: &cephFileSystemName ceph-filesystem
        spec:
          dataPools:
            - failureDomain: host
              replicated:
                size: 3
              name: data0
          metadataPool:
            replicated:
              size: 3
          metadataServer:
            activeCount: 1
            activeStandby: true
            priorityClassName: system-cluster-critical
            placement:
              topologySpreadConstraints:
                - maxSkew: 1
                  topologyKey: kubernetes.io/hostname
                  whenUnsatisfiable: DoNotSchedule
                  labelSelector:
                    matchLabels:
                      app.kubernetes.io/name: ceph-mds
                      app.kubernetes.io/part-of: *cephFileSystemName
            resources:
              requests:
                cpu: 1000m
                memory: 4Gi
              limits:
                memory: 4Gi
        storageClass:
          enabled: true
          isDefault: false
          name: ${CLUSTER_STORAGE_FILESYSTEM}
          pool: data0
          reclaimPolicy: Delete
          allowVolumeExpansion: true
          parameters:
            csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
            csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
            csi.storage.k8s.io/fstype: ext4
    cephFileSystemVolumeSnapshotClass:
      enabled: true
      name: ${CLUSTER_SNAPSHOT_FILESYSTEM}
      isDefault: false
      deletionPolicy: Delete
    cephObjectStores:
      - name: ceph-objectstore
        spec:
          dataPool:
            failureDomain: host
            erasureCoded:
              dataChunks: 2
              codingChunks: 1
          metadataPool:
            failureDomain: host
            replicated:
              size: 3
          preservePoolsOnDelete: true
          gateway:
            hostNetwork: false
            port: 80
            resources:
              requests:
                cpu: 1000m
                memory: 1Gi
              limits:
                memory: 2Gi
            instances: 2
          healthCheck:
            bucket:
              interval: 60s
        storageClass:
          enabled: true
          name: ${CLUSTER_STORAGE_BUCKET}
          reclaimPolicy: Delete
          parameters:
            region: ceph-objectstore