zfs-operator.yaml

# This manifest deploys the OpenEBS ZFS control plane components,
# with associated CRs & RBAC rules.

# Create the OpenEBS namespace
apiVersion: v1
kind: Namespace
metadata:
  name: openebs
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: zfsvolumes.zfs.openebs.io
spec:
  group: zfs.openebs.io
  version: v1alpha1
  scope: Namespaced
  names:
    plural: zfsvolumes
    singular: zfsvolume
    kind: ZFSVolume
    shortNames:
    - zfsvol
    - zv
  additionalPrinterColumns:
  - JSONPath: .spec.poolName
    name: ZPool
    description: ZFS Pool where the volume is created
    type: string
  - JSONPath: .spec.ownerNodeID
    name: Node
    description: Node where the volume is created
    type: string
  - JSONPath: .spec.capacity
    name: Size
    description: Size of the volume
    type: string
  - JSONPath: .spec.volblocksize
    name: volblocksize
    description: volblocksize for the created zvol
    type: string
  - JSONPath: .spec.recordsize
    name: recordsize
    description: recordsize for the created zfs dataset
    type: string
  - JSONPath: .spec.fsType
    name: Filesystem
    description: filesystem created on the volume
    type: string
---
##############################################
###########                       ############
###########   Snapshot CRDs       ############
###########                       ############
##############################################

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: zfssnapshots.zfs.openebs.io
spec:
  group: zfs.openebs.io
  version: v1alpha1
  scope: Namespaced
  names:
    plural: zfssnapshots
    singular: zfssnapshot
    kind: ZFSSnapshot
    shortNames:
    - zfssnapshot
    - zfssnap
---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: (devel)
    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139"
  creationTimestamp: null
  name: volumesnapshotclasses.snapshot.storage.k8s.io
spec:
  group: snapshot.storage.k8s.io
  names:
    kind: VolumeSnapshotClass
    listKind: VolumeSnapshotClassList
    plural: volumesnapshotclasses
    singular: volumesnapshotclass
  scope: Cluster
  #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
  validation:
    openAPIV3Schema:
      description: VolumeSnapshotClass specifies parameters that a underlying storage
        system uses when creating a volume snapshot. A specific VolumeSnapshotClass
        is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses
        are non-namespaced
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        deletionPolicy:
          description: deletionPolicy determines whether a VolumeSnapshotContent created
            through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot
            is deleted. Supported values are "Retain" and "Delete". "Retain" means
            that the VolumeSnapshotContent and its physical snapshot on underlying
            storage system are kept. "Delete" means that the VolumeSnapshotContent
            and its physical snapshot on underlying storage system are deleted. Required.
          enum:
          - Delete
          - Retain
          type: string
        driver:
          description: driver is the name of the storage driver that handles this
            VolumeSnapshotClass. Required.
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        parameters:
          additionalProperties:
            type: string
          description: parameters is a key-value map with storage driver specific
            parameters for creating snapshots. These values are opaque to Kubernetes.
          type: object
      required:
      - deletionPolicy
      - driver
      type: object
  version: v1beta1
  versions:
  - name: v1beta1
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: (devel)
    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139"
  creationTimestamp: null
  name: volumesnapshotcontents.snapshot.storage.k8s.io
spec:
  group: snapshot.storage.k8s.io
  names:
    kind: VolumeSnapshotContent
    listKind: VolumeSnapshotContentList
    plural: volumesnapshotcontents
    singular: volumesnapshotcontent
  scope: Cluster
  subresources:
    status: {}
  #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
  validation:
    openAPIV3Schema:
      description: VolumeSnapshotContent represents the actual "on-disk" snapshot
        object in the underlying storage system
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        spec:
          description: spec defines properties of a VolumeSnapshotContent created
            by the underlying storage system. Required.
          properties:
            deletionPolicy:
              description: deletionPolicy determines whether this VolumeSnapshotContent
                and its physical snapshot on the underlying storage system should
                be deleted when its bound VolumeSnapshot is deleted. Supported values
                are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent
                and its physical snapshot on underlying storage system are kept. "Delete"
                means that the VolumeSnapshotContent and its physical snapshot on
                underlying storage system are deleted. In dynamic snapshot creation
                case, this field will be filled in with the "DeletionPolicy" field
                defined in the VolumeSnapshotClass the VolumeSnapshot refers to. For
                pre-existing snapshots, users MUST specify this field when creating
                the VolumeSnapshotContent object. Required.
              enum:
              - Delete
              - Retain
              type: string
            driver:
              description: driver is the name of the CSI driver used to create the
                physical snapshot on the underlying storage system. This MUST be the
                same as the name returned by the CSI GetPluginName() call for that
                driver. Required.
              type: string
            source:
              description: source specifies from where a snapshot will be created.
                This field is immutable after creation. Required.
              properties:
                snapshotHandle:
                  description: snapshotHandle specifies the CSI "snapshot_id" of a
                    pre-existing snapshot on the underlying storage system. This field
                    is immutable.
                  type: string
                volumeHandle:
                  description: volumeHandle specifies the CSI "volume_id" of the volume
                    from which a snapshot should be dynamically taken from. This field
                    is immutable.
                  type: string
              type: object
            volumeSnapshotClassName:
              description: name of the VolumeSnapshotClass to which this snapshot
                belongs.
              type: string
            volumeSnapshotRef:
              description: volumeSnapshotRef specifies the VolumeSnapshot object to
                which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName
                field must reference to this VolumeSnapshotContent's name for the
                bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent
                object, name and namespace of the VolumeSnapshot object MUST be provided
                for binding to happen. This field is immutable after creation. Required.
              properties:
                apiVersion:
                  description: API version of the referent.
                  type: string
                fieldPath:
                  description: 'If referring to a piece of an object instead of an
                    entire object, this string should contain a valid JSON/Go field
                    access statement, such as desiredState.manifest.containers[2].
                    For example, if the object reference is to a container within
                    a pod, this would take on a value like: "spec.containers{name}"
                    (where "name" refers to the name of the container that triggered
                    the event) or if no container name is specified "spec.containers[2]"
                    (container with index 2 in this pod). This syntax is chosen only
                    to have some well-defined way of referencing a part of an object.
                    TODO: this design is not final and this field is subject to change
                    in the future.'
                  type: string
                kind:
                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
                  type: string
                name:
                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
                  type: string
                namespace:
                  description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
                  type: string
                resourceVersion:
                  description: 'Specific resourceVersion to which this reference is
                    made, if any. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency'
                  type: string
                uid:
                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
                  type: string
              type: object
          required:
          - deletionPolicy
          - driver
          - source
          - volumeSnapshotRef
          type: object
        status:
          description: status represents the current information of a snapshot.
          properties:
            creationTime:
              description: creationTime is the timestamp when the point-in-time snapshot
                is taken by the underlying storage system. In dynamic snapshot creation
                case, this field will be filled in with the "creation_time" value
                returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing
                snapshot, this field will be filled with the "creation_time" value
                returned from the CSI "ListSnapshots" gRPC call if the driver supports
                it. If not specified, it indicates the creation time is unknown. The
                format of this field is a Unix nanoseconds time encoded as an int64.
                On Unix, the command `date +%s%N` returns the current time in nanoseconds
                since 1970-01-01 00:00:00 UTC.
              format: int64
              type: integer
            error:
              description: error is the latest observed error during snapshot creation,
                if any.
              properties:
                message:
                  description: 'message is a string detailing the encountered error
                    during snapshot creation if specified. NOTE: message may be logged,
                    and it should not contain sensitive information.'
                  type: string
                time:
                  description: time is the timestamp when the error was encountered.
                  format: date-time
                  type: string
              type: object
            readyToUse:
              description: readyToUse indicates if a snapshot is ready to be used
                to restore a volume. In dynamic snapshot creation case, this field
                will be filled in with the "ready_to_use" value returned from CSI
                "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this
                field will be filled with the "ready_to_use" value returned from the
                CSI "ListSnapshots" gRPC call if the driver supports it, otherwise,
                this field will be set to "True". If not specified, it means the readiness
                of a snapshot is unknown.
              type: boolean
            restoreSize:
              description: restoreSize represents the complete size of the snapshot
                in bytes. In dynamic snapshot creation case, this field will be filled
                in with the "size_bytes" value returned from CSI "CreateSnapshotRequest"
                gRPC call. For a pre-existing snapshot, this field will be filled
                with the "size_bytes" value returned from the CSI "ListSnapshots"
                gRPC call if the driver supports it. When restoring a volume from
                this snapshot, the size of the volume MUST NOT be smaller than the
                restoreSize if it is specified, otherwise the restoration will fail.
                If not specified, it indicates that the size is unknown.
              format: int64
              minimum: 0
              type: integer
            snapshotHandle:
              description: snapshotHandle is the CSI "snapshot_id" of a snapshot on
                the underlying storage system. If not specified, it indicates that
                dynamic snapshot creation has either failed or it is still in progress.
              type: string
          type: object
      required:
      - spec
      type: object
  version: v1beta1
  versions:
  - name: v1beta1
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---

apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  annotations:
    controller-gen.kubebuilder.io/version: (devel)
    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139"
  creationTimestamp: null
  name: volumesnapshots.snapshot.storage.k8s.io
spec:
  group: snapshot.storage.k8s.io
  names:
    kind: VolumeSnapshot
    listKind: VolumeSnapshotList
    plural: volumesnapshots
    singular: volumesnapshot
  scope: Namespaced
  subresources:
    status: {}
  #preserveUnknownFields: false # this field is supported in kubernetes 1.15+ https://v1-15.docs.kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/
  validation:
    openAPIV3Schema:
      description: VolumeSnapshot is a user's request for either creating a point-in-time
        snapshot of a persistent volume, or binding to a pre-existing snapshot.
      properties:
        apiVersion:
          description: 'APIVersion defines the versioned schema of this representation
            of an object. Servers should convert recognized schemas to the latest
            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
          type: string
        kind:
          description: 'Kind is a string value representing the REST resource this
            object represents. Servers may infer this from the endpoint the client
            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
          type: string
        spec:
          description: 'spec defines the desired characteristics of a snapshot requested
            by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots
            Required.'
          properties:
            source:
              description: source specifies where a snapshot will be created from.
                This field is immutable after creation. Required.
              properties:
                persistentVolumeClaimName:
                  description: persistentVolumeClaimName specifies the name of the
                    PersistentVolumeClaim object in the same namespace as the VolumeSnapshot
                    object where the snapshot should be dynamically taken from. This
                    field is immutable.
                  type: string
                volumeSnapshotContentName:
                  description: volumeSnapshotContentName specifies the name of a pre-existing
                    VolumeSnapshotContent object. This field is immutable.
                  type: string
              type: object
            volumeSnapshotClassName:
              description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass
                requested by the VolumeSnapshot. If not specified, the default snapshot
                class will be used if one exists. If not specified, and there is no
                default snapshot class, dynamic snapshot creation will fail. Empty
                string is not allowed for this field. TODO(xiangqian): a webhook validation
                on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes'
              type: string
          required:
          - source
          type: object
        status:
          description: 'status represents the current information of a snapshot. NOTE:
            status can be modified by sources other than system controllers, and must
            not be depended upon for accuracy. Controllers should only use information
            from the VolumeSnapshotContent object after verifying that the binding
            is accurate and complete.'
          properties:
            boundVolumeSnapshotContentName:
              description: 'boundVolumeSnapshotContentName represents the name of
                the VolumeSnapshotContent object to which the VolumeSnapshot object
                is bound. If not specified, it indicates that the VolumeSnapshot object
                has not been successfully bound to a VolumeSnapshotContent object
                yet. NOTE: Specified boundVolumeSnapshotContentName alone does not
                mean binding       is valid. Controllers MUST always verify bidirectional
                binding between       VolumeSnapshot and VolumeSnapshotContent to
                avoid possible security issues.'
              type: string
            creationTime:
              description: creationTime is the timestamp when the point-in-time snapshot
                is taken by the underlying storage system. In dynamic snapshot creation
                case, this field will be filled in with the "creation_time" value
                returned from CSI "CreateSnapshotRequest" gRPC call. For a pre-existing
                snapshot, this field will be filled with the "creation_time" value
                returned from the CSI "ListSnapshots" gRPC call if the driver supports
                it. If not specified, it indicates that the creation time of the snapshot
                is unknown.
              format: date-time
              type: string
            error:
              description: error is the last observed error during snapshot creation,
                if any. This field could be helpful to upper level controllers(i.e.,
                application controller) to decide whether they should continue on
                waiting for the snapshot to be created based on the type of error
                reported.
              properties:
                message:
                  description: 'message is a string detailing the encountered error
                    during snapshot creation if specified. NOTE: message may be logged,
                    and it should not contain sensitive information.'
                  type: string
                time:
                  description: time is the timestamp when the error was encountered.
                  format: date-time
                  type: string
              type: object
            readyToUse:
              description: readyToUse indicates if a snapshot is ready to be used
                to restore a volume. In dynamic snapshot creation case, this field
                will be filled in with the "ready_to_use" value returned from CSI
                "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, this
                field will be filled with the "ready_to_use" value returned from the
                CSI "ListSnapshots" gRPC call if the driver supports it, otherwise,
                this field will be set to "True". If not specified, it means the readiness
                of a snapshot is unknown.
              type: boolean
            restoreSize:
              description: restoreSize represents the complete size of the snapshot
                in bytes. In dynamic snapshot creation case, this field will be filled
                in with the "size_bytes" value returned from CSI "CreateSnapshotRequest"
                gRPC call. For a pre-existing snapshot, this field will be filled
                with the "size_bytes" value returned from the CSI "ListSnapshots"
                gRPC call if the driver supports it. When restoring a volume from
                this snapshot, the size of the volume MUST NOT be smaller than the
                restoreSize if it is specified, otherwise the restoration will fail.
                If not specified, it indicates that the size is unknown.
              type: string
          type: object
      required:
      - spec
      type: object
  version: v1beta1
  versions:
  - name: v1beta1
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
---

##############################################
###########                       ############
###########   Controller plugin   ############
###########                       ############
##############################################

kind: ServiceAccount
apiVersion: v1
metadata:
  name: openebs-zfs-controller-sa
  namespace: kube-system

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-provisioner-role
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: [""]
    resources: ["namespaces"]
    verbs: ["*"]
  - apiGroups: [""]
    resources: ["persistentvolumes", "services"]
    verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims/status"]
    verbs: ["update", "patch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses", "csinodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]
  - apiGroups: ["*"]
    resources: ["zfsvolumes", "zfssnapshots"]
    verbs: ["*"]
---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-provisioner-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-provisioner-role
  apiGroup: rbac.authorization.k8s.io

---
kind: StatefulSet
apiVersion: apps/v1
metadata:
  name: openebs-zfs-controller
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: openebs-zfs-controller
      role: openebs-zfs
  serviceName: "openebs-zfs"
  replicas: 1
  template:
    metadata:
      labels:
        app: openebs-zfs-controller
        role: openebs-zfs
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - openebs-zfs-controller
            topologyKey: "kubernetes.io/hostname"
      priorityClassName: system-cluster-critical
      serviceAccount: openebs-zfs-controller-sa
      containers:
        - name: csi-resizer
          image: quay.io/k8scsi/csi-resizer:v0.4.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--leader-election"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-snapshotter
          image: quay.io/k8scsi/csi-snapshotter:v2.0.1
          imagePullPolicy: IfNotPresent
          args:
            - "--csi-address=$(ADDRESS)"
            - "--leader-election"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          imagePullPolicy: Always
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: snapshot-controller
          image: quay.io/k8scsi/snapshot-controller:v2.0.1
          args:
            - "--v=5"
            - "--leader-election=false"
          imagePullPolicy: IfNotPresent
        - name: csi-provisioner
          image: quay.io/k8scsi/csi-provisioner:v1.5.0
          imagePullPolicy: IfNotPresent
          args:
            - "--csi-address=$(ADDRESS)"
            - "--v=5"
            - "--feature-gates=Topology=true"
            - "--strict-topology"
            - "--enable-leader-election"
            - "--leader-election-type=leases"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-attacher
          image: quay.io/k8scsi/csi-attacher:v2.0.0
          imagePullPolicy: IfNotPresent
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--leader-election"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-cluster-driver-registrar
          image: quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1
          args:
            - "--v=5"
            - "--driver-requires-attachment=false"
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: openebs-zfs-plugin
          image: quay.io/openebs/zfs-driver:ci
          imagePullPolicy: IfNotPresent
          env:
            - name: OPENEBS_CONTROLLER_DRIVER
              value: controller
            - name: OPENEBS_CSI_ENDPOINT
              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
            - name: OPENEBS_NAMESPACE
              value: openebs
            - name: OPENEBS_IO_INSTALLER_TYPE
              value: "zfs-operator"
            - name: OPENEBS_IO_ENABLE_ANALYTICS
              value: "true"
          args :
            - "--endpoint=$(OPENEBS_CSI_ENDPOINT)"
            - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)"
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
      volumes:
        - name: socket-dir
          emptyDir: {}
---

############################## CSI- Attacher #######################
# Attacher must be able to work with PVs, nodes and VolumeAttachments

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-attacher-role
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["csi.storage.k8s.io"]
    resources: ["csinodeinfos"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments", "csinodes"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-attacher-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-attacher-role
  apiGroup: rbac.authorization.k8s.io

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-snapshotter-role
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents"]
    verbs: ["create", "get", "list", "watch", "update", "delete"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents/status"]
    verbs: ["update"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots/status"]
    verbs: ["update"]
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ["create", "list", "watch", "delete"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-snapshotter-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-snapshotter-role
  apiGroup: rbac.authorization.k8s.io

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-cluster-driver-registrar-role
rules:
  - apiGroups: ["csi.storage.k8s.io"]
    resources: ["csidrivers"]
    verbs: ["create", "delete"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-cluster-driver-registrar-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-controller-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-cluster-driver-registrar-role
  apiGroup: rbac.authorization.k8s.io

---

########################################
###########                 ############
###########   Node plugin   ############
###########                 ############
########################################

apiVersion: v1
kind: ServiceAccount
metadata:
  name: openebs-zfs-node-sa
  namespace: kube-system

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-driver-registrar-role
rules:
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumes", "nodes", "services"]
    verbs: ["get", "list"]
  - apiGroups: ["*"]
    resources: ["zfsvolumes", "zfssnapshots"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]

---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openebs-zfs-driver-registrar-binding
subjects:
  - kind: ServiceAccount
    name: openebs-zfs-node-sa
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: openebs-zfs-driver-registrar-role
  apiGroup: rbac.authorization.k8s.io

---

kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: openebs-zfs-node
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: openebs-zfs-node
  template:
    metadata:
      labels:
        app: openebs-zfs-node
        role: openebs-zfs
    spec:
      priorityClassName: system-node-critical
      serviceAccount: openebs-zfs-node-sa
      hostNetwork: true
      containers:
        - name: csi-node-driver-registrar
          image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0
          args:
            - "--v=5"
            - "--csi-address=$(ADDRESS)"
            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh", "-c", "rm -rf /registration/zfs-localpv /registration/zfs-localpv-reg.sock"]
          env:
            - name: ADDRESS
              value: /plugin/csi.sock
            - name: DRIVER_REG_SOCK_PATH
              value: /var/lib/kubelet/plugins/zfs-localpv/csi.sock
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: NODE_DRIVER
              value: openebs-zfs
          volumeMounts:
            - name: plugin-dir
              mountPath: /plugin
            - name: registration-dir
              mountPath: /registration
        - name: openebs-zfs-plugin
          securityContext:
            privileged: true
            capabilities:
              add: ["CAP_MKNOD", "CAP_SYS_ADMIN", "SYS_ADMIN"]
            allowPrivilegeEscalation: true
          image: quay.io/openebs/zfs-driver:ci
          imagePullPolicy: IfNotPresent
          args:
            - "--nodeid=$(OPENEBS_NODE_ID)"
            - "--endpoint=$(OPENEBS_CSI_ENDPOINT)"
            - "--plugin=$(OPENEBS_NODE_DRIVER)"
          env:
            - name: OPENEBS_NODE_ID
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: OPENEBS_CSI_ENDPOINT
              value: unix:///plugin/csi.sock
            - name: OPENEBS_NODE_DRIVER
              value: agent
            - name: OPENEBS_NAMESPACE
              value: openebs
          volumeMounts:
            - name: plugin-dir
              mountPath: /plugin
            - name: device-dir
              mountPath: /dev
            - name: encr-keys
              mountPath: /home/keys
            - name: zfs-bin
              mountPath: /sbin/zfs
            - name: libzpool
              mountPath: /lib/libzpool.so.2
            - name: libzfscore
              mountPath: /lib/libzfs_core.so.1
            - name: libzfs
              mountPath: /lib/libzfs.so.2
            - name: libuutil
              mountPath: /lib/libuutil.so.1
            - name: libnvpair
              mountPath: /lib/libnvpair.so.1
            - name: pods-mount-dir
              mountPath: /var/lib/kubelet/pods
              # needed so that any mounts setup inside this container are
              # propagated back to the host machine.
              mountPropagation: "Bidirectional"
      volumes:
        - name: device-dir
          hostPath:
            path: /dev
            type: Directory
        - name: encr-keys
          hostPath:
            path: /home/keys
            type: DirectoryOrCreate
        - name: zfs-bin
          hostPath:
            path: /sbin/zfs
            type: File
        - name: libzpool
          hostPath:
            path: /lib/libzpool.so.2.0.0
            type: File
        - name: libzfscore
          hostPath:
            path: /lib/libzfs_core.so.1.0.0
            type: File
        - name: libzfs
          hostPath:
            path: /lib/libzfs.so.2.0.0
            type: File
        - name: libuutil
          hostPath:
            path: /lib/libuutil.so.1.0.1
            type: File
        - name: libnvpair
          hostPath:
            path: /lib/libnvpair.so.1.0.1
            type: File
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry/
            type: DirectoryOrCreate
        - name: plugin-dir
          hostPath:
            path: /var/lib/kubelet/plugins/zfs-localpv/
            type: DirectoryOrCreate
        - name: pods-mount-dir
          hostPath:
            path: /var/lib/kubelet/pods
            type: Directory
---