Automated Local Disk Setup

Dockerfile

@@ -4,10 +4,8 @@ COPY . .
 RUN make build --warn-undefined-variables
 
 FROM quay.io/scylladb/scylla-operator-images:base-ubuntu-22.04
-# sidecar-injection container and existing installations use binary from root,
-# we have to keep it there until we figure out how to properly upgrade them.
+
 COPY --from=builder /go/src/github.com/scylladb/scylla-operator/scylla-operator /usr/bin/
-RUN ln -s /usr/bin/scylla-operator /scylla-operator
 COPY --from=builder /go/src/github.com/scylladb/scylla-operator/scylla-operator-tests /usr/bin/
 COPY ./hack/gke /usr/local/lib/scylla-operator/gke
 ENTRYPOINT ["/usr/bin/scylla-operator"]

deploy/operator/10_validatingwebhook.yaml

@@ -25,3 +25,12 @@ webhooks:
     - UPDATE
     resources:
     - scyllaclusters
+  - apiGroups:
+    - scylla.scylladb.com
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - nodeconfigs

examples/common/operator.yaml

@@ -325,6 +325,71 @@ spec:
                 disableOptimizations:
                   description: disableOptimizations controls if nodes matching placement requirements are going to be optimized. Turning off optimizations on already optimized Nodes does not revert changes.
                   type: boolean
+                localDiskSetup:
+                  description: localDiskSetup contains options of automatic local disk setup.
+                  properties:
+                    filesystems:
+                      description: filesystems is a list of filesystem configurations.
+                      items:
+                        description: FilesystemConfiguration specifies filesystem configuration options.
+                        properties:
+                          device:
+                            description: device is a path to the device where the desired filesystem should be created.
+                            type: string
+                          type:
+                            description: type is a desired filesystem type.
+                            type: string
+                        type: object
+                      type: array
+                    mounts:
+                      description: mounts is a list of mount configuration.
+                      items:
+                        description: MountConfiguration specifies mount configuration options.
+                        properties:
+                          device:
+                            description: device is path to a device that should be mounted.
+                            type: string
+                          fsType:
+                            description: fsType specifies the filesystem on the device.
+                            type: string
+                          mountPoint:
+                            description: mountPoint is a path where the device should be mounted at.
+                            type: string
+                          unsupportedOptions:
+                            description: unsupportedOptions is a list of mount options used during device mounting. unsupported in this field name means that we won't support all the available options passed down using this field.
+                            items:
+                              type: string
+                            type: array
+                        type: object
+                      type: array
+                    raids:
+                      description: raids is a list of raid configurations.
+                      items:
+                        description: RAIDConfiguration is a configuration of a raid array.
+                        properties:
+                          RAID0:
+                            description: RAID0 specifies RAID0 options.
+                            properties:
+                              devices:
+                                description: devices defines which devices constitute the raid array.
+                                properties:
+                                  modelRegex:
+                                    description: modelRegex is a regular expression filtering devices by their model name.
+                                    type: string
+                                  nameRegex:
+                                    description: nameRegex is a regular expression filtering devices by their name.
+                                    type: string
+                                type: object
+                            type: object
+                          name:
+                            description: name specifies the name of the raid device to be created under in `/dev/md/`.
+                            type: string
+                          type:
+                            description: type is a type of raid array.
+                            type: string
+                        type: object
+                      type: array
+                  type: object
                 placement:
                   description: placement contains scheduling rules for NodeConfig Pods.
                   properties:
@@ -847,6 +912,11 @@ spec:
                       message:
                         description: message is a human-readable message indicating details about the transition.
                         type: string
+                      observedGeneration:
+                        description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
+                        format: int64
+                        minimum: 0
+                        type: integer
                       reason:
                         description: reason is the reason for condition's last transition.
                         type: string
@@ -4683,6 +4753,15 @@ webhooks:
     - UPDATE
     resources:
     - scyllaclusters
+  - apiGroups:
+    - scylla.scylladb.com
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - nodeconfigs
 
 ---
 apiVersion: policy/v1

examples/eks/eks-cluster.yaml

@@ -10,12 +10,10 @@ availabilityZones:
 
 nodeGroups:
 - name: scylla-pool
-  instanceType: i3.2xlarge
+  instanceType: i4i.2xlarge
   desiredCapacity: 3
   labels:
-    pool: "scylla-pool"
-  taints:
-    role: "scylla-clusters:NoSchedule"
+    scylla.scylladb.com/node-type: scylla
   ssh:
     allow: true
   kubeletExtraConfig:
@@ -32,9 +30,6 @@ nodeGroups:
     role: "cassandra-stress:NoSchedule"
   ssh:
     allow: true
-  preBootstrapCommands:
-  - curl -o /etc/yum.repos.d/scylla.repo -L http://repositories.scylladb.com/scylla/repo/3e3ec6ec-92b2-46ec-b64f-5b2338bc47de/centos/scylladb-4.0.repo
-  - yum install -y scylla-tools
 
 - name: monitoring-pool
   instanceType: i3.large

examples/eks/eks.sh

@@ -122,10 +122,9 @@ EKS_ZONES_QUOTED=$(printf ',"%s"' "${EKS_ZONES[@]}")
 EKS_ZONES_QUOTED="${EKS_ZONES_QUOTED:1}"
 yq eval -P ".metadata.region = \"${EKS_REGION}\" | .metadata.name = \"${CLUSTER_NAME}\" | .availabilityZones |= [${EKS_ZONES_QUOTED}] | (.nodeGroups[] | select(.name==\"scylla-pool\") | .availabilityZones) |= [${EKS_ZONES_QUOTED}]" eks-cluster.yaml | eksctl create cluster -f -
 
-# Configure node disks and network
-kubectl apply -f node-setup-daemonset.yaml
-wait-for-object-creation default daemonset.apps/node-setup
-kubectl rollout status --timeout=5m daemonset.apps/node-setup
+# Configure nodes
+kubectl apply -f nodeconfig-alpha.yaml
+wait-for-object-creation default nodeconfig.scylla.scylladb.com/cluster
 
 # Install local volume provisioner
 echo "Installing local volume provisioner..."

examples/eks/nodeconfig-alpha.yaml

@@ -0,0 +1,24 @@
+apiVersion: scylla.scylladb.com/v1alpha1
+kind: NodeConfig
+metadata:
+  name: cluster
+spec:
+  localDiskSetup:
+    filesystems:
+    - device: /dev/md/nvmes
+      type: xfs
+    mounts:
+    - device: /dev/md/nvmes
+      mountPoint: /mnt/raid-disks/local
+      unsupportedOptions:
+      - prjquota
+    raids:
+    - name: nvmes
+      type: RAID0
+      RAID0:
+        devices:
+          modelRegex: Amazon EC2 NVMe Instance Storage
+          nameRegex: ^/dev/nvme\d+n\d+$
+  placement:
+    nodeSelector:
+      scylla.scylladb.com/node-type: scylla

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8
 	github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/coreos/go-systemd/v22 v22.3.2
 	github.com/evanphx/json-patch v4.12.0+incompatible
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-git/v5 v5.6.0
@@ -18,6 +19,7 @@ require (
 	github.com/go-openapi/swag v0.22.3
 	github.com/go-openapi/validate v0.22.1
 	github.com/gocql/gocql v1.3.1
+	github.com/godbus/dbus/v5 v5.0.4
 	github.com/google/go-cmp v0.5.9
 	github.com/grafana/grafana-api-golang-client v0.18.3
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed
@@ -39,6 +41,7 @@ require (
 	go.uber.org/multierr v1.9.0
 	go.uber.org/zap v1.24.0
 	golang.org/x/oauth2 v0.6.0
+	golang.org/x/sys v0.6.0
 	google.golang.org/grpc v1.53.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.26.2
@@ -138,7 +141,6 @@ require (
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.9.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/term v0.6.0 // indirect
 	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -82,6 +82,8 @@ github.com/cloudflare/circl v1.3.2/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSb
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
@@ -197,6 +199,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

helm/scylla-operator/templates/validatingwebhook.yaml

@@ -25,3 +25,12 @@ webhooks:
     - UPDATE
     resources:
     - scyllaclusters
+  - apiGroups:
+    - scylla.scylladb.com
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - nodeconfigs