install piraeus-operator 2.5.0

rgl · Apr 7, 2024 · 110b1c2 · 110b1c2
1 parent 01362e3
commit 110b1c2
Show file tree

Hide file tree

Showing 5 changed files with 295 additions and 1 deletion.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,38 @@
+{
+    "cSpell.words": [
+        "cmdline",
+        "containerd",
+        "coredns",
+        "crds",
+        "daemonset",
+        "datasource",
+        "drbdadm",
+        "KUBECONFIG",
+        "kubelet",
+        "libvirtd",
+        "LINSTOR",
+        "lvdisplay",
+        "machineconfigs",
+        "nodeaddresses",
+        "nslookup",
+        "pgrep",
+        "piraeusdatastore",
+        "polkit",
+        "popd",
+        "pushd",
+        "pvdisplay",
+        "resolv",
+        "resourcedefinitions",
+        "siderolabs",
+        "statefulset",
+        "staticpods",
+        "staticpodstatus",
+        "storageclass",
+        "TALOSCONFIG",
+        "talosctl",
+        "usermod",
+        "usermode",
+        "vgdisplay",
+        "virt"
+    ]
+}
diff --git a/README.md b/README.md
@@ -6,6 +6,8 @@ An example [Talos Linux](https://www.talos.dev) Kubernetes cluster in libvirt QE
 
 [Cilium](https://cilium.io) is used to augment the Networking (e.g. the [`LoadBalancer`](https://cilium.io/use-cases/load-balancer/) and [`Ingress`](https://docs.cilium.io/en/stable/network/servicemesh/ingress/) controllers), Observability (e.g. [Service Map](https://cilium.io/use-cases/service-map/)), and Security (e.g. [Network Policy](https://cilium.io/use-cases/network-policy/)).
 
+[LVM](https://en.wikipedia.org/wiki/Logical_Volume_Manager_(Linux)), [DRBD](https://linbit.com/drbd/), [LINSTOR](https://github.com/LINBIT/linstor-server), and the [Piraeus Operator](https://github.com/piraeusdatastore/piraeus-operator), are used for providing persistent storage volumes.
+
 # Usage (Ubuntu 22.04 host)
 
 Install libvirt:
@@ -60,6 +62,20 @@ sudo install hubble /usr/local/bin/hubble
 rm hubble
 ```
 
+Install kubectl-linstor:
+
+```bash
+# NB kubectl linstor storage-pool list is equivalent to:
+#    kubectl -n piraeus-datastore exec deploy/linstor-controller -- linstor storage-pool list
+# see https://github.com/piraeusdatastore/kubectl-linstor/releases
+# renovate: datasource=github-releases depName=piraeusdatastore/kubectl-linstor
+kubectl_linstor_version='0.3.0'
+kubectl_linstor_url="https://github.com/piraeusdatastore/kubectl-linstor/releases/download/v${kubectl_linstor_version}/kubectl-linstor_v${kubectl_linstor_version}_linux_amd64.tar.gz"
+wget -O- "$kubectl_linstor_url" | tar xzf - kubectl-linstor
+sudo install kubectl-linstor /usr/local/bin/kubectl-linstor
+rm kubectl-linstor
+```
+
 Install talosctl:
 
 ```bash
@@ -91,6 +107,7 @@ controllers="$(terraform output -raw controllers)"
 workers="$(terraform output -raw workers)"
 all="$controllers,$workers"
 c0="$(echo $controllers | cut -d , -f 1)"
+w0="$(echo $workers | cut -d , -f 1)"
 talosctl -n $all version
 talosctl -n $all dashboard
 ```
@@ -135,6 +152,79 @@ xdg-open "$example_url"
 kubectl delete -f example.yml
 ```
 
+Execute the [example hello-etcd stateful application](https://github.com/rgl/hello-etcd):
+
+```bash
+install -d tmp/hello-etcd
+pushd tmp/hello-etcd
+wget -qO- https://raw.githubusercontent.com/rgl/hello-etcd/v0.0.2/manifest.yml \
+  | perl -pe 's,(storageClassName:).+,$1 linstor-lvm-r1,g' \
+  | perl -pe 's,(storage:).+,$1 1Gi,g' \
+  > manifest.yml
+kubectl apply -f manifest.yml
+kubectl rollout status deployment hello-etcd
+kubectl rollout status statefulset hello-etcd-etcd
+kubectl get service,statefulset,pod,pvc,pv,sc
+kubectl linstor volume list
+```
+
+Access the `hello-etcd` service from a [kubectl port-forward local port](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/):
+
+```bash
+kubectl port-forward service/hello-etcd 6789:web &
+sleep 3
+wget -qO- http://localhost:6789 # Hello World #1!
+wget -qO- http://localhost:6789 # Hello World #2!
+wget -qO- http://localhost:6789 # Hello World #3!
+```
+
+Delete the etcd pod:
+
+```bash
+# NB the used StorageClass is configured with ReclaimPolicy set to Delete. this
+#    means that, when we delete the application PersistentVolumeClaim, the
+#    volume will be deleted from the linstor storage-pool. this also means
+#    that, to play with this, we cannot delete all the application resource. we
+#    have to keep the persistent volume around by only deleting the etcd pod.
+# NB although we delete the pod, the StatefulSet will create a fresh pod to
+#    replace it. using the same persistent volume as the old one.
+kubectl delete pod/hello-etcd-etcd-0
+kubectl get pod/hello-etcd-etcd-0 # NB its age should be in the seconds range.
+kubectl rollout status deployment hello-etcd
+kubectl rollout status statefulset hello-etcd-etcd
+kubectl get pvc,pv
+kubectl linstor volume list
+```
+
+Access the application, and notice that the counter continues after the previously returned value, which means that although the etcd instance is different, it picked up the same persistent volume:
+
+```bash
+wget -qO- http://localhost:6789 # Hello World #4!
+wget -qO- http://localhost:6789 # Hello World #5!
+wget -qO- http://localhost:6789 # Hello World #6!
+```
+
+Delete everything:
+
+```bash
+kubectl delete -f manifest.yml
+kill %1 # kill the kubectl port-forward background command execution.
+# NB the persistent volume will linger for a bit, until it will be eventually
+#    reclaimed and deleted (because the StorageClass is configured with
+#    ReclaimPolicy set to Delete).
+kubectl get pvc,pv
+kubectl linstor volume list
+# force the persistent volume deletion.
+# NB if you do not do this (or wait until the persistent volume is actually
+#    deleted), the associated AWS EBS volume we be left created in your AWS
+#    account, and you have to manually delete it from there.
+kubectl delete pvc/etcd-data-hello-etcd-etcd-0
+# NB you should wait until its actually deleted.
+kubectl get pvc,pv
+kubectl linstor volume list
+popd
+```
+
 Destroy the infrastructure:
 
 ```bash
@@ -178,6 +268,8 @@ talosctl -n $c0 list -l -r /dev
 talosctl -n $c0 list -l /sys/fs/cgroup
 talosctl -n $c0 read /proc/cmdline | tr ' ' '\n'
 talosctl -n $c0 read /proc/mounts | sort
+talosctl -n $w0 read /proc/modules | sort
+talosctl -n $w0 read /sys/module/drbd/parameters/usermode_helper
 talosctl -n $c0 read /etc/os-release
 talosctl -n $c0 read /etc/resolv.conf
 talosctl -n $c0 read /etc/containerd/config.toml
@@ -217,3 +309,25 @@ kubectl --namespace kube-system run busybox -it --rm --restart=Never --image=bus
 kubectl get crds
 kubectl api-resources
 ```
+
+Storage (lvm/drbd/linstor/piraeus):
+
+```bash
+# NB kubectl linstor node list is equivalent to:
+#    kubectl -n piraeus-datastore exec deploy/linstor-controller -- linstor node list
+kubectl linstor node list
+kubectl linstor storage-pool list
+kubectl linstor volume list
+kubectl -n piraeus-datastore exec daemonset/linstor-satellite.w0 -- drbdadm status
+kubectl -n piraeus-datastore exec daemonset/linstor-satellite.w0 -- lvdisplay
+kubectl -n piraeus-datastore exec daemonset/linstor-satellite.w0 -- vgdisplay
+kubectl -n piraeus-datastore exec daemonset/linstor-satellite.w0 -- pvdisplay
+w0_csi_node_pod_name="$(
+  kubectl -n piraeus-datastore get pods \
+    --field-selector spec.nodeName=w0 \
+    --selector app.kubernetes.io/component=linstor-csi-node \
+    --output 'jsonpath={.items[*].metadata.name}')"
+kubectl -n piraeus-datastore exec "pod/$w0_csi_node_pod_name" -- lsblk
+kubectl -n piraeus-datastore exec "pod/$w0_csi_node_pod_name" -- bash -c 'mount | grep /dev/drbd'
+kubectl -n piraeus-datastore exec "pod/$w0_csi_node_pod_name" -- bash -c 'df -h | grep -P "Filesystem|/dev/drbd"'
+```
diff --git a/do b/do
@@ -6,9 +6,22 @@ set -euo pipefail
 talos_version="1.6.7"
 
 # see https://github.com/siderolabs/extensions/pkgs/container/qemu-guest-agent
+# see https://github.com/siderolabs/extensions/tree/main/guest-agents/qemu-guest-agent
 # renovate: datasource=docker depName=siderolabs/qemu-guest-agent registryUrl=https://ghcr.io
 talos_qemu_guest_agent_extension_version="8.2.2"
 
+# see https://github.com/siderolabs/extensions/pkgs/container/drbd
+# see https://github.com/siderolabs/extensions/tree/main/storage/drbd
+# see https://github.com/LINBIT/drbd
+# NB the full version version is actually $version-v$talos_version, which we
+#    use in the talos systemExtension imageRef.
+# renovate: datasource=docker depName=siderolabs/drbd extractVersion=^(?<version>.+)-v registryUrl=https://ghcr.io
+talos_drbd_extension_version="9.2.8"
+
+# see https://github.com/piraeusdatastore/piraeus-operator/releases
+# renovate: datasource=github-releases depName=piraeusdatastore/piraeus-operator
+piraeus_operator_version="2.5.0"
+
 export CHECKPOINT_DISABLE='1'
 export TF_LOG='DEBUG' # TRACE, DEBUG, INFO, WARN or ERROR.
 export TF_LOG_PATH='terraform.log'
@@ -44,6 +57,7 @@ input:
     imageRef: ghcr.io/siderolabs/installer:$talos_version_tag
   systemExtensions:
     - imageRef: ghcr.io/siderolabs/qemu-guest-agent:$talos_qemu_guest_agent_extension_version
+    - imageRef: ghcr.io/siderolabs/drbd:$talos_drbd_extension_version-v$talos_version
 output:
   kind: image
   imageOptions:
@@ -90,6 +104,7 @@ function apply {
   terraform output -raw talosconfig >talosconfig.yml
   terraform output -raw kubeconfig >kubeconfig.yml
   health
+  piraeus-install
 }
 
 function health {
@@ -104,6 +119,106 @@ function health {
   info
 }
 
+function piraeus-install {
+  # see https://github.com/piraeusdatastore/piraeus-operator
+  # see https://github.com/piraeusdatastore/piraeus-operator/blob/v2.5.0/docs/how-to/talos.md
+  # see https://github.com/piraeusdatastore/piraeus-operator/blob/v2.5.0/docs/tutorial/get-started.md
+  # see https://github.com/piraeusdatastore/piraeus-operator/blob/v2.5.0/docs/tutorial/replicated-volumes.md
+  # see https://github.com/piraeusdatastore/piraeus-operator/blob/v2.5.0/docs/explanation/components.md
+  # see https://github.com/piraeusdatastore/piraeus-operator/blob/v2.5.0/docs/reference/linstorsatelliteconfiguration.md
+  # see https://github.com/piraeusdatastore/piraeus-operator/blob/v2.5.0/docs/reference/linstorcluster.md
+  # see https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/
+  # see https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#ch-kubernetes
+  # see 5.7.1. Available Parameters in a Storage Class at https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-kubernetes-sc-parameters
+  # see https://linbit.com/drbd-user-guide/drbd-guide-9_0-en/
+  # see https://www.talos.dev/v1.6/kubernetes-guides/configuration/storage/#piraeus--linstor
+  step 'piraeus install'
+  kubectl apply --server-side -k "https://github.com/piraeusdatastore/piraeus-operator//config/default?ref=v$piraeus_operator_version"
+  step 'piraeus wait'
+  kubectl wait pod --timeout=5m --for=condition=Ready -n piraeus-datastore -l app.kubernetes.io/component=piraeus-operator
+  step 'piraeus configure'
+  kubectl apply -n piraeus-datastore -f - <<'EOF'
+apiVersion: piraeus.io/v1
+kind: LinstorSatelliteConfiguration
+metadata:
+  name: talos-loader-override
+spec:
+  podTemplate:
+    spec:
+      initContainers:
+        - name: drbd-shutdown-guard
+          $patch: delete
+        - name: drbd-module-loader
+          $patch: delete
+      volumes:
+        - name: run-systemd-system
+          $patch: delete
+        - name: run-drbd-shutdown-guard
+          $patch: delete
+        - name: systemd-bus-socket
+          $patch: delete
+        - name: lib-modules
+          $patch: delete
+        - name: usr-src
+          $patch: delete
+        - name: etc-lvm-backup
+          hostPath:
+            path: /var/etc/lvm/backup
+            type: DirectoryOrCreate
+        - name: etc-lvm-archive
+          hostPath:
+            path: /var/etc/lvm/archive
+            type: DirectoryOrCreate
+EOF
+  kubectl apply -f - <<EOF
+apiVersion: piraeus.io/v1
+kind: LinstorCluster
+metadata:
+  name: linstor
+EOF
+  kubectl apply -f - <<EOF
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+provisioner: linstor.csi.linbit.com
+metadata:
+  name: linstor-lvm-r1
+allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+parameters:
+  csi.storage.k8s.io/fstype: xfs
+  linstor.csi.linbit.com/autoPlace: "1"
+  linstor.csi.linbit.com/storagePool: lvm
+EOF
+  step 'piraeus configure wait'
+  kubectl wait pod --timeout=5m --for=condition=Ready -n piraeus-datastore -l app.kubernetes.io/name=piraeus-datastore
+  kubectl wait LinstorCluster/linstor --timeout=5m --for=condition=Available
+  step 'piraeus create-device-pool'
+  local workers="$(terraform output -raw workers)"
+  local nodes=($(echo "$workers" | tr ',' ' '))
+  for ((n=0; n<${#nodes[@]}; ++n)); do
+    local node="w$((n))"
+    local wwn="$(printf "000000000000ab%02x" $n)"
+    step "piraeus wait node $node"
+    while ! kubectl linstor storage-pool list --node "$node" >/dev/null 2>&1; do sleep 3; done
+    step "piraeus create-device-pool $node"
+    if ! kubectl linstor storage-pool list --node "$node" --storage-pool lvm | grep -q lvm; then
+      kubectl linstor physical-storage create-device-pool \
+        --pool-name lvm \
+        --storage-pool lvm \
+        lvm \
+        "$node" \
+        "/dev/disk/by-id/wwn-0x$wwn"
+    fi
+  done
+  step 'piraeus node list'
+  kubectl linstor node list
+  step 'piraeus storage-pool list'
+  kubectl linstor storage-pool list
+  step 'piraeus volume list'
+  kubectl linstor volume list
+}
+
 function info {
   local controllers="$(terraform output -raw controllers)"
   local workers="$(terraform output -raw workers)"

diff --git a/libvirt.tf b/libvirt.tf
@@ -31,6 +31,14 @@ resource "libvirt_volume" "worker" {
   size             = 40 * 1024 * 1024 * 1024 # 40GiB.
 }
 
+# see https://github.com/dmacvicar/terraform-provider-libvirt/blob/v0.7.6/website/docs/r/volume.html.markdown
+resource "libvirt_volume" "worker_data0" {
+  count  = var.worker_count
+  name   = "${var.prefix}_w${count.index}d0.img"
+  format = "qcow2"
+  size   = 32 * 1024 * 1024 * 1024 # 32GiB.
+}
+
 # see https://github.com/dmacvicar/terraform-provider-libvirt/blob/v0.7.6/website/docs/r/domain.html.markdown
 resource "libvirt_domain" "controller" {
   count      = var.controller_count
@@ -42,7 +50,7 @@ resource "libvirt_domain" "controller" {
     mode = "host-passthrough"
   }
   vcpu   = 4
-  memory = 2 * 1024
+  memory = 4 * 1024
   video {
     type = "qxl"
   }
@@ -83,6 +91,11 @@ resource "libvirt_domain" "worker" {
     volume_id = libvirt_volume.worker[count.index].id
     scsi      = true
   }
+  disk {
+    volume_id = libvirt_volume.worker_data0[count.index].id
+    scsi      = true
+    wwn       = format("000000000000ab%02x", count.index)
+  }
   network_interface {
     network_id     = libvirt_network.talos.id
     addresses      = [local.worker_nodes[count.index].address]

diff --git a/talos.tf b/talos.tf
@@ -27,6 +27,20 @@ locals {
           port    = 7445
         }
       }
+      kernel = {
+        modules = [
+          // piraeus dependencies.
+          {
+            name = "drbd"
+            parameters = [
+              "usermode_helper=disabled",
+            ]
+          },
+          {
+            name = "drbd_transport_tcp"
+          },
+        ]
+      }
     }
     cluster = {
       # see https://www.talos.dev/v1.6/talos-guides/discovery/