sys: add storage node config

- Use a common template for nodes kubelets and kubelet configs
- Install disk-mounter and not rely on it being installed by another systemd

cfssl.tf

@@ -67,39 +67,39 @@ EOS
 }
 
 data "ignition_file" "cfssl-init-ca" {
-  mode       = 493
+  mode = 493
   filesystem = "root"
-  path       = "/opt/bin/cfssl-init-ca"
+  path = "/opt/bin/cfssl-init-ca"
 
   content {
     content = file("${path.module}/resources/cfssl-init-ca.sh")
   }
 }
 
 data "ignition_file" "cfssl-init-proxy-pki" {
-  mode       = 493
+  mode = 493
   filesystem = "root"
-  path       = "/opt/bin/cfssl-init-proxy-pki"
+  path = "/opt/bin/cfssl-init-proxy-pki"
 
   content {
     content = file("${path.module}/resources/cfssl-init-proxy-pki")
   }
 }
 
 data "ignition_file" "cfssl-proxy-ca-csr-json" {
-  mode       = 420
+  mode = 420
   filesystem = "root"
-  path       = "/etc/cfssl/proxy-ca-csr.json"
+  path = "/etc/cfssl/proxy-ca-csr.json"
 
   content {
     content = file("${path.module}/resources/cfssl-proxy-ca-csr.json")
   }
 }
 
 data "ignition_file" "cfssl-proxy-csr-json" {
-  mode       = 420
+  mode = 420
   filesystem = "root"
-  path       = "/etc/cfssl/proxy-csr.json"
+  path = "/etc/cfssl/proxy-csr.json"
 
   content {
     content = file("${path.module}/resources/cfssl-proxy-csr.json")
@@ -110,31 +110,31 @@ data "template_file" "cfssl-server-config" {
   template = file("${path.module}/resources/cfssl-server-config.json")
 
   vars = {
-    expiry_hours     = var.cfssl_node_expiry_hours
+    expiry_hours = var.cfssl_node_expiry_hours
     cfssl_unused_key = random_id.cfssl-auth-key-unused.hex
-    cfssl_auth_key   = random_id.cfssl-auth-key-client.hex
+    cfssl_auth_key = random_id.cfssl-auth-key-client.hex
   }
 }
 
 data "ignition_file" "cfssl-server-config" {
-  mode       = 384
+  mode = 384
   filesystem = "root"
-  path       = "/etc/cfssl/config.json"
+  path = "/etc/cfssl/config.json"
 
   content {
     content = data.template_file.cfssl-server-config.rendered
   }
 }
 
 data "ignition_systemd_unit" "cfssl" {
-  name    = "cfssl.service"
+  name = "cfssl.service"
   content = file("${path.module}/resources/cfssl.service")
 }
 
 data "ignition_file" "cfssl-sk-csr" {
-  mode       = 420
+  mode = 420
   filesystem = "root"
-  path       = "/etc/cfssl/sk-csr.json"
+  path = "/etc/cfssl/sk-csr.json"
 
   content {
     content = <<EOS

etcd.tf

@@ -133,28 +133,28 @@ data "ignition_config" "etcd" {
   count = length(var.etcd_addresses)
 
   files = concat(
-      [
-        data.ignition_file.cfssl.id,
-        data.ignition_file.cfssljson.id,
-        data.ignition_file.cfssl-client-config.id,
-        element(data.ignition_file.etcd-cfssl-new-cert.*.id, count.index),
-        data.ignition_file.etcd-prom-machine-role.id,
-        element(data.ignition_file.etcdctl-wrapper.*.id, count.index),
-        data.ignition_file.format-and-mount.id
-      ],
-      var.etcd_additional_files
-    )
+    [
+      data.ignition_file.cfssl.id,
+      data.ignition_file.cfssljson.id,
+      data.ignition_file.cfssl-client-config.id,
+      element(data.ignition_file.etcd-cfssl-new-cert.*.id, count.index),
+      data.ignition_file.etcd-prom-machine-role.id,
+      element(data.ignition_file.etcdctl-wrapper.*.id, count.index),
+      data.ignition_file.format-and-mount.id
+    ],
+    var.etcd_additional_files
+  )
 
   systemd = concat(
-      [
-        data.ignition_systemd_unit.update-engine.id,
-        data.ignition_systemd_unit.locksmithd_etcd.id,
-        data.ignition_systemd_unit.docker-opts-dropin.id,
-        data.ignition_systemd_unit.node-exporter.id,
-        element(data.ignition_systemd_unit.etcd-member-dropin.*.id, count.index),
-        element(data.ignition_systemd_unit.etcd-disk-mounter.*.id, count.index)
-      ],
-      module.etcd-cert-fetcher.systemd_units,
-      var.etcd_additional_systemd_units
-    )
+    [
+      data.ignition_systemd_unit.update-engine.id,
+      data.ignition_systemd_unit.locksmithd_etcd.id,
+      data.ignition_systemd_unit.docker-opts-dropin.id,
+      data.ignition_systemd_unit.node-exporter.id,
+      element(data.ignition_systemd_unit.etcd-member-dropin.*.id, count.index),
+      element(data.ignition_systemd_unit.etcd-disk-mounter.*.id, count.index)
+    ],
+    module.etcd-cert-fetcher.systemd_units,
+    var.etcd_additional_systemd_units
+  )
 }

node-common.tf

@@ -0,0 +1,115 @@
+// All nodes should belong to system:nodes group
+data "template_file" "node-cfssl-new-cert" {
+  template = file("${path.module}/resources/cfssl-new-cert.sh")
+
+  vars = {
+    cert_name   = "node"
+    user        = "root"
+    group       = "root"
+    profile     = "client"
+    path        = "/etc/kubernetes/ssl"
+    cn          = "system:node:$(${var.node_name_command[var.cloud_provider]})"
+    org         = "system:nodes"
+    get_ip      = var.get_ip_command[var.cloud_provider]
+    extra_names = ""
+  }
+}
+
+data "ignition_file" "node-cfssl-new-cert" {
+  mode       = 493
+  filesystem = "root"
+  path       = "/opt/bin/cfssl-new-cert"
+
+  content {
+    content = data.template_file.node-cfssl-new-cert.rendered
+  }
+}
+
+// Kubeconfig will be the same for all kubernetes nodes as it only
+// contains master address and certs
+data "template_file" "node-kubeconfig" {
+  template = file("${path.module}/resources/node-kubeconfig")
+
+  vars = {
+    master_address = var.master_address
+  }
+}
+
+data "ignition_file" "node-kubeconfig" {
+  mode       = 420
+  filesystem = "root"
+  path       = "/var/lib/kubelet/kubeconfig"
+
+  content {
+    content = data.template_file.node-kubeconfig.rendered
+  }
+}
+
+// Kubelet config
+data "template_file" "node-kubelet-conf" {
+  template = file("${path.module}/resources/node-kubelet-conf.yaml")
+
+  vars = {
+    cluster_dns   = local.cluster_dns_yaml
+    feature_gates = local.feature_gates_yaml_fragment
+  }
+}
+
+data "ignition_file" "node-kubelet-conf" {
+  mode       = 420
+  filesystem = "root"
+  path       = "/etc/kubernetes/config/node-kubelet-conf.yaml"
+
+  content {
+    content = data.template_file.node-kubelet-conf.rendered
+  }
+}
+
+
+data "ignition_file" "node-sysctl-vm" {
+  mode       = 420
+  filesystem = "root"
+  path       = "/etc/sysctl.d/vm.conf"
+
+  content {
+    content = "vm.max_map_count=262144"
+  }
+}
+
+// Common prometheus text-collector metrics for nodes
+data "template_file" "prometheus-tmpfs-dir" {
+  template = file("${path.module}/resources/prometheus-tmpfs-dir.service")
+}
+
+data "ignition_systemd_unit" "prometheus-tmpfs-dir" {
+  name    = "prometheus-tmpfs-dir.service"
+  content = data.template_file.prometheus-tmpfs-dir.rendered
+}
+
+data "template_file" "prometheus-ro-rootfs" {
+  template = file("${path.module}/resources/prometheus-ro-rootfs.service")
+}
+
+data "ignition_systemd_unit" "prometheus-ro-rootfs" {
+  name    = "prometheus-ro-rootfs.service"
+  content = data.template_file.prometheus-ro-rootfs.rendered
+}
+
+data "template_file" "prometheus-ro-rootfs-timer" {
+  template = file("${path.module}/resources/prometheus-ro-rootfs.timer")
+}
+
+data "ignition_systemd_unit" "prometheus-ro-rootfs-timer" {
+  name    = "prometheus-ro-rootfs.timer"
+  content = data.template_file.prometheus-ro-rootfs-timer.rendered
+}
+
+data "ignition_file" "prometheus-ro-rootfs" {
+  mode       = 493
+  filesystem = "root"
+  path       = "/opt/bin/prometheus-ro-rootfs"
+
+  content {
+    content = file("${path.module}/resources/prometheus-ro-rootfs")
+  }
+}

outputs.tf

@@ -10,6 +10,10 @@ output "worker" {
   value = data.ignition_config.worker.rendered
 }
 
+output "storage_node" {
+  value = data.ignition_config.storage-node.rendered
+}
+
 output "etcd" {
   value = data.ignition_config.etcd.*.rendered
 }
@@ -39,10 +43,18 @@ output "worker_ignition_files" {
   value = data.ignition_config.worker.files
 }
 
+output "storage_node_ignition_systemd" {
+  value = data.ignition_config.storage-node.systemd
+}
+
+output "storage_node_ignition_files" {
+  value = data.ignition_config.storage-node.files
+}
+
 output "etcd_ignition_systemd" {
-  value = [data.ignition_config.etcd.*.systemd]
+  value = data.ignition_config.etcd.*.systemd
 }
 
 output "etcd_ignition_files" {
-  value = [data.ignition_config.etcd.*.files]
+  value = data.ignition_config.etcd.*.files
 }

resources/disk-mounter.service

@@ -4,3 +4,5 @@ Description=Mounts device on mountpoint, formatting it if necessary
 Type=oneshot
 RemainAfterExit=yes
 ExecStart=${script_path} ${volume_id} ${filesystem} ${user} ${group} ${mountpoint}
+[Install]
+WantedBy=multi-user.target

resources/node-kubelet.service

@@ -0,0 +1,39 @@
+[Unit]
+Description=Kubernetes Kubelet
+Requires=docker.service
+After=docker.service
+[Service]
+ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/manifests
+ExecStartPre=/usr/bin/mkdir -p /var/log/containers
+ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin
+ExecStartPre=/usr/bin/mkdir -p /var/lib/cni
+ExecStartPre=/usr/bin/mkdir -p /etc/cni/net.d
+ExecStartPre=/usr/bin/mkdir -p /var/run/calico
+ExecStartPre=/usr/bin/mkdir -p /var/lib/calico
+# This is a partial workaround to this upstream Kubernetes issue:
+#  https://github.com/kubernetes/kubernetes/issues/41916#issuecomment-312428731
+ExecStartPre=/sbin/sysctl -w net.ipv4.tcp_retries2=8
+# This is a temporal workaround this upstream Kubernetes issue:
+#  https://github.com/kubernetes/kubernetes/issues/69015
+ExecStartPre=/sbin/sysctl -w fs.inotify.max_user_watches=524288
+ExecStartPre=/opt/bin/cfssl-new-cert
+ExecStart=${kubelet_binary_path} \
+  --allow-privileged \
+%{ if cloud_provider != "" }  --cloud-provider=${cloud_provider} \
+%{ endif ~}
+  --cni-bin-dir=/opt/cni/bin \
+  --cni-conf-dir=/etc/cni/net.d \
+  --config=/etc/kubernetes/config/node-kubelet-conf.yaml \
+  --container-runtime=docker \
+  --exit-on-lock-contention \
+  --kubeconfig=/var/lib/kubelet/kubeconfig \
+  --network-plugin=cni \
+  --node-labels=role=${role} \
+%{ if taints != "" }  --register-with-taints=${taints} \
+%{ endif ~}
+  --lock-file=/var/run/lock/kubelet.lock \
+  --v=0
+Restart=always
+RestartSec=10
+[Install]
+WantedBy=multi-user.target