Add support for Tinkerbell platform

Commit message to be added.

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

assets/lokomotive-kubernetes/bootkube/variables-asset_dir.tf

@@ -0,0 +1 @@
+../terraform-modules/asset_dir/variables.tf

assets/lokomotive-kubernetes/bootkube/variables-cluster_name.tf

@@ -0,0 +1 @@
+../terraform-modules/cluster_name/variables.tf

assets/lokomotive-kubernetes/bootkube/variables-kubernetes.tf

@@ -0,0 +1 @@
+../terraform-modules/kubernetes/variables.tf

assets/lokomotive-kubernetes/bootkube/variables-platform.tf

@@ -0,0 +1 @@
+../terraform-modules/platform/variables.tf

assets/lokomotive-kubernetes/bootkube/variables.tf

@@ -1,8 +1,3 @@
-variable "cluster_name" {
-  description = "Cluster name"
-  type        = string
-}
-
 variable "api_servers" {
   description = "List of domain names used to reach kube-apiserver from within the cluster"
   type        = list(string)
@@ -26,23 +21,12 @@ variable "etcd_servers" {
   type        = list(string)
 }
 
-variable "asset_dir" {
-  description = "Path to a directory where generated assets should be placed (contains secrets)"
-  type        = string
-}
-
 variable "cloud_provider" {
   description = "The provider for cloud services (empty string for no provider)"
   type        = string
   default     = ""
 }
 
-variable "network_mtu" {
-  description = "CNI interface MTU"
-  type        = number
-  default     = 1500
-}
-
 variable "network_encapsulation" {
   description = "Network encapsulation mode either ipip or vxlan (only applies to calico)"
   type        = string
@@ -55,29 +39,6 @@ variable "network_ip_autodetection_method" {
   default     = "first-found"
 }
 
-variable "pod_cidr" {
-  description = "CIDR IP range to assign Kubernetes pods"
-  type        = string
-  default     = "10.2.0.0/16"
-}
-
-variable "service_cidr" {
-  description = <<EOD
-CIDR IP range to assign Kubernetes services.
-The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
-EOD
-
-
-  type    = string
-  default = "10.3.0.0/24"
-}
-
-variable "cluster_domain_suffix" {
-  description = "Queries for domains with the suffix will be answered by kube-dns"
-  type        = string
-  default     = "cluster.local"
-}
-
 variable "container_arch" {
   description = "Architecture suffix for the container image coredns/coredns:coredns- (e.g., arm64)"
   type        = string
@@ -102,30 +63,12 @@ variable "container_images" {
   }
 }
 
-variable "enable_reporting" {
-  type        = bool
-  description = "Enable usage or analytics reporting to upstream component owners (Tigera: Calico)"
-  default     = false
-}
-
 variable "trusted_certs_dir" {
   description = "Path to the directory on cluster nodes where trust TLS certs are kept"
   type        = string
   default     = "/usr/share/ca-certificates"
 }
 
-variable "certs_validity_period_hours" {
-  description = "Validity of all the certificates in hours"
-  type        = number
-  default     = 8760
-}
-
-variable "enable_aggregation" {
-  description = "Enable the Kubernetes Aggregation Layer (defaults to false, recommended)"
-  type        = bool
-  default     = false
-}
-
 # unofficial, temporary, may be removed without notice
 
 variable "external_apiserver_port" {

assets/lokomotive-kubernetes/terraform-modules/asset_dir/variables.tf

@@ -0,0 +1,4 @@
+variable "asset_dir" {
+  description = "Path to a directory where generated assets should be placed (contains secrets)"
+  type        = string
+}

assets/lokomotive-kubernetes/terraform-modules/cluster_name/variables.tf

@@ -0,0 +1,4 @@
+variable "cluster_name" {
+  description = "Cluster name"
+  type        = string
+}

assets/lokomotive-kubernetes/terraform-modules/controller/main.tf

@@ -0,0 +1,72 @@
+locals {
+  kubelet_require_kubeconfig = <<EOF
+systemd:
+  units:
+  - name: kubelet.service
+    dropins:
+    - name: 10-controller.conf
+      contents: |
+        [Service]
+        ConditionPathExists=/etc/kubernetes/kubeconfig
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests
+EOF
+
+  bootkube = templatefile("${path.module}/templates/bootkube.yaml.tmpl", {
+    bootkube_rkt_extra_args = var.bootkube_rkt_extra_args
+    bootkube_image_name     = var.bootkube_image_name
+    bootkube_image_tag      = var.bootkube_image_tag
+    kubelet_image_name      = var.kubelet_image_name
+    kubelet_image_tag       = var.kubelet_image_tag
+  })
+
+  snippets = [
+    local.kubelet_require_kubeconfig,
+    local.bootkube,
+  ]
+}
+
+data "ct_config" "config" {
+  count = var.node_count
+
+  pretty_print = false
+
+  content = templatefile("${path.module}/templates/node.yaml.tmpl", {
+    ssh_keys               = jsonencode(var.ssh_keys)
+    cluster_dns_service_ip = var.cluster_dns_service_ip
+    cluster_domain_suffix  = var.cluster_domain_suffix
+    kubelet_image_name     = var.kubelet_image_name
+    kubelet_image_tag      = var.kubelet_image_tag
+    kubelet_rkt_extra_args = []
+    kubelet_labels = [
+      "node.kubernetes.io/master",
+      "node.kubernetes.io/controller=true",
+    ]
+    kubelet_taints = [
+      "node-role.kubernetes.io/master=:NoSchedule"
+    ]
+  })
+
+  snippets = concat(local.snippets, var.clc_snippets, [
+    templatefile("${path.module}/templates/etcd.yaml.tmpl", {
+      etcd_name   = "etcd${count.index}"
+      etcd_domain = data.template_file.etcds[count.index].rendered
+      # etcd0=https://cluster-etcd0.example.com,etcd1=https://cluster-etcd1.example.com,...
+      etcd_initial_cluster = join(",", [for i, name in data.template_file.etcds.*.rendered : format("etcd%d=https://%s:2380", i, name)])
+    }),
+    # Allow to pass unique snippets per controller node. For example, to set the hostname.
+    format(var.clc_snippet_index, count.index),
+  ])
+}
+
+data "template_file" "etcds" {
+  count = var.node_count
+
+  template = "$${cluster_name}-etcd$${index}.$${dns_zone}"
+
+  vars = {
+    index        = count.index
+    cluster_name = var.cluster_name
+    dns_zone     = var.dns_zone
+  }
+}

assets/lokomotive-kubernetes/terraform-modules/controller/output-node.tf

@@ -0,0 +1 @@
+../node/output.tf

assets/lokomotive-kubernetes/terraform-modules/controller/output.tf

@@ -0,0 +1,3 @@
+output "etcd_servers" {
+  value = data.template_file.etcds.*.rendered
+}

assets/lokomotive-kubernetes/terraform-modules/controller/templates/bootkube.yaml.tmpl

@@ -0,0 +1,47 @@
+systemd:
+  units:
+  - name: bootkube.service
+    contents: |
+      [Unit]
+      Description=Bootstrap a Kubernetes cluster
+      ConditionPathExists=!/opt/bootkube/init_bootkube.done
+      [Service]
+      Type=oneshot
+      RemainAfterExit=true
+      WorkingDirectory=/opt/bootkube
+      ExecStart=/opt/bootkube/bootkube-start
+      ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done
+      [Install]
+      WantedBy=multi-user.target
+storage:
+  files:
+  - path: /opt/bootkube/bootkube-start
+    filesystem: root
+    mode: 0544
+    user:
+      id: 500
+    group:
+      id: 500
+    contents:
+      inline: |
+        #!/bin/bash
+        # Wrapper for bootkube start
+        set -e
+        # Pre-pull hyperkube image because when it is later pulled but takes too long it times out
+        docker pull ${kubelet_image_name}:${kubelet_image_tag}
+        # Move experimental manifests
+        [ -n "$(ls /opt/bootkube/assets/manifests-*/* 2>/dev/null)" ] && mv /opt/bootkube/assets/manifests-*/* /opt/bootkube/assets/manifests && rm -rf /opt/bootkube/assets/manifests-*
+        exec /usr/bin/rkt run \
+          --trust-keys-from-https \
+          --volume assets,kind=host,source=/opt/bootkube/assets \
+          --mount volume=assets,target=/assets \
+          --volume bootstrap,kind=host,source=/etc/kubernetes \
+          --mount volume=bootstrap,target=/etc/kubernetes \
+          %{~ for arg in bootkube_rkt_extra_args ~}
+          ${ arg } \
+          %{~ endfor ~}
+          $${RKT_OPTS} \
+          ${bootkube_image_name}:${bootkube_image_tag} \
+          --net=host \
+          --dns=host \
+          --exec=/bootkube -- start --asset-dir=/assets "$@"

assets/lokomotive-kubernetes/terraform-modules/controller/templates/etcd.yaml.tmpl

@@ -0,0 +1,82 @@
+systemd:
+  units:
+  - name: etcd-member.service
+    enable: true
+    dropins:
+    - name: 40-etcd-cluster.conf
+      contents: |
+        [Service]
+        Environment="ETCD_IMAGE_TAG=v3.4.7"
+        Environment="ETCD_IMAGE_URL=docker://quay.io/coreos/etcd"
+        Environment="RKT_RUN_ARGS=--insecure-options=image"
+        Environment="ETCD_NAME=${etcd_name}"
+        Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
+        Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
+        Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
+        Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
+        Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
+        Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
+        Environment="ETCD_STRICT_RECONFIG_CHECK=true"
+        Environment="ETCD_SSL_DIR=/etc/ssl/etcd"
+        Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
+        Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
+        Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
+        Environment="ETCD_CLIENT_CERT_AUTH=true"
+        Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
+        Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
+        Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
+        Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
+        ExecStopPost=-/opt/etcd-rejoin
+        [Unit]
+        Requires=wait-for-dns.service
+storage:
+  files:
+  - path: /opt/etcd-rejoin
+    filesystem: root
+    mode: 0555
+    contents:
+      inline: |
+        #!/bin/bash
+        set -eou pipefail
+        # Rejoin a cluster as fresh node when etcd cannot join
+        # (e.g., after repovisioning, crashing or node being down).
+        # Set ExecStopPost=-/opt/etcd-rejoin to run when etcd failed and
+        # use env vars of etcd-member.service.
+        # Skip if not provisioned
+        if [ ! -d "/etc/ssl/etcd/" ]; then exit 0; fi
+        # or got stopped.
+        if [ "$EXIT_CODE" = "killed" ]; then exit 0; fi
+        now=$(date +%s)
+        if [ -f /var/lib/etcd-last-fail ]; then
+          last=$(cat /var/lib/etcd-last-fail)
+        else
+          last=0
+        fi
+        echo "$now" > /var/lib/etcd-last-fail
+        let "d = $now - $last"
+        # Skip and restart regularly if it does not fail within 120s.
+        if [ "$d" -gt 120 ]; then exit 0; fi
+        export ETCDCTL_API=3
+        urls=$(echo "$ETCD_INITIAL_CLUSTER" | tr "," "\n" | cut -d "=" -f 2 | tr "\n" "," | head -c -1)
+        # $$ for terraform
+        endpoints="$${urls//2380/2379}"
+        ARGS="--cacert=/etc/ssl/etcd/etcd-client-ca.crt --cert=/etc/ssl/etcd/etcd-client.crt --key=/etc/ssl/etcd/etcd-client.key --endpoints=$endpoints"
+        # Check if unhealthy (should be because etcd is not running)
+        unhealty=$((etcdctl endpoint health $ARGS 2> /dev/stdout | grep "is unhealthy" | grep "$ETCD_NAME") || true)
+        if [ -z "$unhealty" ]; then exit 0; fi
+        # Remove old ID if still exists
+        ID=$((etcdctl member list $ARGS | grep "$ETCD_NAME" | cut -d "," -f 1) || true)
+        if [ ! -z "$ID" ]; then
+          etcdctl member remove "$ID" $ARGS
+        fi
+        # Re-add as new member
+        etcdctl member add "$ETCD_NAME" --peer-urls="$ETCD_INITIAL_ADVERTISE_PEER_URLS" $ARGS
+        # Join fresh without state
+        mv /var/lib/etcd "/var/lib/etcd-bkp-$(date +%s)" || true
+        if [ -z "$(grep ETCD_INITIAL_CLUSTER_STATE=existing /etc/systemd/system/etcd-member.service.d/40-etcd-cluster.conf)" ]; then
+          echo 'Environment="ETCD_INITIAL_CLUSTER_STATE=existing"' >> /etc/systemd/system/etcd-member.service.d/40-etcd-cluster.conf
+          # Apply change
+          systemctl daemon-reload
+        fi
+        # Restart unit (yes, within itself)
+        systemctl restart etcd-member &

assets/lokomotive-kubernetes/terraform-modules/controller/templates/node.yaml.tmpl

@@ -0,0 +1 @@
+../../node/templates/node.yaml.tmpl

assets/lokomotive-kubernetes/terraform-modules/controller/variables-cluster_name.tf

@@ -0,0 +1 @@
+../cluster_name/variables.tf

assets/lokomotive-kubernetes/terraform-modules/controller/variables-kubernetes.tf

@@ -0,0 +1 @@
+../kubernetes/variables.tf

assets/lokomotive-kubernetes/terraform-modules/controller/variables-node.tf

@@ -0,0 +1 @@
+../node/variables.tf

assets/lokomotive-kubernetes/terraform-modules/controller/variables.tf

@@ -0,0 +1,21 @@
+variable "dns_zone" {
+  type = string
+}
+
+variable "bootkube_rkt_extra_args" {
+  description = "Extra parameters to pass to bootkube rkt container"
+  type        = list(string)
+  default     = []
+}
+
+variable "bootkube_image_name" {
+  description = "Docker image name to use for rkt container running bootkube"
+  type        = string
+  default     = "quay.io/kinvolk/bootkube"
+}
+
+variable "bootkube_image_tag" {
+  description = "Docker image tag to use for rkt container running bootkube"
+  type        = string
+  default     = "v0.14.0-helm-amd64"
+}

assets/lokomotive-kubernetes/terraform-modules/controller/versions-node.tf

@@ -0,0 +1 @@
+../node/versions.tf

assets/lokomotive-kubernetes/terraform-modules/controller/versions.tf

@@ -0,0 +1,7 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_providers {
+    template = "~> 2.1"
+  }
+}

assets/lokomotive-kubernetes/terraform-modules/kubernetes/variables.tf

@@ -0,0 +1,5 @@
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Cluster domain suffix. Passed to kubelet as --cluster_domain flag."
+  default     = "cluster.local"
+}

assets/lokomotive-kubernetes/terraform-modules/node/main.tf

@@ -0,0 +1,21 @@
+data "ct_config" "config" {
+  count = var.node_count
+
+  pretty_print = false
+
+  content = templatefile("${path.module}/templates/node.yaml.tmpl", {
+    ssh_keys               = jsonencode(var.ssh_keys)
+    cluster_dns_service_ip = var.cluster_dns_service_ip
+    cluster_domain_suffix  = var.cluster_domain_suffix
+    kubelet_image_name     = var.kubelet_image_name
+    kubelet_image_tag      = var.kubelet_image_tag
+    kubelet_labels         = var.kubelet_labels
+    kubelet_taints         = var.kubelet_taints
+    kubelet_rkt_extra_args = var.kubelet_rkt_extra_args
+  })
+
+  snippets = concat(var.clc_snippets, [
+    # Allow to pass unique snippets per controller node. For example, to set the hostname.
+    var.clc_snippet_index != "" ? format(var.clc_snippet_index, count.index) : "",
+  ])
+}