Add the kvm-libvirt platform to lokoctl

README.md

@@ -35,6 +35,7 @@ Follow one of the quickstart guides for the supported platforms:
 * [Packet quickstart](docs/quickstarts/packet.md)
 * [AWS quickstart](docs/quickstarts/aws.md)
 * [Bare metal quickstart](docs/quickstarts/baremetal.md)
+* [KVM libvirt quickstart](docs/quickstarts/kvm-libvirt.md)
 
 ## Documentation
 

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootkube.tf

@@ -1,3 +1,8 @@
+locals {
+  # api_server = module.bootkube.api_servers
+  api_server = data.template_file.controllernames[0].rendered
+}
+
 module "bootkube" {
   source       = "../../../bootkube"
   cluster_name = var.cluster_name
@@ -19,6 +24,14 @@ module "bootkube" {
   enable_aggregation    = var.enable_aggregation
   encrypt_pod_traffic   = var.encrypt_pod_traffic
 
+  # Disable the self hosted kubelet.
+  disable_self_hosted_kubelet = var.disable_self_hosted_kubelet
+  # Extra flags to API server.
+  kube_apiserver_extra_flags = var.kube_apiserver_extra_flags
+
+  bootstrap_tokens     = var.enable_tls_bootstrap ? concat([local.controller_bootstrap_token], var.worker_bootstrap_tokens) : []
+  enable_tls_bootstrap = var.enable_tls_bootstrap
+
   certs_validity_period_hours = var.certs_validity_period_hours
 }
 

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootstrap-configs.tf

@@ -0,0 +1,24 @@
+locals {
+  controller_bootstrap_token = var.enable_tls_bootstrap ? {
+    token_id     = random_string.bootstrap_token_id[0].result
+    token_secret = random_string.bootstrap_token_secret[0].result
+  } : {}
+}
+
+# Generate a cryptographically random token id (public).
+resource random_string "bootstrap_token_id" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource random_string "bootstrap_token_secret" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 16
+  upper   = false
+  special = false
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -88,7 +88,13 @@ systemd:
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
+          %{~ if enable_tls_bootstrap ~}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          %{~ else ~}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          %{~ endif ~}
           --lock-file=/var/run/lock/kubelet.lock \
           --hostname-override=${etcd_domain} \
           --network-plugin=cni \

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/controllers.tf

@@ -11,7 +11,7 @@ resource "libvirt_pool" "volumetmp" {
 
 resource "libvirt_volume" "base" {
   name   = "${var.cluster_name}-base"
-  source = var.os_image_unpacked
+  source = var.os_image
   pool   = libvirt_pool.volumetmp.name
   format = "qcow2"
 }
@@ -83,10 +83,16 @@ data "template_file" "controller-configs" {
     etcd_name              = "${var.cluster_name}-controller-${count.index}"
     etcd_domain            = "${var.cluster_name}-controller-${count.index}.${var.machine_domain}"
     etcd_initial_cluster   = join(",", data.template_file.etcds.*.rendered)
-    kubeconfig             = indent(10, module.bootkube.kubeconfig-kubelet)
+    kubeconfig = var.enable_tls_bootstrap ? indent(10, templatefile("${path.module}/workers/cl/bootstrap-kubeconfig.yaml.tmpl", {
+      token_id     = random_string.bootstrap_token_id[0].result
+      token_secret = random_string.bootstrap_token_secret[0].result
+      ca_cert      = module.bootkube.ca_cert
+      server       = "https://${local.api_server}:6443"
+    })) : indent(10, module.bootkube.kubeconfig-kubelet)
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
+    enable_tls_bootstrap   = var.enable_tls_bootstrap
   }
 }
 

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/outputs.tf

@@ -6,18 +6,6 @@ output "kubeconfig" {
   value = module.bootkube.kubeconfig-kubelet
 }
 
-output "machine_domain" {
-  value = var.machine_domain
-}
-
-output "cluster_name" {
-  value = var.cluster_name
-}
-
-output "ssh_keys" {
-  value = var.ssh_keys
-}
-
 output "libvirtpool" {
   value = libvirt_pool.volumetmp.name
 }
@@ -50,3 +38,15 @@ output "calico_values" {
 output "lokomotive_values" {
   value = module.bootkube.lokomotive_values
 }
+
+output "ca_cert" {
+  value = module.bootkube.ca_cert
+}
+
+output "bootstrap-secrets_values" {
+  value = module.bootkube.bootstrap-secrets_values
+}
+
+output "apiserver" {
+  value = local.api_server
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/variables.tf

@@ -6,7 +6,7 @@ variable "cluster_name" {
 }
 
 # Nodes
-variable "os_image_unpacked" {
+variable "os_image" {
   type        = string
   description = "Path to unpacked Flatcar Container Linux image flatcar_production_qemu_image.img (probably after a qemu-img resize IMG +5G)"
 }
@@ -105,6 +105,17 @@ variable "enable_aggregation" {
   default     = true
 }
 
+variable "kube_apiserver_extra_flags" {
+  description = "Extra flags passed to self-hosted kube-apiserver."
+  type        = list(string)
+  default     = []
+}
+
+variable "disable_self_hosted_kubelet" {
+  description = "Disable the self hosted kubelet installed by default"
+  type        = bool
+}
+
 # Certificates
 
 variable "certs_validity_period_hours" {
@@ -118,3 +129,13 @@ variable "encrypt_pod_traffic" {
   type        = bool
   default     = false
 }
+
+variable "worker_bootstrap_tokens" {
+  description = "List of token-id and token-secret of each node."
+  type        = list(any)
+}
+
+variable "enable_tls_bootstrap" {
+  description = "Enable TLS Bootstrap for Kubelet."
+  type        = bool
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/versions.tf

@@ -1,13 +1,23 @@
 # Terraform version and plugin versions
 
 terraform {
-  required_version = ">= 0.12.0"
+  required_version = ">= 0.13"
 
   required_providers {
-    ct       = "0.6.0"
+
+    ct = {
+      source  = "poseidon/ct"
+      version = "0.6.1"
+    }
+
+    libvirt = {
+      source = "dmacvicar/libvirt"
+      uri     = "qemu:///system"
+      version = "0.6.2"
+    }
+
     null     = "2.1.2"
     template = "2.1.2"
-    libvirt  = "0.6.0"
     random   = "2.3.0"
   }
 }

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/bootstrap-configs.tf

@@ -0,0 +1,24 @@
+locals {
+  worker_bootstrap_token = var.enable_tls_bootstrap ? {
+    token_id     = random_string.bootstrap_token_id[0].result
+    token_secret = random_string.bootstrap_token_secret[0].result
+  } : {}
+}
+
+# Generate a cryptographically random token id (public).
+resource random_string "bootstrap_token_id" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource random_string "bootstrap_token_secret" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 16
+  upper   = false
+  special = false
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/bootstrap-kubeconfig.yaml.tmpl

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: kubelet
+  user:
+    token: ${token_id}.${token_secret}
+contexts:
+- context:
+    cluster: local
+    user: kubelet

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -65,7 +65,13 @@ systemd:
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
+          %{~ if enable_tls_bootstrap ~}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          %{~ else ~}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          %{~ endif ~}
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=$${NODE_LABELS} \
@@ -146,7 +152,11 @@ storage:
             --net=host \
             --dns=host \
             -- \
+            %{~ if enable_tls_bootstrap ~}
+            kubectl --kubeconfig=/var/lib/kubelet/kubeconfig delete node $(hostname)
+            %{~ else ~}
             kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
+            %{~ endif ~}
 passwd:
   users:
     - name: core

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/outputs.tf

@@ -0,0 +1,3 @@
+output "worker_bootstrap_token" {
+  value = local.worker_bootstrap_token
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/variables.tf

@@ -83,3 +83,18 @@ EOD
   type    = string
   default = "10.2.0.0/16"
 }
+
+variable "ca_cert" {
+  description = "Kubernetes CA certificate needed in the kubeconfig file."
+  type        = string
+}
+
+variable "apiserver" {
+  description = "Apiserver private endpoint needed in the kubeconfig file."
+  type        = string
+}
+
+variable "enable_tls_bootstrap" {
+  description = "Enable TLS Bootstrap for Kubelet."
+  type        = bool
+}

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/versions.tf

@@ -1,11 +1,19 @@
 # Terraform version and plugin versions
 
 terraform {
-  required_version = ">= 0.12.0"
+  required_version = ">= 0.13"
 
   required_providers {
-    ct       = "0.6.0"
+    ct = {
+      source  = "poseidon/ct"
+      version = "0.6.1"
+    }
+
+    libvirt = {
+      source = "dmacvicar/libvirt"
+      uri     = "qemu:///system"
+      version = "0.6.2"
+    }
     template = "2.1.2"
-    libvirt  = "0.6.0"
   }
 }

assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/workers.tf

@@ -48,8 +48,14 @@ data "template_file" "worker-config" {
   template = file("${path.module}/cl/worker.yaml.tmpl")
 
   vars = {
+    enable_tls_bootstrap   = var.enable_tls_bootstrap
     domain_name            = "${var.cluster_name}-${var.pool_name}-worker-${count.index}.${var.machine_domain}"
-    kubeconfig             = indent(10, var.kubeconfig)
+		kubeconfig = var.enable_tls_bootstrap ? indent(10, templatefile("${path.module}/cl/bootstrap-kubeconfig.yaml.tmpl", {
+			token_id     = random_string.bootstrap_token_id[0].result
+			token_secret = random_string.bootstrap_token_secret[0].result
+			ca_cert      = var.ca_cert
+			server       = "https://${var.apiserver}:6443"
+		})) : indent(10, var.kubeconfig)
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix

ci/kvm-libvirt/kvm-libvirt-cluster.lokocfg.envsubst

@@ -0,0 +1,11 @@
+cluster "kvm-libvirt" {
+  asset_dir         = pathexpand("~/lokoctl-assets")
+  ssh_pubkeys       = [file(pathexpand("~/.ssh/id_rsa.pub"))]
+  cluster_name      = "vmcluster"
+  machine_domain    = "vmcluster.k8s"
+  os_image          = "file:///var/tmp/flatcar_production_qemu_image.img"
+
+  worker_pool "one" {
+    count = 2
+  }
+}

cli/cmd/root.go

@@ -26,6 +26,7 @@ import (
 	_ "github.com/kinvolk/lokomotive/pkg/platform/aks"
 	_ "github.com/kinvolk/lokomotive/pkg/platform/aws"
 	_ "github.com/kinvolk/lokomotive/pkg/platform/baremetal"
+	_ "github.com/kinvolk/lokomotive/pkg/platform/kvmlibvirt"
 	_ "github.com/kinvolk/lokomotive/pkg/platform/packet"
 
 	// Register backends by adding an anonymous import.