kvmlibvirt: enable tls bootstrap

Signed-off-by: Imran Pochi <imran@kinvolk.io>
kinvolk · Oct 21, 2020 · bd7ce15 · bd7ce15
1 parent 08a65cb
commit bd7ce15
Show file tree

Hide file tree

Showing 16 changed files with 216 additions and 23 deletions.
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootkube.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootkube.tf
@@ -1,3 +1,8 @@
+locals {
+  # api_server = module.bootkube.api_servers
+  api_server = data.template_file.controllernames[0].rendered
+}
+
 module "bootkube" {
   source       = "../../../bootkube"
   cluster_name = var.cluster_name
@@ -24,6 +29,9 @@ module "bootkube" {
   # Extra flags to API server.
   kube_apiserver_extra_flags = var.kube_apiserver_extra_flags
 
+  bootstrap_tokens     = var.enable_tls_bootstrap ? concat([local.controller_bootstrap_token], var.worker_bootstrap_tokens) : []
+  enable_tls_bootstrap = var.enable_tls_bootstrap
+
   certs_validity_period_hours = var.certs_validity_period_hours
 }
 

diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootstrap-configs.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/bootstrap-configs.tf
@@ -0,0 +1,24 @@
+locals {
+  controller_bootstrap_token = var.enable_tls_bootstrap ? {
+    token_id     = random_string.bootstrap_token_id[0].result
+    token_secret = random_string.bootstrap_token_secret[0].result
+  } : {}
+}
+
+# Generate a cryptographically random token id (public).
+resource random_string "bootstrap_token_id" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource random_string "bootstrap_token_secret" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 16
+  upper   = false
+  special = false
+}
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/cl/controller.yaml.tmpl b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
@@ -88,7 +88,13 @@ systemd:
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
+          %{~ if enable_tls_bootstrap ~}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          %{~ else ~}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          %{~ endif ~}
           --lock-file=/var/run/lock/kubelet.lock \
           --hostname-override=${etcd_domain} \
           --network-plugin=cni \

diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/controllers.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/controllers.tf
@@ -83,10 +83,16 @@ data "template_file" "controller-configs" {
     etcd_name              = "${var.cluster_name}-controller-${count.index}"
     etcd_domain            = "${var.cluster_name}-controller-${count.index}.${var.machine_domain}"
     etcd_initial_cluster   = join(",", data.template_file.etcds.*.rendered)
-    kubeconfig             = indent(10, module.bootkube.kubeconfig-kubelet)
+    kubeconfig = var.enable_tls_bootstrap ? indent(10, templatefile("${path.module}/workers/cl/bootstrap-kubeconfig.yaml.tmpl", {
+      token_id     = random_string.bootstrap_token_id[0].result
+      token_secret = random_string.bootstrap_token_secret[0].result
+      ca_cert      = module.bootkube.ca_cert
+      server       = "https://${local.api_server}:6443"
+    })) : indent(10, module.bootkube.kubeconfig-kubelet)
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix
+    enable_tls_bootstrap   = var.enable_tls_bootstrap
   }
 }
 

diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/outputs.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/outputs.tf
@@ -38,3 +38,15 @@ output "calico_values" {
 output "lokomotive_values" {
   value = module.bootkube.lokomotive_values
 }
+
+output "ca_cert" {
+  value = module.bootkube.ca_cert
+}
+
+output "bootstrap-secrets_values" {
+  value = module.bootkube.bootstrap-secrets_values
+}
+
+output "apiserver" {
+  value = local.api_server
+}
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/variables.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/variables.tf
@@ -129,3 +129,13 @@ variable "encrypt_pod_traffic" {
   type        = bool
   default     = false
 }
+
+variable "worker_bootstrap_tokens" {
+  description = "List of token-id and token-secret of each node."
+  type        = list(any)
+}
+
+variable "enable_tls_bootstrap" {
+  description = "Enable TLS Bootstrap for Kubelet."
+  type        = bool
+}
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/bootstrap-configs.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/bootstrap-configs.tf
@@ -0,0 +1,24 @@
+locals {
+  worker_bootstrap_token = var.enable_tls_bootstrap ? {
+    token_id     = random_string.bootstrap_token_id[0].result
+    token_secret = random_string.bootstrap_token_secret[0].result
+  } : {}
+}
+
+# Generate a cryptographically random token id (public).
+resource random_string "bootstrap_token_id" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret.
+resource random_string "bootstrap_token_secret" {
+  count = var.enable_tls_bootstrap == true ? 1 : 0
+
+  length  = 16
+  upper   = false
+  special = false
+}
diff --git a/...rm-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/bootstrap-kubeconfig.yaml.tmpl b/...rm-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/bootstrap-kubeconfig.yaml.tmpl
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: kubelet
+  user:
+    token: ${token_id}.${token_secret}
+contexts:
+- context:
+    cluster: local
+    user: kubelet
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl
@@ -65,7 +65,13 @@ systemd:
           --cni-conf-dir=/etc/cni/net.d \
           --config=/etc/kubernetes/kubelet.config \
           --exit-on-lock-contention \
+          %{~ if enable_tls_bootstrap ~}
+          --kubeconfig=/var/lib/kubelet/kubeconfig \
+          --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig \
+          --rotate-certificates \
+          %{~ else ~}
           --kubeconfig=/etc/kubernetes/kubeconfig \
+          %{~ endif ~}
           --lock-file=/var/run/lock/kubelet.lock \
           --network-plugin=cni \
           --node-labels=$${NODE_LABELS} \
@@ -146,7 +152,11 @@ storage:
             --net=host \
             --dns=host \
             -- \
+            %{~ if enable_tls_bootstrap ~}
+            kubectl --kubeconfig=/var/lib/kubelet/kubeconfig delete node $(hostname)
+            %{~ else ~}
             kubectl --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname)
+            %{~ endif ~}
 passwd:
   users:
     - name: core

diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/outputs.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/outputs.tf
@@ -0,0 +1,3 @@
+output "worker_bootstrap_token" {
+  value = local.worker_bootstrap_token
+}
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/variables.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/variables.tf
@@ -83,3 +83,18 @@ EOD
   type    = string
   default = "10.2.0.0/16"
 }
+
+variable "ca_cert" {
+  description = "Kubernetes CA certificate needed in the kubeconfig file."
+  type        = string
+}
+
+variable "apiserver" {
+  description = "Apiserver private endpoint needed in the kubeconfig file."
+  type        = string
+}
+
+variable "enable_tls_bootstrap" {
+  description = "Enable TLS Bootstrap for Kubelet."
+  type        = bool
+}
diff --git a/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/workers.tf b/assets/terraform-modules/kvm-libvirt/flatcar-linux/kubernetes/workers/workers.tf
@@ -48,8 +48,14 @@ data "template_file" "worker-config" {
   template = file("${path.module}/cl/worker.yaml.tmpl")
 
   vars = {
+    enable_tls_bootstrap   = var.enable_tls_bootstrap
     domain_name            = "${var.cluster_name}-${var.pool_name}-worker-${count.index}.${var.machine_domain}"
-    kubeconfig             = indent(10, var.kubeconfig)
+		kubeconfig = var.enable_tls_bootstrap ? indent(10, templatefile("${path.module}/cl/bootstrap-kubeconfig.yaml.tmpl", {
+			token_id     = random_string.bootstrap_token_id[0].result
+			token_secret = random_string.bootstrap_token_secret[0].result
+			ca_cert      = var.ca_cert
+			server       = "https://${var.apiserver}:6443"
+		})) : indent(10, var.kubeconfig)
     ssh_keys               = jsonencode(var.ssh_keys)
     cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
     cluster_domain_suffix  = var.cluster_domain_suffix

diff --git a/docs/configuration-reference/platforms/kvm-libvirt.md b/docs/configuration-reference/platforms/kvm-libvirt.md
@@ -66,6 +66,8 @@ cluster "kvm-libvirt" {
 
   certs_validity_period_hours = 8760
 
+  enable_tls_bootstrap = true
+
   worker_pool "worker-pool-1" {
     count = 2
 
@@ -116,6 +118,7 @@ EOF
 | `pod_cidr`                            | CIDR IPv4 range to assign Kubernetes pods.                                                                                                                                                                                                                                        |    "10.2.0.0/16"   |    string    |  false   |
 | `service_cidr`                        | CIDR IPv4 range to assign Kubernetes services.                                                                                                                                                                                                                                    |    "10.3.0.0/16"   |    string    |  false   |
 | `ssh_pubkeys`                         | List of SSH public keys for user `core`. Each element must be specified in a valid OpenSSH public key format, as defined in RFC 4253 Section 6.6, e.g. "ssh-rsa AAAAB3N...".                                                                                                      |          -         | list(string) |   true   |
+| `enable_tls_bootstrap`                | Enable TLS bootstrapping for Kubelet.                                                                                                                                                                                                                                             |        true        |     bool     |  false   |
 | `worker_pool.clc_snippets`            | Flatcar Container Linux Config snippets for nodes in the worker pool.                                                                                                                                                                                                             |          []        | list(string) |  false   |
 | `worker_pool`                         | Configuration block for worker pools. There can be more than one.                                                                                                                                                                                                                 |          -         | list(object) |   true   |
 | `worker_pool.count`                   | Number of workers in the worker pool. Can be changed afterwards to add or delete workers.                                                                                                                                                                                         |          1         |    number    |   true   |

diff --git a/pkg/assets/generated_assets.go b/pkg/assets/generated_assets.go
diff --git a/pkg/platform/kvmlibvirt/kvmlibvirt.go b/pkg/platform/kvmlibvirt/kvmlibvirt.go
@@ -25,6 +25,7 @@ import (
 	"github.com/hashicorp/hcl/v2/gohcl"
 	"github.com/mitchellh/go-homedir"
 
+	"github.com/kinvolk/lokomotive/pkg/assets"
 	"github.com/kinvolk/lokomotive/pkg/platform"
 	"github.com/kinvolk/lokomotive/pkg/terraform"
 )
@@ -60,6 +61,7 @@ type config struct {
 	EnableReporting              bool         `hcl:"enable_reporting,optional"`
 	EnableAggregation            bool         `hcl:"enable_aggregation,optional"`
 	CertsValidityPeriodHours     int          `hcl:"certs_validity_period_hours,optional"`
+	EnableTLSBootstrap           bool         `hcl:"enable_tls_bootstrap,optional"`
 }
 
 func init() {
@@ -91,7 +93,9 @@ func (c *config) Meta() platform.Meta {
 }
 
 func NewConfig() *config {
-	return &config{}
+	return &config{
+		EnableTLSBootstrap: true,
+	}
 }
 
 func (c *config) Apply(ex *terraform.Executor) error {

diff --git a/pkg/platform/kvmlibvirt/template.go b/pkg/platform/kvmlibvirt/template.go
@@ -79,6 +79,14 @@ module "kvm-libvirt-{{.Config.ClusterName}}" {
   {{- if .Config.CertsValidityPeriodHours }}
   certs_validity_period_hours = {{.Config.CertsValidityPeriodHours}}
   {{- end }}
+
+ # Enable TLS Bootstrap
+   enable_tls_bootstrap = {{ .Config.EnableTLSBootstrap }}
+  worker_bootstrap_tokens = [
+    {{- range $index, $pool := .Config.WorkerPools }}
+    module.worker-pool-{{$index}}.worker_bootstrap_token,
+    {{- end }}
+  ]
 }
 
 {{ range $index, $pool := .Config.WorkerPools }}
@@ -99,6 +107,10 @@ module "worker-pool-{{ $index }}" {
   libvirtbaseid         = module.kvm-libvirt-{{$.Config.ClusterName}}.libvirtbaseid
   kubeconfig            = module.kvm-libvirt-{{$.Config.ClusterName}}.kubeconfig
 
+  ca_cert              = module.kvm-libvirt-{{ $.Config.ClusterName }}.ca_cert
+  apiserver            = module.kvm-libvirt-{{ $.Config.ClusterName }}.apiserver
+  enable_tls_bootstrap = {{ $.Config.EnableTLSBootstrap }}
+
   pool_name             = "{{ $pool.Name }}"
   worker_count          = "{{ $pool.Count}}"
 
@@ -170,4 +182,9 @@ output "calico_values" {
   value     = module.kvm-libvirt-{{.Config.ClusterName}}.calico_values
   sensitive = true
 }
+
+output "bootstrap-secrets_values" {
+  value     = module.kvm-libvirt-{{.Config.ClusterName}}.bootstrap-secrets_values
+  sensitive = true
+}
 `