Merge pull request #157 from kinvolk/invidian/merge-bootkube

Move terraform-render-bootkube module into bootkube directory

Makefile

@@ -14,7 +14,3 @@ run-e2e-tests: kube-hunter
 
 kube-hunter:
 	KUBECONFIG=${kubeconfig} ${kubehunter}
-
-.PHONY: update-terraform-render-bootkube
-update-terraform-render-bootkube:
-	./scripts/update-terraform-render-bootkube.sh $(VERSION)

aws/flatcar-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "github.com/kinvolk/terraform-render-bootkube?ref=1b49787f016ce299833d2544b6b983d3f7ea4155"
+  source = "../../../bootkube"
 
   cluster_name          = var.cluster_name
   api_servers           = [format("%s.%s", var.cluster_name, var.dns_zone)]

azure/flatcar-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "github.com/kinvolk/terraform-render-bootkube?ref=1b49787f016ce299833d2544b6b983d3f7ea4155"
+  source = "../../../bootkube"
 
   cluster_name = var.cluster_name
   api_servers  = [format("%s.%s", var.cluster_name, var.dns_zone)]

bare-metal/flatcar-linux/kubernetes/bootkube.tf

@@ -1,6 +1,6 @@
 # Self-hosted Kubernetes assets (kubeconfig, manifests)
 module "bootkube" {
-  source = "github.com/kinvolk/terraform-render-bootkube?ref=1b49787f016ce299833d2544b6b983d3f7ea4155"
+  source = "../../../bootkube"
 
   cluster_name                    = var.cluster_name
   api_servers                     = [var.k8s_domain_name]

bootkube/.gitignore

@@ -0,0 +1,4 @@
+*.tfvars
+.terraform
+*.tfstate*
+assets

bootkube/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Dalton Hubble
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

bootkube/README.md

@@ -0,0 +1,49 @@
+# bootkube
+
+`bootkube` is a Terraform module that renders [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube) assets for bootstrapping a Kubernetes cluster.
+
+## Audience
+
+`bootkube` is a low-level component of the [Lokomotive](https://github.com/kinvolk/lokomotive-kubernetes) Kubernetes distribution. Use Lokomotive modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
+
+## Usage
+
+Use the module to declare bootkube assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
+
+```hcl
+module "bootkube" {
+  source = "git::https://github.com/kinvolk/lokomotive-kubernetes//bootkube?ref=SHA"
+
+  cluster_name = "example"
+  api_servers = ["node1.example.com"]
+  etcd_servers = ["node1.example.com"]
+  asset_dir = "/home/core/clusters/mycluster"
+}
+```
+
+Generate the assets.
+
+```sh
+terraform init
+terraform plan
+terraform apply
+```
+
+Find bootkube assets rendered to the `asset_dir` path. That's it.
+
+### Comparison
+
+Render bootkube assets directly with bootkube v0.14.0.
+
+```sh
+bootkube render --asset-dir=assets --api-servers=https://node1.example.com:6443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379
+```
+
+Compare assets. Rendered assets may differ slightly from bootkube assets to reflect decisions made by the [Lokomotive](https://github.com/kinvolk/lokomotive-kubernetes) distribution.
+
+```sh
+pushd /home/core/mycluster
+mv manifests-networking/* manifests
+popd
+diff -rw assets /home/core/mycluster
+```

bootkube/assets.tf

@@ -0,0 +1,136 @@
+# Self-hosted Kubernetes bootstrap-manifests
+resource "template_dir" "bootstrap-manifests" {
+  source_dir      = "${replace(path.module, path.cwd, ".")}/resources/bootstrap-manifests"
+  destination_dir = "${var.asset_dir}/bootstrap-manifests"
+
+  vars = {
+    hyperkube_image   = var.container_images["hyperkube"]
+    etcd_servers      = join(",", formatlist("https://%s:2379", var.etcd_servers))
+    cloud_provider    = var.cloud_provider
+    pod_cidr          = var.pod_cidr
+    service_cidr      = var.service_cidr
+    trusted_certs_dir = var.trusted_certs_dir
+  }
+}
+
+# Populate kubernetes chart values file named kubernetes.yaml.
+resource "local_file" "kubernetes" {
+  content  = data.template_file.kubernetes.rendered
+  filename = "${var.asset_dir}/charts/kube-system/kubernetes.yaml"
+}
+
+# Populate kubernetes control plane chart.
+# TODO: Currently, there is no way in Terraform to copy local directory, so we use `template_dir` for it.
+# The downside is, that any Terraform templating syntax stored in this directory will be evaluated, which may bring unexpected results.
+resource "template_dir" "kubernetes" {
+  source_dir      = "${replace(path.module, path.cwd, ".")}/resources/charts/kubernetes"
+  destination_dir = "${var.asset_dir}/charts/kube-system/kubernetes"
+}
+
+# Render kubernetes.yaml for kubernetes chart.
+data "template_file" "kubernetes" {
+  template = "${file("${path.module}/resources/charts/kubernetes.yaml")}"
+
+  vars = {
+    hyperkube_image         = var.container_images["hyperkube"]
+    pod_checkpointer_image  = var.container_images["pod_checkpointer"]
+    coredns_image           = "${var.container_images["coredns"]}${var.container_arch}"
+    etcd_servers            = join(",", formatlist("https://%s:2379", var.etcd_servers))
+    control_plane_replicas  = max(2, length(var.etcd_servers))
+    cloud_provider          = var.cloud_provider
+    pod_cidr                = var.pod_cidr
+    service_cidr            = var.service_cidr
+    cluster_domain_suffix   = var.cluster_domain_suffix
+    cluster_dns_service_ip  = cidrhost(var.service_cidr, 10)
+    trusted_certs_dir       = var.trusted_certs_dir
+    ca_cert                 = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+    ca_key                  = base64encode(tls_private_key.kube-ca.private_key_pem)
+    server                  = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    apiserver_key           = base64encode(tls_private_key.apiserver.private_key_pem)
+    apiserver_cert          = base64encode(tls_locally_signed_cert.apiserver.cert_pem)
+    serviceaccount_pub      = base64encode(tls_private_key.service-account.public_key_pem)
+    serviceaccount_key      = base64encode(tls_private_key.service-account.private_key_pem)
+    etcd_ca_cert            = base64encode(tls_self_signed_cert.etcd-ca.cert_pem)
+    etcd_client_cert        = base64encode(tls_locally_signed_cert.client.cert_pem)
+    etcd_client_key         = base64encode(tls_private_key.client.private_key_pem)
+    enable_aggregation      = var.enable_aggregation
+    aggregation_ca_cert     = var.enable_aggregation == true ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""
+    aggregation_client_cert = var.enable_aggregation == true ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""
+    aggregation_client_key  = var.enable_aggregation == true ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""
+  }
+}
+
+# Render kubelet.yaml for kubelet chart
+data "template_file" "kubelet" {
+  template = "${file("${path.module}/resources/charts/kubelet.yaml")}"
+
+  vars = {
+    hyperkube_image         = var.container_images["hyperkube"]
+    cluster_dns_service_ip  = cidrhost(var.service_cidr, 10)
+    cluster_domain_suffix   = var.cluster_domain_suffix
+  }
+}
+
+# Populate kubelet chart values file named kubelet.yaml.
+resource "local_file" "kubelet" {
+  content = data.template_file.kubelet.rendered
+  filename = "${var.asset_dir}/charts/kube-system/kubelet.yaml"
+}
+
+# Populate kubelet chart.
+# TODO: Currently, there is no way in Terraform to copy local directory, so we use `template_dir` for it.
+# The downside is, that any Terraform templating syntax stored in this directory will be evaluated, which may bring unexpected results.
+resource "template_dir" "kubelet" {
+  source_dir = "${replace(path.module, path.cwd, ".")}/resources/charts/kubelet"
+  destination_dir = "${var.asset_dir}/charts/kube-system/kubelet"
+}
+
+# Generated kubeconfig for Kubelets
+resource "local_file" "kubeconfig-kubelet" {
+  content  = data.template_file.kubeconfig-kubelet.rendered
+  filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
+}
+
+# Generated admin kubeconfig (bootkube requires it be at auth/kubeconfig)
+# https://github.com/kubernetes-incubator/bootkube/blob/master/pkg/bootkube/bootkube.go#L42
+resource "local_file" "kubeconfig-admin" {
+  content  = data.template_file.kubeconfig-admin.rendered
+  filename = "${var.asset_dir}/auth/kubeconfig"
+}
+
+# Generated admin kubeconfig in a file named after the cluster
+resource "local_file" "kubeconfig-admin-named" {
+  content  = data.template_file.kubeconfig-admin.rendered
+  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
+}
+
+data "template_file" "kubeconfig-kubelet" {
+  template = file("${path.module}/resources/kubeconfig-kubelet")
+
+  vars = {
+    ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+    kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem)
+    kubelet_key  = base64encode(tls_private_key.kubelet.private_key_pem)
+    server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+  }
+}
+
+# If var.api_servers_external isn't set, use var.api_servers.
+# This is for supporting separate API server URLs for external clients in a backward-compatible way.
+# The use of split() and join() here is because Terraform's conditional operator ('?') cannot be
+# used with lists.
+locals {
+  api_servers_external = split(",", join(",", var.api_servers_external) == "" ? join(",", var.api_servers) : join(",", var.api_servers_external))
+}
+
+data "template_file" "kubeconfig-admin" {
+  template = file("${path.module}/resources/kubeconfig-admin")
+
+  vars = {
+    name         = var.cluster_name
+    ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+    kubelet_cert = base64encode(tls_locally_signed_cert.admin.cert_pem)
+    kubelet_key  = base64encode(tls_private_key.admin.private_key_pem)
+    server       = format("https://%s:%s", local.api_servers_external[0], var.external_apiserver_port)
+  }
+}

bootkube/conditional.tf

@@ -0,0 +1,111 @@
+# Assets generated only when certain options are chosen
+
+# Populate flannel chart values file named flannel.yaml.
+resource "local_file" "flannel" {
+  count    = var.networking == "flannel" ? 1 : 0
+  content  = templatefile("${path.module}/resources/charts/flannel.yaml",{
+    flannel_image     = "${var.container_images["flannel"]}${var.container_arch}"
+    flannel_cni_image = var.container_images["flannel_cni"]
+    pod_cidr          = var.pod_cidr
+  })
+  filename = "${var.asset_dir}/charts/kube-system/flannel.yaml"
+}
+
+# Populate flannel chart.
+# TODO: Currently, there is no way in Terraform to copy local directory, so we use `template_dir` for it.
+# The downside is, that any Terraform templating syntax stored in this directory will be evaluated, which may bring unexpected results.
+resource "template_dir" "flannel" {
+  count           = var.networking == "flannel" ? 1 : 0
+  source_dir      = "${replace(path.module, path.cwd, ".")}/resources/charts/flannel"
+  destination_dir = "${var.asset_dir}/charts/kube-system/flannel"
+}
+
+# Render flannel.yaml for flannel chart.
+data "template_file" "flannel" {
+  count    = var.networking == "flannel" ? 1 : 0
+  template = "${file("${path.module}/resources/charts/flannel.yaml")}"
+
+  vars = {
+    flannel_image     = "${var.container_images["flannel"]}${var.container_arch}"
+    flannel_cni_image = var.container_images["flannel_cni"]
+    pod_cidr          = var.pod_cidr
+  }
+}
+
+# Populate calico chart values file named calico.yaml.
+resource "local_file" "calico" {
+  count    = var.networking == "calico" ? 1 : 0
+  content  = templatefile("${path.module}/resources/charts/calico.yaml",{
+    calico_image                    = var.container_images["calico"]
+    calico_cni_image                = var.container_images["calico_cni"]
+    network_mtu                     = var.network_mtu
+    network_encapsulation           = indent(2, var.network_encapsulation == "vxlan" ? "vxlanMode: Always" : "ipipMode: Always")
+    ipip_enabled                    = var.network_encapsulation == "ipip" ? true : false
+    ipip_readiness                  = var.network_encapsulation == "ipip" ? indent(16, "- --bird-ready") : ""
+    vxlan_enabled                   = var.network_encapsulation == "vxlan" ? true : false
+    network_ip_autodetection_method = var.network_ip_autodetection_method
+    pod_cidr                        = var.pod_cidr
+    enable_reporting                = var.enable_reporting
+  })
+  filename = "${var.asset_dir}/charts/kube-system/calico.yaml"
+}
+
+# Populate calico chart.
+# TODO: Currently, there is no way in Terraform to copy local directory, so we use `template_dir` for it.
+# The downside is, that any Terraform templating syntax stored in this directory will be evaluated, which may bring unexpected results.
+resource "template_dir" "calico" {
+  count           = var.networking == "calico" ? 1 : 0
+  source_dir      = "${replace(path.module, path.cwd, ".")}/resources/charts/calico"
+  destination_dir = "${var.asset_dir}/charts/kube-system/calico"
+}
+
+# Render calico.yaml for calico chart.
+data "template_file" "calico" {
+  count    = var.networking == "calico" ? 1 : 0
+  template = "${file("${path.module}/resources/charts/calico.yaml")}"
+
+  vars = {
+    calico_image                    = var.container_images["calico"]
+    calico_cni_image                = var.container_images["calico_cni"]
+    network_mtu                     = var.network_mtu
+    network_encapsulation           = indent(2, var.network_encapsulation == "vxlan" ? "vxlanMode: Always" : "ipipMode: Always")
+    ipip_enabled                    = var.network_encapsulation == "ipip" ? true : false
+    ipip_readiness                  = var.network_encapsulation == "ipip" ? indent(16, "- --bird-ready") : ""
+    vxlan_enabled                   = var.network_encapsulation == "vxlan" ? true : false
+    network_ip_autodetection_method = var.network_ip_autodetection_method
+    pod_cidr                        = var.pod_cidr
+    enable_reporting                = var.enable_reporting
+  }
+}
+
+# Populate kube-router chart values file named kube-router.yaml.
+resource "local_file" "kube-router" {
+  count    = var.networking == "kube-router" ? 1 : 0
+  content  = templatefile("${path.module}/resources/charts/kube-router.yaml",{
+    kube_router_image = var.container_images["kube_router"]
+    flannel_cni_image = var.container_images["flannel_cni"]
+    network_mtu       = var.network_mtu
+  })
+  filename = "${var.asset_dir}/charts/kube-system/kube-router.yaml"
+}
+
+# Populate kube-router chart.
+# TODO: Currently, there is no way in Terraform to copy local directory, so we use `template_dir` for it.
+# The downside is, that any Terraform templating syntax stored in this directory will be evaluated, which may bring unexpected results.
+resource "template_dir" "kube-router" {
+  count           = var.networking == "kube-router" ? 1 : 0
+  source_dir      = "${replace(path.module, path.cwd, ".")}/resources/charts/kube-router"
+  destination_dir = "${var.asset_dir}/charts/kube-system/kube-router"
+}
+
+# Render kube-router.yaml for kube-router chart.
+data "template_file" "kube-router" {
+  count    = var.networking == "kube-router" ? 1 : 0
+  template = "${file("${path.module}/resources/charts/kube-router.yaml")}"
+
+  vars = {
+    kube_router_image = var.container_images["kube_router"]
+    flannel_cni_image = var.container_images["flannel_cni"]
+    network_mtu       = var.network_mtu
+  }
+}