diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index 055984e8214..9d1b9abe98f 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -80,6 +80,12 @@ tf-validate-exoscale: TF_VERSION: $TERRAFORM_VERSION PROVIDER: exoscale +tf-validate-hetzner: + extends: .terraform_validate + variables: + TF_VERSION: $TERRAFORM_VERSION + PROVIDER: hetzner + tf-validate-vsphere: extends: .terraform_validate variables: diff --git a/contrib/terraform/hetzner/default.tfvars b/contrib/terraform/hetzner/default.tfvars index a56bab1ae49..4e70bf1d938 100644 --- a/contrib/terraform/hetzner/default.tfvars +++ b/contrib/terraform/hetzner/default.tfvars @@ -1,6 +1,6 @@ -prefix = "default" -zone = "hel1" -network_zone = "eu-central" +prefix = "default" +zone = "hel1" +network_zone = "eu-central" inventory_file = "inventory.ini" ssh_public_keys = [ @@ -15,17 +15,17 @@ machines = { "master-0" : { "node_type" : "master", "size" : "cx21", - "image" : "ubuntu-20.04", + "image" : "ubuntu-22.04", }, "worker-0" : { "node_type" : "worker", "size" : "cx21", - "image" : "ubuntu-20.04", + "image" : "ubuntu-22.04", }, "worker-1" : { "node_type" : "worker", "size" : "cx21", - "image" : "ubuntu-20.04", + "image" : "ubuntu-22.04", } } diff --git a/contrib/terraform/hetzner/main.tf b/contrib/terraform/hetzner/main.tf index d02a06484b4..8e38cee302e 100644 --- a/contrib/terraform/hetzner/main.tf +++ b/contrib/terraform/hetzner/main.tf @@ -2,7 +2,7 @@ provider "hcloud" {} module "kubernetes" { source = "./modules/kubernetes-cluster" - #source = "./modules/kubernetes-cluster-flatcar" + # source = "./modules/kubernetes-cluster-flatcar" prefix = var.prefix @@ -14,7 +14,7 @@ module "kubernetes" { #ssh_private_key_path = var.ssh_private_key_path ssh_public_keys = var.ssh_public_keys - network_zone = var.network_zone + network_zone = var.network_zone ssh_whitelist = var.ssh_whitelist api_server_whitelist = var.api_server_whitelist @@ -26,31 +26,32 @@ module "kubernetes" { # Generate ansible inventory # -data "template_file" "inventory" { - template = file("${path.module}/templates/inventory.tpl") - - vars = { - connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d", - keys(module.kubernetes.master_ip_addresses), - values(module.kubernetes.master_ip_addresses).*.public_ip, - values(module.kubernetes.master_ip_addresses).*.private_ip, - range(1, length(module.kubernetes.master_ip_addresses) + 1))) - connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s", - keys(module.kubernetes.worker_ip_addresses), - values(module.kubernetes.worker_ip_addresses).*.public_ip, - values(module.kubernetes.worker_ip_addresses).*.private_ip)) - list_master = join("\n", keys(module.kubernetes.master_ip_addresses)) - list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses)) - network_id = module.kubernetes.network_id - } +locals { + inventory = templatefile( + "${path.module}/templates/inventory.tpl", + { + connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d", + keys(module.kubernetes.master_ip_addresses), + values(module.kubernetes.master_ip_addresses).*.public_ip, + values(module.kubernetes.master_ip_addresses).*.private_ip, + range(1, length(module.kubernetes.master_ip_addresses) + 1))) + connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s", + keys(module.kubernetes.worker_ip_addresses), + values(module.kubernetes.worker_ip_addresses).*.public_ip, + values(module.kubernetes.worker_ip_addresses).*.private_ip)) + list_master = join("\n", keys(module.kubernetes.master_ip_addresses)) + list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses)) + network_id = module.kubernetes.network_id + } + ) } resource "null_resource" "inventories" { provisioner "local-exec" { - command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}" + command = "echo '${local.inventory}' > ${var.inventory_file}" } triggers = { - template = data.template_file.inventory.rendered + template = local.inventory } } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf index 804c5038e3b..b54d360bff3 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/main.tf @@ -15,12 +15,12 @@ resource "hcloud_ssh_key" "first" { public_key = var.ssh_public_keys.0 } -resource "hcloud_server" "master" { +resource "hcloud_server" "machine" { for_each = { for name, machine in var.machines : name => machine - if machine.node_type == "master" } + name = "${var.prefix}-${each.key}" ssh_keys = [hcloud_ssh_key.first.id] # boot into rescue OS @@ -30,11 +30,11 @@ resource "hcloud_server" "master" { server_type = each.value.size location = var.zone connection { - host = self.ipv4_address - timeout = "5m" + host = self.ipv4_address + timeout = "5m" private_key = file(var.ssh_private_key_path) } - firewall_ids = [hcloud_firewall.machine.id] + firewall_ids = each.value.node_type == "master" ? [hcloud_firewall.master.id] : [hcloud_firewall.worker.id] provisioner "file" { content = data.ct_config.machine-ignitions[each.key].rendered destination = "/root/ignition.json" @@ -45,9 +45,9 @@ resource "hcloud_server" "master" { "set -ex", "apt update", "apt install -y gawk", - "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install", + "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/flatcar/init/flatcar-master/bin/flatcar-install", "chmod +x flatcar-install", - "./flatcar-install -s -i /root/ignition.json", + "./flatcar-install -s -i /root/ignition.json -C stable", "shutdown -r +1", ] } @@ -55,9 +55,10 @@ resource "hcloud_server" "master" { # optional: provisioner "remote-exec" { connection { - host = self.ipv4_address - timeout = "3m" - user = var.user_flatcar + host = self.ipv4_address + private_key = file(var.ssh_private_key_path) + timeout = "3m" + user = var.user_flatcar } inline = [ @@ -66,65 +67,11 @@ resource "hcloud_server" "master" { } } -resource "hcloud_server_network" "master" { - for_each = hcloud_server.master - server_id = each.value.id - subnet_id = hcloud_network_subnet.kubernetes.id -} - -resource "hcloud_server" "worker" { +resource "hcloud_server_network" "machine" { for_each = { for name, machine in var.machines : - name => machine - if machine.node_type == "worker" - } - name = "${var.prefix}-${each.key}" - ssh_keys = [hcloud_ssh_key.first.id] - # boot into rescue OS - rescue = "linux64" - # dummy value for the OS because Flatcar is not available - image = each.value.image - server_type = each.value.size - location = var.zone - connection { - host = self.ipv4_address - timeout = "5m" - private_key = file(var.ssh_private_key_path) + name => hcloud_server.machine[name] } - firewall_ids = [hcloud_firewall.machine.id] - provisioner "file" { - content = data.ct_config.machine-ignitions[each.key].rendered - destination = "/root/ignition.json" - } - - provisioner "remote-exec" { - inline = [ - "set -ex", - "apt update", - "apt install -y gawk", - "curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 https://raw.githubusercontent.com/kinvolk/init/flatcar-master/bin/flatcar-install", - "chmod +x flatcar-install", - "./flatcar-install -s -i /root/ignition.json", - "shutdown -r +1", - ] - } - - # optional: - provisioner "remote-exec" { - connection { - host = self.ipv4_address - timeout = "3m" - user = var.user_flatcar - } - - inline = [ - "sudo hostnamectl set-hostname ${self.name}", - ] - } -} - -resource "hcloud_server_network" "worker" { - for_each = hcloud_server.worker server_id = each.value.id subnet_id = hcloud_network_subnet.kubernetes.id } @@ -134,38 +81,33 @@ data "ct_config" "machine-ignitions" { for name, machine in var.machines : name => machine } - content = data.template_file.machine-configs[each.key].rendered -} - -data "template_file" "machine-configs" { - for_each = { - for name, machine in var.machines : - name => machine - } - template = file("${path.module}/templates/machine.yaml.tmpl") - vars = { - ssh_keys = jsonencode(var.ssh_public_keys) - user_flatcar = jsonencode(var.user_flatcar) - name = each.key - } + strict = false + content = templatefile( + "${path.module}/templates/machine.yaml.tmpl", + { + ssh_keys = jsonencode(var.ssh_public_keys) + user_flatcar = var.user_flatcar + name = each.key + } + ) } -resource "hcloud_firewall" "machine" { - name = "${var.prefix}-machine-firewall" +resource "hcloud_firewall" "master" { + name = "${var.prefix}-master-firewall" rule { - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.ssh_whitelist + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.ssh_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "6443" - source_ips = var.api_server_whitelist + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = var.api_server_whitelist } } @@ -173,30 +115,30 @@ resource "hcloud_firewall" "worker" { name = "${var.prefix}-worker-firewall" rule { - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.ssh_whitelist + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.ssh_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "80" - source_ips = var.ingress_whitelist + direction = "in" + protocol = "tcp" + port = "80" + source_ips = var.ingress_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "443" - source_ips = var.ingress_whitelist + direction = "in" + protocol = "tcp" + port = "443" + source_ips = var.ingress_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "30000-32767" - source_ips = var.nodeport_whitelist + direction = "in" + protocol = "tcp" + port = "30000-32767" + source_ips = var.nodeport_whitelist } } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf index 5c31aaa003c..be524deb66d 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/outputs.tf @@ -1,20 +1,22 @@ output "master_ip_addresses" { value = { - for key, instance in hcloud_server.master : - instance.name => { - "private_ip" = hcloud_server_network.master[key].ip - "public_ip" = hcloud_server.master[key].ipv4_address - } + for name, machine in var.machines : + name => { + "private_ip" = hcloud_server_network.machine[name].ip + "public_ip" = hcloud_server.machine[name].ipv4_address + } + if machine.node_type == "master" } } output "worker_ip_addresses" { value = { - for key, instance in hcloud_server.worker : - instance.name => { - "private_ip" = hcloud_server_network.worker[key].ip - "public_ip" = hcloud_server.worker[key].ipv4_address - } + for name, machine in var.machines : + name => { + "private_ip" = hcloud_server_network.machine[name].ip + "public_ip" = hcloud_server.machine[name].ipv4_address + } + if machine.node_type == "worker" } } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl index 426cb09ae0f..95ce1d867ad 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/templates/machine.yaml.tmpl @@ -1,8 +1,11 @@ ---- +variant: flatcar +version: 1.0.0 + passwd: users: - name: ${user_flatcar} ssh_authorized_keys: ${ssh_keys} + storage: files: - path: /home/core/works diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf index 4f2f8f51dab..809377946ec 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/variables.tf @@ -1,6 +1,6 @@ variable "zone" { - type = string + type = string default = "fsn1" } @@ -9,7 +9,7 @@ variable "prefix" { } variable "user_flatcar" { - type = string + type = string default = "core" } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf index 5fa23ff851a..ac98e278469 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster-flatcar/versions.tf @@ -1,13 +1,14 @@ terraform { required_providers { hcloud = { - source = "hetznercloud/hcloud" + source = "hetznercloud/hcloud" } ct = { source = "poseidon/ct" + version = "0.11.0" } null = { - source = "hashicorp/null" + source = "hashicorp/null" } } } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf index d7ec865d7fa..2a0e458815f 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf @@ -75,17 +75,17 @@ resource "hcloud_firewall" "master" { name = "${var.prefix}-master-firewall" rule { - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.ssh_whitelist + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.ssh_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "6443" - source_ips = var.api_server_whitelist + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = var.api_server_whitelist } } @@ -93,30 +93,30 @@ resource "hcloud_firewall" "worker" { name = "${var.prefix}-worker-firewall" rule { - direction = "in" - protocol = "tcp" - port = "22" - source_ips = var.ssh_whitelist + direction = "in" + protocol = "tcp" + port = "22" + source_ips = var.ssh_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "80" - source_ips = var.ingress_whitelist + direction = "in" + protocol = "tcp" + port = "80" + source_ips = var.ingress_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "443" - source_ips = var.ingress_whitelist + direction = "in" + protocol = "tcp" + port = "443" + source_ips = var.ingress_whitelist } rule { - direction = "in" - protocol = "tcp" - port = "30000-32767" - source_ips = var.nodeport_whitelist + direction = "in" + protocol = "tcp" + port = "30000-32767" + source_ips = var.nodeport_whitelist } } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf index 2cea1c20c69..78bc5047b07 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf @@ -1,8 +1,8 @@ terraform { required_providers { hcloud = { - source = "hetznercloud/hcloud" - version = "1.31.1" + source = "hetznercloud/hcloud" + version = "1.38.2" } } required_version = ">= 0.14" diff --git a/contrib/terraform/hetzner/sample-inventory/cluster.tfvars b/contrib/terraform/hetzner/sample-inventory/cluster.tfvars new file mode 100644 index 00000000000..4e70bf1d938 --- /dev/null +++ b/contrib/terraform/hetzner/sample-inventory/cluster.tfvars @@ -0,0 +1,46 @@ +prefix = "default" +zone = "hel1" +network_zone = "eu-central" +inventory_file = "inventory.ini" + +ssh_public_keys = [ + # Put your public SSH key here + "ssh-rsa I-did-not-read-the-docs", + "ssh-rsa I-did-not-read-the-docs 2", +] + +ssh_private_key_path = "~/.ssh/id_rsa" + +machines = { + "master-0" : { + "node_type" : "master", + "size" : "cx21", + "image" : "ubuntu-22.04", + }, + "worker-0" : { + "node_type" : "worker", + "size" : "cx21", + "image" : "ubuntu-22.04", + }, + "worker-1" : { + "node_type" : "worker", + "size" : "cx21", + "image" : "ubuntu-22.04", + } +} + +nodeport_whitelist = [ + "0.0.0.0/0" +] + +ingress_whitelist = [ + "0.0.0.0/0" +] + +ssh_whitelist = [ + "0.0.0.0/0" +] + +api_server_whitelist = [ + "0.0.0.0/0" +] diff --git a/contrib/terraform/hetzner/sample-inventory/group_vars b/contrib/terraform/hetzner/sample-inventory/group_vars new file mode 120000 index 00000000000..37359582379 --- /dev/null +++ b/contrib/terraform/hetzner/sample-inventory/group_vars @@ -0,0 +1 @@ +../../../../inventory/sample/group_vars \ No newline at end of file diff --git a/contrib/terraform/hetzner/variables.tf b/contrib/terraform/hetzner/variables.tf index 8fc8b2c9729..049ce0d4227 100644 --- a/contrib/terraform/hetzner/variables.tf +++ b/contrib/terraform/hetzner/variables.tf @@ -3,7 +3,7 @@ variable "zone" { } variable "network_zone" { description = "The network zone where the cluster is running" - default = "eu-central" + default = "eu-central" } variable "prefix" { diff --git a/contrib/terraform/hetzner/versions.tf b/contrib/terraform/hetzner/versions.tf index 02e5b74ee6e..e331beb4582 100644 --- a/contrib/terraform/hetzner/versions.tf +++ b/contrib/terraform/hetzner/versions.tf @@ -2,14 +2,11 @@ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" - version = "1.31.1" + version = "1.38.2" } null = { source = "hashicorp/null" } - template = { - source = "hashicorp/template" - } } required_version = ">= 0.14" }