feat!: talos multinode initial support

ephemeral/instance.tf

@@ -15,8 +15,9 @@ data "digitalocean_image" "talos" {
   name = "talos-v1.9.1"
 }
 
-resource "digitalocean_droplet" "talos" {
-  name     = "rnd-${local.prefix}-spectrum-cp"
+resource "digitalocean_droplet" "cp" {
+  count    = 1
+  name     = "rnd-${local.prefix}-spectrum-cp-${count.index}"
   size     = "s-8vcpu-16gb"
   image    = data.digitalocean_image.talos.id
   region   = "fra1"
@@ -32,6 +33,6 @@ resource "digitalocean_droplet" "talos" {
 }
 
 resource "digitalocean_reserved_ip" "l2" {
-  droplet_id = digitalocean_droplet.talos.id
-  region     = digitalocean_droplet.talos.region
+  droplet_id = digitalocean_droplet.cp[0].id
+  region     = digitalocean_droplet.cp[0].region
 }

ephemeral/main.tf

@@ -13,10 +13,15 @@ data "vault_generic_secret" "docker" {
 module "talos" {
   source       = "../terraform-modules/talos"
   cluster_name = local.prefix
-  server_ip    = digitalocean_droplet.talos.ipv4_address
 
-  config_patches = [
-    file("${path.root}/config_patch.yml"),
+  control_planes = [
+    {
+      name      = "cp-0"
+      server_ip = digitalocean_droplet.cp[0].ipv4_address
+      config_patches = [
+        file("${path.root}/patches/registry.yml"),
+      ]
+    },
   ]
 }
 
@@ -36,7 +41,7 @@ module "spectrum" {
     DOTOKEN         = base64encode(data.vault_generic_secret.spectrum.data.token)
     DOMAIN          = "${local.prefix}.fluence.dev"
     PREFIX          = local.prefix
-    LOADBALANCER_IP = digitalocean_droplet.talos.ipv4_address
+    LOADBALANCER_IP = digitalocean_droplet.cp[0].ipv4_address
     L2_IP           = digitalocean_reserved_ip.l2.ip_address
   }
 }

examples/talos/README.md

@@ -2,9 +2,9 @@
 
 ## Server customization
 
-In talos machine is configured from a single configuration file in yaml format. Talos terraform module allows to specify overlays of the main configuration file maintained by cloudless labs [here](https://github.com/fluencelabs/spectrum/blob/main/terraform-modules/talos/templates/controlplane_patch.yml) with `config_pathes` option.
+In talos machine is configured from a single configuration file in yaml format. Talos terraform module allows to specify overlays of the main configuration file maintained by cloudless labs [here](https://github.com/fluencelabs/spectrum/blob/main/terraform-modules/talos/base_config.yml) with `config_pathes` option.
 
-You can configure server specific things like layout of disks or network configuration. Checkout [talos documentation](https://www.talos.dev/v1.9/reference/configuration/v1alpha1/config/) and see `config_patch.yml` for an example of a `bond` interface configuration.
+You can configure server specific things like layout of disks or network configuration. Checkout [talos documentation](https://www.talos.dev/v1.9/reference/configuration/v1alpha1/config/) and see [`cp-0.yml`](https://github.com/fluencelabs/blob/main/examples/talos/patches/cp-0.yml) for an example of a `bond` interface configuration.
 
 ## Terraform state
 

examples/talos/main.tf

@@ -22,19 +22,16 @@ provider "helm" {
 
 module "talos" {
   source       = "git::https://github.com/fluencelabs/spectrum.git//terraform-modules/talos?ref=terraform-module-talos-v0.1.0" # x-release-please-version
-  cluster_name = var.cluster_name
-  server_ip    = var.server_ip
-  # config_patches = [
-  #   file("${path.root}/config_patch.yml"),
-  # ]
-}
-
-variable "server_ip" {
-  type        = string
-  description = "IP at which server is accessible"
-}
+  cluster_name = "my-cluster"
 
-variable "cluster_name" {
-  type        = string
-  description = "Name used in k8s and talos to distinguish between clusters"
+  control_planes = [
+    {
+      name      = "cp-0"
+      server_ip = "1.2.3.4"
+      config_patches = [
+        file("${path.root}/patches/base.yml"),
+        file("${path.root}/patches/cp-0.yml"),
+      ]
+    },
+  ]
 }

examples/talos/patches/base.yml

@@ -0,0 +1,4 @@
+machine:
+  time:
+    servers:
+      - time.cloudflare.com

examples/talos/config_patch.yml → examples/talos/patches/cp-0.yml

@@ -30,7 +30,6 @@ machine:
       serial: Y0L0A031T5N8
 
   network:
-    hostname: foobar
     interfaces:
       - interface: bond0
         dhcp: false

terraform-modules/talos/templates/controlplane_patch.yml → terraform-modules/talos/base_config.yml

@@ -1,7 +1,6 @@
 machine:
-  time:
-    servers:
-      - time.cloudflare.com
+  network:
+    hostname: ${hostname}
   install:
       diskSelector:
         size: '>= 100GB'

terraform-modules/talos/talos.tf

@@ -1,61 +1,79 @@
 resource "talos_machine_secrets" "this" {
-  talos_version = "v1.9"
+  talos_version = var.talos_version
 }
 
-data "talos_machine_configuration" "this" {
+locals {
+  virtual_ip       = var.virtual_ip != "" ? var.virtual_ip : var.control_planes[0].server_ip
+  cluster_endpoint = "https://${local.virtual_ip}:6443"
+}
+
+data "talos_machine_configuration" "control_plane" {
+  for_each         = { for control_plane in var.control_planes : control_plane.name => control_plane }
+  talos_version    = var.talos_version
   cluster_name     = var.cluster_name
   machine_type     = "controlplane"
-  cluster_endpoint = "https://${var.server_ip}:6443"
+  cluster_endpoint = local.cluster_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
-  talos_version    = "v1.9"
   config_patches = [
-    templatefile("${path.module}/templates/controlplane_patch.yml", {})
+    templatefile("${path.module}/base_config.yml", { hostname = each.value.name })
   ]
 }
 
-resource "talos_machine_configuration_apply" "this" {
+data "talos_machine_configuration" "worker" {
+  for_each         = { for worker in var.workers : worker.name => worker }
+  talos_version    = var.talos_version
+  cluster_name     = var.cluster_name
+  cluster_endpoint = local.cluster_endpoint
+  machine_type     = "worker"
+  machine_secrets  = talos_machine_secrets.this.machine_secrets
+  config_patches = [
+    templatefile("${path.module}/base_config.yml", { hostname = each.value.name })
+  ]
+}
+
+resource "talos_machine_configuration_apply" "control_plane" {
+  for_each                    = { for control_plane in var.control_planes : control_plane.name => control_plane }
+  client_configuration        = talos_machine_secrets.this.client_configuration
+  machine_configuration_input = data.talos_machine_configuration.control_plane[each.key].machine_configuration
+  node                        = each.value.server_ip
+  config_patches              = each.value.config_patches
+}
+
+resource "talos_machine_configuration_apply" "worker" {
+  for_each                    = { for worker in var.workers : worker.name => worker }
   client_configuration        = talos_machine_secrets.this.client_configuration
-  machine_configuration_input = data.talos_machine_configuration.this.machine_configuration
-  node                        = var.server_ip
-  config_patches              = var.config_patches
+  machine_configuration_input = data.talos_machine_configuration.worker[each.key].machine_configuration
+  node                        = each.value.server_ip
+  config_patches              = each.value.config_patches
 }
 
 data "talos_client_configuration" "this" {
   cluster_name         = var.cluster_name
   client_configuration = talos_machine_secrets.this.client_configuration
   endpoints = [
-    var.server_ip
+    for control_plane in var.control_planes : control_plane.server_ip
   ]
 }
 
 resource "talos_machine_bootstrap" "this" {
-  depends_on           = [talos_machine_configuration_apply.this]
+  depends_on           = [talos_machine_configuration_apply.control_plane]
   client_configuration = talos_machine_secrets.this.client_configuration
-  endpoint             = var.server_ip
-  node                 = var.server_ip
+  endpoint             = var.control_planes[0].server_ip
+  node                 = var.control_planes[0].server_ip
 }
 
 resource "talos_cluster_kubeconfig" "this" {
   client_configuration = talos_machine_secrets.this.client_configuration
-  node                 = var.server_ip
+  node                 = var.control_planes[0].server_ip
 }
 
-# data "talos_cluster_health" "this" {
-#   client_configuration   = data.talos_client_configuration.this.client_configuration
-#   control_plane_nodes    = [var.server_ip]
-#   endpoints              = data.talos_client_configuration.this.endpoints
-#   skip_kubernetes_checks = true
-# }
-
 data "http" "talos_health" {
-  url      = "https://${var.server_ip}:6443/version"
+  url      = "${local.cluster_endpoint}/version"
   insecure = true
   retry {
-    attempts     = 20
+    attempts     = 60
     min_delay_ms = 5000
     max_delay_ms = 5000
   }
-  depends_on = [
-    talos_machine_bootstrap.this,
-  ]
+  depends_on = [talos_machine_bootstrap.this]
 }

terraform-modules/talos/variables.tf

@@ -1,12 +1,31 @@
-variable "server_ip" {
-  type = string
+variable "virtual_ip" {
+  type        = string
+  description = "IP shared between control-plane nodes"
+  default     = ""
 }
 
 variable "cluster_name" {
   type = string
 }
 
-variable "config_patches" {
-  type    = list(string)
+variable "talos_version" {
+  type    = string
+  default = "v1.9"
+}
+
+variable "control_planes" {
+  type = list(object({
+    name           = string
+    server_ip      = string
+    config_patches = list(string)
+  }))
+}
+
+variable "workers" {
+  type = list(object({
+    name           = string
+    server_ip      = string
+    config_patches = list(string)
+  }))
   default = []
 }