feat: Add terraform modules

.github/workflows/terraform.yml

@@ -27,7 +27,7 @@ jobs:
 
     defaults:
       run:
-        working-directory: terraform
+        working-directory: ephemeral
 
     permissions:
       contents: read
@@ -80,7 +80,7 @@ jobs:
           body-includes: "## ephemeral: ${{ steps.sanitize.outputs.workspace }}"
 
       - name: Add comment
-        if: steps.comment.outputs.comment-id == null
+        if: steps.comment.outputs.comment-id == null && (inputs.command == 'create' || inputs.command == 'reset')
         uses: peter-evans/create-or-update-comment@v3
         with:
           comment-id: "${{ steps.comment.outputs.comment-id }}"
@@ -106,10 +106,20 @@ jobs:
         run: |
           case ${{ inputs.command }} in
           create)
-            terraform apply -auto-approve ;;
+            terraform apply -target local_sensitive_file.kubeconfig -auto-approve
+            terraform apply -auto-approve
+          ;;
           destroy)
-            terraform destroy -auto-approve ;;
+            terraform apply -target local_sensitive_file.kubeconfig -auto-approve
+            terraform destroy -auto-approve \
+              -target module.spectrum.helm_release.flux-sync \
+              -target module.spectrum.helm_release.flux \
+              -target module.spectrum.helm_release.cilium
+            terraform destroy -auto-approve 
+            terraform workspace delete ${{ steps.sanitize.outputs.workspace }}
+          ;;
           reset)
+            terraform apply -target local_sensitive_file.kubeconfig -auto-approve
             terraform destroy -auto-approve
             terraform apply -auto-approve
           ;;
@@ -126,8 +136,8 @@ jobs:
         with:
           name: configs
           path: |
-            terraform/kubeconfig
-            terraform/talosconfig
+            ephemeral/kubeconfig
+            ephemeral/talosconfig
 
       - name: Find comment
         if: inputs.command == 'create' || inputs.command == 'reset'

terraform/backend.tf → ephemeral/backend.tf

@@ -27,3 +27,9 @@ data "cloudflare_zone" "fluence_dev" {
 provider "vault" {
   address = "https://vault.fluence.dev"
 }
+
+provider "helm" {
+  kubernetes {
+    config_path = local_sensitive_file.kubeconfig.filename
+  }
+}

terraform/controlplane.tf → ephemeral/controlplane.tf

@@ -1,9 +1,3 @@
-locals {
-  cp = [
-    for i in range(1) : format("%s-%d", "cp", i)
-  ]
-}
-
 resource "digitalocean_loadbalancer" "cp" {
   name     = "rnd-${local.prefix}-controlplane"
   region   = "fra1"
@@ -54,13 +48,38 @@ resource "digitalocean_record" "endpoint" {
   ttl    = 30
 }
 
+resource "digitalocean_droplet" "cp" {
+  name      = "rnd-${local.prefix}-spectrum-cp"
+  size      = "s-4vcpu-8gb"
+  image     = data.digitalocean_image.talos.id
+  region    = "fra1"
+  vpc_uuid  = data.digitalocean_vpc.spectrum.id
+  user_data = data.talos_machine_configuration.cp.machine_configuration
+
+  ssh_keys = [
+    digitalocean_ssh_key.spectrum.id
+  ]
+
+  tags = [
+    local.prefix,
+    "${local.prefix}-controlplane",
+  ]
+}
+
+resource "digitalocean_record" "cp" {
+  name   = "cp"
+  value  = digitalocean_droplet.cp.ipv4_address
+  domain = digitalocean_domain.spectrum.id
+  type   = "A"
+  ttl    = 30
+}
+
+
 resource "talos_machine_secrets" "this" {
   talos_version = "v1.8"
 }
 
 data "talos_machine_configuration" "cp" {
-  for_each = { for index, name in local.cp : name => index }
-
   cluster_name     = terraform.workspace
   machine_type     = "controlplane"
   cluster_endpoint = "https://${local.loadbalancer_dns}:6443"
@@ -69,20 +88,20 @@ data "talos_machine_configuration" "cp" {
   config_patches = [
     templatefile("${path.module}/templates/controlplane_patch.yml", {
       loadbalancerdns = "kube.${local.prefix}.fluence.dev"
-      loadbalancerip  = digitalocean_loadbalancer.cp.ip
-      hostdns         = "${each.key}.${local.prefix}.fluence.dev",
+      hostdns         = "cp.${local.prefix}.fluence.dev",
       subnet          = data.digitalocean_vpc.spectrum.ip_range,
-      branch          = var.github_branch
-      dotoken         = base64encode(data.vault_generic_secret.spectrum.data.token)
-      domain          = "${local.prefix}.fluence.dev"
-      prefix          = local.prefix
-      pr_url          = var.github_pr_url
       docker_username = data.vault_generic_secret.docker.data.username
       docker_password = data.vault_generic_secret.docker.data.password
     })
   ]
 }
 
+resource "talos_machine_configuration_apply" "this" {
+  client_configuration        = talos_machine_secrets.this.client_configuration
+  machine_configuration_input = data.talos_machine_configuration.cp.machine_configuration
+  node                        = digitalocean_droplet.cp.ipv4_address
+}
+
 data "talos_client_configuration" "this" {
   cluster_name         = terraform.workspace
   client_configuration = talos_machine_secrets.this.client_configuration
@@ -91,61 +110,38 @@ data "talos_client_configuration" "this" {
   ]
 }
 
-resource "digitalocean_droplet" "cp" {
-  for_each = { for index, name in local.cp : name => index }
-
-  name      = "rnd-${local.prefix}-spectrum-${each.key}"
-  size      = "s-4vcpu-8gb"
-  image     = data.digitalocean_image.talos.id
-  region    = "fra1"
-  vpc_uuid  = data.digitalocean_vpc.spectrum.id
-  user_data = data.talos_machine_configuration.cp[each.key].machine_configuration
-
-  ssh_keys = [
-    digitalocean_ssh_key.spectrum.id
-  ]
-
-  tags = [
-    local.prefix,
-    "${local.prefix}-controlplane",
-  ]
-}
-
-resource "digitalocean_record" "cp" {
-  for_each = { for index, name in local.cp : name => index }
-
-  name   = each.key
-  value  = digitalocean_droplet.cp[each.key].ipv4_address
-  domain = digitalocean_domain.spectrum.id
-  type   = "A"
-  ttl    = 30
-}
-
 resource "talos_machine_bootstrap" "this" {
   client_configuration = talos_machine_secrets.this.client_configuration
-  endpoint             = digitalocean_droplet.cp["cp-0"].ipv4_address
-  node                 = digitalocean_droplet.cp["cp-0"].ipv4_address
+  endpoint             = digitalocean_droplet.cp.ipv4_address
+  node                 = digitalocean_droplet.cp.ipv4_address
   timeouts = {
     create = "3m"
   }
 
   lifecycle {
     replace_triggered_by = [
-      digitalocean_droplet.cp["cp-0"].id
+      digitalocean_droplet.cp.id
     ]
   }
 }
 
 resource "talos_cluster_kubeconfig" "this" {
   client_configuration = talos_machine_secrets.this.client_configuration
-  node                 = digitalocean_droplet.cp["cp-0"].ipv4_address
+  node                 = digitalocean_droplet.cp.ipv4_address
   timeouts = {
     create = "3m"
   }
 }
 
-data "talos_cluster_health" "health" {
-  client_configuration = data.talos_client_configuration.this.client_configuration
-  control_plane_nodes  = [for droplet in digitalocean_droplet.cp : droplet.ipv4_address_private]
-  endpoints            = data.talos_client_configuration.this.endpoints
+data "http" "talos_health" {
+  url      = "https://${digitalocean_record.endpoint.fqdn}:6443/version"
+  insecure = true
+  retry {
+    attempts     = 20
+    min_delay_ms = 5000
+    max_delay_ms = 5000
+  }
+  depends_on = [
+    talos_machine_bootstrap.this,
+  ]
 }

terraform/dns.tf → ephemeral/dns.tf

@@ -1,7 +1,3 @@
-resource "digitalocean_domain" "spectrum" {
-  name = "${local.prefix}.fluence.dev"
-}
-
 resource "cloudflare_record" "ns" {
   for_each = toset([
     "ns1.digitalocean.com",
@@ -14,3 +10,8 @@ resource "cloudflare_record" "ns" {
   content = each.key
   type    = "NS"
 }
+
+resource "digitalocean_domain" "spectrum" {
+  depends_on = [cloudflare_record.ns]
+  name       = "${local.prefix}.fluence.dev"
+}

ephemeral/main.tf

@@ -0,0 +1,47 @@
+locals {
+  prefix           = terraform.workspace
+  loadbalancer_dns = "kube.${local.prefix}.fluence.dev"
+}
+
+resource "tls_private_key" "spectrum" {
+  algorithm = "ED25519"
+}
+
+resource "digitalocean_ssh_key" "spectrum" {
+  name       = "${local.prefix}-ssh-key"
+  public_key = tls_private_key.spectrum.public_key_openssh
+}
+
+data "digitalocean_image" "talos" {
+  name = "talos-v1.8.2"
+}
+
+data "vault_generic_secret" "spectrum" {
+  path = "kv/digitalocean/spectrum"
+}
+
+data "vault_generic_secret" "docker" {
+  path = "kv/docker-registry/basicauth/ci"
+}
+
+module "spectrum" {
+  depends_on = [
+    data.http.talos_health,
+    local_sensitive_file.kubeconfig,
+  ]
+  source     = "../terraform-modules/spectrum"
+  components = ["kubevirt"]
+  network    = var.github_branch
+  cluster    = "ephemeral"
+
+  cilium_hubble_enabled = true
+
+  flux_variables = {
+    PR_URL          = var.github_pr_url
+    LOADBALANCER_IP = digitalocean_loadbalancer.cp.ip
+    BRANCH          = var.github_branch
+    DOTOKEN         = base64encode(data.vault_generic_secret.spectrum.data.token)
+    DOMAIN          = "${local.prefix}.fluence.dev"
+    PREFIX          = local.prefix
+  }
+}

terraform/outputs.tf → ephemeral/outputs.tf

@@ -8,13 +8,13 @@ output "kubeconfig" {
   sensitive = true
 }
 
-resource "local_file" "kubeconfig" {
+resource "local_sensitive_file" "kubeconfig" {
   content         = talos_cluster_kubeconfig.this.kubeconfig_raw
   filename        = "${path.module}/kubeconfig"
   file_permission = "0600"
 }
 
-resource "local_file" "talosconfig" {
+resource "local_sensitive_file" "talosconfig" {
   content         = data.talos_client_configuration.this.talos_config
   filename        = "${path.module}/talosconfig"
   file_permission = "0600"

ephemeral/templates/controlplane_patch.yml

@@ -0,0 +1,32 @@
+machine:
+  kubelet:
+    nodeIP:
+      validSubnets:
+        - ${subnet}
+  certSANs:
+    - ${loadbalancerdns}
+    - ${hostdns}
+  time:
+    servers:
+      - time.cloudflare.com
+  registries:
+    config:
+      docker.fluence.dev:
+        auth:
+          username: ${docker_username}
+          password: ${docker_password}
+
+cluster:
+  allowSchedulingOnControlPlanes: true
+  apiServer:
+    admissionControl:
+      - name: PodSecurity
+        configuration:
+          exemptions:
+            namespaces:
+              - cilium-test # to run cilium connectivity tests
+  network:
+    cni:
+      name: none
+  proxy:
+    disabled: true

flux/apps/cert-manager/issuer/kustomization.yml

@@ -2,4 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cluster-issuer.yml
-  - secret.yml

flux/apps/cert-manager/ks.yml

@@ -27,7 +27,3 @@ spec:
     namespace: flux-system
   dependsOn:
     - name: cert-manager
-  postBuild:
-    substituteFrom:
-    - kind: ConfigMap
-      name: terraform-config

flux/apps/cert-manager/kustomization.yml

@@ -3,3 +3,4 @@ kind: Kustomization
 resources:
   - ks.yml
   - namespace.yml
+  - secret.yml

flux/clusters/default/kustomization.yml

@@ -0,0 +1,2 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

flux/clusters/ephemeral/hubble-ingress.yml

@@ -0,0 +1,29 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: hubble-ingress
+  namespace: kube-system
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    gethomepage.dev/enabled: "true"
+    gethomepage.dev/group: "Observability"
+    gethomepage.dev/icon: "cilium.png"
+    gethomepage.dev/name: "Hubble"
+    gethomepage.dev/pod-selector: "app.kubernetes.io/name=hubble-ui"
+    cert-manager.io/cluster-issuer: "letsencrypt"
+spec:
+  rules:
+    - host: hubble.${DOMAIN}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: hubble-ui
+                port:
+                  number: 80
+  tls:
+    - hosts:
+        - hubble.${DOMAIN}
+      secretName: hubble-tls

flux/clusters/ephemeral/kustomization.yml

@@ -1,10 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ../../core/cilium
   - ../../core/kubernetes-dashboard
-  - ../../core/kubevirt
   - ../../apps/external-dns
   - ../../apps/ingress-nginx
   - ../../apps/cert-manager
   - ../../apps/homepage
+  - hubble-ingress.yml