diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index a7163895..ce119236 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -27,7 +27,7 @@ jobs: defaults: run: - working-directory: terraform + working-directory: ephemeral permissions: contents: read @@ -80,7 +80,7 @@ jobs: body-includes: "## ephemeral: ${{ steps.sanitize.outputs.workspace }}" - name: Add comment - if: steps.comment.outputs.comment-id == null + if: steps.comment.outputs.comment-id == null && (inputs.command == 'create' || inputs.command == 'reset') uses: peter-evans/create-or-update-comment@v3 with: comment-id: "${{ steps.comment.outputs.comment-id }}" @@ -106,10 +106,20 @@ jobs: run: | case ${{ inputs.command }} in create) - terraform apply -auto-approve ;; + terraform apply -target local_sensitive_file.kubeconfig -auto-approve + terraform apply -auto-approve + ;; destroy) - terraform destroy -auto-approve ;; + terraform apply -target local_sensitive_file.kubeconfig -auto-approve + terraform destroy -auto-approve \ + -target module.spectrum.helm_release.flux-sync \ + -target module.spectrum.helm_release.flux \ + -target module.spectrum.helm_release.cilium + terraform destroy -auto-approve + terraform workspace delete ${{ steps.sanitize.outputs.workspace }} + ;; reset) + terraform apply -target local_sensitive_file.kubeconfig -auto-approve terraform destroy -auto-approve terraform apply -auto-approve ;; @@ -126,8 +136,8 @@ jobs: with: name: configs path: | - terraform/kubeconfig - terraform/talosconfig + ephemeral/kubeconfig + ephemeral/talosconfig - name: Find comment if: inputs.command == 'create' || inputs.command == 'reset' diff --git a/terraform/backend.tf b/ephemeral/backend.tf similarity index 83% rename from terraform/backend.tf rename to ephemeral/backend.tf index 74e5dd31..f6ec94e5 100644 --- a/terraform/backend.tf +++ b/ephemeral/backend.tf @@ -27,3 +27,9 @@ data "cloudflare_zone" "fluence_dev" { provider "vault" { address = "https://vault.fluence.dev" } + +provider "helm" { + kubernetes { + config_path = local_sensitive_file.kubeconfig.filename + } +} diff --git a/terraform/controlplane.tf b/ephemeral/controlplane.tf similarity index 66% rename from terraform/controlplane.tf rename to ephemeral/controlplane.tf index 40339f1b..e6d716f4 100644 --- a/terraform/controlplane.tf +++ b/ephemeral/controlplane.tf @@ -1,9 +1,3 @@ -locals { - cp = [ - for i in range(1) : format("%s-%d", "cp", i) - ] -} - resource "digitalocean_loadbalancer" "cp" { name = "rnd-${local.prefix}-controlplane" region = "fra1" @@ -54,13 +48,38 @@ resource "digitalocean_record" "endpoint" { ttl = 30 } +resource "digitalocean_droplet" "cp" { + name = "rnd-${local.prefix}-spectrum-cp" + size = "s-4vcpu-8gb" + image = data.digitalocean_image.talos.id + region = "fra1" + vpc_uuid = data.digitalocean_vpc.spectrum.id + user_data = data.talos_machine_configuration.cp.machine_configuration + + ssh_keys = [ + digitalocean_ssh_key.spectrum.id + ] + + tags = [ + local.prefix, + "${local.prefix}-controlplane", + ] +} + +resource "digitalocean_record" "cp" { + name = "cp" + value = digitalocean_droplet.cp.ipv4_address + domain = digitalocean_domain.spectrum.id + type = "A" + ttl = 30 +} + + resource "talos_machine_secrets" "this" { talos_version = "v1.8" } data "talos_machine_configuration" "cp" { - for_each = { for index, name in local.cp : name => index } - cluster_name = terraform.workspace machine_type = "controlplane" cluster_endpoint = "https://${local.loadbalancer_dns}:6443" @@ -69,20 +88,20 @@ data "talos_machine_configuration" "cp" { config_patches = [ templatefile("${path.module}/templates/controlplane_patch.yml", { loadbalancerdns = "kube.${local.prefix}.fluence.dev" - loadbalancerip = digitalocean_loadbalancer.cp.ip - hostdns = "${each.key}.${local.prefix}.fluence.dev", + hostdns = "cp.${local.prefix}.fluence.dev", subnet = data.digitalocean_vpc.spectrum.ip_range, - branch = var.github_branch - dotoken = base64encode(data.vault_generic_secret.spectrum.data.token) - domain = "${local.prefix}.fluence.dev" - prefix = local.prefix - pr_url = var.github_pr_url docker_username = data.vault_generic_secret.docker.data.username docker_password = data.vault_generic_secret.docker.data.password }) ] } +resource "talos_machine_configuration_apply" "this" { + client_configuration = talos_machine_secrets.this.client_configuration + machine_configuration_input = data.talos_machine_configuration.cp.machine_configuration + node = digitalocean_droplet.cp.ipv4_address +} + data "talos_client_configuration" "this" { cluster_name = terraform.workspace client_configuration = talos_machine_secrets.this.client_configuration @@ -91,61 +110,38 @@ data "talos_client_configuration" "this" { ] } -resource "digitalocean_droplet" "cp" { - for_each = { for index, name in local.cp : name => index } - - name = "rnd-${local.prefix}-spectrum-${each.key}" - size = "s-4vcpu-8gb" - image = data.digitalocean_image.talos.id - region = "fra1" - vpc_uuid = data.digitalocean_vpc.spectrum.id - user_data = data.talos_machine_configuration.cp[each.key].machine_configuration - - ssh_keys = [ - digitalocean_ssh_key.spectrum.id - ] - - tags = [ - local.prefix, - "${local.prefix}-controlplane", - ] -} - -resource "digitalocean_record" "cp" { - for_each = { for index, name in local.cp : name => index } - - name = each.key - value = digitalocean_droplet.cp[each.key].ipv4_address - domain = digitalocean_domain.spectrum.id - type = "A" - ttl = 30 -} - resource "talos_machine_bootstrap" "this" { client_configuration = talos_machine_secrets.this.client_configuration - endpoint = digitalocean_droplet.cp["cp-0"].ipv4_address - node = digitalocean_droplet.cp["cp-0"].ipv4_address + endpoint = digitalocean_droplet.cp.ipv4_address + node = digitalocean_droplet.cp.ipv4_address timeouts = { create = "3m" } lifecycle { replace_triggered_by = [ - digitalocean_droplet.cp["cp-0"].id + digitalocean_droplet.cp.id ] } } resource "talos_cluster_kubeconfig" "this" { client_configuration = talos_machine_secrets.this.client_configuration - node = digitalocean_droplet.cp["cp-0"].ipv4_address + node = digitalocean_droplet.cp.ipv4_address timeouts = { create = "3m" } } -data "talos_cluster_health" "health" { - client_configuration = data.talos_client_configuration.this.client_configuration - control_plane_nodes = [for droplet in digitalocean_droplet.cp : droplet.ipv4_address_private] - endpoints = data.talos_client_configuration.this.endpoints +data "http" "talos_health" { + url = "https://${digitalocean_record.endpoint.fqdn}:6443/version" + insecure = true + retry { + attempts = 20 + min_delay_ms = 5000 + max_delay_ms = 5000 + } + depends_on = [ + talos_machine_bootstrap.this, + ] } diff --git a/terraform/dns.tf b/ephemeral/dns.tf similarity index 79% rename from terraform/dns.tf rename to ephemeral/dns.tf index 452bc38f..8d2cfe08 100644 --- a/terraform/dns.tf +++ b/ephemeral/dns.tf @@ -1,7 +1,3 @@ -resource "digitalocean_domain" "spectrum" { - name = "${local.prefix}.fluence.dev" -} - resource "cloudflare_record" "ns" { for_each = toset([ "ns1.digitalocean.com", @@ -14,3 +10,8 @@ resource "cloudflare_record" "ns" { content = each.key type = "NS" } + +resource "digitalocean_domain" "spectrum" { + depends_on = [cloudflare_record.ns] + name = "${local.prefix}.fluence.dev" +} diff --git a/ephemeral/main.tf b/ephemeral/main.tf new file mode 100644 index 00000000..3bfaf2b9 --- /dev/null +++ b/ephemeral/main.tf @@ -0,0 +1,47 @@ +locals { + prefix = terraform.workspace + loadbalancer_dns = "kube.${local.prefix}.fluence.dev" +} + +resource "tls_private_key" "spectrum" { + algorithm = "ED25519" +} + +resource "digitalocean_ssh_key" "spectrum" { + name = "${local.prefix}-ssh-key" + public_key = tls_private_key.spectrum.public_key_openssh +} + +data "digitalocean_image" "talos" { + name = "talos-v1.8.2" +} + +data "vault_generic_secret" "spectrum" { + path = "kv/digitalocean/spectrum" +} + +data "vault_generic_secret" "docker" { + path = "kv/docker-registry/basicauth/ci" +} + +module "spectrum" { + depends_on = [ + data.http.talos_health, + local_sensitive_file.kubeconfig, + ] + source = "../terraform-modules/spectrum" + components = ["kubevirt"] + network = var.github_branch + cluster = "ephemeral" + + cilium_hubble_enabled = true + + flux_variables = { + PR_URL = var.github_pr_url + LOADBALANCER_IP = digitalocean_loadbalancer.cp.ip + BRANCH = var.github_branch + DOTOKEN = base64encode(data.vault_generic_secret.spectrum.data.token) + DOMAIN = "${local.prefix}.fluence.dev" + PREFIX = local.prefix + } +} diff --git a/terraform/network.tf b/ephemeral/network.tf similarity index 100% rename from terraform/network.tf rename to ephemeral/network.tf diff --git a/terraform/outputs.tf b/ephemeral/outputs.tf similarity index 84% rename from terraform/outputs.tf rename to ephemeral/outputs.tf index a6751fa2..acd5c383 100644 --- a/terraform/outputs.tf +++ b/ephemeral/outputs.tf @@ -8,13 +8,13 @@ output "kubeconfig" { sensitive = true } -resource "local_file" "kubeconfig" { +resource "local_sensitive_file" "kubeconfig" { content = talos_cluster_kubeconfig.this.kubeconfig_raw filename = "${path.module}/kubeconfig" file_permission = "0600" } -resource "local_file" "talosconfig" { +resource "local_sensitive_file" "talosconfig" { content = data.talos_client_configuration.this.talos_config filename = "${path.module}/talosconfig" file_permission = "0600" diff --git a/ephemeral/templates/controlplane_patch.yml b/ephemeral/templates/controlplane_patch.yml new file mode 100644 index 00000000..0f3e23ea --- /dev/null +++ b/ephemeral/templates/controlplane_patch.yml @@ -0,0 +1,32 @@ +machine: + kubelet: + nodeIP: + validSubnets: + - ${subnet} + certSANs: + - ${loadbalancerdns} + - ${hostdns} + time: + servers: + - time.cloudflare.com + registries: + config: + docker.fluence.dev: + auth: + username: ${docker_username} + password: ${docker_password} + +cluster: + allowSchedulingOnControlPlanes: true + apiServer: + admissionControl: + - name: PodSecurity + configuration: + exemptions: + namespaces: + - cilium-test # to run cilium connectivity tests + network: + cni: + name: none + proxy: + disabled: true diff --git a/terraform/variables.tf b/ephemeral/variables.tf similarity index 100% rename from terraform/variables.tf rename to ephemeral/variables.tf diff --git a/flux/apps/cert-manager/issuer/kustomization.yml b/flux/apps/cert-manager/issuer/kustomization.yml index 30407f72..86fec029 100644 --- a/flux/apps/cert-manager/issuer/kustomization.yml +++ b/flux/apps/cert-manager/issuer/kustomization.yml @@ -2,4 +2,3 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cluster-issuer.yml - - secret.yml diff --git a/flux/apps/cert-manager/ks.yml b/flux/apps/cert-manager/ks.yml index d6a676b3..8723a0cb 100644 --- a/flux/apps/cert-manager/ks.yml +++ b/flux/apps/cert-manager/ks.yml @@ -27,7 +27,3 @@ spec: namespace: flux-system dependsOn: - name: cert-manager - postBuild: - substituteFrom: - - kind: ConfigMap - name: terraform-config diff --git a/flux/apps/cert-manager/kustomization.yml b/flux/apps/cert-manager/kustomization.yml index 27a0edaf..340dc026 100644 --- a/flux/apps/cert-manager/kustomization.yml +++ b/flux/apps/cert-manager/kustomization.yml @@ -3,3 +3,4 @@ kind: Kustomization resources: - ks.yml - namespace.yml + - secret.yml diff --git a/flux/apps/cert-manager/issuer/secret.yml b/flux/apps/cert-manager/secret.yml similarity index 100% rename from flux/apps/cert-manager/issuer/secret.yml rename to flux/apps/cert-manager/secret.yml diff --git a/flux/clusters/default/kustomization.yml b/flux/clusters/default/kustomization.yml new file mode 100644 index 00000000..4ea1bcca --- /dev/null +++ b/flux/clusters/default/kustomization.yml @@ -0,0 +1,2 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization diff --git a/flux/clusters/ephemeral/hubble-ingress.yml b/flux/clusters/ephemeral/hubble-ingress.yml new file mode 100644 index 00000000..589e42ac --- /dev/null +++ b/flux/clusters/ephemeral/hubble-ingress.yml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: hubble-ingress + namespace: kube-system + annotations: + kubernetes.io/ingress.class: "nginx" + gethomepage.dev/enabled: "true" + gethomepage.dev/group: "Observability" + gethomepage.dev/icon: "cilium.png" + gethomepage.dev/name: "Hubble" + gethomepage.dev/pod-selector: "app.kubernetes.io/name=hubble-ui" + cert-manager.io/cluster-issuer: "letsencrypt" +spec: + rules: + - host: hubble.${DOMAIN} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: hubble-ui + port: + number: 80 + tls: + - hosts: + - hubble.${DOMAIN} + secretName: hubble-tls diff --git a/flux/clusters/ephemeral/kustomization.yml b/flux/clusters/ephemeral/kustomization.yml index c2d178ca..d21896ba 100644 --- a/flux/clusters/ephemeral/kustomization.yml +++ b/flux/clusters/ephemeral/kustomization.yml @@ -1,10 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../../core/cilium - ../../core/kubernetes-dashboard - - ../../core/kubevirt - ../../apps/external-dns - ../../apps/ingress-nginx - ../../apps/cert-manager - ../../apps/homepage + - hubble-ingress.yml diff --git a/flux/core/kubevirt/kustomization.yml b/flux/components/kubevirt/app/kustomization.yml similarity index 100% rename from flux/core/kubevirt/kustomization.yml rename to flux/components/kubevirt/app/kustomization.yml diff --git a/flux/components/kubevirt/ks.yml b/flux/components/kubevirt/ks.yml new file mode 100644 index 00000000..a1cacebd --- /dev/null +++ b/flux/components/kubevirt/ks.yml @@ -0,0 +1,13 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: kube-virt + namespace: flux-system +spec: + interval: 1m0s + path: ./flux/components/kube-virt/app + prune: true + sourceRef: + kind: GitRepository + name: spectrum + namespace: flux-system diff --git a/flux/components/kubevirt/kustomization.yml b/flux/components/kubevirt/kustomization.yml new file mode 100644 index 00000000..d1943374 --- /dev/null +++ b/flux/components/kubevirt/kustomization.yml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ks.yaml diff --git a/flux/core/cilium/helm-release.yml b/flux/core/cilium/helm-release.yml deleted file mode 100644 index 78f2b4b0..00000000 --- a/flux/core/cilium/helm-release.yml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: cilium - namespace: kube-system -spec: - interval: 5m - chart: - spec: - chart: cilium - version: 1.16.x - sourceRef: - kind: HelmRepository - name: cilium - namespace: flux-system - valuesFrom: - - kind: ConfigMap - name: cilium-values diff --git a/flux/core/cilium/helm-repository.yml b/flux/core/cilium/helm-repository.yml deleted file mode 100644 index f6b07eff..00000000 --- a/flux/core/cilium/helm-repository.yml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: cilium - namespace: flux-system -spec: - url: https://helm.cilium.io/ - interval: 24h diff --git a/flux/core/cilium/kustomization.yml b/flux/core/cilium/kustomization.yml deleted file mode 100644 index 399b461d..00000000 --- a/flux/core/cilium/kustomization.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - helm-repository.yml - - helm-release.yml - -configMapGenerator: - - name: cilium-values - namespace: kube-system - files: - - values.yaml=./values.yml - -configurations: - - kustomize-config.yml diff --git a/flux/core/cilium/kustomize-config.yml b/flux/core/cilium/kustomize-config.yml deleted file mode 100644 index a80be15a..00000000 --- a/flux/core/cilium/kustomize-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -nameReference: - - kind: ConfigMap - version: v1 - fieldSpecs: - - path: spec/valuesFrom/name - kind: HelmRelease diff --git a/terraform-modules/spectrum/backend.tf b/terraform-modules/spectrum/backend.tf new file mode 100644 index 00000000..7e025848 --- /dev/null +++ b/terraform-modules/spectrum/backend.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.16" + } + } +} diff --git a/terraform-modules/spectrum/cilium.tf b/terraform-modules/spectrum/cilium.tf new file mode 100644 index 00000000..54ef6247 --- /dev/null +++ b/terraform-modules/spectrum/cilium.tf @@ -0,0 +1,28 @@ +locals { + invalid_l2_configuration = var.cilium_l2_enabled && length(var.cilium_devices) == 0 +} + +resource "null_resource" "validate_l2_config" { + count = local.invalid_l2_configuration ? 1 : 0 + + provisioner "local-exec" { + command = "echo 'Validation failed: If cilium_l2_enabled is true, cilium_devices must not be empty.'; exit 1" + } +} + +resource "helm_release" "cilium" { + name = "cilium" + chart = "cilium" + repository = "https://helm.cilium.io/" + namespace = "kube-system" + version = "1.16.4" + wait = true + + values = [ + templatefile("${path.module}/templates/cilium.yml", { + l2_enabled = var.cilium_l2_enabled, + devices = var.cilium_devices + hubble_enabled = var.cilium_hubble_enabled + }) + ] +} diff --git a/terraform-modules/spectrum/flux.tf b/terraform-modules/spectrum/flux.tf new file mode 100644 index 00000000..934dd21f --- /dev/null +++ b/terraform-modules/spectrum/flux.tf @@ -0,0 +1,42 @@ +resource "helm_release" "flux" { + depends_on = [helm_release.cilium] + name = "flux" + chart = "flux2" + repository = "https://fluxcd-community.github.io/helm-charts/" + namespace = "flux-system" + create_namespace = true + wait = true + + set { + name = "notificationController.create" + value = "false" + } + + set { + name = "imageReflectionController.create" + value = "false" + } + + set { + name = "imageAutomationController.create" + value = "false" + } +} + +resource "helm_release" "flux-sync" { + depends_on = [helm_release.flux] + name = "spectrum" + chart = "flux2-sync" + repository = "https://fluxcd-community.github.io/helm-charts/" + namespace = "flux-system" + wait = true + + values = [ + templatefile("${path.module}/templates/flux-sync.yml", { + network = var.network + cluster = var.cluster + variables = var.flux_variables + components = var.components + }) + ] +} diff --git a/flux/core/cilium/values.yml b/terraform-modules/spectrum/templates/cilium.yml similarity index 63% rename from flux/core/cilium/values.yml rename to terraform-modules/spectrum/templates/cilium.yml index 8e675956..a6d0d338 100644 --- a/flux/core/cilium/values.yml +++ b/terraform-modules/spectrum/templates/cilium.yml @@ -8,9 +8,6 @@ envoy.rollOutPods: true k8sServiceHost: localhost k8sServicePort: 7445 -l2announcements: - enabled: true - ipam: mode: kubernetes @@ -41,25 +38,23 @@ cgroup: hostRoot: /sys/fs/cgroup hubble: - enabled: true + enabled: ${hubble_enabled} relay: enabled: true rollOutPods: true ui: enabled: true rollOutPods: true - ingress: - enabled: true - className: nginx - annotations: - gethomepage.dev/enabled: "true" - gethomepage.dev/group: Observability - gethomepage.dev/icon: cilium.png - gethomepage.dev/name: Hubble - cert-manager.io/cluster-issuer: "letsencrypt" - hosts: - - &host "hubble.${DOMAIN}" - tls: - - hosts: - - *host - secretName: hubble-tls + +%{ if l2_enabled } +l2announcements: + enabled: true + +externalIPs: + enabled: true + +devices: + %{ for device in devices } + - ${device} + %{ endfor } +%{ endif } diff --git a/terraform-modules/spectrum/templates/flux-sync.yml b/terraform-modules/spectrum/templates/flux-sync.yml new file mode 100644 index 00000000..a754d153 --- /dev/null +++ b/terraform-modules/spectrum/templates/flux-sync.yml @@ -0,0 +1,33 @@ +gitRepository: + spec: + url: "https://github.com/fluencelabs/spectrum.git" + interval: "1m" + ref: + branch: ${network} + +kustomization: + spec: + interval: "1m0s" + path: "./flux/clusters/${cluster}" + validation: "client" + prune: true + %{ if variables != null } + postBuild: + substitute: + %{ for key, value in variables } + ${key}: ${value} + %{ endfor } + %{ endif } + +kustomizationlist: + - spec: + interval: 1m0s + path: "./flux/components/lightmare/app" + prune: true + + %{ for component in components } + - spec: + interval: 1m0s + path: "./flux/components/${component}/app" + prune: true + %{ endfor } diff --git a/terraform-modules/spectrum/variables.tf b/terraform-modules/spectrum/variables.tf new file mode 100644 index 00000000..1519a567 --- /dev/null +++ b/terraform-modules/spectrum/variables.tf @@ -0,0 +1,34 @@ +variable "network" { + type = string + default = "main" +} + +variable "cluster" { + type = string + default = "default" +} + +variable "cilium_l2_enabled" { + type = bool + default = false +} + +variable "cilium_devices" { + type = list(string) + default = [] +} + +variable "cilium_hubble_enabled" { + type = bool + default = false +} + +variable "flux_variables" { + type = map(string) + default = {} +} + +variable "components" { + type = list(string) + default = [] +} diff --git a/terraform-modules/talos/backend.tf b/terraform-modules/talos/backend.tf new file mode 100644 index 00000000..f9bb76e1 --- /dev/null +++ b/terraform-modules/talos/backend.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + talos = { + source = "siderolabs/talos" + version = "~> 0.6" + } + } +} diff --git a/terraform-modules/talos/outputs.tf b/terraform-modules/talos/outputs.tf new file mode 100644 index 00000000..8b602985 --- /dev/null +++ b/terraform-modules/talos/outputs.tf @@ -0,0 +1,27 @@ +resource "local_sensitive_file" "kubeconfig" { + content = talos_cluster_kubeconfig.this.kubeconfig_raw + filename = "${path.root}/kubeconfig" + file_permission = "0600" +} + +resource "local_sensitive_file" "talosconfig" { + content = data.talos_client_configuration.this.talos_config + filename = "${path.root}/talosconfig" + file_permission = "0600" +} + +output "kubeconfig" { + value = { + path = local_sensitive_file.kubeconfig.filename + content = local_sensitive_file.kubeconfig.content + } + sensitive = true +} + +output "talosconfig" { + value = { + path = local_sensitive_file.talosconfig.filename + content = local_sensitive_file.talosconfig.content + } + sensitive = true +} diff --git a/terraform-modules/talos/talos.tf b/terraform-modules/talos/talos.tf new file mode 100644 index 00000000..ab0753f1 --- /dev/null +++ b/terraform-modules/talos/talos.tf @@ -0,0 +1,48 @@ +resource "talos_machine_secrets" "this" { + talos_version = "v1.8" +} + +data "talos_machine_configuration" "this" { + cluster_name = var.cluster_name + machine_type = "controlplane" + cluster_endpoint = "https://${var.public_ip}:6443" + machine_secrets = talos_machine_secrets.this.machine_secrets + talos_version = "v1.8" + config_patches = [ + file("${path.module}/templates/controlplane_patch.yml") + ] +} + +resource "talos_machine_configuration_apply" "this" { + client_configuration = talos_machine_secrets.this.client_configuration + machine_configuration_input = data.talos_machine_configuration.this.machine_configuration + node = var.public_ip + config_patches = var.config_patches +} + +data "talos_client_configuration" "this" { + cluster_name = var.cluster_name + client_configuration = talos_machine_secrets.this.client_configuration + endpoints = [ + var.public_ip + ] +} + +resource "talos_machine_bootstrap" "this" { + depends_on = [talos_machine_configuration_apply.this] + client_configuration = talos_machine_secrets.this.client_configuration + endpoint = var.public_ip + node = var.public_ip +} + +resource "talos_cluster_kubeconfig" "this" { + client_configuration = talos_machine_secrets.this.client_configuration + node = var.public_ip +} + +data "talos_cluster_health" "this" { + client_configuration = data.talos_client_configuration.this.client_configuration + control_plane_nodes = [var.public_ip] + endpoints = data.talos_client_configuration.this.endpoints + skip_kubernetes_checks = true +} diff --git a/terraform-modules/talos/templates/controlplane_patch.yml b/terraform-modules/talos/templates/controlplane_patch.yml new file mode 100644 index 00000000..7290140e --- /dev/null +++ b/terraform-modules/talos/templates/controlplane_patch.yml @@ -0,0 +1,18 @@ +machine: + time: + servers: + - time.cloudflare.com + install: + diskSelector: + size: '>= 100GB' + image: ghcr.io/siderolabs/installer:v1.8.3 + bootloader: true + wipe: true + +cluster: + allowSchedulingOnControlPlanes: true + network: + cni: + name: none + proxy: + disabled: true diff --git a/terraform-modules/talos/userdata.sh b/terraform-modules/talos/userdata.sh new file mode 100644 index 00000000..2ef0c349 --- /dev/null +++ b/terraform-modules/talos/userdata.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +KERNEL_URL="https://pxe.factory.talos.dev/image/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/v1.8.3/kernel-amd64" +INITRD_URL="https://pxe.factory.talos.dev/image/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/v1.8.3/initramfs-amd64.xz" +CMDLINE="talos.platform=metal talos.exp.wipe=system console=tty0 init_on_alloc=1 slab_nomerge pti=on consoleblank=0 nvme_core.io_timeout=4294967295 printk.devkmsg=on ima_template=ima-ng ima_appraise=fix ima_hash=sha512" + +wget -O /boot/talos-vmlinuz $KERNEL_URL +wget -O /boot/talos-initrd.xz $INITRD_URL + +apt install kexec-tools -y + +kexec -l /boot/talos-vmlinuz --initrd=/boot/talos-initrd.xz --append="$CMDLINE" +kexec -e diff --git a/terraform-modules/talos/variables.tf b/terraform-modules/talos/variables.tf new file mode 100644 index 00000000..edfb117b --- /dev/null +++ b/terraform-modules/talos/variables.tf @@ -0,0 +1,12 @@ +variable "public_ip" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "config_patches" { + type = list(string) + default = [] +} diff --git a/terraform/main.tf b/terraform/main.tf deleted file mode 100644 index 536adb7d..00000000 --- a/terraform/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -locals { - prefix = terraform.workspace - loadbalancer_dns = "kube.${local.prefix}.fluence.dev" -} - -resource "tls_private_key" "spectrum" { - algorithm = "ED25519" -} - -resource "digitalocean_ssh_key" "spectrum" { - name = "${local.prefix}-ssh-key" - public_key = tls_private_key.spectrum.public_key_openssh -} - -data "digitalocean_image" "talos" { - name = "talos-v1.8.2" -} - -data "vault_generic_secret" "spectrum" { - path = "kv/digitalocean/spectrum" -} - -data "vault_generic_secret" "docker" { - path = "kv/docker-registry/basicauth/ci" -} diff --git a/terraform/templates/controlplane_patch.yml b/terraform/templates/controlplane_patch.yml deleted file mode 100644 index 8017aabe..00000000 --- a/terraform/templates/controlplane_patch.yml +++ /dev/null @@ -1,186 +0,0 @@ -machine: - kubelet: - nodeIP: - validSubnets: - - ${subnet} - certSANs: - - ${loadbalancerdns} - - ${hostdns} - time: - servers: - - time.cloudflare.com - registries: - config: - docker.fluence.dev: - auth: - username: ${docker_username} - password: ${docker_password} - -cluster: - allowSchedulingOnControlPlanes: true - apiServer: - admissionControl: - - name: PodSecurity - configuration: - exemptions: - namespaces: - - cilium-test # to run cilium connectivity tests - network: - cni: - name: none - proxy: - disabled: true - extraManifests: - - https://github.com/fluxcd/flux2/releases/download/v2.4.0/install.yaml - inlineManifests: - - name: cilium - contents: |- - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: cilium-install - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - subjects: - - kind: ServiceAccount - name: cilium-install - namespace: kube-system - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cilium-install - namespace: kube-system - --- - apiVersion: batch/v1 - kind: Job - metadata: - name: cilium-install - namespace: kube-system - spec: - backoffLimit: 10 - template: - metadata: - labels: - app: cilium-install - spec: - restartPolicy: OnFailure - tolerations: - - operator: Exists - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - - effect: PreferNoSchedule - operator: Exists - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoSchedule - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: NoExecute - - key: node-role.kubernetes.io/control-plane - operator: Exists - effect: PreferNoSchedule - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - serviceAccount: cilium-install - serviceAccountName: cilium-install - hostNetwork: true - containers: - - name: cilium-install - image: quay.io/cilium/cilium-cli-ci:v0.16.19 - env: - - name: KUBERNETES_SERVICE_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: KUBERNETES_SERVICE_PORT - value: "6443" - command: - - cilium - - install - - --helm-set=ipam.mode=kubernetes - - --helm-set=kubeProxyReplacement=true - - --helm-set=securityContext.capabilities.ciliumAgent={CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID} - - --helm-set=securityContext.capabilities.cleanCiliumState={NET_ADMIN,SYS_ADMIN,SYS_RESOURCE} - - --helm-set=cgroup.autoMount.enabled=false - - --helm-set=cgroup.hostRoot=/sys/fs/cgroup - - --helm-set=k8sServiceHost=localhost - - --helm-set=k8sServicePort=7445 - - name: flux - contents: |- - --- - apiVersion: v1 - kind: Namespace - metadata: - name: flux-system - labels: - app.kubernetes.io/instance: flux-system - app.kubernetes.io/part-of: flux - pod-security.kubernetes.io/warn: restricted - pod-security.kubernetes.io/warn-version: latest - --- - apiVersion: source.toolkit.fluxcd.io/v1 - kind: GitRepository - metadata: - name: spectrum - namespace: flux-system - spec: - interval: 1m0s - url: https://github.com/fluencelabs/spectrum.git - ref: - branch: ${branch} - --- - apiVersion: v1 - kind: ConfigMap - metadata: - name: terraform-config - namespace: flux-system - data: - DOMAIN: ${domain} - LOADBALANCER_IP: ${loadbalancerip} - BRANCH: ${branch} - PREFIX: ${prefix} - DOTOKEN: ${dotoken} - PR_URL: ${pr_url} - --- - apiVersion: source.toolkit.fluxcd.io/v1 - kind: GitRepository - metadata: - name: spectrum - namespace: flux-system - spec: - interval: 1m0s - url: https://github.com/fluencelabs/spectrum.git - ref: - branch: ${branch} - --- - apiVersion: kustomize.toolkit.fluxcd.io/v1 - kind: Kustomization - metadata: - name: deploy-spectrum - namespace: flux-system - spec: - interval: 1m0s - path: "./flux/clusters/ephemeral" - prune: true - sourceRef: - kind: GitRepository - name: spectrum - namespace: flux-system - validation: client - timeout: 2m - postBuild: - substituteFrom: - - kind: ConfigMap - name: terraform-config