fix(iac): Fixes for remote terraform execution and k3s

deployment/terraform/modules/openstack-cogstack-infra/compute-keypair.tf

@@ -21,13 +21,13 @@ resource "openstack_compute_keypair_v2" "compute_keypair" {
 resource "local_file" "private_key" {
   count           = local.is_using_existing_ssh_keypair ? 0 : 1
   content         = openstack_compute_keypair_v2.compute_keypair.private_key
-  filename        = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pem"
+  filename        = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pem"
   file_permission = "0600"
 }
 
 resource "local_file" "public_key" {
   count           = local.is_using_existing_ssh_keypair ? 0 : 1
   content         = openstack_compute_keypair_v2.compute_keypair.public_key
-  filename        = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub"
+  filename        = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub"
   file_permission = "0600"
-}
+}

deployment/terraform/modules/openstack-cogstack-infra/shared-locals.tf

@@ -1,6 +1,7 @@
 
 locals {
   random_prefix = random_id.server.b64_url
+  output_file_directory = var.output_file_directory != null ? var.output_file_directory : "${path.root}/.build"
 }
 
 
@@ -17,8 +18,6 @@ locals {
     ip_address  = var.preexisting_controller_host != null ? var.preexisting_controller_host.ip_address : local.created_controller_host.access_ip_v4
     unique_name = var.preexisting_controller_host != null && var.preexisting_controller_host.unique_name != null ? var.preexisting_controller_host.unique_name : local.created_controller_host.name
   }
-
-
 }
 
 resource "random_id" "server" {

deployment/terraform/modules/openstack-cogstack-infra/variables.tf

@@ -99,4 +99,11 @@ variable "ssh_key_pair" {
     condition     = var.ssh_key_pair == null || fileexists(var.ssh_key_pair.public_key_file)
     error_message = "No file exists in SSH public key path"
   }
+}
+
+
+variable "output_file_directory" {
+  type = string
+  default = null
+  description = "Optional path to write output files to. If directory doesnt exist it will be created"
 }

deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf

@@ -21,13 +21,13 @@ resource "openstack_compute_keypair_v2" "compute_keypair" {
 resource "local_file" "private_key" {
   count           = local.is_using_existing_ssh_keypair ? 0 : 1
   content         = openstack_compute_keypair_v2.compute_keypair.private_key
-  filename        = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pem"
+  filename        = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pem"
   file_permission = "0600"
 }
 
 resource "local_file" "public_key" {
   count           = local.is_using_existing_ssh_keypair ? 0 : 1
   content         = openstack_compute_keypair_v2.compute_keypair.public_key
-  filename        = "${path.root}/.build/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub"
+  filename        = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub"
   file_permission = "0600"
 }

deployment/terraform/modules/openstack-kubernetes-infra/compute.tf

@@ -120,7 +120,7 @@ data "cloudinit_config" "init_docker_controller" {
 }
 
 data "openstack_compute_flavor_v2" "available_compute_flavors" {
-  for_each = toset(["2cpu4ram", "8cpu16ram"])
+  for_each = toset([for vm in var.host_instances : vm.flavour])
   name     = each.value
 }
 
@@ -138,21 +138,3 @@ data "openstack_networking_secgroup_v2" "er_https_from_lbs" {
   name = "er_https_from_lbs"
 }
 
-resource "null_resource" "copy_kubeconfig" {
-  depends_on = [openstack_compute_instance_v2.kubernetes_server]
-
-  provisioner "local-exec" {
-    # Copy the kubeconfig file from the host to a local file using SCP.
-    # Use ssh-keyscan to prevent interactive prompt on unknown host
-    # Use sed to replace the localhost address in the KUBECONFIG file with the actual IP adddress of the created VM. 
-    command = <<EOT
-mkdir -p ${path.module}/.build/ && \
-ssh-keyscan -H ${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} >> ${path.module}/.build/.known_hosts_cogstack && \
-scp -o UserKnownHostsFile=${path.module}/.build/.known_hosts_cogstack -o StrictHostKeyChecking=yes \
-    -i ${local.ssh_keys.private_key_file} \
-    ubuntu@${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}:/etc/rancher/k3s/k3s.yaml \
-    ${local.kubeconfig_file} && \
-sed -i "s/127.0.0.1/${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}/" ${local.kubeconfig_file}
-EOT
-  }
-}

deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf

@@ -0,0 +1,39 @@
+resource "null_resource" "copy_kubeconfig" {
+  depends_on = [openstack_compute_instance_v2.kubernetes_server]
+
+  provisioner "local-exec" {
+    # Copy the kubeconfig file from the host to a local file using SCP.
+    # Use ssh-keyscan to prevent interactive prompt on unknown host
+    # Use sed to replace the localhost address in the KUBECONFIG file with the actual IP adddress of the created VM. 
+    command = <<EOT
+mkdir -p ${path.root}/.build/ && \
+ssh-keyscan -H ${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} >> ${path.root}/.build/.known_hosts_cogstack && \
+ssh -o UserKnownHostsFile=${path.root}/.build/.known_hosts_cogstack -o StrictHostKeyChecking=yes \
+    -i ${local.ssh_keys.private_key_file} \
+    ubuntu@${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} \
+    "sudo cat /etc/rancher/k3s/k3s.yaml" > ${local.kubeconfig_file} && \
+sed -i "s/127.0.0.1/${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}/" ${local.kubeconfig_file}
+EOT
+  }
+}
+
+data "local_file" "kube_config_raw_file" {
+  filename = local.kubeconfig_file
+  depends_on = [ null_resource.copy_kubeconfig ]
+}
+
+# output "kube_config_raw" {
+#   value = data.local_file.kube_config_raw_file
+#   description = "Kubeconfig for this cluster"
+# }
+
+# data "external" "extract_kubeconfig_file" {
+#   program = ["bash", "${path.module}/kubeconfig_extraction.sh"]
+
+#   query = {
+#         PATH_ROOT=path.root
+#         SERVER_IP=openstack_compute_instance_v2.kubernetes_server.access_ip_v4
+#         SSH_KEY=local.ssh_keys.private_key_file
+#         # KUBECONFIG_FILE=local.kubeconfig_file
+#   }
+# }

deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig_extraction.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# ==============================
+# Variables (from Terraform)
+# ==============================
+PATH_ROOT=${path.root}
+SERVER_IP=${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}
+SSH_KEY=${local.ssh_keys.private_key_file}
+KUBECONFIG_FILE=${local.kubeconfig_file}
+
+# ==============================
+# Script Logic
+# ==============================
+
+# Create .build directory if it doesn't exist
+mkdir -p "${PATH_ROOT}/.build/"
+
+# Add server's SSH key to a custom known_hosts file
+ssh-keyscan -H "${SERVER_IP}" >> "${PATH_ROOT}/.build/.known_hosts_cogstack"
+
+# Securely copy the K3s kubeconfig file from the server
+scp \
+  -o UserKnownHostsFile="${PATH_ROOT}/.build/.known_hosts_cogstack" \
+  -o StrictHostKeyChecking=yes \
+  -i "${SSH_KEY}" \
+  "ubuntu@${SERVER_IP}:/etc/rancher/k3s/k3s.yaml" \
+  "${KUBECONFIG_FILE}"
+
+# Replace localhost with the actual server IP in the kubeconfig
+sed -i "s/127\.0\.0\.1/${SERVER_IP}/" "${KUBECONFIG_FILE}"
+
+echo "Kubeconfig successfully fetched and updated at: ${KUBECONFIG_FILE}"

deployment/terraform/modules/openstack-kubernetes-infra/networking.tf

@@ -1,7 +1,7 @@
 
 
 locals {
-  devops_controller_cidr = "${local.controller_host_instance.access_ip_v4}/32"
+  devops_controller_cidr = "${local.controller_host_instance.ip_address}/32"
 
   cogstack_apps_ingress_rules = [
     { port = 22, cidr = var.allowed_ingress_ips_cidr, description = "Expose SSH" },

deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf

@@ -1,21 +1,19 @@
 
-output "created_hosts_2" {
-  value = { for k, value in openstack_compute_instance_v2.kubernetes_nodes : k => {
+output "created_hosts" {
+  value = merge({ for k, value in openstack_compute_instance_v2.kubernetes_nodes : k => {
     ip_address  = value.access_ip_v4
     unique_name = value.name
     name        = k
-  } }
+    } },
+    {
+  (local.controller_host.name) : local.controller_host_instance
+  })
 
   description = "Created Hosts: A map of { hostname: { data } }"
 }
 
 output "created_controller_host" {
-  value = {
-    name        = (local.controller_host.name)
-    ip_address  = local.controller_host_instance.access_ip_v4
-    unique_name = local.controller_host_instance.name
-  }
-
+  value       = local.controller_host_instance
   description = "Created Controller Host: A map of { hostname: { data } }"
 }
 

deployment/terraform/modules/openstack-kubernetes-infra/shared-locals.tf

@@ -6,11 +6,17 @@ locals {
 
 locals {
   controller_host          = one([for host in var.host_instances : host if host.is_controller])
-  controller_host_instance = openstack_compute_instance_v2.kubernetes_server
+  created_controller_host = openstack_compute_instance_v2.kubernetes_server
+  controller_host_instance = {
+    name        = local.controller_host.name
+    ip_address  = local.created_controller_host.access_ip_v4
+    unique_name = local.created_controller_host.name
+  }
 }
 
 locals {
-  kubeconfig_file = "${path.module}/.build/downloaded-kubeconfig.yaml"
+  output_file_directory = var.output_file_directory != null ? var.output_file_directory : "${path.root}/.build"
+  kubeconfig_file = "${local.output_file_directory}/downloaded-kubeconfig.yaml"
 }
 
 resource "random_id" "server" {

deployment/terraform/modules/openstack-kubernetes-infra/variables.tf

@@ -54,4 +54,10 @@ variable "ssh_key_pair" {
     condition     = var.ssh_key_pair == null || fileexists(var.ssh_key_pair.public_key_file)
     error_message = "No file exists in SSH public key path"
   }
+}
+
+variable "output_file_directory" {
+  type = string
+  default = null
+  description = "Optional path to write output files to. If directory doesnt exist it will be created"
 }