MDD Testing PRs 0

.github/workflows/terraform.yml

@@ -23,4 +23,4 @@ jobs:
 
     # Checks that all Terraform configuration files adhere to a canonical format
     - name: Terraform Format
-      run: terraform fmt --recursive -check
+      run: terraform fmt -recursive -check

.gitignore

@@ -1,5 +1,6 @@
 # Local .terraform directories
-**/.terraform/*
+**/.terraform
+**.lock.hcl
 
 # .tfstate files
 *.tfstate
@@ -8,6 +9,9 @@
 # Crash log files
 crash.log
 
+# Ignore test examples directory
+examples/test
+
 # Ignore any .tfvars files that are generated automatically for each Terraform run. Most
 # .tfvars files are managed as part of configuration and so should be included in
 # version control.
@@ -35,3 +39,5 @@ override.tf.json
 *.pem
 rke2.yaml
 admin.conf
+
+**.DS_Store

README.md

@@ -145,7 +145,7 @@ Optional policies have the option of being created by default, but are specified
 |------|---------|
 | aws | n/a |
 | random | n/a |
-| template | n/a |
+| cloudinit | n/a |
 
 ## Inputs
 
@@ -177,6 +177,7 @@ Optional policies have the option of being created by default, but are specified
 | unique\_suffix | Enables/disables generation of a unique suffix to cluster name | `bool` | `true` | yes |
 | vpc\_id | VPC ID to create resources in | `string` | n/a | yes |
 | wait_for_capacity_timeout | How long Terraform should wait for ASG instances to be healthy before timing out. | `string` | `"10m"` | no |
+| metadata_options | Instance Metadata Options | `map` | <pre>{<br>  http_endpoint: "enabled",<br>  http_tokens: "required",<br>  http_put_response_hop_limit: 1,<br>  instance_metadata_tags: "disabled"}</pre> | no |
 
 ## Outputs
 

data.tf

@@ -11,7 +11,7 @@ module "init" {
   agent         = false
 }
 
-data "template_cloudinit_config" "this" {
+data "cloudinit_config" "this" {
   gzip          = true
   base64_encode = true
 
@@ -20,7 +20,8 @@ data "template_cloudinit_config" "this" {
     filename     = "cloud-config.yaml"
     content_type = "text/cloud-config"
     content = templatefile("${path.module}/modules/nodepool/files/cloud-config.yaml", {
-      ssh_authorized_keys = var.ssh_authorized_keys
+      ssh_authorized_keys       = var.ssh_authorized_keys
+      extra_cloud_config_config = var.extra_cloud_config_config
     })
   }
 

examples/quickstart/main.tf

@@ -4,12 +4,13 @@ provider "aws" {
 
 locals {
   cluster_name = "quickstart"
-  aws_region   = "us-gov-west-1"
+  aws_region   = "us-gov-east-1"
 
   tags = {
     "terraform" = "true",
     "env"       = "quickstart",
   }
+  server_iam_role = "K8sUnrestrictedCloudProviderRole"
 }
 
 # Query for defaults
@@ -53,32 +54,31 @@ data "aws_ami" "rhel8" {
 # Server
 #
 module "rke2" {
-  source = "../.."
-
+  source                = "../.."
   cluster_name          = local.cluster_name
   vpc_id                = data.aws_vpc.default.id
   subnets               = [data.aws_subnet.default.id]
   ami                   = data.aws_ami.rhel8.image_id
   ssh_authorized_keys   = [tls_private_key.ssh.public_key_openssh]
+  iam_instance_profile  = local.server_iam_role
   controlplane_internal = false # Note this defaults to best practice of true, but is explicitly set to public for demo purposes
+  tags                  = local.tags
 
-  tags = local.tags
 }
 
 #
 # Generic Agent Pool
 #
 module "agents" {
-  source = "../../modules/agent-nodepool"
-
+  source              = "../../modules/agent-nodepool"
   name                = "generic"
   vpc_id              = data.aws_vpc.default.id
   subnets             = [data.aws_subnet.default.id]
   ami                 = data.aws_ami.rhel8.image_id
   ssh_authorized_keys = [tls_private_key.ssh.public_key_openssh]
   tags                = local.tags
+  cluster_data        = module.rke2.cluster_data
 
-  cluster_data = module.rke2.cluster_data
 }
 
 # For demonstration only, lock down ssh access in production

main.tf

@@ -16,6 +16,7 @@ locals {
     cluster_sg = aws_security_group.cluster.id
     token      = module.statestore.token
   }
+  target_group_arns = module.cp_lb.target_group_arns
 }
 
 resource "random_string" "uid" {
@@ -24,7 +25,7 @@ resource "random_string" "uid" {
   special = false
   lower   = true
   upper   = false
-  number  = true
+  numeric = true
 }
 
 #
@@ -46,7 +47,7 @@ module "statestore" {
 # Controlplane Load Balancer
 #
 module "cp_lb" {
-  source  = "./modules/elb"
+  source  = "./modules/nlb"
   name    = local.uname
   vpc_id  = var.vpc_id
   subnets = var.subnets
@@ -185,13 +186,16 @@ module "servers" {
   instance_type               = var.instance_type
   block_device_mappings       = var.block_device_mappings
   extra_block_device_mappings = var.extra_block_device_mappings
-  vpc_security_group_ids      = concat([aws_security_group.server.id, aws_security_group.cluster.id], var.extra_security_group_ids)
+  vpc_security_group_ids      = concat([aws_security_group.server.id, aws_security_group.cluster.id, module.cp_lb.security_group], var.extra_security_group_ids)
   spot                        = var.spot
-  load_balancers              = [module.cp_lb.name]
+  #load_balancers              = [module.cp_lb.name]
+  target_group_arns           = local.target_group_arns
   wait_for_capacity_timeout   = var.wait_for_capacity_timeout
+  metadata_options            = var.metadata_options
+  associate_public_ip_address = var.associate_public_ip_address
 
   # Overrideable variables
-  userdata             = data.template_cloudinit_config.this.rendered
+  userdata             = data.cloudinit_config.this.rendered
   iam_instance_profile = var.iam_instance_profile == "" ? module.iam[0].iam_instance_profile : var.iam_instance_profile
 
   # Don't allow something not recommended within etcd scaling, set max deliberately and only control desired

modules/agent-nodepool/README.md

@@ -28,7 +28,7 @@
 | tags | Map of additional tags to add to all resources created | `map(string)` | `{}` | no |
 | vpc\_id | VPC ID to create resources in | `string` | n/a | yes |
 | wait_for_capacity_timeout | How long Terraform should wait for ASG instances to be healthy before timing out. | `string` | `"10m"` | no |
-## Outputs
+| metadata_options | Instance Metadata Options | `map` | <pre>{<br>  http_endpoint: "enabled",<br>  http_tokens: "required",<br>  http_put_response_hop_limit: 1,<br>  instance_metadata_tags: "disabled"}</pre> | no |
 
 | Name | Description |
 |------|-------------|

modules/agent-nodepool/data.tf

@@ -37,7 +37,8 @@ data "aws_iam_policy_document" "aws_autoscaler" {
       "autoscaling:DescribeTags",
       "autoscaling:SetDesiredCapacity",
       "autoscaling:TerminateInstanceInAutoScalingGroup",
-      "ec2:DescribeLaunchTemplateVersions"
+      "ec2:DescribeLaunchTemplateVersions",
+      "ec2:DescribeInstanceTypes"
     ]
   }
 }

modules/agent-nodepool/files/cloud-config.yaml

@@ -0,0 +1,13 @@
+#cloud-config
+ssh_authorized_keys:
+%{ for _ in ssh_authorized_keys }
+  - ${_}
+%{ endfor }
+
+users:
+  - default
+  - name: rke2
+    homedir: /var/lib/rancher/rke2
+    system: true
+
+${extra_cloud_config_config}

modules/agent-nodepool/main.tf

@@ -69,16 +69,17 @@ module "init" {
   agent         = true
 }
 
-data "template_cloudinit_config" "init" {
+data "cloudinit_config" "init" {
   gzip          = true
   base64_encode = true
 
   # Main cloud-init config file
   part {
     filename     = "cloud-config.yaml"
     content_type = "text/cloud-config"
-    content = templatefile("${path.module}/../nodepool/files/cloud-config.yaml", {
-      ssh_authorized_keys = var.ssh_authorized_keys
+    content = templatefile("${path.module}/files/cloud-config.yaml", {
+      ssh_authorized_keys       = var.ssh_authorized_keys,
+      extra_cloud_config_config = var.extra_cloud_config_config
     })
   }
 
@@ -116,11 +117,12 @@ module "nodepool" {
   block_device_mappings       = var.block_device_mappings
   extra_block_device_mappings = var.extra_block_device_mappings
   vpc_security_group_ids      = concat([var.cluster_data.cluster_sg], var.extra_security_group_ids)
-  userdata                    = data.template_cloudinit_config.init.rendered
+  userdata                    = data.cloudinit_config.init.rendered
   iam_instance_profile        = var.iam_instance_profile == "" ? module.iam[0].iam_instance_profile : var.iam_instance_profile
   asg                         = var.asg
   spot                        = var.spot
   wait_for_capacity_timeout   = var.wait_for_capacity_timeout
+  metadata_options            = var.metadata_options
 
   tags = merge({
     "Role" = "agent",

modules/agent-nodepool/variables.tf

@@ -61,6 +61,12 @@ variable "block_device_mappings" {
   }
 }
 
+variable "extra_cloud_config_config" {
+  description = "extra config to append to cloud-config"
+  type        = string
+  default     = ""
+}
+
 variable "extra_block_device_mappings" {
   description = "Used to specify additional block device mapping configurations"
   type        = list(map(string))
@@ -95,6 +101,17 @@ variable "extra_security_group_ids" {
   default     = []
 }
 
+variable "metadata_options" {
+  type = map(any)
+  default = {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required" # IMDS-v2
+    http_put_response_hop_limit = 2          # allow pods to use IMDS as well
+    instance_metadata_tags      = "disabled"
+  }
+  description = "Instance Metadata Options"
+}
+
 #
 # RKE2 Variables
 #

modules/common/download.sh

@@ -47,7 +47,7 @@ do_download() {
   get_installer
 
   case $ID in
-  centos)
+  centos | rocky)
     yum install -y unzip
     install_awscli
 
@@ -74,6 +74,7 @@ do_download() {
     7*)
       info "Identified RHEL 7"
 
+      rpm --import http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-7
       yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm
       INSTALL_RKE2_METHOD='yum' INSTALL_RKE2_TYPE="${type}" ./install.sh
       ;;

modules/nlb/main.tf

@@ -0,0 +1,98 @@
+locals {
+  # Handle case where target group/load balancer name exceeds 32 character limit without creating illegal names
+  controlplane_name = "${substr(var.name, 0, 23)}-rke2-cp"
+  server_name       = "${substr(var.name, 0, 18)}-rke2-server"
+  supervisor_name   = "${substr(var.name, 0, 15)}-rke2-supervisor"
+}
+
+resource "aws_security_group" "controlplane" {
+  name        = local.controlplane_name
+  description = "${local.controlplane_name} sg"
+  vpc_id      = var.vpc_id
+
+  tags = merge({}, var.tags)
+}
+
+resource "aws_security_group_rule" "apiserver" {
+  from_port         = var.cp_port
+  to_port           = var.cp_port
+  protocol          = "tcp"
+  security_group_id = aws_security_group.controlplane.id
+  type              = "ingress"
+
+  cidr_blocks = var.cp_ingress_cidr_blocks
+}
+
+resource "aws_security_group_rule" "supervisor" {
+  from_port         = var.cp_supervisor_port
+  to_port           = var.cp_supervisor_port
+  protocol          = "tcp"
+  security_group_id = aws_security_group.controlplane.id
+  type              = "ingress"
+
+  cidr_blocks = var.cp_supervisor_ingress_cidr_blocks
+}
+
+resource "aws_security_group_rule" "egress" {
+  from_port         = "0"
+  to_port           = "0"
+  protocol          = "-1"
+  security_group_id = aws_security_group.controlplane.id
+  type              = "egress"
+
+  cidr_blocks = ["0.0.0.0/0"]
+}
+
+resource "aws_lb_listener" "apiserver" {
+  load_balancer_arn = aws_lb.controlplane.arn
+  port              = var.cp_port
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.apiserver.arn
+  }
+}
+
+resource "aws_lb_target_group" "apiserver" {
+  name     = "${local.controlplane_name}-${var.cp_port}"
+  port     = var.cp_port
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
+}
+
+resource "aws_lb_listener" "supervisor" {
+  load_balancer_arn = aws_lb.controlplane.arn
+  port              = var.cp_supervisor_port
+  protocol          = "TCP"
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.supervisor.arn
+  }
+}
+
+resource "aws_lb_target_group" "supervisor" {
+  name     = "${local.controlplane_name}-${var.cp_supervisor_port}"
+  port     = var.cp_supervisor_port
+  protocol = "TCP"
+  vpc_id   = var.vpc_id
+}
+
+resource "aws_lb" "controlplane" {
+  name = local.controlplane_name
+
+  internal           = var.internal
+  load_balancer_type = "network"
+  subnets            = var.subnets
+
+  enable_cross_zone_load_balancing = var.enable_cross_zone_load_balancing
+
+  access_logs {
+    # the bucket name isn't allowed to be empty in this block, so use its default value as the flag
+    bucket  = var.access_logs_bucket
+    enabled = var.access_logs_bucket != "disabled"
+  }
+
+  tags = merge({}, var.tags)
+}