vsphere upi: terraform update, general updates and reorg

- Update to terraform 0.12.x
- ipam moved to a module and updated for 0.12
- creation of vsphere objects moved to module
- removal of etcd dns records
- domain dns records separated from creating rhcos node a records
- create load balancer using openshift haproxy image and an additional
rhcos virtual machine
- create dns A record for lb ip and api, api-int, *.apps
- change virtual machine ignition extra config to file path string vs
ignition string.  Simplifies bootstrap and other instance configuration
- Updated Dockerfile CI UPI image for 0.12.24

images/installer/Dockerfile.upi.ci

@@ -30,7 +30,7 @@ RUN yum install --setopt=tsflags=nodocs -y \
     yum clean all && rm -rf /var/cache/yum/* && \
     chmod g+w /etc/passwd
 
-ENV TERRAFORM_VERSION=0.11.11
+ENV TERRAFORM_VERSION=0.12.24
 RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
     unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /bin/
 ENV MATCHBOX_PROVIDER_VERSION=v0.3.0

upi/vsphere/cluster_domain/main.tf

@@ -0,0 +1,22 @@
+data "aws_route53_zone" "base" {
+  name = var.base_domain
+}
+
+resource "aws_route53_zone" "cluster" {
+  name          = var.cluster_domain
+  force_destroy = true
+
+  tags = {
+    "Name"     = var.cluster_domain
+    "Platform" = "vSphere"
+  }
+}
+
+resource "aws_route53_record" "name_server" {
+  name    = var.cluster_domain
+  type    = "NS"
+  ttl     = "300"
+  zone_id = data.aws_route53_zone.base.zone_id
+  records = aws_route53_zone.cluster.name_servers
+}
+

upi/vsphere/cluster_domain/outputs.tf

@@ -0,0 +1,3 @@
+output "zone_id" {
+  value = aws_route53_zone.cluster.zone_id
+}

upi/vsphere/cluster_domain/variables.tf

@@ -0,0 +1,9 @@
+variable "cluster_domain" {
+  description = "The domain for the cluster that all DNS records must belong"
+  type        = string
+}
+
+variable "base_domain" {
+  description = "The base domain used for public records."
+  type        = string
+}

upi/vsphere/host_a_record/main.tf

@@ -0,0 +1,9 @@
+resource "aws_route53_record" "a_record" {
+  for_each = var.records
+
+  type    = "A"
+  ttl     = "60"
+  zone_id = var.zone_id
+  name    = each.key
+  records = [each.value]
+}

upi/vsphere/host_a_record/outputs.tf

@@ -0,0 +1,3 @@
+output "fqdns" {
+  value = values(aws_route53_record.a_record)[*].name
+}

upi/vsphere/host_a_record/variables.tf

@@ -0,0 +1,9 @@
+variable "zone_id" {
+  type        = string
+  description = "The ID of the hosted zone to contain this record."
+}
+
+variable "records" {
+  type        = map(string)
+  description = "A records to be added to the zone_id"
+}

upi/vsphere/machine/cidr_to_ip.sh → upi/vsphere/ipam/cidr_to_ip.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# cidr_to_ip - 
+# cidr_to_ip -
 #  https://www.terraform.io/docs/providers/external/data_source.html
 #  Based on info from here: https://gist.github.com/irvingpop/968464132ded25a206ced835d50afa6b
 #  This script takes requests an IP address from an IPAM server
@@ -58,11 +58,11 @@ function produce_output() {
   # The verification and looping is a crude way of overcoming the lack of
   # currency safety in the IPAM server.
   while [[ $SECONDS -lt $timeout ]]
-  do 
+  do
     ip_address=$(curl -s "http://$ipam/api/getFreeIP.php?apiapp=address&apitoken=$ipam_token&subnet=${network}&host=${hostname}")
 
     if [[ "$(is_ip_address "${ip_address}")" != "true" ]]; then error_exit "could not reserve an IP address: ${ip_address}"; fi
-	
+
     if [[ "$ip_address" == "$(get_reservation)" ]]
 	then
       jq -n \

upi/vsphere/ipam/main.tf

@@ -0,0 +1,41 @@
+locals {
+  network      = cidrhost(var.machine_cidr, 0)
+  hostnames    = length(var.static_ip_addresses) == 0 ? var.hostnames : []
+  ip_addresses = length(var.static_ip_addresses) == 0 ? [for result in null_resource.ip_address : jsondecode(data.http.getip[result.triggers.hostname].body)[result.triggers.hostname]] : var.static_ip_addresses
+}
+
+data "http" "getip" {
+  for_each = null_resource.ip_address
+
+  url = "http://${var.ipam}/api/getIPs.php?apiapp=address&apitoken=${var.ipam_token}&domain=${null_resource.ip_address[each.key].triggers.hostname}"
+
+  request_headers = {
+    Accept = "application/json"
+  }
+}
+
+resource "null_resource" "ip_address" {
+  for_each = local.hostnames
+
+  triggers = {
+    ipam       = var.ipam
+    ipam_token = var.ipam_token
+    network    = local.network
+    hostname   = each.key
+  }
+
+  provisioner "local-exec" {
+    command = <<EOF
+echo '{"network":"${self.triggers.network}","hostname":"${self.triggers.hostname}","ipam":"${self.triggers.ipam}","ipam_token":"${self.triggers.ipam_token}"}' | ${path.module}/cidr_to_ip.sh
+EOF
+
+  }
+  provisioner "local-exec" {
+    when = destroy
+
+    command = <<EOF
+curl -s "http://${self.triggers.ipam}/api/removeHost.php?apiapp=address&apitoken=${self.triggers.ipam_token}&host=${self.triggers.hostname}"
+EOF
+
+  }
+}

upi/vsphere/ipam/outputs.tf

@@ -0,0 +1,3 @@
+output "ip_addresses" {
+  value = local.ip_addresses
+}

upi/vsphere/ipam/variables.tf

@@ -0,0 +1,20 @@
+variable "hostnames" {
+  type = set(string)
+}
+
+variable "machine_cidr" {
+  type = string
+}
+
+variable "ipam" {
+  type = string
+}
+
+variable "ipam_token" {
+  type = string
+}
+
+variable "static_ip_addresses" {
+  type    = list(string)
+  default = []
+}

upi/vsphere/ipam/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}

upi/vsphere/lb/haproxy.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=haproxy
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+TimeoutStartSec=0
+ExecStartPre=-/bin/podman kill haproxy
+ExecStartPre=-/bin/podman rm haproxy
+ExecStartPre=/bin/podman pull quay.io/openshift/origin-haproxy-router
+ExecStart=/bin/podman run --name haproxy \
+  --net=host \
+  --privileged \
+  --entrypoint=/usr/sbin/haproxy \
+  -v /etc/haproxy/haproxy.conf:/var/lib/haproxy/conf/haproxy.conf:Z \
+  quay.io/openshift/origin-haproxy-router -f /var/lib/haproxy/conf/haproxy.conf
+
+[Install]
+WantedBy=multi-user.target
+

upi/vsphere/lb/haproxy.tmpl

@@ -0,0 +1,55 @@
+defaults
+  maxconn 20000
+  mode    tcp
+  log     /var/run/haproxy/haproxy-log.sock local0
+  option  dontlognull
+  retries 3
+  timeout http-request 10s
+  timeout queue        1m
+  timeout connect      10s
+  timeout client       86400s
+  timeout server       86400s
+  timeout tunnel       86400s
+
+frontend api-server
+    bind ${lb_ip_address}:6443
+    default_backend api-server
+
+frontend machine-config-server
+    bind ${lb_ip_address}:22623
+    default_backend machine-config-server
+
+frontend router-http
+    bind ${lb_ip_address}:80
+    default_backend router-http
+
+frontend router-https
+    bind ${lb_ip_address}:443
+    default_backend router-https
+
+backend api-server
+    balance roundrobin
+    %{ for addr in api ~}
+    server ${addr} ${addr}:6443 check
+    %{ endfor ~}
+
+backend machine-config-server
+    balance roundrobin
+    %{ for addr in api ~}
+    server ${addr} ${addr}:22623 check
+    %{ endfor ~}
+
+backend router-http
+    balance source
+    mode tcp
+    %{ for addr in ingress ~}
+    server ${addr} ${addr}:80 check
+    %{ endfor ~}
+
+backend router-https
+    balance source
+    mode tcp
+    %{ for addr in ingress ~}
+    server ${addr} ${addr}:443 check
+    %{ endfor ~}
+

upi/vsphere/lb/main.tf

@@ -0,0 +1,29 @@
+data "ignition_systemd_unit" "haproxy" {
+  name    = "haproxy.service"
+  content = file("${path.module}/haproxy.service")
+}
+
+data "ignition_file" "haproxy" {
+  filesystem = "root"
+  path       = "/etc/haproxy/haproxy.conf"
+  mode       = 0755
+  content {
+    content = templatefile("${path.module}/haproxy.tmpl", {
+      lb_ip_address = var.lb_ip_address,
+      api           = var.api_backend_addresses,
+      ingress       = var.ingress_backend_addresses
+    })
+  }
+}
+
+data "ignition_user" "core" {
+  name                = "core"
+  ssh_authorized_keys = [file("${var.ssh_public_key_path}")]
+}
+
+data "ignition_config" "lb" {
+  users   = [data.ignition_user.core.rendered]
+  files   = [data.ignition_file.haproxy.rendered]
+  systemd = [data.ignition_systemd_unit.haproxy.rendered]
+}
+

upi/vsphere/lb/outputs.tf

@@ -0,0 +1,4 @@
+output "ignition" {
+  value = data.ignition_config.lb.rendered
+}
+

upi/vsphere/lb/variables.tf

@@ -0,0 +1,15 @@
+variable "lb_ip_address" {
+  type = string
+}
+
+variable "api_backend_addresses" {
+  type = list(string)
+}
+
+variable "ingress_backend_addresses" {
+  type = list(string)
+}
+
+variable "ssh_public_key_path" {
+  type = string
+}