azure: update assets directory

Updates to new version and according file changes for the respective version. Part of #314 Signed-off-by: Kautilya Tripathi <kautilya@kinvolk.io>
kinvolk · Jan 17, 2022 · fcbb436 · fcbb436
1 parent f50166a
commit fcbb436
Show file tree

Hide file tree

Showing 15 changed files with 337 additions and 101 deletions.
diff --git a/assets/terraform-modules/azure/flatcar-linux/kubernetes/bootkube.tf b/assets/terraform-modules/azure/flatcar-linux/kubernetes/bootkube.tf
@@ -7,6 +7,8 @@ module "bootkube" {
   etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone)
   asset_dir    = var.asset_dir
 
+  networking = var.networking
+
   network_encapsulation = "vxlan"
 
   # we should be able to use 1450 MTU, but in practice, 1410 was needed

diff --git a/assets/terraform-modules/azure/flatcar-linux/kubernetes/cl/controller.yaml.tmpl b/assets/terraform-modules/azure/flatcar-linux/kubernetes/cl/controller.yaml.tmpl
@@ -2,31 +2,32 @@
 systemd:
   units:
     - name: etcd-member.service
-      enable: true
-      dropins:
-        - name: 40-etcd-cluster.conf
-          contents: |
-            [Service]
-            Environment="IMAGE_TAG=v3.4.16"
-            Environment="IMAGE_URL=docker://quay.io/coreos/etcd"
-            Environment="RKT_RUN_ARGS=--insecure-options=image"
-            Environment="SSL_DIR=/etc/ssl/etcd"
-            Environment="ETCD_NAME=${etcd_name}"
-            Environment="ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379"
-            Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380"
-            Environment="ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379"
-            Environment="ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380"
-            Environment="ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381"
-            Environment="ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}"
-            Environment="ETCD_STRICT_RECONFIG_CHECK=true"
-            Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt"
-            Environment="ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt"
-            Environment="ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key"
-            Environment="ETCD_CLIENT_CERT_AUTH=true"
-            Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt"
-            Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt"
-            Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key"
-            Environment="ETCD_PEER_CLIENT_CERT_AUTH=true"
+      enabled: true
+      contents: |
+        [Unit]
+        Description=etcd (System Container)
+        Documentation=https://github.com/etcd-io/etcd
+        Requires=docker.service
+        After=docker.service
+        [Service]
+        Environment=ETCD_IMAGE=quay.io/coreos/etcd:v3.5.1
+        ExecStartPre=/usr/bin/docker run -d \
+          --name etcd \
+          --network host \
+          --env-file /etc/etcd/etcd.env \
+          --user 232:232 \
+          --volume /etc/ssl/etcd:/etc/ssl/certs:ro \
+          --volume /var/lib/etcd:/var/lib/etcd:rw \
+          $${ETCD_IMAGE}
+        ExecStart=docker logs -f etcd
+        ExecStop=docker stop etcd
+        ExecStopPost=docker rm etcd
+        Restart=always
+        RestartSec=10s
+        TimeoutStartSec=0
+        LimitNOFILE=40000
+        [Install]
+        WantedBy=multi-user.target
     - name: docker.service
       enable: true
     - name: locksmithd.service
@@ -90,7 +91,6 @@ systemd:
           --exit-on-lock-contention \
           --kubeconfig=/etc/kubernetes/kubeconfig \
           --lock-file=/var/run/lock/kubelet.lock \
-          --network-plugin=cni \
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
@@ -137,6 +137,28 @@ storage:
       contents:
         inline: |
           fs.inotify.max_user_watches=16184
+    - path: /etc/etcd/etcd.env
+      filesystem: root
+      mode: 0644
+      contents:
+          inline: |
+            ETCD_NAME=${etcd_name}
+            ETCD_DATA_DIR=/var/lib/etcd
+            ETCD_ADVERTISE_CLIENT_URLS=https://${etcd_domain}:2379
+            ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${etcd_domain}:2380
+            ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:2379
+            ETCD_LISTEN_PEER_URLS=https://0.0.0.0:2380
+            ETCD_LISTEN_METRICS_URLS=http://0.0.0.0:2381
+            ETCD_INITIAL_CLUSTER=${etcd_initial_cluster}
+            ETCD_STRICT_RECONFIG_CHECK=true
+            ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/server-ca.crt
+            ETCD_CERT_FILE=/etc/ssl/certs/etcd/server.crt
+            ETCD_KEY_FILE=/etc/ssl/certs/etcd/server.key
+            ETCD_CLIENT_CERT_AUTH=true
+            ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/etcd/peer-ca.crt
+            ETCD_PEER_CERT_FILE=/etc/ssl/certs/etcd/peer.crt
+            ETCD_PEER_KEY_FILE=/etc/ssl/certs/etcd/peer.key
+            ETCD_PEER_CLIENT_CERT_AUTH=tru
     - path: /opt/bootkube/bootkube-start
       filesystem: root
       mode: 0544

diff --git a/assets/terraform-modules/azure/flatcar-linux/kubernetes/controllers.tf b/assets/terraform-modules/azure/flatcar-linux/kubernetes/controllers.tf
@@ -11,12 +11,12 @@ resource "azurerm_dns_a_record" "etcds" {
   ttl  = 300
 
   # private IPv4 address for etcd
-  records = [azurerm_network_interface.controllers[count.index].private_ip_address]
+  records = [azurerm_network_interface.controllers.*.private_ip_address[count.index]]
 }
 
 locals {
-  # Channel for a CoreOS Container Linux derivative
-  # coreos-stable -> CoreOS Container Linux Stable
+  # Container Linux derivative
+  # flatcar-stable -> Flatcar Linux Stable
   channel = split("-", var.os_image)[1]
 }
 
@@ -37,33 +37,45 @@ data "azurerm_image" "custom" {
 }
 
 # Controller instances
-resource "azurerm_virtual_machine" "controllers" {
+resource "azurerm_linux_virtual_machine" "controllers" {
   count               = var.controller_count
   resource_group_name = azurerm_resource_group.cluster.name
 
   name                = "${var.cluster_name}-controller-${count.index}"
   location            = var.region
   availability_set_id = azurerm_availability_set.controllers.id
-  vm_size             = var.controller_type
 
-  # boot
-  storage_image_reference {
-    id = data.azurerm_image.custom.id
-  }
+  size        = var.controller_type
+  custom_data = base64encode(data.ct_config.controller-ignitions.*.rendered[count.index])
 
   # storage
-  storage_os_disk {
-    name              = "${var.cluster_name}-controller-${count.index}"
-    create_option     = "FromImage"
-    caching           = "ReadWrite"
-    disk_size_gb      = var.disk_size
-    os_type           = "Linux"
-    managed_disk_type = "Premium_LRS"
+  os_disk {
+    name                 = "${var.cluster_name}-controller-${count.index}"
+    caching              = "None"
+    disk_size_gb         = var.disk_size
+    storage_account_type = "Premium_LRS"
+  }
+
+  # Flatcar Container Linux
+  source_image_reference {
+    publisher = "Kinvolk"
+    offer     = "flatcar-container-linux-free"
+    sku       = local.channel
+    version   = "latest"
+  }
+
+  plan {
+    name      = local.channel
+    publisher = "kinvolk"
+    product   = "flatcar-container-linux-free"
   }
 
   # network
-  network_interface_ids = [azurerm_network_interface.controllers[count.index].id]
+  network_interface_ids = [
+    azurerm_network_interface.controllers.*.id[count.index]
+  ]
 
+  # Azure requires setting admin_ssh_key, though Ignition custom_data handles it too
   os_profile {
     computer_name  = "${var.cluster_name}-controller-${count.index}"
     admin_username = "core"
@@ -87,7 +99,8 @@ resource "azurerm_virtual_machine" "controllers" {
 
   lifecycle {
     ignore_changes = [
-      storage_os_disk,
+      custom_data,
+      os_disk,
       os_profile,
     ]
   }
@@ -112,7 +125,16 @@ resource "azurerm_network_interface" "controllers" {
   }
 }
 
-# Add controller NICs to the controller backend address pool
+# Associate controller network interface with controller security group
+resource "azurerm_network_interface_security_group_association" "controllers" {
+  count = var.controller_count
+
+  network_interface_id      = azurerm_network_interface.controllers[count.index].id
+  network_security_group_id = azurerm_network_security_group.controller.id
+}
+
+
+# Associate controller network interface with controller backend address pool
 resource "azurerm_network_interface_backend_address_pool_association" "controllers" {
   count = var.controller_count
 

diff --git a/assets/terraform-modules/azure/flatcar-linux/kubernetes/lb.tf b/assets/terraform-modules/azure/flatcar-linux/kubernetes/lb.tf
@@ -94,6 +94,21 @@ resource "azurerm_lb_rule" "ingress-https" {
   probe_id                = azurerm_lb_probe.ingress.id
 }
 
+# Worker outbound TCP/UDP SNAT
+resource "azurerm_lb_outbound_rule" "worker-outbound" {
+  resource_group_name = azurerm_resource_group.cluster.name
+
+  name            = "worker"
+  loadbalancer_id = azurerm_lb.cluster.id
+  frontend_ip_configuration {
+    name = "ingress"
+  }
+
+  protocol                = "All"
+  backend_address_pool_id = azurerm_lb_backend_address_pool.worker.id
+}
+
+
 # Address pool of controllers
 resource "azurerm_lb_backend_address_pool" "controller" {
   resource_group_name = azurerm_resource_group.cluster.name

diff --git a/assets/terraform-modules/azure/flatcar-linux/kubernetes/network.tf b/assets/terraform-modules/azure/flatcar-linux/kubernetes/network.tf
@@ -24,10 +24,20 @@ resource "azurerm_subnet" "controller" {
   address_prefix       = cidrsubnet(var.host_cidr, 1, 0)
 }
 
+resource "azurerm_subnet_network_security_group_association" "controller" {
+  subnet_id                 = azurerm_subnet.controller.id
+  network_security_group_id = azurerm_network_security_group.controller.id
+}
+
 resource "azurerm_subnet" "worker" {
   resource_group_name = azurerm_resource_group.cluster.name
 
   name                 = "worker"
   virtual_network_name = azurerm_virtual_network.network.name
   address_prefix       = cidrsubnet(var.host_cidr, 1, 1)
 }
+
+resource "azurerm_subnet_network_security_group_association" "worker" {
+  subnet_id                 = azurerm_subnet.worker.id
+  network_security_group_id = azurerm_network_security_group.worker.id
+}
diff --git a/assets/terraform-modules/azure/flatcar-linux/kubernetes/outputs.tf b/assets/terraform-modules/azure/flatcar-linux/kubernetes/outputs.tf
@@ -19,6 +19,10 @@ output "resource_group_name" {
   value = azurerm_resource_group.cluster.name
 }
 
+output "resource_group_id" {
+  value = azurerm_resource_group.cluster.id
+}
+
 output "subnet_id" {
   value = azurerm_subnet.worker.id
 }