azure: add OutboundType for controlling egress

data/data/azure/bootstrap/main.tf

@@ -137,16 +137,19 @@ resource "azurerm_network_interface" "bootstrap" {
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v4" {
-  // should be 'count = var.use_ipv4 && ! var.emulate_single_stack_ipv6 ? 1 : 0', but we need a V4 LB for egress for quay
-  count = var.use_ipv4 ? 1 : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = (! var.private || ! var.outbound_udr) ? 1 : 0
 
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.elb_backend_pool_v4_id
   ip_configuration_name   = local.bootstrap_nic_ip_v4_configuration_name
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v6" {
-  count = var.use_ipv6 ? 1 : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = var.use_ipv6 && (! var.private || ! var.outbound_udr) ? 1 : 0
 
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.elb_backend_pool_v6_id

data/data/azure/bootstrap/variables.tf

@@ -93,3 +93,16 @@ variable "emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "outbound_udr" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+
+This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+EOF
+}

data/data/azure/main.tf

@@ -41,6 +41,7 @@ module "bootstrap" {
   storage_account        = azurerm_storage_account.cluster
   nsg_name               = module.vnet.cluster_nsg_name
   private                = module.vnet.private
+  outbound_udr           = var.azure_outbound_user_defined_routing
 
   use_ipv4                  = var.use_ipv4 || var.azure_emulate_single_stack_ipv6
   use_ipv6                  = var.use_ipv6
@@ -62,6 +63,7 @@ module "vnet" {
   master_subnet               = var.azure_control_plane_subnet
   worker_subnet               = var.azure_compute_subnet
   private                     = var.azure_private
+  outbound_udr                = var.azure_outbound_user_defined_routing
 
   use_ipv4                  = var.use_ipv4 || var.azure_emulate_single_stack_ipv6
   use_ipv6                  = var.use_ipv6
@@ -88,6 +90,7 @@ module "master" {
   os_volume_type         = var.azure_master_root_volume_type
   os_volume_size         = var.azure_master_root_volume_size
   private                = module.vnet.private
+  outbound_udr           = var.azure_outbound_user_defined_routing
 
   use_ipv4                  = var.use_ipv4 || var.azure_emulate_single_stack_ipv6
   use_ipv6                  = var.use_ipv6

data/data/azure/master/master.tf

@@ -46,16 +46,19 @@ resource "azurerm_network_interface" "master" {
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "master_v4" {
-  // should be 'count = var.use_ipv4 && ! var.emulate_single_stack_ipv6 ? var.instance_count : 0', but we need a V4 LB for egress for quay
-  count = var.use_ipv4 ? var.instance_count : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = (! var.private || ! var.outbound_udr) ? var.instance_count : 0
 
   network_interface_id    = element(azurerm_network_interface.master.*.id, count.index)
   backend_address_pool_id = var.elb_backend_pool_v4_id
   ip_configuration_name   = local.ip_v4_configuration_name
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "master_v6" {
-  count = var.use_ipv6 ? var.instance_count : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = var.use_ipv6 && (! var.private || ! var.outbound_udr) ? var.instance_count : 0
 
   network_interface_id    = element(azurerm_network_interface.master.*.id, count.index)
   backend_address_pool_id = var.elb_backend_pool_v6_id

data/data/azure/master/variables.tf

@@ -110,3 +110,16 @@ variable "emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "outbound_udr" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+
+This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+EOF
+}

data/data/azure/variables-azure.tf

@@ -121,3 +121,13 @@ variable "azure_emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "azure_outbound_user_defined_routing" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+EOF
+}

data/data/azure/vnet/outputs.tf

@@ -1,9 +1,9 @@
 output "public_lb_backend_pool_v4_id" {
-  value = var.use_ipv4 ? azurerm_lb_backend_address_pool.public_lb_pool_v4[0].id : null
+  value = local.need_public_ipv4 ? azurerm_lb_backend_address_pool.public_lb_pool_v4[0].id : null
 }
 
 output "public_lb_backend_pool_v6_id" {
-  value = var.use_ipv6 ? azurerm_lb_backend_address_pool.public_lb_pool_v6[0].id : null
+  value = local.need_public_ipv6 ? azurerm_lb_backend_address_pool.public_lb_pool_v6[0].id : null
 }
 
 output "internal_lb_backend_pool_v4_id" {
@@ -19,11 +19,11 @@ output "public_lb_id" {
 }
 
 output "public_lb_pip_v4_fqdn" {
-  value = var.private || ! var.use_ipv4 ? null : data.azurerm_public_ip.cluster_public_ip_v4[0].fqdn
+  value = local.need_public_ipv4 ? data.azurerm_public_ip.cluster_public_ip_v4[0].fqdn : null
 }
 
 output "public_lb_pip_v6_fqdn" {
-  value = var.private || ! var.use_ipv6 ? null : data.azurerm_public_ip.cluster_public_ip_v6[0].fqdn
+  value = local.need_public_ipv6 ? data.azurerm_public_ip.cluster_public_ip_v6[0].fqdn : null
 }
 
 output "internal_lb_ip_v4_address" {

data/data/azure/vnet/public-lb.tf

@@ -3,9 +3,16 @@ locals {
   public_lb_frontend_ip_v6_configuration_name = "public-lb-ip-v6"
 }
 
-resource "azurerm_public_ip" "cluster_public_ip_v4" {
+locals {
   // DEBUG: Azure apparently requires dual stack LB for v6
-  count = var.use_ipv4 || true ? 1 : 0
+  need_public_ipv4 = ! var.private || ! var.outbound_udr
+
+  need_public_ipv6 = var.use_ipv6 && (! var.private || ! var.outbound_udr)
+}
+
+
+resource "azurerm_public_ip" "cluster_public_ip_v4" {
+  count = local.need_public_ipv4 ? 1 : 0
 
   sku                 = "Standard"
   location            = var.region
@@ -17,14 +24,15 @@ resource "azurerm_public_ip" "cluster_public_ip_v4" {
 
 data "azurerm_public_ip" "cluster_public_ip_v4" {
   // DEBUG: Azure apparently requires dual stack LB for v6
-  count = var.use_ipv4 || true ? 1 : 0
+  count = local.need_public_ipv4 ? 1 : 0
 
   name                = azurerm_public_ip.cluster_public_ip_v4[0].name
   resource_group_name = var.resource_group_name
 }
 
+
 resource "azurerm_public_ip" "cluster_public_ip_v6" {
-  count = var.use_ipv6 ? 1 : 0
+  count = local.need_public_ipv6 ? 1 : 0
 
   ip_version          = "IPv6"
   sku                 = "Standard"
@@ -36,7 +44,7 @@ resource "azurerm_public_ip" "cluster_public_ip_v6" {
 }
 
 data "azurerm_public_ip" "cluster_public_ip_v6" {
-  count = var.use_ipv6 ? 1 : 0
+  count = local.need_public_ipv6 ? 1 : 0
 
   name                = azurerm_public_ip.cluster_public_ip_v6[0].name
   resource_group_name = var.resource_group_name
@@ -51,8 +59,18 @@ resource "azurerm_lb" "public" {
   dynamic "frontend_ip_configuration" {
     for_each = [for ip in [
       // DEBUG: Azure apparently requires dual stack LB for external load balancers v6
-      { name : local.public_lb_frontend_ip_v4_configuration_name, value : azurerm_public_ip.cluster_public_ip_v4[0].id, include : true, ipv6 : false },
-      { name : local.public_lb_frontend_ip_v6_configuration_name, value : var.use_ipv6 ? azurerm_public_ip.cluster_public_ip_v6[0].id : null, include : var.use_ipv6, ipv6 : true },
+      {
+        name : local.public_lb_frontend_ip_v4_configuration_name,
+        value : local.need_public_ipv4 ? azurerm_public_ip.cluster_public_ip_v4[0].id : null,
+        include : local.need_public_ipv4,
+        ipv6 : false,
+      },
+      {
+        name : local.public_lb_frontend_ip_v6_configuration_name,
+        value : local.need_public_ipv6 ? azurerm_public_ip.cluster_public_ip_v6[0].id : null,
+        include : local.need_public_ipv6,
+        ipv6 : true,
+      },
       ] : {
       name : ip.name
       value : ip.value
@@ -70,24 +88,29 @@ resource "azurerm_lb" "public" {
   }
 }
 
+// The backends are only created when frontend configuration exists, because of the following error from Azure API;
+// ```
+// Load Balancer /subscriptions/xx/resourceGroups/xx/providers/Microsoft.Network/loadBalancers/xx-public-lb does not have Frontend IP Configuration, 
+// but it has other child resources. This setup is not supported.
+// ```
 resource "azurerm_lb_backend_address_pool" "public_lb_pool_v4" {
-  count = var.use_ipv4 ? 1 : 0
+  count = local.need_public_ipv4 ? 1 : 0
 
   resource_group_name = var.resource_group_name
   loadbalancer_id     = azurerm_lb.public.id
   name                = var.cluster_id
 }
 
 resource "azurerm_lb_backend_address_pool" "public_lb_pool_v6" {
-  count = var.use_ipv6 ? 1 : 0
+  count = local.need_public_ipv6 ? 1 : 0
 
   resource_group_name = var.resource_group_name
   loadbalancer_id     = azurerm_lb.public.id
   name                = "${var.cluster_id}-IPv6"
 }
 
 resource "azurerm_lb_rule" "public_lb_rule_api_internal_v4" {
-  count = var.private || ! var.use_ipv4 ? 0 : 1
+  count = var.use_ipv4 && ! var.private ? 1 : 0
 
   name                           = "api-internal-v4"
   resource_group_name            = var.resource_group_name
@@ -104,7 +127,7 @@ resource "azurerm_lb_rule" "public_lb_rule_api_internal_v4" {
 }
 
 resource "azurerm_lb_rule" "public_lb_rule_api_internal_v6" {
-  count = var.private || ! var.use_ipv6 ? 0 : 1
+  count = var.use_ipv6 && ! var.private ? 1 : 0
 
   name                           = "api-internal-v6"
   resource_group_name            = var.resource_group_name
@@ -120,36 +143,32 @@ resource "azurerm_lb_rule" "public_lb_rule_api_internal_v6" {
   probe_id                       = azurerm_lb_probe.public_lb_probe_api_internal[0].id
 }
 
-resource "azurerm_lb_rule" "internal_outbound_rule_v4" {
-  count = var.private && var.use_ipv4 ? 1 : 0
+resource "azurerm_lb_outbound_rule" "public_lb_outbound_rule_v4" {
+  count = var.use_ipv4 && var.private && ! var.outbound_udr ? 1 : 0
 
-  name                           = "internal_outbound_rule_v4"
-  resource_group_name            = var.resource_group_name
-  protocol                       = "Tcp"
-  backend_address_pool_id        = azurerm_lb_backend_address_pool.public_lb_pool_v4[0].id
-  loadbalancer_id                = azurerm_lb.public.id
-  frontend_port                  = 27627
-  backend_port                   = 27627
-  frontend_ip_configuration_name = local.public_lb_frontend_ip_v4_configuration_name
-  enable_floating_ip             = false
-  idle_timeout_in_minutes        = 30
-  load_distribution              = "Default"
+  name                    = "outbound-rule-v4"
+  resource_group_name     = var.resource_group_name
+  loadbalancer_id         = azurerm_lb.public.id
+  backend_address_pool_id = azurerm_lb_backend_address_pool.public_lb_pool_v4[0].id
+  protocol                = "All"
+
+  frontend_ip_configuration {
+    name = local.public_lb_frontend_ip_v4_configuration_name
+  }
 }
 
-resource "azurerm_lb_rule" "internal_outbound_rule_v6" {
-  count = var.private && var.use_ipv6 ? 1 : 0
+resource "azurerm_lb_outbound_rule" "public_lb_outbound_rule_v6" {
+  count = var.use_ipv6 && var.private && ! var.outbound_udr ? 1 : 0
 
-  name                           = "internal_outbound_rule_v6"
-  resource_group_name            = var.resource_group_name
-  protocol                       = "Tcp"
-  backend_address_pool_id        = azurerm_lb_backend_address_pool.public_lb_pool_v6[0].id
-  loadbalancer_id                = azurerm_lb.public.id
-  frontend_port                  = 27627
-  backend_port                   = 27627
-  frontend_ip_configuration_name = local.public_lb_frontend_ip_v6_configuration_name
-  enable_floating_ip             = false
-  idle_timeout_in_minutes        = 30
-  load_distribution              = "Default"
+  name                    = "outbound-rule-v6"
+  resource_group_name     = var.resource_group_name
+  loadbalancer_id         = azurerm_lb.public.id
+  backend_address_pool_id = azurerm_lb_backend_address_pool.public_lb_pool_v6[0].id
+  protocol                = "All"
+
+  frontend_ip_configuration {
+    name = local.public_lb_frontend_ip_v6_configuration_name
+  }
 }
 
 resource "azurerm_lb_probe" "public_lb_probe_api_internal" {

data/data/azure/vnet/variables.tf

@@ -76,3 +76,13 @@ variable "emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "outbound_udr" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+EOF
+}

data/data/install.openshift.io_installconfigs.yaml

@@ -909,6 +909,15 @@ spec:
                     description: NetworkResourceGroupName specifies the network resource
                       group that contains an existing VNet
                     type: string
+                  outboundType:
+                    default: Loadbalancer
+                    description: OutboundType is a strategy for how egress from cluster
+                      is achieved. When not specified default is "Loadbalancer".
+                    enum:
+                    - ""
+                    - Loadbalancer
+                    - UserDefinedRouting
+                    type: string
                   region:
                     description: Region specifies the Azure region where the cluster
                       will be created.

docs/user/azure/customization.md

@@ -12,7 +12,10 @@ The following options are available when using Azure:
 * `networkResourceGroupName` (optional string): The resource group where the Azure VNet is found.
 * `virtualNetwork` (optional string): The name of an existing VNet where the cluster infrastructure should be provisioned.
 * `controlPlaneSubnet` (optional string): An existing subnet which should be used for the cluster control plane.
-* `computeSubnet` (optional string): An existing subnet which should be used by cluster nodes. 
+* `computeSubnet` (optional string): An existing subnet which should be used by cluster nodes.
+* `outboundType` (optional string):  OutboundType is a strategy for how egress from cluster is achieved. Valid values are `Loadbalancer` or `UserDefinedRouting`
+    * `Loadbalancer` (default): LoadbalancerOutboundType uses Standard loadbalancer for egress from the cluster, see [docs][azure-lb-outbound]
+    * `UserDefinedRouting`: UserDefinedRoutingOutboundType uses user defined routing for egress from the cluster, see [docs][azure-udr-outbound]. User defined routing for egress can only be used when deploying clusters to pre-existing virtual networks.
 
 ## Machine pools
 
@@ -93,6 +96,7 @@ platform:
 pullSecret: '{"auths": ...}'
 sshKey: ssh-ed25519 AAAA...
 ```
+
 ### Existing VNet
 
 An example Azure install config to use a pre-existing VNet and subnets:
@@ -116,3 +120,6 @@ platform:
 pullSecret: '{"auths": ...}'
 sshKey: ssh-ed25519 AAAA...
 ```
+
+[azure-lb-outbound]: https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#lb
+[azure-udr-outbound]: https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview

pkg/asset/cluster/tfvars.go

@@ -285,6 +285,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				ImageURL:                    string(*rhcosImage),
 				PreexistingNetwork:          preexistingnetwork,
 				Publish:                     installConfig.Config.Publish,
+				OutboundType:                installConfig.Config.Azure.OutboundType,
 			},
 		)
 		if err != nil {

pkg/asset/machines/azure/machines.go

@@ -94,6 +94,11 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		mpool.OSDisk.DiskType = "Premium_LRS"
 	}
 
+	publicLB := clusterID
+	if platform.OutboundType == azure.UserDefinedRoutingOutboundType {
+		publicLB = ""
+	}
+
 	return &azureprovider.AzureMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: "azureproviderconfig.openshift.io/v1beta1",
@@ -119,6 +124,7 @@ func provider(platform *azure.Platform, mpool *azure.MachinePool, osImage string
 		Vnet:                 virtualNetwork,
 		ResourceGroup:        fmt.Sprintf("%s-rg", clusterID),
 		NetworkResourceGroup: networkResourceGroup,
+		PublicLoadBalancer:   publicLB,
 	}, nil
 }