Merge branch 'master' into installer_telemetry_metric_builder

.github/ISSUE_TEMPLATE.md

@@ -16,7 +16,13 @@ $ openshift-install version
 
 # Platform:
 <!--
-Please specify the platform type: aws, libvirt, openstack, baremetal, or none (UPI)
+Please specify the platform type: aws, libvirt, openstack or baremetal
+-->
+
+<!--
+Please specify:
+* IPI (automated install with `openshift-install`. If you don't know, then it's IPI)
+* UPI (semi-manual installation on customised infrastructure)
 -->
 
 # What happened?

cmd/openshift-install/migrate/azure/eligible.go

@@ -1,23 +1,33 @@
 package azure
 
 import (
+	"fmt"
+
 	"github.com/spf13/cobra"
 
 	azmigrate "github.com/openshift/installer/pkg/migrate/azure"
+	"github.com/openshift/installer/pkg/types/azure"
 )
 
-func runMigrateAzurePrivateDNSEligibleCmd(cmd *cobra.Command, args []string) error {
-	return azmigrate.Eligible()
-}
-
 // NewMigrateAzurePrivateDNSEligibleCmd adds the eligble command to openshift-install
 func NewMigrateAzurePrivateDNSEligibleCmd() *cobra.Command {
+	var cloudName string
+
 	cmd := &cobra.Command{
 		Use:   "azure-privatedns-eligible",
 		Short: "Show legacy Azure zones that are eligible to be migrated",
 		Long:  "This will show legacy Azure private zones that can be migrated to new private zones.",
-		RunE:  runMigrateAzurePrivateDNSEligibleCmd,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return azmigrate.Eligible(azure.CloudEnvironment(cloudName))
+		},
 	}
 
+	cmd.Flags().StringVar(
+		&cloudName,
+		"cloud-name",
+		string(azure.PublicCloud),
+		fmt.Sprintf("cloud environment name, one of: %s, %s, %s, %s", azure.PublicCloud, azure.USGovernmentCloud, azure.ChinaCloud, azure.GermanCloud),
+	)
+
 	return cmd
 }

cmd/openshift-install/migrate/azure/migrate.go

@@ -1,14 +1,18 @@
 package azure
 
 import (
+	"fmt"
+
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 
 	azmigrate "github.com/openshift/installer/pkg/migrate/azure"
+	"github.com/openshift/installer/pkg/types/azure"
 )
 
 var (
 	azureMigrateOpts struct {
+		cloudName         string
 		zone              string
 		resourceGroup     string
 		virtualNetwork    string
@@ -18,6 +22,11 @@ var (
 )
 
 func runMigrateAzurePrivateDNSMigrateCmd(cmd *cobra.Command, args []string) error {
+	switch azure.CloudEnvironment(azureMigrateOpts.cloudName) {
+	case azure.PublicCloud, azure.USGovernmentCloud, azure.ChinaCloud, azure.GermanCloud:
+	default:
+		return errors.Errorf("cloud-name must be one of %s, %s, %s, %s", azure.PublicCloud, azure.USGovernmentCloud, azure.ChinaCloud, azure.GermanCloud)
+	}
 	if azureMigrateOpts.zone == "" {
 		return errors.New("zone is a required argument")
 	}
@@ -31,7 +40,14 @@ func runMigrateAzurePrivateDNSMigrateCmd(cmd *cobra.Command, args []string) erro
 		return errors.New("virtual-network requires virtual-network-resource-group to be set")
 	}
 
-	return azmigrate.Migrate(azureMigrateOpts.resourceGroup, azureMigrateOpts.zone, azureMigrateOpts.virtualNetwork, azureMigrateOpts.vnetResourceGroup, azureMigrateOpts.link)
+	return azmigrate.Migrate(
+		azure.CloudEnvironment(azureMigrateOpts.cloudName),
+		azureMigrateOpts.resourceGroup,
+		azureMigrateOpts.zone,
+		azureMigrateOpts.virtualNetwork,
+		azureMigrateOpts.vnetResourceGroup,
+		azureMigrateOpts.link,
+	)
 }
 
 // NewMigrateAzurePrivateDNSMigrateCmd adds the migrate command to openshift-install
@@ -43,6 +59,12 @@ func NewMigrateAzurePrivateDNSMigrateCmd() *cobra.Command {
 		RunE:  runMigrateAzurePrivateDNSMigrateCmd,
 	}
 
+	cmd.PersistentFlags().StringVar(
+		&azureMigrateOpts.cloudName,
+		"cloud-name",
+		string(azure.PublicCloud),
+		fmt.Sprintf("cloud environment name, one of: %s, %s, %s, %s", azure.PublicCloud, azure.USGovernmentCloud, azure.ChinaCloud, azure.GermanCloud),
+	)
 	cmd.PersistentFlags().StringVar(&azureMigrateOpts.zone, "zone", "", "The zone to migrate")
 	cmd.PersistentFlags().StringVar(&azureMigrateOpts.resourceGroup, "resource-group", "", "The resource group of the zone")
 	cmd.PersistentFlags().StringVar(&azureMigrateOpts.virtualNetwork, "virtual-network", "", "The virtual network to create the private zone in")

data/data/azure/bootstrap/main.tf

@@ -137,16 +137,19 @@ resource "azurerm_network_interface" "bootstrap" {
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v4" {
-  // should be 'count = var.use_ipv4 && ! var.emulate_single_stack_ipv6 ? 1 : 0', but we need a V4 LB for egress for quay
-  count = var.use_ipv4 ? 1 : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = (! var.private || ! var.outbound_udr) ? 1 : 0
 
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.elb_backend_pool_v4_id
   ip_configuration_name   = local.bootstrap_nic_ip_v4_configuration_name
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "public_lb_bootstrap_v6" {
-  count = var.use_ipv6 ? 1 : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = var.use_ipv6 && (! var.private || ! var.outbound_udr) ? 1 : 0
 
   network_interface_id    = azurerm_network_interface.bootstrap.id
   backend_address_pool_id = var.elb_backend_pool_v6_id

data/data/azure/bootstrap/variables.tf

@@ -93,3 +93,16 @@ variable "emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "outbound_udr" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+
+This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+EOF
+}

data/data/azure/main.tf

@@ -13,6 +13,7 @@ provider "azurerm" {
   client_id       = var.azure_client_id
   client_secret   = var.azure_client_secret
   tenant_id       = var.azure_tenant_id
+  environment     = var.azure_environment
 }
 
 provider "azureprivatedns" {
@@ -40,6 +41,7 @@ module "bootstrap" {
   storage_account        = azurerm_storage_account.cluster
   nsg_name               = module.vnet.cluster_nsg_name
   private                = module.vnet.private
+  outbound_udr           = var.azure_outbound_user_defined_routing
 
   use_ipv4                  = var.use_ipv4 || var.azure_emulate_single_stack_ipv6
   use_ipv6                  = var.use_ipv6
@@ -61,6 +63,7 @@ module "vnet" {
   master_subnet               = var.azure_control_plane_subnet
   worker_subnet               = var.azure_compute_subnet
   private                     = var.azure_private
+  outbound_udr                = var.azure_outbound_user_defined_routing
 
   use_ipv4                  = var.use_ipv4 || var.azure_emulate_single_stack_ipv6
   use_ipv6                  = var.use_ipv6
@@ -87,6 +90,7 @@ module "master" {
   os_volume_type         = var.azure_master_root_volume_type
   os_volume_size         = var.azure_master_root_volume_size
   private                = module.vnet.private
+  outbound_udr           = var.azure_outbound_user_defined_routing
 
   use_ipv4                  = var.use_ipv4 || var.azure_emulate_single_stack_ipv6
   use_ipv6                  = var.use_ipv6
@@ -169,7 +173,7 @@ resource "azurerm_storage_blob" "rhcos_image" {
   name                   = "rhcos${random_string.storage_suffix.result}.vhd"
   storage_account_name   = azurerm_storage_account.cluster.name
   storage_container_name = azurerm_storage_container.vhd.name
-  type                   = "Block"
+  type                   = "Page"
   source_uri             = var.azure_image_url
   metadata               = map("source_uri", var.azure_image_url)
 }

data/data/azure/master/master.tf

@@ -46,16 +46,19 @@ resource "azurerm_network_interface" "master" {
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "master_v4" {
-  // should be 'count = var.use_ipv4 && ! var.emulate_single_stack_ipv6 ? var.instance_count : 0', but we need a V4 LB for egress for quay
-  count = var.use_ipv4 ? var.instance_count : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = (! var.private || ! var.outbound_udr) ? var.instance_count : 0
 
   network_interface_id    = element(azurerm_network_interface.master.*.id, count.index)
   backend_address_pool_id = var.elb_backend_pool_v4_id
   ip_configuration_name   = local.ip_v4_configuration_name
 }
 
 resource "azurerm_network_interface_backend_address_pool_association" "master_v6" {
-  count = var.use_ipv6 ? var.instance_count : 0
+  // This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+  // conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+  count = var.use_ipv6 && (! var.private || ! var.outbound_udr) ? var.instance_count : 0
 
   network_interface_id    = element(azurerm_network_interface.master.*.id, count.index)
   backend_address_pool_id = var.elb_backend_pool_v6_id

data/data/azure/master/variables.tf

@@ -110,3 +110,16 @@ variable "emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "outbound_udr" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+
+This is required because terraform cannot calculate counts during plan phase completely and therefore the `vnet/public-lb.tf`
+conditional need to be recreated. See https://github.com/hashicorp/terraform/issues/12570
+EOF
+}

data/data/azure/variables-azure.tf

@@ -8,6 +8,11 @@ EOF
   default = "0.1"
 }
 
+variable "azure_environment" {
+  type = string
+  description = "The target Azure cloud environment for the cluster."
+}
+
 variable "azure_region" {
   type = string
   description = "The target Azure region for the cluster."
@@ -116,3 +121,13 @@ variable "azure_emulate_single_stack_ipv6" {
   type        = bool
   description = "This determines whether a dual-stack cluster is configured to emulate single-stack IPv6."
 }
+
+variable "azure_outbound_user_defined_routing" {
+  type    = bool
+  default = false
+
+  description = <<EOF
+This determined whether User defined routing will be used for egress to Internet.
+When false, Standard LB will be used for egress to the Internet.
+EOF
+}

data/data/azure/vnet/outputs.tf

@@ -1,9 +1,9 @@
 output "public_lb_backend_pool_v4_id" {
-  value = var.use_ipv4 ? azurerm_lb_backend_address_pool.public_lb_pool_v4[0].id : null
+  value = local.need_public_ipv4 ? azurerm_lb_backend_address_pool.public_lb_pool_v4[0].id : null
 }
 
 output "public_lb_backend_pool_v6_id" {
-  value = var.use_ipv6 ? azurerm_lb_backend_address_pool.public_lb_pool_v6[0].id : null
+  value = local.need_public_ipv6 ? azurerm_lb_backend_address_pool.public_lb_pool_v6[0].id : null
 }
 
 output "internal_lb_backend_pool_v4_id" {
@@ -19,11 +19,11 @@ output "public_lb_id" {
 }
 
 output "public_lb_pip_v4_fqdn" {
-  value = var.private || ! var.use_ipv4 ? null : data.azurerm_public_ip.cluster_public_ip_v4[0].fqdn
+  value = local.need_public_ipv4 ? data.azurerm_public_ip.cluster_public_ip_v4[0].fqdn : null
 }
 
 output "public_lb_pip_v6_fqdn" {
-  value = var.private || ! var.use_ipv6 ? null : data.azurerm_public_ip.cluster_public_ip_v6[0].fqdn
+  value = local.need_public_ipv6 ? data.azurerm_public_ip.cluster_public_ip_v6[0].fqdn : null
 }
 
 output "internal_lb_ip_v4_address" {