Add subscription feature MTU3900

FeatureFlagMTU3900 is the feature in the subscription that causes new
OpenShift cluster nodes to use the largest available Maximum Transmission
Unit (MTU) on Azure virtual networks, which as of late 2021 is 3900 bytes.
Otherwise cluster nodes will use the DHCP-provided MTU of 1500 bytes.

pkg/api/featureflags.go

@@ -14,4 +14,10 @@ const (
 	// We need a feature flag to make sure we don't open a security hole in existing
 	// clusters before customer had a chance to patch their API RBAC
 	FeatureFlagAdminKubeconfig = "Microsoft.RedHatOpenShift/AdminKubeconfig"
+
+	// FeatureFlagMTU3900 is the feature in the subscription that causes new
+	// OpenShift cluster nodes to use the largest available Maximum Transmission
+	// Unit (MTU) on Azure virtual networks, which as of late 2021 is 3900 bytes.
+	// Otherwise cluster nodes will use the DHCP-provided MTU of 1500 bytes.
+	FeatureFlagMTU3900 = "Microsoft.RedHatOpenShift/MTU3900"
 )

pkg/cluster/deploystorage.go

@@ -28,6 +28,7 @@ import (
 	"github.com/Azure/ARO-RP/pkg/cluster/graph"
 	"github.com/Azure/ARO-RP/pkg/env"
 	"github.com/Azure/ARO-RP/pkg/util/arm"
+	"github.com/Azure/ARO-RP/pkg/util/feature"
 	"github.com/Azure/ARO-RP/pkg/util/stringutils"
 	"github.com/Azure/ARO-RP/pkg/util/subnet"
 )
@@ -209,6 +210,15 @@ func (m *manager) ensureGraph(ctx context.Context, installConfig *installconfig.
 		}
 	}
 
+	// Handle MTU3900 feature flag
+	subProperties := m.subscriptionDoc.Subscription.Properties
+	if feature.IsRegisteredForFeature(subProperties, api.FeatureFlagMTU3900) {
+		m.log.Printf("applying feature flag %s", api.FeatureFlagMTU3900)
+		if err = m.overrideEthernetMTU(g); err != nil {
+			return err
+		}
+	}
+
 	// the graph is quite big so we store it in a storage account instead of in cosmosdb
 	return m.graph.Save(ctx, resourceGroup, clusterStorageAccountName, g)
 }

pkg/cluster/overridemtu.go

@@ -0,0 +1,99 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"fmt"
+
+	"github.com/coreos/ignition/v2/config/v3_2/types"
+	"github.com/openshift/installer/pkg/asset/ignition"
+	"github.com/openshift/installer/pkg/asset/ignition/bootstrap"
+	"github.com/openshift/installer/pkg/asset/machines/machineconfig"
+	mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	"github.com/pkg/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/Azure/ARO-RP/pkg/cluster/graph"
+)
+
+const (
+	IgnFilePath = "/etc/NetworkManager/dispatcher.d/30-eth0-mtu-3900"
+	IgnFileData = `#!/bin/bash
+
+if [ "$1" == "eth0" ] && [ "$2" == "up" ]; then
+    ip link set $1 mtu 3900
+fi`
+)
+
+func newMTUMachineConfigIgnitionFile(role string) (types.File, error) {
+	mtuIgnitionConfig := types.Config{
+		Ignition: types.Ignition{
+			Version: types.MaxVersion.String(),
+		},
+		Storage: types.Storage{
+			Files: []types.File{
+				ignition.FileFromString(IgnFilePath, "root", 0555, IgnFileData),
+			},
+		},
+	}
+
+	rawExt, err := ignition.ConvertToRawExtension(mtuIgnitionConfig)
+	if err != nil {
+		return types.File{}, err
+	}
+
+	mtuMachineConfig := &mcv1.MachineConfig{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: mcv1.SchemeGroupVersion.String(),
+			Kind:       "MachineConfig",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fmt.Sprintf("99-%s-mtu", role),
+			Labels: map[string]string{
+				"machineconfiguration.openshift.io/role": role,
+			},
+		},
+		Spec: mcv1.MachineConfigSpec{
+			Config: rawExt,
+		},
+	}
+
+	configs := []*mcv1.MachineConfig{mtuMachineConfig}
+	manifests, err := machineconfig.Manifests(configs, role, "/opt/openshift/openshift")
+	if err != nil {
+		return types.File{}, err
+	}
+
+	return ignition.FileFromBytes(manifests[0].Filename, "root", 0644, manifests[0].Data), nil
+}
+
+func (m *manager) overrideEthernetMTU(g graph.Graph) error {
+	// This adds the following MachineConfig manifest files to the bootstrap
+	// node's Ignition config:
+	//
+	// /opt/openshift/openshift/99_openshift-machineconfig_99-master-mtu.yaml
+	// /opt/openshift/openshift/99_openshift-machineconfig_99-worker-mtu.yaml
+
+	bootstrap := g.Get(&bootstrap.Bootstrap{}).(*bootstrap.Bootstrap)
+
+	ignitionFile, err := newMTUMachineConfigIgnitionFile("master")
+	if err != nil {
+		return nil
+	}
+	bootstrap.Config.Storage.Files = append(bootstrap.Config.Storage.Files, ignitionFile)
+
+	ignitionFile, err = newMTUMachineConfigIgnitionFile("worker")
+	if err != nil {
+		return nil
+	}
+	bootstrap.Config.Storage.Files = append(bootstrap.Config.Storage.Files, ignitionFile)
+
+	data, err := ignition.Marshal(bootstrap.Config)
+	if err != nil {
+		return errors.Wrap(err, "failed to Marshal Ignition config")
+	}
+	bootstrap.File.Data = data
+
+	return nil
+}