diff --git a/cmd/aro/operator.go b/cmd/aro/operator.go index 6dbef283db7..07b85de50d7 100644 --- a/cmd/aro/operator.go +++ b/cmd/aro/operator.go @@ -10,6 +10,7 @@ import ( configclient "github.com/openshift/client-go/config/clientset/versioned" consoleclient "github.com/openshift/client-go/console/clientset/versioned" + imageregistryclient "github.com/openshift/client-go/imageregistry/clientset/versioned" securityclient "github.com/openshift/client-go/security/clientset/versioned" maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned" mcoclient "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned" @@ -36,6 +37,7 @@ import ( "github.com/Azure/ARO-RP/pkg/operator/controllers/pullsecret" "github.com/Azure/ARO-RP/pkg/operator/controllers/rbac" "github.com/Azure/ARO-RP/pkg/operator/controllers/routefix" + "github.com/Azure/ARO-RP/pkg/operator/controllers/storageaccounts" "github.com/Azure/ARO-RP/pkg/operator/controllers/subnets" "github.com/Azure/ARO-RP/pkg/operator/controllers/workaround" "github.com/Azure/ARO-RP/pkg/util/dynamichelper" @@ -99,6 +101,10 @@ func operator(ctx context.Context, log *logrus.Entry) error { if err != nil { return err } + imageregistrycli, err := imageregistryclient.NewForConfig(restConfig) + if err != nil { + return err + } // TODO (NE): dh is sometimes passed, sometimes created later. Can we standardize? dh, err := dynamichelper.New(log, restConfig) if err != nil { @@ -191,6 +197,11 @@ func operator(ctx context.Context, log *logrus.Entry) error { arocli, configcli)).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create controller ImageConfig: %v", err) } + if err = (storageaccounts.NewReconciler( + log.WithField("controller", controllers.StorageAccountsControllerName), + arocli, maocli, kubernetescli, imageregistrycli)).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller %s: %v", controllers.StorageAccountsControllerName, err) + } } if err = (checker.NewReconciler( diff --git a/deploy/rp-development-predeploy.json b/deploy/rp-development-predeploy.json index ca72319c96a..061de1d8829 100644 --- a/deploy/rp-development-predeploy.json +++ b/deploy/rp-development-predeploy.json @@ -103,6 +103,14 @@ "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-pe-nsg')]", "tags": null }, + "serviceEndpoints": [ + { + "service": "Microsoft.Storage", + "locations": [ + "*" + ] + } + ], "privateEndpointNetworkPolicies": "Disabled" }, "name": "rp-pe-subnet" diff --git a/deploy/rp-production-predeploy.json b/deploy/rp-production-predeploy.json index d6fc304256f..3d4d10421a8 100644 --- a/deploy/rp-production-predeploy.json +++ b/deploy/rp-production-predeploy.json @@ -205,6 +205,12 @@ "locations": [ "*" ] + }, + { + "service": "Microsoft.Storage", + "locations": [ + "*" + ] } ] }, @@ -235,6 +241,14 @@ "id": "[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-pe-nsg')]", "tags": null }, + "serviceEndpoints": [ + { + "service": "Microsoft.Storage", + "locations": [ + "*" + ] + } + ], "privateEndpointNetworkPolicies": "Disabled" }, "name": "rp-pe-subnet" diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index fd0e714da97..f26b143eee2 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -49,36 +49,25 @@ const FLAG_FALSE string = "false" // Default flags for new clusters & ones that have not been AdminUpdated var DefaultOperatorFlags OperatorFlags = OperatorFlags{ - "aro.alertwebhook.enabled": FLAG_TRUE, - - "aro.azuresubnets.enabled": FLAG_TRUE, - - "aro.banner.enabled": FLAG_FALSE, - - "aro.checker.enabled": FLAG_TRUE, - - "aro.dnsmasq.enabled": FLAG_TRUE, - - "aro.genevalogging.enabled": FLAG_TRUE, - - "aro.imageconfig.enabled": FLAG_TRUE, - - "aro.machine.enabled": FLAG_TRUE, - - "aro.machineset.enabled": FLAG_TRUE, - - "aro.monitoring.enabled": FLAG_TRUE, - - "aro.nodedrainer.enabled": FLAG_TRUE, - - "aro.pullsecret.enabled": FLAG_TRUE, - "aro.pullsecret.managed": FLAG_TRUE, - - "aro.rbac.enabled": FLAG_TRUE, - - "aro.routefix.enabled": FLAG_TRUE, - - "aro.workaround.enabled": FLAG_TRUE, + "aro.alertwebhook.enabled": FLAG_TRUE, + "aro.azuresubnets.enabled": FLAG_TRUE, + "aro.azuresubnets.nsg.managed": FLAG_TRUE, + "aro.azuresubnets.serviceendpoint.managed": FLAG_TRUE, + "aro.banner.enabled": FLAG_FALSE, + "aro.checker.enabled": FLAG_TRUE, + "aro.dnsmasq.enabled": FLAG_TRUE, + "aro.genevalogging.enabled": FLAG_TRUE, + "aro.imageconfig.enabled": FLAG_TRUE, + "aro.machine.enabled": FLAG_TRUE, + "aro.machineset.enabled": FLAG_TRUE, + "aro.monitoring.enabled": FLAG_TRUE, + "aro.nodedrainer.enabled": FLAG_TRUE, + "aro.pullsecret.enabled": FLAG_TRUE, + "aro.pullsecret.managed": FLAG_TRUE, + "aro.rbac.enabled": FLAG_TRUE, + "aro.routefix.enabled": FLAG_TRUE, + "aro.storageaccounts.enabled": FLAG_TRUE, + "aro.workaround.enabled": FLAG_TRUE, } func (d OperatorFlags) Copy() OperatorFlags { diff --git a/pkg/api/subnets.go b/pkg/api/subnets.go new file mode 100644 index 00000000000..cd080382d1c --- /dev/null +++ b/pkg/api/subnets.go @@ -0,0 +1,11 @@ +package api + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +var ( + SubnetsEndpoints = []string{ + "Microsoft.ContainerRegistry", + "Microsoft.Storage", + } +) diff --git a/pkg/cluster/adminupdate_test.go b/pkg/cluster/adminupdate_test.go index ea2d824bf6e..fa1ea56b331 100644 --- a/pkg/cluster/adminupdate_test.go +++ b/pkg/cluster/adminupdate_test.go @@ -67,6 +67,9 @@ func TestAdminUpdateSteps(t *testing.T) { "[Action fixInfraID-fm]", "[AuthorizationRefreshingAction [Action ensureResourceGroup-fm]]", "[Action createOrUpdateDenyAssignment-fm]", + "[AuthorizationRefreshingAction [Action enableServiceEndpoints-fm]]", + "[Action populateRegistryStorageAccountName-fm]", + "[Action migrateStorageAccounts-fm]", "[Action fixSSH-fm]", "[Action populateDatabaseIntIP-fm]", "[Action startVMs-fm]", @@ -102,6 +105,9 @@ func TestAdminUpdateSteps(t *testing.T) { "[Action fixInfraID-fm]", "[AuthorizationRefreshingAction [Action ensureResourceGroup-fm]]", "[Action createOrUpdateDenyAssignment-fm]", + "[AuthorizationRefreshingAction [Action enableServiceEndpoints-fm]]", + "[Action populateRegistryStorageAccountName-fm]", + "[Action migrateStorageAccounts-fm]", "[Action fixSSH-fm]", "[Action populateDatabaseIntIP-fm]", "[Action startVMs-fm]", diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 43c7ab43e1d..db9a7cc7268 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -80,16 +80,16 @@ type manager struct { subnet subnet.Manager graph graph.Manager - kubernetescli kubernetes.Interface - extensionscli extensionsclient.Interface - maocli maoclient.Interface - mcocli mcoclient.Interface - operatorcli operatorclient.Interface - configcli configclient.Interface - samplescli samplesclient.Interface - securitycli securityclient.Interface - arocli aroclient.Interface - registryclient imageregistryclient.Interface + kubernetescli kubernetes.Interface + extensionscli extensionsclient.Interface + maocli maoclient.Interface + mcocli mcoclient.Interface + operatorcli operatorclient.Interface + configcli configclient.Interface + samplescli samplesclient.Interface + securitycli securityclient.Interface + arocli aroclient.Interface + imageregistrycli imageregistryclient.Interface } // New returns a cluster manager diff --git a/pkg/cluster/deploystorage.go b/pkg/cluster/deploystorage.go index d0a9b3ba854..db14aa07dae 100644 --- a/pkg/cluster/deploystorage.go +++ b/pkg/cluster/deploystorage.go @@ -132,10 +132,10 @@ func (m *manager) deployStorageTemplate(ctx context.Context, installConfig *inst clusterStorageAccountName := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix resources := []*arm.Resource{ - m.storageAccount(clusterStorageAccountName, installConfig.Config.Azure.Region), + m.storageAccount(clusterStorageAccountName, installConfig.Config.Azure.Region, true), m.storageAccountBlobContainer(clusterStorageAccountName, "ignition"), m.storageAccountBlobContainer(clusterStorageAccountName, "aro"), - m.storageAccount(m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName, installConfig.Config.Azure.Region), + m.storageAccount(m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName, installConfig.Config.Azure.Region, true), m.storageAccountBlobContainer(m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName, "image-registry"), m.clusterNSG(infraID, installConfig.Config.Azure.Region), m.clusterServicePrincipalRBAC(), diff --git a/pkg/cluster/deploystorage_resources.go b/pkg/cluster/deploystorage_resources.go index cf3bd494591..be50d922a53 100644 --- a/pkg/cluster/deploystorage_resources.go +++ b/pkg/cluster/deploystorage_resources.go @@ -74,19 +74,93 @@ func (m *manager) clusterServicePrincipalRBAC() *arm.Resource { ) } -func (m *manager) storageAccount(name, region string) *arm.Resource { - return &arm.Resource{ - Resource: &mgmtstorage.Account{ - Sku: &mgmtstorage.Sku{ - Name: "Standard_LRS", - }, - AccountProperties: &mgmtstorage.AccountProperties{ - MinimumTLSVersion: mgmtstorage.TLS12, +// storageAccount will return storage account resource. +// Legacy storage accounts (public) are not encrypted and cannot be retrofitted. +// The flag controls this behavior in update/create. +func (m *manager) storageAccount(name, region string, encrypted bool) *arm.Resource { + + virtualNetworkRules := []mgmtstorage.VirtualNetworkRule{ + { + VirtualNetworkResourceID: to.StringPtr(m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID), + Action: mgmtstorage.Allow, + }, + { + VirtualNetworkResourceID: to.StringPtr(m.doc.OpenShiftCluster.Properties.WorkerProfiles[0].SubnetID), + Action: mgmtstorage.Allow, + }, + { + VirtualNetworkResourceID: to.StringPtr("/subscriptions/" + m.env.SubscriptionID() + "/resourceGroups/" + m.env.ResourceGroup() + "/providers/Microsoft.Network/virtualNetworks/rp-pe-vnet-001/subnets/rp-pe-subnet"), + Action: mgmtstorage.Allow, + }, + { + VirtualNetworkResourceID: to.StringPtr("/subscriptions/" + m.env.SubscriptionID() + "/resourceGroups/" + m.env.ResourceGroup() + "/providers/Microsoft.Network/virtualNetworks/rp-vnet/subnets/rp-subnet"), + Action: mgmtstorage.Allow, + }, + } + + // Prod includes a gateway rule as well + // Once we reach a PLS limit (1000) within a vnet , we may need some refactoring here + // https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/azure-subscription-service-limits#private-link-limits + if !m.env.IsLocalDevelopmentMode() { + virtualNetworkRules = append(virtualNetworkRules, mgmtstorage.VirtualNetworkRule{ + VirtualNetworkResourceID: to.StringPtr("/subscriptions/" + m.env.SubscriptionID() + "/resourceGroups/" + m.env.GatewayResourceGroup() + "/providers/Microsoft.Network/virtualNetworks/gateway-vnet/subnets/gateway-subnet"), + Action: mgmtstorage.Allow, + }) + } + + sa := &mgmtstorage.Account{ + Kind: mgmtstorage.StorageV2, + Sku: &mgmtstorage.Sku{ + Name: "Standard_LRS", + }, + AccountProperties: &mgmtstorage.AccountProperties{ + AllowBlobPublicAccess: to.BoolPtr(false), + EnableHTTPSTrafficOnly: to.BoolPtr(true), + MinimumTLSVersion: mgmtstorage.TLS12, + NetworkRuleSet: &mgmtstorage.NetworkRuleSet{ + Bypass: mgmtstorage.AzureServices, + VirtualNetworkRules: &virtualNetworkRules, + DefaultAction: "Deny", }, - Name: &name, - Location: ®ion, - Type: to.StringPtr("Microsoft.Storage/storageAccounts"), }, + Name: &name, + Location: ®ion, + Type: to.StringPtr("Microsoft.Storage/storageAccounts"), + } + + // In development API calls originates from user laptop so we allow all. + // TODO: Move to development on VPN so we can make this IPRule. Will be done as part of Simply secure v2 work + if m.env.IsLocalDevelopmentMode() { + sa.NetworkRuleSet.DefaultAction = mgmtstorage.DefaultActionAllow + } + // When migrating storage accounts for old clusters we are not able to change + // encryption which is why we have this encryption flag. We will not add this + // retrospectively to old clusters + // If a storage account already has encryption enabled and the encrypted + // bool is set to false, it will still maintain the encryption on the storage account. + if encrypted { + sa.AccountProperties.Encryption = &mgmtstorage.Encryption{ + RequireInfrastructureEncryption: to.BoolPtr(true), + Services: &mgmtstorage.EncryptionServices{ + Blob: &mgmtstorage.EncryptionService{ + KeyType: mgmtstorage.KeyTypeAccount, + }, + File: &mgmtstorage.EncryptionService{ + KeyType: mgmtstorage.KeyTypeAccount, + }, + Table: &mgmtstorage.EncryptionService{ + KeyType: mgmtstorage.KeyTypeAccount, + }, + Queue: &mgmtstorage.EncryptionService{ + KeyType: mgmtstorage.KeyTypeAccount, + }, + }, + KeySource: mgmtstorage.KeySourceMicrosoftStorage, + } + } + + return &arm.Resource{ + Resource: sa, APIVersion: azureclient.APIVersion("Microsoft.Storage"), } } diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go index bf2ce48f2ac..1355a4058ad 100644 --- a/pkg/cluster/install.go +++ b/pkg/cluster/install.go @@ -52,6 +52,9 @@ func (m *manager) adminUpdate() []steps.Step { toRun = append(toRun, steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.ensureResourceGroup)), // re-create RP RBAC if needed after tenant migration steps.Action(m.createOrUpdateDenyAssignment), + steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.enableServiceEndpoints)), + steps.Action(m.populateRegistryStorageAccountName), // must go before migrateStorageAccounts + steps.Action(m.migrateStorageAccounts), steps.Action(m.fixSSH), steps.Action(m.populateDatabaseIntIP), //steps.Action(m.removePrivateDNSZone), // TODO(mj): re-enable once we communicate this out @@ -142,6 +145,7 @@ func (m *manager) Install(ctx context.Context) error { return m.ensureInfraID(ctx, installConfig) }), steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.ensureResourceGroup)), + steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.enableServiceEndpoints)), steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(m.setMasterSubnetPolicies)), steps.AuthorizationRefreshingAction(m.fpAuthorizer, steps.Action(func(ctx context.Context) error { return m.deployStorageTemplate(ctx, installConfig) @@ -288,7 +292,7 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error { return err } - m.registryclient, err = imageregistryclient.NewForConfig(restConfig) + m.imageregistrycli, err = imageregistryclient.NewForConfig(restConfig) return err } diff --git a/pkg/cluster/storageaccount_test.go b/pkg/cluster/storageaccount_test.go new file mode 100644 index 00000000000..bcb1727e06b --- /dev/null +++ b/pkg/cluster/storageaccount_test.go @@ -0,0 +1,138 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "testing" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + + "github.com/Azure/ARO-RP/pkg/api" + mock_subnet "github.com/Azure/ARO-RP/pkg/util/mocks/subnet" +) + +var ( + subscriptionId = "0000000-0000-0000-0000-000000000000" + vnetResourceGroup = "vnet-rg" + vnetName = "vnet" + subnetNameWorker = "worker" + subnetNameMaster = "master" +) + +func getValidSubnet(endpoints bool, state *mgmtnetwork.ProvisioningState) *mgmtnetwork.Subnet { + s := &mgmtnetwork.Subnet{ + SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{}, + } + if endpoints { + s.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{ + ServiceEndpoints: &[]mgmtnetwork.ServiceEndpointPropertiesFormat{ + { + Service: to.StringPtr("Microsoft.ContainerRegistry"), + Locations: &[]string{"*"}, + }, + { + Service: to.StringPtr("Microsoft.Storage"), + Locations: &[]string{"*"}, + }, + }, + } + if state != nil { + for i := range *s.SubnetPropertiesFormat.ServiceEndpoints { + (*s.SubnetPropertiesFormat.ServiceEndpoints)[i].ProvisioningState = *state + } + } + } + return s +} + +func TestEnableServiceEndpoints(t *testing.T) { + ctx := context.Background() + + type test struct { + name string + oc *api.OpenShiftCluster + mock func(subnetMock *mock_subnet.MockManager, tt test) + } + + for _, tt := range []test{ + { + name: "nothing to do", + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + MasterProfile: api.MasterProfile{ + SubnetID: "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster, + }, + WorkerProfiles: []api.WorkerProfile{ + { + SubnetID: "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker, + }, + }, + }, + }, + mock: func(subnetClient *mock_subnet.MockManager, tt test) { + subnets := []string{ + tt.oc.Properties.MasterProfile.SubnetID, + } + for _, wp := range tt.oc.Properties.WorkerProfiles { + subnets = append(subnets, wp.SubnetID) + } + + for _, subnetId := range subnets { + state := mgmtnetwork.Succeeded + subnetClient.EXPECT().Get(gomock.Any(), subnetId).Return(getValidSubnet(true, &state), nil) + } + }, + }, + { + name: "enable endpoints", + oc: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + MasterProfile: api.MasterProfile{ + SubnetID: "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster, + }, + WorkerProfiles: []api.WorkerProfile{ + { + SubnetID: "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker, + }, + }, + }, + }, + mock: func(subnetClient *mock_subnet.MockManager, tt test) { + subnets := []string{ + tt.oc.Properties.MasterProfile.SubnetID, + } + for _, wp := range tt.oc.Properties.WorkerProfiles { + subnets = append(subnets, wp.SubnetID) + } + + for _, subnetId := range subnets { + subnetClient.EXPECT().Get(gomock.Any(), subnetId).Return(getValidSubnet(false, nil), nil) + subnetClient.EXPECT().CreateOrUpdate(gomock.Any(), subnetId, getValidSubnet(true, nil)).Return(nil) + } + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + subnetClient := mock_subnet.NewMockManager(controller) + + tt.mock(subnetClient, tt) + + m := &manager{ + subnet: subnetClient, + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: tt.oc, + }, + } + + // we don't test errors as all of them would be out of our control + _ = m.enableServiceEndpoints(ctx) + }) + } +} diff --git a/pkg/cluster/storageaccounts.go b/pkg/cluster/storageaccounts.go index 1f7ec425760..89d512c2b87 100644 --- a/pkg/cluster/storageaccounts.go +++ b/pkg/cluster/storageaccounts.go @@ -5,18 +5,91 @@ package cluster import ( "context" + "strings" + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/go-autorest/autorest/to" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/arm" + "github.com/Azure/ARO-RP/pkg/util/stringutils" ) +// enableServiceEndpoints should enable service endpoints on +// subnets for storage account access +func (m *manager) enableServiceEndpoints(ctx context.Context) error { + subnets := []string{ + m.doc.OpenShiftCluster.Properties.MasterProfile.SubnetID, + } + + for _, wp := range m.doc.OpenShiftCluster.Properties.WorkerProfiles { + subnets = append(subnets, wp.SubnetID) + } + + for _, subnetId := range subnets { + subnet, err := m.subnet.Get(ctx, subnetId) + if err != nil { + return err + } + + var changed bool + for _, endpoint := range api.SubnetsEndpoints { + var found bool + if subnet != nil && subnet.ServiceEndpoints != nil { + for _, se := range *subnet.ServiceEndpoints { + if strings.EqualFold(*se.Service, endpoint) && + se.ProvisioningState == mgmtnetwork.Succeeded { + found = true + } + } + } + if !found { + if subnet.ServiceEndpoints == nil { + subnet.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{} + } + *subnet.ServiceEndpoints = append(*subnet.ServiceEndpoints, mgmtnetwork.ServiceEndpointPropertiesFormat{ + Service: to.StringPtr(endpoint), + Locations: &[]string{"*"}, + }) + changed = true + } + } + if changed { + err := m.subnet.CreateOrUpdate(ctx, subnetId, subnet) + if err != nil { + return err + } + } + } + return nil +} + +// migrateStorageAccounts redeploys storage accounts with firewall rules preventing external access +// The encryption flag is set to false/disabled for legacy storage accounts. +func (m *manager) migrateStorageAccounts(ctx context.Context) error { + resourceGroup := stringutils.LastTokenByte(m.doc.OpenShiftCluster.Properties.ClusterProfile.ResourceGroupID, '/') + clusterStorageAccountName := "cluster" + m.doc.OpenShiftCluster.Properties.StorageSuffix + registryStorageAccountName := m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName + + t := &arm.Template{ + Schema: "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + ContentVersion: "1.0.0.0", + Resources: []*arm.Resource{ + m.storageAccount(clusterStorageAccountName, m.doc.OpenShiftCluster.Location, false), + m.storageAccount(registryStorageAccountName, m.doc.OpenShiftCluster.Location, false), + }, + } + + return m.deployARMTemplate(ctx, resourceGroup, "storage", t, nil) +} + func (m *manager) populateRegistryStorageAccountName(ctx context.Context) error { if m.doc.OpenShiftCluster.Properties.ImageRegistryStorageAccountName != "" { return nil } - rc, err := m.registryclient.ImageregistryV1().Configs().Get(ctx, "cluster", metav1.GetOptions{}) + rc, err := m.imageregistrycli.ImageregistryV1().Configs().Get(ctx, "cluster", metav1.GetOptions{}) if err != nil { return err } diff --git a/pkg/deploy/bindata.go b/pkg/deploy/bindata.go index 20ca0350692..14802c223e3 100644 --- a/pkg/deploy/bindata.go +++ b/pkg/deploy/bindata.go @@ -276,7 +276,7 @@ func rbacDevelopmentJson() (*asset, error) { return a, nil } -var _rpDevelopmentPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x58\xdb\x6f\xdb\xb6\x17\x7e\xf7\x5f\x21\xf0\xf7\x03\xec\x0c\xbe\xc8\x5e\x0a\xac\x7e\x2b\xd0\x61\x08\x76\x69\xd0\x14\x7d\x31\x82\x80\xa6\x8e\x1d\x36\x14\x49\x90\x94\x5b\x6f\xc8\xff\x3e\x50\xb2\x7c\x91\xa8\x5b\x2b\x6f\xa9\x27\xe5\x21\xb0\x78\x2e\xe4\xe1\x77\xbe\x8f\xe2\x5f\x3d\xcf\xf3\x3c\xf4\x7f\x4d\x1e\x21\xc4\x68\xee\xa1\x47\x63\xa4\x9e\x4f\x26\xc9\x9b\x71\x88\x39\x5e\x43\x08\xdc\x8c\xf1\x9f\x91\x82\x31\x11\xe1\x6e\x4c\x4f\x66\xfe\xf4\xd5\xc8\x9f\x8e\xfc\xe9\x24\x00\xc9\xc4\xd6\xda\x7d\x80\x50\x32\x6c\x60\xfc\x49\x0b\xfe\x3f\x34\x4c\x32\x10\xc1\x0d\x70\xf3\x11\x94\xa6\x82\xdb\x44\xd3\xb1\x6f\xff\x52\x03\x89\x15\x0e\xc1\x80\xd2\x68\xee\x25\xd3\x8a\xdf\xe3\x20\xa4\xfc\xdd\xf2\x13\x10\x73\x13\x9c\x0c\xc5\xc3\x66\x2b\xc1\x46\xd3\x46\x51\xbe\x46\xfb\xc1\xe7\xe1\x21\xc4\x4a\xde\x81\xda\x50\x02\xb7\x8a\x72\x42\x25\x66\x5f\x1b\xe9\x09\xb6\x1b\x1c\x31\x73\xab\x60\x45\xbf\x54\xc6\x18\x9e\x8e\x86\xf8\xcb\x6f\xc0\xd7\xe6\x11\xcd\xbd\x99\xef\x4c\xa0\xbe\x6d\xaa\xbd\xa3\x78\x48\x81\x16\x91\x22\x60\x0b\xba\xd8\xdb\x64\x42\x49\x25\x24\x28\x43\x41\xe7\xd2\xc4\xe3\x1a\x48\xa4\xa8\xd9\xbe\x8f\x58\x26\xd0\xf1\x93\x77\xac\x9b\x20\x6b\x6b\x04\x11\xcc\xae\xed\x03\x91\x99\xfa\xe5\xe7\x16\x2f\xef\x56\x28\xf3\x1e\xf3\x75\x5c\x91\x1f\xaa\x7c\x02\xd0\x86\x72\x6c\xa8\xe0\x27\x8e\xd7\xd7\x3f\xd6\x4b\xf7\x26\x08\x14\x68\xbd\x47\x40\xa3\x94\xcd\x9d\x31\x21\xa0\x6d\xe9\xd0\x1b\xc6\xc4\xe7\x2a\x73\xa9\xa8\xb0\xdb\x85\xe6\xde\x74\xe6\x57\x4d\x8c\x2a\x20\x66\xd7\x8e\x37\x7c\x29\x22\x1e\xa0\x42\x97\xe7\xe2\x68\x88\xe3\x30\xae\xa2\x92\x0f\x94\x3f\x60\x15\xba\xc3\x14\x84\xf8\xfe\xd1\x33\x9b\x5d\x1c\x78\x5e\xfd\xe3\xe0\xd1\xfa\xf1\x81\xf2\x02\xe4\xe4\xde\xde\xf7\x4a\xe2\x1f\x01\x72\xc4\x75\x8e\x88\x53\xfe\xfc\x9d\x12\x25\xb4\x58\x99\xf1\x1f\x60\x3e\x0b\xf5\x34\xe1\xc9\xff\xbb\x1d\xeb\xfd\xa2\x44\x24\x75\xd6\x9d\x09\x82\xd3\x95\x2f\x52\x96\x8d\x4d\x07\x57\xe3\x74\xf0\x3e\xeb\x85\x25\x3d\x52\xbf\x99\x3f\xf3\x47\xfe\x4f\x23\x7f\xea\x54\x9a\x52\x96\x2e\x59\xac\x84\x0b\x5c\x6f\x5e\x95\x70\xd2\x0a\x77\x12\x13\x28\x24\x86\xd4\x2a\x69\x98\x12\xf5\x8a\x8d\xa7\x7e\x72\x1e\x99\xcc\xae\xdd\x10\xbc\xcf\xbd\x75\x80\x1a\xe9\x68\xc9\xc1\x9c\x5d\x29\x71\x96\x0b\x8e\xa7\x5f\xd1\xb9\xae\x3d\xaf\xcc\x18\x7b\xd2\xe0\x04\x02\x37\xc1\xa0\x5f\x13\x52\xfd\xa1\xd7\x4f\x7a\xb1\x7f\x95\x45\x8a\x33\x95\xc1\x6b\x5b\x06\x1e\x31\x56\x6a\x9c\x27\x86\xfd\x48\x2d\xbd\x1a\x25\x1b\xd6\x3a\xeb\x6c\x6c\xd0\xda\x6d\xb8\xa1\xca\x44\x98\xed\x7e\x9e\xbb\x01\x33\x76\x01\x48\xe0\x81\x7e\xc7\x9d\xa0\x6d\x61\xbb\x7b\xee\x0a\xbe\xbc\xf6\xbf\xb6\xfd\x33\xfb\x6e\xdb\x7f\x37\xfd\x97\xdb\xfe\x89\x3a\xb5\xcd\x00\xd5\xa7\x9a\x0d\x36\xf0\x33\x0f\xa4\xa0\xdc\xec\xe6\x79\x2b\x18\x25\x49\xad\xd1\x5b\xaa\xf1\x92\xc1\x37\x9f\x7d\xed\xfa\xce\x44\x27\x12\x62\x46\x19\xf9\xf9\xf6\xfd\xcf\xb1\xca\x01\x45\x8d\x89\x25\xad\xe8\x82\x08\x4e\xb0\x19\x1c\x2e\x1d\x06\xfd\xd3\x6f\xfb\xfe\xd5\xd0\xeb\x8f\x08\xd3\x79\xb4\x3a\x2a\xfe\x2b\x6c\x3f\x5a\xdf\x49\x1c\xa1\xa5\x4a\x57\xb1\xa0\x01\x8e\x79\x72\x29\x82\x16\x3a\x5a\x6a\xa2\xa8\xb4\x91\x06\x57\xe3\x74\xcc\xd5\x69\x48\x3f\x45\xc5\xac\xb9\xc2\x21\x65\xdb\xf8\x9b\xa1\xa0\x4b\x0f\x07\x76\x83\x79\x80\x95\xa3\x6f\x5c\xd4\x98\x7c\x8c\x1c\xf5\x5d\x63\x86\xfc\xda\x05\xef\x03\x88\xc3\x35\x12\x5a\x1c\xef\xbd\xeb\x86\xa8\x9c\xa6\x90\x04\x15\x52\x6d\xfb\xa0\x06\x5f\x6b\x20\xaa\x4c\x16\x4e\x8c\xd7\x45\xfc\x91\x3e\xf7\x15\x8c\x47\x2c\x6a\x56\x94\x60\x53\xa1\x7a\x07\x0f\x05\xd8\x40\x1d\x56\x0e\x80\x41\x3d\xcb\x75\xee\x00\xe4\x34\x8b\x64\x60\x53\x97\x2f\xb8\x98\x98\x5b\xba\x67\x38\x1f\xb4\x4e\xee\x2f\xdb\xc4\x54\xf3\x5d\xae\xb9\x23\x8c\xea\x2a\x00\x36\xdd\x8f\xbc\xf0\x39\x08\x02\xb8\x55\xe1\x3b\xb1\x32\x6f\x13\x90\xcd\x3d\xa3\x22\x28\x95\xc8\xac\x1a\x4d\x5f\x8f\xfc\xd7\x35\x3f\x32\x9b\x8a\x41\xb0\x34\x9d\x18\xe4\x4c\x2e\x4f\x0c\x5c\x77\xf0\x2f\x58\x0c\x3a\x6e\xcc\x1b\x37\x2a\xb1\x6e\x8d\x16\x5b\xd7\xe5\xd6\xd5\x96\x86\x52\xa8\x4e\x05\x8e\x7c\x9b\xaa\x80\x14\xaa\x53\x81\x9c\x49\xa7\x02\x9d\x0a\x74\x2a\x10\x3f\x9d\x0a\xec\x9e\x4b\x56\x01\xbd\x21\x9d\x0a\xe4\x4c\x3a\x15\x38\xbb\x0a\xfc\x3b\x3d\xdb\xa9\x45\xc6\xb8\xe6\x56\x74\xa2\xe2\xb2\xbc\x30\x51\xe9\x25\x33\x7e\xee\xfd\x1d\x00\x00\xff\xff\xe9\x07\x8c\x5b\x4d\x29\x00\x00") +var _rpDevelopmentPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x59\x5b\x6f\xdb\x36\x14\x7e\xf7\xaf\x10\xb8\x01\x76\x0a\x5f\x64\x2f\x05\x56\xbf\x15\xe8\x30\x04\xbb\x34\xa8\x8b\xbe\x18\x41\x40\x53\xc7\x0e\x1b\x8a\x24\x48\xca\xad\x37\xe4\xbf\x0f\x92\x2c\x5b\x96\xa8\x5b\x2b\x6f\x89\x27\xe6\x21\x88\x78\xce\x21\x79\xf8\x9d\xef\x23\x99\xbf\x7b\x8e\xe3\x38\xe8\x47\x4d\x1e\xc0\xc7\x68\xee\xa0\x07\x63\xa4\x9e\x4f\x26\xf1\x97\xb1\x8f\x39\xde\x80\x0f\xdc\x8c\xf1\x5f\x81\x82\x31\x11\xfe\xbe\x4f\x4f\x66\xee\xf4\xf5\xc8\x9d\x8e\xdc\xe9\xc4\x03\xc9\xc4\x2e\xb4\xfb\x08\xbe\x64\xd8\xc0\xf8\xb3\x16\xfc\x07\x34\x8c\x47\x20\x82\x1b\xe0\xe6\x13\x28\x4d\x05\x0f\x07\x9a\x8e\xdd\xf0\x27\x31\x90\x58\x61\x1f\x0c\x28\x8d\xe6\x4e\x3c\xad\xe8\x3b\xf6\x7c\xca\xdf\xaf\x3e\x03\x31\x37\xde\x49\x57\xd4\x6d\x76\x12\xc2\x68\xda\x28\xca\x37\xe8\xd0\xf9\x34\x3c\x86\x58\xcb\x05\xa8\x2d\x25\x70\xab\x28\x27\x54\x62\xf6\xad\x91\x1e\x61\xb7\xc5\x01\x33\xb7\x0a\xd6\xf4\x6b\x65\x8c\xe1\x69\xaf\x8f\xbf\xfe\x0e\x7c\x63\x1e\xd0\xdc\x99\xb9\xd6\x01\xd4\xf7\x4d\xb5\x97\x8a\x87\x14\x68\x11\x28\x02\x61\x42\x97\x07\x9b\x4c\x28\xa9\x84\x04\x65\x28\xe8\xdc\x30\x51\xbf\x06\x12\x28\x6a\x76\x1f\x02\x96\x09\x94\x6e\x79\xc7\xba\x03\x64\x6d\x8d\x20\x82\x85\x6b\xfb\x48\x64\x26\x7f\xf9\xb9\x45\xcb\xbb\x15\xca\x7c\xc0\x7c\x13\x65\xe4\x55\x95\x8f\x07\xda\x50\x8e\x0d\x15\xfc\xc4\xf1\xfa\xfa\xa7\x7a\xc3\xbd\xf5\x3c\x05\x5a\x1f\x10\xd0\x68\xc8\xe6\xce\x98\x10\xd0\x61\xea\xd0\x5b\xc6\xc4\x97\x2a\x73\xa9\xa8\x08\xb7\x0b\xcd\x9d\xe9\xcc\xad\x9a\x18\x55\x40\xcc\xbe\x1c\x6f\xf8\x4a\x04\xdc\x43\x85\x2e\x4f\xc5\xd1\x10\xc7\x7e\x94\x45\x25\xef\x29\xbf\xc7\xca\xb7\x87\x29\x08\xf1\xf2\xd1\x33\x9b\x5d\x1c\x78\x5e\xff\xeb\xe0\xd1\xfa\xe1\x9e\xf2\x02\xe4\xe4\xbe\xde\xf5\x4a\xe2\xa7\x00\x39\xe2\x3a\x47\xc4\x09\x7f\xfe\x41\x89\x12\x5a\xac\xcd\xf8\x4f\x30\x5f\x84\x7a\x9c\xf0\xf8\xf7\x62\xcf\x7a\xbf\x2a\x11\x48\x9d\x75\x67\x82\xe0\x64\xe5\xcb\x84\x65\x23\xd3\xc1\xd5\x38\xe9\xbc\xcb\x7a\x61\x49\x53\xea\x37\x73\x67\xee\xc8\xfd\x79\xe4\x4e\xad\x4a\x53\xca\xd2\x25\x8b\x95\x70\x81\xeb\xcd\xab\x12\x8e\x4b\x61\x21\x31\x81\x42\x62\x48\xac\xe2\x82\x29\x51\xaf\xc8\x78\xea\xc6\xe7\x91\xc9\xec\xda\x0e\xc1\xbb\xdc\x57\x0b\xa8\x91\x0e\x56\x1c\xcc\xd9\x95\x12\x67\xb9\x20\x3d\xfd\x8a\xca\xb5\xed\x79\xe5\x88\x91\x27\xf5\x4e\x20\x70\xe3\x0d\xfa\x35\x21\xd5\x1f\x3a\xfd\xb8\x16\xfb\x57\x59\xa4\x58\x87\x32\x78\x13\xa6\x81\x07\x8c\x95\x1a\xe7\x89\xe1\xd0\x53\x4b\xaf\x46\xf1\x86\xb5\xce\x3a\xdb\x30\x68\xed\x32\xdc\x52\x65\x02\xcc\xf6\x7f\x9e\xbb\x00\x33\x76\x1e\x48\xe0\x9e\x7e\xcf\xad\xa0\x6d\x61\xbb\x7b\xf6\x0c\x3e\xbf\xf2\xbf\x0e\xeb\x67\xf6\x62\xcb\x7f\x3f\xfd\xe7\x5b\xfe\xb1\x3a\xb5\xcd\x00\x55\x27\xaf\xf8\x42\xf5\x0b\xf7\xa4\xa0\xbc\x64\x6f\xd2\xad\x3a\x1b\xe9\xd8\xa7\x45\xbd\x30\x42\xe1\x0d\xd4\x58\xa3\x93\x2e\xee\x7a\x13\x3b\xb8\xbd\x2a\x3e\x6c\xa5\x5b\x1e\xb6\xd9\x56\xcc\xa0\x91\x7f\xf5\x99\x71\x8b\xcd\x21\xbb\x7b\x14\xdc\x0a\x46\x49\x8c\x64\xf4\x8e\x6a\xbc\x62\xf0\xdd\x37\x8b\x10\x3d\x67\x22\x6b\x09\x11\x5f\x8f\xdc\x3c\x39\xfe\xef\x38\xfb\x58\xa3\x8d\x69\x3b\xc9\xe8\x92\x08\x4e\xb0\x19\x1c\x9f\x74\x06\xfd\xd3\x97\x93\xfe\xd5\xd0\xe9\x8f\x08\xd3\x79\x2e\xb0\x64\xfc\x37\xd8\x7d\x0a\x7d\x27\x51\x84\x96\x32\x5d\xa5\x31\x06\x38\xe6\xf1\x93\x13\x5a\xea\x60\xa5\x89\xa2\x32\x8c\x34\xb8\x1a\x27\x7d\x36\x1e\x43\xfa\x31\x28\xd6\xa4\x35\xf6\x29\xdb\x45\x37\xb2\x02\x7e\x38\x5e\x87\x0c\xe6\x1e\x56\x96\xba\xb1\x09\x4f\x7c\xd5\x4b\xd5\x5d\x63\xfd\xf9\xd6\x05\x1f\x02\x88\xe3\x23\x1d\x5a\xa6\xf7\xde\xf6\xfe\x56\x2e\x02\x48\x82\xf2\xa9\xd6\x7b\x5a\xac\x50\x43\x0d\x44\x95\x89\xee\x89\xf1\xa6\x88\x3f\x92\x56\xc5\x78\x24\x44\xcd\x9a\x12\x6c\x2a\xce\x14\x47\x0f\x05\xd8\xd4\xd1\x03\xe4\x01\x83\x7a\x96\x9b\xdc\xf1\xd2\x6a\x16\x48\x2f\x1c\xba\x7c\xc1\xc5\xc4\xdc\xd2\x2b\xce\xf9\xa0\x75\xf2\x3a\xdc\x26\xa6\x9a\xef\x72\xcd\x1d\x61\x54\x57\x01\xb0\xe9\x7e\xe4\x85\xcf\x42\x10\xc0\x43\x15\x5e\x88\xb5\x79\x17\x83\x6c\xee\x18\x15\x40\xa9\x44\x66\xd5\x68\xfa\x66\xe4\xbe\xa9\x79\x85\x6f\x2a\x06\xde\xca\x74\x62\x90\x33\xb9\x3c\x31\xb0\xfd\x87\xe3\x19\x8b\x41\xc7\x8d\x79\xe3\x46\x29\xd6\xad\xd1\x62\xeb\xba\xdc\xba\xda\x52\x5f\x0a\xd5\xa9\x40\xca\xb7\xa9\x0a\x48\xa1\x3a\x15\xc8\x99\x74\x2a\xd0\xa9\x40\xa7\x02\x51\xeb\x54\x60\xdf\x2e\x59\x05\xf4\x96\x74\x2a\x90\x33\xe9\x54\xe0\xec\x2a\xf0\xdf\xd4\x6c\xa7\x16\x19\xe3\x9a\x5b\xd1\x89\x8a\xcd\xf2\xc2\x44\xa5\x17\xcf\xf8\xa9\xf7\x4f\x00\x00\x00\xff\xff\xc9\xd1\x85\xe4\xab\x2a\x00\x00") func rpDevelopmentPredeployJsonBytes() ([]byte, error) { return bindataRead( @@ -436,7 +436,7 @@ func rpProductionPredeployParametersJson() (*asset, error) { return a, nil } -var _rpProductionPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5a\x5f\x4f\x1b\x39\x10\x7f\xe7\x53\xac\xf6\x4e\x4a\xa8\x9a\x64\x93\x52\x89\xe6\xad\x2d\xa7\x0a\xf5\xca\xa1\xa6\xe2\x05\x21\xe4\x78\x27\xc1\xc5\x6b\x5b\xb6\x37\x90\xab\xf8\xee\x27\xef\x6e\xc2\xfe\xf1\x7a\x03\x24\x5c\x28\x59\x1e\x48\xb2\xe3\x19\xcf\xcf\x3f\xcf\x78\x6c\xff\xda\xf3\x3c\xcf\xf3\xff\x54\xf8\x0a\x22\xe4\x0f\x3d\xff\x4a\x6b\xa1\x86\xbd\x5e\xfa\x4b\x37\x42\x0c\x4d\x21\x02\xa6\xbb\xe8\xdf\x58\x42\x17\xf3\x28\x7b\xa7\x7a\x83\xa0\xff\xbe\x13\xf4\x3b\x41\xbf\x17\x82\xa0\x7c\x6e\xe4\x7e\x40\x24\x28\xd2\xd0\xfd\xa9\x38\xfb\xc3\x7f\x9b\x5a\xc0\x9c\x69\x60\xfa\x0c\xa4\x22\x9c\x19\x43\xfd\x6e\x60\xfe\x16\x02\x33\x24\x09\x1a\x53\x50\xfe\xd0\x4b\x7b\x95\xb6\xa3\xb1\xd2\x20\xbf\xc2\x7c\x86\x62\xaa\x3f\x62\x0c\x4a\x9d\x72\x4a\x30\x49\x44\xcf\x97\xa2\xe6\xf9\x55\xf8\x96\x28\xd0\xc0\x10\xd3\xc7\xa1\x31\x79\xae\xe2\xb1\xc2\x92\x08\x4d\x38\x6b\xef\x77\x17\xef\x2e\xb2\x4e\x14\x1a\xf2\xf1\x4f\xc0\x8b\x86\x02\x49\x14\x81\x06\xa9\xda\xad\x89\x18\x81\x9c\x11\x0c\xa7\x92\x30\x4c\x04\xa2\xc7\x61\x6b\xdf\xaa\x43\x80\x8c\x88\x32\x1e\x17\xdd\x2a\x08\x29\xc0\x12\x74\xd5\x99\x82\xd0\x14\xb4\x6f\x7d\x7b\x51\x35\x9b\x02\x07\x52\x93\x09\xc1\x48\x5b\x70\x2a\x4a\x4a\x40\x1a\x2c\xdd\x5f\x4a\x84\x40\xc1\x2d\x61\xba\xe7\x78\x1d\x8b\xd0\x98\xb0\x3b\x50\xf9\xf5\x6e\xcf\xfe\x2d\xe7\xaa\x1f\x8e\x7f\xf0\x6b\x60\xdb\xc4\x0b\xb9\x5d\xbc\x78\x14\xac\x82\x4b\x8d\xe8\x0e\xd5\xf5\xa2\xaa\x52\x0f\x76\xb0\xba\xa3\x04\x25\xea\xe9\xb0\x27\x9f\xee\xb2\xa4\x72\xef\x71\x31\xab\xa4\xd9\xea\x64\xf4\xa5\xea\x91\xaf\xe7\x02\x0c\x5c\x63\xce\x69\xa9\xb7\x7e\x08\x13\x33\x80\x67\x88\xc6\x46\x66\x82\xa8\x82\xa5\xc4\x5d\x6e\xc4\xe1\x56\x4b\xf4\xb9\x21\x77\xd5\x18\x46\x52\xa2\x79\x83\xe5\xf3\x8b\x7a\xb3\x47\x9f\xdc\xa1\x71\x43\x66\x4f\xdd\xa1\x63\x43\x56\x47\x0d\x53\x6b\xfd\x66\x6d\xe9\xbf\xde\x90\xd2\x92\xb0\xa9\x6f\xd5\x34\x45\x1a\x6e\xd0\xfc\x3b\x28\x1e\x4b\x0c\x5f\x24\x8f\xc5\x09\x8a\xe0\x71\xda\xae\x33\x08\x4e\x25\x4c\xc8\x6d\xa3\x8e\x92\xef\x11\xba\xfd\x1b\xd8\x54\x5f\xf9\x43\x6f\x10\x58\x0d\x48\x71\xa2\xa6\xa3\xa4\xab\x1f\xc3\x50\x1a\x9c\x13\x53\x1b\xc2\xd9\x16\xa1\x1e\x80\x4c\x21\x0e\xc8\x0c\xe2\x62\x7c\x2a\xa9\x12\x92\x0b\xb3\x68\xb2\xf8\xe3\x65\x21\x2e\x96\x44\xcf\xbf\xc7\xd4\xb1\xa4\xb2\xc7\xc7\x55\x0c\x94\x65\x35\xc7\x9c\x1a\xdf\x7e\x60\xe1\x08\x9b\x69\xdf\x12\xf7\xcc\x14\xfc\x8e\xd8\x34\x41\xe4\x4d\x53\x9b\x10\x94\x26\x0c\x99\x34\x52\x68\x78\x70\xf0\x6e\x35\x73\x05\x16\x98\x96\x1f\x4d\x85\xb0\xa0\xf3\xb7\xa4\x72\x90\x0f\xe8\x45\x45\x5f\xa3\x0b\x28\x99\xee\x89\x69\x4a\xf9\x4d\x93\xb8\x90\x84\x9b\x11\xf4\x87\x5e\x7f\x10\x34\x75\x8c\x48\xc0\x3a\x2b\x56\x8e\xd9\x98\xc7\x2c\xb4\x67\x27\xaf\xc8\xdc\x8a\x26\x96\xce\x69\x5f\x8a\x4b\xc2\x2e\x91\x8c\xec\x6a\x6a\x54\xfc\x0e\x84\x7a\xff\x48\x42\xf5\x4d\x89\x78\xd8\x0d\x7a\x83\x83\x6d\xe5\xd1\xc1\xf3\xf3\x28\x1c\x6b\x93\xe0\x0d\x99\xb2\x14\x72\x69\x20\x7e\x5d\xa4\x3a\xfc\xbd\x59\xd5\xdf\x06\x56\x1d\xbe\x3e\x5a\x35\x36\xdc\x1a\x4e\x1d\x01\x2b\xaf\xac\x2a\xd2\x05\x4a\xbd\x7f\x7e\x4a\x01\x9b\xe7\xf8\xf4\xca\xa8\xf4\xb8\x75\x54\xe2\x54\xb9\x56\xaf\x5b\x78\xdb\x2b\xf6\xba\xde\x3d\x6b\x04\x7b\xf7\x7f\xad\xaf\xa6\xc0\x60\x86\x6a\xb8\x56\xf9\xb5\xb8\xb3\x50\xb2\x92\xd3\xdc\x61\xaa\x52\x33\x2d\x6a\x8f\x6f\x04\x4b\xae\xf8\x44\x77\x4f\x40\xdf\x70\x79\xdd\x63\xe9\xff\x51\x56\x31\x24\x95\x9d\x2a\x37\xa7\x1c\xa3\x85\xff\xe7\x32\x5f\x04\xb6\xf7\xbb\x8b\x97\xe5\xf1\xf5\x31\x67\x21\x59\x36\xcb\xd3\xe4\x7e\x53\xa3\x4a\x0b\x1f\x09\x92\xdb\x7b\x1f\x04\x83\xa0\x13\x1c\x76\x82\xbe\xb5\x96\x74\x16\x46\x0e\x8c\x04\xec\x60\xaa\xaf\x1f\x51\x3a\xfd\x46\x02\xe1\x6a\x89\x5f\x96\xca\x05\x03\xc7\x86\x5a\x92\x75\x82\x34\xeb\x58\xa5\x2c\x1b\x67\x96\x1d\x3e\x15\x8f\x99\x6b\xf3\x6e\x4d\xb1\x18\x59\x93\x66\xb0\x52\xd2\xb4\x51\xa5\xd1\x62\xd2\x92\x84\x05\xe6\x1c\x87\xed\xd6\x8a\x4c\x6c\xbd\xf5\x5a\xe9\xcc\x6f\x8e\xb3\x89\x29\x8d\xa6\x06\x06\x16\x53\xea\x14\x76\x04\x33\x2f\xb7\x5f\xfc\x17\x0b\x05\x27\xac\x61\x57\x75\xf1\x34\x43\x91\xd7\x5d\x9c\x90\x5f\x61\x7e\x86\x62\xea\xda\x9d\x2d\x28\x59\x4c\xbb\xd5\x7a\xb6\x6c\xf6\xa6\x3e\xbe\xe7\x9f\x2a\x69\xcb\x4f\x03\x80\xde\x13\xe1\x48\xf6\x31\x3e\x73\x15\x71\x75\xf4\xe9\xc5\x60\xe2\x94\xa8\x6f\xbf\x5a\x66\xed\xa4\x31\x62\xed\x69\x75\xc6\x2a\x67\x02\x8e\x84\x31\x23\x52\xc7\x88\x66\x5f\xd7\x94\x2a\x6a\x63\x7e\x65\xdb\x52\x00\x0b\xd5\x3f\xcc\x3a\xc0\x6b\x88\x30\x7b\x76\x04\xb7\x2f\xe3\x1c\x98\x90\x3d\x78\xb1\x19\x27\xeb\xfe\xf6\x66\x9c\x74\x1d\xf5\xcc\x49\x47\x48\x32\x43\x7a\x99\x74\xb2\x7e\xe6\x8e\x52\xfc\x23\xa2\xd0\x98\xc2\x93\x57\xe9\xc6\xbf\x0d\x85\x13\x01\x49\x44\xe9\x04\xd5\xe9\xfb\xea\xa2\xca\x3d\x8b\x1e\x1c\x58\x16\x88\x9e\x63\xce\x30\xd2\xed\xfc\xfa\xb9\x78\xce\xd4\xda\x7f\xeb\xb5\x3a\x98\xda\x16\xd3\x55\xc4\x17\xeb\x8c\x5e\xa2\x61\x4d\x48\x37\x45\xc1\x47\x1f\x9e\xab\xeb\xb8\x3e\x6a\x4e\x50\x44\xe8\x3c\x29\x8d\x6b\x66\xe9\x12\x45\xa5\x11\x0b\x91\xb4\xcc\x1b\x5b\x68\x44\xe5\x23\xcc\xe5\x28\x2c\xaf\x48\xb5\x5b\xce\x6b\x51\x66\x4c\xf2\x23\xd6\x7c\x18\xdd\xda\xb7\x1f\xfd\x03\x33\xf6\x46\x7c\xa2\x8f\xd2\x2b\x40\x43\x4f\xcb\x18\x9c\xb3\xb1\x4c\xfc\xfe\x87\x4e\xf0\x61\xc5\x12\xea\xa1\xbc\x0b\xc7\x7a\xc7\xbb\x8a\xc8\x06\x79\xe7\xbc\x76\x65\xe5\x9d\xf3\x36\xc2\x4b\xe5\x9d\xe0\x72\xc7\xbb\x8a\xc8\x06\x79\xe7\xba\x97\x66\xa5\x9d\xeb\x36\xca\x4b\x65\x9d\x9a\xe1\x1d\xeb\x2a\x22\x1b\x64\x9d\xf3\xde\x9e\x95\x76\xce\xeb\x48\xcf\xcc\xbb\xe4\xd3\xc5\xde\xdd\xde\x7f\x01\x00\x00\xff\xff\xa8\x8f\xbc\x95\xe5\x2d\x00\x00") +var _rpProductionPredeployJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x5a\xdf\x4f\xdb\xba\x17\x7f\xe7\xaf\x88\xf2\xfd\x4a\x2d\xd3\xda\xa6\x1d\x93\x58\xdf\xb6\x71\x35\xa1\xdd\x71\xd1\x3a\xed\x05\x21\xe4\x3a\xa7\xc5\xc3\xb1\x2d\xdb\x29\xf4\x4e\xfc\xef\x57\x4e\xd2\x92\x1f\x8e\x53\xa0\x65\x05\x1a\x1e\x68\x92\xe3\x73\x7c\x3e\xfe\xf8\x1c\x1f\xc7\xbf\xf7\x3c\xcf\xf3\xfc\xff\x2b\x7c\x09\x11\xf2\x87\x9e\x7f\xa9\xb5\x50\xc3\x5e\x2f\x7d\xd2\x8d\x10\x43\x53\x88\x80\xe9\x2e\xfa\x37\x96\xd0\xc5\x3c\xca\xde\xa9\xde\x20\xe8\xbf\xef\x04\xfd\x4e\xd0\xef\x85\x20\x28\x9f\x1b\xb9\x1f\x10\x09\x8a\x34\x74\x7f\x29\xce\xfe\xe7\xbf\x4d\x2d\x60\xce\x34\x30\xfd\x13\xa4\x22\x9c\x19\x43\xfd\x6e\x60\xfe\x16\x02\x33\x24\x09\x1a\x53\x50\xfe\xd0\x4b\x7b\x95\xb6\xa3\xb1\xd2\x20\xbf\xc2\x7c\x86\x62\xaa\x3f\x62\x0c\x4a\x9d\x72\x4a\x30\x49\x44\xcf\x96\xa2\xe6\xfa\x5d\xb8\x4b\x14\x68\x60\x88\xe9\xe3\xd0\x98\x3c\x53\xf1\x58\x61\x49\x84\x26\x9c\xb5\xf7\xbb\x8b\x77\xe7\x59\x27\x0a\x0d\xf9\xf8\x17\xe0\x45\x43\x81\x24\x8a\x40\x83\x54\xed\xd6\x44\x8c\x40\xce\x08\x86\x53\x49\x18\x26\x02\xd1\xe3\xb0\xb5\x6f\xd5\x21\x40\x46\x44\x19\x8f\x8b\x6e\x15\x84\x14\x60\x09\xba\xea\x4c\x41\x68\x0a\xda\xb7\xbe\x3d\xaf\x9a\x4d\x81\x03\xa9\xc9\x84\x60\xa4\x2d\x38\x15\x25\x25\x20\x0d\x96\xee\x2f\x25\x42\xa0\xe0\x96\x30\xdd\x73\xbc\x8e\x45\x68\x4c\xd8\x1d\xa8\x3c\xbd\xdd\xb3\xdf\xe5\x5c\xf5\xc3\xf1\x0f\x7e\x05\x6c\x9b\x78\x21\xb7\x8b\x17\x0f\x82\x55\x70\xa9\x11\xdd\xa1\xba\x5e\x54\x55\xea\xc1\x0e\x56\x77\x94\xa0\x44\x3d\x1e\xf6\xe4\xd7\x6d\x96\x54\xee\x3c\x2e\x66\x95\x34\x5b\x9d\x8c\xbe\x54\x3d\xf2\xf5\x5c\x80\x81\x6b\xcc\x39\x2d\xf5\xd6\x0f\x61\x62\x06\xf0\x27\xa2\xb1\x91\x99\x20\xaa\x60\x29\x71\x9b\x1b\x71\xb8\xd1\x12\x7d\x6e\xc8\x5d\x35\x86\x91\x94\x68\xde\x60\xf9\xec\xbc\xde\xec\xd1\x27\x77\x68\xdc\x90\xd9\x53\x77\xe8\xd8\x90\xd5\x51\xc3\xd4\x5a\xbf\x59\x5b\xfa\xaf\x37\xa4\xb4\x24\x6c\xea\x5b\x35\x4d\x91\x86\x6b\x34\xff\x0e\x8a\xc7\x12\xc3\x17\xc9\x63\x71\x82\x22\x78\x98\xb6\xab\x0c\x82\x53\x09\x13\x72\xd3\xa8\xa3\xe4\x7b\x84\x6e\xfe\x06\x36\xd5\x97\xfe\xd0\x1b\x04\x56\x03\x52\x9c\xa8\xe9\x28\xe9\xea\xc7\x30\x94\x06\xe7\xc4\xd4\x86\x70\xb6\x45\xa8\x7b\x20\x53\x88\x03\x32\x83\xb8\x18\x9f\x4a\xaa\x84\xe4\xc2\x2c\x9a\x2c\xfe\x78\x59\x88\x8b\x25\xd1\xf3\xef\x31\x75\x2c\xa9\xec\xf1\x71\x15\x03\x65\x59\xcd\x31\xa7\xc6\xb7\x1f\x58\x38\xc2\x66\xda\xb7\xc4\x3d\x33\x05\xbf\x23\x36\x4d\x10\x79\xd3\xd4\x26\x04\xa5\x09\x43\x26\x8d\x14\x1a\x1e\x1c\xbc\x5b\xcd\x5c\x81\x05\xa6\xe5\x47\x53\x21\x2c\xe8\xfc\x2d\xa9\x1c\xe4\x3d\x7a\x51\xd1\xd7\xe8\x02\x4a\xa6\x7b\x62\x9a\x52\x7e\xdd\x24\x2e\x24\xe1\x66\x04\xfd\xa1\xd7\x1f\x04\x4d\x1d\x23\x12\xb0\xce\x8a\x95\x63\x36\xe6\x31\x0b\xed\xd9\xc9\x2b\x32\xb7\xa2\x89\xa5\x73\xda\x97\xe2\x82\xb0\x0b\x24\x23\xbb\x9a\x1a\x15\x2f\x81\x50\xef\x1f\x48\xa8\xbe\x29\x11\x0f\xbb\x41\x6f\x70\xb0\xad\x3c\x3a\x78\x7a\x1e\x85\x63\x6d\x12\xbc\x21\x53\x96\x42\x2e\x0c\xc4\xaf\x8b\x54\x87\x2f\x9b\x55\xfd\x6d\x60\xd5\xe1\xeb\xa3\x55\x63\xc3\xad\xe1\xd4\x11\xb0\xf2\xca\xaa\x22\x5d\xa0\xd4\xfb\xa7\xa7\x14\xb0\x79\x8e\x4f\xaf\x8c\x4a\x0f\x5b\x47\x25\x4e\x95\x6b\xf5\xba\x85\xb7\xbd\x62\xaf\xeb\xdd\x93\x46\xb0\x77\x7f\x6a\x7d\x35\x05\x06\x33\x54\xc3\xb5\xca\xd3\xe2\xce\x42\xc9\x4a\x4e\x73\x87\xa9\x4a\xcd\xb4\xa8\x3d\xbe\x11\x2c\xb9\xe2\x13\xdd\x3d\x01\x7d\xcd\xe5\x55\x8f\xa5\xff\x47\x59\xc5\x90\x54\x76\xaa\xdc\x9c\x72\x8c\x16\xfe\x9f\xc9\x7c\x11\xd8\xde\xef\x2e\x5e\x96\xc7\xd7\xc7\x9c\x85\x64\xd9\x2c\x4f\x93\xbb\x4d\x8d\x2a\x2d\x7c\x24\x48\x6e\xef\x7d\x10\x0c\x82\x4e\x70\xd8\x09\xfa\xd6\x5a\xd2\x59\x18\x39\x30\x12\xb0\x83\xa9\xbe\x7e\x44\xe9\xf4\x1b\x09\x84\xab\x25\x7e\x59\x2a\x17\x0c\x1c\x1b\x6a\x49\xd6\x09\xd2\xac\x63\x95\xb2\x6c\x9c\x59\x76\xf8\x54\x3c\x66\xae\xcd\xbb\x35\xc5\x62\x64\x4d\x9a\xc1\x4a\x49\xd3\x46\x95\x46\x8b\x49\x4b\x12\x16\x98\x73\x1c\xb6\x5b\x2b\x32\xb1\xf5\xd6\x6b\xa5\x33\xbf\x39\xce\x26\xa6\x34\x9a\x1a\x18\x58\x4c\xa9\x53\xd8\x11\xcc\xbc\xdc\x7e\xf1\x5f\x2c\x14\x9c\xb0\x86\x5d\xd5\xc5\xd5\x0c\x45\x5e\x77\x71\x42\x7e\x85\xf9\x4f\x14\x53\xd7\xee\x6c\x41\xc9\x62\xda\xad\xd6\xb3\x65\xb3\x37\xf5\xf1\x3d\x7f\x55\x49\x5b\xbe\x1a\x00\xf4\x1e\x09\x47\xb2\x8f\xf1\x99\xab\x88\xab\xa3\x4f\x3b\x4c\x12\x4c\x46\x9a\x4b\x34\x75\x7d\x04\x2c\xe8\xf8\xe3\x68\x38\x25\xea\xdb\xaf\xb6\xce\xe8\xa4\x11\x73\xed\x8b\x8c\x19\xab\x7c\x21\x71\xa4\xcf\x19\x91\x3a\x46\x34\xbb\x5d\x53\xe2\xac\xcd\x80\x95\x4d\x5c\x01\x2c\x54\xff\x30\xeb\x00\xaf\x21\xde\xee\xd9\x11\xdc\xbe\xfc\x7b\x60\x12\xd8\xe0\xd9\xe6\xdf\xac\xfb\xdb\x9b\x7f\xd3\x55\xe5\x4b\x4f\xc1\x2f\x2b\xbe\x36\x17\x8a\x33\xa4\x97\xe8\x66\x2c\xc8\x7d\xb6\xf3\x8f\x88\x42\x63\x0a\x8f\xae\x08\x0d\x7b\x36\x14\xac\x05\x24\xf1\xba\x13\x54\x83\xe3\xab\x8b\xd9\x77\x73\xf4\xde\x61\x7b\x81\xe8\x19\xe6\x0c\x23\xdd\xce\xd7\x6a\xc5\x6f\x9a\xad\xfd\xb7\x5e\xab\x83\xa9\xad\x70\xab\x22\xbe\x58\xd3\xf6\x12\x0d\x6b\x42\xba\x29\xc7\x3c\xf8\xa0\x86\xba\x8a\xeb\x73\xd2\x04\x45\x84\xce\x93\x6d\x98\x9a\xf8\xb0\x44\x51\x69\xc4\x42\x24\x2d\xf3\xc6\x96\x78\x50\xf9\x73\xf9\x72\x14\x96\xc7\xf1\xda\x2d\xe7\x11\x3c\x33\x26\xf9\x11\x6b\x3e\xf8\xd0\xda\xb7\x1f\x33\x01\x66\xec\x8d\xf8\x44\x1f\xa5\xc7\xcd\x86\x9e\x96\x31\x38\x67\x63\x99\xf8\xfd\x0f\x9d\xe0\xc3\x8a\xe5\xfa\x7d\x79\x17\x8e\xf5\x8e\x77\x15\x91\x0d\xf2\xce\x79\xc4\xcf\xca\x3b\xe7\xc9\x97\xe7\xca\x3b\xc1\xe5\x8e\x77\x15\x91\x0d\xf2\xce\x75\x06\xd2\x4a\x3b\xd7\xc9\xa7\xe7\xca\x3a\x35\xc3\x3b\xd6\x55\x44\x36\xc8\x3a\xe7\x19\x51\x2b\xed\x9c\x47\xdf\x9e\x98\x77\xc9\xaf\xf3\xbd\xdb\xbd\xff\x02\x00\x00\xff\xff\x87\x9e\x67\x0d\x51\x30\x00\x00") func rpProductionPredeployJsonBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/deploy/generator/resources_rp.go b/pkg/deploy/generator/resources_rp.go index 4fc621f754c..0c1b1707e37 100644 --- a/pkg/deploy/generator/resources_rp.go +++ b/pkg/deploy/generator/resources_rp.go @@ -156,6 +156,10 @@ func (g *generator) rpVnet() *arm.Resource { Service: to.StringPtr("Microsoft.AzureCosmosDB"), Locations: &[]string{"*"}, }, + { + Service: to.StringPtr("Microsoft.Storage"), + Locations: &[]string{"*"}, + }, } } @@ -171,6 +175,12 @@ func (g *generator) rpPEVnet() *arm.Resource { ID: to.StringPtr("[resourceId('Microsoft.Network/networkSecurityGroups', 'rp-pe-nsg')]"), }, PrivateEndpointNetworkPolicies: to.StringPtr("Disabled"), + ServiceEndpoints: &[]mgmtnetwork.ServiceEndpointPropertiesFormat{ + { + Service: to.StringPtr("Microsoft.Storage"), + Locations: &[]string{"*"}, + }, + }, }, Name: to.StringPtr("rp-pe-subnet"), }, diff --git a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go index f41c78b16b9..daead04a8c9 100644 --- a/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go +++ b/pkg/operator/apis/aro.openshift.io/v1alpha1/cluster_types.go @@ -83,6 +83,7 @@ type ClusterSpec struct { AZEnvironment string `json:"azEnvironment,omitempty"` Location string `json:"location,omitempty"` InfraID string `json:"infraId,omitempty"` + StorageSuffix string `json:"storageSuffix,omitempty"` ArchitectureVersion int `json:"architectureVersion,omitempty"` GenevaLogging GenevaLoggingSpec `json:"genevaLogging,omitempty"` InternetChecker InternetCheckerSpec `json:"internetChecker,omitempty"` @@ -92,6 +93,7 @@ type ClusterSpec struct { GatewayDomains []string `json:"gatewayDomains,omitempty"` GatewayPrivateEndpointIP string `json:"gatewayPrivateEndpointIP,omitempty"` Banner Banner `json:"banner,omitempty"` + ServiceSubnets []string `json:"serviceSubnets,omitempty"` // OperatorFlags defines feature gates for the ARO Operator OperatorFlags OperatorFlags `json:"operatorflags,omitempty"` diff --git a/pkg/operator/apis/aro.openshift.io/v1alpha1/zz_generated.deepcopy.go b/pkg/operator/apis/aro.openshift.io/v1alpha1/zz_generated.deepcopy.go index 9a2ed694ca2..7e830da7b89 100644 --- a/pkg/operator/apis/aro.openshift.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/operator/apis/aro.openshift.io/v1alpha1/zz_generated.deepcopy.go @@ -94,6 +94,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { copy(*out, *in) } out.Banner = in.Banner + if in.ServiceSubnets != nil { + in, out := &in.ServiceSubnets, &out.ServiceSubnets + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.OperatorFlags != nil { in, out := &in.OperatorFlags, &out.OperatorFlags *out = make(OperatorFlags, len(*in)) diff --git a/pkg/operator/controllers/const.go b/pkg/operator/controllers/const.go index 5f31009f3a7..b482873050b 100644 --- a/pkg/operator/controllers/const.go +++ b/pkg/operator/controllers/const.go @@ -22,4 +22,5 @@ const ( MachineControllerName = "Machine" MachineSetControllerName = "MachineSet" ImageConfigControllerName = "ImageConfig" + StorageAccountsControllerName = "StorageAccounts" ) diff --git a/pkg/operator/controllers/storageaccounts/storageaccount_controller.go b/pkg/operator/controllers/storageaccounts/storageaccount_controller.go new file mode 100644 index 00000000000..c12302a1915 --- /dev/null +++ b/pkg/operator/controllers/storageaccounts/storageaccount_controller.go @@ -0,0 +1,126 @@ +package storageaccounts + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/go-autorest/autorest/azure" + imageregistryclient "github.com/openshift/client-go/imageregistry/clientset/versioned" + machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" + maoclient "github.com/openshift/machine-api-operator/pkg/generated/clientset/versioned" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + aroclient "github.com/Azure/ARO-RP/pkg/operator/clientset/versioned" + "github.com/Azure/ARO-RP/pkg/operator/controllers" + "github.com/Azure/ARO-RP/pkg/util/azureclient" + "github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage" + "github.com/Azure/ARO-RP/pkg/util/clusterauthorizer" + "github.com/Azure/ARO-RP/pkg/util/subnet" +) + +const ( + CONFIG_NAMESPACE string = "aro.storageaccounts" + ENABLED string = CONFIG_NAMESPACE + ".enabled" +) + +// Reconciler is the controller struct +type Reconciler struct { + log *logrus.Entry + + arocli aroclient.Interface + kubernetescli kubernetes.Interface + maocli maoclient.Interface + imageregistrycli imageregistryclient.Interface +} + +// reconcileManager is instance of manager instantiated per request +type reconcileManager struct { + log *logrus.Entry + + instance *arov1alpha1.Cluster + subscriptionID string + + imageregistrycli imageregistryclient.Interface + kubeSubnets subnet.KubeManager + storage storage.AccountsClient +} + +// NewReconciler creates a new Reconciler +func NewReconciler(log *logrus.Entry, arocli aroclient.Interface, maocli maoclient.Interface, kubernetescli kubernetes.Interface, imageregistrycli imageregistryclient.Interface) *Reconciler { + return &Reconciler{ + log: log, + arocli: arocli, + kubernetescli: kubernetescli, + imageregistrycli: imageregistrycli, + maocli: maocli, + } +} + +// Reconcile ensures the firewall is set on storage accounts as per user subnets +func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + instance, err := r.arocli.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{}) + if err != nil { + return reconcile.Result{}, err + } + + if !instance.Spec.OperatorFlags.GetSimpleBoolean(ENABLED) { + // controller is disabled + return reconcile.Result{}, nil + } + + // Get endpoints from operator + azEnv, err := azureclient.EnvironmentFromName(instance.Spec.AZEnvironment) + if err != nil { + return reconcile.Result{}, err + } + + resource, err := azure.ParseResourceID(instance.Spec.ResourceID) + if err != nil { + return reconcile.Result{}, err + } + + // create refreshable authorizer from token + authorizer, err := clusterauthorizer.NewAzRefreshableAuthorizer(ctx, r.log, &azEnv, r.kubernetescli) + if err != nil { + return reconcile.Result{}, err + } + + manager := reconcileManager{ + log: r.log, + instance: instance, + subscriptionID: resource.SubscriptionID, + + imageregistrycli: r.imageregistrycli, + kubeSubnets: subnet.NewKubeManager(r.maocli, resource.SubscriptionID), + storage: storage.NewAccountsClient(&azEnv, resource.SubscriptionID, authorizer), + } + + return reconcile.Result{}, manager.reconcileAccounts(ctx) +} + +// SetupWithManager creates the controller +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + aroClusterPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == arov1alpha1.SingletonClusterName + }) + + return ctrl.NewControllerManagedBy(mgr). + For(&arov1alpha1.Cluster{}, builder.WithPredicates(aroClusterPredicate)). + Watches(&source.Kind{Type: &machinev1beta1.Machine{}}, &handler.EnqueueRequestForObject{}). // to reconcile on machine replacement + Watches(&source.Kind{Type: &corev1.Node{}}, &handler.EnqueueRequestForObject{}). // to reconcile on node status change + Named(controllers.StorageAccountsControllerName). + Complete(r) +} diff --git a/pkg/operator/controllers/storageaccounts/storageaccounts.go b/pkg/operator/controllers/storageaccounts/storageaccounts.go new file mode 100644 index 00000000000..f7ee569196d --- /dev/null +++ b/pkg/operator/controllers/storageaccounts/storageaccounts.go @@ -0,0 +1,86 @@ +package storageaccounts + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "strings" + + mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/go-autorest/autorest/to" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Azure/ARO-RP/pkg/util/stringutils" +) + +func (r *reconcileManager) reconcileAccounts(ctx context.Context) error { + resourceGroup := stringutils.LastTokenByte(r.instance.Spec.ClusterResourceGroupID, '/') + + subnets, err := r.kubeSubnets.List(ctx) + if err != nil { + return err + } + + serviceSubnets := r.instance.Spec.ServiceSubnets + for _, subnet := range subnets { + serviceSubnets = append(serviceSubnets, subnet.ResourceID) + } + + rc, err := r.imageregistrycli.ImageregistryV1().Configs().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + storageAccounts := []string{ + "cluster" + r.instance.Spec.StorageSuffix, // this is our creation, so name is deterministic + rc.Spec.Storage.Azure.AccountName, + } + + for _, accountName := range storageAccounts { + var changed bool + + account, err := r.storage.GetProperties(ctx, resourceGroup, accountName, "") + if err != nil { + return err + } + + for _, subnet := range serviceSubnets { + // if subnet ResourceID was found and we need to append + found := false + + if account.AccountProperties.NetworkRuleSet != nil && account.AccountProperties.NetworkRuleSet.VirtualNetworkRules != nil { + for _, rule := range *account.AccountProperties.NetworkRuleSet.VirtualNetworkRules { + if strings.EqualFold(to.String(rule.VirtualNetworkResourceID), subnet) { + found = true + break + } + } + } + + // if rule was not found - we add it + if !found { + *account.AccountProperties.NetworkRuleSet.VirtualNetworkRules = append(*account.AccountProperties.NetworkRuleSet.VirtualNetworkRules, mgmtstorage.VirtualNetworkRule{ + VirtualNetworkResourceID: to.StringPtr(subnet), + Action: mgmtstorage.Allow, + }) + changed = true + } + } + + if changed { + sa := mgmtstorage.AccountUpdateParameters{ + AccountPropertiesUpdateParameters: &mgmtstorage.AccountPropertiesUpdateParameters{ + NetworkRuleSet: account.AccountProperties.NetworkRuleSet, + }, + } + + _, err = r.storage.Update(ctx, resourceGroup, accountName, sa) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/pkg/operator/controllers/storageaccounts/storageaccounts_test.go b/pkg/operator/controllers/storageaccounts/storageaccounts_test.go new file mode 100644 index 00000000000..9ccc247a79a --- /dev/null +++ b/pkg/operator/controllers/storageaccounts/storageaccounts_test.go @@ -0,0 +1,250 @@ +package storageaccounts + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "strconv" + "testing" + + mgmtstorage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + imageregistryv1 "github.com/openshift/api/imageregistry/v1" + imageregistryclient "github.com/openshift/client-go/imageregistry/clientset/versioned" + imageregistryfake "github.com/openshift/client-go/imageregistry/clientset/versioned/fake" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + mock_storage "github.com/Azure/ARO-RP/pkg/util/mocks/azureclient/mgmt/storage" + mock_subnet "github.com/Azure/ARO-RP/pkg/util/mocks/subnet" + "github.com/Azure/ARO-RP/pkg/util/subnet" +) + +var ( + subscriptionId = "0000000-0000-0000-0000-000000000000" + clusterResourceGroupName = "aro-iljrzb5a" + clusterResourceGroupId = "/subscriptions/" + subscriptionId + "/resourcegroups/" + clusterResourceGroupName + vnetResourceGroup = "vnet-rg" + vnetName = "vnet" + subnetNameWorker = "worker" + subnetNameMaster = "master" + + storageSuffix = "random-suffix" + clusterStorageAccountName = "cluster" + storageSuffix + registryStorageAccountName = "image-registry-account" + + resourceIdMaster = "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster + resourceIdWorker = "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker +) + +func getValidClusterInstance(operatorFlag bool) *arov1alpha1.Cluster { + return &arov1alpha1.Cluster{ + Spec: arov1alpha1.ClusterSpec{ + ClusterResourceGroupID: clusterResourceGroupId, + StorageSuffix: storageSuffix, + OperatorFlags: arov1alpha1.OperatorFlags{ + ENABLED: strconv.FormatBool(operatorFlag), + }, + }, + } +} + +func getValidAccount(virtualNetworkResourceIDs []string) *mgmtstorage.Account { + account := &mgmtstorage.Account{ + AccountProperties: &mgmtstorage.AccountProperties{ + NetworkRuleSet: &mgmtstorage.NetworkRuleSet{ + VirtualNetworkRules: &[]mgmtstorage.VirtualNetworkRule{}, + }, + }, + } + + for _, rule := range virtualNetworkResourceIDs { + *account.AccountProperties.NetworkRuleSet.VirtualNetworkRules = append(*account.AccountProperties.NetworkRuleSet.VirtualNetworkRules, mgmtstorage.VirtualNetworkRule{ + VirtualNetworkResourceID: to.StringPtr(rule), + Action: mgmtstorage.Allow, + }) + } + return account +} + +func TestReconcileManager(t *testing.T) { + log := logrus.NewEntry(logrus.StandardLogger()) + + for _, tt := range []struct { + name string + mocks func(*mock_storage.MockAccountsClient, *mock_subnet.MockKubeManager) + imageregistrycli imageregistryclient.Interface + instance func(*arov1alpha1.Cluster) + operatorFlag bool + wantErr error + }{ + { + name: "Operator Flag enabled - nothing to do", + operatorFlag: true, + mocks: func(storage *mock_storage.MockAccountsClient, kubeSubnet *mock_subnet.MockKubeManager) { + // cluster subnets + kubeSubnet.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: resourceIdMaster, + IsMaster: true, + }, + { + ResourceID: resourceIdWorker, + IsMaster: false, + }, + }, nil) + + // storage objects in azure + result := getValidAccount([]string{resourceIdMaster, resourceIdWorker}) + storage.EXPECT().GetProperties(gomock.Any(), clusterResourceGroupName, clusterStorageAccountName, gomock.Any()).Return(*result, nil) + storage.EXPECT().GetProperties(gomock.Any(), clusterResourceGroupName, registryStorageAccountName, gomock.Any()).Return(*result, nil) + + }, + imageregistrycli: imageregistryfake.NewSimpleClientset( + &imageregistryv1.Config{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: imageregistryv1.ImageRegistrySpec{ + Storage: imageregistryv1.ImageRegistryConfigStorage{ + Azure: &imageregistryv1.ImageRegistryConfigStorageAzure{ + AccountName: registryStorageAccountName, + }, + }, + }, + }, + ), + }, + { + name: "Operator Flag disabled - nothing to do", + operatorFlag: false, + mocks: func(storage *mock_storage.MockAccountsClient, kubeSubnet *mock_subnet.MockKubeManager) { + // cluster subnets + kubeSubnet.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: resourceIdMaster, + IsMaster: true, + }, + { + ResourceID: resourceIdWorker, + IsMaster: false, + }, + }, nil) + + // storage objects in azure + result := getValidAccount([]string{resourceIdMaster, resourceIdWorker}) + storage.EXPECT().GetProperties(gomock.Any(), clusterResourceGroupName, clusterStorageAccountName, gomock.Any()).Return(*result, nil) + storage.EXPECT().GetProperties(gomock.Any(), clusterResourceGroupName, registryStorageAccountName, gomock.Any()).Return(*result, nil) + + }, + imageregistrycli: imageregistryfake.NewSimpleClientset( + &imageregistryv1.Config{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: imageregistryv1.ImageRegistrySpec{ + Storage: imageregistryv1.ImageRegistryConfigStorage{ + Azure: &imageregistryv1.ImageRegistryConfigStorageAzure{ + AccountName: registryStorageAccountName, + }, + }, + }, + }, + ), + }, + { + name: "Operator Flag enabled - all rules to all accounts", + operatorFlag: true, + mocks: func(storage *mock_storage.MockAccountsClient, kubeSubnet *mock_subnet.MockKubeManager) { + + // cluster subnets + kubeSubnet.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: resourceIdMaster, + IsMaster: true, + }, + { + ResourceID: resourceIdWorker, + IsMaster: false, + }, + }, nil) + + // storage objects in azure + result := getValidAccount([]string{}) + updated := mgmtstorage.AccountUpdateParameters{ + AccountPropertiesUpdateParameters: &mgmtstorage.AccountPropertiesUpdateParameters{ + NetworkRuleSet: getValidAccount([]string{resourceIdMaster, resourceIdWorker}).NetworkRuleSet, + }, + } + + storage.EXPECT().GetProperties(gomock.Any(), clusterResourceGroupName, clusterStorageAccountName, gomock.Any()).Return(*result, nil) + storage.EXPECT().Update(gomock.Any(), clusterResourceGroupName, clusterStorageAccountName, gomock.Eq(updated)) + + // we can't reuse these from above due to fact how gomock handles objects. + // they are modified by the functions so they are not the same anymore + result = getValidAccount([]string{}) + updated = mgmtstorage.AccountUpdateParameters{ + AccountPropertiesUpdateParameters: &mgmtstorage.AccountPropertiesUpdateParameters{ + NetworkRuleSet: getValidAccount([]string{resourceIdMaster, resourceIdWorker}).NetworkRuleSet, + }, + } + + storage.EXPECT().GetProperties(gomock.Any(), clusterResourceGroupName, registryStorageAccountName, gomock.Any()).Return(*result, nil) + storage.EXPECT().Update(gomock.Any(), clusterResourceGroupName, registryStorageAccountName, gomock.Eq(updated)) + }, + imageregistrycli: imageregistryfake.NewSimpleClientset( + &imageregistryv1.Config{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: imageregistryv1.ImageRegistrySpec{ + Storage: imageregistryv1.ImageRegistryConfigStorage{ + Azure: &imageregistryv1.ImageRegistryConfigStorageAzure{ + AccountName: registryStorageAccountName, + }, + }, + }, + }, + ), + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + storage := mock_storage.NewMockAccountsClient(controller) + kubeSubnet := mock_subnet.NewMockKubeManager(controller) + + if tt.mocks != nil { + tt.mocks(storage, kubeSubnet) + } + + instance := getValidClusterInstance(tt.operatorFlag) + if tt.instance != nil { + tt.instance(instance) + } + + r := reconcileManager{ + log: log, + instance: instance, + subscriptionID: subscriptionId, + storage: storage, + kubeSubnets: kubeSubnet, + imageregistrycli: tt.imageregistrycli, + } + + err := r.reconcileAccounts(context.Background()) + if err != nil { + if tt.wantErr == nil { + t.Fatal(err) + } + if err.Error() != tt.wantErr.Error() || err == nil && tt.wantErr != nil { + t.Errorf("Expected Error %s, got %s when processing %s testcase", tt.wantErr.Error(), err.Error(), tt.name) + } + } + }) + } +} diff --git a/pkg/operator/controllers/subnets/subnet_controller_test.go b/pkg/operator/controllers/subnets/subnet_controller_test.go new file mode 100644 index 00000000000..45f908a7d4a --- /dev/null +++ b/pkg/operator/controllers/subnets/subnet_controller_test.go @@ -0,0 +1,354 @@ +package subnets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "strconv" + "testing" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/go-autorest/autorest/to" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + + "github.com/Azure/ARO-RP/pkg/api" + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + mock_subnet "github.com/Azure/ARO-RP/pkg/util/mocks/subnet" + "github.com/Azure/ARO-RP/pkg/util/subnet" +) + +var ( + subscriptionId = "0000000-0000-0000-0000-000000000000" + clusterResourceGroupName = "aro-iljrzb5a" + infraId = "abcd" + clusterResourceGroupId = "/subscriptions/" + subscriptionId + "/resourcegroups/" + clusterResourceGroupName + vnetResourceGroup = "vnet-rg" + vnetName = "vnet" + subnetNameWorker = "worker" + subnetNameMaster = "master" + + nsgv1NodeResourceId = clusterResourceGroupId + "/providers/Microsoft.Network/networkSecurityGroups/" + infraId + subnet.NSGNodeSuffixV1 + nsgv1MasterResourceId = clusterResourceGroupId + "/providers/Microsoft.Network/networkSecurityGroups/" + infraId + subnet.NSGControlPlaneSuffixV1 + nsgv2ResourceId = clusterResourceGroupId + "/providers/Microsoft.Network/networkSecurityGroups/" + infraId + subnet.NSGSuffixV2 + subnetResourceIdMaster = "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster + subnetResourceIdWorker = "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker +) + +func getValidClusterInstance(operatorFlagEnabled bool, operatorFlagNSG bool, operatorFlagServiceEndpoint bool) *arov1alpha1.Cluster { + return &arov1alpha1.Cluster{ + Spec: arov1alpha1.ClusterSpec{ + ArchitectureVersion: 0, + ClusterResourceGroupID: clusterResourceGroupId, + InfraID: infraId, + OperatorFlags: arov1alpha1.OperatorFlags{ + ENABLED: strconv.FormatBool(operatorFlagEnabled), + NSG_MANAGED: strconv.FormatBool(operatorFlagNSG), + SERVICE_ENDPOINT_MANAGED: strconv.FormatBool(operatorFlagServiceEndpoint), + }, + }, + } +} + +func getValidSubnet() *mgmtnetwork.Subnet { + s := &mgmtnetwork.Subnet{ + SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{ + NetworkSecurityGroup: &mgmtnetwork.SecurityGroup{ + ID: to.StringPtr(nsgv1MasterResourceId), + }, + ServiceEndpoints: &[]mgmtnetwork.ServiceEndpointPropertiesFormat{}, + }, + } + for _, endpoint := range api.SubnetsEndpoints { + *s.SubnetPropertiesFormat.ServiceEndpoints = append(*s.SubnetPropertiesFormat.ServiceEndpoints, mgmtnetwork.ServiceEndpointPropertiesFormat{ + Service: to.StringPtr(endpoint), + ProvisioningState: mgmtnetwork.Succeeded, + }) + } + return s +} + +func TestReconcileManager(t *testing.T) { + log := logrus.NewEntry(logrus.StandardLogger()) + + for _, tt := range []struct { + name string + subnetMock func(*mock_subnet.MockManager, *mock_subnet.MockKubeManager) + instance func(*arov1alpha1.Cluster) + operatorFlagEnabled bool + operatorFlagNSG bool + operatorFlagServiceEndpoint bool + wantErr error + }{ + { + name: "Operator Disabled - no change", + operatorFlagEnabled: false, + operatorFlagNSG: false, + operatorFlagServiceEndpoint: false, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdMaster, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + subnetObjectMaster := getValidSubnet() + mock.EXPECT().Get(gomock.Any(), subnetResourceIdMaster).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + }, + }, + { + name: "Architecture V1 - no change", + operatorFlagEnabled: true, + operatorFlagNSG: true, + operatorFlagServiceEndpoint: true, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdMaster, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + subnetObjectMaster := getValidSubnet() + mock.EXPECT().Get(gomock.Any(), subnetResourceIdMaster).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + }, + }, + { + name: "Architecture V1 - all fixup", + operatorFlagEnabled: true, + operatorFlagNSG: true, + operatorFlagServiceEndpoint: true, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdMaster, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + subnetObjectMaster := getValidSubnet() + subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv1MasterResourceId + "new") + mock.EXPECT().Get(gomock.Any(), subnetResourceIdMaster).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectMasterUpdate := getValidSubnet() + subnetObjectMasterUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv1MasterResourceId) + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdMaster, subnetObjectMasterUpdate).Return(nil) + + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId + "new") + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + + subnetObjectWorkerUpdate := getValidSubnet() + subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdWorker, subnetObjectWorkerUpdate).Return(nil) + }, + }, + { + name: "Architecture V1 - node only fixup", + operatorFlagEnabled: true, + operatorFlagNSG: true, + operatorFlagServiceEndpoint: true, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdMaster, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + subnetObjectMaster := getValidSubnet() + mock.EXPECT().Get(gomock.Any(), subnetResourceIdMaster).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId + "new") + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + + subnetObjectWorkerUpdate := getValidSubnet() + subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdWorker, subnetObjectWorkerUpdate).Return(nil) + }, + }, + { + name: "Architecture V2 - no fixups", + operatorFlagEnabled: true, + operatorFlagNSG: true, + operatorFlagServiceEndpoint: true, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdMaster, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + subnetObjectMaster := getValidSubnet() + subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + mock.EXPECT().Get(gomock.Any(), subnetResourceIdMaster).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + }, + instance: func(instace *arov1alpha1.Cluster) { + instace.Spec.ArchitectureVersion = int(api.ArchitectureVersionV2) + }, + }, + { + name: "Architecture V2 - all nodes fixup", + operatorFlagEnabled: true, + operatorFlagNSG: true, + operatorFlagServiceEndpoint: true, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdMaster, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + subnetObjectMaster := getValidSubnet() + subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId + "new") + mock.EXPECT().Get(gomock.Any(), subnetResourceIdMaster).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectMasterUpdate := getValidSubnet() + subnetObjectMasterUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdMaster, subnetObjectMasterUpdate).Return(nil) + + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId + "new") + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + + subnetObjectWorkerUpdate := getValidSubnet() + subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdWorker, subnetObjectWorkerUpdate).Return(nil) + }, + instance: func(instace *arov1alpha1.Cluster) { + instace.Spec.ArchitectureVersion = int(api.ArchitectureVersionV2) + }, + }, + { + name: "Architecture V2 - endpoint fixup", + operatorFlagEnabled: true, + operatorFlagNSG: true, + operatorFlagServiceEndpoint: true, + subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { + + kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ + { + ResourceID: subnetResourceIdWorker, + IsMaster: true, + }, + { + ResourceID: subnetResourceIdWorker, + IsMaster: false, + }, + }, nil) + + // master + subnetObjectMaster := getValidSubnet() + subnetObjectMaster.SubnetPropertiesFormat.ServiceEndpoints = nil + subnetObjectMaster.ServiceEndpoints = nil + subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectMaster, nil).MaxTimes(2) + + subnetObjectMasterUpdate := getValidSubnet() + subnetObjectMasterUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + for i := range *subnetObjectMasterUpdate.SubnetPropertiesFormat.ServiceEndpoints { + (*subnetObjectMasterUpdate.SubnetPropertiesFormat.ServiceEndpoints)[i].Locations = &[]string{"*"} + (*subnetObjectMasterUpdate.SubnetPropertiesFormat.ServiceEndpoints)[i].ProvisioningState = "" + } + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdWorker, subnetObjectMasterUpdate).Return(nil) + + // worker + subnetObjectWorker := getValidSubnet() + subnetObjectWorker.SubnetPropertiesFormat.ServiceEndpoints = nil + subnetObjectWorker.ServiceEndpoints = nil + subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + mock.EXPECT().Get(gomock.Any(), subnetResourceIdWorker).Return(subnetObjectWorker, nil).MaxTimes(2) + + subnetObjectWorkerUpdate := getValidSubnet() + subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) + for i := range *subnetObjectWorkerUpdate.SubnetPropertiesFormat.ServiceEndpoints { + (*subnetObjectWorkerUpdate.SubnetPropertiesFormat.ServiceEndpoints)[i].Locations = &[]string{"*"} + (*subnetObjectWorkerUpdate.SubnetPropertiesFormat.ServiceEndpoints)[i].ProvisioningState = "" + } + mock.EXPECT().CreateOrUpdate(gomock.Any(), subnetResourceIdWorker, subnetObjectWorkerUpdate).Return(nil) + }, + instance: func(instace *arov1alpha1.Cluster) { + instace.Spec.ArchitectureVersion = int(api.ArchitectureVersionV2) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + subnets := mock_subnet.NewMockManager(controller) + kubeSubnets := mock_subnet.NewMockKubeManager(controller) + if tt.subnetMock != nil { + tt.subnetMock(subnets, kubeSubnets) + } + + instance := getValidClusterInstance(tt.operatorFlagEnabled, tt.operatorFlagNSG, tt.operatorFlagServiceEndpoint) + if tt.instance != nil { + tt.instance(instance) + } + + r := reconcileManager{ + log: log, + instance: instance, + subscriptionID: subscriptionId, + subnets: subnets, + kubeSubnets: kubeSubnets, + } + + err := r.reconcileSubnets(context.Background(), instance) + if err != nil { + if tt.wantErr == nil { + t.Fatal(err) + } + if err.Error() != tt.wantErr.Error() || err == nil && tt.wantErr != nil { + t.Errorf("Expected Error %s, got %s when processing %s testcase", tt.wantErr.Error(), err.Error(), tt.name) + } + } + }) + } +} diff --git a/pkg/operator/controllers/subnets/subnet_nsg.go b/pkg/operator/controllers/subnets/subnet_nsg.go index de23e93ebdd..a4cb767806e 100644 --- a/pkg/operator/controllers/subnets/subnet_nsg.go +++ b/pkg/operator/controllers/subnets/subnet_nsg.go @@ -14,21 +14,6 @@ import ( "github.com/Azure/ARO-RP/pkg/util/subnet" ) -func (r *reconcileManager) reconcileSubnets(ctx context.Context) error { - subnets, err := r.kubeSubnets.List(ctx) - if err != nil { - return err - } - - for _, s := range subnets { - err = r.ensureSubnetNSG(ctx, s) - if err != nil { - return err - } - } - return nil -} - func (r *reconcileManager) ensureSubnetNSG(ctx context.Context, s subnet.Subnet) error { architectureVersion := api.ArchitectureVersion(r.instance.Spec.ArchitectureVersion) diff --git a/pkg/operator/controllers/subnets/subnet_serviceendpoint.go b/pkg/operator/controllers/subnets/subnet_serviceendpoint.go new file mode 100644 index 00000000000..eb05ce1b5c9 --- /dev/null +++ b/pkg/operator/controllers/subnets/subnet_serviceendpoint.go @@ -0,0 +1,61 @@ +package subnets + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "fmt" + "strings" + + mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" + "github.com/Azure/go-autorest/autorest/to" + + "github.com/Azure/ARO-RP/pkg/api" + "github.com/Azure/ARO-RP/pkg/util/subnet" +) + +func (r *reconcileManager) ensureSubnetServiceEndpoints(ctx context.Context, s subnet.Subnet) error { + subnetObject, err := r.subnets.Get(ctx, s.ResourceID) + if err != nil { + return err + } + + if subnetObject == nil { // just in case + return fmt.Errorf("subnet can't be nil") + } + + var changed bool + if subnetObject.SubnetPropertiesFormat == nil { + subnetObject.SubnetPropertiesFormat = &mgmtnetwork.SubnetPropertiesFormat{} + } + if subnetObject.SubnetPropertiesFormat.ServiceEndpoints == nil { + subnetObject.SubnetPropertiesFormat.ServiceEndpoints = &[]mgmtnetwork.ServiceEndpointPropertiesFormat{} + } + + for _, endpoint := range api.SubnetsEndpoints { + var found bool + for _, se := range *subnetObject.SubnetPropertiesFormat.ServiceEndpoints { + if strings.EqualFold(*se.Service, endpoint) && + se.ProvisioningState == mgmtnetwork.Succeeded { + found = true + } + } + if !found { + *subnetObject.SubnetPropertiesFormat.ServiceEndpoints = append(*subnetObject.SubnetPropertiesFormat.ServiceEndpoints, mgmtnetwork.ServiceEndpointPropertiesFormat{ + Service: to.StringPtr(endpoint), + Locations: &[]string{"*"}, + }) + changed = true + } + } + + if changed { + err = r.subnets.CreateOrUpdate(ctx, s.ResourceID, subnetObject) + if err != nil { + return err + } + + } + return nil +} diff --git a/pkg/operator/controllers/subnets/subnetnsg_test.go b/pkg/operator/controllers/subnets/subnetnsg_test.go deleted file mode 100644 index 1872f25bc2d..00000000000 --- a/pkg/operator/controllers/subnets/subnetnsg_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package subnets - -// Copyright (c) Microsoft Corporation. -// Licensed under the Apache License 2.0. - -import ( - "context" - "testing" - - mgmtnetwork "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2020-08-01/network" - "github.com/Azure/go-autorest/autorest/to" - "github.com/golang/mock/gomock" - "github.com/sirupsen/logrus" - - "github.com/Azure/ARO-RP/pkg/api" - arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" - mock_subnet "github.com/Azure/ARO-RP/pkg/util/mocks/subnet" - "github.com/Azure/ARO-RP/pkg/util/subnet" -) - -var ( - subscriptionId = "0000000-0000-0000-0000-000000000000" - clusterResourceGroupName = "aro-iljrzb5a" - infraId = "abcd" - clusterResourceGroupId = "/subscriptions/" + subscriptionId + "/resourcegroups/" + clusterResourceGroupName - vnetResourceGroup = "vnet-rg" - vnetName = "vnet" - subnetNameWorker = "worker" - subnetNameMaster = "master" - - nsgv1NodeResourceId = clusterResourceGroupId + "/providers/Microsoft.Network/networkSecurityGroups/" + infraId + subnet.NSGNodeSuffixV1 - nsgv1MasterResourceId = clusterResourceGroupId + "/providers/Microsoft.Network/networkSecurityGroups/" + infraId + subnet.NSGControlPlaneSuffixV1 - nsgv2ResourceId = clusterResourceGroupId + "/providers/Microsoft.Network/networkSecurityGroups/" + infraId + subnet.NSGSuffixV2 -) - -func getValidClusterInstance() *arov1alpha1.Cluster { - return &arov1alpha1.Cluster{ - Spec: arov1alpha1.ClusterSpec{ - ArchitectureVersion: 0, - ClusterResourceGroupID: clusterResourceGroupId, - InfraID: infraId, - }, - } -} - -func getValidSubnet() *mgmtnetwork.Subnet { - return &mgmtnetwork.Subnet{ - SubnetPropertiesFormat: &mgmtnetwork.SubnetPropertiesFormat{ - NetworkSecurityGroup: &mgmtnetwork.SecurityGroup{ - ID: to.StringPtr(nsgv1MasterResourceId), - }, - }, - } -} - -func TestReconcileManager(t *testing.T) { - log := logrus.NewEntry(logrus.StandardLogger()) - - for _, tt := range []struct { - name string - subnetMock func(*mock_subnet.MockManager, *mock_subnet.MockKubeManager) - instance func(*arov1alpha1.Cluster) - wantErr error - }{ - { - name: "Architecture V1 - no change", - subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { - resourceIdMaster := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster - resourceIdWorker := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker - - kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ - { - ResourceID: resourceIdMaster, - IsMaster: true, - }, - { - ResourceID: resourceIdWorker, - IsMaster: false, - }, - }, nil) - - subnetObjectMaster := getValidSubnet() - mock.EXPECT().Get(gomock.Any(), resourceIdMaster).Return(subnetObjectMaster, nil) - - subnetObjectWorker := getValidSubnet() - subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) - mock.EXPECT().Get(gomock.Any(), resourceIdWorker).Return(subnetObjectWorker, nil) - }, - }, - { - name: "Architecture V1 - all fixup", - subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { - - resourceIdMaster := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster - resourceIdWorker := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker - - kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ - { - ResourceID: resourceIdMaster, - IsMaster: true, - }, - { - ResourceID: resourceIdWorker, - IsMaster: false, - }, - }, nil) - - subnetObjectMaster := getValidSubnet() - subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv1MasterResourceId + "new") - mock.EXPECT().Get(gomock.Any(), resourceIdMaster).Return(subnetObjectMaster, nil) - - subnetObjectMasterUpdate := getValidSubnet() - subnetObjectMasterUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv1MasterResourceId) - mock.EXPECT().CreateOrUpdate(gomock.Any(), resourceIdMaster, subnetObjectMasterUpdate).Return(nil) - - subnetObjectWorker := getValidSubnet() - subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId + "new") - mock.EXPECT().Get(gomock.Any(), resourceIdWorker).Return(subnetObjectWorker, nil) - - subnetObjectWorkerUpdate := getValidSubnet() - subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) - mock.EXPECT().CreateOrUpdate(gomock.Any(), resourceIdWorker, subnetObjectWorkerUpdate).Return(nil) - }, - }, - { - name: "Architecture V1 - node only fixup", - subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { - - resourceIdMaster := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster - resourceIdWorker := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker - - kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ - { - ResourceID: resourceIdMaster, - IsMaster: true, - }, - { - ResourceID: resourceIdWorker, - IsMaster: false, - }, - }, nil) - - subnetObjectMaster := getValidSubnet() - mock.EXPECT().Get(gomock.Any(), resourceIdMaster).Return(subnetObjectMaster, nil) - - subnetObjectWorker := getValidSubnet() - subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId + "new") - mock.EXPECT().Get(gomock.Any(), resourceIdWorker).Return(subnetObjectWorker, nil) - - subnetObjectWorkerUpdate := getValidSubnet() - subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv1NodeResourceId) - mock.EXPECT().CreateOrUpdate(gomock.Any(), resourceIdWorker, subnetObjectWorkerUpdate).Return(nil) - }, - }, - { - name: "Architecture V2 - no fixups", - subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { - - resourceIdMaster := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster - resourceIdWorker := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker - - kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ - { - ResourceID: resourceIdMaster, - IsMaster: true, - }, - { - ResourceID: resourceIdWorker, - IsMaster: false, - }, - }, nil) - - subnetObjectMaster := getValidSubnet() - subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) - mock.EXPECT().Get(gomock.Any(), resourceIdMaster).Return(subnetObjectMaster, nil) - - subnetObjectWorker := getValidSubnet() - subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) - mock.EXPECT().Get(gomock.Any(), resourceIdWorker).Return(subnetObjectWorker, nil) - }, - instance: func(instace *arov1alpha1.Cluster) { - instace.Spec.ArchitectureVersion = int(api.ArchitectureVersionV2) - }, - }, - { - name: "Architecture V2 - all nodes fixup", - subnetMock: func(mock *mock_subnet.MockManager, kmock *mock_subnet.MockKubeManager) { - - resourceIdMaster := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameMaster - resourceIdWorker := "/subscriptions/" + subscriptionId + "/resourceGroups/" + vnetResourceGroup + "/providers/Microsoft.Network/virtualNetworks/" + vnetName + "/subnets/" + subnetNameWorker - - kmock.EXPECT().List(gomock.Any()).Return([]subnet.Subnet{ - { - ResourceID: resourceIdMaster, - IsMaster: true, - }, - { - ResourceID: resourceIdWorker, - IsMaster: false, - }, - }, nil) - - subnetObjectMaster := getValidSubnet() - subnetObjectMaster.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId + "new") - mock.EXPECT().Get(gomock.Any(), resourceIdMaster).Return(subnetObjectMaster, nil) - - subnetObjectMasterUpdate := getValidSubnet() - subnetObjectMasterUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) - mock.EXPECT().CreateOrUpdate(gomock.Any(), resourceIdMaster, subnetObjectMasterUpdate).Return(nil) - - subnetObjectWorker := getValidSubnet() - subnetObjectWorker.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId + "new") - mock.EXPECT().Get(gomock.Any(), resourceIdWorker).Return(subnetObjectWorker, nil) - - subnetObjectWorkerUpdate := getValidSubnet() - subnetObjectWorkerUpdate.NetworkSecurityGroup.ID = to.StringPtr(nsgv2ResourceId) - mock.EXPECT().CreateOrUpdate(gomock.Any(), resourceIdWorker, subnetObjectWorkerUpdate).Return(nil) - }, - instance: func(instace *arov1alpha1.Cluster) { - instace.Spec.ArchitectureVersion = int(api.ArchitectureVersionV2) - }, - }, - } { - t.Run(tt.name, func(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - subnets := mock_subnet.NewMockManager(controller) - kubeSubnets := mock_subnet.NewMockKubeManager(controller) - if tt.subnetMock != nil { - tt.subnetMock(subnets, kubeSubnets) - } - - instance := getValidClusterInstance() - if tt.instance != nil { - tt.instance(instance) - } - - r := reconcileManager{ - log: log, - instance: instance, - subscriptionID: subscriptionId, - subnets: subnets, - kubeSubnets: kubeSubnets, - } - - err := r.reconcileSubnets(context.Background()) - if err != nil { - if tt.wantErr == nil { - t.Fatal(err) - } - if err != nil && err.Error() != tt.wantErr.Error() || err == nil && tt.wantErr != nil { - t.Errorf("Expected Error %s, got %s when processing %s testcase", tt.wantErr.Error(), err.Error(), tt.name) - } - } - // we don't need to compare as mock should do the job - - }) - } -} diff --git a/pkg/operator/controllers/subnets/subnets_controller.go b/pkg/operator/controllers/subnets/subnets_controller.go index d63065fb6e1..376e0c5e01c 100644 --- a/pkg/operator/controllers/subnets/subnets_controller.go +++ b/pkg/operator/controllers/subnets/subnets_controller.go @@ -5,6 +5,8 @@ package subnets import ( "context" + "fmt" + "strings" "github.com/Azure/go-autorest/autorest/azure" machinev1beta1 "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1" @@ -30,8 +32,10 @@ import ( ) const ( - CONFIG_NAMESPACE string = "aro.azuresubnets" - ENABLED string = CONFIG_NAMESPACE + ".enabled" + CONFIG_NAMESPACE string = "aro.azuresubnets" + ENABLED string = CONFIG_NAMESPACE + ".enabled" + NSG_MANAGED string = CONFIG_NAMESPACE + ".nsg.managed" + SERVICE_ENDPOINT_MANAGED string = CONFIG_NAMESPACE + ".serviceendpoint.managed" ) // Reconciler is the controller struct @@ -43,7 +47,7 @@ type Reconciler struct { maocli maoclient.Interface } -// reconcileManager is instance of manager instanciated per request +// reconcileManager is an instance of the manager instantiated per request type reconcileManager struct { log *logrus.Entry @@ -76,7 +80,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return reconcile.Result{}, nil } - // Get endpoints from operator + if !instance.Spec.OperatorFlags.GetSimpleBoolean(NSG_MANAGED) && !instance.Spec.OperatorFlags.GetSimpleBoolean(SERVICE_ENDPOINT_MANAGED) { + // controller is disabled + return reconcile.Result{}, nil + } + + // Get endpoints from the operator azEnv, err := azureclient.EnvironmentFromName(instance.Spec.AZEnvironment) if err != nil { return reconcile.Result{}, err @@ -87,7 +96,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. return reconcile.Result{}, err } - // create refreshable authorizer from token + // create a refreshable authorizer from token authorizer, err := clusterauthorizer.NewAzRefreshableAuthorizer(ctx, r.log, &azEnv, r.kubernetescli) if err != nil { return reconcile.Result{}, err @@ -101,7 +110,42 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. subnets: subnet.NewManager(&azEnv, resource.SubscriptionID, authorizer), } - return reconcile.Result{}, manager.reconcileSubnets(ctx) + return reconcile.Result{}, manager.reconcileSubnets(ctx, instance) +} + +func (r *reconcileManager) reconcileSubnets(ctx context.Context, instance *arov1alpha1.Cluster) error { + + subnets, err := r.kubeSubnets.List(ctx) + if err != nil { + return err + } + + var combinedErrors []string + + // This potentially calls an update twice for the same loop, but this is the price + // to pay for keeping logic split, separate, and simple + for _, s := range subnets { + + if instance.Spec.OperatorFlags.GetSimpleBoolean(NSG_MANAGED) { + err = r.ensureSubnetNSG(ctx, s) + if err != nil { + combinedErrors = append(combinedErrors, err.Error()) + } + } + + if instance.Spec.OperatorFlags.GetSimpleBoolean(SERVICE_ENDPOINT_MANAGED) { + err = r.ensureSubnetServiceEndpoints(ctx, s) + if err != nil { + combinedErrors = append(combinedErrors, err.Error()) + } + } + } + + if len(combinedErrors) > 0 { + return fmt.Errorf(strings.Join(combinedErrors, "\n")) + } + + return nil } // SetupWithManager creates the controller diff --git a/pkg/operator/deploy/bindata.go b/pkg/operator/deploy/bindata.go index 5ce45227ee3..3e4996c4283 100644 --- a/pkg/operator/deploy/bindata.go +++ b/pkg/operator/deploy/bindata.go @@ -87,7 +87,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _aroOpenshiftIo_clustersYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x18\x4d\x73\xda\xc8\xf2\xce\xaf\xe8\xca\x3b\xf8\xf0\x1e\x02\xe2\x3c\xef\x86\x9b\x17\x27\x59\x2a\xd9\x98\xb2\x5d\xb9\xa4\x72\x68\x46\x2d\x31\xb1\x34\xa3\x9d\x6e\x91\x38\x5b\xfb\xdf\xb7\x66\x24\x81\xc0\x02\xa3\xa4\x72\x59\x5d\x40\x3d\xfd\xfd\x3d\x1a\x0c\x87\xc3\x01\x16\xfa\x03\x39\xd6\xd6\x4c\x01\x0b\x4d\x5f\x85\x8c\x7f\xe3\xe8\xfe\x57\x8e\xb4\x1d\xad\x27\x83\x7b\x6d\xe2\x29\xcc\x4a\x16\x9b\xdf\x10\xdb\xd2\x29\xba\xa2\x44\x1b\x2d\xda\x9a\x41\x4e\x82\x31\x0a\x4e\x07\x00\x68\x8c\x15\xf4\x60\xf6\xaf\x00\xca\x1a\x71\x36\xcb\xc8\x0d\x53\x32\xd1\x7d\xb9\xa4\x65\xa9\xb3\x98\x5c\x60\xde\x88\x5e\x8f\xa3\x8b\xe8\x7c\x38\x8e\x9e\x8f\x9f\x4f\xc6\x2f\x27\x17\x93\xf3\xf1\x2f\x2f\x2e\x86\x2f\x5f\xbc\x18\x4f\x2e\xfe\x3f\xc1\x0b\x75\x3e\x00\x50\x8e\x02\xf3\x3b\x9d\x13\x0b\xe6\xc5\x14\x4c\x99\x65\x03\x00\x83\x39\x4d\x41\x65\x25\x0b\x39\x8e\xd0\xd9\xc8\x16\x64\x78\xa5\x13\x89\xb4\x1d\x70\x41\xca\x6b\x94\x3a\x5b\x16\x53\x78\x74\x5e\x71\xa8\x95\xae\x0d\xae\x98\x05\x48\xa6\x59\xde\xb6\xa1\xef\x34\x4b\x38\x29\xb2\xd2\x61\xb6\x15\x1d\x80\xac\x4d\x5a\x66\xe8\x36\xe0\x01\x00\x2b\x5b\x50\x9b\x6b\x6d\x7c\x90\x39\xac\x0d\x58\x4f\x30\x2b\x56\x38\xa9\xb8\xa8\x15\xe5\x58\xa9\x04\xe0\xd5\xbd\x5c\xcc\x3f\x9c\xdf\xee\x80\x01\x62\x62\xe5\x74\x21\xc1\x91\x35\x7b\xd0\x0c\xb2\x22\xa8\x70\x21\xb1\x2e\xbc\x36\x4a\xc2\xe5\x62\xbe\xa1\x2f\x9c\x2d\xc8\x89\x6e\xac\xaf\x9e\x56\x62\xb4\xa0\x7b\xd2\xce\xbc\x42\x15\x16\xc4\x3e\x23\xa8\x12\x5b\x9b\x46\x71\x6d\x03\xd8\x04\x64\xa5\x19\x1c\x15\x8e\x98\x4c\x95\x23\x3b\x8c\xc1\x23\xa1\x01\xbb\xfc\x4c\x4a\x22\xb8\x25\xe7\xd9\x00\xaf\x6c\x99\xc5\x3e\x91\xd6\xe4\x04\x1c\x29\x9b\x1a\xfd\x6d\xc3\x9b\x41\x6c\x10\x9a\xa1\x50\x1d\x94\xed\xa3\x8d\x90\x33\x98\xc1\x1a\xb3\x92\xfe\x07\x68\x62\xc8\xf1\x01\x1c\x79\x29\x50\x9a\x16\xbf\x80\xc2\x11\xfc\x61\x1d\x81\x36\x89\x9d\xc2\x4a\xa4\xe0\xe9\x68\x94\x6a\x69\x0a\x42\xd9\x3c\x2f\x8d\x96\x87\x51\xc8\x6d\xbd\x2c\xc5\x3a\x1e\xc5\xb4\xa6\x6c\xc4\x3a\x1d\xa2\x53\x2b\x2d\xa4\xa4\x74\x34\xc2\x42\x0f\x83\xea\x26\x14\x45\x94\xc7\xff\x71\x75\x09\xf1\xd9\x8e\xae\xf2\xe0\xd3\x83\xc5\x69\x93\xb6\x0e\x42\x2e\x1e\x89\x80\xcf\x4a\x1f\x6d\xac\x49\x2b\x2b\xb6\x8e\xf6\x20\xef\x9d\x9b\x57\xb7\x77\xd0\x88\x0e\xc1\xd8\xf7\x7e\xf0\xfb\x96\x90\xb7\x21\xf0\x0e\xd3\x26\x21\x57\x05\x31\x71\x36\x0f\x3c\xc9\xc4\x85\xd5\x46\xea\xdc\xd2\x64\xf6\xdd\xcf\xe5\x32\xd7\xe2\xe3\xfe\x67\x49\x2c\x3e\x56\x11\xcc\x42\x97\x80\x25\x41\x59\xc4\x28\x14\x47\x30\x37\x30\xc3\x9c\xb2\x19\x32\xfd\xf4\x00\x78\x4f\xf3\xd0\x3b\xf6\xb4\x10\xb4\x1b\xdc\x3e\x72\xe5\xb5\xd6\x41\xd3\x68\x0e\xc4\xab\xae\xcf\xdb\x82\xd4\x4e\xc5\xc4\xc4\xda\xf9\x9c\x16\x14\xf2\x95\xd0\xee\x3e\xcd\xd3\x5d\xa9\xfe\x41\xe5\xae\x6c\x8e\xda\xec\x1f\x1c\x34\x0a\xaa\x1a\x9f\x1b\x99\x2f\xfa\x11\xb5\xbc\xdb\xd9\x21\xb6\xf4\xbe\xf8\xd2\x3d\x1b\x00\xf0\xdb\x2b\xb3\xd6\xce\x9a\x9c\x8c\xf4\x12\xbd\x44\x63\xc8\x3d\x26\xd9\xf1\xf0\x6f\x01\x69\xe3\x5c\x9d\x00\x36\xb0\xba\x95\x2c\xc9\xff\xfb\x62\x9a\xc6\xa1\xc2\x64\x7b\xa4\xe7\x31\x7f\x43\x3d\xda\x3a\x2d\x78\xc2\x8a\x83\xa9\x13\x98\x56\x61\x6f\xc6\xec\x1b\x3f\xae\xe6\x71\x2f\x2f\xc5\xfd\x13\x21\x45\xa1\x2f\xf8\x50\xa5\x50\x87\xb1\x5a\x28\xef\xf4\xc1\x09\x66\xa2\x73\xf8\xd0\x2d\x6f\xe1\xf4\x1a\x85\x5e\xd5\x6d\xa4\x67\x22\xa6\x64\x68\x8d\xef\x6c\x9a\x6a\x93\x3e\xa6\x7c\x32\x78\x89\x4e\x0f\xe6\x6f\x60\x80\xe2\x67\xc7\x14\xce\x3e\x8e\x87\x2f\x3f\xfd\x37\xaa\x7e\xce\xfa\xc7\x1b\x20\xb7\x46\x8b\xf5\x87\x6f\x66\xb7\x97\x4a\xd9\xf2\x50\xe2\x90\x29\xf3\xee\x93\x21\x5c\xde\x5c\x37\xeb\x87\x4d\x79\xfe\xfe\xee\x24\xbc\xc5\xcd\xf5\xd5\x49\x88\x3f\x6c\xd8\xd1\xba\x7e\xca\xb8\x2b\x8d\xa9\xb1\x2c\x5a\xf1\xc2\xd9\xf8\x00\xd6\xdd\xe3\x11\xdf\x1c\xcd\xf0\x35\x6a\x97\xe0\xd7\x1f\xb6\xe3\xbd\x5f\x05\x0b\x54\xf4\x2f\x08\xd1\x91\x5e\xa3\x4d\xe2\xb0\x67\x73\xd1\x26\x75\xc4\xdc\xb3\x54\xab\x2d\x8c\x64\xb6\x22\x75\xdf\xd5\xc1\x8f\x17\x6b\xe9\xb2\x4e\xf8\x91\xc6\xf4\x84\x42\x6d\x84\xae\x06\x75\xd4\x6f\x99\x55\x61\x7d\xed\xe5\x02\x6f\x1f\x8a\x75\x49\x86\x69\x87\xc6\x18\xc7\xe1\x36\x85\xd9\xe2\xa8\x2b\x8e\xda\xb4\x33\x07\xaf\x6b\x81\xaf\xbd\xc0\xcd\x38\x4c\x08\xfd\xe0\x0e\x1d\x98\x37\xf7\x82\xcb\x9b\xeb\x0d\x7e\x1f\x4f\x34\xfb\x64\x57\x12\xed\x28\xd3\xcc\xb3\xf9\x55\x73\x33\xb9\xfc\xe6\xd5\xd8\x32\xa8\xae\x08\xd4\xba\x30\x9d\x6c\xf8\xda\x90\xf4\x4a\xe3\x43\xab\x9b\xa0\x94\x7c\xc2\xf2\x16\xf0\x76\xd6\x37\xbb\x64\xbf\x2b\x7f\xf7\xfe\xa6\xac\xa9\xe2\xdf\x67\xf8\x76\x86\x7b\xd6\x70\xf2\x8e\xfe\x5c\x72\xb5\x9c\xb3\xa0\x89\xd1\xc5\x5b\x41\x90\x68\xca\x62\x8e\x3a\xf8\x1e\x2f\x46\x80\x0c\x59\xee\x1c\x1a\xd6\xcd\x5d\xfc\x50\x05\x26\xd6\xe5\x28\x53\xf0\x5b\xfe\x50\x74\x4e\xdf\x5b\xa9\x39\x31\x63\x7a\x50\xce\x93\xf4\x8e\x90\x0f\xcd\xf9\x13\xc8\xbb\x32\xa3\x07\x79\x40\xf8\x3e\xe2\x23\xc5\x77\xac\x7f\x35\xdd\xe6\x89\x05\xbd\x53\xac\xa3\xf8\x77\x94\xb7\xf4\xc0\x8b\xea\x26\xf8\xb3\xf7\xc1\x4e\x1b\x1f\x01\xab\x02\x9b\x82\xb8\xb2\x4a\x23\x16\xeb\x7c\x4e\xb4\x20\xe5\x72\x73\xb9\x6e\xb4\xab\x43\x07\x7f\xfd\x3d\xd8\x46\x11\x95\xa2\x42\x28\x7e\xbf\xff\xcd\xe7\xd9\xb3\x9d\x8f\x3a\xe1\xb5\x55\x9b\xf0\xf1\xd3\xa0\x12\x4c\xf1\x87\xe6\xf3\x8d\x07\xfe\x13\x00\x00\xff\xff\xec\xb4\xc0\xad\x4c\x13\x00\x00") +var _aroOpenshiftIo_clustersYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xdc\x58\xcf\x72\xdb\x36\x13\xbf\xeb\x29\x76\xf2\x1d\x7c\xf8\x4a\x4a\x8a\x53\xb7\xd1\xcd\x95\x93\x54\x93\x34\xd6\x58\x9e\x5c\x32\x39\xac\xc0\x25\x85\x98\x04\x58\x60\xa1\xc4\xe9\xf4\xdd\x3b\x00\x49\x89\x92\x29\x59\x74\x26\x3d\x94\x17\x89\x8b\xc5\xfe\xfd\xed\x2e\xc0\x41\x14\x45\x03\x2c\xe5\x07\x32\x56\x6a\x35\x01\x2c\x25\x7d\x65\x52\xfe\xcd\xc6\x77\xbf\xda\x58\xea\xe1\x7a\x3c\xb8\x93\x2a\x99\xc0\xd4\x59\xd6\xc5\x0d\x59\xed\x8c\xa0\x2b\x4a\xa5\x92\x2c\xb5\x1a\x14\xc4\x98\x20\xe3\x64\x00\x80\x4a\x69\x46\x4f\xb6\xfe\x15\x40\x68\xc5\x46\xe7\x39\x99\x28\x23\x15\xdf\xb9\x25\x2d\x9d\xcc\x13\x32\x41\x78\xa3\x7a\x3d\x8a\x2f\xe2\xf3\x68\x14\x3f\x1f\x3d\x1f\x8f\x5e\x8e\x2f\xc6\xe7\xa3\x5f\x5e\x5c\x44\x2f\x5f\xbc\x18\x8d\x2f\x7e\x1e\xe3\x85\x38\x1f\x00\x08\x43\x41\xf8\xad\x2c\xc8\x32\x16\xe5\x04\x94\xcb\xf3\x01\x80\xc2\x82\x26\x20\x72\x67\x99\x8c\x8d\xd1\xe8\x58\x97\xa4\xec\x4a\xa6\x1c\x4b\x3d\xb0\x25\x09\x6f\x51\x66\xb4\x2b\x27\xf0\x60\xbd\x92\x50\x1b\x5d\x3b\x5c\x09\x0b\x94\x5c\x5a\x7e\xdb\xa6\xbe\x93\x96\xc3\x4a\x99\x3b\x83\xf9\x56\x75\x20\x5a\xa9\x32\x97\xa3\xd9\x90\x07\x00\x56\xe8\x92\xda\x52\x6b\xe7\x83\xce\xa8\x76\x60\x3d\xc6\xbc\x5c\xe1\xb8\x92\x22\x56\x54\x60\x65\x12\x80\x37\xf7\x72\x3e\xfb\x70\xbe\xd8\x21\x03\x24\x64\x85\x91\x25\x87\x40\xd6\xe2\x41\x5a\xe0\x15\x41\xc5\x0b\xa9\x36\xe1\xb5\x31\x12\x2e\xe7\xb3\xcd\xfe\xd2\xe8\x92\x0c\xcb\xc6\xfb\xea\x69\x01\xa3\x45\xdd\xd3\x76\xe6\x0d\xaa\xb8\x20\xf1\x88\xa0\x4a\x6d\xed\x1a\x25\xb5\x0f\xa0\x53\xe0\x95\xb4\x60\xa8\x34\x64\x49\x55\x18\xd9\x11\x0c\x9e\x09\x15\xe8\xe5\x67\x12\x1c\xc3\x82\x8c\x17\x03\x76\xa5\x5d\x9e\x78\x20\xad\xc9\x30\x18\x12\x3a\x53\xf2\xdb\x46\xb6\x05\xd6\x41\x69\x8e\x4c\x75\x52\xb6\x8f\x54\x4c\x46\x61\x0e\x6b\xcc\x1d\xfd\x04\xa8\x12\x28\xf0\x1e\x0c\x79\x2d\xe0\x54\x4b\x5e\x60\xb1\x31\xfc\xa1\x0d\x81\x54\xa9\x9e\xc0\x8a\xb9\xb4\x93\xe1\x30\x93\xdc\x14\x84\xd0\x45\xe1\x94\xe4\xfb\x61\xc0\xb6\x5c\x3a\xd6\xc6\x0e\x13\x5a\x53\x3e\xb4\x32\x8b\xd0\x88\x95\x64\x12\xec\x0c\x0d\xb1\x94\x51\x30\x5d\x85\xa2\x88\x8b\xe4\x7f\xa6\x2e\x21\x7b\xb6\x63\x2b\xdf\x7b\x78\x58\x36\x52\x65\xad\x85\x80\xc5\x23\x19\xf0\xa8\xf4\xd9\xc6\x7a\x6b\xe5\xc5\x36\xd0\x9e\xe4\xa3\x73\xf3\x6a\x71\x0b\x8d\xea\x90\x8c\xfd\xe8\x87\xb8\x6f\x37\xda\x6d\x0a\x7c\xc0\xa4\x4a\xc9\x54\x49\x4c\x8d\x2e\x82\x4c\x52\x49\xa9\xa5\xe2\x1a\x5b\x92\xd4\x7e\xf8\xad\x5b\x16\x92\x7d\xde\xff\x74\x64\xd9\xe7\x2a\x86\x69\xe8\x12\xb0\x24\x70\x65\x82\x4c\x49\x0c\x33\x05\x53\x2c\x28\x9f\xa2\xa5\x1f\x9e\x00\x1f\x69\x1b\xf9\xc0\x9e\x96\x82\x76\x83\xdb\x67\xae\xa2\xd6\x5a\x68\x1a\xcd\x81\x7c\xd5\xf5\xb9\x28\x49\xec\x54\x4c\x42\x56\x1a\x8f\x69\x46\x26\x5f\x09\xed\xee\xd3\x3c\xdd\x95\xea\x1f\x14\xe6\x4a\x17\x28\xd5\xfe\xc2\x41\xa7\xa0\xaa\xf1\x99\xe2\xd9\xbc\xdf\xa6\x56\x74\x3b\x3b\xc4\x76\xbf\x2f\xbe\x6c\xcf\x07\x00\xfc\xf6\x4a\xad\xa5\xd1\xaa\x20\xc5\xbd\x54\x2f\x51\x29\x32\x0f\xb7\xec\x44\xf8\xb7\xc0\xb4\x09\xae\x4c\x01\x1b\x5a\xdd\x4a\x96\xe4\xff\x7d\x51\x4d\xe3\x10\x61\xb2\x3d\xb0\xf3\x58\xbc\xa1\x1e\x6d\x9d\x1e\x3c\xe2\xc5\x41\xe8\x04\xa1\x55\xda\x9b\x31\xfb\xc6\x8f\xab\x59\xd2\x2b\x4a\x49\x7f\x20\x64\xc8\xf4\x05\xef\x2b\x08\x75\x38\x2b\x99\x8a\xce\x18\x9c\xe0\x26\x1a\x83\xf7\xdd\xfa\xe6\x46\xae\x91\xe9\x55\xdd\x46\x7a\x02\x31\x23\x45\x6b\x7c\xa7\xb3\x4c\xaa\xec\xe1\xce\x47\x93\x97\xca\xec\x20\x7e\x83\x00\x64\x3f\x3b\x26\x70\xf6\x71\x14\xbd\xfc\xf4\xff\xb8\xfa\x39\xeb\x9f\x6f\x80\x42\x2b\xc9\xda\x2f\xbe\x99\x2e\x2e\x85\xd0\xee\x10\x70\x48\xb9\xa2\x7b\x25\x82\xcb\x9b\xeb\xe6\xf8\xa1\x33\x3b\x7b\x7f\x7b\x12\xdf\xfc\xe6\xfa\xea\x24\xc6\xef\x76\xec\x68\x5d\x3f\xe6\xdc\x95\xc4\x4c\x69\xcb\x52\xd8\xb9\xd1\xc9\x01\xae\xdb\x87\x23\xbe\x59\x9a\xe2\x6b\x94\x26\xc5\xaf\xdf\xed\xc7\x7b\x7f\x14\x2c\x51\xd0\x7f\x20\x45\x47\x7a\x8d\x54\xa9\xc1\x9e\xcd\x45\xaa\xcc\x90\xb5\x3d\x4b\xb5\x3a\x85\x11\x4f\x57\x24\xee\xba\x3a\xf8\xf1\x62\x75\x26\xef\xa4\x1f\x69\x4c\x8f\x18\xd4\x66\xe8\x6a\x50\x47\xe3\x96\x6b\x11\x8e\xaf\xbd\x42\xe0\xfd\x43\xd6\x26\xcd\x31\xeb\xb0\x18\x93\x24\xdc\xa6\x30\x9f\x1f\x0d\xc5\x51\x9f\x76\xe6\xe0\x75\xad\xf0\xb5\x57\xb8\x19\x87\x29\xa1\x1f\xdc\xa1\x03\xdb\xcd\xbd\xe0\xf2\xe6\x7a\xc3\xdf\x27\x12\xcd\x79\xb2\x0b\x44\x3b\xc6\x34\xf3\x6c\x76\xd5\xdc\x4c\x2e\xbf\x79\x33\xb6\x02\xaa\x2b\x02\xb5\x2e\x4c\x27\x3b\x6e\xc9\xac\xa5\xa0\x85\x5b\x2a\xe2\x7f\x61\x7a\x59\xd6\x06\x33\x5a\xb8\x34\x95\x5f\x7b\x81\x60\xad\x88\x7b\x15\xdc\xa1\x43\x26\x23\x3b\x7b\xc2\x31\x33\xf0\xed\x1c\x34\xf5\xd2\x87\xeb\xe9\x27\x4d\xa1\x55\x85\xd4\x3e\x81\xee\x04\xe6\xb4\x91\xe4\x21\xf1\xd9\xd9\xea\x1a\x61\x19\x55\x82\x26\xd9\x2a\x82\x54\x52\x9e\xd8\xb8\x43\xee\xf1\xb6\x01\x90\xa3\xe5\x5b\x83\xca\xca\xe6\xab\xc1\xa1\x5e\x91\x6a\x53\x20\x4f\xc0\xdf\x47\x22\x96\x05\x3d\xb5\xa7\x14\x64\x2d\x66\x07\xf5\x3c\xba\xdf\x10\xda\x43\x27\x92\x13\xb6\x77\x21\xa3\xc7\xf6\xc0\xf0\xb4\xcd\x47\xda\xc4\xb1\x62\x6a\xfa\xe2\x23\x57\x89\x4e\xb5\x86\x92\xdf\x91\xdf\xd2\xbd\x9d\x57\x77\xd6\x1f\x5d\xfb\x9d\x3e\x3e\x20\x56\x05\x36\x01\x36\xae\x82\x51\xdd\x30\xda\x14\xb7\xdc\x7c\x06\x68\xac\xab\x53\x07\x7f\xfd\x3d\xd8\x66\x11\x85\xa0\x92\x29\x79\xbf\xff\x75\xea\xd9\xb3\xf0\xd2\x7c\x7e\x0a\xaf\xad\xda\x84\x8f\x9f\x06\x95\x62\x4a\x3e\x34\x1f\x9a\x3c\xf1\x9f\x00\x00\x00\xff\xff\x78\x43\x46\x8f\xf6\x13\x00\x00") func aroOpenshiftIo_clustersYamlBytes() ([]byte, error) { return bindataRead( diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go index 7b08ba15dad..ddfe1d59403 100644 --- a/pkg/operator/deploy/deploy.go +++ b/pkg/operator/deploy/deploy.go @@ -138,6 +138,16 @@ func (o *operator) resources() ([]kruntime.Object, error) { domain += "." + o.env.Domain() } + serviceSubnets := []string{ + "/subscriptions/" + o.env.SubscriptionID() + "/resourceGroups/" + o.env.ResourceGroup() + "/providers/Microsoft.Network/virtualNetworks/rp-pe-vnet-001/subnets/rp-pe-subnet", + "/subscriptions/" + o.env.SubscriptionID() + "/resourceGroups/" + o.env.ResourceGroup() + "/providers/Microsoft.Network/virtualNetworks/rp-vnet/subnets/rp-subnet", + } + + // Avoiding issues with dev environment when gateway is not present + if o.oc.Properties.FeatureProfile.GatewayEnabled { + serviceSubnets = append(serviceSubnets, "/subscriptions/"+o.env.SubscriptionID()+"/resourceGroups/"+o.env.GatewayResourceGroup()+"/providers/Microsoft.Network/virtualNetworks/gateway-vnet/subnets/gateway-subnet") + } + cluster := &arov1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: arov1alpha1.SingletonClusterName, @@ -152,12 +162,14 @@ func (o *operator) resources() ([]kruntime.Object, error) { InfraID: o.oc.Properties.InfraID, ArchitectureVersion: int(o.oc.Properties.ArchitectureVersion), VnetID: vnetID, + StorageSuffix: o.oc.Properties.StorageSuffix, GenevaLogging: arov1alpha1.GenevaLoggingSpec{ ConfigVersion: o.env.ClusterGenevaLoggingConfigVersion(), MonitoringGCSAccount: o.env.ClusterGenevaLoggingAccount(), MonitoringGCSEnvironment: o.env.ClusterGenevaLoggingEnvironment(), MonitoringGCSNamespace: o.env.ClusterGenevaLoggingNamespace(), }, + ServiceSubnets: serviceSubnets, InternetChecker: arov1alpha1.InternetCheckerSpec{ URLs: []string{ fmt.Sprintf("https://%s/", o.env.ACRDomain()), diff --git a/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml b/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml index cfa60206090..be06e5e72f1 100644 --- a/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml +++ b/pkg/operator/deploy/staticresources/aro.openshift.io_clusters.yaml @@ -105,6 +105,12 @@ spec: resourceId: description: ResourceID is the Azure resourceId of the cluster type: string + serviceSubnets: + items: + type: string + type: array + storageSuffix: + type: string vnetId: type: string type: object diff --git a/pkg/util/azureclient/mgmt/storage/generate.go b/pkg/util/azureclient/mgmt/storage/generate.go new file mode 100644 index 00000000000..89cd32bdcb6 --- /dev/null +++ b/pkg/util/azureclient/mgmt/storage/generate.go @@ -0,0 +1,8 @@ +package storage + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +//go:generate rm -rf ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE +//go:generate go run ../../../../../vendor/github.com/golang/mock/mockgen -destination=../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/$GOPACKAGE AccountsClient +//go:generate go run ../../../../../vendor/golang.org/x/tools/cmd/goimports -local=github.com/Azure/ARO-RP -e -w ../../../../util/mocks/azureclient/mgmt/$GOPACKAGE/$GOPACKAGE.go diff --git a/pkg/util/mocks/azureclient/mgmt/storage/storage.go b/pkg/util/mocks/azureclient/mgmt/storage/storage.go new file mode 100644 index 00000000000..f5eafb8b0be --- /dev/null +++ b/pkg/util/mocks/azureclient/mgmt/storage/storage.go @@ -0,0 +1,96 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/Azure/ARO-RP/pkg/util/azureclient/mgmt/storage (interfaces: AccountsClient) + +// Package mock_storage is a generated GoMock package. +package mock_storage + +import ( + context "context" + reflect "reflect" + + storage "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + gomock "github.com/golang/mock/gomock" +) + +// MockAccountsClient is a mock of AccountsClient interface. +type MockAccountsClient struct { + ctrl *gomock.Controller + recorder *MockAccountsClientMockRecorder +} + +// MockAccountsClientMockRecorder is the mock recorder for MockAccountsClient. +type MockAccountsClientMockRecorder struct { + mock *MockAccountsClient +} + +// NewMockAccountsClient creates a new mock instance. +func NewMockAccountsClient(ctrl *gomock.Controller) *MockAccountsClient { + mock := &MockAccountsClient{ctrl: ctrl} + mock.recorder = &MockAccountsClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAccountsClient) EXPECT() *MockAccountsClientMockRecorder { + return m.recorder +} + +// GetProperties mocks base method. +func (m *MockAccountsClient) GetProperties(arg0 context.Context, arg1, arg2 string, arg3 storage.AccountExpand) (storage.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProperties", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(storage.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProperties indicates an expected call of GetProperties. +func (mr *MockAccountsClientMockRecorder) GetProperties(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProperties", reflect.TypeOf((*MockAccountsClient)(nil).GetProperties), arg0, arg1, arg2, arg3) +} + +// ListAccountSAS mocks base method. +func (m *MockAccountsClient) ListAccountSAS(arg0 context.Context, arg1, arg2 string, arg3 storage.AccountSasParameters) (storage.ListAccountSasResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListAccountSAS", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(storage.ListAccountSasResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListAccountSAS indicates an expected call of ListAccountSAS. +func (mr *MockAccountsClientMockRecorder) ListAccountSAS(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAccountSAS", reflect.TypeOf((*MockAccountsClient)(nil).ListAccountSAS), arg0, arg1, arg2, arg3) +} + +// ListKeys mocks base method. +func (m *MockAccountsClient) ListKeys(arg0 context.Context, arg1, arg2 string, arg3 storage.ListKeyExpand) (storage.AccountListKeysResult, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListKeys", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(storage.AccountListKeysResult) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListKeys indicates an expected call of ListKeys. +func (mr *MockAccountsClientMockRecorder) ListKeys(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListKeys", reflect.TypeOf((*MockAccountsClient)(nil).ListKeys), arg0, arg1, arg2, arg3) +} + +// Update mocks base method. +func (m *MockAccountsClient) Update(arg0 context.Context, arg1, arg2 string, arg3 storage.AccountUpdateParameters) (storage.Account, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(storage.Account) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockAccountsClientMockRecorder) Update(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockAccountsClient)(nil).Update), arg0, arg1, arg2, arg3) +} diff --git a/pkg/util/subnet/cluster_subnets.go b/pkg/util/subnet/cluster_subnets.go index 44d91cfd4a6..404f3aa56e0 100644 --- a/pkg/util/subnet/cluster_subnets.go +++ b/pkg/util/subnet/cluster_subnets.go @@ -34,7 +34,7 @@ func NewKubeManager(maocli maoclient.Interface, subscriptionID string) KubeManag } // List reconstructs subnetId used in machines object in the cluster -// In cases we interat with customer vnets, we don't know which subnets are used in ARO. +// In cases when we interact with customer vnets, we don't know which subnets are used in ARO. // Example : /subscriptions/{subscriptionID}/resourceGroups/{vnet-resource-group}/providers/Microsoft.Network/virtualNetworks/{vnet-name}/subnets/{subnet-name} func (m *kubeManager) List(ctx context.Context) ([]Subnet, error) { subnetMap := []Subnet{} diff --git a/test/e2e/operator.go b/test/e2e/operator.go index 956b1d4dba5..07655ecf4e1 100644 --- a/test/e2e/operator.go +++ b/test/e2e/operator.go @@ -27,7 +27,6 @@ import ( arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" "github.com/Azure/ARO-RP/pkg/operator/controllers/machineset" "github.com/Azure/ARO-RP/pkg/operator/controllers/monitoring" - "github.com/Azure/ARO-RP/pkg/operator/controllers/subnets" "github.com/Azure/ARO-RP/pkg/util/conditions" "github.com/Azure/ARO-RP/pkg/util/ready" "github.com/Azure/ARO-RP/pkg/util/subnet" @@ -343,17 +342,6 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { const nsg = "e2e-nsg" - // TODO (robryan) rm this func once default to on https://github.com/Azure/ARO-RP/issues/1735 - enableReconcileSubnet := func() { - instance, err := clients.AROClusters.AroV1alpha1().Clusters().Get(ctx, arov1alpha1.SingletonClusterName, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - if !instance.Spec.OperatorFlags.GetSimpleBoolean(subnets.ENABLED) { - instance.Spec.OperatorFlags[subnets.ENABLED] = "true" - _, err = clients.AROClusters.AroV1alpha1().Clusters().Update(ctx, instance, metav1.UpdateOptions{}) - Expect(err).NotTo(HaveOccurred()) - } - } // Gathers vnet name, resource group, location, and adds master/worker subnets to list to reconcile. gatherNetworkInfo := func() { oc, err := clients.OpenshiftClustersv20200430.Get(ctx, vnetResourceGroup, clusterName) @@ -391,7 +379,6 @@ var _ = Describe("ARO Operator - Azure Subnet Reconciler", func() { } BeforeEach(func() { - enableReconcileSubnet() gatherNetworkInfo() createE2ENSG() }) diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/clientset_generated.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 00000000000..76c495c45c9 --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,66 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/openshift/client-go/imageregistry/clientset/versioned" + imageregistryv1 "github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1" + fakeimageregistryv1 "github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var _ clientset.Interface = &Clientset{} + +// ImageregistryV1 retrieves the ImageregistryV1Client +func (c *Clientset) ImageregistryV1() imageregistryv1.ImageregistryV1Interface { + return &fakeimageregistryv1.FakeImageregistryV1{Fake: &c.Fake} +} diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/doc.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/doc.go new file mode 100644 index 00000000000..3630ed1cd17 --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/doc.go @@ -0,0 +1,4 @@ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/register.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/register.go new file mode 100644 index 00000000000..4ddf2ee1e93 --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/fake/register.go @@ -0,0 +1,40 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + imageregistryv1 "github.com/openshift/api/imageregistry/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + imageregistryv1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/doc.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/doc.go new file mode 100644 index 00000000000..2b5ba4c8e44 --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/doc.go @@ -0,0 +1,4 @@ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_config.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_config.go new file mode 100644 index 00000000000..80a3a955a71 --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_config.go @@ -0,0 +1,117 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + imageregistryv1 "github.com/openshift/api/imageregistry/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeConfigs implements ConfigInterface +type FakeConfigs struct { + Fake *FakeImageregistryV1 +} + +var configsResource = schema.GroupVersionResource{Group: "imageregistry.operator.openshift.io", Version: "v1", Resource: "configs"} + +var configsKind = schema.GroupVersionKind{Group: "imageregistry.operator.openshift.io", Version: "v1", Kind: "Config"} + +// Get takes name of the config, and returns the corresponding config object, and an error if there is any. +func (c *FakeConfigs) Get(ctx context.Context, name string, options v1.GetOptions) (result *imageregistryv1.Config, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(configsResource, name), &imageregistryv1.Config{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.Config), err +} + +// List takes label and field selectors, and returns the list of Configs that match those selectors. +func (c *FakeConfigs) List(ctx context.Context, opts v1.ListOptions) (result *imageregistryv1.ConfigList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(configsResource, configsKind, opts), &imageregistryv1.ConfigList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &imageregistryv1.ConfigList{ListMeta: obj.(*imageregistryv1.ConfigList).ListMeta} + for _, item := range obj.(*imageregistryv1.ConfigList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested configs. +func (c *FakeConfigs) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(configsResource, opts)) +} + +// Create takes the representation of a config and creates it. Returns the server's representation of the config, and an error, if there is any. +func (c *FakeConfigs) Create(ctx context.Context, config *imageregistryv1.Config, opts v1.CreateOptions) (result *imageregistryv1.Config, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(configsResource, config), &imageregistryv1.Config{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.Config), err +} + +// Update takes the representation of a config and updates it. Returns the server's representation of the config, and an error, if there is any. +func (c *FakeConfigs) Update(ctx context.Context, config *imageregistryv1.Config, opts v1.UpdateOptions) (result *imageregistryv1.Config, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(configsResource, config), &imageregistryv1.Config{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.Config), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeConfigs) UpdateStatus(ctx context.Context, config *imageregistryv1.Config, opts v1.UpdateOptions) (*imageregistryv1.Config, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(configsResource, "status", config), &imageregistryv1.Config{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.Config), err +} + +// Delete takes name of the config and deletes it. Returns an error if one occurs. +func (c *FakeConfigs) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(configsResource, name), &imageregistryv1.Config{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeConfigs) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(configsResource, listOpts) + + _, err := c.Fake.Invokes(action, &imageregistryv1.ConfigList{}) + return err +} + +// Patch applies the patch and returns the patched config. +func (c *FakeConfigs) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *imageregistryv1.Config, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(configsResource, name, pt, data, subresources...), &imageregistryv1.Config{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.Config), err +} diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_imagepruner.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_imagepruner.go new file mode 100644 index 00000000000..7678aedf8ab --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_imagepruner.go @@ -0,0 +1,117 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + imageregistryv1 "github.com/openshift/api/imageregistry/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeImagePruners implements ImagePrunerInterface +type FakeImagePruners struct { + Fake *FakeImageregistryV1 +} + +var imageprunersResource = schema.GroupVersionResource{Group: "imageregistry.operator.openshift.io", Version: "v1", Resource: "imagepruners"} + +var imageprunersKind = schema.GroupVersionKind{Group: "imageregistry.operator.openshift.io", Version: "v1", Kind: "ImagePruner"} + +// Get takes name of the imagePruner, and returns the corresponding imagePruner object, and an error if there is any. +func (c *FakeImagePruners) Get(ctx context.Context, name string, options v1.GetOptions) (result *imageregistryv1.ImagePruner, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(imageprunersResource, name), &imageregistryv1.ImagePruner{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.ImagePruner), err +} + +// List takes label and field selectors, and returns the list of ImagePruners that match those selectors. +func (c *FakeImagePruners) List(ctx context.Context, opts v1.ListOptions) (result *imageregistryv1.ImagePrunerList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(imageprunersResource, imageprunersKind, opts), &imageregistryv1.ImagePrunerList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &imageregistryv1.ImagePrunerList{ListMeta: obj.(*imageregistryv1.ImagePrunerList).ListMeta} + for _, item := range obj.(*imageregistryv1.ImagePrunerList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested imagePruners. +func (c *FakeImagePruners) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(imageprunersResource, opts)) +} + +// Create takes the representation of a imagePruner and creates it. Returns the server's representation of the imagePruner, and an error, if there is any. +func (c *FakeImagePruners) Create(ctx context.Context, imagePruner *imageregistryv1.ImagePruner, opts v1.CreateOptions) (result *imageregistryv1.ImagePruner, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(imageprunersResource, imagePruner), &imageregistryv1.ImagePruner{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.ImagePruner), err +} + +// Update takes the representation of a imagePruner and updates it. Returns the server's representation of the imagePruner, and an error, if there is any. +func (c *FakeImagePruners) Update(ctx context.Context, imagePruner *imageregistryv1.ImagePruner, opts v1.UpdateOptions) (result *imageregistryv1.ImagePruner, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(imageprunersResource, imagePruner), &imageregistryv1.ImagePruner{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.ImagePruner), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeImagePruners) UpdateStatus(ctx context.Context, imagePruner *imageregistryv1.ImagePruner, opts v1.UpdateOptions) (*imageregistryv1.ImagePruner, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(imageprunersResource, "status", imagePruner), &imageregistryv1.ImagePruner{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.ImagePruner), err +} + +// Delete takes name of the imagePruner and deletes it. Returns an error if one occurs. +func (c *FakeImagePruners) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteAction(imageprunersResource, name), &imageregistryv1.ImagePruner{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeImagePruners) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(imageprunersResource, listOpts) + + _, err := c.Fake.Invokes(action, &imageregistryv1.ImagePrunerList{}) + return err +} + +// Patch applies the patch and returns the patched imagePruner. +func (c *FakeImagePruners) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *imageregistryv1.ImagePruner, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(imageprunersResource, name, pt, data, subresources...), &imageregistryv1.ImagePruner{}) + if obj == nil { + return nil, err + } + return obj.(*imageregistryv1.ImagePruner), err +} diff --git a/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_imageregistry_client.go b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_imageregistry_client.go new file mode 100644 index 00000000000..4c384a34672 --- /dev/null +++ b/vendor/github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake/fake_imageregistry_client.go @@ -0,0 +1,28 @@ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeImageregistryV1 struct { + *testing.Fake +} + +func (c *FakeImageregistryV1) Configs() v1.ConfigInterface { + return &FakeConfigs{c} +} + +func (c *FakeImageregistryV1) ImagePruners() v1.ImagePrunerInterface { + return &FakeImagePruners{c} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeImageregistryV1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 40ab65b36b4..17550763041 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -818,8 +818,10 @@ github.com/openshift/client-go/console/clientset/versioned/typed/console/v1/fake github.com/openshift/client-go/console/clientset/versioned/typed/console/v1alpha1 github.com/openshift/client-go/console/clientset/versioned/typed/console/v1alpha1/fake github.com/openshift/client-go/imageregistry/clientset/versioned +github.com/openshift/client-go/imageregistry/clientset/versioned/fake github.com/openshift/client-go/imageregistry/clientset/versioned/scheme github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1 +github.com/openshift/client-go/imageregistry/clientset/versioned/typed/imageregistry/v1/fake github.com/openshift/client-go/operator/clientset/versioned github.com/openshift/client-go/operator/clientset/versioned/fake github.com/openshift/client-go/operator/clientset/versioned/scheme