diff --git a/go.mod b/go.mod index 52d4d339df..5f93e41cf4 100644 --- a/go.mod +++ b/go.mod @@ -130,6 +130,11 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 golang.org/x/sync v0.6.0 gotest.tools/v3 v3.5.1 k8s.io/kubectl v0.28.5 diff --git a/go.sum b/go.sum index ecd557e43e..03025387cf 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,21 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1 h1:EHlWMHGZaYhRHWNbawWxEzlZ2Eh8aTXNs8t7iAibq+I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1/go.mod h1:noQIdW75SiQFB3mSFJBr4iRRH83S9skaFiBv4C0uEs0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0 h1:MRPU8Bge2f9tkfG3PCr4vEnqXl8XOSjlhuK3l+8Hvkc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0/go.mod h1:xYrOYxajQvXMlp6M1E3amlaqPDXspyJxmjqTsGo6Jmw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0/go.mod h1:jj6P8ybImR+5topJ+eH6fgcemSFBmU6/6bFF8KkwuDI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0 h1:9CrwzqQ+e8EqD+A2bh547GjBU4K0o30FhiTB981LFNI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0/go.mod h1:Wfx7a5UHfOLG6O4NZ7Q0BPZUYwvlNCBR/OlIBpP3dlA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/test/e2e/framework/azure/create-cilium-cluster.go b/test/e2e/framework/azure/create-cilium-cluster.go new file mode 100644 index 0000000000..671a25945b --- /dev/null +++ b/test/e2e/framework/azure/create-cilium-cluster.go @@ -0,0 +1,213 @@ +package azure + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" + "github.com/Azure/azure-container-networking/test/e2e/manifests" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubectl/pkg/scheme" +) + +var ( + ErrResourceNameTooLong = fmt.Errorf("resource name too long") + ErrEmptyFile = fmt.Errorf("empty file") +) + +type CreateBYOCiliumCluster struct { + SubscriptionID string + ResourceGroupName string + Location string + ClusterName string + VnetName string + SubnetName string + PodCidr string + DNSServiceIP string + ServiceCidr string +} + +func printjson(data interface{}) { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(data); err != nil { + panic(err) + } +} + +func (c *CreateBYOCiliumCluster) Prevalidate() error { + for _, dir := range manifests.CiliumV14Directories { + files, err := manifests.CiliumManifests.ReadDir(dir) + if err != nil { + return fmt.Errorf("error reading manifest directory %s from embed: %w", dir, err) + } + + for _, file := range files { + b, err := manifests.CiliumManifests.ReadFile(fmt.Sprintf("%s/%s", dir, file.Name())) + if err != nil { + return fmt.Errorf("error reading manifest file %s from embed: %w", file.Name(), err) + } + if len(b) == 0 { + return fmt.Errorf("manifest file %s from embed is empty: %w", file.Name(), ErrEmptyFile) + } + + } + } + + if len(c.ResourceGroupName) > 80 { //nolint:gomnd // 80 is the max length for resource group names + return fmt.Errorf("resource group name for nodes cannot exceed 80 characters: %w", ErrResourceNameTooLong) + } + + return nil +} + +func (c *CreateBYOCiliumCluster) Postvalidate() error { + return nil +} + +func (c *CreateBYOCiliumCluster) Run() error { + // Start with default cluster template + ciliumCluster := GetStarterClusterTemplate(c.Location) + ciliumCluster.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginNone) + ciliumCluster.Properties.NetworkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay) + ciliumCluster.Properties.NetworkProfile.PodCidr = to.Ptr(c.PodCidr) + ciliumCluster.Properties.NetworkProfile.DNSServiceIP = to.Ptr(c.DNSServiceIP) + ciliumCluster.Properties.NetworkProfile.ServiceCidr = to.Ptr(c.ServiceCidr) + subnetkey := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", c.SubscriptionID, c.ResourceGroupName, c.VnetName, c.SubnetName) + ciliumCluster.Properties.AgentPoolProfiles[0].VnetSubnetID = to.Ptr(subnetkey) + + // Set the kubeproxy config + kubeProxyConfig := armcontainerservice.NetworkProfileKubeProxyConfig{ + Mode: to.Ptr(armcontainerservice.ModeIPVS), + Enabled: to.Ptr(false), + IpvsConfig: to.Ptr(armcontainerservice.NetworkProfileKubeProxyConfigIpvsConfig{ + Scheduler: to.Ptr(armcontainerservice.IpvsSchedulerLeastConnection), + TCPTimeoutSeconds: to.Ptr(int32(900)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json + TCPFinTimeoutSeconds: to.Ptr(int32(120)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json + UDPTimeoutSeconds: to.Ptr(int32(300)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json + }), + } + + log.Printf("using kube-proxy config:\n") + printjson(kubeProxyConfig) + ciliumCluster.Properties.NetworkProfile.KubeProxyConfig = to.Ptr(kubeProxyConfig) + + // Deploy cluster + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout) + defer cancel() + + clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create az client: %w", err) + } + + log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + + poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil) + if err != nil { + return fmt.Errorf("failed to finish the create cluster request: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to create cluster: %w", err) + } + + // get kubeconfig + log.Printf("getting kubeconfig for cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + clientset, err := c.getKubeConfig() + if err != nil { + return fmt.Errorf("failed to get kubeconfig for cluster \"%s\": %w", c.ClusterName, err) + } + + // Deploy the cilium components once the cluster is created + log.Printf("deploying cilium components to cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + err = c.deployCiliumComponents(clientset) + if err != nil { + return fmt.Errorf("failed to deploy cilium components: %w", err) + } + + // wait for cilium pods to be ready + err = k8s.WaitForPodReady(ctx, clientset, "kube-system", "k8s-app=cilium") + if err != nil { + return fmt.Errorf("failed to wait for cilium pods to be ready: %w", err) + } + + return nil +} + +func (c *CreateBYOCiliumCluster) getKubeConfig() (*kubernetes.Clientset, error) { + // create temporary directory for kubeconfig, as we need access to deploy cilium things + dir, err := os.MkdirTemp("", "cilium-e2e") + if err != nil { + return nil, fmt.Errorf("failed to create temporary directory to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + // reuse getKubeConfig job + kubeconfigpath := dir + "/kubeconfig" + getKubeconfigJob := GetAKSKubeConfig{ + ClusterName: c.ClusterName, + SubscriptionID: c.SubscriptionID, + ResourceGroupName: c.ResourceGroupName, + Location: c.Location, + KubeConfigFilePath: kubeconfigpath, + } + + err = getKubeconfigJob.Run() + if err != nil { + return nil, fmt.Errorf("failed to get kubeconfig to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + return nil, fmt.Errorf("failed to build kubeconfig to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes client to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + return clientset, nil +} + +func (c *CreateBYOCiliumCluster) deployCiliumComponents(clientset *kubernetes.Clientset) error { + // traverse the predefined Cilium component folders + for _, dir := range manifests.CiliumV14Directories { + + files, err := manifests.CiliumManifests.ReadDir(dir) + if err != nil { + return fmt.Errorf("error reading manifest directory %s from embed: %w", dir, err) + } + + for _, file := range files { + yamlFile, err := manifests.CiliumManifests.ReadFile(fmt.Sprintf("%s/%s", dir, file.Name())) + if err != nil { + return fmt.Errorf("error reading YAML file: %w", err) + } + + // Decode the YAML file into a Kubernetes object + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(yamlFile), nil, nil) + if err != nil { + return fmt.Errorf("error decoding YAML file: %w", err) + } + + // create the resource + err = k8s.CreateResource(context.Background(), obj, clientset) + if err != nil { + return fmt.Errorf("error creating resource: %w", err) + } + } + } + + return nil +} diff --git a/test/e2e/framework/azure/create-cluster.go b/test/e2e/framework/azure/create-cluster.go new file mode 100644 index 0000000000..54250bbabc --- /dev/null +++ b/test/e2e/framework/azure/create-cluster.go @@ -0,0 +1,115 @@ +package azure + +import ( + "context" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" +) + +const ( + MaxNumberOfNodes = 3 + MaxPodsPerNode = 250 + AgentSKU = "Standard_D4s_v3" +) + +var defaultClusterCreateTimeout = 30 * time.Minute + +type CreateCluster struct { + SubscriptionID string + ResourceGroupName string + Location string + ClusterName string +} + +func (c *CreateCluster) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout) + defer cancel() + clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, GetStarterClusterTemplate(c.Location), nil) + if err != nil { + return fmt.Errorf("failed to finish the create cluster request: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the create cluster result: %w", err) + } + + return nil +} + +func GetStarterClusterTemplate(location string) armcontainerservice.ManagedCluster { + id := armcontainerservice.ResourceIdentityTypeSystemAssigned + return armcontainerservice.ManagedCluster{ + Location: to.Ptr(location), + Tags: map[string]*string{ + "archv2": to.Ptr(""), + "tier": to.Ptr("production"), + }, + Properties: &armcontainerservice.ManagedClusterProperties{ + AddonProfiles: map[string]*armcontainerservice.ManagedClusterAddonProfile{}, + /* Moving this to a separate stage to enable AMA since it takes some time to provision + AzureMonitorProfile: &armcontainerservice.ManagedClusterAzureMonitorProfile{ + Metrics: &armcontainerservice.ManagedClusterAzureMonitorProfileMetrics{ + Enabled: to.Ptr(true), + }, + }, + */ + AgentPoolProfiles: []*armcontainerservice.ManagedClusterAgentPoolProfile{ + { + Type: to.Ptr(armcontainerservice.AgentPoolTypeVirtualMachineScaleSets), + AvailabilityZones: []*string{to.Ptr("1")}, + Count: to.Ptr[int32](MaxNumberOfNodes), + EnableNodePublicIP: to.Ptr(false), + Mode: to.Ptr(armcontainerservice.AgentPoolModeSystem), + OSType: to.Ptr(armcontainerservice.OSTypeLinux), + ScaleDownMode: to.Ptr(armcontainerservice.ScaleDownModeDelete), + VMSize: to.Ptr(AgentSKU), + Name: to.Ptr("nodepool1"), + MaxPods: to.Ptr(int32(MaxPodsPerNode)), + }, + }, + KubernetesVersion: to.Ptr(""), + DNSPrefix: to.Ptr("dnsprefix1"), + EnablePodSecurityPolicy: to.Ptr(false), + EnableRBAC: to.Ptr(true), + LinuxProfile: nil, + NetworkProfile: &armcontainerservice.NetworkProfile{ + LoadBalancerSKU: to.Ptr(armcontainerservice.LoadBalancerSKUStandard), + OutboundType: to.Ptr(armcontainerservice.OutboundTypeLoadBalancer), + NetworkPlugin: to.Ptr(armcontainerservice.NetworkPluginAzure), + }, + WindowsProfile: &armcontainerservice.ManagedClusterWindowsProfile{ + AdminPassword: to.Ptr("replacePassword1234$"), + AdminUsername: to.Ptr("azureuser"), + }, + }, + Identity: &armcontainerservice.ManagedClusterIdentity{ + Type: &id, + }, + + SKU: &armcontainerservice.ManagedClusterSKU{ + Name: to.Ptr(armcontainerservice.ManagedClusterSKUName("Base")), + Tier: to.Ptr(armcontainerservice.ManagedClusterSKUTierStandard), + }, + } +} + +func (c *CreateCluster) Prevalidate() error { + return nil +} + +func (c *CreateCluster) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/create-rg.go b/test/e2e/framework/azure/create-rg.go new file mode 100644 index 0000000000..efa1d88ea5 --- /dev/null +++ b/test/e2e/framework/azure/create-rg.go @@ -0,0 +1,48 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" +) + +type CreateResourceGroup struct { + SubscriptionID string + ResourceGroupName string + Location string +} + +func (c *CreateResourceGroup) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armresources.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create resource group client: %w", err) + } + log.Printf("creating resource group %s in location %s...", c.ResourceGroupName, c.Location) + + _, err = clientFactory.NewResourceGroupsClient().CreateOrUpdate(ctx, c.ResourceGroupName, armresources.ResourceGroup{ + Location: to.Ptr(c.Location), + }, nil) + if err != nil { + return fmt.Errorf("failed to finish the request: %w", err) + } + + log.Printf("resource group created %s in location %s", c.ResourceGroupName, c.Location) + return nil +} + +func (c *CreateResourceGroup) Prevalidate() error { + return nil +} + +func (c *CreateResourceGroup) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/create-vnet.go b/test/e2e/framework/azure/create-vnet.go new file mode 100644 index 0000000000..0149a14c8d --- /dev/null +++ b/test/e2e/framework/azure/create-vnet.go @@ -0,0 +1,110 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" +) + +const FlowTimeoutInMinutes = 10 + +type CreateVNet struct { + SubscriptionID string + ResourceGroupName string + Location string + VnetName string + VnetAddressSpace string +} + +func (c *CreateVNet) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armnetwork.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + log.Printf("creating vnet \"%s\" in resource group \"%s\"...", c.VnetName, c.ResourceGroupName) + + poller, err := clientFactory.NewVirtualNetworksClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.VnetName, armnetwork.VirtualNetwork{ + Location: to.Ptr(c.Location), + Properties: &armnetwork.VirtualNetworkPropertiesFormat{ + AddressSpace: &armnetwork.AddressSpace{ + AddressPrefixes: []*string{ + to.Ptr(c.VnetAddressSpace), + }, + }, + FlowTimeoutInMinutes: to.Ptr[int32](FlowTimeoutInMinutes), + }, + }, nil) + if err != nil { + return fmt.Errorf("failed to finish the request for create vnet: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result for create vnet: %w", err) + } + return nil +} + +func (c *CreateVNet) Prevalidate() error { + return nil +} + +func (c *CreateVNet) Postvalidate() error { + return nil +} + +type CreateSubnet struct { + SubscriptionID string + ResourceGroupName string + Location string + VnetName string + SubnetName string + SubnetAddressSpace string +} + +func (c *CreateSubnet) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armnetwork.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + log.Printf("creating subnet \"%s\" in vnet \"%s\" in resource group \"%s\"...", c.SubnetName, c.VnetName, c.ResourceGroupName) + + poller, err := clientFactory.NewSubnetsClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.VnetName, c.SubnetName, armnetwork.Subnet{ + Properties: &armnetwork.SubnetPropertiesFormat{ + AddressPrefix: to.Ptr(c.SubnetAddressSpace), + }, + }, nil) + if err != nil { + return fmt.Errorf("failed to finish the request for create subnet: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result for create subnet: %w", err) + } + return nil +} + +func (c *CreateSubnet) Prevalidate() error { + return nil +} + +func (c *CreateSubnet) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/delete-cluster.go b/test/e2e/framework/azure/delete-cluster.go new file mode 100644 index 0000000000..674a2fbb62 --- /dev/null +++ b/test/e2e/framework/azure/delete-cluster.go @@ -0,0 +1,48 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" +) + +type DeleteCluster struct { + ClusterName string + SubscriptionID string + ResourceGroupName string + Location string +} + +func (d *DeleteCluster) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armcontainerservice.NewClientFactory(d.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + log.Printf("deleting cluster %s in resource group %s...", d.ClusterName, d.ResourceGroupName) + poller, err := clientFactory.NewManagedClustersClient().BeginDelete(ctx, d.ResourceGroupName, d.ClusterName, nil) + if err != nil { + return fmt.Errorf("failed to finish the request: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result: %w", err) + } + return nil +} + +func (d *DeleteCluster) Prevalidate() error { + return nil +} + +func (d *DeleteCluster) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/delete-rg.go b/test/e2e/framework/azure/delete-rg.go new file mode 100644 index 0000000000..0d65df6d5f --- /dev/null +++ b/test/e2e/framework/azure/delete-rg.go @@ -0,0 +1,52 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" +) + +type DeleteResourceGroup struct { + SubscriptionID string + ResourceGroupName string + Location string +} + +func (d *DeleteResourceGroup) Run() error { + log.Printf("deleting resource group \"%s\"...", d.ResourceGroupName) + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armresources.NewClientFactory(d.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create resource group client: %w", err) + } + + forceDeleteType := "Microsoft.Compute/virtualMachines,Microsoft.Compute/virtualMachineScaleSets" + poller, err := clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)}) + if err != nil { + return fmt.Errorf("failed to finish the delete resource group request: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result for delete resource group: %w", err) + } + + log.Printf("resource group \"%s\" deleted successfully", d.ResourceGroupName) + return nil +} + +func (d *DeleteResourceGroup) Prevalidate() error { + return nil +} + +func (d *DeleteResourceGroup) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/enable-ama.go b/test/e2e/framework/azure/enable-ama.go new file mode 100644 index 0000000000..e6bdaf4e61 --- /dev/null +++ b/test/e2e/framework/azure/enable-ama.go @@ -0,0 +1,117 @@ +package azure + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" +) + +const fileperms = 0o600 + +type CreateAzureMonitor struct { + SubscriptionID string + ResourceGroupName string + Location string + ClusterName string +} + +func (c *CreateAzureMonitor) Run() error { + log.Printf(`this will deploy azure monitor workspace and grafana, but as of 1/9/2024, the api docs don't show how to do +az aks update --enable-azure-monitor-metrics \ +-n $NAME \ +-g $CLUSTER_RESOURCE_GROUP \ +--azure-monitor-workspace-resource-id $AZMON_RESOURCE_ID \ +--grafana-resource-id $GRAFANA_RESOURCE_ID +`) + + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + + ctx := context.Background() + amaClientFactory, err := armmonitor.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create azure monitor workspace client: %w", err) + } + log.Printf("creating resource group %s in location %s...", c.ResourceGroupName, c.Location) + + // create azure monitor + _, err = amaClientFactory.NewAzureMonitorWorkspacesClient().Create(ctx, c.ResourceGroupName, "test", armmonitor.AzureMonitorWorkspaceResource{ + Location: &c.Location, + }, &armmonitor.AzureMonitorWorkspacesClientCreateOptions{}) + if err != nil { + return fmt.Errorf("failed to azure monitor workspace: %w", err) + } + + // Create grafana + + granafaClientFactory, err := armdashboard.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create grafana client: %w", err) + } + + _, err = granafaClientFactory.NewGrafanaClient().BeginCreate(ctx, c.ResourceGroupName, "test", armdashboard.ManagedGrafana{}, &armdashboard.GrafanaClientBeginCreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create grafana: %w", err) + } + + log.Printf("azure monitor workspace %s in location %s", c.ResourceGroupName, c.Location) + + // update aks cluster + + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout) + defer cancel() + aksClientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + cluster, err := aksClientFactory.NewManagedClustersClient().Get(ctx, c.ResourceGroupName, c.ClusterName, nil) + if err != nil { + return fmt.Errorf("failed to get cluster to enable AMA: %w", err) + } + + // enable Azure Monitor Metrics + cluster.Properties.AzureMonitorProfile.Metrics.Enabled = to.Ptr(true) + + // Marshal the struct into a JSON byte array with indentation + jsonData, err := json.MarshalIndent(cluster, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal cluster to JSON for AMA: %w", err) + } + + // Write the JSON data to a file + err = os.WriteFile("cluster.json", jsonData, fileperms) + if err != nil { + return fmt.Errorf("failed to write cluster JSON to file for AMA: %w", err) + } + + poller, err := aksClientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, GetStarterClusterTemplate(c.Location), nil) + if err != nil { + return fmt.Errorf("failed to finish the update cluster request for AMA: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to enable AMA on cluster %s: %w", *cluster.Name, err) + } + + return nil +} + +func (c *CreateAzureMonitor) Prevalidate() error { + return nil +} + +func (c *CreateAzureMonitor) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/get-kubeconfig.go b/test/e2e/framework/azure/get-kubeconfig.go new file mode 100644 index 0000000000..08cb841403 --- /dev/null +++ b/test/e2e/framework/azure/get-kubeconfig.go @@ -0,0 +1,53 @@ +package azure + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" +) + +const KubeConfigPerms = 0o600 + +type GetAKSKubeConfig struct { + ClusterName string + SubscriptionID string + ResourceGroupName string + Location string + KubeConfigFilePath string +} + +func (c *GetAKSKubeConfig) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + res, err := clientFactory.NewManagedClustersClient().ListClusterUserCredentials(ctx, c.ResourceGroupName, c.ClusterName, nil) + if err != nil { + return fmt.Errorf("failed to finish the get managed cluster client request: %w", err) + } + + err = os.WriteFile(c.KubeConfigFilePath, []byte(res.Kubeconfigs[0].Value), KubeConfigPerms) + if err != nil { + return fmt.Errorf("failed to write kubeconfig to file \"%s\": %w", c.KubeConfigFilePath, err) + } + + log.Printf("kubeconfig for cluster \"%s\" in resource group \"%s\" written to \"%s\"\n", c.ClusterName, c.ResourceGroupName, c.KubeConfigFilePath) + return nil +} + +func (c *GetAKSKubeConfig) Prevalidate() error { + return nil +} + +func (c *GetAKSKubeConfig) Postvalidate() error { + return nil +}