ci: E2E Framework [Azure steps] [4/6]

go.mod

@@ -130,6 +130,11 @@ require (
 )
 
 require (
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
 	golang.org/x/sync v0.6.0
 	gotest.tools/v3 v3.5.1
 	k8s.io/kubectl v0.28.5

go.sum

@@ -12,6 +12,21 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM=
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw=
 github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1 h1:EHlWMHGZaYhRHWNbawWxEzlZ2Eh8aTXNs8t7iAibq+I=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1/go.mod h1:noQIdW75SiQFB3mSFJBr4iRRH83S9skaFiBv4C0uEs0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0 h1:MRPU8Bge2f9tkfG3PCr4vEnqXl8XOSjlhuK3l+8Hvkc=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0/go.mod h1:xYrOYxajQvXMlp6M1E3amlaqPDXspyJxmjqTsGo6Jmw=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0/go.mod h1:jj6P8ybImR+5topJ+eH6fgcemSFBmU6/6bFF8KkwuDI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0 h1:9CrwzqQ+e8EqD+A2bh547GjBU4K0o30FhiTB981LFNI=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0/go.mod h1:Wfx7a5UHfOLG6O4NZ7Q0BPZUYwvlNCBR/OlIBpP3dlA=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE=
+
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA=
 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

test/e2e/framework/azure/create-cilium-cluster.go

@@ -0,0 +1,213 @@
+package azure
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes"
+	"github.com/Azure/azure-container-networking/test/e2e/manifests"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/kubectl/pkg/scheme"
+)
+
+var (
+	ErrResourceNameTooLong = fmt.Errorf("resource name too long")
+	ErrEmptyFile           = fmt.Errorf("empty file")
+)
+
+type CreateBYOCiliumCluster struct {
+	SubscriptionID    string
+	ResourceGroupName string
+	Location          string
+	ClusterName       string
+	VnetName          string
+	SubnetName        string
+	PodCidr           string
+	DNSServiceIP      string
+	ServiceCidr       string
+}
+
+func printjson(data interface{}) {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "    ")
+	if err := enc.Encode(data); err != nil {
+		panic(err)
+	}
+}
+
+func (c *CreateBYOCiliumCluster) Prevalidate() error {
+	for _, dir := range manifests.CiliumV14Directories {
+		files, err := manifests.CiliumManifests.ReadDir(dir)
+		if err != nil {
+			return fmt.Errorf("error reading manifest directory %s from embed: %w", dir, err)
+		}
+
+		for _, file := range files {
+			b, err := manifests.CiliumManifests.ReadFile(fmt.Sprintf("%s/%s", dir, file.Name()))
+			if err != nil {
+				return fmt.Errorf("error reading manifest file %s from embed: %w", file.Name(), err)
+			}
+			if len(b) == 0 {
+				return fmt.Errorf("manifest file %s from embed is empty: %w", file.Name(), ErrEmptyFile)
+			}
+
+		}
+	}
+
+	if len(c.ResourceGroupName) > 80 { //nolint:gomnd // 80 is the max length for resource group names
+		return fmt.Errorf("resource group name for nodes cannot exceed 80 characters: %w", ErrResourceNameTooLong)
+	}
+
+	return nil
+}
+
+func (c *CreateBYOCiliumCluster) Postvalidate() error {
+	return nil
+}
+
+func (c *CreateBYOCiliumCluster) Run() error {
+	// Start with default cluster template
+	ciliumCluster := GetStarterClusterTemplate(c.Location)
+	ciliumCluster.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginNone)
+	ciliumCluster.Properties.NetworkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay)
+	ciliumCluster.Properties.NetworkProfile.PodCidr = to.Ptr(c.PodCidr)
+	ciliumCluster.Properties.NetworkProfile.DNSServiceIP = to.Ptr(c.DNSServiceIP)
+	ciliumCluster.Properties.NetworkProfile.ServiceCidr = to.Ptr(c.ServiceCidr)
+	subnetkey := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", c.SubscriptionID, c.ResourceGroupName, c.VnetName, c.SubnetName)
+	ciliumCluster.Properties.AgentPoolProfiles[0].VnetSubnetID = to.Ptr(subnetkey)
+
+	// Set the kubeproxy config
+	kubeProxyConfig := armcontainerservice.NetworkProfileKubeProxyConfig{
+		Mode:    to.Ptr(armcontainerservice.ModeIPVS),
+		Enabled: to.Ptr(false),
+		IpvsConfig: to.Ptr(armcontainerservice.NetworkProfileKubeProxyConfigIpvsConfig{
+			Scheduler:            to.Ptr(armcontainerservice.IpvsSchedulerLeastConnection),
+			TCPTimeoutSeconds:    to.Ptr(int32(900)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json
+			TCPFinTimeoutSeconds: to.Ptr(int32(120)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json
+			UDPTimeoutSeconds:    to.Ptr(int32(300)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json
+		}),
+	}
+
+	log.Printf("using kube-proxy config:\n")
+	printjson(kubeProxyConfig)
+	ciliumCluster.Properties.NetworkProfile.KubeProxyConfig = to.Ptr(kubeProxyConfig)
+
+	// Deploy cluster
+	cred, err := azidentity.NewAzureCLICredential(nil)
+	if err != nil {
+		return fmt.Errorf("failed to obtain a credential: %w", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout)
+	defer cancel()
+
+	clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create az client: %w", err)
+	}
+
+	log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName)
+
+	poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil)
+	if err != nil {
+		return fmt.Errorf("failed to finish the create cluster request: %w", err)
+	}
+	_, err = poller.PollUntilDone(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create cluster: %w", err)
+	}
+
+	// get kubeconfig
+	log.Printf("getting kubeconfig for cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName)
+	clientset, err := c.getKubeConfig()
+	if err != nil {
+		return fmt.Errorf("failed to get kubeconfig for cluster \"%s\": %w", c.ClusterName, err)
+	}
+
+	// Deploy the cilium components once the cluster is created
+	log.Printf("deploying cilium components to cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName)
+	err = c.deployCiliumComponents(clientset)
+	if err != nil {
+		return fmt.Errorf("failed to deploy cilium components: %w", err)
+	}
+
+	// wait for cilium pods to be ready
+	err = k8s.WaitForPodReady(ctx, clientset, "kube-system", "k8s-app=cilium")
+	if err != nil {
+		return fmt.Errorf("failed to wait for cilium pods to be ready: %w", err)
+	}
+
+	return nil
+}
+
+func (c *CreateBYOCiliumCluster) getKubeConfig() (*kubernetes.Clientset, error) {
+	// create temporary directory for kubeconfig, as we need access to deploy cilium things
+	dir, err := os.MkdirTemp("", "cilium-e2e")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary directory to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err)
+	}
+	// reuse getKubeConfig job
+	kubeconfigpath := dir + "/kubeconfig"
+	getKubeconfigJob := GetAKSKubeConfig{
+		ClusterName:        c.ClusterName,
+		SubscriptionID:     c.SubscriptionID,
+		ResourceGroupName:  c.ResourceGroupName,
+		Location:           c.Location,
+		KubeConfigFilePath: kubeconfigpath,
+	}
+
+	err = getKubeconfigJob.Run()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get kubeconfig to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err)
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build kubeconfig to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Kubernetes client to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err)
+	}
+	return clientset, nil
+}
+
+func (c *CreateBYOCiliumCluster) deployCiliumComponents(clientset *kubernetes.Clientset) error {
+	// traverse the predefined Cilium component folders
+	for _, dir := range manifests.CiliumV14Directories {
+
+		files, err := manifests.CiliumManifests.ReadDir(dir)
+		if err != nil {
+			return fmt.Errorf("error reading manifest directory %s from embed: %w", dir, err)
+		}
+
+		for _, file := range files {
+			yamlFile, err := manifests.CiliumManifests.ReadFile(fmt.Sprintf("%s/%s", dir, file.Name()))
+			if err != nil {
+				return fmt.Errorf("error reading YAML file: %w", err)
+			}
+
+			// Decode the YAML file into a Kubernetes object
+			decode := scheme.Codecs.UniversalDeserializer().Decode
+			obj, _, err := decode([]byte(yamlFile), nil, nil)
+			if err != nil {
+				return fmt.Errorf("error decoding YAML file: %w", err)
+			}
+
+			// create the resource
+			err = k8s.CreateResource(context.Background(), obj, clientset)
+			if err != nil {
+				return fmt.Errorf("error creating resource: %w", err)
+			}
+		}
+	}
+
+	return nil
+}

test/e2e/framework/azure/create-cluster.go

@@ -0,0 +1,115 @@
+package azure
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4"
+)
+
+const (
+	MaxNumberOfNodes = 3
+	MaxPodsPerNode   = 250
+	AgentSKU         = "Standard_D4s_v3"
+)
+
+var defaultClusterCreateTimeout = 30 * time.Minute
+
+type CreateCluster struct {
+	SubscriptionID    string
+	ResourceGroupName string
+	Location          string
+	ClusterName       string
+}
+
+func (c *CreateCluster) Run() error {
+	cred, err := azidentity.NewAzureCLICredential(nil)
+	if err != nil {
+		return fmt.Errorf("failed to obtain a credential: %w", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout)
+	defer cancel()
+	clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create client: %w", err)
+	}
+
+	poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, GetStarterClusterTemplate(c.Location), nil)
+	if err != nil {
+		return fmt.Errorf("failed to finish the create cluster request: %w", err)
+	}
+	_, err = poller.PollUntilDone(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("failed to pull the create cluster result: %w", err)
+	}
+
+	return nil
+}
+
+func GetStarterClusterTemplate(location string) armcontainerservice.ManagedCluster {
+	id := armcontainerservice.ResourceIdentityTypeSystemAssigned
+	return armcontainerservice.ManagedCluster{
+		Location: to.Ptr(location),
+		Tags: map[string]*string{
+			"archv2": to.Ptr(""),
+			"tier":   to.Ptr("production"),
+		},
+		Properties: &armcontainerservice.ManagedClusterProperties{
+			AddonProfiles: map[string]*armcontainerservice.ManagedClusterAddonProfile{},
+			/* Moving this to a separate stage to enable AMA since it takes some time to provision
+			AzureMonitorProfile: &armcontainerservice.ManagedClusterAzureMonitorProfile{
+				Metrics: &armcontainerservice.ManagedClusterAzureMonitorProfileMetrics{
+					Enabled: to.Ptr(true),
+				},
+			},
+			*/
+			AgentPoolProfiles: []*armcontainerservice.ManagedClusterAgentPoolProfile{
+				{
+					Type:               to.Ptr(armcontainerservice.AgentPoolTypeVirtualMachineScaleSets),
+					AvailabilityZones:  []*string{to.Ptr("1")},
+					Count:              to.Ptr[int32](MaxNumberOfNodes),
+					EnableNodePublicIP: to.Ptr(false),
+					Mode:               to.Ptr(armcontainerservice.AgentPoolModeSystem),
+					OSType:             to.Ptr(armcontainerservice.OSTypeLinux),
+					ScaleDownMode:      to.Ptr(armcontainerservice.ScaleDownModeDelete),
+					VMSize:             to.Ptr(AgentSKU),
+					Name:               to.Ptr("nodepool1"),
+					MaxPods:            to.Ptr(int32(MaxPodsPerNode)),
+				},
+			},
+			KubernetesVersion:       to.Ptr(""),
+			DNSPrefix:               to.Ptr("dnsprefix1"),
+			EnablePodSecurityPolicy: to.Ptr(false),
+			EnableRBAC:              to.Ptr(true),
+			LinuxProfile:            nil,
+			NetworkProfile: &armcontainerservice.NetworkProfile{
+				LoadBalancerSKU: to.Ptr(armcontainerservice.LoadBalancerSKUStandard),
+				OutboundType:    to.Ptr(armcontainerservice.OutboundTypeLoadBalancer),
+				NetworkPlugin:   to.Ptr(armcontainerservice.NetworkPluginAzure),
+			},
+			WindowsProfile: &armcontainerservice.ManagedClusterWindowsProfile{
+				AdminPassword: to.Ptr("replacePassword1234$"),
+				AdminUsername: to.Ptr("azureuser"),
+			},
+		},
+		Identity: &armcontainerservice.ManagedClusterIdentity{
+			Type: &id,
+		},
+
+		SKU: &armcontainerservice.ManagedClusterSKU{
+			Name: to.Ptr(armcontainerservice.ManagedClusterSKUName("Base")),
+			Tier: to.Ptr(armcontainerservice.ManagedClusterSKUTierStandard),
+		},
+	}
+}
+
+func (c *CreateCluster) Prevalidate() error {
+	return nil
+}
+
+func (c *CreateCluster) Postvalidate() error {
+	return nil
+}