diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 5e45e0261b..114cb615c1 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -182,6 +182,8 @@ type PodInfo interface { Equals(PodInfo) bool // String implements string for logging PodInfos String() string + // SecondaryInterfacesExist returns true if there exist a secondary interface for this pod + SecondaryInterfacesExist() bool } type KubernetesPodInfo struct { @@ -194,9 +196,10 @@ var _ PodInfo = (*podInfo)(nil) // podInfo implements PodInfo for multiple schemas of Key type podInfo struct { KubernetesPodInfo - PodInfraContainerID string - PodInterfaceID string - Version podInfoScheme + PodInfraContainerID string + PodInterfaceID string + Version podInfoScheme + SecondaryInterfaceSet bool } func (p podInfo) String() string { @@ -250,6 +253,10 @@ func (p *podInfo) OrchestratorContext() (json.RawMessage, error) { return jsonContext, nil } +func (p *podInfo) SecondaryInterfacesExist() bool { + return p.SecondaryInterfaceSet +} + // NewPodInfo returns an implementation of PodInfo that returns the passed // configuration for their namesake functions. func NewPodInfo(infraContainerID, interfaceID, name, namespace string) PodInfo { @@ -287,6 +294,7 @@ func NewPodInfoFromIPConfigsRequest(req IPConfigsRequest) (PodInfo, error) { } p.(*podInfo).PodInfraContainerID = req.InfraContainerID p.(*podInfo).PodInterfaceID = req.PodInterfaceID + p.(*podInfo).SecondaryInterfaceSet = req.SecondaryInterfacesExist return p, nil } @@ -442,11 +450,12 @@ type IPConfigRequest struct { // Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice type IPConfigsRequest struct { - DesiredIPAddresses []string `json:"desiredIPAddresses"` - PodInterfaceID string `json:"podInterfaceID"` - InfraContainerID string `json:"infraContainerID"` - OrchestratorContext json.RawMessage `json:"orchestratorContext"` - Ifname string `json:"ifname"` // Used by delegated IPAM + DesiredIPAddresses []string `json:"desiredIPAddresses"` + PodInterfaceID string `json:"podInterfaceID"` + InfraContainerID string `json:"infraContainerID"` + OrchestratorContext json.RawMessage `json:"orchestratorContext"` + Ifname string `json:"ifname"` // Used by delegated IPAM + SecondaryInterfacesExist bool `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func } // IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD diff --git a/cns/api.go b/cns/api.go index 705907b238..19f8a63da8 100644 --- a/cns/api.go +++ b/cns/api.go @@ -48,8 +48,18 @@ type HTTPService interface { GetPendingReleaseIPConfigs() []IPConfigurationStatus GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) + AttachSWIFTv2Middleware(middleware SWIFTv2Middleware) } +// Middleware interface for testing later on +type SWIFTv2Middleware interface { + ValidateIPConfigsRequest(context.Context, *IPConfigsRequest) (types.ResponseCode, string) + GetIPConfig(context.Context, PodInfo) (PodIpInfo, error) + SetRoutes(*PodIpInfo) error +} + +type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) + // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC type IPConfigurationStatus struct { diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 5713235df0..3d391ef579 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -11,13 +11,23 @@ const ( EnvNodeName = "NODENAME" // EnvNodeIP is the IP of the node running this CNS binary EnvNodeIP = "NODE_IP" - // LabelSwiftV2 is the Node label for Swift V2 - LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy" + // LabelNodeSwiftV2 is the Node label for Swift V2 + LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" + // LabelPodSwiftV2 is the Pod label for Swift V2 + LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" + EnvPodCIDRs = "POD_CIDRs" + EnvServiceCIDRs = "SERVICE_CIDRs" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) +// ErrPodCIDRsUnset indicates the the $EnvPodCIDRs variable is unset in the environment. +var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDRs) + +// ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment. +var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs) + // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { nodeName := os.Getenv(EnvNodeName) @@ -31,3 +41,19 @@ func NodeName() (string, error) { func NodeIP() string { return os.Getenv(EnvNodeIP) } + +func PodCIDRs() (string, error) { + podCIDRs := os.Getenv(EnvPodCIDRs) + if podCIDRs == "" { + return "", ErrPodCIDRsUnset + } + return podCIDRs, nil +} + +func ServiceCIDRs() (string, error) { + serviceCIDRs := os.Getenv(EnvServiceCIDRs) + if serviceCIDRs == "" { + return "", ErrServiceCIDRsUnset + } + return serviceCIDRs, nil +} diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index df06b6d610..cc411ec583 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -17,3 +17,23 @@ func TestNodeName(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "test", name) } + +func TestPodCIDRs(t *testing.T) { + _, err := PodCIDRs() + require.Error(t, err) + require.ErrorIs(t, err, ErrPodCIDRsUnset) + os.Setenv(EnvPodCIDRs, "test") + cidr, err := PodCIDRs() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} + +func TestServiceCIDRs(t *testing.T) { + _, err := ServiceCIDRs() + require.Error(t, err) + require.ErrorIs(t, err, ErrServiceCIDRsUnset) + os.Setenv(EnvServiceCIDRs, "test") + cidr, err := ServiceCIDRs() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} diff --git a/cns/fakes/cnsfake.go b/cns/fakes/cnsfake.go index 07618b01e4..aed304f976 100644 --- a/cns/fakes/cnsfake.go +++ b/cns/fakes/cnsfake.go @@ -276,3 +276,5 @@ func (fake *HTTPServiceFake) Init(*common.ServiceConfig) error { } func (fake *HTTPServiceFake) Stop() {} + +func (fake *HTTPServiceFake) AttachSWIFTv2Middleware(cns.SWIFTv2Middleware) {} diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go new file mode 100644 index 0000000000..6432d55852 --- /dev/null +++ b/cns/middlewares/mock/mockClient.go @@ -0,0 +1,65 @@ +package middlewares + +import ( + "context" + "errors" + + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + errPodNotFound = errors.New("pod not found") + errMTPNCNotFound = errors.New("mtpnc not found") +) + +// MockClient implements the client.Client interface for testing. We only care about Get, the rest is nil ops. +type MockClient struct { + client.Client + mtPodCache map[string]*v1.Pod + mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig +} + +// NewMockClient returns a new MockClient. +func NewMockClient() *MockClient { + testPod1 := v1.Pod{} + testPod1.Labels = make(map[string]string) + testPod1.Labels[configuration.LabelPodSwiftV2] = "true" + + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC1.Status.PrimaryIP = "192.168.0.1" + testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" + testMTPNC1.Status.GatewayIP = "10.0.0.1" + testMTPNC1.Status.NCID = "testncid" + + testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{} + + return &MockClient{ + mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, + mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ + "testpod1namespace/testpod1": &testMTPNC1, + "testpod3namespace/testpod3": &testMTPNC3, + }, + } +} + +// Get implements client.Client.Get. +func (c *MockClient) Get(_ context.Context, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error { + switch o := obj.(type) { + case *v1.Pod: + if pod, ok := c.mtPodCache[key.String()]; ok { + *o = *pod + } else { + return errPodNotFound + } + case *v1alpha1.MultitenantPodNetworkConfig: + if mtpnc, ok := c.mtpncCache[key.String()]; ok { + *o = *mtpnc + } else { + return errMTPNCNotFound + } + } + return nil +} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go new file mode 100644 index 0000000000..dd0966c76e --- /dev/null +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -0,0 +1,92 @@ +package middlewares + +import ( + "context" + "errors" + "fmt" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + v1 "k8s.io/api/core/v1" + k8types "k8s.io/apimachinery/pkg/types" +) + +var ( + errMTPNCNotReady = errors.New("mtpnc is not ready") + errFailedToGetPod = errors.New("failed to get pod") +) + +type MockSWIFTv2Middleware struct { + mtPodState map[string]*v1.Pod + mtpncState map[string]*v1alpha1.MultitenantPodNetworkConfig +} + +func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { + testPod1 := v1.Pod{} + testPod1.Labels = make(map[string]string) + testPod1.Labels[configuration.LabelPodSwiftV2] = "true" + + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC1.Status.PrimaryIP = "192.168.0.1" + testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" + testMTPNC1.Status.GatewayIP = "10.0.0.1" + testMTPNC1.Status.NCID = "testncid" + + return &MockSWIFTv2Middleware{ + mtPodState: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, + mtpncState: map[string]*v1alpha1.MultitenantPodNetworkConfig{"testpod1namespace/testpod1": &testMTPNC1}, + } +} + +// validateMultitenantIPConfigsRequest validates if pod is multitenant +// nolint +func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) + return types.UnexpectedError, errBuf + } + podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod, ok := m.mtPodState[podNamespacedName.String()] + if !ok { + errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) + return types.UnexpectedError, errBuf + } + // check the pod labels for Swift V2, enrich the request with the multitenant flag. + if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { + req.SecondaryInterfacesExist = true + } + return types.Success, "" +} + +// GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) +// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD +func (m *MockSWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] + if !ok { + return cns.PodIpInfo{}, errFailedToGetPod + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return cns.PodIpInfo{}, errMTPNCNotReady + } + podIPInfo := cns.PodIpInfo{} + podIPInfo.PodIPConfig = cns.IPSubnet{ + IPAddress: mtpnc.Status.PrimaryIP, + } + podIPInfo.MacAddress = mtpnc.Status.MacAddress + podIPInfo.NICType = cns.DelegatedVMNIC + podIPInfo.SkipDefaultRoutes = false + + return podIPInfo, nil +} + +func (m *MockSWIFTv2Middleware) SetRoutes(_ *cns.PodIpInfo) error { + return nil +} diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go new file mode 100644 index 0000000000..cea9e2f118 --- /dev/null +++ b/cns/middlewares/swiftV2.go @@ -0,0 +1,169 @@ +package middlewares + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + v1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + ErrMTPNCNotReady = errors.New("mtpnc is not ready") + ErrInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") +) + +const ( + prefixLength = 32 + overlayGatewayv4 = "169.254.1.1" + overlayGatewayV6 = "fe80::1234:5678:9abc" +) + +type SWIFTv2Middleware struct { + Cli client.Client +} + +// ValidateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario. +// nolint +func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) + return types.UnexpectedError, errBuf + } + podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { + errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) + return types.UnexpectedError, errBuf + } + + // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true. + if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { + req.SecondaryInterfacesExist = true + } + return types.Success, "" +} + +// GetIPConfig returns the pod's SWIFT V2 IP configuration. +func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return cns.PodIpInfo{}, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return cns.PodIpInfo{}, ErrMTPNCNotReady + } + podIPInfo := cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: mtpnc.Status.PrimaryIP, + PrefixLength: prefixLength, + }, + MacAddress: mtpnc.Status.MacAddress, + NICType: cns.DelegatedVMNIC, + SkipDefaultRoutes: false, + // InterfaceName is empty for DelegatedVMNIC + } + + return podIPInfo, nil +} + +// SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. +func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + podIPInfo.Routes = []cns.Route{} + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + // default route via SWIFT v2 interface + route := cns.Route{ + IPAddress: "0.0.0.0/0", + } + podIPInfo.Routes = []cns.Route{route} + case cns.InfraNIC: + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return fmt.Errorf("failed to get podCIDRs from env : %w", err) + } + podCIDRsV4, podCIDRv6, err := parseCIDRs(podCIDRs) + if err != nil { + return fmt.Errorf("failed to parse podCIDRs : %w", err) + } + + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) + } + serviceCIDRsV4, serviceCIDRsV6, err := parseCIDRs(serviceCIDRs) + if err != nil { + return fmt.Errorf("failed to parse serviceCIDRs : %w", err) + } + // Check if the podIPInfo is IPv4 or IPv6 + if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { + // routes for IPv4 podCIDR traffic + for _, podCIDRv4 := range podCIDRsV4 { + podCIDRv4Route := cns.Route{ + IPAddress: podCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route) + } + // route for IPv4 serviceCIDR traffic + for _, serviceCIDRv4 := range serviceCIDRsV4 { + serviceCIDRv4Route := cns.Route{ + IPAddress: serviceCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) + } + + } else { + // routes for IPv6 podCIDR traffic + for _, podCIDRv6 := range podCIDRv6 { + podCIDRv6Route := cns.Route{ + IPAddress: podCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route) + } + // route for IPv6 serviceCIDR traffic + for _, serviceCIDRv6 := range serviceCIDRsV6 { + serviceCIDRv6Route := cns.Route{ + IPAddress: serviceCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) + } + } + default: + return ErrInvalidSWIFTv2NICType + } + return nil +} + +// parseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. +func parseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { + for _, cidr := range strings.Split(cidrs, ",") { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse cidr %s : %w", cidr, err) + } + if ip.To4() != nil { + v4IPs = append(v4IPs, cidr) + } else { + v6IPs = append(v6IPs, cidr) + } + } + return v4IPs, v6IPs, nil +} diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go new file mode 100644 index 0000000000..d2ffd18d50 --- /dev/null +++ b/cns/middlewares/swiftV2_test.go @@ -0,0 +1,175 @@ +package middlewares + +import ( + "context" + "os" + "testing" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + mock "github.com/Azure/azure-container-networking/cns/middlewares/mock" + "github.com/Azure/azure-container-networking/cns/types" + "gotest.tools/v3/assert" +) + +var ( + testPod1GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" + testPod1Info = cns.NewPodInfo("898fb8-eth0", testPod1GUID, "testpod1", "testpod1namespace") + + testPod2GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" + testPod2Info = cns.NewPodInfo("b21e1e-eth0", testPod2GUID, "testpod2", "testpod2namespace") + + testPod3GUID = "718e04ac-5a13-4dce-84b3-040accaa9b41" + testPod3Info = cns.NewPodInfo("718e04-eth0", testPod3GUID, "testpod3", "testpod3namespace") +) + +func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + + happyReq := &cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + happyReq.OrchestratorContext = b + happyReq.SecondaryInterfacesExist = false + + respCode, err := middleware.ValidateIPConfigsRequest(context.TODO(), happyReq) + assert.Equal(t, err, "") + assert.Equal(t, respCode, types.Success) + assert.Equal(t, happyReq.SecondaryInterfacesExist, true) +} + +func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + + // Fail to unmarshal pod info test + failReq := &cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + failReq.OrchestratorContext = []byte("invalid") + respCode, _ := middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) + + // Pod doesn't exist in cache test + failReq = &cns.IPConfigsRequest{ + PodInterfaceID: testPod2Info.InterfaceID(), + InfraContainerID: testPod2Info.InfraContainerID(), + } + b, _ := testPod2Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) +} + +func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") + + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + + ipInfo, err := middleware.GetIPConfig(context.TODO(), testPod1Info) + assert.Equal(t, err, nil) + assert.Equal(t, ipInfo.NICType, cns.DelegatedVMNIC) + assert.Equal(t, ipInfo.SkipDefaultRoutes, false) +} + +func TestGetSWIFTv2IPConfigFailure(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + + // Pod's MTPNC doesn't exist in cache test + _, err := middleware.GetIPConfig(context.TODO(), testPod2Info) + assert.Error(t, err, "failed to get pod's mtpnc from cache : mtpnc not found") + + // Pod's MTPNC is not ready test + _, err = middleware.GetIPConfig(context.TODO(), testPod3Info) + assert.Error(t, err, ErrMTPNCNotReady.Error()) +} + +func TestSetRoutesSuccess(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24;16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16;16A0:0010:AB00:0000::/32") + podIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + }, + } + desiredPodIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "10.0.1.10/24", + GatewayIPAddress: overlayGatewayv4, + }, + { + IPAddress: "10.0.0.0/16", + GatewayIPAddress: overlayGatewayv4, + }, + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "16A0:0010:AB00:001E::2/32", + GatewayIPAddress: overlayGatewayV6, + }, + { + IPAddress: "16A0:0010:AB00:0000::/32", + GatewayIPAddress: overlayGatewayV6, + }, + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + Routes: []cns.Route{ + { + IPAddress: "0.0.0.0/0", + }, + }, + }, + } + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := middleware.SetRoutes(ipInfo) + assert.Equal(t, err, nil) + } + for i := range podIPInfo { + assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) + } +} diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go index 854fb3bc81..34ec365d9a 100644 --- a/cns/restserver/internalapi_test.go +++ b/cns/restserver/internalapi_test.go @@ -30,6 +30,10 @@ import ( const ( primaryIp = "10.0.0.5" + SWIFTv2IP = "192.168.0.1" + SWIFTv2MAC = "00:00:00:00:00:00" + SWIFTv2GatewayIP = "10.0.0.1" + SWIFTv2NCID = "testncid" gatewayIp = "10.0.0.1" subnetPrfixLength = 24 dockerContainerType = cns.Docker diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 93984a8578..2e51ef6bde 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "net" "net/http" @@ -24,8 +25,9 @@ var ( ) // requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response -func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) +func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -43,7 +45,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %s", err, ipconfigsRequest), + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), }, PodIPInfo: podIPInfo, }, err @@ -71,6 +73,36 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn } } + // Check if request is for pod with secondary interface(s) + if podInfo.SecondaryInterfacesExist() { + // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here + SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) + } + podIPInfo = append(podIPInfo, SWIFTv2PodIPInfo) + // Setting up routes for SWIFTv2 scenario + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := service.SWIFTv2Middleware.SetRoutes(ipInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to set SWIFTv2 routes %v", ipconfigsRequest) + } + } + } + return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.Success, @@ -117,7 +149,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r } } - ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(ipconfigsRequest) //nolint:contextcheck // appease linter + ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) //nolint:contextcheck // appease linter if errResp != nil { // As this API is expected to return IPConfigResponse, generate it from the IPConfigsResponse returned above reserveResp := &cns.IPConfigResponse{ @@ -164,7 +196,7 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r return } - ipConfigsResp, err := service.requestIPConfigHandlerHelper(ipconfigsRequest) // nolint:contextcheck // appease linter + ipConfigsResp, err := service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) // nolint:contextcheck // appease linter if err != nil { w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String()) err = service.Listener.Encode(w, &ipConfigsResp) @@ -240,8 +272,8 @@ func (service *HTTPRestService) updateEndpointState(ipconfigsRequest cns.IPConfi } // releaseIPConfigHandlerHelper validates the request and removes the endpoint associated with the pod -func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) { - podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) +func (service *HTTPRestService) releaseIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) { + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.Response{ ReturnCode: returnCode, @@ -255,7 +287,7 @@ func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cn ReturnCode: types.UnexpectedError, Message: err.Error(), } - return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error + return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %+v", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error } } @@ -263,7 +295,7 @@ func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cn return &cns.Response{ ReturnCode: types.UnexpectedError, Message: err.Error(), - }, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %s", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error + }, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %+v", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error } return &cns.Response{ @@ -313,7 +345,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r Ifname: ipconfigRequest.Ifname, } - resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest) + resp, err := service.releaseIPConfigHandlerHelper(r.Context(), ipconfigsRequest) if err != nil { w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) @@ -335,14 +367,14 @@ func (service *HTTPRestService) releaseIPConfigsHandler(w http.ResponseWriter, r ReturnCode: types.UnexpectedError, Message: err.Error(), } - logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) + logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %+v", resp.Message, ipconfigsRequest) w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) logger.ResponseEx(service.Name, ipconfigsRequest, resp, resp.ReturnCode, err) return } - resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest) + resp, err := service.releaseIPConfigHandlerHelper(r.Context(), ipconfigsRequest) if err != nil { w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index b552b15774..9b17b79393 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "net" "net/netip" @@ -13,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/fakes" + middlewares "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/Azure/azure-container-networking/store" @@ -65,6 +67,7 @@ func getTestService() *HTTPRestService { var config common.ServiceConfig httpsvc, _ := NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, store.NewMockStore(""), nil, nil) svc = httpsvc + svc.ipConfigsRequestValidators = append(svc.ipConfigsRequestValidators, svc.validateDefaultIPConfigsRequest) httpsvc.IPAMPoolMonitor = &fakes.MonitorFake{} setOrchestratorTypeInternal(cns.KubernetesCRD) @@ -1529,3 +1532,60 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { t.Fatalf("Expected fail requesting IPs due to only having one in the ipconfig map, IPs in the pool will not be assigned") } } + +func TestIPAMGetSWIFTv2IP(t *testing.T) { + svc := getTestService() + svc.AttachSWIFTv2Middleware(middlewares.NewMockSWIFTv2Middleware()) + + ncStates := []ncState{ + { + ncID: testNCID, + ips: []string{ + testIP1, + }, + }, + { + ncID: testNCIDv6, + ips: []string{ + testIP1v6, + }, + }, + } + + // Add Available Pod IP to state + for i := range ncStates { + ipconfigs := make(map[string]cns.IPConfigurationStatus, 0) + state := NewPodState(ncStates[i].ips[0], ipIDs[i][0], ncStates[i].ncID, types.Available, 0) + ipconfigs[state.ID] = state + err := UpdatePodIPConfigState(t, svc, ipconfigs, ncStates[i].ncID) + if err != nil { + t.Fatalf("Expected to not fail adding IPs to state: %+v", err) + } + } + + req := cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + req.OrchestratorContext = b + req.DesiredIPAddresses = make([]string, 2) + req.DesiredIPAddresses[0] = testIP1 + req.DesiredIPAddresses[1] = testIP1v6 + + resp, err := svc.requestIPConfigHandlerHelper(context.TODO(), req) + if err != nil { + t.Fatalf("Expected to not fail requesting IPs: %+v", err) + } + podIPInfo := resp.PodIPInfo + + if len(podIPInfo) != 3 { + t.Fatalf("Expected to get 3 pod IP info (IPv4, IPv6, Multitenant IP), actual %d", len(podIPInfo)) + } + + // Asserting that SWIFT v2 IP is returned + assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) + assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MacAddress) + assert.Equal(t, cns.DelegatedVMNIC, podIPInfo[2].NICType) + assert.False(t, podIPInfo[2].SkipDefaultRoutes) +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 70676c28e3..08fff69664 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -30,10 +30,8 @@ import ( // All helper/utility functions - util.go // Constants - const.go -var ( - // Named Lock for accessing different states in httpRestServiceState - namedLock = acn.InitNamedLock() -) +// Named Lock for accessing different states in httpRestServiceState +var namedLock = acn.InitNamedLock() type interfaceGetter interface { GetInterfaces(ctx context.Context) (*wireserver.GetInterfacesResult, error) @@ -69,11 +67,13 @@ type HTTPRestService struct { state *httpRestServiceState podsPendingIPAssignment *bounded.TimedSet sync.RWMutex - dncPartitionKey string - EndpointState map[string]*EndpointInfo // key : container id - EndpointStateStore store.KeyValueStore - cniConflistGenerator CNIConflistGenerator - generateCNIConflistOnce sync.Once + dncPartitionKey string + EndpointState map[string]*EndpointInfo // key : container id + EndpointStateStore store.KeyValueStore + cniConflistGenerator CNIConflistGenerator + generateCNIConflistOnce sync.Once + ipConfigsRequestValidators []cns.IPConfigsRequestValidator + SWIFTv2Middleware cns.SWIFTv2Middleware } type CNIConflistGenerator interface { @@ -228,6 +228,9 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { return err } + // Adding the default ipconfigs request validator + service.ipConfigsRequestValidators = []cns.IPConfigsRequestValidator{service.validateDefaultIPConfigsRequest} + // Add handlers. listener := service.Listener // default handlers @@ -349,3 +352,9 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { } }) } + +func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { + service.SWIFTv2Middleware = middleware + // adding the SWIFT v2 ipconfigs request validator + service.ipConfigsRequestValidators = append(service.ipConfigsRequestValidators, middleware.ValidateIPConfigsRequest) +} diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 1f6fe6b23c..08139f8a35 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -766,17 +766,13 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, } } -func (service *HTTPRestService) validateIPConfigsRequest( - ipConfigsRequest cns.IPConfigsRequest, -) (cns.PodInfo, types.ResponseCode, string) { - if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { - return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" - } - - if ipConfigsRequest.OrchestratorContext == nil { - return nil, - types.EmptyOrchestratorContext, - fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) +func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) { + // looping through all the ipconfigs request validators, if any validator fails, return the error + for _, validator := range service.ipConfigsRequestValidators { + respCode, message := validator(ctx, &ipConfigsRequest) + if respCode != types.Success { + return nil, respCode, message + } } // retrieve podinfo from orchestrator context @@ -787,6 +783,18 @@ func (service *HTTPRestService) validateIPConfigsRequest( return podInfo, types.Success, "" } +// validateDefaultIPConfigsRequest validates the request for default IP configs request +func (service *HTTPRestService) validateDefaultIPConfigsRequest(_ context.Context, ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { + return types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" + } + + if ipConfigsRequest.OrchestratorContext == nil { + return types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) + } + return types.Success, "" +} + // getPrimaryHostInterface returns the cached InterfaceInfo, if available, otherwise // queries the IMDS to get the primary interface info and caches it in the server state // before returning the result. @@ -828,6 +836,7 @@ func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus podIPInfo.HostPrimaryIPInfo.PrimaryIP = primaryHostInterface.PrimaryIP podIPInfo.HostPrimaryIPInfo.Subnet = primaryHostInterface.Subnet podIPInfo.HostPrimaryIPInfo.Gateway = primaryHostInterface.Gateway + podIPInfo.NICType = cns.InfraNIC return nil } diff --git a/cns/service/main.go b/cns/service/main.go index 6f53df9a4a..875aad6747 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -37,6 +37,7 @@ import ( nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" podctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/pod" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" @@ -1154,7 +1155,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } // check the Node labels for Swift V2 - if _, ok := node.Labels[configuration.LabelSwiftV2]; ok { + if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok { cnsconfig.EnableSwiftV2 = true cnsconfig.WatchPods = true // TODO(rbtr): create the NodeInfo for Swift V2 @@ -1322,6 +1323,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err := mtpncctrl.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") } + // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service + swiftV2Middleware := middlewares.SWIFTv2Middleware{Cli: manager.GetClient()} + httpRestService.AttachSWIFTv2Middleware(&swiftV2Middleware) } // adding some routes to the root service mux diff --git a/go.mod b/go.mod index ff94c8add3..9a2ce367b0 100644 --- a/go.mod +++ b/go.mod @@ -132,7 +132,10 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) -require sigs.k8s.io/yaml v1.3.0 +require ( + gotest.tools/v3 v3.0.3 + sigs.k8s.io/yaml v1.3.0 +) require github.com/emicklei/go-restful/v3 v3.11.0 // indirect diff --git a/go.sum b/go.sum index 6d1a09c595..d744e44258 100644 --- a/go.sum +++ b/go.sum @@ -1270,7 +1270,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=