diff --git a/.github/workflows/test-forked.yml b/.github/workflows/test-forked.yml index e3bf9e9740..cdf508fea6 100644 --- a/.github/workflows/test-forked.yml +++ b/.github/workflows/test-forked.yml @@ -265,6 +265,7 @@ jobs: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }} DATADOG_KEY: ${{ secrets.DATADOG_KEY }} run: | helm version diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9facfc23b..85b38a1595 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -263,6 +263,7 @@ jobs: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }} DATADOG_KEY: ${{ secrets.DATADOG_KEY }} run: | helm version diff --git a/go.mod b/go.mod index 3e5949b1be..91e913b510 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Azure/go-autorest/autorest v0.11.24 github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/aws/aws-sdk-go v1.44.15 + github.com/aws/aws-sdk-go v1.44.16 github.com/fatih/structtag v1.2.0 github.com/go-logr/zapr v1.2.3 github.com/google/go-cmp v0.5.7 @@ -22,6 +22,7 @@ require ( go.mongodb.org/atlas v0.16.0 go.mongodb.org/mongo-driver v1.8.3 go.uber.org/zap v1.21.0 + google.golang.org/api v0.70.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8s.io/api v0.23.4 k8s.io/apimachinery v0.23.4 @@ -55,6 +56,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/gax-go/v2 v2.1.1 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect @@ -78,6 +80,7 @@ require ( github.com/xdg-go/scram v1.1.0 // indirect github.com/xdg-go/stringprep v1.0.2 // indirect github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect + go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.7.0 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect @@ -91,6 +94,8 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf // indirect + google.golang.org/grpc v1.44.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index 50f90c11ec..ac41536f49 100644 --- a/go.sum +++ b/go.sum @@ -97,10 +97,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.15 h1:z02BVeV6k7hZMfWEQmKh3X23s3F9PBHFCcIVfNlut7A= -github.com/aws/aws-sdk-go v1.44.15/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.43.23 h1:/YmZzPMK6Xzi0B/W9O/Pq7nyIXpBv6mTiJdDDFC7u94= -github.com/aws/aws-sdk-go v1.43.23/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.16 h1:6voHuNZZNWo71MdNlym4eRlcogTeTSk9Ipo6qDJWzoU= +github.com/aws/aws-sdk-go v1.44.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -315,6 +313,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= @@ -589,6 +588,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= @@ -960,6 +960,7 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0 h1:67zQnAE0T2rB0A3CwLSas0K+SbVzSxP+zTLkQLexeiw= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1035,6 +1036,7 @@ google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf h1:SVYXkUz2yZS9FWb2Gm8ivSlbNQzL2Z/NpPKE3RG2jWk= google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1062,6 +1064,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/test/e2e/actions/cloud/aws.go b/test/e2e/actions/cloud/aws.go index 73df381731..e646fc052f 100644 --- a/test/e2e/actions/cloud/aws.go +++ b/test/e2e/actions/cloud/aws.go @@ -4,30 +4,37 @@ import ( "errors" "fmt" + v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" aws "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/aws" ) type awsAction struct{} -func (awsAction *awsAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (string, string, error) { +func (awsAction *awsAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (v1.PrivateEndpoint, error) { fmt.Print("create AWS LINK") session := aws.SessionAWS(pe.Region) vpcID, err := session.GetVPCID() if err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } subnetID, err := session.GetSubnetID() if err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } privateEndpointID, err := session.CreatePrivateEndpoint(vpcID, subnetID, pe.ServiceName, privatelinkName) if err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } - - return privateEndpointID, "", nil + cResponse := v1.PrivateEndpoint{ + ID: privateEndpointID, + IP: "", + Provider: provider.ProviderAWS, + Region: pe.Region, + } + return cResponse, nil } func (awsAction *awsAction) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkID string) error { diff --git a/test/e2e/actions/cloud/azure.go b/test/e2e/actions/cloud/azure.go index 4a9fbb2827..0da396db32 100644 --- a/test/e2e/actions/cloud/azure.go +++ b/test/e2e/actions/cloud/azure.go @@ -5,6 +5,8 @@ import ( "os" "path" + v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" + "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/azure" "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config" @@ -19,20 +21,26 @@ var ( subnetName = "default" ) -func (azureAction *azureAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (string, string, error) { +func (azureAction *azureAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (v1.PrivateEndpoint, error) { session, err := azure.SessionAzure(os.Getenv("AZURE_SUBSCRIPTION_ID"), config.TagName) if err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } err = session.DisableNetworkPolicies(resourceGroup, vpc, subnetName) if err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } id, ip, err := session.CreatePrivateEndpoint(pe.Region, resourceGroup, privatelinkName, pe.ServiceResourceID) if err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } - return id, ip, nil + cResponse := v1.PrivateEndpoint{ + ID: id, + IP: ip, + Provider: provider.ProviderAzure, + Region: pe.Region, + } + return cResponse, nil } func (azureAction *azureAction) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) error { diff --git a/test/e2e/actions/cloud/cloud.go b/test/e2e/actions/cloud/cloud.go index 0e593ac2eb..05c0328092 100644 --- a/test/e2e/actions/cloud/cloud.go +++ b/test/e2e/actions/cloud/cloud.go @@ -3,12 +3,13 @@ package cloud import ( "errors" + v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" ) type CloudActions interface { - createPrivateEndpoint(pe status.ProjectPrivateEndpoint, name string) (string, string, error) + createPrivateEndpoint(pe status.ProjectPrivateEndpoint, name string) (v1.PrivateEndpoint, error) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, name string) error statusPrivateEndpointPending(region, privateID string) bool statusPrivateEndpointAvailable(region, privateID string) bool @@ -19,6 +20,11 @@ type PEActions struct { PrivateEndpoint status.ProjectPrivateEndpoint } +type Endpoints struct { + IP string + Name string +} + func CreatePEActions(pe status.ProjectPrivateEndpoint) (PEActions, error) { peActions := PEActions{PrivateEndpoint: pe} switch pe.Provider { @@ -48,16 +54,18 @@ func (peActions *PEActions) validation() error { return errors.New("Azure. PrivateEndpoint.ServiceResourceID is empty") } case provider.ProviderGCP: - return errors.New("work with GCP is not implemented") + if len(peActions.PrivateEndpoint.ServiceAttachmentNames) < 1 { + return errors.New("GCP. PrivateEndpoint.ServiceAttachmentNames should not be empty") + } default: return errors.New("Check Provider") } return nil } -func (peActions *PEActions) CreatePrivateEndpoint(name string) (string, string, error) { +func (peActions *PEActions) CreatePrivateEndpoint(name string) (v1.PrivateEndpoint, error) { if err := peActions.validation(); err != nil { - return "", "", err + return v1.PrivateEndpoint{}, err } return peActions.CloudActions.createPrivateEndpoint(peActions.PrivateEndpoint, name) } @@ -71,6 +79,7 @@ func (peActions *PEActions) DeletePrivateEndpoint(name string) error { // privateID is different for different clouds: privateID for AWS or PEname for AZURE // AWS = PrivateID, AZURE = privateEndpoint Name +// GCP = prefix func (peActions *PEActions) IsStatusPrivateEndpointPending(privateID string) bool { return peActions.CloudActions.statusPrivateEndpointPending(peActions.PrivateEndpoint.Region, privateID) } diff --git a/test/e2e/actions/cloud/gcp.go b/test/e2e/actions/cloud/gcp.go index 46995fb34b..651e55d8d2 100644 --- a/test/e2e/actions/cloud/gcp.go +++ b/test/e2e/actions/cloud/gcp.go @@ -2,28 +2,97 @@ package cloud import ( "fmt" + "time" + v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1" "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status" + "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/gcp" ) type gcpAction struct{} -func (gcpAction *gcpAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (string, string, error) { - fmt.Print("NOT IMPLEMENTED create GCP LINK") - return "some test", "IP if req", nil +var ( + // TODO get from GCP + googleProjectID = "atlasoperator" // Google Cloud Project ID + googleVPC = "atlas-operator-test" // VPC Name + googleSubnetName = "atlas-operator-subnet-leo" // Subnet Name + googleConnectPrefix = "ao" // Private Service Connect Endpoint Prefix +) + +func (gcpAction *gcpAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (v1.PrivateEndpoint, error) { + session, err := gcp.SessionGCP(googleProjectID) + if err != nil { + return v1.PrivateEndpoint{}, err + } + var cResponse v1.PrivateEndpoint + for i, target := range pe.ServiceAttachmentNames { + addressName := formAddressName(privatelinkName, i) + ruleName := formRuleName(privatelinkName, i) + ip, err := session.AddIPAdress(pe.Region, addressName, googleSubnetName) + if err != nil { + return v1.PrivateEndpoint{}, err + } + + cResponse.Endpoints = append(cResponse.Endpoints, v1.GCPEndpoint{ + EndpointName: ruleName, + IPAddress: ip, + }) + cResponse.EndpointGroupName = googleVPC + cResponse.Region = pe.Region + cResponse.Provider = pe.Provider + cResponse.GCPProjectID = googleProjectID + + session.AddForwardRule(pe.Region, ruleName, addressName, googleVPC, googleSubnetName, target) + } + return cResponse, nil } func (gcpAction *gcpAction) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) error { - fmt.Print("NOT IMPLEMENTED delete GCP LINK") + session, err := gcp.SessionGCP(googleProjectID) + if err != nil { + return err + } + for i := range pe.Endpoints { + session.DeleteForwardRule(pe.Region, formRuleName(privatelinkName, i), 10, 20*time.Second) + session.DeleteIPAdress(pe.Region, formAddressName(privatelinkName, i)) + } return nil } func (gcpAction *gcpAction) statusPrivateEndpointPending(region, privateID string) bool { - fmt.Print("NOT IMPLEMENTED delete GCP LINK") - return true + session, err := gcp.SessionGCP(googleProjectID) + if err != nil { + fmt.Print(err) + return false + } + ruleName := formRuleName(privateID, 1) + result, err := session.DescribePrivateLinkStatus(region, ruleName) + if err != nil { + fmt.Print(err) + return false + } + return (result == "PENDING") } func (gcpAction *gcpAction) statusPrivateEndpointAvailable(region, privateID string) bool { - fmt.Print("NOT IMPLEMENTED delete GCP LINK") - return true + session, err := gcp.SessionGCP(googleProjectID) + if err != nil { + fmt.Print(err) + return false + } + ruleName := formRuleName(privateID, 1) + result, err := session.DescribePrivateLinkStatus(region, ruleName) + if err != nil { + fmt.Print(err) + return false + } + return (result == "ACCEPTED") +} + +func formAddressName(name string, i int) string { + return fmt.Sprintf("%s%s-ip-%d", googleConnectPrefix, name, i) +} + +func formRuleName(name string, i int) string { + return fmt.Sprintf("%s%s-%d", googleConnectPrefix, name, i) } diff --git a/test/e2e/actions/steps.go b/test/e2e/actions/steps.go index 484b264ae4..4e6de8a582 100644 --- a/test/e2e/actions/steps.go +++ b/test/e2e/actions/steps.go @@ -64,7 +64,7 @@ func WaitCluster(input model.UserInputs, generation string) { } func WaitProject(data *model.TestDataProvider, generation string) { - EventuallyWithOffset(1, kube.GetReadyProjectStatus(data), "5m", "10s").Should(Equal("True"), "Kubernetes resource: Project status `Ready` should be 'True'") + EventuallyWithOffset(1, kube.GetReadyProjectStatus(data), "15m", "10s").Should(Equal("True"), "Kubernetes resource: Project status `Ready` should be 'True'") ExpectWithOffset(1, kubecli.GetGeneration(data.Resources.Namespace, data.Resources.GetAtlasProjectFullKubeName())).Should(Equal(generation), "Kubernetes resource: Generation should be upgraded") atlasProject, err := kube.GetProjectResource(data) Expect(err).ShouldNot(HaveOccurred()) diff --git a/test/e2e/api/gcp/gcp.go b/test/e2e/api/gcp/gcp.go new file mode 100644 index 0000000000..653092ce6d --- /dev/null +++ b/test/e2e/api/gcp/gcp.go @@ -0,0 +1,153 @@ +package gcp + +import ( + "context" + "fmt" + "time" + + "google.golang.org/api/compute/v1" + "google.golang.org/api/googleapi" +) + +type sessionGCP struct { + computeService *compute.Service + gProjectID string +} + +func SessionGCP(gProjectID string) (sessionGCP, error) { + computeService, err := compute.NewService(context.Background()) + if err != nil { + return sessionGCP{}, fmt.Errorf("compute.NewClient: %w", err) + } + return sessionGCP{computeService, gProjectID}, nil +} + +func (s *sessionGCP) AddIPAdress(region, addressName, subnet string) (string, error) { + address := &compute.Address{ + AddressType: "INTERNAL", + Description: addressName, + Name: addressName, + Network: "", + Region: region, + Subnetwork: s.formSubnetURL(region, subnet), + ServerResponse: googleapi.ServerResponse{}, + } + _, err := s.computeService.Addresses.Insert(s.gProjectID, region, address).Context(context.Background()).Do() + if err != nil { + return "", fmt.Errorf("computeService.Addresses.Insert: %w", err) + } + ip, err := s.GetIP(region, addressName, 20, 10) + if err != nil { + return "", fmt.Errorf("computeService.Addresses.Get: %w", err) + } + return ip, nil +} + +func (s *sessionGCP) GetIP(region, addressName string, try, interval int) (string, error) { + for i := 0; i < try; i++ { + r, err := s.computeService.Addresses.Get(s.gProjectID, region, addressName).Do() + if err != nil { + return "", err + } + if r.Address != "" { + return r.Address, nil + } + time.Sleep(time.Duration(interval) * time.Second) + } + return "", fmt.Errorf("timeout computeService.Addresses.Get") +} + +func (s *sessionGCP) DeleteIPAdress(region, addressName string) error { + _, err := s.computeService.Addresses.Delete(s.gProjectID, region, addressName).Context(context.Background()).Do() + if err != nil { + return fmt.Errorf("computeService.Addresses.Delete: %w", err) + } + return nil +} + +func (s *sessionGCP) AddForwardRule(region, ruleName, addressName, network, subnet, target string) error { + rules := &compute.ForwardingRule{ + IPAddress: s.formAddressURL(region, addressName), + Labels: map[string]string{}, + Name: ruleName, + Network: s.formNetworkURL(network), + Ports: []string{}, + Region: region, + ServiceDirectoryRegistrations: []*compute.ForwardingRuleServiceDirectoryRegistration{}, + Subnetwork: "", + Target: target, + ServerResponse: googleapi.ServerResponse{}, + } + _, err := s.computeService.ForwardingRules.Insert(s.gProjectID, region, rules).Context(context.Background()).Do() + if err != nil { + return fmt.Errorf("computeService.ForwardingRules.Insert: %w", err) + } + return nil +} + +func (s *sessionGCP) DeleteForwardRule(region, ruleName string, try int, interval time.Duration) error { + _, err := s.computeService.ForwardingRules.Delete(s.gProjectID, region, ruleName).Do() + if err != nil { + return fmt.Errorf("computeService.ForwardingRules.Delete: %w", err) + } + + contain := func(list []*compute.ForwardingRule, name string) bool { + for _, item := range list { + if item.Name == name { + return true + } + } + return false + } + + deleted := false + for i := 0; i < try; i++ { + r, err := s.computeService.ForwardingRules.List(s.gProjectID, region).Do() + if err != nil { + return fmt.Errorf("computeService.ForwardingRule.List: %w", err) + } + if !contain(r.Items, ruleName) { + deleted = true + break + } + time.Sleep(interval) + } + if !deleted { + return fmt.Errorf("computeService.ForwardingRules.Delete. Could not delete forward rule after %d retries", try) + } + + return nil +} + +// Possible values: +// "ACCEPTED" - The connection has been accepted by the producer. +// "CLOSED" - The connection has been closed by the producer and will +// not serve traffic going forward. +// "PENDING" - The connection is pending acceptance by the producer. +// "REJECTED" - The connection has been rejected by the producer. +// "STATUS_UNSPECIFIED" +func (s *sessionGCP) DescribePrivateLinkStatus(region, ruleName string) (string, error) { + resp, err := s.computeService.ForwardingRules.Get(s.gProjectID, region, ruleName).Context(context.Background()).Do() + if err != nil { + return "", fmt.Errorf("computeService.Addresses.Get: %w", err) + } + return resp.PscConnectionStatus, nil +} + +func (s *sessionGCP) formNetworkURL(network string) string { + return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/%s", + s.gProjectID, network, + ) +} + +func (s *sessionGCP) formSubnetURL(region, subnet string) string { + return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/subnetworks/%s", + s.gProjectID, region, subnet, + ) +} + +func (s *sessionGCP) formAddressURL(region, addressName string) string { + return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/addresses/%s", + s.gProjectID, region, addressName, + ) +} diff --git a/test/e2e/config/config.go b/test/e2e/config/config.go index eebcb43289..eaa6a6f80e 100644 --- a/test/e2e/config/config.go +++ b/test/e2e/config/config.go @@ -29,4 +29,7 @@ const ( // AWS Tags for test TagName = "atlas-operator-test" TagBusy = "busy" + + // GCP + FileNameSAGCP = "gcp_service_account.json" ) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1c2e4576ac..ba7cee1e57 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -9,6 +9,8 @@ import ( . "github.com/onsi/gomega" mongocli "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/cli/mongocli" + "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config" + "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/utils" ) const ( @@ -60,3 +62,10 @@ func checkUpAzureEnviroment() { Expect(os.Getenv("AZURE_CLIENT_SECRET")).ShouldNot(BeEmpty(), "Please, setup AZURE_CLIENT_SECRET environment variable for test with Azure") Expect(os.Getenv("AZURE_SUBSCRIPTION_ID")).ShouldNot(BeEmpty(), "Please, setup AZURE_SUBSCRIPTION_ID environment variable for test with Azure") } + +func checkNSetUpGCPEnviroment() { + Expect(os.Getenv("GCP_SA_CRED")).ShouldNot(BeEmpty(), "Please, setup GCP_SA_CRED environment variable for test with GCP (req. Service Account)") + Expect(utils.SaveToFile(config.FileNameSAGCP, []byte(os.Getenv("GCP_SA_CRED")))).ShouldNot(HaveOccurred()) + Expect(os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", config.FileNameSAGCP)).ShouldNot(HaveOccurred()) + Expect(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")).ShouldNot(BeEmpty(), "Please, setup GOOGLE_APPLICATION_CREDENTIALS environment variable for test with GCP") +} diff --git a/test/e2e/model/project.go b/test/e2e/model/project.go index 9a1a5665ac..454970a271 100644 --- a/test/e2e/model/project.go +++ b/test/e2e/model/project.go @@ -80,23 +80,24 @@ func (p *AProject) UpdatePrivateLinkByOrder(i int, id string) *AProject { return p } -func (p *AProject) UpdatePrivateLinkID(provider provider.ProviderName, region, id, ip string) *AProject { +func (p *AProject) UpdatePrivateLinkID(test v1.PrivateEndpoint) *AProject { for i, peItem := range p.Spec.PrivateEndpoints { - if (peItem.Provider == provider) && (peItem.Region == region) { - p.Spec.PrivateEndpoints[i].ID = id - p.Spec.PrivateEndpoints[i].IP = ip + if (peItem.Provider == test.Provider) && (peItem.Region == test.Region) { + p.Spec.PrivateEndpoints[i] = test } } return p } -func (p *AProject) GetPrivateIDByProviderRegion(provider provider.ProviderName, region string) string { - for i, peItem := range p.Spec.PrivateEndpoints { - if (peItem.Provider == provider) && (peItem.Region == region) { - return p.Spec.PrivateEndpoints[i].ID +func (p *AProject) GetPrivateIDByProviderRegion(statusItem status.ProjectPrivateEndpoint) string { + if statusItem.Provider == provider.ProviderAWS { + for i, peItem := range p.Spec.PrivateEndpoints { + if (peItem.Provider == statusItem.Provider) && (peItem.Region == statusItem.Region) { + return p.Spec.PrivateEndpoints[i].ID + } } } - return "" + return statusItem.ID } func (p *AProject) DeletePrivateLink(id string) *AProject { diff --git a/test/e2e/private_link_test.go b/test/e2e/private_link_test.go index 1fab4f5436..249040df36 100644 --- a/test/e2e/private_link_test.go +++ b/test/e2e/private_link_test.go @@ -12,6 +12,7 @@ import ( cloud "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/actions/cloud" "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/actions/deploy" kube "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/actions/kube" + "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/utils" kubecli "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/cli/kubecli" @@ -39,6 +40,7 @@ var _ = Describe("UserLogin", Label("privatelink"), func() { Eventually(kubecli.GetVersionOutput()).Should(Say(K8sVersion)) checkUpAWSEnviroment() checkUpAzureEnviroment() + checkNSetUpGCPEnviroment() }) _ = AfterEach(func() { @@ -63,7 +65,6 @@ var _ = Describe("UserLogin", Label("privatelink"), func() { data.Resources.Namespace, ) }) - } By("Clean Cloud", func() { DeleteAllPrivateEndpoints(&data) @@ -176,6 +177,28 @@ var _ = Describe("UserLogin", Label("privatelink"), func() { }, }, ), + Entry("Test[privatelink-gpc-1]: User has project which was updated with 2 AWS PrivateEndpoint", Label("privatelink-gpc-1"), + model.NewTestDataProvider( + "privatelink-gpc-1", + model.AProject{}, + model.NewEmptyAtlasKeyType().UseDefaulFullAccess(), + []string{"data/atlascluster_backup.yaml"}, + []string{}, + []model.DBUser{ + *model.NewDBUser("user1"). + WithSecretRef("dbuser-secret-u1"). + AddBuildInAdminRole(), + }, + 40000, + []func(*model.TestDataProvider){}, + ), + []privateEndpoint{ + { + provider: "GCP", + region: "europe-west1", + }, + }, + ), ) }) @@ -212,15 +235,13 @@ func privateFlow(userData *model.TestDataProvider, requstedPE []privateEndpoint) for _, peitem := range project.Status.PrivateEndpoints { cloudTest, err := cloud.CreatePEActions(peitem) Expect(err).ShouldNot(HaveOccurred()) - privateLinkID, ip, err := cloudTest.CreatePrivateEndpoint(peitem.ID) + + privateEndpointID := peitem.ID + Expect(privateEndpointID).ShouldNot(BeEmpty()) + + output, err := cloudTest.CreatePrivateEndpoint(privateEndpointID) Expect(err).ShouldNot(HaveOccurred()) - Expect(privateLinkID).ShouldNot(BeEmpty()) - Eventually( - func() bool { - return cloudTest.IsStatusPrivateEndpointPending(privateLinkID) - }, - ).Should(BeTrue()) - userData.Resources.Project.UpdatePrivateLinkID(peitem.Provider, peitem.Region, privateLinkID, ip) + userData.Resources.Project = userData.Resources.Project.UpdatePrivateLinkID(output) } }) @@ -230,7 +251,7 @@ func privateFlow(userData *model.TestDataProvider, requstedPE []privateEndpoint) }) By("Check statuses", func() { - Eventually(kube.GetProjectPEndpointStatus(userData)).Should(Equal("True"), "Condition status 'PrivateEndpointServiceReady' is not'True'") + Eventually(kube.GetProjectPEndpointStatus(userData)).Should(Equal("True"), "Condition status 'PrivateEndpointReady' is not'True'") Eventually(kube.GetReadyProjectStatus(userData)).Should(Equal("True"), "Condition status 'Ready' is not 'True'") project, err := kube.GetProjectResource(userData) @@ -238,7 +259,7 @@ func privateFlow(userData *model.TestDataProvider, requstedPE []privateEndpoint) for _, peitem := range project.Status.PrivateEndpoints { cloudTest, err := cloud.CreatePEActions(peitem) Expect(err).ShouldNot(HaveOccurred()) - privateEndpointID := userData.Resources.Project.GetPrivateIDByProviderRegion(peitem.Provider, peitem.Region) + privateEndpointID := userData.Resources.Project.GetPrivateIDByProviderRegion(peitem) Expect(privateEndpointID).ShouldNot(BeEmpty()) Eventually( func() bool { @@ -257,7 +278,7 @@ func DeleteAllPrivateEndpoints(data *model.TestDataProvider) { for _, peitem := range project.Status.PrivateEndpoints { cloudTest, err := cloud.CreatePEActions(peitem) if err == nil { - privateEndpointID := data.Resources.Project.GetPrivateIDByProviderRegion(peitem.Provider, peitem.Region) + privateEndpointID := data.Resources.Project.GetPrivateIDByProviderRegion(peitem) if privateEndpointID != "" { err = cloudTest.DeletePrivateEndpoint(privateEndpointID) if err != nil {