CLOUDP-20220: GCP test

.github/workflows/test-forked.yml

@@ -265,6 +265,7 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }}
           DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
         run: |
           helm version

.github/workflows/test.yml

@@ -263,6 +263,7 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }}
           DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
         run: |
           helm version

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Azure/go-autorest/autorest v0.11.24
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/Azure/go-autorest/autorest/to v0.4.0
-	github.com/aws/aws-sdk-go v1.44.15
+	github.com/aws/aws-sdk-go v1.44.16
 	github.com/fatih/structtag v1.2.0
 	github.com/go-logr/zapr v1.2.3
 	github.com/google/go-cmp v0.5.7
@@ -22,6 +22,7 @@ require (
 	go.mongodb.org/atlas v0.16.0
 	go.mongodb.org/mongo-driver v1.8.3
 	go.uber.org/zap v1.21.0
+	google.golang.org/api v0.70.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.23.4
 	k8s.io/apimachinery v0.23.4
@@ -55,6 +56,7 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
@@ -78,6 +80,7 @@ require (
 	github.com/xdg-go/scram v1.1.0 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
+	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
@@ -91,6 +94,8 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf // indirect
+	google.golang.org/grpc v1.44.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect

go.sum

@@ -97,10 +97,8 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws/aws-sdk-go v1.44.15 h1:z02BVeV6k7hZMfWEQmKh3X23s3F9PBHFCcIVfNlut7A=
-github.com/aws/aws-sdk-go v1.44.15/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.43.23 h1:/YmZzPMK6Xzi0B/W9O/Pq7nyIXpBv6mTiJdDDFC7u94=
-github.com/aws/aws-sdk-go v1.43.23/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.16 h1:6voHuNZZNWo71MdNlym4eRlcogTeTSk9Ipo6qDJWzoU=
+github.com/aws/aws-sdk-go v1.44.16/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -315,6 +313,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
+github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
 github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
@@ -589,6 +588,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
@@ -960,6 +960,7 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr
 google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
 google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
 google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
+google.golang.org/api v0.70.0 h1:67zQnAE0T2rB0A3CwLSas0K+SbVzSxP+zTLkQLexeiw=
 google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1035,6 +1036,7 @@ google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ6
 google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
+google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf h1:SVYXkUz2yZS9FWb2Gm8ivSlbNQzL2Z/NpPKE3RG2jWk=
 google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@@ -1062,6 +1064,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
+google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=

test/e2e/actions/cloud/aws.go

@@ -4,30 +4,37 @@ import (
 	"errors"
 	"fmt"
 
+	v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1"
+	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider"
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status"
 	aws "github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/aws"
 )
 
 type awsAction struct{}
 
-func (awsAction *awsAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (string, string, error) {
+func (awsAction *awsAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (v1.PrivateEndpoint, error) {
 	fmt.Print("create AWS LINK")
 	session := aws.SessionAWS(pe.Region)
 	vpcID, err := session.GetVPCID()
 	if err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
 	subnetID, err := session.GetSubnetID()
 	if err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
 
 	privateEndpointID, err := session.CreatePrivateEndpoint(vpcID, subnetID, pe.ServiceName, privatelinkName)
 	if err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
-
-	return privateEndpointID, "", nil
+	cResponse := v1.PrivateEndpoint{
+		ID:       privateEndpointID,
+		IP:       "",
+		Provider: provider.ProviderAWS,
+		Region:   pe.Region,
+	}
+	return cResponse, nil
 }
 
 func (awsAction *awsAction) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkID string) error {

test/e2e/actions/cloud/azure.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"path"
 
+	v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1"
+	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider"
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status"
 	"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/azure"
 	"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config"
@@ -19,20 +21,26 @@ var (
 	subnetName    = "default"
 )
 
-func (azureAction *azureAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (string, string, error) {
+func (azureAction *azureAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (v1.PrivateEndpoint, error) {
 	session, err := azure.SessionAzure(os.Getenv("AZURE_SUBSCRIPTION_ID"), config.TagName)
 	if err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
 	err = session.DisableNetworkPolicies(resourceGroup, vpc, subnetName)
 	if err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
 	id, ip, err := session.CreatePrivateEndpoint(pe.Region, resourceGroup, privatelinkName, pe.ServiceResourceID)
 	if err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
-	return id, ip, nil
+	cResponse := v1.PrivateEndpoint{
+		ID:       id,
+		IP:       ip,
+		Provider: provider.ProviderAzure,
+		Region:   pe.Region,
+	}
+	return cResponse, nil
 }
 
 func (azureAction *azureAction) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) error {

test/e2e/actions/cloud/cloud.go

@@ -3,12 +3,13 @@ package cloud
 import (
 	"errors"
 
+	v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1"
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/provider"
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status"
 )
 
 type CloudActions interface {
-	createPrivateEndpoint(pe status.ProjectPrivateEndpoint, name string) (string, string, error)
+	createPrivateEndpoint(pe status.ProjectPrivateEndpoint, name string) (v1.PrivateEndpoint, error)
 	deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, name string) error
 	statusPrivateEndpointPending(region, privateID string) bool
 	statusPrivateEndpointAvailable(region, privateID string) bool
@@ -19,6 +20,11 @@ type PEActions struct {
 	PrivateEndpoint status.ProjectPrivateEndpoint
 }
 
+type Endpoints struct {
+	IP   string
+	Name string
+}
+
 func CreatePEActions(pe status.ProjectPrivateEndpoint) (PEActions, error) {
 	peActions := PEActions{PrivateEndpoint: pe}
 	switch pe.Provider {
@@ -48,16 +54,18 @@ func (peActions *PEActions) validation() error {
 			return errors.New("Azure. PrivateEndpoint.ServiceResourceID is empty")
 		}
 	case provider.ProviderGCP:
-		return errors.New("work with GCP is not implemented")
+		if len(peActions.PrivateEndpoint.ServiceAttachmentNames) < 1 {
+			return errors.New("GCP. PrivateEndpoint.ServiceAttachmentNames should not be empty")
+		}
 	default:
 		return errors.New("Check Provider")
 	}
 	return nil
 }
 
-func (peActions *PEActions) CreatePrivateEndpoint(name string) (string, string, error) {
+func (peActions *PEActions) CreatePrivateEndpoint(name string) (v1.PrivateEndpoint, error) {
 	if err := peActions.validation(); err != nil {
-		return "", "", err
+		return v1.PrivateEndpoint{}, err
 	}
 	return peActions.CloudActions.createPrivateEndpoint(peActions.PrivateEndpoint, name)
 }
@@ -71,6 +79,7 @@ func (peActions *PEActions) DeletePrivateEndpoint(name string) error {
 
 // privateID is different for different clouds: privateID for AWS or PEname for AZURE
 // AWS = PrivateID, AZURE = privateEndpoint Name
+// GCP = prefix
 func (peActions *PEActions) IsStatusPrivateEndpointPending(privateID string) bool {
 	return peActions.CloudActions.statusPrivateEndpointPending(peActions.PrivateEndpoint.Region, privateID)
 }

test/e2e/actions/cloud/gcp.go

@@ -2,28 +2,97 @@ package cloud
 
 import (
 	"fmt"
+	"time"
 
+	v1 "github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1"
 	"github.com/mongodb/mongodb-atlas-kubernetes/pkg/api/v1/status"
+	"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/gcp"
 )
 
 type gcpAction struct{}
 
-func (gcpAction *gcpAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (string, string, error) {
-	fmt.Print("NOT IMPLEMENTED create GCP LINK")
-	return "some test", "IP if req", nil
+var (
+	// TODO get from GCP
+	googleProjectID     = "atlasoperator"             // Google Cloud Project ID
+	googleVPC           = "atlas-operator-test"       // VPC Name
+	googleSubnetName    = "atlas-operator-subnet-leo" // Subnet Name
+	googleConnectPrefix = "ao"                        // Private Service Connect Endpoint Prefix
+)
+
+func (gcpAction *gcpAction) createPrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) (v1.PrivateEndpoint, error) {
+	session, err := gcp.SessionGCP(googleProjectID)
+	if err != nil {
+		return v1.PrivateEndpoint{}, err
+	}
+	var cResponse v1.PrivateEndpoint
+	for i, target := range pe.ServiceAttachmentNames {
+		addressName := formAddressName(privatelinkName, i)
+		ruleName := formRuleName(privatelinkName, i)
+		ip, err := session.AddIPAdress(pe.Region, addressName, googleSubnetName)
+		if err != nil {
+			return v1.PrivateEndpoint{}, err
+		}
+
+		cResponse.Endpoints = append(cResponse.Endpoints, v1.GCPEndpoint{
+			EndpointName: ruleName,
+			IPAddress:    ip,
+		})
+		cResponse.EndpointGroupName = googleVPC
+		cResponse.Region = pe.Region
+		cResponse.Provider = pe.Provider
+		cResponse.GCPProjectID = googleProjectID
+
+		session.AddForwardRule(pe.Region, ruleName, addressName, googleVPC, googleSubnetName, target)
+	}
+	return cResponse, nil
 }
 
 func (gcpAction *gcpAction) deletePrivateEndpoint(pe status.ProjectPrivateEndpoint, privatelinkName string) error {
-	fmt.Print("NOT IMPLEMENTED delete GCP LINK")
+	session, err := gcp.SessionGCP(googleProjectID)
+	if err != nil {
+		return err
+	}
+	for i := range pe.Endpoints {
+		session.DeleteForwardRule(pe.Region, formRuleName(privatelinkName, i), 10, 20*time.Second)
+		session.DeleteIPAdress(pe.Region, formAddressName(privatelinkName, i))
+	}
 	return nil
 }
 
 func (gcpAction *gcpAction) statusPrivateEndpointPending(region, privateID string) bool {
-	fmt.Print("NOT IMPLEMENTED delete GCP LINK")
-	return true
+	session, err := gcp.SessionGCP(googleProjectID)
+	if err != nil {
+		fmt.Print(err)
+		return false
+	}
+	ruleName := formRuleName(privateID, 1)
+	result, err := session.DescribePrivateLinkStatus(region, ruleName)
+	if err != nil {
+		fmt.Print(err)
+		return false
+	}
+	return (result == "PENDING")
 }
 
 func (gcpAction *gcpAction) statusPrivateEndpointAvailable(region, privateID string) bool {
-	fmt.Print("NOT IMPLEMENTED delete GCP LINK")
-	return true
+	session, err := gcp.SessionGCP(googleProjectID)
+	if err != nil {
+		fmt.Print(err)
+		return false
+	}
+	ruleName := formRuleName(privateID, 1)
+	result, err := session.DescribePrivateLinkStatus(region, ruleName)
+	if err != nil {
+		fmt.Print(err)
+		return false
+	}
+	return (result == "ACCEPTED")
+}
+
+func formAddressName(name string, i int) string {
+	return fmt.Sprintf("%s%s-ip-%d", googleConnectPrefix, name, i)
+}
+
+func formRuleName(name string, i int) string {
+	return fmt.Sprintf("%s%s-%d", googleConnectPrefix, name, i)
 }

test/e2e/actions/steps.go

@@ -64,7 +64,7 @@ func WaitCluster(input model.UserInputs, generation string) {
 }
 
 func WaitProject(data *model.TestDataProvider, generation string) {
-	EventuallyWithOffset(1, kube.GetReadyProjectStatus(data), "5m", "10s").Should(Equal("True"), "Kubernetes resource: Project status `Ready` should be 'True'")
+	EventuallyWithOffset(1, kube.GetReadyProjectStatus(data), "15m", "10s").Should(Equal("True"), "Kubernetes resource: Project status `Ready` should be 'True'")
 	ExpectWithOffset(1, kubecli.GetGeneration(data.Resources.Namespace, data.Resources.GetAtlasProjectFullKubeName())).Should(Equal(generation), "Kubernetes resource: Generation should be upgraded")
 	atlasProject, err := kube.GetProjectResource(data)
 	Expect(err).ShouldNot(HaveOccurred())