Skip to content

Commit

Permalink
kubetest2-kops: automatically create buckets
Browse files Browse the repository at this point in the history
When working with boskos, this means we don't need to have a shared
bucket with dynamic permissions.
  • Loading branch information
justinsb committed Jun 22, 2023
1 parent 0e5d198 commit e79b08c
Show file tree
Hide file tree
Showing 6 changed files with 201 additions and 22 deletions.
2 changes: 1 addition & 1 deletion tests/e2e/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ replace k8s.io/kops => ../../.
replace k8s.io/client-go => k8s.io/client-go v0.24.2

require (
github.com/aws/aws-sdk-go v1.44.283
github.com/blang/semver/v4 v4.0.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/octago/sflags v0.2.0
Expand Down Expand Up @@ -35,7 +36,6 @@ require (
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/apparentlymart/go-cidr v1.1.0 // indirect
github.com/aws/aws-sdk-go v1.44.283 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
Expand Down
126 changes: 126 additions & 0 deletions tests/e2e/kubetest2-kops/aws/s3.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/sts"
"k8s.io/klog/v2"
)

// We need to pick some region to query the AWS APIs through, even if we are not running on AWS.
const defaultRegion = "us-east-2"

type awsClient struct {
sts *sts.STS
s3 *s3.S3
}

func newAWSClient(ctx context.Context, creds *credentials.Credentials) (*awsClient, error) {
awsConfig := aws.NewConfig().WithRegion(defaultRegion).WithUseDualStack(true)
awsConfig = awsConfig.WithCredentialsChainVerboseErrors(true)
if creds != nil {
awsConfig = awsConfig.WithCredentials(creds)
}

awsSession, err := session.NewSessionWithOptions(session.Options{
Config: *awsConfig,
SharedConfigState: session.SharedConfigEnable,
})
if err != nil {
return nil, fmt.Errorf("error starting new AWS session: %v", err)
}

return &awsClient{
sts: sts.New(awsSession, awsConfig),
s3: s3.New(awsSession, awsConfig),
}, nil
}

// AWSBucketName constructs a bucket name that is unique to the AWS account.
func AWSBucketName(ctx context.Context, creds *credentials.Credentials) (string, error) {
client, err := newAWSClient(ctx, creds)
if err != nil {
return "", err
}

callerIdentity, err := client.sts.GetCallerIdentity(&sts.GetCallerIdentityInput{})
if err != nil {
return "", fmt.Errorf("error getting AWS caller identity from STS: %w", err)
}
bucket := "kops-test-" + aws.StringValue(callerIdentity.Account)
return bucket, nil
}

// EnsureAWSBucket creates a bucket if it does not exist in the account.
// If a different account has already created the bucket, that is treated as an error to prevent "preimage" attacks.
func EnsureAWSBucket(ctx context.Context, creds *credentials.Credentials, bucketName string) error {
// These don't need to be in the same region, so we pick a region arbitrarily
location := "us-east-2"

client, err := newAWSClient(ctx, creds)
if err != nil {
return err
}

// Note that this lists only our buckets, so we know that someone else hasn't created the bucket
buckets, err := client.s3.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
if err != nil {
return fmt.Errorf("error listing buckets: %w", err)
}

var existingBucket *s3.Bucket
for _, bucket := range buckets.Buckets {
if aws.StringValue(bucket.Name) == bucketName {
existingBucket = bucket
}
}

if existingBucket == nil {
klog.Infof("creating S3 bucket s3://%s", bucketName)
if _, err := client.s3.CreateBucketWithContext(ctx, &s3.CreateBucketInput{
Bucket: &bucketName,
CreateBucketConfiguration: &s3.CreateBucketConfiguration{
LocationConstraint: &location,
},
}); err != nil {
return fmt.Errorf("error creating bucket s3://%v: %w", bucketName, err)
}
}

return nil
}

// DeleteAWSBucket deletes an AWS bucket.
func DeleteAWSBucket(ctx context.Context, creds *credentials.Credentials, bucketName string) error {
client, err := newAWSClient(ctx, creds)
if err != nil {
return err
}

klog.Infof("deleting S3 bucket s3://%s", bucketName)
if _, err := client.s3.DeleteBucketWithContext(ctx, &s3.DeleteBucketInput{Bucket: &bucketName}); err != nil {
return fmt.Errorf("error deleting bucket: %w", err)
}
return nil
}
42 changes: 33 additions & 9 deletions tests/e2e/kubetest2-kops/deployer/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ import (
"path/filepath"
"strings"

"github.com/aws/aws-sdk-go/aws/credentials"
"k8s.io/klog/v2"
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
"k8s.io/kops/tests/e2e/pkg/kops"
"k8s.io/kops/tests/e2e/pkg/target"
Expand Down Expand Up @@ -73,7 +75,7 @@ func (d *deployer) initialize(ctx context.Context) error {
if err != nil {
return fmt.Errorf("init failed to get resource %q from boskos: %w", d.BoskosResourceType, err)
}
klog.V(1).Infof("Got AWS account %s from boskos", resource.Name)
klog.Infof("got AWS account %q from boskos", resource.Name)

accessKeyIDObj, ok := resource.UserData.Load("access-key-id")
if !ok {
Expand All @@ -83,10 +85,8 @@ func (d *deployer) initialize(ctx context.Context) error {
if !ok {
return fmt.Errorf("secret-access-key not found in boskos resource %q", resource.Name)
}
d.awsStaticCredentials = &awsStaticCredentials{
AccessKeyID: accessKeyIDObj.(string),
SecretAccessKey: secretAccessKeyObj.(string),
}
d.awsCredentials = credentials.NewStaticCredentials(accessKeyIDObj.(string), secretAccessKeyObj.(string), "")
d.createBucket = true
}

if d.SSHPrivateKeyPath == "" || d.SSHPublicKeyPath == "" {
Expand Down Expand Up @@ -213,9 +213,23 @@ func (d *deployer) env() []string {
// https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62
vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath))

if d.awsStaticCredentials != nil {
vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", d.awsStaticCredentials.AccessKeyID))
vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", d.awsStaticCredentials.SecretAccessKey))
if d.awsCredentials != nil {
credentials, err := d.awsCredentials.Get()
if err != nil {
klog.Fatalf("error getting aws credentials: %v", err)
}
if credentials.AccessKeyID != "" {
klog.Infof("setting AWS_ACCESS_KEY_ID")
vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", credentials.AccessKeyID))
} else {
klog.Warningf("AWS credentials configured but AWS_ACCESS_KEY_ID was empty")
}
if credentials.SecretAccessKey != "" {
klog.Infof("setting AWS_SECRET_ACCESS_KEY")
vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", credentials.SecretAccessKey))
} else {
klog.Warningf("AWS credentials configured but AWS_SECRET_ACCESS_KEY was empty")
}
}
} else if d.CloudProvider == "digitalocean" {
// Pass through some env vars if set
Expand Down Expand Up @@ -277,11 +291,21 @@ func defaultClusterName(cloudProvider string) (string, error) {
// stateStore returns the kops state store to use
// defaulting to values used in prow jobs
func (d *deployer) stateStore() string {
ctx := context.TODO()

ss := os.Getenv("KOPS_STATE_STORE")
if ss == "" {
switch d.CloudProvider {
case "aws":
ss = "s3://k8s-kops-prow"
if d.createBucket {
bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
if err != nil {
klog.Fatalf("error building aws bucket name: %v", err)
}
ss = "s3://" + bucketName
} else {
ss = "s3://k8s-kops-prow"
}
case "gce":
d.createBucket = true
ss = "gs://" + gce.GCSBucketName(d.GCPProject)
Expand Down
10 changes: 3 additions & 7 deletions tests/e2e/kubetest2-kops/deployer/deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"sync"
"time"

"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/octago/sflags/gen/gpflag"
"github.com/spf13/pflag"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -85,13 +86,8 @@ type deployer struct {

boskos boskosHelper

// awsStaticCredentials holds credentials for AWS loaded from boskos
awsStaticCredentials *awsStaticCredentials
}

type awsStaticCredentials struct {
AccessKeyID string
SecretAccessKey string
// awsCredentials holds credentials for AWS loaded from boskos
awsCredentials *credentials.Credentials
}

// assert that New implements types.NewDeployer
Expand Down
20 changes: 18 additions & 2 deletions tests/e2e/kubetest2-kops/deployer/down.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ package deployer

import (
"context"
"fmt"
"strings"

"k8s.io/klog/v2"
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
"sigs.k8s.io/kubetest2/pkg/exec"
)
Expand Down Expand Up @@ -55,8 +57,22 @@ func (d *deployer) Down() error {
return err
}

if d.CloudProvider == "gce" && d.createBucket {
gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
if d.createBucket {
switch d.CloudProvider {
case "gce":
gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
case "aws":
bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
if err != nil {
return fmt.Errorf("error building aws bucket name: %w", err)
}

if err := aws.DeleteAWSBucket(ctx, d.awsCredentials, bucketName); err != nil {
klog.Warningf("error deleting AWS bucket: %w", err)
}
default:
return fmt.Errorf("bucket cleanup not implemented for cloud %q", d.CloudProvider)
}
}

if err := d.boskos.Cleanup(ctx); err != nil {
Expand Down
23 changes: 20 additions & 3 deletions tests/e2e/kubetest2-kops/deployer/up.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package deployer

import (
"context"
"errors"
"fmt"
osexec "os/exec"
Expand All @@ -36,6 +37,8 @@ import (
)

func (d *deployer) Up() error {
ctx := context.TODO()

if err := d.init(); err != nil {
return err
}
Expand All @@ -47,9 +50,23 @@ func (d *deployer) Up() error {
_ = d.Down()
}

if d.CloudProvider == "gce" && d.createBucket {
if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
return err
if d.createBucket {
switch d.CloudProvider {
case "gce":
if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
return err
}
case "aws":
bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
if err != nil {
return fmt.Errorf("error building aws bucket name: %w", err)
}

if err := aws.EnsureAWSBucket(ctx, d.awsCredentials, bucketName); err != nil {
return err
}
default:
return fmt.Errorf("bucket creation not implemented for cloud %q", d.CloudProvider)
}
}

Expand Down

0 comments on commit e79b08c

Please sign in to comment.