From d01ac5d70bfcfa6f0c1da7e57e24dc0554e3122c Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 19 Sep 2018 20:48:53 -0700 Subject: [PATCH] pkg/rhcos: Default to the most-recent AMI We're pushing public AMIs since openshift/os@6dd20dc6 (jenkins: Make RHCOS AMI Public, 2018-09-18, openshift/os#304). There's still no public analog to [1], so I'm just scraping this from metadata on images available via the AWS API. The analogous AWS command line invocation is: $ AWS_DEFAULT_REGION=us-east-1 aws ec2 describe-images --filter 'Name=name,Values=rhcos*' --query 'sort_by(Images, &CreationDate)[-1].ImageId' --output text with a few extra filters thrown in. The full set of metadata on the most recent current image is: $ AWS_DEFAULT_REGION=us-east-1 aws ec2 describe-images --filter 'Name=name,Values=rhcos*' --query 'sort_by(Images, &CreationDate)[-1]' --output json { "VirtualizationType": "hvm", "Description": "Red Hat CoreOS 4.0.5846 (c9a6bb48b837b5bcfeb9bd427be9a18b5bd75b6c57cb289245f211ff98b2a740)", "Hypervisor": "xen", "EnaSupport": true, "SriovNetSupport": "simple", "ImageId": "ami-08a5792a684330602", "State": "available", "BlockDeviceMappings": [ { "DeviceName": "/dev/xvda", "Ebs": { "Encrypted": false, "DeleteOnTermination": true, "VolumeType": "gp2", "VolumeSize": 8, "SnapshotId": "snap-00a45db4ad6173805" } }, { "DeviceName": "/dev/xvdb", "VirtualName": "ephemeral0" } ], "Architecture": "x86_64", "ImageLocation": "531415883065/rhcos_dev_c9a6bb4-hvm", "RootDeviceType": "ebs", "OwnerId": "531415883065", "RootDeviceName": "/dev/xvda", "CreationDate": "2018-09-19T23:40:54.000Z", "Public": true, "ImageType": "machine", "Name": "rhcos_dev_c9a6bb4-hvm" } That doesn't include the "tested" information, so there's still no support for changing channels. We'll need to wait for a public analog of [1], which is blocked on getting stable, production hosting for the release metadata. I'd prefer to use JMESPath and server-side filtering in Go as well, to only return the latest matching AMI. But the AWS Go library doesn't seem to support server-side filtering at the moment [2]. Docs for the AWS Go APIs I'm using are in [3,4,5,6,7,8]. The filters I'm adding here are similar to those we used for Container Linux before they were dropped in 702ee7bb (*: Remove stale Container Linux references, 2018-09-11, #233). I added a few more just to be conservative (e.g. we don't want to match a pending or failed image, so I require state to be available). I haven't pushed the Context variables all the way up the stack yet, so there are some context.TODO() entries. The 30-second timeout keeps us from hanging excessively when the caller lacks AWS credentials; the error messages look like: failed to init cluster: failed to parse test config: failed to determine default AMI: NoCredentialProviders: no valid providers in chain. Deprecated. For verbose messaging see aws.Config.CredentialsChainVerboseErrors You can test this error condition by removing the explicit AMI values I've added to our fixtures in this commit and running: $ AWS_PROFILE=does-not-exist go test ./installer/pkg/... [1]: http://aos-ostree.rhev-ci-vms.eng.rdu2.redhat.com/rhcos/images/aws-us-east-1-tested.json [2]: https://github.com/aws/aws-sdk-go/issues/2156 [3]: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#NewSessionWithOptions [4]: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#Options [5]: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#Must [6]: https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#New [7]: https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#EC2.DescribeImagesWithContext [8]: https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#DescribeImagesInput --- .../config-generator/fixtures/test-aws.yaml | 1 + .../pkg/config-generator/fixtures/test.yaml | 2 + installer/pkg/config-generator/generator.go | 3 +- .../pkg/workflow/fixtures/aws.basic.yaml | 1 + pkg/asset/manifests/machine-api-operator.go | 3 +- pkg/rhcos/ami.go | 72 +++++++++++++++++-- pkg/types/config/parser.go | 7 +- 7 files changed, 82 insertions(+), 7 deletions(-) diff --git a/installer/pkg/config-generator/fixtures/test-aws.yaml b/installer/pkg/config-generator/fixtures/test-aws.yaml index 0cd5ffdd6a3..21fd9ed9130 100644 --- a/installer/pkg/config-generator/fixtures/test-aws.yaml +++ b/installer/pkg/config-generator/fixtures/test-aws.yaml @@ -17,6 +17,7 @@ admin: email: test@coreos.com password: asd123 aws: + ec2AMIOverride: ami-0af8953af3ec06b7c region: us-east-1 sshKey: tectonic vpcCIDRBlock: 10.0.0.0/16 diff --git a/installer/pkg/config-generator/fixtures/test.yaml b/installer/pkg/config-generator/fixtures/test.yaml index 0ec4099a156..e8784ac3f2c 100644 --- a/installer/pkg/config-generator/fixtures/test.yaml +++ b/installer/pkg/config-generator/fixtures/test.yaml @@ -7,3 +7,5 @@ master: nodePools: - name: master count: 3 +aws: + ec2AMIOverride: ami-0af8953af3ec06b7c diff --git a/installer/pkg/config-generator/generator.go b/installer/pkg/config-generator/generator.go index aba44ae7532..68e9278437b 100644 --- a/installer/pkg/config-generator/generator.go +++ b/installer/pkg/config-generator/generator.go @@ -1,6 +1,7 @@ package configgenerator import ( + "context" "crypto/rand" "encoding/base64" "encoding/hex" @@ -119,7 +120,7 @@ func (c *ConfigGenerator) maoConfig(clusterDir string) (*maoOperatorConfig, erro if c.AWS.EC2AMIOverride != "" { ami = c.AWS.EC2AMIOverride } else { - ami, err = rhcos.AMI(rhcos.DefaultChannel, c.Region) + ami, err = rhcos.AMI(context.TODO(), rhcos.DefaultChannel, c.Region) if err != nil { return nil, fmt.Errorf("failed to lookup RHCOS AMI: %v", err) } diff --git a/installer/pkg/workflow/fixtures/aws.basic.yaml b/installer/pkg/workflow/fixtures/aws.basic.yaml index 9cfb628ad02..a2f2ca7f46d 100644 --- a/installer/pkg/workflow/fixtures/aws.basic.yaml +++ b/installer/pkg/workflow/fixtures/aws.basic.yaml @@ -2,6 +2,7 @@ admin: email: fake-email@example.com password: fake-password aws: + ec2AMIOverride: ami-0af8953af3ec06b7c master: ec2Type: m4.large rootVolume: diff --git a/pkg/asset/manifests/machine-api-operator.go b/pkg/asset/manifests/machine-api-operator.go index a1f61ee14f9..7ec5ca1f009 100644 --- a/pkg/asset/manifests/machine-api-operator.go +++ b/pkg/asset/manifests/machine-api-operator.go @@ -1,6 +1,7 @@ package manifests import ( + "context" "fmt" "path/filepath" @@ -111,7 +112,7 @@ func (mao *machineAPIOperator) maoConfig(dependencies map[asset.Asset]*asset.Sta if mao.installConfig.Platform.AWS != nil { var ami string - ami, err := rhcos.AMI(DefaultChannel, mao.installConfig.Platform.AWS.Region) + ami, err := rhcos.AMI(context.TODO(), DefaultChannel, mao.installConfig.Platform.AWS.Region) if err != nil { return "", fmt.Errorf("failed to lookup RHCOS AMI: %v", err) } diff --git a/pkg/rhcos/ami.go b/pkg/rhcos/ami.go index c5612fbf249..ba382769d60 100644 --- a/pkg/rhcos/ami.go +++ b/pkg/rhcos/ami.go @@ -1,7 +1,13 @@ package rhcos import ( + "context" "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" ) const ( @@ -10,14 +16,72 @@ const ( ) // AMI calculates a Red Hat CoreOS AMI. -func AMI(channel, region string) (ami string, err error) { +func AMI(ctx context.Context, channel, region string) (ami string, err error) { if channel != DefaultChannel { return "", fmt.Errorf("channel %q is not yet supported", channel) } - if region != "us-east-1" { - return "", fmt.Errorf("region %q is not yet supported", region) + ssn := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String(region), + }, + })) + + svc := ec2.New(ssn) + + result, err := svc.DescribeImagesWithContext(ctx, &ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("name"), + Values: aws.StringSlice([]string{"rhcos*"}), + }, + { + Name: aws.String("architecture"), + Values: aws.StringSlice([]string{"x86_64"}), + }, + { + Name: aws.String("virtualization-type"), + Values: aws.StringSlice([]string{"hvm"}), + }, + { + Name: aws.String("image-type"), + Values: aws.StringSlice([]string{"machine"}), + }, + { + Name: aws.String("owner-id"), + Values: aws.StringSlice([]string{"531415883065"}), + }, + { + Name: aws.String("state"), + Values: aws.StringSlice([]string{"available"}), + }, + }, + }) + if err != nil { + return "", err + } + + var image *ec2.Image + var created time.Time + for _, nextImage := range result.Images { + if nextImage.ImageId == nil || nextImage.CreationDate == nil { + continue + } + nextCreated, err := time.Parse(time.RFC3339, *nextImage.CreationDate) + if err != nil { + return "", err + } + + if image == nil || nextCreated.After(created) { + image = nextImage + created = nextCreated + } + } + + if image == nil { + return "", fmt.Errorf("no RHCOS AMIs found in %s", region) } - return "ami-0af8953af3ec06b7c", nil + return *image.ImageId, nil } diff --git a/pkg/types/config/parser.go b/pkg/types/config/parser.go index 09f286a7f3f..d2b92acfcfd 100644 --- a/pkg/types/config/parser.go +++ b/pkg/types/config/parser.go @@ -1,9 +1,11 @@ package config import ( + "context" "errors" "fmt" "io/ioutil" + "time" "gopkg.in/yaml.v2" @@ -32,7 +34,10 @@ func ParseConfig(data []byte) (*Cluster, error) { } if cluster.EC2AMIOverride == "" { - ami, err := rhcos.AMI(rhcos.DefaultChannel, cluster.AWS.Region) + ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) + defer cancel() + + ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, cluster.AWS.Region) if err != nil { return nil, fmt.Errorf("failed to determine default AMI: %v", err) }