pkg/rhcos: Default to the most-recent AMI

We're pushing public AMIs since openshift/os@6dd20dc6 (jenkins: Make
RHCOS AMI Public, 2018-09-18, openshift/os#304).  There's still no
public analog to [1], so I'm just scraping this from metadata on
images available via the AWS API.  The analogous AWS command line
invocation is:

  $ AWS_DEFAULT_REGION=us-east-1 aws ec2 describe-images --filter 'Name=name,Values=rhcos*' --query 'sort_by(Images, &CreationDate)[-1].ImageId' --output text

with a few extra filters thrown in.  The full set of metadata on the
most recent current image is:

  $ AWS_DEFAULT_REGION=us-east-1 aws ec2 describe-images --filter 'Name=name,Values=rhcos*' --query 'sort_by(Images, &CreationDate)[-1]' --output json
  {
      "VirtualizationType": "hvm",
      "Description": "Red Hat CoreOS 4.0.5846 (c9a6bb48b837b5bcfeb9bd427be9a18b5bd75b6c57cb289245f211ff98b2a740)",
      "Hypervisor": "xen",
      "EnaSupport": true,
      "SriovNetSupport": "simple",
      "ImageId": "ami-08a5792a684330602",
      "State": "available",
      "BlockDeviceMappings": [
          {
              "DeviceName": "/dev/xvda",
              "Ebs": {
                  "Encrypted": false,
                  "DeleteOnTermination": true,
                  "VolumeType": "gp2",
                  "VolumeSize": 8,
                  "SnapshotId": "snap-00a45db4ad6173805"
              }
          },
          {
              "DeviceName": "/dev/xvdb",
              "VirtualName": "ephemeral0"
          }
      ],
      "Architecture": "x86_64",
      "ImageLocation": "531415883065/rhcos_dev_c9a6bb4-hvm",
      "RootDeviceType": "ebs",
      "OwnerId": "531415883065",
      "RootDeviceName": "/dev/xvda",
      "CreationDate": "2018-09-19T23:40:54.000Z",
      "Public": true,
      "ImageType": "machine",
      "Name": "rhcos_dev_c9a6bb4-hvm"
  }

That doesn't include the "tested" information, so there's still no
support for changing channels.  We'll need to wait for a public analog
of [1], which is blocked on getting stable, production hosting for the
release metadata.

I'd prefer to use JMESPath and server-side filtering in Go as well, to
only return the latest matching AMI.  But the AWS Go library doesn't
seem to support server-side filtering at the moment [2].  Docs for the
AWS Go APIs I'm using are in [3,4,5,6,7,8].

The filters I'm adding here are similar to those we used for Container
Linux before they were dropped in 702ee7b (*: Remove stale Container
Linux references, 2018-09-11, openshift#233).  I added a few more just to be
conservative (e.g. we don't want to match a pending or failed image,
so I require state to be available).

I haven't pushed the Context variables all the way up the stack yet,
so there are some context.TODO() entries.  The 30-second timeout keeps
us from hanging excessively when the caller lacks AWS credentials; the
error messages look like:

  failed to init cluster: failed to parse test config: failed to determine default AMI: NoCredentialProviders: no valid providers in chain. Deprecated.
    For verbose messaging see aws.Config.CredentialsChainVerboseErrors

You can test this error condition by removing the explicit AMI values
I've added to our fixtures in this commit and running:

  $ AWS_PROFILE=does-not-exist go test ./installer/pkg/...

[1]: http://aos-ostree.rhev-ci-vms.eng.rdu2.redhat.com/rhcos/images/aws-us-east-1-tested.json
[2]: aws/aws-sdk-go#2156
[3]: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#NewSessionWithOptions
[4]: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#Options
[5]: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/#Must
[6]: https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#New
[7]: https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#EC2.DescribeImagesWithContext
[8]: https://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#DescribeImagesInput

installer/pkg/config-generator/fixtures/test-aws.yaml

@@ -17,6 +17,7 @@ admin:
   email: test@coreos.com
   password: asd123
 aws:
+  ec2AMIOverride: ami-0af8953af3ec06b7c
   region: us-east-1
   sshKey: tectonic
   vpcCIDRBlock: 10.0.0.0/16

installer/pkg/config-generator/fixtures/test.yaml

@@ -7,3 +7,5 @@ master:
 nodePools:
   - name: master
     count: 3
+aws:
+  ec2AMIOverride: ami-0af8953af3ec06b7c

installer/pkg/config-generator/generator.go

@@ -1,6 +1,7 @@
 package configgenerator
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
@@ -119,7 +120,7 @@ func (c *ConfigGenerator) maoConfig(clusterDir string) (*maoOperatorConfig, erro
 		if c.AWS.EC2AMIOverride != "" {
 			ami = c.AWS.EC2AMIOverride
 		} else {
-			ami, err = rhcos.AMI(rhcos.DefaultChannel, c.Region)
+			ami, err = rhcos.AMI(context.TODO(), rhcos.DefaultChannel, c.Region)
 			if err != nil {
 				return nil, fmt.Errorf("failed to lookup RHCOS AMI: %v", err)
 			}

installer/pkg/workflow/fixtures/aws.basic.yaml

@@ -2,6 +2,7 @@ admin:
   email: fake-email@example.com
   password: fake-password
 aws:
+  ec2AMIOverride: ami-0af8953af3ec06b7c
   master:
     ec2Type: m4.large
     rootVolume:

pkg/asset/manifests/machine-api-operator.go

@@ -1,6 +1,7 @@
 package manifests
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -111,7 +112,7 @@ func (mao *machineAPIOperator) maoConfig(dependencies map[asset.Asset]*asset.Sta
 	if mao.installConfig.Platform.AWS != nil {
 		var ami string
 
-		ami, err := rhcos.AMI(DefaultChannel, mao.installConfig.Platform.AWS.Region)
+		ami, err := rhcos.AMI(context.TODO(), DefaultChannel, mao.installConfig.Platform.AWS.Region)
 		if err != nil {
 			return "", fmt.Errorf("failed to lookup RHCOS AMI: %v", err)
 		}

pkg/rhcos/ami.go

@@ -1,7 +1,13 @@
 package rhcos
 
 import (
+	"context"
 	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
 )
 
 const (
@@ -10,14 +16,72 @@ const (
 )
 
 // AMI calculates a Red Hat CoreOS AMI.
-func AMI(channel, region string) (ami string, err error) {
+func AMI(ctx context.Context, channel, region string) (ami string, err error) {
 	if channel != DefaultChannel {
 		return "", fmt.Errorf("channel %q is not yet supported", channel)
 	}
 
-	if region != "us-east-1" {
-		return "", fmt.Errorf("region %q is not yet supported", region)
+	ssn := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config: aws.Config{
+			Region: aws.String(region),
+		},
+	}))
+
+	svc := ec2.New(ssn)
+
+	result, err := svc.DescribeImagesWithContext(ctx, &ec2.DescribeImagesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("name"),
+				Values: aws.StringSlice([]string{"rhcos*"}),
+			},
+			{
+				Name:   aws.String("architecture"),
+				Values: aws.StringSlice([]string{"x86_64"}),
+			},
+			{
+				Name:   aws.String("virtualization-type"),
+				Values: aws.StringSlice([]string{"hvm"}),
+			},
+			{
+				Name:   aws.String("image-type"),
+				Values: aws.StringSlice([]string{"machine"}),
+			},
+			{
+				Name:   aws.String("owner-id"),
+				Values: aws.StringSlice([]string{"531415883065"}),
+			},
+			{
+				Name:   aws.String("state"),
+				Values: aws.StringSlice([]string{"available"}),
+			},
+		},
+	})
+	if err != nil {
+		return "", err
+	}
+
+	var image *ec2.Image
+	var created time.Time
+	for _, nextImage := range result.Images {
+		if nextImage.ImageId == nil || nextImage.CreationDate == nil {
+			continue
+		}
+		nextCreated, err := time.Parse(time.RFC3339, *nextImage.CreationDate)
+		if err != nil {
+			return "", err
+		}
+
+		if image == nil || nextCreated.After(created) {
+			image = nextImage
+			created = nextCreated
+		}
+	}
+
+	if image == nil {
+		return "", fmt.Errorf("no RHCOS AMIs found in %s", region)
 	}
 
-	return "ami-0af8953af3ec06b7c", nil
+	return *image.ImageId, nil
 }

pkg/types/config/parser.go

@@ -1,9 +1,11 @@
 package config
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"time"
 
 	"gopkg.in/yaml.v2"
 
@@ -32,7 +34,10 @@ func ParseConfig(data []byte) (*Cluster, error) {
 	}
 
 	if cluster.EC2AMIOverride == "" {
-		ami, err := rhcos.AMI(rhcos.DefaultChannel, cluster.AWS.Region)
+		ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
+		defer cancel()
+
+		ami, err := rhcos.AMI(ctx, rhcos.DefaultChannel, cluster.AWS.Region)
 		if err != nil {
 			return nil, fmt.Errorf("failed to determine default AMI: %v", err)
 		}