rhcos: Embed full build metadata in binary, add override envvar

This way we avoid needing a service available at least for AWS
installs.  The AMIs now get hardcoded into the binary.

Further, in order to make it easier for developers to test
alternative builds of RHCOS, add an override environment variable
just like is available for the release payload.

`hack/update-rhcos-bootimage.py` is a small script which accepts
a URL to build metadata and updates our cached version with just
the subset of keys we care about (or potentially care about); e.g.
not the pkgdiff.

data/data/rhcos.json

@@ -0,0 +1,83 @@
+{
+    "amis": [
+        {
+            "hvm": "ami-088e16048b7067a11",
+            "name": "ap-northeast-1"
+        },
+        {
+            "hvm": "ami-02d307be73b793f42",
+            "name": "ap-northeast-2"
+        },
+        {
+            "hvm": "ami-0af24a18980d2d990",
+            "name": "ap-south-1"
+        },
+        {
+            "hvm": "ami-0bda8cde9fb69a545",
+            "name": "ap-southeast-1"
+        },
+        {
+            "hvm": "ami-00e1827fc950fb38c",
+            "name": "ap-southeast-2"
+        },
+        {
+            "hvm": "ami-02047f83d438def3e",
+            "name": "ca-central-1"
+        },
+        {
+            "hvm": "ami-0472155d9fea5056f",
+            "name": "eu-central-1"
+        },
+        {
+            "hvm": "ami-08b6d57cc42948927",
+            "name": "eu-west-1"
+        },
+        {
+            "hvm": "ami-0ed12e3715aee7cdb",
+            "name": "eu-west-2"
+        },
+        {
+            "hvm": "ami-036e9773fa7520f7b",
+            "name": "eu-west-3"
+        },
+        {
+            "hvm": "ami-0218529bb6b2a7b6a",
+            "name": "sa-east-1"
+        },
+        {
+            "hvm": "ami-0e6248dbf1f9e164c",
+            "name": "us-east-1"
+        },
+        {
+            "hvm": "ami-026d285b42705e30e",
+            "name": "us-east-2"
+        },
+        {
+            "hvm": "ami-0646b7bd4b7771505",
+            "name": "us-west-1"
+        },
+        {
+            "hvm": "ami-06d3e0b5f0d941b0e",
+            "name": "us-west-2"
+        }
+    ],
+    "images": {
+        "openstack": {
+            "path": "rhcos-maipo-400.7.20190306.0-openstack.qcow2.gz",
+            "sha256": "20c8e556958d49f0baf2ef9dfbc8bc98066219a4c5fceabdfe2faa95f8ba7854",
+            "uncompressed-sha256": "ac3acb54327724560dbf882537f02c9c9d54feaf97820325e60fed5a4c775443"
+        },
+        "qemu": {
+            "path": "rhcos-maipo-400.7.20190306.0-qemu.qcow2.gz",
+            "sha256": "2b662fc001eb026c2ba54a98c2b167c3da3a50ddb4a833dbc6299ea1230d331d",
+            "uncompressed-sha256": "e4ee475ec31243424f1e7bc61ef0bd9dcfe9a9817d8fa05d0e7dd072d9d3b60a"
+        }
+    },
+    "buildid": "400.7.20190306.0",
+    "oscontainer": {
+        "digest": "sha256:c09f455cc09673a1a13ae7b54cc4348cda0411e06dfa79ecd0130b35d62e8670",
+        "image": "docker-registry-default.cloud.registry.upshift.redhat.com/redhat-coreos/maipo"
+    },
+    "ostree-commit": "109ec0b9b06e5070a4aec54370fa3787d8ec414d0860cb596771cc585899f2f2",
+    "ostree-version": "400.7.20190306.0"
+}

hack/build.sh

@@ -2,8 +2,6 @@
 
 set -ex
 
-RHCOS_BUILD_NAME="${RHCOS_BUILD_NAME:-400.7.20190306.0}"
-
 # shellcheck disable=SC2068
 version() { IFS="."; printf "%03d%03d%03d\\n" $@; unset IFS;}
 
@@ -46,10 +44,6 @@ release)
 	then
 		LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/asset/ignition/bootstrap.defaultReleaseImage=${RELEASE_IMAGE}"
 	fi
-	if test -n "${RHCOS_BUILD_NAME}"
-	then
-		LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/rhcos.buildName=${RHCOS_BUILD_NAME}"
-	fi
 	if test "${SKIP_GENERATION}" != y
 	then
 		go generate ./data

hack/update-rhcos-bootimage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/python3
+import os,sys,json,argparse
+import urllib.request
+
+dn = os.path.abspath(os.path.dirname(sys.argv[0]))
+
+parser = argparse.ArgumentParser()
+parser.add_argument("meta", action='store')
+args = parser.parse_args()
+
+with urllib.request.urlopen(args.meta) as f:
+    meta = json.load(f)
+newmeta = {}
+for k in ['amis', 'images', 'buildid', 'oscontainer',
+          'ostree-commit', 'ostree-version']:
+    newmeta[k] = meta[k]
+with open(os.path.join(dn, "../data/data/rhcos.json"), 'w') as f:
+    json.dump(newmeta, f, indent=4)

pkg/asset/rhcos/image.go

@@ -55,9 +55,9 @@ func (i *Image) Generate(p asset.Parents) error {
 	defer cancel()
 	switch config.Platform.Name() {
 	case aws.Name:
-		osimage, err = rhcos.AMI(ctx, rhcos.DefaultChannel, config.Platform.AWS.Region)
+		osimage, err = rhcos.AMI(ctx, config.Platform.AWS.Region)
 	case libvirt.Name:
-		osimage, err = rhcos.QEMU(ctx, rhcos.DefaultChannel)
+		osimage, err = rhcos.QEMU(ctx)
 	case openstack.Name:
 		osimage = "rhcos"
 	case none.Name:

pkg/rhcos/ami.go

@@ -6,9 +6,9 @@ import (
 	"github.com/pkg/errors"
 )
 
-// AMI fetches the HVM AMI ID of the latest Red Hat Enterprise Linux CoreOS release.
-func AMI(ctx context.Context, channel, region string) (string, error) {
-	meta, err := fetchLatestMetadata(ctx, channel)
+// AMI fetches the HVM AMI ID of the Red Hat Enterprise Linux CoreOS release.
+func AMI(ctx context.Context, region string) (string, error) {
+	meta, err := fetchRHCOSBuild(ctx)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
 	}

pkg/rhcos/builds.go

@@ -6,20 +6,19 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+
+	"github.com/openshift/installer/data"
 )
 
 var (
-	// DefaultChannel is the default RHCOS channel for the cluster.
-	DefaultChannel = "maipo"
-
-	// buildName is the name of the build in the channel that will be picked up
-	// empty string means the first one in the build list (latest) will be used
-	buildName = ""
+	defaultChannel = "releases/ootpa"
 
-	baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases"
+	baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage"
 )
 
 type metadata struct {
@@ -36,42 +35,77 @@ type metadata struct {
 	OSTreeVersion string `json:"ostree-version"`
 }
 
-func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) {
-	build := buildName
-	var err error
-	if build == "" {
-		build, err = fetchLatestBuild(ctx, channel)
-		if err != nil {
-			return metadata{}, errors.Wrap(err, "failed to fetch latest build")
+// rhcosBuildID is a version number e.g. 410.8.20180312.0
+var rhcosBuildID string
+
+// rhcosBuildChannel is a name for a stream of builds
+var rhcosBuildChannel string
+
+func fetchRHCOSBuild(ctx context.Context) (metadata, error) {
+	// Example uses:
+	// env OPENSHIFT_INSTALL_RHCOS_OVERRIDE=latest
+	// env OPENSHIFT_INSTALL_RHCOS_OVERRIDE=410.8.20190314.0
+	// env OPENSHIFT_INSTALL_RHCOS_OVERRIDE=testing/walters@410.8.20190312.0
+	if rc, ok := os.LookupEnv("OPENSHIFT_INSTALL_RHCOS_OVERRIDE"); ok && rc != "" {
+		logrus.Warn("Found override for RHEL CoreOS. Please be warned, this is not advised")
+		parts := strings.Split(rc, "@")
+		if len(parts) == 2 {
+			rhcosBuildChannel = parts[0]
+			rhcosBuildID = parts[1]
+		} else {
+			rhcosBuildChannel = defaultChannel
+			rhcosBuildID = parts[0]
 		}
 	}
+	var body []byte
+	// If there's no override, load our hardcoded version
+	if rhcosBuildID == "" {
+		// TODO(walters) for now
+		rhcosBuildChannel = "releases/maipo"
+		file, err := data.Assets.Open("rhcos.json")
+		if err != nil {
+			return metadata{}, err
+		}
+		defer file.Close()
+		body, err = ioutil.ReadAll(file)
+	} else {
+		var build string
+		var err error
+		if rhcosBuildID == "latest" {
+			build, err = fetchLatestBuild(ctx, rhcosBuildChannel)
+			if err != nil {
+				return metadata{}, errors.Wrap(err, "failed to fetch latest build")
+			}
+		} else {
+			build = rhcosBuildID
+		}
+		url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, rhcosBuildChannel, build)
+		logrus.Debugf("Fetching RHCOS metadata from %q", url)
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			return metadata{}, errors.Wrap(err, "failed to build request")
+		}
 
-	url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build)
-	logrus.Debugf("Fetching RHCOS metadata from %q", url)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return metadata{}, errors.Wrap(err, "failed to build request")
-	}
-
-	client := &http.Client{}
-	resp, err := client.Do(req.WithContext(ctx))
-	if err != nil {
-		return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
-	}
-	defer resp.Body.Close()
+		client := &http.Client{}
+		resp, err := client.Do(req.WithContext(ctx))
+		if err != nil {
+			return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
+		}
+		defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
-		return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
-	}
+		if resp.StatusCode != http.StatusOK {
+			return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
+		}
 
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return metadata{}, errors.Wrap(err, "failed to read HTTP response")
+		body, err = ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return metadata{}, errors.Wrap(err, "failed to read HTTP response")
+		}
 	}
 
 	var meta metadata
 	if err := json.Unmarshal(body, &meta); err != nil {
-		return meta, errors.Wrap(err, "failed to parse HTTP response")
+		return meta, errors.Wrap(err, "failed to parse RHCOS build metadata")
 	}
 
 	return meta, nil

pkg/rhcos/qemu.go

@@ -7,12 +7,12 @@ import (
 	"github.com/pkg/errors"
 )
 
-// QEMU fetches the URL of the latest Red Hat Enterprise Linux CoreOS release.
-func QEMU(ctx context.Context, channel string) (string, error) {
-	meta, err := fetchLatestMetadata(ctx, channel)
+// QEMU fetches the URL of the Red Hat Enterprise Linux CoreOS release.
+func QEMU(ctx context.Context) (string, error) {
+	meta, err := fetchRHCOSBuild(ctx)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
 	}
 
-	return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
+	return fmt.Sprintf("%s/%s/%s/%s", baseURL, rhcosBuildChannel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
 }