squash! rhcos: Embed full build metadata in binary

Drop the HTTP stuff in favor of just the local asset or
OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE.

The codecs business works around the lack of byte-stream support in
json.load before Python 3.6 [1].

[1]: https://docs.python.org/3/library/json.html#json.load

data/data/rhcos.json

@@ -1,66 +1,52 @@
 {
-    "amis": [
-        {
-            "hvm": "ami-0fbc0018a8310e53e",
-            "name": "ap-northeast-1"
-        },
-        {
-            "hvm": "ami-0ba6edf20991dee91",
-            "name": "ap-northeast-2"
-        },
-        {
-            "hvm": "ami-0acf1668760f8f8e9",
-            "name": "ap-south-1"
-        },
-        {
-            "hvm": "ami-0bcecfdf9ff5fd5ca",
-            "name": "ap-southeast-1"
-        },
-        {
-            "hvm": "ami-0480abd0220d56ae2",
-            "name": "ap-southeast-2"
-        },
-        {
-            "hvm": "ami-0de4e822461cb671b",
-            "name": "ca-central-1"
-        },
-        {
-            "hvm": "ami-056c9291dce6d5023",
-            "name": "eu-central-1"
-        },
-        {
-            "hvm": "ami-0f0159b00648b0bf4",
-            "name": "eu-west-1"
-        },
-        {
-            "hvm": "ami-044f299a3abcb4d96",
-            "name": "eu-west-2"
-        },
-        {
-            "hvm": "ami-095776c2e71c62b2f",
-            "name": "eu-west-3"
-        },
-        {
-            "hvm": "ami-00aefabf6d653ff5d",
-            "name": "sa-east-1"
-        },
-        {
-            "hvm": "ami-07f604c5f18a1e7a9",
-            "name": "us-east-1"
-        },
-        {
-            "hvm": "ami-0eef624367320ec26",
-            "name": "us-east-2"
-        },
-        {
-            "hvm": "ami-0f89dba68d747846b",
-            "name": "us-west-1"
-        },
-        {
-            "hvm": "ami-0eac581fbaa9fa9c6",
-            "name": "us-west-2"
+    "amis": {
+        "ap-northeast-1": {
+            "hvm": "ami-0fbc0018a8310e53e"
+        },
+        "ap-northeast-2": {
+            "hvm": "ami-0ba6edf20991dee91"
+        },
+        "ap-south-1": {
+            "hvm": "ami-0acf1668760f8f8e9"
+        },
+        "ap-southeast-1": {
+            "hvm": "ami-0bcecfdf9ff5fd5ca"
+        },
+        "ap-southeast-2": {
+            "hvm": "ami-0480abd0220d56ae2"
+        },
+        "ca-central-1": {
+            "hvm": "ami-0de4e822461cb671b"
+        },
+        "eu-central-1": {
+            "hvm": "ami-056c9291dce6d5023"
+        },
+        "eu-west-1": {
+            "hvm": "ami-0f0159b00648b0bf4"
+        },
+        "eu-west-2": {
+            "hvm": "ami-044f299a3abcb4d96"
+        },
+        "eu-west-3": {
+            "hvm": "ami-095776c2e71c62b2f"
+        },
+        "sa-east-1": {
+            "hvm": "ami-00aefabf6d653ff5d"
+        },
+        "us-east-1": {
+            "hvm": "ami-07f604c5f18a1e7a9"
+        },
+        "us-east-2": {
+            "hvm": "ami-0eef624367320ec26"
+        },
+        "us-west-1": {
+            "hvm": "ami-0f89dba68d747846b"
+        },
+        "us-west-2": {
+            "hvm": "ami-0eac581fbaa9fa9c6"
         }
-    ],
+    },
+    "baseURI": "https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190325.0/",
     "buildid": "410.8.20190325.0",
     "images": {
         "metal-bios": {

hack/update-rhcos-bootimage.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 # Usage: ./hack/update-rhcos-bootimage.py https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190401.0/meta.json
-import os,sys,json,argparse
+import codecs,os,sys,json,argparse
+import urllib.parse
 import urllib.request
 
 dn = os.path.abspath(os.path.dirname(sys.argv[0]))
@@ -10,10 +11,18 @@
 args = parser.parse_args()
 
 with urllib.request.urlopen(args.meta) as f:
-    meta = json.load(f)
+    string_f = codecs.getreader('utf-8')(f)  # support for Python < 3.6
+    meta = json.load(string_f)
 newmeta = {}
-for k in ['amis', 'images', 'buildid', 'oscontainer',
+for k in ['images', 'buildid', 'oscontainer',
           'ostree-commit', 'ostree-version']:
     newmeta[k] = meta[k]
+newmeta['amis'] = {
+    entry['name']: {
+        'hvm': entry['hvm'],
+    }
+    for entry in meta['amis']
+}
+newmeta['baseURI'] = urllib.parse.urljoin(args.meta, '.')
 with open(os.path.join(dn, "../data/data/rhcos.json"), 'w') as f:
     json.dump(newmeta, f, sort_keys=True, indent=4)

pkg/rhcos/ami.go

@@ -13,11 +13,10 @@ func AMI(ctx context.Context, region string) (string, error) {
 		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
 	}
 
-	for _, ami := range meta.AMIs {
-		if ami.Name == region {
-			return ami.HVM, nil
-		}
+	ami, ok := meta.AMIs[region]
+	if !ok {
+		return "", errors.Errorf("no RHCOS AMIs found in %s", region)
 	}
 
-	return "", errors.Errorf("no RHCOS AMIs found in %s", region)
+	return ami.HVM, nil
 }

pkg/rhcos/builds.go

@@ -3,29 +3,18 @@ package rhcos
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
-	"net/http"
-
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 
 	"github.com/openshift/installer/data"
-)
-
-var (
-	// DefaultChannel is the stream of builds used by default
-	defaultChannel = "releases/ootpa"
-
-	baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage"
+	"github.com/pkg/errors"
 )
 
 type metadata struct {
-	AMIs []struct {
-		HVM  string `json:"hvm"`
-		Name string `json:"name"`
+	AMIs map[string]struct {
+		HVM string `json:"hvm"`
 	} `json:"amis"`
-	Images struct {
+	BaseURI string `json:"baseURI"`
+	Images  struct {
 		QEMU struct {
 			Path   string `json:"path"`
 			SHA256 string `json:"sha256"`
@@ -34,104 +23,22 @@ type metadata struct {
 	OSTreeVersion string `json:"ostree-version"`
 }
 
-// rhcosBuildID is a version number e.g. 410.8.20180312.0
-var rhcosBuildID string
-
-// rhcosBuildChannel is a name for a stream of builds
-var rhcosBuildChannel string
-
-func fetchRHCOSBuild(ctx context.Context) (metadata, error) {
-	// https://github.com/openshift/installer/pull/1423 is trying
-	// to add more configuration here, for now it's removed, so
-	// you need to use OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE
-	var body []byte
-	// If there's no override, load our hardcoded version
-	if rhcosBuildID == "" {
-		// TODO(walters) for now
-		rhcosBuildChannel = "releases/maipo"
-		file, err := data.Assets.Open("rhcos.json")
-		if err != nil {
-			return metadata{}, err
-		}
-		defer file.Close()
-		body, err = ioutil.ReadAll(file)
-	} else {
-		var build string
-		var err error
-		if rhcosBuildID == "latest" {
-			build, err = fetchLatestBuild(ctx, rhcosBuildChannel)
-			if err != nil {
-				return metadata{}, errors.Wrap(err, "failed to fetch latest build")
-			}
-		} else {
-			build = rhcosBuildID
-		}
-		url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, rhcosBuildChannel, build)
-		logrus.Debugf("Fetching RHCOS metadata from %q", url)
-		req, err := http.NewRequest("GET", url, nil)
-		if err != nil {
-			return metadata{}, errors.Wrap(err, "failed to build request")
-		}
-
-		client := &http.Client{}
-		resp, err := client.Do(req.WithContext(ctx))
-		if err != nil {
-			return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode != http.StatusOK {
-			return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
-		}
-
-		body, err = ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return metadata{}, errors.Wrap(err, "failed to read HTTP response")
-		}
-	}
-
-	var meta metadata
-	if err := json.Unmarshal(body, &meta); err != nil {
-		return meta, errors.Wrap(err, "failed to parse RHCOS build metadata")
-	}
-
-	return meta, nil
-}
-
-func fetchLatestBuild(ctx context.Context, channel string) (string, error) {
-	url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel)
-	logrus.Debugf("Fetching RHCOS builds from %q", url)
-	req, err := http.NewRequest("GET", url, nil)
+func fetchRHCOSBuild(ctx context.Context) (*metadata, error) {
+	file, err := data.Assets.Open("rhcos.json")
 	if err != nil {
-		return "", errors.Wrap(err, "failed to build request")
+		return nil, err
 	}
+	defer file.Close()
 
-	client := &http.Client{}
-	resp, err := client.Do(req.WithContext(ctx))
+	body, err := ioutil.ReadAll(file)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to fetch builds")
+		return nil, err
 	}
-	defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
-		return "", errors.Errorf("incorrect HTTP response (%s)", resp.Status)
-	}
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to read HTTP response")
-	}
-
-	var builds struct {
-		Builds []string `json:"builds"`
-	}
-	if err := json.Unmarshal(body, &builds); err != nil {
-		return "", errors.Wrap(err, "failed to parse HTTP response")
-	}
-
-	if len(builds.Builds) == 0 {
-		return "", errors.Errorf("no builds found")
+	var meta *metadata
+	if err := json.Unmarshal(body, &meta); err != nil {
+		return meta, errors.Wrap(err, "failed to parse RHCOS build metadata")
 	}
 
-	return builds.Builds[0], nil
+	return meta, nil
 }

pkg/rhcos/qemu.go

@@ -2,7 +2,7 @@ package rhcos
 
 import (
 	"context"
-	"fmt"
+	"path"
 
 	"github.com/pkg/errors"
 )
@@ -14,5 +14,5 @@ func QEMU(ctx context.Context) (string, error) {
 		return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
 	}
 
-	return fmt.Sprintf("%s/%s/%s/%s", baseURL, rhcosBuildChannel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
+	return path.Join(meta.BaseURI, meta.Images.QEMU.Path), nil
 }