Skip to content

Commit

Permalink
rhcos: Embed full build metadata in binary
Browse files Browse the repository at this point in the history 
This way we avoid needing a service available at least for AWS
installs.  The AMIs now get hardcoded into the binary.

`hack/update-rhcos-bootimage.py` is a small script which accepts
a URL to build metadata and updates our cached version with just
the subset of keys we care about (or potentially care about); e.g.
not the pkgdiff.

From Trevor:

Drop the HTTP stuff in favor of just the local asset or
`OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE`.

The codecs business works around the lack of byte-stream support in
json.load before Python 3.6 [1].

[1]: https://docs.python.org/3/library/json.html#json.load

Co-authored-by: Trevor King <wking@tremily.us>
  • Loading branch information
@cgwalters @wking
cgwalters and wking committed Apr 3, 2019
1 parent b3b8a89 commit e080f04
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 106 deletions.
94 changes: 94 additions & 0 deletions data/data/rhcos.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"amis": {
"ap-northeast-1": {
"hvm": "ami-0fbc0018a8310e53e"
},
"ap-northeast-2": {
"hvm": "ami-0ba6edf20991dee91"
},
"ap-south-1": {
"hvm": "ami-0acf1668760f8f8e9"
},
"ap-southeast-1": {
"hvm": "ami-0bcecfdf9ff5fd5ca"
},
"ap-southeast-2": {
"hvm": "ami-0480abd0220d56ae2"
},
"ca-central-1": {
"hvm": "ami-0de4e822461cb671b"
},
"eu-central-1": {
"hvm": "ami-056c9291dce6d5023"
},
"eu-west-1": {
"hvm": "ami-0f0159b00648b0bf4"
},
"eu-west-2": {
"hvm": "ami-044f299a3abcb4d96"
},
"eu-west-3": {
"hvm": "ami-095776c2e71c62b2f"
},
"sa-east-1": {
"hvm": "ami-00aefabf6d653ff5d"
},
"us-east-1": {
"hvm": "ami-07f604c5f18a1e7a9"
},
"us-east-2": {
"hvm": "ami-0eef624367320ec26"
},
"us-west-1": {
"hvm": "ami-0f89dba68d747846b"
},
"us-west-2": {
"hvm": "ami-0eac581fbaa9fa9c6"
}
},
"baseURI": "https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190325.0/",
"buildid": "410.8.20190325.0",
"images": {
"metal-bios": {
"path": "rhcos-410.8.20190325.0-metal-bios.raw",
"sha256": "3caaf8d714ac8bb820af6b31d595b8afae5ef951c68f8cee3701e236e9600f30",
"size": "735841103",
"uncompressed-sha256": "32d19b94cfb2aa45799caed2dda70f4e7ac1d0fca2b834a1cbd4459a6b0d056a",
"uncompressed-size": "17179869184"
},
"metal-uefi": {
"path": "rhcos-410.8.20190325.0-metal-uefi.raw",
"sha256": "1686bb2e3b01873804325b50e286d858435afe1fbc3e88479d54b7a21e58d0f1",
"size": "732781221",
"uncompressed-sha256": "4c9a34f8b30ff7c5b107c87faac0f34a6b591d07a43ae729c45d98d15f3a8fa1",
"uncompressed-size": "17179869184"
},
"openstack": {
"path": "rhcos-410.8.20190325.0-openstack.qcow2",
"sha256": "1b16214e59d38b5c1e6d291aea35f2539845232ca3300462104d022f9877fede",
"size": "721092654",
"uncompressed-sha256": "f2f1362c155d8d331387ce759a46b16c83c8a454f59d2ab5087d9781cdc29c97",
"uncompressed-size": "2014380032"
},
"qemu": {
"path": "rhcos-410.8.20190325.0-qemu.qcow2",
"sha256": "12b2db0cae8ea4019f24183dfc905609ab27e8a0fa01bf3631dc6ad7d2afe664",
"size": "721096885",
"uncompressed-sha256": "3b288cf2e02b63f8852c731836d5923a75230a836c63814a3988a96c26268257",
"uncompressed-size": "2014314496"
},
"vmware": {
"path": "rhcos-410.8.20190325.0-vmware.ova",
"sha256": "d8c78e90f3a6fccd21c848a53b669c1c51080972377ad3f6bc81cdcc2cb48300",
"size": "710257080",
"uncompressed-sha256": "f73f5ef77095b0c79a648e2957a965f60544da08c78176028393948500da0dfc",
"uncompressed-size": "743802880"
}
},
"oscontainer": {
"digest": "sha256:007512169406dfa78f2eefdcd1553d94c399340bd1e871b4c884baf609a967ba",
"image": "docker-registry-default.cloud.registry.upshift.redhat.com/redhat-coreos/ootpa"
},
"ostree-commit": "ed3e5f10b22db9db37e48b72e907ee60113ed259eeef5c019faa0109feeeccf8",
"ostree-version": "410.8.20190325.0"
}
6 changes: 0 additions & 6 deletions hack/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

set -ex

RHCOS_BUILD_NAME="${RHCOS_BUILD_NAME:-410.8.20190325.0}"

# shellcheck disable=SC2068
version() { IFS="."; printf "%03d%03d%03d\\n" $@; unset IFS;}

Expand Down Expand Up @@ -46,10 +44,6 @@ release)
then
LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/asset/ignition/bootstrap.defaultReleaseImage=${RELEASE_IMAGE}"
fi
if test -n "${RHCOS_BUILD_NAME}"
then
LDFLAGS="${LDFLAGS} -X github.com/openshift/installer/pkg/rhcos.buildName=${RHCOS_BUILD_NAME}"
fi
if test "${SKIP_GENERATION}" != y
then
go generate ./data
Expand Down
28 changes: 28 additions & 0 deletions hack/update-rhcos-bootimage.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/python3
# Usage: ./hack/update-rhcos-bootimage.py https://releases-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190401.0/meta.json
import codecs,os,sys,json,argparse
import urllib.parse
import urllib.request

dn = os.path.abspath(os.path.dirname(sys.argv[0]))

parser = argparse.ArgumentParser()
parser.add_argument("meta", action='store')
args = parser.parse_args()

with urllib.request.urlopen(args.meta) as f:
string_f = codecs.getreader('utf-8')(f) # support for Python < 3.6
meta = json.load(string_f)
newmeta = {}
for k in ['images', 'buildid', 'oscontainer',
'ostree-commit', 'ostree-version']:
newmeta[k] = meta[k]
newmeta['amis'] = {
entry['name']: {
'hvm': entry['hvm'],
}
for entry in meta['amis']
}
newmeta['baseURI'] = urllib.parse.urljoin(args.meta, '.')
with open(os.path.join(dn, "../data/data/rhcos.json"), 'w') as f:
json.dump(newmeta, f, sort_keys=True, indent=4)
4 changes: 2 additions & 2 deletions pkg/asset/rhcos/image.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ func (i *Image) Generate(p asset.Parents) error {
defer cancel()
switch config.Platform.Name() {
case aws.Name:
osimage, err = rhcos.AMI(ctx, rhcos.DefaultChannel, config.Platform.AWS.Region)
osimage, err = rhcos.AMI(ctx, config.Platform.AWS.Region)
case libvirt.Name:
osimage, err = rhcos.QEMU(ctx, rhcos.DefaultChannel)
osimage, err = rhcos.QEMU(ctx)
case openstack.Name:
osimage = "rhcos"
case none.Name:
Expand Down
15 changes: 7 additions & 8 deletions pkg/rhcos/ami.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,17 @@ import (
"github.com/pkg/errors"
)

// AMI fetches the HVM AMI ID of the latest Red Hat Enterprise Linux CoreOS release.
func AMI(ctx context.Context, channel, region string) (string, error) {
meta, err := fetchLatestMetadata(ctx, channel)
// AMI fetches the HVM AMI ID of the Red Hat Enterprise Linux CoreOS release.
func AMI(ctx context.Context, region string) (string, error) {
meta, err := fetchRHCOSBuild(ctx)
if err != nil {
return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
}

for _, ami := range meta.AMIs {
if ami.Name == region {
return ami.HVM, nil
}
ami, ok := meta.AMIs[region]
if !ok {
return "", errors.Errorf("no RHCOS AMIs found in %s", region)
}

return "", errors.Errorf("no RHCOS AMIs found in %s", region)
return ami.HVM, nil
}
98 changes: 13 additions & 85 deletions pkg/rhcos/builds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,18 @@ package rhcos
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"

"github.com/openshift/installer/data"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)

var (
// DefaultChannel is the default RHCOS channel for the cluster.
DefaultChannel = "ootpa"

// buildName is the name of the build in the channel that will be picked up
// empty string means the first one in the build list (latest) will be used
buildName = ""

baseURL = "https://releases-rhcos.svc.ci.openshift.org/storage/releases"
)

type metadata struct {
AMIs []struct {
HVM string `json:"hvm"`
Name string `json:"name"`
AMIs map[string]struct {
HVM string `json:"hvm"`
} `json:"amis"`
Images struct {
BaseURI string `json:"baseURI"`
Images struct {
QEMU struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
Expand All @@ -36,81 +23,22 @@ type metadata struct {
OSTreeVersion string `json:"ostree-version"`
}

func fetchLatestMetadata(ctx context.Context, channel string) (metadata, error) {
build := buildName
var err error
if build == "" {
build, err = fetchLatestBuild(ctx, channel)
if err != nil {
return metadata{}, errors.Wrap(err, "failed to fetch latest build")
}
}

url := fmt.Sprintf("%s/%s/%s/meta.json", baseURL, channel, build)
logrus.Debugf("Fetching RHCOS metadata from %q", url)
req, err := http.NewRequest("GET", url, nil)
func fetchRHCOSBuild(ctx context.Context) (*metadata, error) {
file, err := data.Assets.Open("rhcos.json")
if err != nil {
return metadata{}, errors.Wrap(err, "failed to build request")
return nil, err
}
defer file.Close()

client := &http.Client{}
resp, err := client.Do(req.WithContext(ctx))
body, err := ioutil.ReadAll(file)
if err != nil {
return metadata{}, errors.Wrapf(err, "failed to fetch metadata for build %s", build)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return metadata{}, errors.Errorf("incorrect HTTP response (%s)", resp.Status)
return nil, err
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return metadata{}, errors.Wrap(err, "failed to read HTTP response")
}

var meta metadata
var meta *metadata
if err := json.Unmarshal(body, &meta); err != nil {
return meta, errors.Wrap(err, "failed to parse HTTP response")
return meta, errors.Wrap(err, "failed to parse RHCOS build metadata")
}

return meta, nil
}

func fetchLatestBuild(ctx context.Context, channel string) (string, error) {
url := fmt.Sprintf("%s/%s/builds.json", baseURL, channel)
logrus.Debugf("Fetching RHCOS builds from %q", url)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", errors.Wrap(err, "failed to build request")
}

client := &http.Client{}
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
return "", errors.Wrap(err, "failed to fetch builds")
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return "", errors.Errorf("incorrect HTTP response (%s)", resp.Status)
}

body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "failed to read HTTP response")
}

var builds struct {
Builds []string `json:"builds"`
}
if err := json.Unmarshal(body, &builds); err != nil {
return "", errors.Wrap(err, "failed to parse HTTP response")
}

if len(builds.Builds) == 0 {
return "", errors.Errorf("no builds found")
}

return builds.Builds[0], nil
}
20 changes: 15 additions & 5 deletions pkg/rhcos/qemu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,27 @@ package rhcos

import (
"context"
"fmt"
"net/url"

"github.com/pkg/errors"
)

// QEMU fetches the URL of the latest Red Hat Enterprise Linux CoreOS release.
func QEMU(ctx context.Context, channel string) (string, error) {
meta, err := fetchLatestMetadata(ctx, channel)
// QEMU fetches the URL of the Red Hat Enterprise Linux CoreOS release.
func QEMU(ctx context.Context) (string, error) {
meta, err := fetchRHCOSBuild(ctx)
if err != nil {
return "", errors.Wrap(err, "failed to fetch RHCOS metadata")
}

return fmt.Sprintf("%s/%s/%s/%s", baseURL, channel, meta.OSTreeVersion, meta.Images.QEMU.Path), nil
base, err := url.Parse(meta.BaseURI)
if err != nil {
return "", err
}

relQEMU, err := url.Parse(meta.Images.QEMU.Path)
if err != nil {
return "", err
}

return base.ResolveReference(relQEMU).String(), nil
}

0 comments on commit e080f04

Please sign in to comment.