Merge pull request #944 from cgwalters/8gb-isnt-enough-for-everyone

ore: Introspect disk for size rather than hardcode 8GB

cmd/ore/aws/upload.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -122,6 +123,31 @@ func defaultBucketURL(urlPrefix, imageName, board, file, region string) (*url.UR
 	return s3URL, nil
 }
 
+type qemuInfo struct {
+	VirtualSize uint64 `json:"virtual-size"`
+}
+
+func getDiskSizeGiB(path string) (uint32, error) {
+	out, err := exec.Command("qemu-img", "info", "--output=json", path).Output()
+	if err != nil {
+		return 0, err
+	}
+
+	var info qemuInfo
+	err = json.Unmarshal(out, &info)
+	if err != nil {
+		return 0, err
+	}
+
+	const GiB = 1024 * 1024 * 1024
+	quotient, remainder := info.VirtualSize/GiB, info.VirtualSize%GiB
+	// Round up if there's leftover
+	if remainder > 0 {
+		quotient = quotient + 1
+	}
+	return uint32(quotient), nil
+}
+
 func runUpload(cmd *cobra.Command, args []string) error {
 	if len(args) != 0 {
 		fmt.Fprintf(os.Stderr, "Unrecognized args in aws upload cmd: %v\n", args)
@@ -132,6 +158,13 @@ func runUpload(cmd *cobra.Command, args []string) error {
 		os.Exit(2)
 	}
 
+	imageSizeGiB, err := getDiskSizeGiB(uploadFile)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to query size of disk: %v\n", err)
+		os.Exit(1)
+	}
+	plog.Debugf("Image size: %v\n", imageSizeGiB)
+
 	// if an image name is unspecified try to use version.txt
 	imageName := uploadImageName
 	if imageName == "" {
@@ -155,7 +188,6 @@ func runUpload(cmd *cobra.Command, args []string) error {
 	}
 
 	var s3URL *url.URL
-	var err error
 	if uploadSourceObject != "" {
 		s3URL, err = url.Parse(uploadSourceObject)
 		if err != nil {
@@ -225,7 +257,7 @@ func runUpload(cmd *cobra.Command, args []string) error {
 	}
 
 	// create AMIs and grant permissions
-	hvmID, err := API.CreateHVMImage(sourceSnapshot, amiName+"-hvm", uploadAMIDescription)
+	hvmID, err := API.CreateHVMImage(sourceSnapshot, amiName+"-hvm", imageSizeGiB, uploadAMIDescription)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to create HVM image: %v\n", err)
 		os.Exit(1)
@@ -257,7 +289,7 @@ func runUpload(cmd *cobra.Command, args []string) error {
 
 	var pvID string
 	if uploadCreatePV {
-		pvImageID, err := API.CreatePVImage(sourceSnapshot, amiName, uploadAMIDescription)
+		pvImageID, err := API.CreatePVImage(sourceSnapshot, amiName, imageSizeGiB, uploadAMIDescription)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to create PV image: %v\n", err)
 			os.Exit(1)

cmd/plume/prerelease.go

@@ -407,12 +407,12 @@ func awsUploadToPartition(spec *channelSpec, part *awsPartitionSpec, imageName,
 
 	plog.Printf("Creating AMIs from %v...", snapshot.SnapshotID)
 
-	hvmImageID, err := api.CreateHVMImage(snapshot.SnapshotID, imageName+"-hvm", imageDescription+" (HVM)")
+	hvmImageID, err := api.CreateHVMImage(snapshot.SnapshotID, imageName+"-hvm", 0, imageDescription+" (HVM)")
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to create HVM image: %v", err)
 	}
 
-	pvImageID, err := api.CreatePVImage(snapshot.SnapshotID, imageName, imageDescription+" (PV)")
+	pvImageID, err := api.CreatePVImage(snapshot.SnapshotID, imageName, 0, imageDescription+" (PV)")
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to create PV image: %v", err)
 	}

platform/api/aws/images.go

@@ -36,6 +36,14 @@ var (
 
 type EC2ImageType string
 
+// The size of Container Linux on AWS, in GiB. See discussion in
+// https://github.com/coreos/mantle/pull/944
+// This is used if the provided size is less than 8, to ensure
+// compatibility with CL which has 4.5GiB disks right now but
+// wants 8GiB in AWS.  Otherwise, if a larger disk is provided,
+// its size will be honored.
+const containerLinuxDiskSize = 8
+
 const (
 	EC2ImageTypeHVM EC2ImageType = "hvm"
 	EC2ImageTypePV  EC2ImageType = "paravirtual"
@@ -350,18 +358,18 @@ func (a *API) CreateImportRole(bucket string) error {
 	return nil
 }
 
-func (a *API) CreateHVMImage(snapshotID string, name string, description string) (string, error) {
-	params := registerImageParams(snapshotID, name, description, "xvd", EC2ImageTypeHVM)
+func (a *API) CreateHVMImage(snapshotID string, name string, sizeGiB uint32, description string) (string, error) {
+	params := registerImageParams(snapshotID, name, sizeGiB, description, "xvd", EC2ImageTypeHVM)
 	params.EnaSupport = aws.Bool(true)
 	params.SriovNetSupport = aws.String("simple")
 	return a.createImage(params)
 }
 
-func (a *API) CreatePVImage(snapshotID string, name string, description string) (string, error) {
+func (a *API) CreatePVImage(snapshotID string, name string, sizeGiB uint32, description string) (string, error) {
 	if !RegionSupportsPV(a.opts.Region) {
 		return "", NoRegionPVSupport
 	}
-	params := registerImageParams(snapshotID, name, description, "sd", EC2ImageTypePV)
+	params := registerImageParams(snapshotID, name, sizeGiB, description, "sd", EC2ImageTypePV)
 	params.KernelId = aws.String(akis[a.opts.Region])
 	return a.createImage(params)
 }
@@ -403,9 +411,12 @@ func (a *API) createImage(params *ec2.RegisterImageInput) (string, error) {
 	return imageID, nil
 }
 
-const diskSize = 8 // GB
-
-func registerImageParams(snapshotID, name, description string, diskBaseName string, imageType EC2ImageType) *ec2.RegisterImageInput {
+func registerImageParams(snapshotID string, name string, sizeGiB uint32, description string, diskBaseName string, imageType EC2ImageType) *ec2.RegisterImageInput {
+	// See comments around the containerLinuxDiskSize constant above; this
+	// basically converts the 4.5 Container Linux disk to 8 for AWS.
+	if sizeGiB < containerLinuxDiskSize {
+		sizeGiB = containerLinuxDiskSize
+	}
 	return &ec2.RegisterImageInput{
 		Name:               aws.String(name),
 		Description:        aws.String(description),
@@ -418,7 +429,7 @@ func registerImageParams(snapshotID, name, description string, diskBaseName stri
 				Ebs: &ec2.EbsBlockDevice{
 					SnapshotId:          aws.String(snapshotID),
 					DeleteOnTermination: aws.Bool(true),
-					VolumeSize:          aws.Int64(diskSize),
+					VolumeSize:          aws.Int64(int64(sizeGiB)),
 					VolumeType:          aws.String("gp2"),
 				},
 			},