Merge pull request #3546 from nalind/all-the-platform

buildah build: add --all-platforms

cmd/buildah/build.go

@@ -329,6 +329,7 @@ func buildCmd(c *cobra.Command, inputArgs []string, iopts buildOptions) error {
 	options := define.BuildOptions{
 		AddCapabilities:         iopts.CapAdd,
 		AdditionalTags:          tags,
+		AllPlatforms:            iopts.AllPlatforms,
 		Annotations:             iopts.Annotation,
 		Architecture:            systemContext.ArchitectureChoice,
 		Args:                    args,

contrib/completions/bash/buildah

@@ -378,6 +378,7 @@ return 1
 
  _buildah_bud() {
      local boolean_options="
+     --all-platforms
      --help
      -h
      --layers

define/build.go

@@ -234,4 +234,8 @@ type BuildOptions struct {
 	// to build the image for.  If this slice has items in it, the OS and
 	// Architecture fields above are ignored.
 	Platforms []struct{ OS, Arch, Variant string }
+	// AllPlatforms tells the builder to set the list of target platforms
+	// to match the set of platforms for which all of the build's base
+	// images are available.  If this field is set, Platforms is ignored.
+	AllPlatforms bool
 }

docs/buildah-build.1.md

@@ -38,6 +38,10 @@ Add a custom host-to-IP mapping (host:ip)
 
 Add a line to /etc/hosts. The format is hostname:ip. The **--add-host** option can be set multiple times.
 
+**--all-platforms**
+
+Instead of building for a set of platforms specified using the **--platform** option, inspect the build's base images, and build for all of the platforms for which they are all available.  Stages that use *scratch* as a starting point can not be inspected, so at least one non-*scratch* stage must be present for detection to work usefully.
+
 **--annotation** *annotation*
 
 Add an image *annotation* (e.g. annotation=*value*) to the image metadata. Can be used multiple times.
@@ -809,6 +813,8 @@ buildah bud --platform linux/s390x,linux/ppc64le,linux/amd64 --manifest myimage
 
 buildah bud --platform linux/arm64 --platform linux/amd64 --manifest myimage /tmp/mysrc
 
+buildah bud --all-platforms --manifest myimage /tmp/mysrc
+
 ### Building an image using a URL
 
   This will clone the specified GitHub repository from the URL and use it as context. The Containerfile or Dockerfile at the root of the repository is used as the context of the build. This only works if the GitHub repository is a dedicated repository.

imagebuildah/build.go

@@ -10,15 +10,19 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 
+	"github.com/containerd/containerd/platforms"
 	"github.com/containers/buildah/define"
 	"github.com/containers/buildah/util"
 	"github.com/containers/common/libimage"
 	"github.com/containers/common/pkg/config"
+	"github.com/containers/image/v5/docker"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/shortnames"
 	istorage "github.com/containers/image/v5/storage"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
@@ -193,23 +197,33 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
 		})
 	}
 
+	if options.AllPlatforms {
+		options.Platforms, err = platformsForBaseImages(ctx, logger, paths, files, options.From, options.Args, options.SystemContext)
+		if err != nil {
+			return "", nil, err
+		}
+	}
+
 	systemContext := options.SystemContext
 	for _, platform := range options.Platforms {
 		platformContext := *systemContext
-		platformContext.OSChoice = platform.OS
-		platformContext.ArchitectureChoice = platform.Arch
-		platformContext.VariantChoice = platform.Variant
+		platformSpec := platforms.Normalize(v1.Platform{
+			OS:           platform.OS,
+			Architecture: platform.Arch,
+			Variant:      platform.Variant,
+		})
+		if platformSpec.OS != "" && platformSpec.Architecture != "" {
+			platformContext.OSChoice = platformSpec.OS
+			platformContext.ArchitectureChoice = platformSpec.Architecture
+			platformContext.VariantChoice = platformSpec.Variant
+		}
 		platformOptions := options
 		platformOptions.SystemContext = &platformContext
-		platformOptions.OS = platform.OS
-		platformOptions.Architecture = platform.Arch
+		platformOptions.OS = platformContext.OSChoice
+		platformOptions.Architecture = platformContext.ArchitectureChoice
 		logPrefix := ""
 		if len(options.Platforms) > 1 {
-			logPrefix = "[" + platform.OS + "/" + platform.Arch
-			if platform.Variant != "" {
-				logPrefix += "/" + platform.Variant
-			}
-			logPrefix += "] "
+			logPrefix = "[" + platforms.Format(platformSpec) + "] "
 		}
 		builds.Go(func() error {
 			thisID, thisRef, err := buildDockerfilesOnce(ctx, store, logger, logPrefix, platformOptions, paths, files)
@@ -219,12 +233,8 @@ func BuildDockerfiles(ctx context.Context, store storage.Store, options define.B
 			id, ref = thisID, thisRef
 			instancesLock.Lock()
 			instances = append(instances, instance{
-				ID: thisID,
-				Platform: v1.Platform{
-					OS:           platformContext.OSChoice,
-					Architecture: platformContext.ArchitectureChoice,
-					Variant:      platformContext.VariantChoice,
-				},
+				ID:       thisID,
+				Platform: platformSpec,
 			})
 			instancesLock.Unlock()
 			return nil
@@ -431,3 +441,194 @@ func preprocessContainerfileContents(logger *logrus.Logger, containerfile string
 	}
 	return &stdoutBuffer, nil
 }
+
+// platformsForBaseImages resolves the names of base images from the
+// dockerfiles, and if they are all valid references to manifest lists, returns
+// the list of platforms that are supported by all of the base images.
+func platformsForBaseImages(ctx context.Context, logger *logrus.Logger, dockerfilepaths []string, dockerfiles [][]byte, from string, args map[string]string, systemContext *types.SystemContext) ([]struct{ OS, Arch, Variant string }, error) {
+	baseImages, err := baseImages(dockerfilepaths, dockerfiles, from, args)
+	if err != nil {
+		return nil, errors.Wrapf(err, "determining list of base images")
+	}
+	logrus.Debugf("unresolved base images: %v", baseImages)
+	if len(baseImages) == 0 {
+		return nil, errors.Wrapf(err, "build uses no non-scratch base images")
+	}
+	targetPlatforms := make(map[string]struct{})
+	var platformList []struct{ OS, Arch, Variant string }
+	for baseImageIndex, baseImage := range baseImages {
+		resolved, err := shortnames.Resolve(systemContext, baseImage)
+		if err != nil {
+			return nil, errors.Wrapf(err, "resolving image name %q", baseImage)
+		}
+		var manifestBytes []byte
+		var manifestType string
+		for _, candidate := range resolved.PullCandidates {
+			ref, err := docker.NewReference(candidate.Value)
+			if err != nil {
+				logrus.Debugf("parsing image reference %q: %v", candidate.Value.String(), err)
+				continue
+			}
+			src, err := ref.NewImageSource(ctx, systemContext)
+			if err != nil {
+				logrus.Debugf("preparing to read image manifest for %q: %v", baseImage, err)
+				continue
+			}
+			candidateBytes, candidateType, err := src.GetManifest(ctx, nil)
+			_ = src.Close()
+			if err != nil {
+				logrus.Debugf("reading image manifest for %q: %v", baseImage, err)
+				continue
+			}
+			if !manifest.MIMETypeIsMultiImage(candidateType) {
+				logrus.Debugf("base image %q is not a reference to a manifest list: %v", baseImage, err)
+				continue
+			}
+			if err := candidate.Record(); err != nil {
+				logrus.Debugf("error recording name %q for base image %q: %v", candidate.Value.String(), baseImage, err)
+				continue
+			}
+			baseImage = candidate.Value.String()
+			manifestBytes, manifestType = candidateBytes, candidateType
+			break
+		}
+		if len(manifestBytes) == 0 {
+			if len(resolved.PullCandidates) > 0 {
+				return nil, errors.Errorf("base image name %q didn't resolve to a manifest list", baseImage)
+			}
+			return nil, errors.Errorf("base image name %q didn't resolve to anything", baseImage)
+		}
+		if manifestType != v1.MediaTypeImageIndex {
+			list, err := manifest.ListFromBlob(manifestBytes, manifestType)
+			if err != nil {
+				return nil, errors.Wrapf(err, "parsing manifest list from base image %q", baseImage)
+			}
+			list, err = list.ConvertToMIMEType(v1.MediaTypeImageIndex)
+			if err != nil {
+				return nil, errors.Wrapf(err, "converting manifest list from base image %q to v2s2 list", baseImage)
+			}
+			manifestBytes, err = list.Serialize()
+			if err != nil {
+				return nil, errors.Wrapf(err, "encoding converted v2s2 manifest list for base image %q", baseImage)
+			}
+		}
+		index, err := manifest.OCI1IndexFromManifest(manifestBytes)
+		if err != nil {
+			return nil, errors.Wrapf(err, "decoding manifest list for base image %q", baseImage)
+		}
+		if baseImageIndex == 0 {
+			// populate the list with the first image's normalized platforms
+			for _, instance := range index.Manifests {
+				if instance.Platform == nil {
+					continue
+				}
+				platform := platforms.Normalize(*instance.Platform)
+				targetPlatforms[platforms.Format(platform)] = struct{}{}
+				logger.Debugf("image %q supports %q", baseImage, platforms.Format(platform))
+			}
+		} else {
+			// prune the list of any normalized platforms this base image doesn't support
+			imagePlatforms := make(map[string]struct{})
+			for _, instance := range index.Manifests {
+				if instance.Platform == nil {
+					continue
+				}
+				platform := platforms.Normalize(*instance.Platform)
+				imagePlatforms[platforms.Format(platform)] = struct{}{}
+				logger.Debugf("image %q supports %q", baseImage, platforms.Format(platform))
+			}
+			var removed []string
+			for platform := range targetPlatforms {
+				if _, present := imagePlatforms[platform]; !present {
+					removed = append(removed, platform)
+					logger.Debugf("image %q does not support %q", baseImage, platform)
+				}
+			}
+			for _, remove := range removed {
+				delete(targetPlatforms, remove)
+			}
+		}
+		if baseImageIndex == len(baseImages)-1 && len(targetPlatforms) > 0 {
+			// extract the list
+			for platform := range targetPlatforms {
+				platform, err := platforms.Parse(platform)
+				if err != nil {
+					return nil, errors.Wrapf(err, "parsing platform double/triple %q", platform)
+				}
+				platformList = append(platformList, struct{ OS, Arch, Variant string }{
+					OS:      platform.OS,
+					Arch:    platform.Architecture,
+					Variant: platform.Variant,
+				})
+				logger.Debugf("base images all support %q", platform)
+			}
+		}
+	}
+	if len(platformList) == 0 {
+		return nil, errors.New("base images have no platforms in common")
+	}
+	return platformList, nil
+}
+
+// baseImages parses the dockerfilecontents, possibly replacing the first
+// stage's base image with FROM, and returns the list of base images as
+// provided.  Each entry in the dockerfilenames slice corresponds to a slice in
+// dockerfilecontents.
+func baseImages(dockerfilenames []string, dockerfilecontents [][]byte, from string, args map[string]string) ([]string, error) {
+	mainNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(dockerfilecontents[0]))
+	if err != nil {
+		return nil, errors.Wrapf(err, "error parsing main Dockerfile: %s", dockerfilenames[0])
+	}
+
+	for i, d := range dockerfilecontents[1:] {
+		additionalNode, err := imagebuilder.ParseDockerfile(bytes.NewReader(d))
+		if err != nil {
+			return nil, errors.Wrapf(err, "error parsing additional Dockerfile %s", dockerfilenames[i])
+		}
+		mainNode.Children = append(mainNode.Children, additionalNode.Children...)
+	}
+
+	b := imagebuilder.NewBuilder(args)
+	defaultContainerConfig, err := config.Default()
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get container config")
+	}
+	b.Env = defaultContainerConfig.GetDefaultEnv()
+	stages, err := imagebuilder.NewStages(mainNode, b)
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading multiple stages")
+	}
+	var baseImages []string
+	nicknames := make(map[string]bool)
+	for stageIndex, stage := range stages {
+		node := stage.Node // first line
+		for node != nil {  // each line
+			for _, child := range node.Children { // tokens on this line, though we only care about the first
+				switch strings.ToUpper(child.Value) { // first token - instruction
+				case "FROM":
+					if child.Next != nil { // second token on this line
+						// If we have a fromOverride, replace the value of
+						// image name for the first FROM in the Containerfile.
+						if from != "" {
+							child.Next.Value = from
+							from = ""
+						}
+						base := child.Next.Value
+						if base != "scratch" && !nicknames[base] {
+							// TODO: this didn't undergo variable and arg
+							// expansion, so if the AS clause in another
+							// FROM instruction uses argument values,
+							// we might not record the right value here.
+							baseImages = append(baseImages, base)
+						}
+					}
+				}
+			}
+			node = node.Next // next line
+		}
+		if stage.Name != strconv.Itoa(stageIndex) {
+			nicknames[stage.Name] = true
+		}
+	}
+	return baseImages, nil
+}

pkg/cli/common.go

@@ -48,6 +48,7 @@ type NameSpaceResults struct {
 
 // BudResults represents the results for Build flags
 type BudResults struct {
+	AllPlatforms        bool
 	Annotation          []string
 	Authfile            string
 	BuildArg            []string
@@ -175,6 +176,7 @@ func GetLayerFlags(flags *LayerResults) pflag.FlagSet {
 // GetBudFlags returns common build flags
 func GetBudFlags(flags *BudResults) pflag.FlagSet {
 	fs := pflag.FlagSet{}
+	fs.BoolVar(&flags.AllPlatforms, "all-platforms", false, "attempt to build for all base image platforms")
 	fs.String("arch", runtime.GOARCH, "set the ARCH of the image to the provided value instead of the architecture of the host")
 	fs.StringArrayVar(&flags.Annotation, "annotation", []string{}, "Set metadata for an image (default [])")
 	fs.StringVar(&flags.Authfile, "authfile", "", "path of the authentication file.")

pkg/parse/parse.go

@@ -9,11 +9,11 @@ import (
 	"net"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 	"unicode"
 
+	"github.com/containerd/containerd/platforms"
 	"github.com/containers/buildah/define"
 	"github.com/containers/buildah/pkg/sshagent"
 	"github.com/containers/common/pkg/parse"
@@ -669,7 +669,7 @@ const platformSep = "/"
 
 // DefaultPlatform returns the standard platform for the current system
 func DefaultPlatform() string {
-	return runtime.GOOS + platformSep + runtime.GOARCH
+	return platforms.DefaultString()
 }
 
 // Platform separates the platform string into os, arch and variant,

tests/bud.bats

@@ -3366,10 +3366,21 @@ _EOF
   if test $status -ne 0 ; then
     skip "unable to run arm container, assuming emulation is not available"
   fi
+  outputlist=localhost/testlist
   run_buildah 125 build --signature-policy ${TESTSDIR}/policy.json --jobs=0 --platform=linux/arm64,linux/amd64 --manifest $outputlist -f ${TESTSDIR}/bud/multiarch/Dockerfile.fail-multistage ${TESTSDIR}/bud/multiarch
   expect_output --substring 'error building at STEP "RUN false"'
 }
 
+@test "bud-multiple-platform-no-run" {
+  outputlist=localhost/testlist
+  run_buildah build --signature-policy ${TESTSDIR}/policy.json --jobs=0 --all-platforms --manifest $outputlist -f ${TESTSDIR}/bud/multiarch/Dockerfile.no-run ${TESTSDIR}/bud/multiarch
+  run_buildah manifest inspect $outputlist
+  echo "$output"
+  run jq '.manifests | length' <<< "$output"
+  echo "$output"
+  [[ "$output" -gt 1 ]] # should at least be more than one entry in there, right?
+}
+
 # * Performs multi-stage build with label1=value1 and verifies
 # * Relabels build with label1=value2 and verifies
 # * Rebuild with label1=value1 and makes sure everything is used from cache

tests/bud/multiarch/Dockerfile.no-run

@@ -0,0 +1,7 @@
+# A base image that is known to be a manifest list.
+FROM docker.io/library/alpine
+COPY Dockerfile.no-run /root/
+# A different base image that is known to be a manifest list, supporting a
+# different but partially-overlapping set of platforms.
+FROM registry.access.redhat.com/ubi8-micro
+COPY --from=0 /root/Dockerfile.no-run /root/