Merge pull request #3527 from nalind/zstd-import

unmarshalConvertedConfig(): handle zstd compression

config.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containers/buildah/define"
 	"github.com/containers/buildah/docker"
 	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/transports"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage/pkg/stringid"
@@ -28,18 +29,24 @@ func unmarshalConvertedConfig(ctx context.Context, dest interface{}, img types.I
 		return errors.Wrapf(err, "error getting manifest MIME type for %q", transports.ImageName(img.Reference()))
 	}
 	if wantedManifestMIMEType != actualManifestMIMEType {
+		layerInfos := img.LayerInfos()
+		for i := range layerInfos { // force the "compression" to gzip, which is supported by all of the formats we care about
+			layerInfos[i].CompressionOperation = types.Compress
+			layerInfos[i].CompressionAlgorithm = &compression.Gzip
+		}
 		updatedImg, err := img.UpdatedImage(ctx, types.ManifestUpdateOptions{
+			LayerInfos: layerInfos,
+		})
+		if err != nil {
+			return errors.Wrapf(err, "resetting recorded compression for %q", transports.ImageName(img.Reference()))
+		}
+		secondUpdatedImg, err := updatedImg.UpdatedImage(ctx, types.ManifestUpdateOptions{
 			ManifestMIMEType: wantedManifestMIMEType,
-			InformationOnly: types.ManifestUpdateInformation{ // Strictly speaking, every value in here is invalid. But…
-				Destination:  nil, // Destination is technically required, but actually necessary only for conversion _to_ v2s1.  Leave it nil, we will crash if that ever changes.
-				LayerInfos:   nil, // LayerInfos is necessary for size information in v2s2/OCI manifests, but the code can work with nil, and we are not reading the converted manifest at all.
-				LayerDiffIDs: nil, // LayerDiffIDs are actually embedded in the converted manifest, but the code can work with nil, and the values are not needed until pushing the finished image, at which time containerImageRef.NewImageSource builds the values from scratch.
-			},
 		})
 		if err != nil {
 			return errors.Wrapf(err, "error converting image %q from %q to %q", transports.ImageName(img.Reference()), actualManifestMIMEType, wantedManifestMIMEType)
 		}
-		img = updatedImg
+		img = secondUpdatedImg
 	}
 	config, err := img.ConfigBlob(ctx)
 	if err != nil {

tests/copy/copy.go

@@ -3,15 +3,19 @@ package main
 import (
 	"context"
 	"os"
+	"strings"
 
 	"github.com/containers/buildah"
 	cp "github.com/containers/image/v5/copy"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/pkg/compression"
 	"github.com/containers/image/v5/signature"
 	imageStorage "github.com/containers/image/v5/storage"
 	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/unshare"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -22,6 +26,9 @@ func main() {
 	var systemContext types.SystemContext
 	var logLevel string
 	var maxParallelDownloads uint
+	var compressionFormat string
+	var manifestFormat string
+	compressionLevel := -1
 
 	if buildah.InitReexec() {
 		return
@@ -36,6 +43,22 @@ func main() {
 			if err := cobra.ExactArgs(2)(cmd, args); err != nil {
 				return err
 			}
+			if compressionLevel != -1 {
+				systemContext.CompressionLevel = &compressionLevel
+			}
+			if compressionFormat != "" {
+				alg, err := compression.AlgorithmByName(compressionFormat)
+				if err != nil {
+					return err
+				}
+				systemContext.CompressionFormat = &alg
+			}
+			switch strings.ToLower(manifestFormat) {
+			case "oci":
+				manifestFormat = v1.MediaTypeImageManifest
+			case "docker", "dockerv2s2":
+				manifestFormat = manifest.DockerV2Schema2MediaType
+			}
 
 			level, err := logrus.ParseLevel(logLevel)
 			if err != nil {
@@ -79,10 +102,11 @@ func main() {
 			}()
 
 			options := cp.Options{
-				ReportWriter:         os.Stdout,
-				SourceCtx:            &systemContext,
-				DestinationCtx:       &systemContext,
-				MaxParallelDownloads: maxParallelDownloads,
+				ReportWriter:          os.Stdout,
+				SourceCtx:             &systemContext,
+				DestinationCtx:        &systemContext,
+				MaxParallelDownloads:  maxParallelDownloads,
+				ForceManifestMIMEType: manifestFormat,
 			}
 			if _, err = cp.Image(context.TODO(), policyContext, dest, src, &options); err != nil {
 				return err
@@ -108,6 +132,11 @@ func main() {
 	rootCmd.PersistentFlags().StringVar(&systemContext.UserShortNameAliasConfPath, "short-name-alias-conf", "", "`pathname` of short name alias cache file (not usually used)")
 	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "warn", "logging level")
 	rootCmd.PersistentFlags().UintVar(&maxParallelDownloads, "max-parallel-downloads", 0, "maximum `number` of blobs to copy at once")
+	rootCmd.PersistentFlags().StringVar(&manifestFormat, "format", "", "image manifest type")
+	rootCmd.PersistentFlags().BoolVar(&systemContext.DirForceCompress, "dest-compress", false, "force compression of layers for dir: destinations")
+	rootCmd.PersistentFlags().BoolVar(&systemContext.DirForceDecompress, "dest-decompress", false, "force decompression of layers for dir: destinations")
+	rootCmd.PersistentFlags().StringVar(&compressionFormat, "dest-compress-format", "", "compression type")
+	rootCmd.PersistentFlags().IntVar(&compressionLevel, "dest-compress-level", 0, "compression level")
 	if err := rootCmd.Execute(); err != nil {
 		os.Exit(1)
 	}

tests/from.bats

@@ -612,3 +612,8 @@ load helpers
   run_buildah inspect --format '{{.CNIPluginPath}}' $cid
   expect_output "${cni_plugin_path}"
 }
+
+@test "from-image-with-zstd-compression" {
+  copy --format oci --dest-compress --dest-compress-format zstd docker://quay.io/libpod/alpine_nginx:latest dir:${TESTDIR}/base-image
+  run_buildah from dir:${TESTDIR}/base-image
+}