pkg/compression: new zstd variant zstd:chunked

add a new custom variant of the zstd compression that permits to
retrieve each file separately.

The idea is based on CRFS and its stargz format for having seekable
and indexable tarballs.

One disadvantage of the stargz format is that a custom file is added
to the tarball to store the layer metadata, and the metadata file is
part of the image itself.  Clients that are not aware of the stargz
format will propagate the metadata file inside of the containers.

The zstd compression supports embeddeding additional data as part of
the stream that the zstd decompressor will ignore (skippable frame),
so the issue above with CRFS can be solved directly within the zstd
compression format.

Beside this minor advantage, zstd is much faster and compresses better
than gzip, so take this opportunity to push the zstd format further.

The zstd compression is supported by the OCI image specs since August
2019: opencontainers/image-spec#788 and has
been supported by containers/image since then.

Clients that are not aware of the zstd:chunked format, won't notice
any difference when handling a blob that uses the variant.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

pkg/compression/compression.go

@@ -27,12 +27,15 @@ var (
 	Xz = internal.NewAlgorithm("Xz", "xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor)
 	// Zstd compression.
 	Zstd = internal.NewAlgorithm("zstd", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor)
+	// Zstd:chunked compression.
+	ZstdChunked = internal.NewAlgorithm("zstd:chunked", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, getZstdCompressor)
 
 	compressionAlgorithms = map[string]Algorithm{
-		Gzip.Name():  Gzip,
-		Bzip2.Name(): Bzip2,
-		Xz.Name():    Xz,
-		Zstd.Name():  Zstd,
+		Gzip.Name():        Gzip,
+		Bzip2.Name():       Bzip2,
+		Xz.Name():          Xz,
+		Zstd.Name():        Zstd,
+		ZstdChunked.Name(): ZstdChunked,
 	}
 )
 

pkg/compression/compression_linux.go

@@ -0,0 +1,11 @@
+package compression
+
+import (
+	"io"
+
+	"github.com/containers/storage/pkg/chunked"
+)
+
+func getZstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	return chunked.ZstdCompressor(r, metadata, level)
+}

pkg/compression/compression_unsupported.go

@@ -0,0 +1,12 @@
+// +build !linux
+
+package compression
+
+import (
+	"fmt"
+	"io"
+)
+
+func getZstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
+	return nil, fmt.Errorf("zstd compressor not supported on this arch")
+}