From 9c8e4fb6e557e2394b2389eba21b95052c59b9d8 Mon Sep 17 00:00:00 2001 From: Valentin Delaye Date: Fri, 28 Feb 2025 10:04:42 +0100 Subject: [PATCH] Support for no compression for directories Signed-off-by: Valentin Delaye --- .../java/land/oras/utils/ArchiveUtils.java | 25 +--- .../java/land/oras/utils/DigestUtils.java | 17 --- .../land/oras/utils/SupportedAlgorithm.java | 8 -- .../land/oras/utils/SupportedCompression.java | 134 ++++++++++++++++++ src/test/java/land/oras/RegistryTest.java | 44 ++++++ 5 files changed, 184 insertions(+), 44 deletions(-) create mode 100644 src/main/java/land/oras/utils/SupportedCompression.java diff --git a/src/main/java/land/oras/utils/ArchiveUtils.java b/src/main/java/land/oras/utils/ArchiveUtils.java index 2f0b7486..aa6b7bbe 100644 --- a/src/main/java/land/oras/utils/ArchiveUtils.java +++ b/src/main/java/land/oras/utils/ArchiveUtils.java @@ -195,15 +195,7 @@ public static void untar(InputStream fis, Path target) { * @return The path to the tar.gz file or the tar.zstd file */ public static LocalPath compress(LocalPath path, String mediaType) { - if (!path.getMediaType().equals(Const.DEFAULT_BLOB_MEDIA_TYPE)) { - throw new OrasException("Can only compress tar media type. Given " + path.getMediaType()); - } - if (mediaType.equals(Const.BLOB_DIR_ZSTD_MEDIA_TYPE)) { - return compressZstd(path); - } else if (mediaType.equals(Const.DEFAULT_BLOB_DIR_MEDIA_TYPE)) { - return compressGzip(path); - } - throw new OrasException("Unsupported compression type: " + mediaType); + return SupportedCompression.fromMediaType(mediaType).compress(path); } /** @@ -213,15 +205,10 @@ public static LocalPath compress(LocalPath path, String mediaType) { * @return The path to the tar.gz file or the tar.zstd file */ public static LocalPath uncompress(InputStream is, String mediaType) { - if (mediaType.equals(Const.BLOB_DIR_ZSTD_MEDIA_TYPE)) { - return uncompressZstd(is); - } else if (mediaType.equals(Const.DEFAULT_BLOB_DIR_MEDIA_TYPE)) { - return uncompressGzip(is); - } - throw new OrasException("Unsupported compression type: " + mediaType); + return SupportedCompression.fromMediaType(mediaType).uncompress(is); } - private static LocalPath compressZstd(LocalPath tarFile) { + static LocalPath compressZstd(LocalPath tarFile) { LOG.trace("Compressing tar file to zstd archive"); Path tarGzFile = Paths.get(tarFile.toString() + ".gz"); try (InputStream fis = Files.newInputStream(tarFile.getPath()); @@ -237,7 +224,7 @@ private static LocalPath compressZstd(LocalPath tarFile) { return LocalPath.of(tarGzFile, Const.BLOB_DIR_ZSTD_MEDIA_TYPE); } - private static LocalPath compressGzip(LocalPath tarFile) { + static LocalPath compressGzip(LocalPath tarFile) { LOG.trace("Compressing tar file to gz archive"); Path tarGzFile = Paths.get(tarFile.toString() + ".gz"); try (InputStream fis = Files.newInputStream(tarFile.getPath()); @@ -253,7 +240,7 @@ private static LocalPath compressGzip(LocalPath tarFile) { return LocalPath.of(tarGzFile, Const.DEFAULT_BLOB_DIR_MEDIA_TYPE); } - private static LocalPath uncompressGzip(InputStream inputStream) { + static LocalPath uncompressGzip(InputStream inputStream) { LOG.trace("Uncompressing tar.gz file"); Path tarFile = createTempTar(); try (BufferedInputStream bis = new BufferedInputStream(inputStream); @@ -268,7 +255,7 @@ private static LocalPath uncompressGzip(InputStream inputStream) { return LocalPath.of(tarFile, Const.DEFAULT_BLOB_MEDIA_TYPE); } - private static LocalPath uncompressZstd(InputStream inputStream) { + static LocalPath uncompressZstd(InputStream inputStream) { LOG.trace("Uncompressing zstd file"); Path tarFile = createTempTar(); try (BufferedInputStream bis = new BufferedInputStream(inputStream); diff --git a/src/main/java/land/oras/utils/DigestUtils.java b/src/main/java/land/oras/utils/DigestUtils.java index db33e82e..50c722ba 100644 --- a/src/main/java/land/oras/utils/DigestUtils.java +++ b/src/main/java/land/oras/utils/DigestUtils.java @@ -113,21 +113,4 @@ public static String digest(String algorithm, InputStream input) { throw new OrasException("Failed to calculate digest", e); } } - - /** - * Bytes to hex string - * @param bytes of bytes[] - * @return hex string - */ - public static String bytesToHex(byte[] bytes) { - StringBuilder hexString = new StringBuilder(); - for (byte b : bytes) { - String hex = Integer.toHexString(0xff & b); - if (hex.length() == 1) { - hexString.append('0'); - } - hexString.append(hex); - } - return hexString.toString(); - } } diff --git a/src/main/java/land/oras/utils/SupportedAlgorithm.java b/src/main/java/land/oras/utils/SupportedAlgorithm.java index 4605bb8a..0d0d158b 100644 --- a/src/main/java/land/oras/utils/SupportedAlgorithm.java +++ b/src/main/java/land/oras/utils/SupportedAlgorithm.java @@ -63,14 +63,6 @@ public enum SupportedAlgorithm { this.prefix = prefix; } - /** - * Get the algorithm - * @return The algorithm - */ - public String getAlgorithm() { - return algorithm; - } - /** * Get the prefix * @return The prefix diff --git a/src/main/java/land/oras/utils/SupportedCompression.java b/src/main/java/land/oras/utils/SupportedCompression.java new file mode 100644 index 00000000..1c9e2158 --- /dev/null +++ b/src/main/java/land/oras/utils/SupportedCompression.java @@ -0,0 +1,134 @@ +/*- + * =LICENSE= + * ORAS Java SDK + * === + * Copyright (C) 2024 - 2025 ORAS + * === + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * =LICENSEEND= + */ + +package land.oras.utils; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.function.Function; +import land.oras.LocalPath; +import land.oras.exception.OrasException; +import org.jspecify.annotations.NullMarked; + +/** + * Supported compression method for archive + * See @link https://github.com/opencontainers/image-spec/blob/main/descriptor.md#digests + * See @link https://github.com/opencontainers/image-spec/blob/main/descriptor.md#registered-algorithms + */ +@NullMarked +public enum SupportedCompression { + + /** + * No compression + */ + NO_COMPRESSION(Const.DEFAULT_BLOB_MEDIA_TYPE, (localPath -> localPath), (is -> { + // This is just a tar we need to copy the stream to a temporary file + try { + Path temp = ArchiveUtils.createTempTar(); + Files.copy(is, temp, StandardCopyOption.REPLACE_EXISTING); + return LocalPath.of(temp, Const.DEFAULT_BLOB_MEDIA_TYPE); + } catch (Exception e) { + throw new OrasException("Failed to copy stream to temporary file", e); + } + })), + + /** + * GZIP + */ + GZIP(Const.DEFAULT_BLOB_DIR_MEDIA_TYPE, ArchiveUtils::compressGzip, ArchiveUtils::uncompressGzip), + + /** + * ZSTD + */ + ZSTD(Const.BLOB_DIR_ZSTD_MEDIA_TYPE, ArchiveUtils::compressZstd, ArchiveUtils::uncompressZstd); + + /** + * The media type + */ + private final String mediaType; + + /** + * The compress function + */ + private final Function compressFunction; + + /** + * The uncompress function + */ + private final Function uncompressFunction; + + /** + * Get the supported compression + * @param mediaType The media type + */ + SupportedCompression( + String mediaType, + Function compressFunction, + Function uncompressFunction) { + this.mediaType = mediaType; + this.compressFunction = compressFunction; + this.uncompressFunction = uncompressFunction; + } + + /** + * Get the media type + * @return The media type + */ + public String getMediaType() { + return mediaType; + } + + /** + * Compress + * @param path The path + * @return The compressed path + */ + LocalPath compress(LocalPath path) { + if (!path.getMediaType().equals(Const.DEFAULT_BLOB_MEDIA_TYPE)) { + throw new OrasException("Can only compress tar media type. Given " + path.getMediaType()); + } + return compressFunction.apply(path); + } + + /** + * Uncompress + * @param inputStream The input stream + * @return The uncompressed path + */ + LocalPath uncompress(InputStream inputStream) { + return uncompressFunction.apply(inputStream); + } + + /** + * Get the algorithm from a digest + * @param mediaType The media type + * @return The supported algorithm + */ + static SupportedCompression fromMediaType(String mediaType) { + for (SupportedCompression compression : SupportedCompression.values()) { + if (mediaType.equalsIgnoreCase(compression.getMediaType())) { + return compression; + } + } + throw new OrasException("Unsupported mediaType: " + mediaType); + } +} diff --git a/src/test/java/land/oras/RegistryTest.java b/src/test/java/land/oras/RegistryTest.java index eac6344d..d80584ba 100644 --- a/src/test/java/land/oras/RegistryTest.java +++ b/src/test/java/land/oras/RegistryTest.java @@ -605,6 +605,50 @@ void testShouldPushAndPullCompressedTarGzDirectory() throws IOException { assertEquals("barfoo", Files.readString(extractDir.resolve("file3.txt"))); } + @Test + void testShouldPushAndPullUncompressedTarDirectory() throws IOException { + + Registry registry = Registry.Builder.builder() + .defaults("myuser", "mypass") + .withInsecure(true) + .build(); + ContainerRef containerRef = + ContainerRef.parse("%s/library/artifact-not-compressed".formatted(this.registry.getRegistry())); + + Path file1 = blobDir.resolve("file1.txt"); + Path file2 = blobDir.resolve("file2.txt"); + Path file3 = blobDir.resolve("file3.txt"); + Files.writeString(file1, "foobar"); + Files.writeString(file2, "test1234"); + Files.writeString(file3, "barfoo"); + + // Upload blob dir + Manifest manifest = registry.pushArtifact(containerRef, LocalPath.of(blobDir, Const.DEFAULT_BLOB_MEDIA_TYPE)); + assertEquals(1, manifest.getLayers().size()); + + Layer layer = manifest.getLayers().get(0); + + // A compressed directory file + assertEquals(Const.DEFAULT_BLOB_MEDIA_TYPE, layer.getMediaType()); + Map annotations = layer.getAnnotations(); + + // Assert annotations of the layer + assertEquals(3, annotations.size()); + assertEquals(blobDir.getFileName().toString(), annotations.get(Const.ANNOTATION_TITLE)); + assertEquals("true", annotations.get(Const.ANNOTATION_ORAS_UNPACK)); + assertEquals( + SupportedAlgorithm.SHA256, + SupportedAlgorithm.fromDigest(annotations.get(Const.ANNOTATION_ORAS_CONTENT_DIGEST))); + + // Pull + registry.pullArtifact(containerRef, extractDir, true); + + // Assert extracted files + assertEquals("foobar", Files.readString(extractDir.resolve("file1.txt"))); + assertEquals("test1234", Files.readString(extractDir.resolve("file2.txt"))); + assertEquals("barfoo", Files.readString(extractDir.resolve("file3.txt"))); + } + @Test void testShouldPushAndPullCompressedZstdDirectory() throws IOException {