From 9037ba264b33de2119de089684268eea86aaf160 Mon Sep 17 00:00:00 2001 From: Valentin Delaye Date: Wed, 5 Mar 2025 14:29:54 +0100 Subject: [PATCH] Add support for docker content types Signed-off-by: Valentin Delaye --- src/main/java/land/oras/Registry.java | 101 +++++++++++++----- src/main/java/land/oras/utils/Const.java | 24 ++++- .../java/land/oras/RegistryWireMockTest.java | 48 +++++++-- 3 files changed, 135 insertions(+), 38 deletions(-) diff --git a/src/main/java/land/oras/Registry.java b/src/main/java/land/oras/Registry.java index dc6aa374..92bbceb1 100644 --- a/src/main/java/land/oras/Registry.java +++ b/src/main/java/land/oras/Registry.java @@ -302,7 +302,8 @@ public Manifest pushArtifact( public void pullArtifact(ContainerRef containerRef, Path path, boolean overwrite) { // Only collect layer that are files - List layers = collectLayers(containerRef, false); + String contentType = getContentType(containerRef); + List layers = collectLayers(containerRef, contentType, false); if (layers.isEmpty()) { LOG.info("Skipped pulling layers without file name in '{}'", Const.ANNOTATION_TITLE); return; @@ -387,8 +388,14 @@ public Manifest pushArtifact( return manifest; } - private void writeManifest(Manifest manifest, ManifestDescriptor descriptor, Path folder) throws IOException { + private void writeIndex(Index index, Path folder) throws IOException { + Path indexFile = folder.resolve(Const.OCI_LAYOUT_INDEX); + Files.writeString(indexFile, index.getJson() != null ? index.getJson() : index.toJson()); + } + + private void writeManifest(Manifest manifest, Path folder) throws IOException { Path blobs = folder.resolve(Const.OCI_LAYOUT_BLOBS); + ManifestDescriptor descriptor = manifest.getDescriptor(); String manifestDigest = descriptor.getDigest(); SupportedAlgorithm manifestAlgorithm = SupportedAlgorithm.fromDigest(manifestDigest); Path manifestFile = blobs.resolve(manifestAlgorithm.getPrefix()) @@ -456,44 +463,54 @@ public void copy(ContainerRef containerRef, Path folder) { // Write oci layout Files.writeString(folder.resolve(Const.OCI_LAYOUT_FOLDER), ociLayout.toJson()); - String contentType = getContentType(containerRef); + Map headers = getHeaders(containerRef); + String contentType = headers.get(Const.CONTENT_TYPE_HEADER.toLowerCase()); + if (contentType == null) { + throw new OrasException("Content type not found in headers"); + } + String manifestDigest = headers.get(Const.DOCKER_CONTENT_DIGEST_HEADER.toLowerCase()); + if (manifestDigest == null) { + throw new OrasException("Manifest digest not found in headers"); + } + + LOG.debug("Content type: {}", contentType); + LOG.debug("Manifest digest: {}", manifestDigest); // Single manifest - if (contentType.equals(Const.DEFAULT_MANIFEST_MEDIA_TYPE)) { + if (isManifestMediaType(contentType)) { // Write manifest as any blob Manifest manifest = getManifest(containerRef); - ManifestDescriptor descriptor = manifest.getDescriptor(); - Config sourceConfig = manifest.getConfig(); - writeManifest(manifest, descriptor, folder); + writeManifest(manifest, folder); - // Write the index.json - Index index = Index.fromManifests(List.of(descriptor)); - Path indexFile = folder.resolve(Const.OCI_LAYOUT_INDEX); - Files.writeString(indexFile, index.toJson()); + // Write the index.json containing this manifest + Index index = Index.fromManifests(List.of(manifest.getDescriptor())); + writeIndex(index, folder); // Write config as any blob - writeConfig(containerRef, sourceConfig, folder); + writeConfig(containerRef, manifest.getConfig(), folder); } // Index - else { + else if (isIndexMediaType(contentType)) { + Index index = getIndex(containerRef); // Write all manifests and their config for (ManifestDescriptor descriptor : index.getManifests()) { Manifest manifest = getManifest(containerRef.withDigest(descriptor.getDigest())); - writeManifest(manifest, descriptor, folder); - Config config = manifest.getConfig(); - writeConfig(containerRef, config, folder); + writeManifest(manifest.withDescriptor(descriptor), folder); + writeConfig(containerRef, manifest.getConfig(), folder); } - // Write index as is - Path indexFile = folder.resolve(Const.OCI_LAYOUT_INDEX); - Files.writeString(indexFile, index.getJson()); + // Write the index + writeIndex(index, folder); + + } else { + throw new OrasException("Unsupported content type: %s".formatted(contentType)); } // Write all layer - for (Layer layer : collectLayers(containerRef, true)) { + for (Layer layer : collectLayers(containerRef, contentType, true)) { try (InputStream is = fetchBlob(containerRef.withDigest(layer.getDigest()))) { // Algorithm @@ -818,7 +835,7 @@ public Manifest getManifest(ContainerRef containerRef) { logResponse(response); handleError(response); String contentType = response.headers().get(Const.CONTENT_TYPE_HEADER.toLowerCase()); - if (contentType.equals(Const.DEFAULT_INDEX_MEDIA_TYPE)) { + if (!isManifestMediaType(contentType)) { throw new OrasException( "Expected manifest but got index. Probably a multi-platform image instead of artifact"); } @@ -839,7 +856,7 @@ public Index getIndex(ContainerRef containerRef) { logResponse(response); handleError(response); String contentType = response.headers().get(Const.CONTENT_TYPE_HEADER.toLowerCase()); - if (!contentType.equals(Const.DEFAULT_INDEX_MEDIA_TYPE)) { + if (!isIndexMediaType(contentType)) { throw new OrasException("Expected index but got %s".formatted(contentType)); } String size = response.headers().get(Const.CONTENT_LENGTH_HEADER.toLowerCase()); @@ -988,7 +1005,40 @@ public InputStream getBlobStream(ContainerRef containerRef) { return fetchBlob(containerRef); } + /** + * Return if a media type is an index media type + * @param mediaType The media type + * @return True if it is a index media type + */ + private boolean isIndexMediaType(String mediaType) { + return mediaType.equals(Const.DEFAULT_INDEX_MEDIA_TYPE) || mediaType.equals(Const.DOCKER_INDEX_MEDIA_TYPE); + } + + /** + * Return if a media type is a manifest media type + * @param mediaType The media type + * @return True if it is a manifest media type + */ + private boolean isManifestMediaType(String mediaType) { + return mediaType.equals(Const.DEFAULT_MANIFEST_MEDIA_TYPE) + || mediaType.equals(Const.DOCKER_MANIFEST_MEDIA_TYPE); + } + + /** + * Get the content type of the container + * @param containerRef The container + * @return The content type + */ private String getContentType(ContainerRef containerRef) { + return getHeaders(containerRef).get(Const.CONTENT_TYPE_HEADER.toLowerCase()); + } + + /** + * Execute a head request on the manifest URL and return the headers + * @param containerRef The container + * @return The headers + */ + private Map getHeaders(ContainerRef containerRef) { URI uri = URI.create("%s://%s".formatted(getScheme(), containerRef.getManifestsPath())); OrasHttpClient.ResponseWrapper response = client.head(uri, Map.of(Const.ACCEPT_HEADER, Const.MANIFEST_ACCEPT_TYPE)); @@ -1000,7 +1050,7 @@ private String getContentType(ContainerRef containerRef) { logResponse(response); } handleError(response); - return response.headers().get(Const.CONTENT_TYPE_HEADER.toLowerCase()); + return response.headers(); } /** @@ -1009,10 +1059,9 @@ private String getContentType(ContainerRef containerRef) { * @param includeAll Include all layers or only the ones with title annotation * @return The layers */ - private List collectLayers(ContainerRef containerRef, boolean includeAll) { + private List collectLayers(ContainerRef containerRef, String contentType, boolean includeAll) { List layers = new LinkedList<>(); - String contentType = getContentType(containerRef); - if (contentType.equals(Const.DEFAULT_MANIFEST_MEDIA_TYPE)) { + if (isManifestMediaType(contentType)) { return getManifest(containerRef).getLayers(); } Index index = getIndex(containerRef); diff --git a/src/main/java/land/oras/utils/Const.java b/src/main/java/land/oras/utils/Const.java index 02612149..b3e79788 100644 --- a/src/main/java/land/oras/utils/Const.java +++ b/src/main/java/land/oras/utils/Const.java @@ -108,6 +108,21 @@ private Const() { */ public static final String ARTIFACT_MANIFEST_MEDIA_TYPE = "application/vnd.oci.artifact.manifest.v1+json"; + /** + * Docker distribution manifest type + */ + public static final String DOCKER_MANIFEST_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v2+json"; + + /** + * Docker index media type (manifest list or fat manifest) + */ + public static final String DOCKER_INDEX_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json"; + + /** + * Docker legacy media type + */ + public static final String DOCKER_LEGACY_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws"; + /** * The default manifest media type */ @@ -116,8 +131,13 @@ private Const() { /** * The default accept type for the manifest */ - public static final String MANIFEST_ACCEPT_TYPE = - "%s, %s, %s".formatted(DEFAULT_INDEX_MEDIA_TYPE, DEFAULT_MANIFEST_MEDIA_TYPE, ARTIFACT_MANIFEST_MEDIA_TYPE); + public static final String MANIFEST_ACCEPT_TYPE = "%s, %s, %s, %s, %s" + .formatted( + DEFAULT_INDEX_MEDIA_TYPE, + DEFAULT_MANIFEST_MEDIA_TYPE, + ARTIFACT_MANIFEST_MEDIA_TYPE, + DOCKER_INDEX_MEDIA_TYPE, + DOCKER_MANIFEST_MEDIA_TYPE); /** * Annotation for the title diff --git a/src/test/java/land/oras/RegistryWireMockTest.java b/src/test/java/land/oras/RegistryWireMockTest.java index d79efefa..dca6aa29 100644 --- a/src/test/java/land/oras/RegistryWireMockTest.java +++ b/src/test/java/land/oras/RegistryWireMockTest.java @@ -38,6 +38,7 @@ import land.oras.auth.UsernamePasswordProvider; import land.oras.credentials.FileStore; import land.oras.exception.OrasException; +import land.oras.utils.Const; import land.oras.utils.JsonUtils; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -58,6 +59,9 @@ public class RegistryWireMockTest { @TempDir private Path configDir; + @TempDir + private Path ociLayout; + @Test void shouldRedirectWhenDownloadingBlob(WireMockRuntimeInfo wmRuntimeInfo) { @@ -202,16 +206,6 @@ void shouldHandle500Error(WireMockRuntimeInfo wmRuntimeInfo) { // Get exception and assert OrasException exception = assertThrows(OrasException.class, () -> registry.getTags(ref)); assertEquals(500, exception.getStatusCode()); - - wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/error-artifact/manifests/latest")) - .willReturn(WireMock.noContent())); - wireMock.register(WireMock.get(WireMock.urlEqualTo("/v2/library/error-artifact/manifests/latest")) - .willReturn(WireMock.okJson(Manifest.empty().toJson()))); - wireMock.register(WireMock.post(WireMock.urlPathMatching("/v2/library/error-artifact/blobs/uploads/.*")) - .willReturn(WireMock.serverError().withBody("Internal Server Error"))); - - exception = assertThrows(OrasException.class, () -> registry.copy(registry, ref, ref)); - assertEquals(500, exception.getStatusCode()); } // Timeout with similar structure as previous test and request 408 with different artifact name @@ -233,6 +227,40 @@ void shouldHandleTimeout(WireMockRuntimeInfo wmRuntimeInfo) { assertEquals(408, exception.getStatusCode()); } + // Timeout with similar structure as previous test and request 408 with different artifact name + @Test + void copyToOciLayoutMissingInvalidContentType(WireMockRuntimeInfo wmRuntimeInfo) { + + WireMock wireMock = wmRuntimeInfo.getWireMock(); + String registryUrl = wmRuntimeInfo.getHttpBaseUrl().replace("http://", ""); + + // Using here a unique container reference to avoid conflicts when running in parallel + ContainerRef ref = ContainerRef.parse("%s/library/invalid-copy-artifact".formatted(registryUrl)); + + wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/invalid-copy-artifact/manifests/latest")) + .willReturn(WireMock.noContent())); + + // No content type + Registry registry = Registry.Builder.builder().withInsecure(true).build(); + OrasException exception = assertThrows(OrasException.class, () -> registry.copy(ref, ociLayout)); + assertEquals("Content type not found in headers", exception.getMessage()); + + // No manifest digest + wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/invalid-copy-artifact/manifests/latest")) + .willReturn( + WireMock.noContent().withHeader(Const.CONTENT_TYPE_HEADER, Const.DEFAULT_MANIFEST_MEDIA_TYPE))); + exception = assertThrows(OrasException.class, () -> registry.copy(ref, ociLayout)); + assertEquals("Manifest digest not found in headers", exception.getMessage()); + + // Invalid content type + wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/invalid-copy-artifact/manifests/latest")) + .willReturn(WireMock.noContent() + .withHeader(Const.CONTENT_TYPE_HEADER, "application/json") + .withHeader(Const.DOCKER_CONTENT_DIGEST_HEADER, "sha256:1234"))); + exception = assertThrows(OrasException.class, () -> registry.copy(ref, ociLayout)); + assertEquals("Unsupported content type: application/json", exception.getMessage()); + } + // Note: Currently this test is @Disabled because the retry functionality isn't implemented. // remove the @Disabled annotation and the test should pass. @Test