Skip to content

Add support for docker content types #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jonesbusy merged 1 commit into from
Mar 5, 2025
Merged

Add support for docker content types #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 75 additions & 26 deletions src/main/java/land/oras/Registry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,8 @@ public Manifest pushArtifact(
public void pullArtifact(ContainerRef containerRef, Path path, boolean overwrite) {

// Only collect layer that are files
List<Layer> layers = collectLayers(containerRef, false);
String contentType = getContentType(containerRef);
List<Layer> layers = collectLayers(containerRef, contentType, false);
if (layers.isEmpty()) {
LOG.info("Skipped pulling layers without file name in '{}'", Const.ANNOTATION_TITLE);
return;
Expand Down Expand Up @@ -387,8 +388,14 @@ public Manifest pushArtifact(
return manifest;
}

private void writeManifest(Manifest manifest, ManifestDescriptor descriptor, Path folder) throws IOException {
private void writeIndex(Index index, Path folder) throws IOException {
Path indexFile = folder.resolve(Const.OCI_LAYOUT_INDEX);
Files.writeString(indexFile, index.getJson() != null ? index.getJson() : index.toJson());
}

private void writeManifest(Manifest manifest, Path folder) throws IOException {
Path blobs = folder.resolve(Const.OCI_LAYOUT_BLOBS);
ManifestDescriptor descriptor = manifest.getDescriptor();
String manifestDigest = descriptor.getDigest();
SupportedAlgorithm manifestAlgorithm = SupportedAlgorithm.fromDigest(manifestDigest);
Path manifestFile = blobs.resolve(manifestAlgorithm.getPrefix())
Expand Down Expand Up @@ -456,44 +463,54 @@ public void copy(ContainerRef containerRef, Path folder) {
// Write oci layout
Files.writeString(folder.resolve(Const.OCI_LAYOUT_FOLDER), ociLayout.toJson());

String contentType = getContentType(containerRef);
Map<String, String> headers = getHeaders(containerRef);
String contentType = headers.get(Const.CONTENT_TYPE_HEADER.toLowerCase());
if (contentType == null) {
throw new OrasException("Content type not found in headers");
}
String manifestDigest = headers.get(Const.DOCKER_CONTENT_DIGEST_HEADER.toLowerCase());
if (manifestDigest == null) {
throw new OrasException("Manifest digest not found in headers");
}

LOG.debug("Content type: {}", contentType);
LOG.debug("Manifest digest: {}", manifestDigest);

// Single manifest
if (contentType.equals(Const.DEFAULT_MANIFEST_MEDIA_TYPE)) {
if (isManifestMediaType(contentType)) {

// Write manifest as any blob
Manifest manifest = getManifest(containerRef);
ManifestDescriptor descriptor = manifest.getDescriptor();
Config sourceConfig = manifest.getConfig();
writeManifest(manifest, descriptor, folder);
writeManifest(manifest, folder);

// Write the index.json
Index index = Index.fromManifests(List.of(descriptor));
Path indexFile = folder.resolve(Const.OCI_LAYOUT_INDEX);
Files.writeString(indexFile, index.toJson());
// Write the index.json containing this manifest
Index index = Index.fromManifests(List.of(manifest.getDescriptor()));
writeIndex(index, folder);

// Write config as any blob
writeConfig(containerRef, sourceConfig, folder);
writeConfig(containerRef, manifest.getConfig(), folder);
}
// Index
else {
else if (isIndexMediaType(contentType)) {

Index index = getIndex(containerRef);

// Write all manifests and their config
for (ManifestDescriptor descriptor : index.getManifests()) {
Manifest manifest = getManifest(containerRef.withDigest(descriptor.getDigest()));
writeManifest(manifest, descriptor, folder);
Config config = manifest.getConfig();
writeConfig(containerRef, config, folder);
writeManifest(manifest.withDescriptor(descriptor), folder);
writeConfig(containerRef, manifest.getConfig(), folder);
}

// Write index as is
Path indexFile = folder.resolve(Const.OCI_LAYOUT_INDEX);
Files.writeString(indexFile, index.getJson());
// Write the index
writeIndex(index, folder);

} else {
throw new OrasException("Unsupported content type: %s".formatted(contentType));
}

// Write all layer
for (Layer layer : collectLayers(containerRef, true)) {
for (Layer layer : collectLayers(containerRef, contentType, true)) {
try (InputStream is = fetchBlob(containerRef.withDigest(layer.getDigest()))) {

// Algorithm
Expand Down Expand Up @@ -818,7 +835,7 @@ public Manifest getManifest(ContainerRef containerRef) {
logResponse(response);
handleError(response);
String contentType = response.headers().get(Const.CONTENT_TYPE_HEADER.toLowerCase());
if (contentType.equals(Const.DEFAULT_INDEX_MEDIA_TYPE)) {
if (!isManifestMediaType(contentType)) {
throw new OrasException(
"Expected manifest but got index. Probably a multi-platform image instead of artifact");
}
Expand All @@ -839,7 +856,7 @@ public Index getIndex(ContainerRef containerRef) {
logResponse(response);
handleError(response);
String contentType = response.headers().get(Const.CONTENT_TYPE_HEADER.toLowerCase());
if (!contentType.equals(Const.DEFAULT_INDEX_MEDIA_TYPE)) {
if (!isIndexMediaType(contentType)) {
throw new OrasException("Expected index but got %s".formatted(contentType));
}
String size = response.headers().get(Const.CONTENT_LENGTH_HEADER.toLowerCase());
Expand Down Expand Up @@ -988,7 +1005,40 @@ public InputStream getBlobStream(ContainerRef containerRef) {
return fetchBlob(containerRef);
}

/**
* Return if a media type is an index media type
* @param mediaType The media type
* @return True if it is a index media type
*/
private boolean isIndexMediaType(String mediaType) {
return mediaType.equals(Const.DEFAULT_INDEX_MEDIA_TYPE) || mediaType.equals(Const.DOCKER_INDEX_MEDIA_TYPE);
}

/**
* Return if a media type is a manifest media type
* @param mediaType The media type
* @return True if it is a manifest media type
*/
private boolean isManifestMediaType(String mediaType) {
return mediaType.equals(Const.DEFAULT_MANIFEST_MEDIA_TYPE)
|| mediaType.equals(Const.DOCKER_MANIFEST_MEDIA_TYPE);
}

/**
* Get the content type of the container
* @param containerRef The container
* @return The content type
*/
private String getContentType(ContainerRef containerRef) {
return getHeaders(containerRef).get(Const.CONTENT_TYPE_HEADER.toLowerCase());
}

/**
* Execute a head request on the manifest URL and return the headers
* @param containerRef The container
* @return The headers
*/
private Map<String, String> getHeaders(ContainerRef containerRef) {
URI uri = URI.create("%s://%s".formatted(getScheme(), containerRef.getManifestsPath()));
OrasHttpClient.ResponseWrapper<String> response =
client.head(uri, Map.of(Const.ACCEPT_HEADER, Const.MANIFEST_ACCEPT_TYPE));
Expand All @@ -1000,7 +1050,7 @@ private String getContentType(ContainerRef containerRef) {
logResponse(response);
}
handleError(response);
return response.headers().get(Const.CONTENT_TYPE_HEADER.toLowerCase());
return response.headers();
}

/**
Expand All @@ -1009,10 +1059,9 @@ private String getContentType(ContainerRef containerRef) {
* @param includeAll Include all layers or only the ones with title annotation
* @return The layers
*/
private List<Layer> collectLayers(ContainerRef containerRef, boolean includeAll) {
private List<Layer> collectLayers(ContainerRef containerRef, String contentType, boolean includeAll) {
List<Layer> layers = new LinkedList<>();
String contentType = getContentType(containerRef);
if (contentType.equals(Const.DEFAULT_MANIFEST_MEDIA_TYPE)) {
if (isManifestMediaType(contentType)) {
return getManifest(containerRef).getLayers();
}
Index index = getIndex(containerRef);
Expand Down
24 changes: 22 additions & 2 deletions src/main/java/land/oras/utils/Const.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,21 @@ private Const() {
*/
public static final String ARTIFACT_MANIFEST_MEDIA_TYPE = "application/vnd.oci.artifact.manifest.v1+json";

/**
* Docker distribution manifest type
*/
public static final String DOCKER_MANIFEST_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v2+json";

/**
* Docker index media type (manifest list or fat manifest)
*/
public static final String DOCKER_INDEX_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.list.v2+json";

/**
* Docker legacy media type
*/
public static final String DOCKER_LEGACY_MEDIA_TYPE = "application/vnd.docker.distribution.manifest.v1+prettyjws";

/**
* The default manifest media type
*/
Expand All @@ -116,8 +131,13 @@ private Const() {
/**
* The default accept type for the manifest
*/
public static final String MANIFEST_ACCEPT_TYPE =
"%s, %s, %s".formatted(DEFAULT_INDEX_MEDIA_TYPE, DEFAULT_MANIFEST_MEDIA_TYPE, ARTIFACT_MANIFEST_MEDIA_TYPE);
public static final String MANIFEST_ACCEPT_TYPE = "%s, %s, %s, %s, %s"
.formatted(
DEFAULT_INDEX_MEDIA_TYPE,
DEFAULT_MANIFEST_MEDIA_TYPE,
ARTIFACT_MANIFEST_MEDIA_TYPE,
DOCKER_INDEX_MEDIA_TYPE,
DOCKER_MANIFEST_MEDIA_TYPE);

/**
* Annotation for the title
Expand Down
48 changes: 38 additions & 10 deletions src/test/java/land/oras/RegistryWireMockTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import land.oras.auth.UsernamePasswordProvider;
import land.oras.credentials.FileStore;
import land.oras.exception.OrasException;
import land.oras.utils.Const;
import land.oras.utils.JsonUtils;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
Expand All @@ -58,6 +59,9 @@ public class RegistryWireMockTest {
@TempDir
private Path configDir;

@TempDir
private Path ociLayout;

@Test
void shouldRedirectWhenDownloadingBlob(WireMockRuntimeInfo wmRuntimeInfo) {

Expand Down Expand Up @@ -202,16 +206,6 @@ void shouldHandle500Error(WireMockRuntimeInfo wmRuntimeInfo) {
// Get exception and assert
OrasException exception = assertThrows(OrasException.class, () -> registry.getTags(ref));
assertEquals(500, exception.getStatusCode());

wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/error-artifact/manifests/latest"))
.willReturn(WireMock.noContent()));
wireMock.register(WireMock.get(WireMock.urlEqualTo("/v2/library/error-artifact/manifests/latest"))
.willReturn(WireMock.okJson(Manifest.empty().toJson())));
wireMock.register(WireMock.post(WireMock.urlPathMatching("/v2/library/error-artifact/blobs/uploads/.*"))
.willReturn(WireMock.serverError().withBody("Internal Server Error")));

exception = assertThrows(OrasException.class, () -> registry.copy(registry, ref, ref));
assertEquals(500, exception.getStatusCode());
}

// Timeout with similar structure as previous test and request 408 with different artifact name
Expand All @@ -233,6 +227,40 @@ void shouldHandleTimeout(WireMockRuntimeInfo wmRuntimeInfo) {
assertEquals(408, exception.getStatusCode());
}

// Timeout with similar structure as previous test and request 408 with different artifact name
@Test
void copyToOciLayoutMissingInvalidContentType(WireMockRuntimeInfo wmRuntimeInfo) {

WireMock wireMock = wmRuntimeInfo.getWireMock();
String registryUrl = wmRuntimeInfo.getHttpBaseUrl().replace("http://", "");

// Using here a unique container reference to avoid conflicts when running in parallel
ContainerRef ref = ContainerRef.parse("%s/library/invalid-copy-artifact".formatted(registryUrl));

wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/invalid-copy-artifact/manifests/latest"))
.willReturn(WireMock.noContent()));

// No content type
Registry registry = Registry.Builder.builder().withInsecure(true).build();
OrasException exception = assertThrows(OrasException.class, () -> registry.copy(ref, ociLayout));
assertEquals("Content type not found in headers", exception.getMessage());

// No manifest digest
wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/invalid-copy-artifact/manifests/latest"))
.willReturn(
WireMock.noContent().withHeader(Const.CONTENT_TYPE_HEADER, Const.DEFAULT_MANIFEST_MEDIA_TYPE)));
exception = assertThrows(OrasException.class, () -> registry.copy(ref, ociLayout));
assertEquals("Manifest digest not found in headers", exception.getMessage());

// Invalid content type
wireMock.register(WireMock.head(WireMock.urlEqualTo("/v2/library/invalid-copy-artifact/manifests/latest"))
.willReturn(WireMock.noContent()
.withHeader(Const.CONTENT_TYPE_HEADER, "application/json")
.withHeader(Const.DOCKER_CONTENT_DIGEST_HEADER, "sha256:1234")));
exception = assertThrows(OrasException.class, () -> registry.copy(ref, ociLayout));
assertEquals("Unsupported content type: application/json", exception.getMessage());
}

// Note: Currently this test is @Disabled because the retry functionality isn't implemented.
// remove the @Disabled annotation and the test should pass.
@Test
Expand Down