From fc51669803b594485dafb939f80f1a8eec57e2f8 Mon Sep 17 00:00:00 2001 From: Valentin Delaye Date: Sun, 6 Apr 2025 14:03:48 +0200 Subject: [PATCH] Prepare for extract for copy Signed-off-by: Valentin Delaye --- src/main/java/land/oras/LayoutRef.java | 28 +++++ src/main/java/land/oras/OCI.java | 8 ++ src/main/java/land/oras/OCILayout.java | 93 ++++++++++++++-- src/main/java/land/oras/Registry.java | 9 +- src/test/java/land/oras/LayoutRefTest.java | 10 ++ src/test/java/land/oras/OCILayoutTest.java | 122 +++++++++++++++++---- 6 files changed, 230 insertions(+), 40 deletions(-) diff --git a/src/main/java/land/oras/LayoutRef.java b/src/main/java/land/oras/LayoutRef.java index 0c8c8b8b..a3f34a29 100644 --- a/src/main/java/land/oras/LayoutRef.java +++ b/src/main/java/land/oras/LayoutRef.java @@ -69,6 +69,34 @@ public LayoutRef withDigest(String digest) { return withTag(digest); } + /** + * Convert the manifest to a layout ref. + * @param layout The OCI layout. + * @param manifest The manifest. + * @return The layout ref. + */ + public static LayoutRef fromManifest(OCILayout layout, Manifest manifest) { + ManifestDescriptor descriptor = manifest.getDescriptor(); + if (descriptor == null) { + throw new OrasException("Manifest descriptor is null"); + } + return new LayoutRef(layout.getPath(), manifest.getDescriptor().getDigest()); + } + + /** + * Convert the manifest to a layout ref. + * @param layout The OCI layout. + * @param index The manifest. + * @return The layout ref. + */ + public static LayoutRef fromIndex(OCILayout layout, Index index) { + ManifestDescriptor descriptor = index.getDescriptor(); + if (descriptor == null) { + throw new OrasException("Index descriptor is null"); + } + return new LayoutRef(layout.getPath(), index.getDescriptor().getDigest()); + } + /** * Parse the layout ref with folder and tag. * @param name The layout ref. diff --git a/src/main/java/land/oras/OCI.java b/src/main/java/land/oras/OCI.java index e6e3457d..9ee4252a 100644 --- a/src/main/java/land/oras/OCI.java +++ b/src/main/java/land/oras/OCI.java @@ -211,6 +211,14 @@ public abstract Manifest pushArtifact( */ public abstract Manifest pushManifest(T ref, Manifest manifest); + /** + * Push an index + * @param ref The ref + * @param index The index + * @return The index + */ + public abstract Index pushIndex(T ref, Index index); + /** * Retrieve an index * @param ref The ref diff --git a/src/main/java/land/oras/OCILayout.java b/src/main/java/land/oras/OCILayout.java index f7f38ebe..049ceda2 100644 --- a/src/main/java/land/oras/OCILayout.java +++ b/src/main/java/land/oras/OCILayout.java @@ -152,13 +152,54 @@ public Manifest pushManifest(LayoutRef layoutRef, Manifest manifest) { // Write blobs try { writeManifest(manifest); - writeIndex(index); + writeOCIIndex(index); } catch (IOException e) { throw new OrasException("Failed to write manifest", e); } return manifest; } + @Override + public Index pushIndex(LayoutRef layoutRef, Index index) { + byte[] indexData = index.getJson() != null + ? index.getJson().getBytes() + : index.toJson().getBytes(); + + String indexDigest = layoutRef + .getAlgorithm() + .digest( + index.getJson() != null + ? index.getJson().getBytes() + : index.toJson().getBytes()); + + ManifestDescriptor indexDescriptor = ManifestDescriptor.of( + Const.DEFAULT_INDEX_MEDIA_TYPE, indexDigest, indexData.length) + .withAnnotations( + index.getAnnotations() == null || index.getAnnotations().isEmpty() + ? null + : index.getAnnotations()) + .withArtifactType(index.getMediaType()); + if (layoutRef.getTag() != null && !layoutRef.isValidDigest()) { + Map newAnnotations = new HashMap<>(); + if (index.getAnnotations() != null) { + newAnnotations.putAll(index.getAnnotations()); + } + newAnnotations.put(Const.ANNOTATION_REF, layoutRef.getTag()); + indexDescriptor = indexDescriptor.withAnnotations(newAnnotations); + } + + Index ociIndex = Index.fromPath(getIndexPath()).withNewManifests(indexDescriptor); + + // Write blobs + try { + writeIndex(index); + writeOCIIndex(ociIndex); + } catch (IOException e) { + throw new OrasException("Failed to write manifest", e); + } + return index; + } + @Override public Index getIndex(LayoutRef ref) { Path path = getIndexPath(); @@ -348,7 +389,9 @@ public void copy(Registry registry, ContainerRef containerRef, boolean recursive // Write manifest as any blob Manifest manifest = registry.getManifest(containerRef); - writeManifest(manifest); + String tag = containerRef.getTag(); + LayoutRef layoutRef = LayoutRef.fromManifest(this, manifest).withTag(tag); + pushManifest(layoutRef, manifest); if (recursive) { LOG.debug("Recursively copy referrers"); @@ -359,10 +402,6 @@ public void copy(Registry registry, ContainerRef containerRef, boolean recursive } } - // Write the index.json containing this manifest - Index index = Index.fromManifests(List.of(manifest.getDescriptor())); - writeIndex(index); - // Write config as any blob writeConfig(registry, containerRef, manifest.getConfig()); } @@ -370,17 +409,18 @@ public void copy(Registry registry, ContainerRef containerRef, boolean recursive else if (registry.isIndexMediaType(contentType)) { Index index = registry.getIndex(containerRef); + String tag = containerRef.getTag(); + LayoutRef layoutRef = LayoutRef.fromIndex(this, index); + pushIndex(layoutRef.withTag(tag), index); // Write all manifests and their config for (ManifestDescriptor descriptor : index.getManifests()) { Manifest manifest = registry.getManifest(containerRef.withDigest(descriptor.getDigest())); - writeManifest(manifest.withDescriptor(descriptor)); + LayoutRef manifestLayoutRef = LayoutRef.fromManifest(this, manifest); + pushManifest(manifestLayoutRef, manifest.withDescriptor(descriptor)); writeConfig(registry, containerRef, manifest.getConfig()); } - // Write the index - writeIndex(index); - } else { throw new OrasException("Unsupported content type: %s".formatted(contentType)); } @@ -497,7 +537,7 @@ private Path getBlobAlgorithmPath(String digest) { return getBlobPath().resolve(algorithm.getPrefix()); } - private void writeIndex(Index index) throws IOException { + private void writeOCIIndex(Index index) throws IOException { Path indexFile = getIndexPath(); Files.writeString(indexFile, index.getJson() != null ? index.getJson() : index.toJson()); if (index.getJson() != null) { @@ -528,6 +568,29 @@ private void writeManifest(Manifest manifest) throws IOException { } } + private void writeIndex(Index index) throws IOException { + ManifestDescriptor descriptor = index.getDescriptor(); + Path manifestFile = getBlobPath(descriptor); + Path manifestPrefixDirectory = + getBlobAlgorithmPath(index.getDescriptor().getDigest()); + + if (!Files.exists(manifestPrefixDirectory)) { + Files.createDirectory(manifestPrefixDirectory); + } + // Skip if already exists + if (Files.exists(manifestFile)) { + LOG.debug("Manifest already exists: {}", manifestFile); + return; + } + if (index.getJson() == null) { + LOG.debug("Writing new manifest: {}", manifestFile); + Files.writeString(manifestFile, index.toJson()); + } else { + LOG.debug("Writing existing manifest: {}", manifestFile); + Files.writeString(manifestFile, index.getJson()); + } + } + private void writeConfig(Registry registry, ContainerRef containerRef, Config config) throws IOException { String configDigest = config.getDigest(); Path configFile = getBlobPath(config); @@ -576,6 +639,14 @@ public static OCILayout fromLayoutIndex(Path layoutPath) { return layout; } + /** + * Return the path to the OCI layout + * @return The path to the OCI layout + */ + public Path getPath() { + return path; + } + /** * Builder for the registry */ diff --git a/src/main/java/land/oras/Registry.java b/src/main/java/land/oras/Registry.java index 8af2d08c..d82a8461 100644 --- a/src/main/java/land/oras/Registry.java +++ b/src/main/java/land/oras/Registry.java @@ -205,12 +205,7 @@ public Manifest pushManifest(ContainerRef containerRef, Manifest manifest) { return getManifest(containerRef); } - /** - * Push a manifest - * @param containerRef The container - * @param index The index - * @return The location - */ + @Override public Index pushIndex(ContainerRef containerRef, Index index) { URI uri = URI.create("%s://%s".formatted(getScheme(), containerRef.getManifestsPath())); OrasHttpClient.ResponseWrapper response = client.put( @@ -571,7 +566,7 @@ private byte[] ensureDigest(ContainerRef ref, byte[] data) { SupportedAlgorithm algorithm = SupportedAlgorithm.fromDigest(ref.getDigest()); String dataDigest = algorithm.digest(data); if (!ref.getDigest().equals(dataDigest)) { - throw new OrasException("Digest mismatch: %s != %s".formatted(ref.getTag(), dataDigest)); + throw new OrasException("Digest mismatch: %s != %s".formatted(ref.getDigest(), dataDigest)); } return data; } diff --git a/src/test/java/land/oras/LayoutRefTest.java b/src/test/java/land/oras/LayoutRefTest.java index e646be06..6df5fe22 100644 --- a/src/test/java/land/oras/LayoutRefTest.java +++ b/src/test/java/land/oras/LayoutRefTest.java @@ -45,6 +45,16 @@ void shouldParseLayoutWithAllParts() { assertFalse(layoutRef.isValidDigest(), "v1 is not a valid digest"); } + @Test + void shouldCreateLayoutRefFromManifest() { + Manifest manifest = Manifest.empty(); + OCILayout layout = OCILayout.Builder.builder().defaults(tempDir).build(); + LayoutRef layoutRef = LayoutRef.fromManifest(layout, manifest); + assertNotNull(layoutRef); + assertEquals(layout.getPath().toString(), layoutRef.getFolder().toString()); + assertEquals("sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", layoutRef.getTag()); + } + @Test void shouldParseLayoutWithDigest() { String ociLayout = tempDir.resolve("foo").toString(); diff --git a/src/test/java/land/oras/OCILayoutTest.java b/src/test/java/land/oras/OCILayoutTest.java index 6a28a41b..6587ee81 100644 --- a/src/test/java/land/oras/OCILayoutTest.java +++ b/src/test/java/land/oras/OCILayoutTest.java @@ -69,7 +69,7 @@ void shouldPushEmptyManifest() { // Assertion assertOciLayout(path); - assertIndex(path, manifest); + assertIndex(path, manifest, 1, 0); assertBlobExists(path, manifest.getDescriptor().getDigest()); assertEquals(425, manifest.getDescriptor().getSize()); @@ -104,6 +104,58 @@ void shouldPushEmptyManifest() { assertEquals("bar", index.getManifests().get(1).getAnnotations().get("foo")); } + @Test + void shouldPushIndex() { + Path path = layoutPath.resolve("shouldPushIndex"); + LayoutRef layoutRef = LayoutRef.parse("%s".formatted(path.toString())); + OCILayout ociLayout = OCILayout.Builder.builder().defaults(path).build(); + Index index = Index.fromManifests(List.of(Manifest.empty().getDescriptor())); + byte[] manifestContent = index.toJson().getBytes(StandardCharsets.UTF_8); + String indexDigest = SupportedAlgorithm.getDefault().digest(manifestContent); + index = index.withDescriptor( + ManifestDescriptor.of(Const.DEFAULT_INDEX_MEDIA_TYPE, indexDigest, manifestContent.length)); + index = ociLayout.pushIndex(layoutRef, index); + + // Assertion + assertOciLayout(path); + assertIndex(path, index, 1); + assertBlobExists(path, index.getDescriptor().getDigest()); + assertEquals(229, index.getDescriptor().getSize()); + + // One element in the index + Index ociIndex = Index.fromPath(path.resolve(Const.OCI_LAYOUT_INDEX)); + assertEquals(1, ociIndex.getManifests().size()); + + // Check latest tag + assertNull(index.getManifests().get(0).getAnnotations()); + } + + @Test + void shouldPushIndexWithTag() { + Path path = layoutPath.resolve("shouldPushIndexWithTag"); + LayoutRef layoutRef = LayoutRef.parse("%s".formatted(path.toString())); + OCILayout ociLayout = OCILayout.Builder.builder().defaults(path).build(); + Index index = Index.fromManifests(List.of(Manifest.empty().getDescriptor())); + byte[] manifestContent = index.toJson().getBytes(StandardCharsets.UTF_8); + String indexDigest = SupportedAlgorithm.getDefault().digest(manifestContent); + index = index.withDescriptor( + ManifestDescriptor.of(Const.DEFAULT_INDEX_MEDIA_TYPE, indexDigest, manifestContent.length)); + index = ociLayout.pushIndex(layoutRef.withTag("latest"), index); + + // Assertion + assertOciLayout(path); + assertIndex(path, index, 1); + assertBlobExists(path, index.getDescriptor().getDigest()); + assertEquals(229, index.getDescriptor().getSize()); + + // One element in the index + Index ociIndex = Index.fromPath(path.resolve(Const.OCI_LAYOUT_INDEX)); + assertEquals(1, ociIndex.getManifests().size()); + + // Check latest tag + assertEquals("latest", ociIndex.getManifests().get(0).getAnnotations().get(Const.ANNOTATION_REF)); + } + @Test void shouldPushManifestFromFile() { @@ -119,7 +171,7 @@ void shouldPushManifestFromFile() { // Assertion assertOciLayout(path); - assertIndex(path, manifest); + assertIndex(path, manifest, 1, 0); assertBlobExists(path, manifest.getDescriptor().getDigest()); assertEquals(556, manifest.getDescriptor().getSize()); @@ -153,7 +205,7 @@ void shouldPushEmptyManifestWithRef() { // Assertion assertOciLayout(path); - assertIndex(path, manifest); + assertIndex(path, manifest, 1, 0); assertBlobExists(path, manifest.getDescriptor().getDigest()); assertEquals(425, manifest.getDescriptor().getSize()); @@ -244,7 +296,7 @@ void shouldPushToOciLayoutWithoutTag() throws IOException { assertBlobContent(ociLayoutPath, Config.empty().getDigest(), "{}"); // Check index exists - assertIndex(ociLayoutPath, manifest); + assertIndex(ociLayoutPath, manifest, 1, 0); // Assert blobs and their content assertBlobExists(ociLayoutPath, SupportedAlgorithm.SHA256.digest(artifactPath)); @@ -255,7 +307,7 @@ void shouldPushToOciLayoutWithoutTag() throws IOException { layoutRef, ArtifactType.from("foo/bar"), annotations, LocalPath.of(artifactPath, "text/plain")); // Check index exists - assertIndex(ociLayoutPath, manifest1); + assertIndex(ociLayoutPath, manifest1, 1, 0); Index index = Index.fromPath(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX)); @@ -311,7 +363,7 @@ void shouldPushToOciLayoutWithTag() throws IOException { assertBlobContent(ociLayoutPath, Config.empty().getDigest(), "{}"); // Check index exists - assertIndex(ociLayoutPath, manifest); + assertIndex(ociLayoutPath, manifest, 1, 0); // Assert blobs and their content assertBlobExists(ociLayoutPath, SupportedAlgorithm.SHA256.digest(artifactPath)); @@ -322,7 +374,7 @@ void shouldPushToOciLayoutWithTag() throws IOException { layoutRef, ArtifactType.from("foo/bar"), annotations, LocalPath.of(artifactPath, "text/plain")); // Check index exists - assertIndex(ociLayoutPath, manifest1); + assertIndex(ociLayoutPath, manifest1, 1, 0); Index index = Index.fromPath(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX)); @@ -550,7 +602,7 @@ void testShouldCopyArtifactFromRegistryIntoOciLayout() throws IOException { assertBlobContent(layoutPath, Config.empty().getDigest(), "{}"); // Check index exists - assertIndex(layoutPath, manifest); + assertIndex(layoutPath, manifest, 1, 0); // Assert blobs and their content assertBlobExists(layoutPath, SupportedAlgorithm.SHA256.digest(file1)); @@ -599,7 +651,7 @@ void testShouldCopyRecursivelyArtifactFromRegistryIntoOciLayout() throws IOExcep assertBlobContent(layoutPath, Config.empty().getDigest(), "{}"); // Check index exists - assertIndex(layoutPath, manifest); + assertIndex(layoutPath, manifest, 3, 0); // Assert blobs and their content assertBlobExists(layoutPath, SupportedAlgorithm.SHA256.digest(file1)); @@ -611,7 +663,7 @@ void testShouldCopyRecursivelyArtifactFromRegistryIntoOciLayout() throws IOExcep } @Test - void testShouldCopyImageIntoOciLayoutWithoutIndex() { + void testShouldCopyImageIntoOciLayoutWithoutIndexAndTag() { Registry registry = Registry.Builder.builder() .defaults("myuser", "mypass") @@ -645,7 +697,7 @@ void testShouldCopyImageIntoOciLayoutWithoutIndex() { assertOciLayout(layoutPath); // Check index exists - assertIndex(layoutPath, pushedManifest); + assertIndex(layoutPath, pushedManifest, 1, 0); // Check manifest exists assertTrue(Files.exists(layoutPath @@ -674,10 +726,18 @@ void testShouldCopyImageIntoOciLayoutWithoutIndex() { // Check manifest exists assertBlobExists(layoutPath, pushedManifest.getDescriptor().getDigest()); + + // Ensure the manifest on index contains the ref tag + assertIndex(layoutPath, pushedManifest, 1, 0); + + Index index = Index.fromPath(layoutPath.resolve(Const.OCI_LAYOUT_INDEX)); + + // Check latest tag + assertEquals("latest", index.getManifests().get(0).getAnnotations().get(Const.ANNOTATION_REF)); } @Test - void testShouldCopyImageIntoOciLayoutWithIndex() throws IOException { + void testShouldCopyImageIntoOciLayoutWithIndex() { Path layoutPathIndex = layoutPath.resolve("testShouldCopyImageIntoOciLayoutWithIndex"); @@ -711,8 +771,9 @@ void testShouldCopyImageIntoOciLayoutWithIndex() throws IOException { assertOciLayout(layoutPathIndex); - // Check index exists - assertIndex(layoutPathIndex, pushedManifest); + // Check index and manifest are stored in index + assertIndex(layoutPathIndex, index, 2); + assertIndex(layoutPathIndex, pushedManifest, 2, 1); // Check manifest exists assertBlobExists(layoutPathIndex, pushedManifest.getDescriptor().getDigest()); @@ -743,6 +804,11 @@ void testShouldCopyImageIntoOciLayoutWithIndex() throws IOException { assertLayerExists(layoutPathIndex, layer1); assertBlobExists(layoutPathIndex, index.getDescriptor().getDigest()); assertBlobExists(layoutPathIndex, pushedManifest.getDescriptor().getDigest()); + + // Check latest tag + Index ociIndex = Index.fromPath(layoutPathIndex.resolve(Const.OCI_LAYOUT_INDEX)); + assertEquals(2, ociIndex.getManifests().size()); + assertEquals("latest", ociIndex.getManifests().get(0).getAnnotations().get(Const.ANNOTATION_REF)); } @Test @@ -775,7 +841,7 @@ void testShouldCopyIntoOciLayoutWithBlobConfig() throws IOException { assertBlobContent(layoutPath, layer.getDigest(), "foobartest"); // Check index exists - assertIndex(layoutPath, manifest); + assertIndex(layoutPath, manifest, 1, 0); } private void assertOciLayout(Path layoutPath) { @@ -784,15 +850,27 @@ private void assertOciLayout(Path layoutPath) { assertEquals("1.0.0", layoutFile.getImageLayoutVersion()); } - private void assertIndex(Path ociLayoutPath, Manifest manifest) { + private void assertIndex(Path ociLayoutPath, Manifest manifest, int size, int index) { assertTrue(Files.exists(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX))); - Index index = Index.fromPath(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX)); - LOG.debug("Index is {}", index.toJson()); - assertEquals(2, index.getSchemaVersion()); - assertEquals(1, index.getManifests().size()); - assertEquals(Const.DEFAULT_INDEX_MEDIA_TYPE, index.getMediaType()); + Index indexObject = Index.fromPath(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX)); + LOG.debug("Index is {}", indexObject.toJson()); + assertEquals(2, indexObject.getSchemaVersion()); + assertEquals(size, indexObject.getManifests().size()); + assertEquals(Const.DEFAULT_INDEX_MEDIA_TYPE, indexObject.getMediaType()); + assertEquals( + manifest.getDescriptor().getSize(), + indexObject.getManifests().get(index).getSize()); + } + + private void assertIndex(Path ociLayoutPath, Index index, int size) { + assertTrue(Files.exists(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX))); + Index ociIndex = Index.fromPath(ociLayoutPath.resolve(Const.OCI_LAYOUT_INDEX)); + LOG.debug("Index is {}", ociIndex.toJson()); + assertEquals(2, ociIndex.getSchemaVersion()); + assertEquals(size, ociIndex.getManifests().size()); + assertEquals(Const.DEFAULT_INDEX_MEDIA_TYPE, ociIndex.getMediaType()); assertEquals( - manifest.getDescriptor().getSize(), index.getManifests().get(0).getSize()); + index.getDescriptor().getSize(), ociIndex.getManifests().get(0).getSize()); } private void assertLayerExists(Path ociLayoutPath, Layer layer) {