diff --git a/gc/gc-base/src/test/java/org/projectnessie/gc/repository/TestNessieRepositoryConnectorBatching.java b/gc/gc-base/src/test/java/org/projectnessie/gc/repository/TestNessieRepositoryConnectorBatching.java index e15fa54874..80e8beb864 100644 --- a/gc/gc-base/src/test/java/org/projectnessie/gc/repository/TestNessieRepositoryConnectorBatching.java +++ b/gc/gc-base/src/test/java/org/projectnessie/gc/repository/TestNessieRepositoryConnectorBatching.java @@ -26,6 +26,7 @@ import static org.projectnessie.gc.repository.NessieRepositoryConnector.CONTENT_BATCH_SIZE; import java.util.Map; +import java.util.UUID; import java.util.function.IntFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -76,7 +77,9 @@ public void batchContentsFetchingSomeKeysNoValues() throws Exception { IntFunction key = i -> ContentKey.of("key-" + i); IntFunction keyEntry = - i -> EntriesResponse.Entry.entry(key.apply(i), Content.Type.ICEBERG_TABLE); + i -> + EntriesResponse.Entry.entry( + key.apply(i), Content.Type.ICEBERG_TABLE, UUID.randomUUID().toString()); GetEntriesBuilder getEntries = mock(GetEntriesBuilder.class); when(getEntries.reference(ref)).thenReturn(getEntries); @@ -107,7 +110,9 @@ public void batchContentsFetchingSomeKeysAndValues() throws Exception { IntFunction key = i -> ContentKey.of("key-" + i); IntFunction content = i -> IcebergTable.of("meta-" + i, 42, 43, 44, 45, "cid-" + i); IntFunction keyEntry = - i -> EntriesResponse.Entry.entry(key.apply(i), Content.Type.ICEBERG_TABLE); + i -> + EntriesResponse.Entry.entry( + key.apply(i), Content.Type.ICEBERG_TABLE, UUID.randomUUID().toString()); GetEntriesBuilder getEntries = mock(GetEntriesBuilder.class); when(getEntries.reference(ref)).thenReturn(getEntries); @@ -142,7 +147,9 @@ public void batchContentsFetchingBatchKeys() throws Exception { IntFunction key = i -> ContentKey.of("key-" + i); IntFunction content = i -> IcebergTable.of("meta-" + i, 42, 43, 44, 45, "cid-" + i); IntFunction keyEntry = - i -> EntriesResponse.Entry.entry(key.apply(i), Content.Type.ICEBERG_TABLE); + i -> + EntriesResponse.Entry.entry( + key.apply(i), Content.Type.ICEBERG_TABLE, UUID.randomUUID().toString()); GetEntriesBuilder getEntries = mock(GetEntriesBuilder.class); when(getEntries.reference(ref)).thenReturn(getEntries); @@ -180,7 +187,9 @@ public void batchContentsFetchingBatchKeysPlus1() throws Exception { IntFunction key = i -> ContentKey.of("key-" + i); IntFunction content = i -> IcebergTable.of("meta-" + i, 42, 43, 44, 45, "cid-" + i); IntFunction keyEntry = - i -> EntriesResponse.Entry.entry(key.apply(i), Content.Type.ICEBERG_TABLE); + i -> + EntriesResponse.Entry.entry( + key.apply(i), Content.Type.ICEBERG_TABLE, UUID.randomUUID().toString()); GetEntriesBuilder getEntries = mock(GetEntriesBuilder.class); when(getEntries.reference(ref)).thenReturn(getEntries); @@ -231,7 +240,9 @@ public void batchContentsFetchingTenBatches() throws Exception { IntFunction key = i -> ContentKey.of("key-" + i); IntFunction content = i -> IcebergTable.of("meta-" + i, 42, 43, 44, 45, "cid-" + i); IntFunction keyEntry = - i -> EntriesResponse.Entry.entry(key.apply(i), Content.Type.ICEBERG_TABLE); + i -> + EntriesResponse.Entry.entry( + key.apply(i), Content.Type.ICEBERG_TABLE, UUID.randomUUID().toString()); GetEntriesBuilder getEntries = mock(GetEntriesBuilder.class); when(getEntries.reference(ref)).thenReturn(getEntries); diff --git a/gc/gc-base/src/test/java/org/projectnessie/gc/roundtrip/TestMarkAndSweep.java b/gc/gc-base/src/test/java/org/projectnessie/gc/roundtrip/TestMarkAndSweep.java index f10ec5d18a..894053503a 100644 --- a/gc/gc-base/src/test/java/org/projectnessie/gc/roundtrip/TestMarkAndSweep.java +++ b/gc/gc-base/src/test/java/org/projectnessie/gc/roundtrip/TestMarkAndSweep.java @@ -60,7 +60,6 @@ import org.projectnessie.model.Content; import org.projectnessie.model.ContentKey; import org.projectnessie.model.Detached; -import org.projectnessie.model.EntriesResponse; import org.projectnessie.model.IcebergTable; import org.projectnessie.model.LogResponse.LogEntry; import org.projectnessie.model.Operation.Put; @@ -232,14 +231,11 @@ public Stream> allContents( // 1L is the very first, commit - the first non-live commit // 2L is the oldest live-commit - need to fetch the visible keys from that one soft.assertThat(l).isEqualTo(2L); - Stream keysStream = + List keys = IntStream.range(0, markAndSweep.numKeysAtCutOff) .mapToObj( i -> markAndSweep.numToContentKey(markAndSweep.numCommits + i)) - .map(ck -> EntriesResponse.Entry.entry(ck, ICEBERG_TABLE)); - - List keys = - keysStream.map(EntriesResponse.Entry::getName).collect(Collectors.toList()); + .collect(Collectors.toList()); soft.assertThat(keys).hasSize(markAndSweep.numKeysAtCutOff); return keys.stream() .map( diff --git a/model/src/main/java/org/projectnessie/model/EntriesResponse.java b/model/src/main/java/org/projectnessie/model/EntriesResponse.java index 53b820418f..2451c69afe 100644 --- a/model/src/main/java/org/projectnessie/model/EntriesResponse.java +++ b/model/src/main/java/org/projectnessie/model/EntriesResponse.java @@ -15,11 +15,14 @@ */ package org.projectnessie.model; +import com.fasterxml.jackson.annotation.JsonView; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import java.util.List; +import javax.annotation.Nullable; import javax.validation.constraints.NotNull; import org.immutables.value.Value; +import org.projectnessie.model.ser.Views; @Value.Immutable @JsonSerialize(as = ImmutableEntriesResponse.class) @@ -50,8 +53,17 @@ static ImmutableEntry.Builder builder() { @Value.Parameter(order = 1) ContentKey getName(); + @JsonView(Views.V2.class) + @Value.Parameter(order = 3) + @Nullable // for V1 backwards compatibility + String getContentId(); + static Entry entry(ContentKey name, Content.Type type) { - return ImmutableEntry.of(name, type); + return entry(name, type, null); + } + + static Entry entry(ContentKey name, Content.Type type, String contentId) { + return ImmutableEntry.of(name, type, contentId); } } } diff --git a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestContents.java b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestContents.java index 8459918344..87b9a794bb 100644 --- a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestContents.java +++ b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestContents.java @@ -177,17 +177,18 @@ public void verifyAllContentAndOperationTypes() throws BaseNessieClientServerExc List entries = getApi().getEntries().refName(branch.getName()).stream().collect(Collectors.toList()); - List expect = + Map expect = contentAndOps.stream() .filter(c -> c.operation instanceof Put) - .map(c -> Entry.entry(c.operation.getKey(), c.type)) - .collect(Collectors.toList()); - List notExpect = + .collect(Collectors.toMap(c -> c.operation.getKey(), c -> c.type)); + Map notExpect = contentAndOps.stream() .filter(c -> c.operation instanceof Delete) - .map(c -> Entry.entry(c.operation.getKey(), c.type)) - .collect(Collectors.toList()); - soft.assertThat(entries).containsAll(expect).doesNotContainAnyElementsOf(notExpect); + .collect(Collectors.toMap(c -> c.operation.getKey(), c -> c.type)); + soft.assertThat(entries) + .map(e -> Maps.immutableEntry(e.getName(), e.getType())) + .containsAll(expect.entrySet()) + .doesNotContainAnyElementsOf(notExpect.entrySet()); // Diff against of committed HEAD and previous commit must yield the content in the // Put operations @@ -277,7 +278,9 @@ public void verifyContentAndOperationTypesIndividually( List entries = getApi().getEntries().refName(branch.getName()).stream().collect(Collectors.toList()); soft.assertThat(entries) - .containsExactly(Entry.entry(fixedContentKey, contentAndOperationType.type)); + .hasSize(1) + .extracting(Entry::getName, Entry::getType) + .containsExactly(tuple(fixedContentKey, contentAndOperationType.type)); // Diff against of committed HEAD and previous commit must yield the content in the // Put operation @@ -334,7 +337,8 @@ public void verifyContentAndOperationTypesIndividually( List entries = getApi().getEntries().refName(branch.getName()).stream().collect(Collectors.toList()); soft.assertThat(entries) - .containsExactly(Entry.entry(fixedContentKey, contentAndOperationType.type)); + .extracting(Entry::getName, Entry::getType) + .containsExactly(tuple(fixedContentKey, contentAndOperationType.type)); // Diff against of committed HEAD and previous commit must yield the content in the // Put operations diff --git a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestEntries.java b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestEntries.java index eeefc2635b..95037ab5a1 100644 --- a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestEntries.java +++ b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestEntries.java @@ -15,11 +15,13 @@ */ package org.projectnessie.jaxrs.tests; -import static java.util.Arrays.asList; +import static com.google.common.collect.Maps.immutableEntry; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -68,10 +70,11 @@ public void filterEntriesByType(ReferenceMode refMode) throws BaseNessieClientSe .commit(); List entries = getApi().getEntries().reference(refMode.transform(branch)).get().getEntries(); - List expected = - asList( - Entry.entry(a, Content.Type.ICEBERG_TABLE), Entry.entry(b, Content.Type.ICEBERG_VIEW)); - soft.assertThat(entries).containsExactlyInAnyOrderElementsOf(expected); + Map expect = + ImmutableMap.of(a, Content.Type.ICEBERG_TABLE, b, Content.Type.ICEBERG_VIEW); + soft.assertThat(entries) + .map(e -> immutableEntry(e.getName(), e.getType())) + .containsExactlyInAnyOrderElementsOf(expect.entrySet()); entries = getApi() @@ -80,7 +83,9 @@ public void filterEntriesByType(ReferenceMode refMode) throws BaseNessieClientSe .filter("entry.contentType=='ICEBERG_TABLE'") .get() .getEntries(); - soft.assertThat(entries).containsExactly(expected.get(0)); + soft.assertThat(entries) + .map(e -> immutableEntry(e.getName(), e.getType())) + .containsExactly(immutableEntry(a, Content.Type.ICEBERG_TABLE)); entries = getApi() @@ -89,7 +94,9 @@ public void filterEntriesByType(ReferenceMode refMode) throws BaseNessieClientSe .filter("entry.contentType=='ICEBERG_VIEW'") .get() .getEntries(); - soft.assertThat(entries).containsExactly(expected.get(1)); + soft.assertThat(entries) + .map(e -> immutableEntry(e.getName(), e.getType())) + .containsExactly(immutableEntry(b, Content.Type.ICEBERG_VIEW)); entries = getApi() @@ -98,7 +105,9 @@ public void filterEntriesByType(ReferenceMode refMode) throws BaseNessieClientSe .filter("entry.contentType in ['ICEBERG_TABLE', 'ICEBERG_VIEW']") .get() .getEntries(); - soft.assertThat(entries).containsExactlyInAnyOrderElementsOf(expected); + soft.assertThat(entries) + .map(e -> immutableEntry(e.getName(), e.getType())) + .containsExactlyInAnyOrderElementsOf(expect.entrySet()); } @ParameterizedTest @@ -321,7 +330,8 @@ public void filterEntriesByNamespaceAndPrefixDepth(ReferenceMode refMode) .getEntries(); soft.assertThat(entries) .hasSize(1) - .containsExactly(Entry.entry(ContentKey.of("a"), Content.Type.NAMESPACE)); + .map(e -> immutableEntry(e.getName(), e.getType())) + .containsExactly(immutableEntry(ContentKey.of("a"), Content.Type.NAMESPACE)); entries = getApi() @@ -333,10 +343,11 @@ public void filterEntriesByNamespaceAndPrefixDepth(ReferenceMode refMode) .getEntries(); soft.assertThat(entries) .hasSize(3) + .map(e -> immutableEntry(e.getName(), e.getType())) .containsExactlyInAnyOrder( - Entry.entry(ContentKey.of("a", "boo"), Content.Type.NAMESPACE), - Entry.entry(ContentKey.of("a", "b"), Content.Type.NAMESPACE), - Entry.entry(ContentKey.of("a", "thirdTable"), Content.Type.ICEBERG_TABLE)); + immutableEntry(ContentKey.of("a", "boo"), Content.Type.NAMESPACE), + immutableEntry(ContentKey.of("a", "b"), Content.Type.NAMESPACE), + immutableEntry(ContentKey.of("a", "thirdTable"), Content.Type.ICEBERG_TABLE)); entries = getApi() @@ -348,9 +359,10 @@ public void filterEntriesByNamespaceAndPrefixDepth(ReferenceMode refMode) .getEntries(); soft.assertThat(entries) .hasSize(2) + .map(e -> immutableEntry(e.getName(), e.getType())) .containsExactlyInAnyOrder( - Entry.entry(ContentKey.of("a", "b", "fourthTable"), Content.Type.ICEBERG_TABLE), - Entry.entry(ContentKey.of("a", "b", "c"), Content.Type.NAMESPACE)); + immutableEntry(ContentKey.of("a", "b", "fourthTable"), Content.Type.ICEBERG_TABLE), + immutableEntry(ContentKey.of("a", "b", "c"), Content.Type.NAMESPACE)); entries = getApi() @@ -362,9 +374,10 @@ public void filterEntriesByNamespaceAndPrefixDepth(ReferenceMode refMode) .getEntries(); soft.assertThat(entries) .hasSize(2) + .map(e -> immutableEntry(e.getName(), e.getType())) .containsExactlyInAnyOrder( - Entry.entry(ContentKey.of("a", "b", "c", "secondTable"), Content.Type.ICEBERG_TABLE), - Entry.entry(ContentKey.of("a", "b", "c", "firstTable"), Content.Type.ICEBERG_TABLE)); + immutableEntry(ContentKey.of("a", "b", "c", "secondTable"), Content.Type.ICEBERG_TABLE), + immutableEntry(ContentKey.of("a", "b", "c", "firstTable"), Content.Type.ICEBERG_TABLE)); entries = getApi() @@ -386,10 +399,11 @@ public void filterEntriesByNamespaceAndPrefixDepth(ReferenceMode refMode) .getEntries(); soft.assertThat(entries) .hasSize(3) + .map(e -> immutableEntry(e.getName(), e.getType())) .containsExactlyInAnyOrder( - Entry.entry(ContentKey.of("a", "boo", "fifthTable"), Content.Type.ICEBERG_TABLE), - Entry.entry(ContentKey.of("a", "b", "fourthTable"), Content.Type.ICEBERG_TABLE), - Entry.entry(ContentKey.of("a", "b", "c"), Content.Type.NAMESPACE)); + immutableEntry(ContentKey.of("a", "boo", "fifthTable"), Content.Type.ICEBERG_TABLE), + immutableEntry(ContentKey.of("a", "b", "fourthTable"), Content.Type.ICEBERG_TABLE), + immutableEntry(ContentKey.of("a", "b", "c"), Content.Type.NAMESPACE)); if (ReferenceMode.DETACHED != refMode) { // check that implicit namespaces are properly detected @@ -418,10 +432,10 @@ public void fetchEntriesByNamelessReference() throws BaseNessieClientServerExcep .commit(); List entries = getApi().getEntries().hashOnRef(branch.getHash()).get().getEntries(); soft.assertThat(entries) - .containsExactlyInAnyOrderElementsOf( - Arrays.asList( - Entry.entry(a, Content.Type.ICEBERG_TABLE), - Entry.entry(b, Content.Type.ICEBERG_VIEW))); + .map(e -> immutableEntry(e.getName(), e.getType())) + .containsExactlyInAnyOrder( + immutableEntry(a, Content.Type.ICEBERG_TABLE), + immutableEntry(b, Content.Type.ICEBERG_VIEW)); } private void checkNamespaces( diff --git a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestNamespace.java b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestNamespace.java index 7ca3d5f097..a6360397cb 100644 --- a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestNamespace.java +++ b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractRestNamespace.java @@ -214,10 +214,10 @@ public void testNamespaceDeletion() throws BaseNessieClientServerException { contentAndOps.stream().map(c -> c.operation).forEach(commit::operation); commit.commit(); - List entries = + List entries = contentAndOps.stream() .filter(c -> c.operation instanceof Put) - .map(c -> Entry.entry(c.operation.getKey(), c.type)) + .map(c -> c.operation.getKey()) .collect(Collectors.toList()); CommitMultipleOperationsBuilder commit2 = @@ -226,16 +226,13 @@ public void testNamespaceDeletion() throws BaseNessieClientServerException { .branch(branch) .commitMeta(CommitMeta.fromMessage("create namespaces")); entries.stream() - .map(e -> e.getName().getNamespace()) + .map(ContentKey::getNamespace) .distinct() - .forEach( - ns -> { - commit2.operation(Put.of(ContentKey.of(ns.getElements()), ns)); - }); + .forEach(ns -> commit2.operation(Put.of(ContentKey.of(ns.getElements()), ns))); commit2.commit(); - for (Entry e : entries) { - Namespace namespace = e.getName().getNamespace(); + for (ContentKey contentKey : entries) { + Namespace namespace = contentKey.getNamespace(); soft.assertThat( getApi() .getNamespace() diff --git a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV1Test.java b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV1Test.java index f7d69ae767..185707f482 100644 --- a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV1Test.java +++ b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV1Test.java @@ -43,6 +43,7 @@ import org.projectnessie.model.ContentKey; import org.projectnessie.model.DiffResponse; import org.projectnessie.model.DiffResponse.DiffEntry; +import org.projectnessie.model.EntriesResponse; import org.projectnessie.model.IcebergTable; import org.projectnessie.model.ImmutableBranch; import org.projectnessie.model.ImmutableOperations; @@ -121,6 +122,17 @@ public void testBasic() { .as(Branch.class); Assertions.assertNotEquals(newReference.getHash(), commitResponse.getHash()); + EntriesResponse entries = + rest() + .get("trees/tree/{branch}/entries", newReference.getName()) + .then() + .statusCode(200) + .extract() + .as(EntriesResponse.class); + assertThat(entries.getEntries()) + .hasSize(1) + .allSatisfy(e -> assertThat(e.getContentId()).isNull()); + // fetch the content IcebergTable table = rest() diff --git a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV2Test.java b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV2Test.java index 262bb5b1e6..66ec42301f 100644 --- a/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV2Test.java +++ b/servers/jax-rs-tests/src/main/java/org/projectnessie/jaxrs/tests/AbstractResteasyV2Test.java @@ -44,6 +44,7 @@ import org.projectnessie.model.ContentKey; import org.projectnessie.model.ContentResponse; import org.projectnessie.model.DiffResponse; +import org.projectnessie.model.EntriesResponse; import org.projectnessie.model.GetMultipleContentsResponse; import org.projectnessie.model.IcebergTable; import org.projectnessie.model.ImmutableOperations; @@ -142,7 +143,18 @@ void testGetSeveralContents(String branchName) { branch = commit(branch, key1, table1); branch = commit(branch, key2, table2); - Stream> entries = + EntriesResponse entries = + rest() + .get("trees/{ref}/entries", branch.toPathString()) + .then() + .statusCode(200) + .extract() + .as(EntriesResponse.class); + assertThat(entries.getEntries()) + .hasSize(2) + .allSatisfy(e -> assertThat(e.getContentId()).isNotNull()); + + Stream> contents = rest() .queryParam("key", key1.toPathString(), key2.toPathString()) .get("trees/{ref}/contents", branch.toPathString()) @@ -158,7 +170,7 @@ void testGetSeveralContents(String branchName) { content.getKey(), ((IcebergTable) content.getContent()).getMetadataLocation())); - assertThat(entries).containsExactlyInAnyOrder(entry(key1, "loc1"), entry(key2, "loc2")); + assertThat(contents).containsExactlyInAnyOrder(entry(key1, "loc1"), entry(key2, "loc2")); } /** Dedicated test for human-readable references in URL paths. */ diff --git a/servers/services/src/main/java/org/projectnessie/services/impl/TreeApiImpl.java b/servers/services/src/main/java/org/projectnessie/services/impl/TreeApiImpl.java index 4830b61f22..a884613e7f 100644 --- a/servers/services/src/main/java/org/projectnessie/services/impl/TreeApiImpl.java +++ b/servers/services/src/main/java/org/projectnessie/services/impl/TreeApiImpl.java @@ -587,7 +587,10 @@ public EntriesResponse getEntries( try (Stream entryStream = getStore().getKeys(refWithHash.getHash())) { Stream entriesStream = filterEntries(refWithHash, entryStream, filter) - .map(key -> EntriesResponse.Entry.entry(fromKey(key.getKey()), key.getType())); + .map( + key -> + EntriesResponse.Entry.entry( + fromKey(key.getKey()), key.getType(), key.getContentId())); if (namespaceDepth != null && namespaceDepth > 0) { entriesStream = entriesStream @@ -610,7 +613,7 @@ private static EntriesResponse.Entry truncate(EntriesResponse.Entry entry, Integ Content.Type type = entry.getName().getElements().size() > depth ? Content.Type.NAMESPACE : entry.getType(); ContentKey key = ContentKey.of(entry.getName().getElements().subList(0, depth)); - return EntriesResponse.Entry.entry(key, type); + return EntriesResponse.Entry.entry(key, type, null); } /**