optimize get-entries prefix matching

projectnessie · May 13, 2023 · 99db9e0 · 99db9e0
1 parent ca7331a
commit 99db9e0
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 12 deletions.
diff --git a/...age/common/src/main/java/org/projectnessie/versioned/storage/common/indexes/StoreKey.java b/...age/common/src/main/java/org/projectnessie/versioned/storage/common/indexes/StoreKey.java
@@ -24,6 +24,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
@@ -243,6 +244,21 @@ public boolean startsWith(StoreKey prefix) {
     return key.startsWith(prefix.key);
   }
 
+  public boolean startsWithElementsOrParts(StoreKey prefix) {
+    int m = CharBuffer.wrap(key).mismatch(CharBuffer.wrap(prefix.key));
+    if (m == -1) {
+      // equal
+      return true;
+    }
+    if (m < prefix.key.length()) {
+      // prefix does not match at all
+      return false;
+    }
+    // check for element or part border
+    char c = key.charAt(m);
+    return c == (char) 0 || c == (char) 1;
+  }
+
   /** Tests whether this store key ends with the given element. */
   public boolean endsWithElement(String element) {
     int elLen = element.length();

diff --git a/...common/src/test/java/org/projectnessie/versioned/storage/common/indexes/TestStoreKey.java b/...common/src/test/java/org/projectnessie/versioned/storage/common/indexes/TestStoreKey.java
@@ -165,6 +165,41 @@ void keyEndsWithElement() {
     soft.assertThat(key("a", "b").endsWithElement("b")).isTrue();
   }
 
+  @Test
+  void keyStartsWithElementsOrParts() {
+    soft.assertThat(key("a").startsWithElementsOrParts(key("a"))).isTrue();
+    soft.assertThat(key("b").startsWithElementsOrParts(key("a"))).isFalse();
+    soft.assertThat(key("b", "a").startsWithElementsOrParts(key("a"))).isFalse();
+    soft.assertThat(key("a", "b").startsWithElementsOrParts(key("a"))).isTrue();
+    soft.assertThat(key("a", "b").startsWithElementsOrParts(key("a", "b"))).isTrue();
+    soft.assertThat(key("a", "b", "c").startsWithElementsOrParts(key("a", "b"))).isTrue();
+    soft.assertThat(key("a", "b\u0001b", "c").startsWithElementsOrParts(key("a", "b"))).isTrue();
+    soft.assertThat(key("a", "b\u0001b", "c").startsWithElementsOrParts(key("a", "b\u0001b")))
+        .isTrue();
+    soft.assertThat(
+            key("a", "b\u0001b\u0001b", "c").startsWithElementsOrParts(key("a", "b\u0001b")))
+        .isTrue();
+    soft.assertThat(key("a", "bb\u0001b", "c").startsWithElementsOrParts(key("a", "b"))).isFalse();
+    soft.assertThat(key("a", "bb", "c").startsWithElementsOrParts(key("a", "b"))).isFalse();
+    soft.assertThat(key("a", "a\u0001b", "c").startsWithElementsOrParts(key("a", "b"))).isFalse();
+    soft.assertThat(key("a", "bπ\u0001b", "c").startsWithElementsOrParts(key("a", "b")))
+        .isFalse(); // UNICODE CHAR
+    soft.assertThat(key("a", "bπ", "c").startsWithElementsOrParts(key("a", "b")))
+        .isFalse(); // UNICODE CHAR
+    soft.assertThat(key("a", "bπ\u0001b", "c").startsWithElementsOrParts(key("a", "bπ")))
+        .isTrue(); // UNICODE CHAR
+    soft.assertThat(key("a", "bπ", "c").startsWithElementsOrParts(key("a", "bπ")))
+        .isTrue(); // UNICODE CHAR
+    soft.assertThat(key("a", "πb\u0001b", "c").startsWithElementsOrParts(key("a", "π")))
+        .isFalse(); // UNICODE CHAR
+    soft.assertThat(key("a", "πb", "c").startsWithElementsOrParts(key("a", "π")))
+        .isFalse(); // UNICODE CHAR
+    soft.assertThat(key("a", "πb\u0001b", "c").startsWithElementsOrParts(key("a", "πb")))
+        .isTrue(); // UNICODE CHAR
+    soft.assertThat(key("a", "πb", "c").startsWithElementsOrParts(key("a", "πb")))
+        .isTrue(); // UNICODE CHAR
+  }
+
   @Test
   void invalidKeys() {
     soft.assertThatIllegalArgumentException().isThrownBy(() -> key(""));

diff --git a/...age/store/src/main/java/org/projectnessie/versioned/storage/versionstore/TypeMapping.java b/...age/store/src/main/java/org/projectnessie/versioned/storage/versionstore/TypeMapping.java
@@ -136,6 +136,13 @@ public static StoreKey keyToStoreKey(@Nonnull @jakarta.annotation.Nonnull Conten
     return keyToStoreKeyVariant(key, CONTENT_DISCRIMINATOR);
   }
 
+  @Nonnull
+  @jakarta.annotation.Nonnull
+  public static StoreKey keyToStoreKeyNoVariant(
+      @Nonnull @jakarta.annotation.Nonnull ContentKey key) {
+    return StoreKey.keyFromString(keyToStoreKeyPrepare(key).toString());
+  }
+
   @Nonnull
   @jakarta.annotation.Nonnull
   public static StoreKey keyToStoreKeyVariant(

diff --git a/...tore/src/main/java/org/projectnessie/versioned/storage/versionstore/VersionStoreImpl.java b/...tore/src/main/java/org/projectnessie/versioned/storage/versionstore/VersionStoreImpl.java
@@ -48,6 +48,7 @@
 import static org.projectnessie.versioned.storage.versionstore.TypeMapping.hashToObjId;
 import static org.projectnessie.versioned.storage.versionstore.TypeMapping.keyToStoreKey;
 import static org.projectnessie.versioned.storage.versionstore.TypeMapping.keyToStoreKeyMin;
+import static org.projectnessie.versioned.storage.versionstore.TypeMapping.keyToStoreKeyNoVariant;
 import static org.projectnessie.versioned.storage.versionstore.TypeMapping.objIdToHash;
 import static org.projectnessie.versioned.storage.versionstore.TypeMapping.storeKeyToKey;
 import static org.projectnessie.versioned.storage.versionstore.TypeMapping.toCommitMeta;
@@ -520,13 +521,11 @@ public PaginationIterator<KeyEntry> getKeys(
                 return key != null && contentKeyPredicate.test(key);
               });
     }
+
     Predicate<StoreIndexElement<CommitOp>> stopPredicate;
     if (prefixKey != null) {
-      stopPredicate =
-          indexElement -> {
-            ContentKey key = storeKeyToKey(indexElement.key());
-            return key != null && !key.startsWith(prefixKey);
-          };
+      StoreKey prefix = keyToStoreKeyNoVariant(prefixKey);
+      stopPredicate = indexElement -> !indexElement.key().startsWithElementsOrParts(prefix);
     } else {
       stopPredicate = x -> false;
     }
@@ -807,13 +806,13 @@ public PaginationIterator<Diff> getDiffs(
             }
             : x -> true;
 
-    Predicate<DiffEntry> stopPredicate =
-        prefixKey != null
-            ? d -> {
-              ContentKey key = storeKeyToKey(d.key());
-              return key != null && !key.startsWith(prefixKey);
-            }
-            : x -> false;
+    Predicate<DiffEntry> stopPredicate;
+    if (prefixKey != null) {
+      StoreKey prefix = keyToStoreKeyNoVariant(prefixKey);
+      stopPredicate = d -> !d.key().startsWithElementsOrParts(prefix);
+    } else {
+      stopPredicate = x -> false;
+    }
 
     return new FilteringPaginationIterator<DiffEntry, Diff>(
         diffIter,