upstream: b=master,r=6e0764d8bcb819bd8aa41418b58f9ae2500a6f14,t=2021-…

…06-16-0826-10942

components/nexus-repository/src/main/java/org/sonatype/nexus/repository/storage/MetadataNodeEntityAdapter.java

@@ -12,14 +12,19 @@
  */
 package org.sonatype.nexus.repository.storage;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import javax.annotation.Nullable;
 
 import org.sonatype.nexus.common.collect.NestedAttributesMap;
+import org.sonatype.nexus.common.entity.Continuation;
+import org.sonatype.nexus.common.entity.Continuations;
+import org.sonatype.nexus.common.entity.DetachedEntityId;
 import org.sonatype.nexus.common.entity.EntityHelper;
 import org.sonatype.nexus.orient.entity.AttachedEntityId;
 import org.sonatype.nexus.orient.entity.IterableEntityAdapter;
@@ -78,6 +83,17 @@ public abstract class MetadataNodeEntityAdapter<T extends MetadataNode<?>>
    */
   static final String P_LAST_UPDATED = "last_updated";
 
+
+  private static final String RID = "rid";
+
+  /**
+   * According to the Orient documentation #-1:-1 is a begin id
+   * So this is the start point for fetching the data.
+   */
+  private static final String START_RID = "#-1:-1";
+
+  private static final String SKIP_AS_QUERY_IS_EMPTY = "Skipped finding {}s as query is empty, parameters: {}";
+
   protected final BucketEntityAdapter bucketEntityAdapter;
 
   public MetadataNodeEntityAdapter(final String typeName, final BucketEntityAdapter bucketEntityAdapter) {
@@ -201,7 +217,7 @@ private Iterable<T> browseByQuery(final ODatabaseDocumentTx db,
     String query = buildQuery(false, whereClause, buckets, querySuffix);
 
     if (isBlank(query)) {
-      log.debug("Skipped finding {}s as query is empty, parameters: {}", getTypeName(), parameters);
+      log.debug(SKIP_AS_QUERY_IS_EMPTY, getTypeName(), parameters);
       return Collections.emptyList();
     }
 
@@ -216,6 +232,41 @@ private Iterable<T> browseByQuery(final ODatabaseDocumentTx db,
     }
   }
 
+  Iterable<T> browseAllPartiallyByLimit(final ODatabaseDocumentTx db,
+                                        @Nullable final String whereClause,
+                                        @Nullable final Map<String, Object> parameters,
+                                        @Nullable final Iterable<Bucket> buckets)
+  {
+    if (parameters != null && parameters.containsKey(RID)) {
+      throw new IllegalArgumentException("Using '" + RID + "' is unsupported.");
+    }
+
+    Map<String, Object> queryParams = new HashMap<>();
+    if (parameters != null) {
+      queryParams.putAll(parameters);
+    }
+
+    String querySuffix = " and @rid > :rid order by @rid LIMIT " + Continuations.BROWSE_LIMIT;
+    String query = buildQuery(false, whereClause, buckets, querySuffix);
+
+    if (isBlank(query)) {
+      log.debug(SKIP_AS_QUERY_IS_EMPTY, getTypeName(), parameters);
+      return Collections.emptySet();
+    }
+
+    return Continuations.iterableOf((limit, token) -> {
+      log.debug("Finding {}s with query: {}, parameters: {}", getTypeName(), query, parameters);
+      String rid = token == null ? START_RID : recordIdentity(new DetachedEntityId(token)).toString();
+      queryParams.put(RID, rid);
+      Iterable<T> docs = transform(db.command(new OCommandSQL(query)).execute(queryParams));
+
+      Continuation<T> continuation = new ContinuationArrayList<>();
+      docs.forEach(continuation::add);
+
+      return continuation;
+    });
+  }
+
   private Iterable<T> browseByQuery(final ODatabaseDocumentTx db,
                                     @Nullable final String whereClause,
                                     @Nullable final Map<String, Object> parameters,
@@ -228,7 +279,7 @@ private Iterable<T> browseByQuery(final ODatabaseDocumentTx db,
     String query = buildQuery(false, whereClause, buckets, querySuffix);
 
     if (isBlank(query)) {
-      log.debug("Skipped finding {}s as query is empty, parameters: {}", getTypeName(), parameters);
+      log.debug(SKIP_AS_QUERY_IS_EMPTY, getTypeName(), parameters);
       return Collections.emptyList();
     }
 
@@ -336,4 +387,17 @@ protected void addBucketConstraints(@Nullable final String whereClause,
       query.append(')');
     }
   }
+
+  private static class ContinuationArrayList<T extends MetadataNode<?>>
+      extends ArrayList<T>
+      implements Continuation<T>
+  {
+    private static final long serialVersionUID = -8278643802740770499L;
+
+    @Override
+    public String nextContinuationToken() {
+      checkState(!isEmpty(), "No more results");
+      return EntityHelper.id(get(size() - 1)).getValue();
+    }
+  }
 }

components/nexus-repository/src/main/java/org/sonatype/nexus/repository/storage/StorageTx.java

@@ -136,6 +136,18 @@ Iterable<ODocument> browse(final String selectSql,
    */
   Iterable<Asset> browseAssets(Query query, Bucket bucket);
 
+  /**
+   * Gets all assets owned by the specified bucket for a given query with a limit option.
+   * This method will NOT see uncommitted changes performed in this same TX, if any.
+   * The returned {@link Iterable} may throw {@link RuntimeException} if timeout to
+   * receive new elements is breached.
+   *
+   * For setting the limit @see Continuations.BROWSE_LIMIT
+   *
+   * @since 3.next
+   */
+  Iterable<Asset> browseAllPartiallyByLimit(Query query, Bucket bucket);
+
   /**
    * Gets all assets owned by the specified bucket for a given query. This method will NOT see uncommitted changes
    * performed in this same TX, if any. The returned {@link Iterable} may throw {@link RuntimeException} if timeout to

components/nexus-repository/src/main/java/org/sonatype/nexus/repository/storage/StorageTxImpl.java

@@ -315,6 +315,14 @@ public Iterable<Asset> browseAssets(final Query query, final Bucket bucket) {
         query.getQuerySuffix());
   }
 
+  @Override
+  @Guarded(by = ACTIVE)
+  public Iterable<Asset> browseAllPartiallyByLimit(final Query query, final Bucket bucket)
+  {
+    return assetEntityAdapter
+        .browseAllPartiallyByLimit(db, query.getWhere(), query.getParameters(), ImmutableList.of(bucket));
+  }
+
   @Override
   @Guarded(by = ACTIVE)
   public Iterable<Asset> browseAssets(

plugins/nexus-repository-apt/src/main/java/org/sonatype/nexus/repository/apt/datastore/internal/browse/AptBrowseNodeGenerator.java

@@ -47,7 +47,9 @@ public class AptBrowseNodeGenerator
   public List<BrowsePath> computeAssetPaths(final Asset asset) {
     Component component = asset.component().orElse(null);
     if (component != null) {
-      return computeComponentPaths(asset);
+      List<BrowsePath> path = computeComponentPaths(asset);
+      BrowsePathBuilder.appendPath(path, asset.path().substring(1));
+      return path;
     }
 
     List<String> pathParts = new ArrayList<>();
@@ -78,7 +80,6 @@ public List<BrowsePath> computeComponentPaths(final Asset asset) {
     pathParts.add(component.version());
     pathParts.add(component.namespace());
     pathParts.add(component.name());
-    pathParts.add(asset.path().substring(1));
     return BrowsePathBuilder.fromPaths(pathParts, true);
   }
 }

plugins/nexus-repository-apt/src/test/java/org/sonatype/nexus/repository/apt/datastore/internal/AptBrowseNodeGeneratorTest.java

@@ -49,16 +49,15 @@ public void computeComponentPath() {
 
     List<BrowsePath> paths = generator.computeComponentPaths(asset);
 
-    assertThat(paths.size(), is(7));
+    assertThat(paths.size(), is(6));
 
     assertThat(paths, containsInAnyOrder(
         new BrowsePath("packages", "/packages/"),
         new BrowsePath("n", "/packages/n/"),
         new BrowsePath("nano", "/packages/n/nano/"),
         new BrowsePath("1.0.0", "/packages/n/nano/1.0.0/"),
         new BrowsePath("amd64", "/packages/n/nano/1.0.0/amd64/"),
-        new BrowsePath("nano", "/packages/n/nano/1.0.0/amd64/nano/"),
-        new BrowsePath("path/assetName", "/packages/n/nano/1.0.0/amd64/nano/path/assetName/")));
+        new BrowsePath("nano", "/packages/n/nano/1.0.0/amd64/nano/")));
   }
 
 

revision.txt

@@ -1 +1 @@
-b=master,r=1883828adcd5274cb28a13192732d118c4e7969f,t=2021-06-16-0600-59514
+b=master,r=6e0764d8bcb819bd8aa41418b58f9ae2500a6f14,t=2021-06-16-0826-10942