Parallel indexing of repositories on startup

And avoid redundant copying of blobs
git-as-svn · Sep 15, 2017 · e42a66e · e42a66e
1 parent 14c28ea
commit e42a66e
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 12 deletions.
diff --git a/src/main/java/svnserver/repository/git/GitRepository.java b/src/main/java/svnserver/repository/git/GitRepository.java
@@ -204,19 +204,19 @@ public boolean loadRevisions() throws IOException, SVNException {
       final long beginTime = System.currentTimeMillis();
       int processed = 0;
       long reportTime = beginTime;
-      log.info("Loading cached revision changes: {} revision", newRevs.size());
+      log.info("[{}]: loading cached revision changes: {} revisions", context.getName(), newRevs.size());
       for (int i = newRevs.size() - 1; i >= 0; i--) {
         loadRevisionInfo(newRevs.get(i));
         processed++;
         long currentTime = System.currentTimeMillis();
         if (currentTime - reportTime > REPORT_DELAY) {
-          log.info("  processed cached revision: {} ({} rev/sec)", newRevs.size() - i, 1000.0f * processed / (currentTime - reportTime));
+          log.info("[{}]: processed cached revision: {}/{} ({} rev/sec)", context.getName(), newRevs.size() - i, newRevs.size(), 1000.0f * processed / (currentTime - reportTime));
           reportTime = currentTime;
           processed = 0;
         }
       }
       final long endTime = System.currentTimeMillis();
-      log.info("Cached revision loaded: {} ms", endTime - beginTime);
+      log.info("[{}]: {} cached revision loaded: {} ms", context.getName(), newRevs.size(), endTime - beginTime);
       return true;
     } finally {
       lock.writeLock().unlock();
@@ -282,7 +282,7 @@ public boolean cacheRevisions() throws IOException, SVNException {
         final long beginTime = System.currentTimeMillis();
         int processed = 0;
         long reportTime = beginTime;
-        log.info("Loading revision changes: {} revision", newRevs.size());
+        log.info("[{}]: Loading revision changes: {} revision", context.getName(), newRevs.size());
         int revisionId = revisions.size();
         ObjectId cacheId = revisions.get(revisions.size() - 1).getCacheCommit();
         for (int i = newRevs.size() - 1; i >= 0; i--) {
@@ -485,7 +485,7 @@ private GitProperty[] parseGitProperty(@NotNull String fileName, @NotNull GitObj
   private GitProperty[] cachedParseGitProperty(GitObject<ObjectId> objectId, GitPropertyFactory factory) throws IOException, SVNException {
     GitProperty[] property = filePropertyCache.get(objectId.getObject());
     if (property == null) {
-      property = factory.create(loadContent(objectId));
+      property = factory.create(loadContent(repository.newObjectReader(), objectId.getObject()));
       if (property.length == 0) {
         property = GitProperty.emptyArray;
       }
@@ -619,8 +619,8 @@ public int getLastChange(@NotNull String nodePath, int beforeRevision) {
   }
 
   @NotNull
-  public static String loadContent(@NotNull GitObject<? extends ObjectId> objectId) throws IOException {
-    final byte[] bytes = objectId.getRepo().newObjectReader().open(objectId.getObject()).getBytes();
+  public static String loadContent(@NotNull ObjectReader reader, @NotNull ObjectId objectId) throws IOException {
+    final byte[] bytes = reader.open(objectId).getCachedBytes();
     return new String(bytes, StandardCharsets.UTF_8);
   }
 

diff --git a/src/main/java/svnserver/repository/git/LayoutHelper.java b/src/main/java/svnserver/repository/git/LayoutHelper.java
@@ -260,7 +260,7 @@ public static String loadRepositoryId(@NotNull ObjectReader objectReader, Object
     RevWalk revWalk = new RevWalk(objectReader);
     TreeWalk treeWalk = TreeWalk.forPath(objectReader, ENTRY_UUID, revWalk.parseCommit(commit).getTree());
     if (treeWalk != null) {
-      return new String(objectReader.open(treeWalk.getObjectId(0)).getBytes(), StandardCharsets.UTF_8);
+      return GitRepository.loadContent(objectReader, treeWalk.getObjectId(0));
     }
     throw new FileNotFoundException(ENTRY_UUID);
   }

diff --git a/src/main/java/svnserver/repository/mapping/RepositoryListMapping.java b/src/main/java/svnserver/repository/mapping/RepositoryListMapping.java
@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Simple repository mapping by predefined list.
@@ -53,10 +54,13 @@ public RepositoryInfo getRepository(@NotNull SVNURL url) throws SVNException {
 
   @Override
   public void initRevisions() throws IOException, SVNException {
-    for (Map.Entry<String, VcsRepository> entry : mapping.entrySet()) {
-      log.info("Repository initialize: {}", entry.getKey());
-      entry.getValue().updateRevisions();
-    }
+    new ConcurrentHashMap<>(mapping).forEachEntry(0, entry -> {
+      try {
+        entry.getValue().updateRevisions();
+      } catch (IOException | SVNException e) {
+        throw new RuntimeException(e);
+      }
+    });
   }
 
   @Nullable