Merge branch 'stable-3.5' into stable-3.6

* stable-3.5:
  Set version to 3.5.5-SNAPSHOT
  Set version to 3.5.4
  CopyApprovalsIT: remove unnecessary comments
  Log progress of the copy-approvals program
  copy-approvals: do not lock loose refs when executing BatchRefUpdate
  copy-approvals: multi-threaded, slice based
  Update git submodules
  Set version to 3.4.9-SNAPSHOT
  Set version to 3.4.8
  Fix bulk loading of entries with PassthroughLoadingCache
  Update git submodules
  Set version to 3.4.8-SNAPSHOT
  Set version to 3.4.7
  Introduce a PassthroughLoadingCache for disabled caches
  Export commons:lang3 dependency to plugin API
  copy-approvals: continue when there are corrupt meta-refs in a project
  copy-approvals: don't stop when it fails on one project
  Fix CopyApprovalsIT.multipleProjects: make sure to use the secondProject

Release-Notes: skip
Change-Id: Ie771563ec237a96f591c2e20772d07a9d06c502d

Documentation/config-gerrit.txt

@@ -870,8 +870,14 @@ Default is 1024 for most caches, except:
 * `"plugin_resources"`: default is 2m (2 MiB of memory)
 
 +
-If set to 0 the cache is disabled. Entries are removed immediately
-after being stored by the cache. This is primarily useful for testing.
+If set to 0 the cache is disabled; entries are loaded but not stored
+in-memory.
++
+**NOTE**: When the cache is disabled, there is no locking when accessing
+the same key/value, and therefore multiple threads may
+load the same value concurrently with a higher memory footprint.
+To keep a minimum caching and avoid concurrent loading of the same
+key/value, set `memoryLimit` to `1` and `maxAge` to `1`.
 
 [[cache.name.expireFromMemoryAfterAccess]]cache.<name>.expireFromMemoryAfterAccess::
 +

java/com/google/gerrit/server/cache/mem/DefaultMemoryCacheFactory.java

@@ -55,14 +55,15 @@ public <K, V> Cache<K, V> build(CacheDef<K, V> def) {
 
   @Override
   public <K, V> LoadingCache<K, V> build(CacheDef<K, V> def, CacheLoader<K, V> loader) {
-    return CaffeinatedGuava.build(create(def), loader);
+    return cacheMaximumWeight(def) == 0
+        ? new PassthroughLoadingCache<>(loader)
+        : CaffeinatedGuava.build(create(def), loader);
   }
 
   private <K, V> Caffeine<K, V> create(CacheDef<K, V> def) {
     Caffeine<K, V> builder = newCacheBuilder();
     builder.recordStats();
-    builder.maximumWeight(
-        cfg.getLong("cache", def.configKey(), "memoryLimit", def.maximumWeight()));
+    builder.maximumWeight(cacheMaximumWeight(def));
     builder = builder.removalListener(newRemovalListener(def.name()));
     builder.weigher(newWeigher(def.weigher()));
 
@@ -109,6 +110,10 @@ private <K, V> Caffeine<K, V> create(CacheDef<K, V> def) {
     return builder;
   }
 
+  private <K, V> long cacheMaximumWeight(CacheDef<K, V> def) {
+    return cfg.getLong("cache", def.configKey(), "memoryLimit", def.maximumWeight());
+  }
+
   private static long toSeconds(@Nullable Duration duration) {
     return duration != null ? duration.getSeconds() : 0;
   }

java/com/google/gerrit/server/cache/mem/PassthroughLoadingCache.java

@@ -0,0 +1,141 @@
+// Copyright (C) 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.server.cache.mem;
+
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.CacheLoader.UnsupportedLoadingOperationException;
+import com.google.common.cache.CacheStats;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableMap;
+import com.google.gerrit.common.Nullable;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+
+/** Implementation of a NOOP cache that just passes all gets to the loader */
+public class PassthroughLoadingCache<K, V> implements LoadingCache<K, V> {
+
+  private final CacheLoader<? super K, V> cacheLoader;
+
+  public PassthroughLoadingCache(CacheLoader<? super K, V> cacheLoader) {
+    this.cacheLoader = cacheLoader;
+  }
+
+  @Override
+  public @Nullable V getIfPresent(Object key) {
+    return null;
+  }
+
+  @Override
+  public V get(K key, Callable<? extends V> loader) throws ExecutionException {
+    try {
+      return loader.call();
+    } catch (Exception e) {
+      throw new ExecutionException(e);
+    }
+  }
+
+  @Override
+  public ImmutableMap<K, V> getAllPresent(Iterable<?> keys) {
+    return ImmutableMap.of();
+  }
+
+  @Override
+  public void put(K key, V value) {}
+
+  @Override
+  public void putAll(Map<? extends K, ? extends V> m) {}
+
+  @Override
+  public void invalidate(Object key) {}
+
+  @Override
+  public void invalidateAll(Iterable<?> keys) {}
+
+  @Override
+  public void invalidateAll() {}
+
+  @Override
+  public long size() {
+    return 0;
+  }
+
+  @Override
+  public CacheStats stats() {
+    return new CacheStats(0, 0, 0, 0, 0, 0);
+  }
+
+  @Override
+  public void cleanUp() {}
+
+  @Override
+  public V get(K key) throws ExecutionException {
+    try {
+      return cacheLoader.load(key);
+    } catch (Exception e) {
+      throw new ExecutionException(e);
+    }
+  }
+
+  @Override
+  public V getUnchecked(K key) {
+    try {
+      return cacheLoader.load(key);
+    } catch (Exception e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  @Override
+  public ImmutableMap<K, V> getAll(Iterable<? extends K> keys) throws ExecutionException {
+    try {
+      try {
+        return getAllBulk(keys);
+      } catch (UnsupportedLoadingOperationException e) {
+        return getAllIndividually(keys);
+      }
+    } catch (Exception e) {
+      throw new ExecutionException(e);
+    }
+  }
+
+  private ImmutableMap<K, V> getAllIndividually(Iterable<? extends K> keys) throws Exception {
+    ImmutableMap.Builder<K, V> builder = ImmutableMap.builder();
+    for (K k : keys) {
+      builder.put(k, cacheLoader.load(k));
+    }
+    return builder.build();
+  }
+
+  @SuppressWarnings("unchecked")
+  private ImmutableMap<K, V> getAllBulk(Iterable<? extends K> keys) throws Exception {
+    return (ImmutableMap<K, V>) ImmutableMap.copyOf(cacheLoader.loadAll(keys));
+  }
+
+  @Override
+  public V apply(K key) {
+    return getUnchecked(key);
+  }
+
+  @Override
+  public void refresh(K key) {}
+
+  @Override
+  public ConcurrentMap<K, V> asMap() {
+    return new ConcurrentHashMap<>();
+  }
+}

java/com/google/gerrit/server/notedb/NoteDbUpdateManager.java

@@ -306,6 +306,12 @@ private void stage() throws IOException {
     }
   }
 
+  public BatchRefUpdate prepare() throws IOException {
+    checkNotExecuted();
+    stage();
+    return prepare(changeRepo, false, pushCert);
+  }
+
   @Nullable
   public BatchRefUpdate execute() throws IOException {
     return execute(false);
@@ -353,7 +359,7 @@ public BatchRefUpdate execute(boolean dryrun) throws IOException {
     }
   }
 
-  private BatchRefUpdate execute(OpenRepo or, boolean dryrun, @Nullable PushCertificate pushCert)
+  private BatchRefUpdate prepare(OpenRepo or, boolean dryrun, @Nullable PushCertificate pushCert)
       throws IOException {
     if (or == null || or.cmds.isEmpty()) {
       return null;
@@ -382,7 +388,13 @@ private BatchRefUpdate execute(OpenRepo or, boolean dryrun, @Nullable PushCertif
       bru = listener.beforeUpdateRefs(bru);
     }
 
-    if (!dryrun) {
+    return bru;
+  }
+
+  private BatchRefUpdate execute(OpenRepo or, boolean dryrun, @Nullable PushCertificate pushCert)
+      throws IOException {
+    BatchRefUpdate bru = prepare(or, dryrun, pushCert);
+    if (bru != null && !dryrun) {
       RefUpdateUtil.executeChecked(bru, or.rw);
     }
     return bru;

java/com/google/gerrit/server/update/BatchUpdate.java

@@ -434,6 +434,11 @@ public void execute() throws UpdateException, RestApiException {
     execute(ImmutableList.of(this), ImmutableList.of(), false);
   }
 
+  public BatchRefUpdate prepareRefUpdates() throws Exception {
+    ChangesHandle handle = executeChangeOps(ImmutableList.of(), false);
+    return handle.prepare();
+  }
+
   public boolean isExecuted() {
     return executed;
   }
@@ -610,6 +615,10 @@ void setResult(Change.Id id, ChangeResult result) {
       checkArgument(old == null, "result for change %s already set: %s", id, old);
     }
 
+    public BatchRefUpdate prepare() throws IOException {
+      return manager.prepare();
+    }
+
     void execute() throws IOException {
       BatchUpdate.this.batchRefUpdate = manager.execute(dryrun);
       BatchUpdate.this.executed = manager.isExecuted();

javatests/com/google/gerrit/server/cache/mem/BUILD

@@ -0,0 +1,14 @@
+load("//tools/bzl:junit.bzl", "junit_tests")
+
+junit_tests(
+    name = "tests",
+    srcs = glob(["*Test.java"]),
+    deps = [
+        "//java/com/google/gerrit/server",
+        "//java/com/google/gerrit/server/cache/mem",
+        "//lib:jgit",
+        "//lib:junit",
+        "//lib/guice",
+        "//lib/truth",
+    ],
+)