chimera: do not maintain time-based cached value of FsStat

NFS server by itself can cache the results of fsstat.
JdbcFs sill keeps a cached result, but it updated on demand.
Nevertheless, as this is a time consuming operation,
start an extra thread to perform the update. As long as update
is still in the progress, the last value is returned.

The update part takes care that only one thread is started in parallel.

Introduce two new properties:

nfs.fs-stat-cache.time
nfs.fs-stat-cache.time.unit

which will turn on caching in the nfs server.

Acked-by: Gerd Behrmann
Acked-by: Dmitry Litvintsev
Target: master
Target: 2.12, 2.11, 2.10 in the future
Require-book: no
Require-notes: yes
(cherry picked from commit f1f765d)
Signed-off-by: Tigran Mkrtchyan <tigran.mkrtchyan@desy.de>

modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java

@@ -16,6 +16,11 @@
  */
 package org.dcache.chimera;
 
+import com.google.common.base.Throwables;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -31,10 +36,13 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.StringTokenizer;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import diskCacheV111.util.AccessLatency;
 import diskCacheV111.util.RetentionPolicy;
-
 import org.dcache.acl.ACE;
 import org.dcache.chimera.posix.Stat;
 import org.dcache.chimera.store.InodeStorageInformation;
@@ -85,7 +93,35 @@ public class JdbcFs implements FileSystemProvider {
      * Database connection pool
      */
     private final DataSource _dbConnectionsPool;
-    private final FsStatCache _fsStatCache;
+
+    /*
+     * A dummy constant key force bay cache interface. the value doesn't
+     * matter - only that it's the same value every time
+     */
+    private final Integer DUMMY_KEY = 0;
+    /**
+     * Cache value of FsStat
+     */
+    private final Executor _fsStatUpdateExecutor =
+            Executors.newSingleThreadExecutor(
+                    new ThreadFactoryBuilder()
+                        .setNameFormat("fsstat-updater-thread-%d")
+                        .build()
+            );
+
+    private final LoadingCache<Object, FsStat> _fsStatCache
+            = CacheBuilder.newBuilder()
+                .refreshAfterWrite(100, TimeUnit.MILLISECONDS)
+                .build(
+                    CacheLoader.asyncReloading(new CacheLoader<Object, FsStat>() {
+
+                        @Override
+                        public FsStat load(Object k) throws Exception {
+                            return JdbcFs.this.getFsStat0();
+                        }
+                    }
+            , _fsStatUpdateExecutor));
+
     /**
      * current fs id
      */
@@ -125,7 +161,6 @@ public JdbcFs(DataSource dataSource, String dialect, int id) {
         } catch (Exception e) {
         }
         _wormID = wormID;
-        _fsStatCache = new FsStatCache(this);
     }
 
     private FsInode getWormID() throws ChimeraFsException {
@@ -2286,46 +2321,29 @@ private static void checkNameLength(String name) throws InvalidNameChimeraExcept
         }
     }
 
-    /**
-     * internal class to hide caching mechanism.
-     */
-    static class FsStatCache {
-
-        private FsStat _fsStatCached;
-        private long _fsStatLastUpdate;
-        // FIXME: make it configurable
-        private long _fsStateLifetime = 3600000;
-        private final JdbcFs _fs;
-
-        FsStatCache(JdbcFs fs) {
-            _fs = fs;
-        }
-
-        public synchronized FsStat getFsStat(DataSource dbConnectionsPool, FsSqlDriver driver) throws ChimeraFsException {
-
-            if (_fsStatLastUpdate == 0 || _fsStatLastUpdate + _fsStateLifetime < System.currentTimeMillis()) {
-                Connection dbConnection = null;
-                try {
-                    dbConnection = dbConnectionsPool.getConnection();
-                    _fsStatCached = driver.getFsStat(dbConnection);
-                } catch (SQLException e) {
-                    throw new IOHimeraFsException(e.getMessage());
-                } finally {
-                    tryToClose(dbConnection);
-                }
-                _log.debug("updateing cached value of FsStat");
-                _fsStatLastUpdate = System.currentTimeMillis();
-            } else {
-                _log.debug("using cached value of FsStat");
-            }
-
-            return _fsStatCached;
+    public FsStat getFsStat0() throws ChimeraFsException {
+        FsStat fsStat = null;
+        Connection dbConnection = null;
+        try {
+            dbConnection = _dbConnectionsPool.getConnection();
+            fsStat = _sqlDriver.getFsStat(dbConnection);
+        } catch (SQLException e) {
+            _log.error("Failed to obtain FsStat: {}", e.getMessage());
+        } finally {
+            tryToClose(dbConnection);
         }
+        return fsStat;
     }
 
     @Override
     public FsStat getFsStat() throws ChimeraFsException {
-        return _fsStatCache.getFsStat(_dbConnectionsPool, _sqlDriver);
+        try {
+            return _fsStatCache.get(DUMMY_KEY);
+        }catch(ExecutionException e) {
+            Throwable t = e.getCause();
+            Throwables.propagateIfPossible(t, ChimeraFsException.class);
+            throw new ChimeraFsException(t.getMessage(), t);
+        }
     }
 
     ///////////////////////////////////////////////////////////////

modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml

@@ -172,6 +172,8 @@
         <property name="maxEntries" value="${nfs.namespace-cache.size}" />
         <property name="lifeTime" value="${nfs.namespace-cache.time}" />
         <property name="timeUnit" value="${nfs.namespace-cache.time.unit}" />
+        <property name="fsStatLifeTime" value="${nfs.fs-stat-cache.time}" />
+        <property name="fsStatTimeUnit" value="${nfs.fs-stat-cache.time.unit}" />
     </bean>
 
     <beans  profile="portmap-true">

skel/share/defaults/nfs.properties

@@ -157,6 +157,13 @@ nfs.db.connections.idle = 1
 #
 # Each cached entry takes 120 bytes of additional memory
 nfs.namespace-cache.time = 3
-nfs.namespace-cache.time.unit = SECONDS
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)nfs.namespace-cache.time.unit = SECONDS
 nfs.namespace-cache.size = 0
 
+# FS stat cache update interval. This variable controls frequency of
+# aggregate queries to underlying db back-end when reporting
+# total size and total number of files in namespace (e.g. when executing
+# 'df' command). Depending on database implementation, the aggregate queries
+# could be costly (PostgreSQL is one such implementation).
+nfs.fs-stat-cache.time = 3600
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)nfs.fs-stat-cache.time.unit = SECONDS