Namespace: add FS stat cache table

Motivation:

On startup NFS door executes a heavy query that calculates
total number of files and total space used. On large installations,
such as Fermilab dCache, that query takes 10-20 minutes (depending
on hardware). NFS mounts hang for as much trying to mount the
file system. A df command would hang as well. If NFS server is
restarted (which is sometimes admin's reaction to hanging mounts)
another query will start running. And so on

Modification:

Add a table t_fstat that captures usedFiles and usedSpace to
be queried by NFS door. Insert default starting values (1M) for both.
Add a function updateFsStat that runs at configurable intervals
to update usedFiles and usedSpace by running aggregare select
on t_inodes table effectively caching the result.

Result:

Mount (and df) command works without hangs. On upgrade the initial
value of used space shown is 1M that gets updated after one hour
based on default. Defaults can be tweaked in dcache.conf or layout
files.

Release Notes:

Added functionality to cache total files and total space used on DB
backend. Added two properties that control the behavior how often heavy
query is run:

pnfsmanager.fs-stat-cache.time = 3600
pnfsmanager.fs-stat-cache.time.unit = SECONDS

Target: trunk
Require-notes: yes
Require-book: yes
Patch: https://rb.dcache.org/r/13030/
Acked-by: tigran.mkrtchyan@desy.de, arossi@fnal.gov

modules/chimera/src/main/java/org/dcache/chimera/FileSystemProvider.java

@@ -375,6 +375,11 @@ Set<Checksum> getInodeChecksums(FsInode inode)
 
     String getInfo();
 
+    /**
+     * Update file system statistics.
+     */
+    void updateFsStat() throws ChimeraFsException;
+
     /**
      * Get file system statistic.
      *

modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java

@@ -154,18 +154,28 @@ long getRootInumber()
         return _root;
     }
 
+    /**
+     * Update file system cache table.
+     */
+    void updateFsStat() {
+        _jdbc.update("UPDATE t_fstat SET iusedFiles = t.usedFiles, "+
+                     "iusedSpace = t.usedSpace "+
+                     "FROM (SELECT count(*) AS usedFiles, "+
+                     "SUM(isize) AS usedSpace FROM t_inodes "+
+                     "WHERE itype=32768) t");
+    }
+
     /**
      * Get FsStat for a given filesystem.
      * @return fsStat
      */
     FsStat getFsStat() {
         return _jdbc.queryForObject(
-                "SELECT count(*) AS usedFiles, SUM(isize) AS usedSpace FROM t_inodes WHERE itype=32768",
+                                    "SELECT iusedfiles AS usedFiles, iusedspace AS usedSpace from t_fstat",
                 (rs, rowNum) -> {
                     long usedFiles = rs.getBigDecimal("usedFiles")
                             .min(BigDecimal.valueOf(Long.MAX_VALUE))
                             .longValue();
-                    // SUM(isize) of an empty table returns null
                     BigDecimal usedSpaceB = rs.getBigDecimal("usedSpace");
                     long usedSpace = usedSpaceB == null? 0L : usedSpaceB.min(BigDecimal.valueOf(Long.MAX_VALUE)).longValue();
                     return new FsStat(JdbcFs.AVAILABLE_SPACE, JdbcFs.TOTAL_FILES, usedSpace, usedFiles);

modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java

@@ -1330,6 +1330,11 @@ private static void checkNameLength(String name) throws InvalidNameChimeraExcept
         }
     }
 
+    @Override
+    public void updateFsStat() throws ChimeraFsException {
+        _sqlDriver.updateFsStat();
+    }
+
     public FsStat getFsStat0() throws ChimeraFsException {
         return _sqlDriver.getFsStat();
     }

modules/chimera/src/main/resources/org/dcache/chimera/changelog/changelog-master.xml

@@ -23,4 +23,5 @@
     <include file="org/dcache/chimera/changelog/changeset-6.0.xml"/>
     <include file="org/dcache/chimera/changelog/changeset-6.2.xml"/>
     <include file="org/dcache/chimera/changelog/changeset-7.1.xml"/>
+    <include file="org/dcache/chimera/changelog/changeset-7.2.xml"/>
 </databaseChangeLog>

modules/chimera/src/main/resources/org/dcache/chimera/changelog/changeset-7.2.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+     http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
+
+    <changeSet id="32" author="litvinse">
+      <comment>add FS statiscs table</comment>
+
+      <createTable tableName="t_fstat">
+        <column name="iusedfiles" type="bigint"/>
+        <column name="iusedspace" type="bigint"/>
+      </createTable>
+
+      <insert tableName="t_fstat">
+        <column name="iusedfiles" value="1048576"/>
+        <column name="iusedspace" value="1048576"/>
+      </insert>
+
+    </changeSet>
+
+</databaseChangeLog>

modules/chimera/src/test/java/org/dcache/chimera/BasicTest.java

@@ -1344,11 +1344,11 @@ public void testOverflowOnJomboFiles() throws ChimeraFsException {
         assertThat(fsStat.getUsedFiles(),greaterThan(0L));
     }
     @Test
-    public void testEmptyFsStat() throws ChimeraFsException {
+    public void testFsStat() throws ChimeraFsException {
 
         FsStat fsStat = _fs.getFsStat();
-        assertThat(fsStat.getUsedSpace(), is(0L));
-        assertThat(fsStat.getUsedFiles(), is(0L));
+        assertThat(fsStat.getUsedSpace(), is(1048576L));
+        assertThat(fsStat.getUsedFiles(), is(1048576L));
     }
 
     @Test

modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraNameSpaceProvider.java

@@ -1607,6 +1607,15 @@ public Collection<FileAttributes> cancelUpload(Subject subject, FsPath temporary
         return deleted;
     }
 
+    @Override
+    public void updateFsStat() throws CacheException {
+        try {
+            _fs.updateFsStat();
+        } catch (ChimeraFsException e) {
+            throw new CacheException(CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.getMessage());
+        }
+    }
+
     private void removeRecursively(ExtendedInode parent, String name, ExtendedInode inode,
             Consumer<ExtendedInode> deleted) throws ChimeraFsException, CacheException
     {

modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml

@@ -3,7 +3,9 @@
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xmlns:context="http://www.springframework.org/schema/context"
        xsi:schemaLocation="http://www.springframework.org/schema/beans
-           http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd">
+                        http://www.springframework.org/schema/beans/spring-beans.xsd
+                        http://www.springframework.org/schema/context
+                        http://www.springframework.org/schema/context/spring-context.xsd">
 
     <context:property-placeholder/>
     <context:annotation-config/>
@@ -39,6 +41,10 @@
       <property name="atimeGap" value="${pnfsmanager.atime-gap}" />
       <property name="flushNotificationTarget" value="${pnfsmanager.destination.flush-notification}"/>
       <property name="cancelUploadNotificationTarget" value="${pnfsmanager.destination.cancel-upload-notification}"/>
+      <property name="scheduledExecutor" ref="scheduled-executor"/>
+      <property name="updateFsStatInterval" value="${pnfsmanager.fs-stat-cache.time}"/>
+      <property name="updateFsStatIntervalUnit" value="${pnfsmanager.fs-stat-cache.time.unit}"/>
+
   </bean>
 
     <bean id="data-source" class="org.dcache.db.AlarmEnabledDataSource" destroy-method="close">
@@ -119,6 +125,27 @@
       <property name="nameSpaceProvider" ref="name-space-provider"/>
   </bean>
 
+
+  <bean id="scheduled-executor"
+        class="org.dcache.util.CDCScheduledExecutorServiceDecorator">
+      <description>Task scheduler</description>
+      <constructor-arg>
+          <bean class="java.util.concurrent.Executors"
+                factory-method="newScheduledThreadPool"
+                destroy-method="shutdown">
+              <constructor-arg value="2"/>
+          </bean>
+      </constructor-arg>
+  </bean>
+
+  <bean id="ha-service-leadership-manager" class="org.dcache.cells.HAServiceLeadershipManager"
+        init-method="initZkLeaderListener" destroy-method="shutdown">
+      <constructor-arg value="pnfsmanager"/>
+      <property name="leadershipListener">
+          <ref bean="leaderlistener-group"/>
+      </property>
+  </bean>
+
   <beans profile="inotify-true">
       <bean id="event-notifier" class="diskCacheV111.namespace.EventNotifier">
           <description>Service allowing other dCache components to be notified of namespace events</description>
@@ -146,19 +173,41 @@
                     value="${pnfsmanager.inotify-generation.backlog.per-door}"/>
       </bean>
 
-      <bean parent="pnfs-manager">
+      <bean id="leaderlistener-group" class="org.dcache.cells.LeadershipListenerGroup">
+        <description>Propagates leadership change notifications to managed listeners</description>
+        <property name="leaderElectionAwareComponents">
+          <set>
+            <ref bean="pnfs-manager-ionotify"/>
+          </set>
+        </property>
+      </bean>
+
+      <bean id="pnfs-manager-ionotify" parent="pnfs-manager">
           <property name="nameSpaceProvider">
               <bean class="diskCacheV111.namespace.MonitoringNameSpaceProvider">
                   <property name="nameSpaceProvider" ref="name-space-provider"/>
                   <property name="eventReceiver" ref="event-notifier"/>
               </bean>
           </property>
       </bean>
+
   </beans>
 
   <beans profile="inotify-false">
-      <bean parent="pnfs-manager">
+
+      <bean id="leaderlistener-group" class="org.dcache.cells.LeadershipListenerGroup">
+	<description>Propagates leadership change notifications to managed listeners</description>
+	<property name="leaderElectionAwareComponents">
+          <set>
+            <ref bean="pnfs-manager-no-ionotify"/>
+          </set>
+	</property>
+      </bean>
+
+      <bean id="pnfs-manager-no-ionotify" parent="pnfs-manager">
           <property name="nameSpaceProvider" ref="name-space-provider"/>
       </bean>
+
+
   </beans>
 </beans>

modules/dcache/src/main/java/diskCacheV111/namespace/ForwardingNameSpaceProvider.java

@@ -229,4 +229,10 @@ public void removeExtendedAttribute(Subject subject, FsPath path, String name)
     {
         delegate().removeExtendedAttribute(subject, path, name);
     }
+
+    @Override
+    public void updateFsStat() throws CacheException {
+        delegate().updateFsStat();
+    }
+
 }

modules/dcache/src/main/java/diskCacheV111/namespace/NameSpaceProvider.java

@@ -418,4 +418,10 @@ Set<String> listExtendedAttributes(Subject subject, FsPath path)
      */
     void removeExtendedAttribute(Subject subject, FsPath path, String name)
             throws CacheException;
+
+    /**
+     * Update FS stat cache.
+     */
+    void updateFsStat() throws CacheException;
+
 }

modules/dcache/src/main/java/diskCacheV111/namespace/PnfsManagerV3.java

@@ -8,6 +8,7 @@
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Required;
@@ -37,6 +38,8 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -107,6 +110,7 @@
 import org.dcache.util.ChecksumType;
 import org.dcache.util.ColumnWriter;
 import org.dcache.util.ColumnWriter.TabulatedRow;
+import org.dcache.util.FireAndForgetTask;
 import org.dcache.util.TimeUtils;
 import org.dcache.vehicles.FileAttributes;
 import org.dcache.vehicles.PnfsCreateSymLinkMessage;
@@ -124,7 +128,7 @@
 
 public class PnfsManagerV3
     extends AbstractCellComponent
-    implements CellCommandListener, CellMessageReceiver, CellInfoProvider
+    implements CellCommandListener, CellMessageReceiver, CellInfoProvider, LeaderLatchListener
 {
     private static final Logger _log =
         LoggerFactory.getLogger(PnfsManagerV3.class);
@@ -162,6 +166,11 @@ public class PnfsManagerV3
     private int _listThreads;
     private long _logSlowThreshold;
 
+    private ScheduledFuture<?> updateFsFuture;
+    private ScheduledExecutorService scheduledExecutor;
+    private TimeUnit updateFsStatIntervalUnit;
+    private long updateFsStatInterval;
+
     /**
      * Whether to use folding.
      */
@@ -240,6 +249,24 @@ public PnfsManagerV3()
         populateRequestMap();
     }
 
+    @Required
+    public void setUpdateFsStatInterval(long updateFsStatInterval)
+    {
+        this.updateFsStatInterval = updateFsStatInterval;
+    }
+
+    @Required
+    public void setUpdateFsStatIntervalUnit(TimeUnit updateFsStatIntervalUnit)
+    {
+        this.updateFsStatIntervalUnit = updateFsStatIntervalUnit;
+    }
+
+    @Required
+    public void setScheduledExecutor(ScheduledExecutorService executor)
+    {
+        scheduledExecutor = executor;
+    }
+
     @Required
     public void setThreads(int threads)
     {
@@ -388,6 +415,33 @@ private void drainQueue(BlockingQueue<CellMessage> queue)
         queue.offer(SHUTDOWN_SENTINEL);
     }
 
+    @Override
+    public void isLeader()
+    {
+        updateFsFuture = scheduledExecutor.
+            scheduleWithFixedDelay(
+                                   new FireAndForgetTask(new Runnable() {
+                                           @Override
+                                           public void run() {
+                                               try {
+                                                   updateFsStat();
+                                               } catch (CacheException ignore) {
+                                                   _log.error("Failed to tun updateFsStat: {}", ignore.getMessage());
+                                               }
+                                           }
+                                       }),
+                                   100000,
+                                   updateFsStatIntervalUnit.toMillis(updateFsStatInterval),
+                                   TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public void notLeader()
+    {
+        updateFsFuture.cancel(true);
+    }
+
+
     @Override
     public void getInfo( PrintWriter pw ){
         pw.print("atime precision: ");
@@ -2662,4 +2716,11 @@ private static void checkRestriction(Restriction restriction, Set<AccessMask> ma
             throw new PermissionDeniedCacheException("Restriction " + restriction + " denied activity " + activity + " on " + path);
         }
     }
+
+    /**
+     * This function to be run periodically to update FS stat cache
+     */
+    public void updateFsStat() throws CacheException {
+        _nameSpaceProvider.updateFsStat();
+    }
 }

modules/dcache/src/main/java/org/dcache/auth/RemoteNameSpaceProvider.java

@@ -255,6 +255,10 @@ public Collection<FileAttributes> cancelUpload(Subject subject, FsPath uploadPat
                 uploadPath, path, requested, explanation)).getDeletedFiles();
     }
 
+    @Override
+    public void updateFsStat() {
+    }
+
     public byte[] readExtendedAttribute(Subject subject, FsPath path, String name)
             throws CacheException
     {

skel/share/defaults/nfs.properties

@@ -196,11 +196,8 @@ nfs.namespace-cache.time = 3
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)nfs.namespace-cache.time.unit = SECONDS
 nfs.namespace-cache.size = 0
 
-# FS stat cache update interval. This variable controls frequency of
-# aggregate queries to underlying db back-end when reporting
-# total size and total number of files in namespace (e.g. when executing
-# 'df' command). Depending on database implementation, the aggregate queries
-# could be costly (PostgreSQL is one such implementation).
+# FS stat cache update interval. This variable controls how often
+# total number of files and total space used numbers are updated if memory
 nfs.fs-stat-cache.time = 3600
 (one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)nfs.fs-stat-cache.time.unit = SECONDS
 

skel/share/defaults/pnfsmanager.properties

@@ -164,6 +164,13 @@ pnfsmanager.limits.queue-length = 0
 #
 (one-of?true|false)pnfsmanager.enable.move-to-directory-with-different-storageclass = true
 
+# FS stat cache table update interval. This variable controls frequency of
+# scheduled updated on underlying db back-end  capturing total size and total
+# number of files in namespace
+pnfsmanager.fs-stat-cache.time = 3600
+(one-of?MILLISECONDS|SECONDS|MINUTES|HOURS|DAYS)pnfsmanager.fs-stat-cache.time.unit = SECONDS
+
+
 #  ---- Enabled ACL support
 #
 #   Set to true to enable ACL support.