chimera: let JdbcFs run maintenance tasks for all backend implementation

Motivation:
A file system backend might require some periodic maintenance
activities, like a cleanup, merging of temporary information
or synchronization.

Modification:
Add method FsSqlDriver#performMaintenanceTask which is periodically
called by JdbcFs. Update PnfsManager and NFS door to participate in
leader election to ensure that only one component runs periodic tasks.

Result:
A possibility to run periodic chimera backend specific tasks.

Acked-by: Paul Millar
Target: master
Require-book: no
Require-notes: yes

modules/chimera/pom.xml

@@ -46,6 +46,10 @@
       <groupId>org.springframework</groupId>
       <artifactId>spring-jdbc</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.curator</groupId>
+      <artifactId>curator-recipes</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.dcache</groupId>
       <artifactId>dcache-common</artifactId>

modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java

@@ -2212,4 +2212,16 @@ void removeLabel(FsInode inode, String labelname) throws ChimeraFsException {
         setInodeAttributes(inode, 0, new Stat());
     }
 
+
+    /**
+     * File system backend specific periodic maintenance task. It's recommended to run the
+     * maintenance task runs as a single transaction, or as a series of transactions, if sub-tasks
+     * are involved. The thread running the tasks might be interrupted, for example, due to shutdown
+     * of the dCache processes.
+     *
+     * The implementation should handle multiple instances of the maintenance task running concurrently,
+     */
+    void performMaintenanceTask() {
+
+    }
 }

modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java

@@ -48,8 +48,11 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import javax.sql.DataSource;
+import org.apache.curator.framework.recipes.leader.LeaderLatchListener;
 import org.dcache.acl.ACE;
 import org.dcache.acl.enums.RsType;
 import org.dcache.chimera.posix.Stat;
@@ -75,7 +78,7 @@
  * @Immutable
  * @Threadsafe
  */
-public class JdbcFs implements FileSystemProvider {
+public class JdbcFs implements FileSystemProvider, LeaderLatchListener {
 
     /**
      * common error message for unimplemented
@@ -181,6 +184,16 @@ public FsStat load(Object k) throws Exception {
      */
     private RetentionPolicy _defaultRetentionPolicy;
 
+
+    private final ScheduledExecutorService maintenanceTaskExecutor = Executors.newSingleThreadScheduledExecutor(
+          new ThreadFactoryBuilder()
+                .setNameFormat("chimera-maintenance-executor-%d")
+                .build()
+    );
+
+    private ScheduledFuture<?> maintenanceTask;
+
+
     public JdbcFs(DataSource dataSource, PlatformTransactionManager txManager)
           throws SQLException, ChimeraFsException {
         this(dataSource, txManager, 0);
@@ -1479,7 +1492,7 @@ public String getInfo() {
      */
     @Override
     public void close() throws IOException {
-        // enforced by the interface
+        maintenanceTaskExecutor.shutdown();
     }
 
     @Override
@@ -1651,4 +1664,30 @@ private void checkQuota(int uid, int gid, RetentionPolicy rp)
                   String.format("%s user quota exceeded for uid=%d", rp, uid));
         }
     }
+
+    private synchronized void enableMaintenanceTask() {
+        if (maintenanceTask == null) {
+            maintenanceTask = maintenanceTaskExecutor.scheduleWithFixedDelay(
+                  () -> _sqlDriver.performMaintenanceTask(), 10, 20, TimeUnit.SECONDS
+            );
+        }
+    }
+
+    private synchronized void disableMaintenanceTask() {
+        if (maintenanceTask != null) {
+            maintenanceTask.cancel(false);
+            maintenanceTask = null;
+        }
+    }
+
+    @Override
+    public void isLeader() {
+        enableMaintenanceTask();
+    }
+
+    @Override
+    public void notLeader() {
+        disableMaintenanceTask();
+    }
+
 }

modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml

@@ -107,6 +107,24 @@
       <property name="defaultRetentionPolicy" value="#{ T(diskCacheV111.util.RetentionPolicy).getRetentionPolicy('${pnfsmanager.default-retention-policy}') }"/>
   </bean>
 
+  <bean id="chimera-maintenance-leadership-manager" class="org.dcache.cells.HAServiceLeadershipManager"
+    init-method="initZkLeaderListener" destroy-method="shutdown">
+    <description>Coordinates which components performs maintenance tasks</description>
+    <constructor-arg value="chimera-maintenance"/>
+    <property name="leadershipListener">
+      <ref bean="chimera-maintenance-leaderlistener-group"/>
+    </property>
+  </bean>
+
+  <bean id="chimera-maintenance-leaderlistener-group" class="org.dcache.cells.LeadershipListenerGroup">
+    <description>Propagates leadership change notifications to managed listeners</description>
+    <property name="leaderElectionAwareComponents">
+      <set>
+        <ref bean="file-system"/>
+      </set>
+    </property>
+  </bean>
+
   <bean id="quota-refresh-executor"
         class="org.dcache.util.CDCScheduledExecutorServiceDecorator">
       <description>Task scheduler</description>

modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml

@@ -145,6 +145,24 @@
 
     </bean>
 
+    <bean id="chimera-maintenance-leadership-manager" class="org.dcache.cells.HAServiceLeadershipManager"
+      init-method="initZkLeaderListener" destroy-method="shutdown">
+        <description>Coordinates which components performs maintenance tasks</description>
+        <constructor-arg value="chimera-maintenance"/>
+        <property name="leadershipListener">
+            <ref bean="chimera-maintenance-leaderlistener-group"/>
+        </property>
+    </bean>
+
+    <bean id="chimera-maintenance-leaderlistener-group" class="org.dcache.cells.LeadershipListenerGroup">
+        <description>Propagates leadership change notifications to managed listeners</description>
+        <property name="leaderElectionAwareComponents">
+            <set>
+                <ref bean="fileSystem"/>
+            </set>
+        </property>
+    </bean>
+
     <bean id="export" class="org.dcache.nfs.ExportFile">
         <description>NFS export file</description>
         <constructor-arg index = "0">
@@ -350,4 +368,5 @@
     <beans profile="inotify-false">
         <bean parent="door"/>
     </beans>
+
 </beans>

modules/dcache/src/main/java/org/dcache/cells/HAServiceLeadershipManager.java

@@ -1,7 +1,7 @@
 /*
  * dCache - http://www.dcache.org/
  *
- * Copyright (C) 2020 Deutsches Elektronen-Synchrotron
+ * Copyright (C) 2020 - 2023 Deutsches Elektronen-Synchrotron
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -28,8 +28,10 @@
 import dmg.cells.zookeeper.CDCLeaderLatchListener;
 import dmg.util.CommandException;
 import dmg.util.command.Command;
+import dmg.util.command.CommandPrefix;
 import java.io.PrintWriter;
 import java.util.Collection;
+import java.util.Objects;
 import java.util.concurrent.Callable;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.recipes.leader.LeaderLatch;
@@ -55,7 +57,11 @@ public class HAServiceLeadershipManager implements CellIdentityAware, CellComman
     private LeaderLatch zkLeaderLatch;
     private LeaderLatchListener leadershipListener;
 
+    @CommandPrefix
+    private final String serviceName;
+
     public HAServiceLeadershipManager(String serviceName) {
+        this.serviceName = Objects.requireNonNull(serviceName);
         createZkLeadershipPath(serviceName);
     }
 

skel/bin/chimera

@@ -9,7 +9,7 @@ classpath=$(printLimitedClassPath dcache-vehicles dcache-chimera chimera HikariC
     guava jline common-cli dcache-common acl-vehicles acl \
     slf4j-api logback-classic logback-core logback-console-config jcl-over-slf4j \
     spring-core spring-beans spring-jdbc spring-tx \
-    postgresql h2 hsqldb)
+    postgresql h2 hsqldb curator-recipes)
 
 CLASSPATH="$classpath" quickJava -Dlog=${DCACHE_LOG:-warn} \
     org.dcache.chimera.cli.Shell \