pnfsmanager: add instrumentation to support sending events

Motivation:

Update PnfsManager deployment so that sending of inotify-events is
supported.

Modification:

Add configuration properties.

Update spring to include inotify instrumentation, if so configured.

Result:

PnfsManager can generate inotify events, if any cell has subscribed to
events.

Target: master
Require-notes: no
Require-book: no
Patch: https://rb.dcache.org/r/11162/
Acked-by: Albert Rossi

modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml

@@ -23,7 +23,7 @@
         <property name="excludedDestinations" value="${pnfsmanager.destination.cache-notification}"/>
     </bean>
 
-  <bean id="pnfs-manager" class="diskCacheV111.namespace.PnfsManagerV3"
+  <bean abstract="true" id="pnfs-manager" class="diskCacheV111.namespace.PnfsManagerV3"
         init-method="init" destroy-method="shutdown">
       <description>Request processor</description>
       <property name="threads" value="${pnfsmanager.limits.threads}"/>
@@ -33,7 +33,6 @@
       <property name="folding" value="${pnfsmanager.enable.folding}"/>
       <property name="directoryListLimit" value="${pnfsmanager.limits.list-chunk-size}"/>
       <property name="permissionHandler" ref="permission-handler"/>
-      <property name="nameSpaceProvider" ref="name-space-provider"/>
       <property name="queueMaxSize" value="${pnfsmanager.limits.queue-length}"/>
       <property name="atimeGap" value="${pnfsmanager.atime-gap}" />
       <property name="flushNotificationTarget" value="${pnfsmanager.destination.flush-notification}"/>
@@ -108,4 +107,47 @@
       <description>ACL command line</description>
       <property name="nameSpaceProvider" ref="name-space-provider"/>
   </bean>
+
+  <beans profile="inotify-true">
+      <bean id="event-notifier" class="diskCacheV111.namespace.EventNotifier">
+          <description>Service allowing other dCache components to be notified of namespace events</description>
+          <property name="cellStub">
+              <bean class="org.dcache.cells.CellStub">
+                  <property name="timeout" value="10"/>
+                  <property name="timeoutUnit" value="SECONDS"/>
+              </bean>
+          </property>
+          <property name="dispatchExecutor">
+              <bean class="org.dcache.util.BoundedCachedExecutor"
+                    destroy-method="shutdownNow">
+                  <constructor-arg value="1"/>  <!-- Only a single thread -->
+                  <constructor-arg value="${pnfsmanager.inotify-generation.backlog.initial}"/>
+              </bean>
+          </property>
+          <property name="senderExecutor">
+              <bean class="java.util.concurrent.Executors"
+                    factory-method="newSingleThreadExecutor"
+                    destroy-method="shutdownNow"/>
+          </property>
+          <property name="eventBatchSize"
+                    value="${pnfsmanager.inotify-generation.message-batch-size}"/>
+          <property name="maximumQueuedEvents"
+                    value="${pnfsmanager.inotify-generation.backlog.per-door}"/>
+      </bean>
+
+      <bean parent="pnfs-manager">
+          <property name="nameSpaceProvider">
+              <bean class="diskCacheV111.namespace.MonitoringNameSpaceProvider">
+                  <property name="nameSpaceProvider" ref="name-space-provider"/>
+                  <property name="eventReceiver" ref="event-notifier"/>
+              </bean>
+          </property>
+      </bean>
+  </beans>
+
+  <beans profile="inotify-false">
+      <bean parent="pnfs-manager">
+          <property name="nameSpaceProvider" ref="name-space-provider"/>
+      </bean>
+  </beans>
 </beans>

skel/share/defaults/dcache.properties

@@ -1171,6 +1171,63 @@ dcache.oidc.hostnames =
 dcache.kafka.bootstrap-servers = localhost:9092
 
 
+#  ---- Support for inotify events
+#
+#  dCache services (e.g., frontend) can request that they be notified
+#  of activity similar to the inotify(7) events.  The following
+#  property controls whether such events are sent.
+#
+(one-of?true|false)dcache.inotify-generation.enable = true
+
+#  The inotify support is designed so that, under overload situations,
+#  only limited resources are available for sending these events.  If
+#  that proves insufficient then inotify events will start to be
+#  queued.  To avoid memory exhausion, these queues have a limit
+#  beyond which the events are discarded and some inotify clients will
+#  receive the IN_Q_OVERFLOW event.
+
+#  Maximum number of inotify events to queue while deciding to which
+#  doors (currently just frontend) an inotify event should be sent.
+#  If events are dropped then all inotify clients of all doors will
+#  receive an IN_Q_OVERFLOW event.
+#
+#  This is a single queue, independent of the number of frontend doors
+#  running.
+#
+#  Setting this value too small will result in unnecessary loss of
+#  events when processing a door's changing watch list.  Setting this
+#  value too large risks running out of memory if large number of
+#  events are queued.
+#
+dcache.inotify-generation.backlog.initial = 1000
+
+#  Maximum number of inotify events to queue while waiting to send
+#  inotify messages to a frontend door.  If the limit is reached then
+#  all clients of that door will receive the IN_Q_OVERFLOW event.
+#  Other doors are unaffected.
+#
+#  There is a queue per frontend door with at least one inotify
+#  client.
+#
+#  Setting this value too small will result in unnecessary loss of
+#  events if overload situations are sufficiently short.  Setting this
+#  value too large risks running out of memory during a sustained
+#  overload.
+#
+dcache.inotify-generation.backlog.per-door = 1000
+
+#  When sending inotify events to a door, multiple events may be sent
+#  in a single message.  Doing so improves the throughput by reducing
+#  overhead for sending a single event.  Note that batching only
+#  occurs in overload situation.
+#
+#  Setting this value too small increases the likelihood of events
+#  being lost during overload situation.  Setting this value too large
+#  results in cell messaging being slow to respond and (in extreme
+#  cases) running out of memory.
+#
+dcache.inotify-generation.message-batch-size = 100
+
 #  -----------------------------------------------------------------------
 #  ---- Unused properties
 #  -----------------------------------------------------------------------

skel/share/defaults/pnfsmanager.properties

@@ -182,6 +182,16 @@ pnfsmanager.topic.upload-cancelled = ${dcache.topic.upload-cancelled}
 # Comma separated list of cell addresses to which to send notifications when a partially uploaded file is deleted
 pnfsmanager.destination.cancel-upload-notification = ${pnfsmanager.topic.upload-cancelled}
 
+#  ---- Support for inotify events
+pnfsmanager.inotify-generation.enable = \
+    ${dcache.inotify-generation.enable}
+pnfsmanager.inotify-generation.backlog.initial = \
+    ${dcache.inotify-generation.backlog.initial}
+pnfsmanager.inotify-generation.backlog.per-door = \
+    ${dcache.inotify-generation.backlog.per-door}
+pnfsmanager.inotify-generation.message-batch-size = \
+    ${dcache.inotify-generation.message-batch-size}
+
 (immutable)pnfsmanager.destination.flush-notification-when-space-reservation-is-true=${pnfsmanager.service.spacemanager}
 (immutable)pnfsmanager.destination.flush-notification-when-space-reservation-is-false=
 

skel/share/services/pnfsmanager.batch

@@ -31,4 +31,5 @@ create org.dcache.cells.UniversalSpringCell ${pnfsmanager.cell.name} \
    "classpath:diskCacheV111/namespace/pnfsmanager-chimera.xml \
         -consume=${pnfsmanager.cell.consume} -subscribe=${pnfsmanager.cell.subscribe} \
         -namespace-provider=org.dcache.chimera.namespace.ChimeraNameSpaceProviderFactory \
+        -profiles=inotify-${pnfsmanager.inotify-generation.enable} \
        "