dcache-qos: throttle verifier requests to engine (qos rule engine 5)

Motivation:

Implement the rule engine extension to QoS services.

Modification:

Small patch which adds throttling (in the
form of a semaphore which is configurable)
to the concurrent requests for requirements
send to the engine.

Result:

Less pressure on the QoS engine.

Target: master
Patch: https://rb.dcache.org/r/14073
Depends-on: #14072
Acked-by: Tigran

modules/dcache-qos/src/main/java/org/dcache/qos/services/verifier/handlers/VerifyOperationHandler.java

@@ -74,6 +74,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import java.time.temporal.ChronoUnit;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
 import org.dcache.alarms.AlarmMarkerFactory;
 import org.dcache.alarms.PredefinedAlarm;
 import org.dcache.qos.QoSException;
@@ -198,6 +199,11 @@ private static void sendOutOfSyncAlarm() {
      */
     private ExecutorService updateExecutor;
 
+    /**
+     * Throttles requests to engine.
+     */
+    private Semaphore semaphore;
+
     /**
      * Statistics.
      */
@@ -330,7 +336,13 @@ public void handleVerification(PnfsId pnfsId) {
             FileQoSUpdate data = new FileQoSUpdate(operation.getPnfsId(),
                   operation.getPrincipalPool(),
                   operation.getMessageType());
-            requirements = requirementsListener.fileQoSRequirementsRequested(data);
+
+            semaphore.acquire();
+            try {
+                requirements = requirementsListener.fileQoSRequirementsRequested(data);
+            } finally {
+                semaphore.release();
+            }
 
             if (requirements == null) {
                 if (operation.getMessageType() == CLEAR_CACHE_LOCATION) {
@@ -500,6 +512,10 @@ public void setManager(VerifyOperationManager manager) {
         this.manager = manager;
     }
 
+    public void setMaxConcurrentRequirementRequests(int maxRequests) {
+        semaphore = new Semaphore(maxRequests);
+    }
+
     public void setPoolInfoMap(PoolInfoMap poolInfoMap) {
         this.poolInfoMap = poolInfoMap;
     }

modules/dcache-qos/src/main/resources/org/dcache/qos/qos-verifier.xml

@@ -286,6 +286,7 @@
     <property name="manager" ref="verify-operation-manager"/>
     <property name="scanRecordMap" ref="verify-scan-record-map"/>
     <property name="statusVerifier" ref="file-status-verifier"/>
+    <property name="maxConcurrentRequirementRequests" value="${qos.limits.messages.max-requirement-requests}"/>
     <property name="updateExecutor">
       <bean class="org.dcache.util.BoundedCachedExecutor" destroy-method="shutdownNow">
         <constructor-arg value="${qos.limits.verifier.submit-threads}"/>

skel/share/defaults/dcache.properties

@@ -83,6 +83,9 @@ dcache.log.zookeeper.max-history=30
 # How many days to keep resilience logs
 dcache.log.resilience.max-history=30
 
+How many days to keep resilience logs
+dcache.log.qos.max-history=30
+
 # Host on which the remote log server will run
 # relative to this dCache installation
 #