dcache,pnfs: implement qos policy support (qos engine 3)

Motivation:

Implement the rule engine extension to QoS services.

Modification:

Adds vehicles and message handling to PnfsManager,
along with admin commands supporting policies.

Also adds FileAttribute support to the PoolManager
message and to the WriteHandleImpl on the Pool.

Result:

Support for qos file policies is now present
in the dCache infrastructure.  The next patches
will modify the qos-related services.

NOTE:  This  patch which forces
an update of the pools;

Target: master
Patch: https://rb.dcache.org/r/14066/
Depends-on: #14065
Acked-by: Tigran

modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml

@@ -50,6 +50,7 @@
       <property name="quotaSystem" ref="quota-system"/>
       <property name="quotaEnabled" value="${pnfsmanager.enable.quota}"/>
       <property name="useParentHashOnCreate" value="${pnfsmanager.use-parent-hash-on-create}"/>
+      <property name="qosManager" ref="qos-manager"/>
   </bean>
 
     <bean id="data-source" class="org.dcache.db.AlarmEnabledDataSource" destroy-method="close">
@@ -151,6 +152,11 @@
       <constructor-arg value="#{ T(diskCacheV111.util.RetentionPolicy).getRetentionPolicy('${pnfsmanager.default-retention-policy}') }"/>
   </bean>
 
+  <bean id="qos-manager" class="org.dcache.chimera.qos.JdbcQos" depends-on="liquibase">
+    <description>Quota</description>
+    <constructor-arg ref="data-source"/>
+  </bean>
+
   <bean id="name-space-provider"
         class="org.dcache.chimera.namespace.ChimeraNameSpaceProvider">
       <description>Name space provider</description>

modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/PnfsCreateEntryMessage.java

@@ -15,6 +15,8 @@
 import static org.dcache.namespace.FileAttribute.OWNER;
 import static org.dcache.namespace.FileAttribute.OWNER_GROUP;
 import static org.dcache.namespace.FileAttribute.PNFSID;
+import static org.dcache.namespace.FileAttribute.QOS_POLICY;
+import static org.dcache.namespace.FileAttribute.QOS_STATE;
 import static org.dcache.namespace.FileAttribute.RETENTION_POLICY;
 import static org.dcache.namespace.FileAttribute.SIMPLE_TYPE;
 import static org.dcache.namespace.FileAttribute.SIZE;
@@ -43,14 +45,14 @@ public class PnfsCreateEntryMessage extends PnfsSetFileAttributes {
     public static final EnumSet INVALID_CREATE_DIRECTORY_ATTRIBUTES =
           EnumSet.of(CACHECLASS, CHECKSUM, CREATION_TIME, FLAGS, HSM,
                 LOCATIONS, NLINK, PNFSID, RETENTION_POLICY, SIMPLE_TYPE,
-                SIZE, STORAGECLASS, STORAGEINFO);
+                SIZE, STORAGECLASS, STORAGEINFO, QOS_POLICY, QOS_STATE);
     public static final EnumSet INVALID_CREATE_FILE_ATTRIBUTES =
           EnumSet.of(CACHECLASS, CREATION_TIME, NLINK, PNFSID, STORAGECLASS,
                 STORAGEINFO, SIMPLE_TYPE);
     public static final EnumSet INVALID_CREATE_SYM_LINK_ATTRIBUTES =
           EnumSet.of(ACCESS_LATENCY, CACHECLASS, CHECKSUM, CREATION_TIME,
                 FLAGS, HSM, LOCATIONS, NLINK, PNFSID, RETENTION_POLICY,
-                SIZE, STORAGECLASS, STORAGEINFO, SIMPLE_TYPE);
+                SIZE, STORAGECLASS, STORAGEINFO, SIMPLE_TYPE, QOS_POLICY, QOS_STATE);
 
     public PnfsCreateEntryMessage(String path, FileAttributes attributes) {
         this(path, attributes, Collections.emptySet());
@@ -62,7 +64,7 @@ public PnfsCreateEntryMessage(String path, FileAttributes attributes,
               EnumSet.of(OWNER, OWNER_GROUP, MODE, TYPE, SIZE,
                     CREATION_TIME, ACCESS_TIME, MODIFICATION_TIME,
                     PNFSID, STORAGEINFO, STORAGECLASS, CACHECLASS, HSM,
-                    ACCESS_LATENCY, RETENTION_POLICY))));
+                    ACCESS_LATENCY, RETENTION_POLICY, QOS_POLICY, QOS_STATE))));
         checkArgument(attributes.isDefined(TYPE));
 
         switch (attributes.getFileType()) {

modules/dcache-vehicles/src/main/java/org/dcache/vehicles/qos/PnfsManagerAddQoSPolicyMessage.java

@@ -0,0 +1,76 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a  world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes.  All documents and software
+available from this server are protected under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information  (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the  URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED  WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED  WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR  ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY  OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT  OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE  POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws.  Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+ */
+package org.dcache.vehicles.qos;
+
+import org.dcache.qos.QoSPolicy;
+
+public class PnfsManagerAddQoSPolicyMessage extends PnfsManagerQoSPolicyMessage {
+
+    private static final long serialVersionUID = -7885598791426942261L;
+    private final QoSPolicy policy;
+
+    public PnfsManagerAddQoSPolicyMessage(QoSPolicy policy) {
+        this.policy = policy;
+    }
+
+    public QoSPolicy getPolicy() {
+        return policy;
+    }
+}

modules/dcache-vehicles/src/main/java/org/dcache/vehicles/qos/PnfsManagerGetQoSPolicyMessage.java

@@ -0,0 +1,85 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a  world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes.  All documents and software
+available from this server are protected under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information  (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the  URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED  WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED  WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR  ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY  OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT  OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE  POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws.  Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+ */
+package org.dcache.vehicles.qos;
+
+import org.dcache.qos.QoSPolicy;
+
+public class PnfsManagerGetQoSPolicyMessage extends PnfsManagerQoSPolicyMessage {
+
+    private static final long serialVersionUID = 1132558931827401229L;
+    private final String policyName;
+    private QoSPolicy policy;
+
+    public String getPolicyName() {
+        return policyName;
+    }
+
+    public PnfsManagerGetQoSPolicyMessage(String policyName) {
+        this.policyName = policyName;
+    }
+
+    public QoSPolicy getPolicy() {
+        return policy;
+    }
+
+    public void setPolicy(QoSPolicy policy) {
+        this.policy = policy;
+    }
+}

modules/dcache-vehicles/src/main/java/org/dcache/vehicles/qos/PnfsManagerGetQoSPolicyStatsMessage.java

@@ -0,0 +1,77 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a  world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes.  All documents and software
+available from this server are protected under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information  (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the  URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED  WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED  WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR  ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY  OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT  OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE  POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws.  Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+ */
+package org.dcache.vehicles.qos;
+
+import java.util.List;
+import org.dcache.qos.QoSPolicyStat;
+
+public class PnfsManagerGetQoSPolicyStatsMessage extends PnfsManagerQoSPolicyMessage {
+
+    private static final long serialVersionUID = -6481629980312661139L;
+    private List<QoSPolicyStat> policyStats;
+
+    public List<QoSPolicyStat> getPolicyStats() {
+        return policyStats;
+    }
+
+    public void setPolicyStats(List<QoSPolicyStat> policyStats) {
+        this.policyStats = policyStats;
+    }
+}

modules/dcache-vehicles/src/main/java/org/dcache/vehicles/qos/PnfsManagerInvalidateQoSPolicyMessage.java

@@ -0,0 +1,73 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a  world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes.  All documents and software
+available from this server are protected under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information  (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the  URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED  WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED  WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR  ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY  OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT  OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE  POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws.  Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+ */
+package org.dcache.vehicles.qos;
+
+public class PnfsManagerInvalidateQoSPolicyMessage extends PnfsManagerQoSPolicyMessage {
+
+    private final Integer id;
+
+    public PnfsManagerInvalidateQoSPolicyMessage(Integer id) {
+        this.id = id;
+    }
+
+    public Integer getPolicyId() {
+        return id;
+    }
+}