Add support for simple read-only WebDAV properties

This patch adds initial support in dCache for supporting WebDAV.  Only
three read-only file-based properties are added (AccessLatency,
RetentionPolicy and Checksums), but I've marked in the code where dCache
is notified of any requested changes to property values.

Some observations:

1. Milton provides two levels of abstraction when answering queries
   about properties.  One (implemented in this patch) is based on the
   Resource implementing the MultiNamespaceCustomPropertyResource
   interface.  The other involves creating classes for handling property
   queries and updates (the former is implemented based on the latter).
   The latter has the benefit of separating the property-handling code
   from the other aspects of file and directory management.  If the
   property-handling code grows to dominate the
   Dcache{File,Directory}Resource classes then this might be a
   convenient way of factoring out that code.

2. Milton asks for the Resource before parsing the XML body of the
   client's PROPFIND or PROPPATCH request.  After processing the XML,
   Milton iterates over the list of properties to modify, query or
   remove.  Static metadata is obtained before any operation, so it can
   filter-out attempts to query an unknown property, or to modify a
   read-only property.

   The combination of processing the XML and processing the properties
   iteratively means the webdav door must either fetch all potentially
   useful attributes when building the Resource, or it will send a
   message for each requested property.  Neither is terribly appealing;
   I went for the former for now, but we should fix this in the future.

3. If a client sends a malformed XML statement then Milton will log a
   stack-trace.  This seems overkill, but I've left that as-is for now
   as it may need to be fixed upstream.

4. I've unilaterally adopted the SRM namespace for reporting access
   latency and retention policy of the file.

The patch also modifies the RFC-3230 support to honour the 'Want-Digest'
request header only if the method is GET or HEAD.  RFC-3230 is only
defined for GET (and HEAD must return the same headers as GET would have
returned).

The patch adds some support in FileAttributes for returning a Guava
Optional object.  The semantics are:

    get<ATTRIBUTE-NAME> method is use when a missing attribute is a bug.

    get<ATTRIBUTE-NAME>IfPresent is used when a missing attribute is not
       (necessarily) a bug.

Target: master
Requires-notes: yes
Requires-book: yes
Patch: http://rb.dcache.org/r/6270/
Acked-by: Gerd Behrmann

modules/common/src/main/java/org/dcache/util/Checksums.java

@@ -2,7 +2,9 @@
 
 import com.google.common.base.Function;
 import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
 import com.google.common.base.Splitter;
+import com.google.common.collect.Collections2;
 import com.google.common.collect.Maps.EntryTransformer;
 import com.google.common.collect.Sets;
 import org.slf4j.Logger;
@@ -68,12 +70,9 @@ private Checksums()
 
     /**
      * This Function maps an instance of Checksum to the corresponding
-     * fragment of an RFC 3230 response.  For further details, see:
-     *
-     *     http://tools.ietf.org/html/rfc3230
-     *     http://www.iana.org/assignments/http-dig-alg/http-dig-alg.xml
+     * fragment of an RFC 3230 response.
      */
-    private static final Function<Checksum,String> FOR_RFC3230 =
+    private static final Function<Checksum,String> TO_RFC3230_FRAGMENT =
             new Function<Checksum,String>() {
         @Override
         public String apply(Checksum f)
@@ -93,17 +92,23 @@ public String apply(Checksum f)
         }
     };
 
-
     /**
-     * Encode the supplied checksum values as a comma-separated list as
-     * per RFC-3230.  The resulting string may be used as the value part of
-     * a Digest HTTP header in the response to a GET or HEAD request.
+     * This Function maps a collection of Checksum objects to the corresponding
+     * RFC 3230 string.  For further details, see:
+     *
+     *     http://tools.ietf.org/html/rfc3230
+     *     http://www.iana.org/assignments/http-dig-alg/http-dig-alg.xml
      */
-    public static String rfc3230Encoded(Collection<Checksum> checksums)
-    {
-        Iterable<String> rfc3230Parts = transform(checksums, FOR_RFC3230);
-        return Joiner.on(',').skipNulls().join(rfc3230Parts);
-    }
+    public static final Function<Collection<Checksum>,String> TO_RFC3230 =
+            new Function<Collection<Checksum>,String>() {
+        @Override
+        public String apply(Collection<Checksum> checksums)
+        {
+            Iterable<String> parts = transform(checksums, TO_RFC3230_FRAGMENT);
+            return Joiner.on(',').skipNulls().join(parts);
+        }
+    };
+
 
     /**
      * Parse the RFC-3230 Digest response header value.  If there is no

modules/common/src/test/java/org/dcache/util/ChecksumsTests.java

@@ -257,7 +257,7 @@ private void givenSet(ChecksumBuilder... builders)
 
     private void whenGeneratingRfc3230ForSetOfChecksums()
     {
-        _rfc3230 = Checksums.rfc3230Encoded(_checksums);
+        _rfc3230 = Checksums.TO_RFC3230.apply(_checksums);
     }
 
     private ChecksumBuilder checksum()

modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheFileResource.java

@@ -1,42 +1,82 @@
 package org.dcache.webdav;
 
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import io.milton.http.Auth;
 import io.milton.http.Range;
 import io.milton.http.Request;
 import io.milton.http.exceptions.BadRequestException;
 import io.milton.http.exceptions.ConflictException;
 import io.milton.http.exceptions.NotAuthorizedException;
+import io.milton.property.PropertySource.PropertyMetaData;
+import io.milton.property.PropertySource.PropertySetException;
 import io.milton.resource.DeletableResource;
 import io.milton.resource.GetableResource;
+import io.milton.resource.MultiNamespaceCustomPropertyResource;
+
+import javax.xml.namespace.QName;
 
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.FileNameMap;
 import java.net.URISyntaxException;
 import java.net.URLConnection;
+import java.util.List;
 import java.util.Map;
 
+import diskCacheV111.util.AccessLatency;
 import diskCacheV111.util.CacheException;
 import diskCacheV111.util.FileNotFoundCacheException;
 import diskCacheV111.util.FsPath;
 import diskCacheV111.util.NotInTrashCacheException;
 import diskCacheV111.util.PermissionDeniedCacheException;
+import diskCacheV111.util.RetentionPolicy;
 
-import org.dcache.namespace.FileAttribute;
-import org.dcache.util.Checksums;
 import org.dcache.vehicles.FileAttributes;
 
+import static io.milton.property.PropertySource.PropertyAccessibility.READ_ONLY;
+import static org.dcache.util.Checksums.TO_RFC3230;
+
 /**
  * Exposes regular dCache files as resources in the Milton WebDAV
  * framework.
  */
 public class DcacheFileResource
     extends DcacheResource
-    implements GetableResource, DeletableResource
+    implements GetableResource, DeletableResource,
+    MultiNamespaceCustomPropertyResource
 {
     private static final FileNameMap MIME_TYPE_MAP =
         URLConnection.getFileNameMap();
 
+    private static final String DCACHE_NAMESPACE_URI =
+            "http://www.dcache.org/2013/webdav";
+
+    // We use the SRM 2.2 WSDL's TargetNamespace for the WebDAV properties
+    // associated with SRM concepts.
+    private static final String SRM_NAMESPACE_URI =
+            "http://srm.lbl.gov/StorageResourceManager";
+
+    /*
+     * Our dCache WebDAV properties.
+     */
+    private static final String PROPERTY_CHECKSUMS = "Checksums";
+    /*
+     * Our SRM WebDAV properties.
+     */
+    private static final String PROPERTY_ACCESS_LATENCY = "AccessLatency";
+    private static final String PROPERTY_RETENTION_POLICY = "RetentionPolicy";
+
+    private static final ImmutableMap<QName,PropertyMetaData> PROPERTY_METADATA =
+            new ImmutableMap.Builder<QName,PropertyMetaData>()
+                    .put(new QName(SRM_NAMESPACE_URI, PROPERTY_ACCESS_LATENCY),
+                            new PropertyMetaData(READ_ONLY, AccessLatency.class))
+                    .put(new QName(SRM_NAMESPACE_URI, PROPERTY_RETENTION_POLICY),
+                            new PropertyMetaData(READ_ONLY, RetentionPolicy.class))
+                    .put(new QName(DCACHE_NAMESPACE_URI, PROPERTY_CHECKSUMS),
+                            new PropertyMetaData(READ_ONLY, String.class))
+                    .build();
+
     public DcacheFileResource(DcacheResourceFactory factory,
                               FsPath path, FileAttributes attributes)
     {
@@ -114,10 +154,71 @@ public void delete()
 
     public String getRfc3230Digest()
     {
-        if(_attributes.isDefined(FileAttribute.CHECKSUM)) {
-            return Checksums.rfc3230Encoded(_attributes.getChecksums());
-        } else {
-            return "";
+        return _attributes.getChecksumsIfPresent().transform(TO_RFC3230).or("");
+    }
+
+    @Override
+    public Object getProperty(QName qname)
+    {
+        switch (qname.getNamespaceURI()) {
+        case DCACHE_NAMESPACE_URI:
+            return getDcacheProperty(qname.getLocalPart());
+        case SRM_NAMESPACE_URI:
+            return getSrmProperty(qname.getLocalPart());
+        }
+
+        // Milton filters out unknown properties by checking with the
+        // PropertyMetaData, so if we get here then it's a bug.
+        throw new RuntimeException("unknown property " + qname);
+    }
+
+    private Object getDcacheProperty(String localPart)
+    {
+        switch(localPart) {
+        case PROPERTY_CHECKSUMS:
+            return _attributes.getChecksumsIfPresent().transform(TO_RFC3230).orNull();
+        }
+
+        throw new RuntimeException("unknown dCache property " + localPart);
+    }
+
+    private Object getSrmProperty(String localPart)
+    {
+        switch(localPart) {
+        case PROPERTY_ACCESS_LATENCY:
+            return _attributes.getAccessLatencyIfPresent().orNull();
+        case PROPERTY_RETENTION_POLICY:
+            return _attributes.getRetentionPolicyIfPresent().orNull();
         }
+
+        throw new RuntimeException("unknown SRM property " + localPart);
+    }
+
+    @Override
+    public void setProperty(QName qname, Object o) throws PropertySetException,
+            NotAuthorizedException
+    {
+        // Handle any updates here.
+
+        // We should not see any read-only or unknown properties as Milton
+        // discovers them from PropertyMetaData and filters out any attempt by
+        // end-users.
+        throw new RuntimeException("Attempt to update " +
+                (PROPERTY_METADATA.containsKey(qname) ? "read-only" : "unknown") +
+                "property " + qname);
+    }
+
+    @Override
+    public PropertyMetaData getPropertyMetaData(QName qname)
+    {
+        // Milton accepts null and PropertyMetaData.UNKNOWN to mean the
+        // property is unknown.
+        return PROPERTY_METADATA.get(qname);
+    }
+
+    @Override
+    public List<QName> getAllPropertyNames()
+    {
+        return PROPERTY_METADATA.keySet().asList();
     }
 }

modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java

@@ -12,6 +12,7 @@
 import io.milton.http.Request;
 import io.milton.http.ResourceFactory;
 import io.milton.resource.Resource;
+import io.milton.servlet.ServletRequest;
 import org.jboss.netty.handler.codec.http.HttpHeaders;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 import org.slf4j.Logger;
@@ -109,6 +110,11 @@ public class DcacheResourceFactory
         EnumSet.of(TYPE, PNFSID, CREATION_TIME, MODIFICATION_TIME, SIZE,
                    MODE, OWNER, OWNER_GROUP);
 
+    // Additional attributes needed for PROPFIND requests; e.g., to supply
+    // values for properties.
+    private static final Set<FileAttribute> PROPFIND_ATTRIBUTES =
+        EnumSet.of(CHECKSUM, ACCESS_LATENCY, RETENTION_POLICY);
+
     private static final String PROTOCOL_INFO_NAME = "Http";
     private static final int PROTOCOL_INFO_MAJOR_VERSION = 1;
     private static final int PROTOCOL_INFO_MINOR_VERSION = 1;
@@ -541,10 +547,7 @@ public DcacheResource getResource(FsPath path)
                 try {
                     PnfsHandler pnfs = new PnfsHandler(_pnfs, subject);
                     Set<FileAttribute> requestedAttributes =
-                            EnumSet.copyOf(REQUIRED_ATTRIBUTES);
-                    if (isDigestRequested()) {
-                        requestedAttributes.add(CHECKSUM);
-                    }
+                            buildRequestedAttributes();
                     FileAttributes attributes =
                         pnfs.getFileAttributes(path.toString(), requestedAttributes);
                     return getResource(path, attributes);
@@ -749,7 +752,7 @@ public List<DcacheResource> list(final FsPath path)
                 @Override
                 public Set<FileAttribute> getRequiredAttributes()
                 {
-                    return REQUIRED_ATTRIBUTES;
+                    return buildRequestedAttributes();
                 }
 
                 @Override
@@ -1067,12 +1070,43 @@ private void initializeTransfer(HttpTransfer transfer, Subject subject)
         transfer.setOverwriteAllowed(_isOverwriteAllowed);
     }
 
+    private Set<FileAttribute> buildRequestedAttributes()
+    {
+        Set<FileAttribute> attributes = EnumSet.copyOf(REQUIRED_ATTRIBUTES);
+
+        if (isDigestRequested()) {
+            attributes.add(CHECKSUM);
+        }
+
+        if (isPropfindRequest()) {
+            // FIXME: Unfortunately, Milton parses the request body after
+            // requesting the Resource, so we cannot know which additional
+            // attributes are being requested; therefore, we must request all
+            // of them.
+            attributes.addAll(PROPFIND_ATTRIBUTES);
+        }
+
+        return attributes;
+    }
+
     private static boolean isDigestRequested()
     {
-        // TODO: parse the Want-Digest to see if the requested digest(s) are
-        //       supported.  If not then we can omit fetching the checksum
-        //       values.
-        return HttpManager.request().getHeaders().containsKey("Want-Digest");
+        switch (HttpManager.request().getMethod()) {
+        case HEAD:
+        case GET:
+            // TODO: parse the Want-Digest to see if the requested digest(s) are
+            //       supported.  If not then we can omit fetching the checksum
+            //       values.
+            return HttpManager.request().getHeaders().containsKey("Want-Digest");
+        default:
+            return false;
+        }
+    }
+
+
+    private boolean isPropfindRequest()
+    {
+        return HttpManager.request().getMethod() == Request.Method.PROPFIND;
     }
 
 

modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java

@@ -50,6 +50,7 @@
 import org.dcache.vehicles.FileAttributes;
 
 import static java.util.Arrays.asList;
+import static org.dcache.util.Checksums.TO_RFC3230;
 import static org.dcache.util.StringMarkup.percentEncode;
 import static org.dcache.util.StringMarkup.quotedString;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.*;
@@ -673,12 +674,6 @@ private long write(RepositoryChannel file, ChannelBuffer channelBuffer)
     private static String buildDigest(MoverChannel<HttpProtocolInfo> file)
     {
         FileAttributes attributes = file.getFileAttributes();
-
-        if(attributes.isDefined(FileAttribute.CHECKSUM)) {
-            Set<Checksum> checksums = attributes.getChecksums();
-            return Checksums.rfc3230Encoded(checksums);
-        } else {
-            return "";
-        }
+        return attributes.getChecksumsIfPresent().transform(TO_RFC3230).or("");
     }
 }

modules/dcache/src/main/java/org/dcache/vehicles/FileAttributes.java

@@ -1,6 +1,8 @@
 package org.dcache.vehicles;
 
 import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+import com.google.common.base.Optional;
 
 import java.io.Serializable;
 import java.util.Collection;
@@ -184,6 +186,10 @@ public AccessLatency getAccessLatency() {
         return _accessLatency;
     }
 
+    public Optional<AccessLatency> getAccessLatencyIfPresent() {
+        return toOptional(ACCESS_LATENCY, _accessLatency);
+    }
+
     public long getAccessTime()
     {
         guard(ACCESS_TIME);
@@ -201,6 +207,10 @@ public Set<Checksum> getChecksums() {
         return _checksums;
     }
 
+    public Optional<Set<Checksum>> getChecksumsIfPresent() {
+        return toOptional(CHECKSUM, _checksums);
+    }
+
     /**
      * Get {@link FileType} corresponding to the file.
      * @return file type
@@ -266,6 +276,10 @@ public RetentionPolicy getRetentionPolicy() {
         return _retentionPolicy;
     }
 
+    public Optional<RetentionPolicy> getRetentionPolicyIfPresent() {
+        return toOptional(RETENTION_POLICY, _retentionPolicy);
+    }
+
     public long getSize() {
         guard(SIZE);
         return _size;
@@ -407,4 +421,10 @@ public String toString()
                 .omitNullValues()
                 .toString();
     }
+
+    private <T> Optional<T> toOptional(FileAttribute attribute, T value)
+    {
+        return isDefined(attribute) ? Optional.of(value) :
+                (Optional<T>)Optional.absent();
+    }
 }