webdav: support redirecting to next replica for NOT FOUND or FORBIDDE…

…N responses

To support the concept of federation, clients must survive catalogues or other
starting points containing incomplete or out-of-date information.  In particular,
a starting point could redirect a client requesting some data to a storage system
that no longer contains that data or that applies additional constraints on which
users may read that file.

A normal WebDAV implementation would return an error to the client, so failing
the request.  This is undesirable as, no matter how active the starting point is
at maintaining its state, there is always the possibility for the information to
become stale.

The solution adopted by CERN is to encode a stack of replicas in the query-string
part of the request.  With WebDAV, this is not used for anything, so can carry
this extra information.

This patch adds support for redirecting the client to the next replica, if there
is an error and an additional replica is known.

Target: master
Request: 2.7
Request: 2.6
Patch: http://rb.dcache.org/r/6359
Acked-by: Gerd Behrmann

modules/dcache-webdav/src/main/java/org/dcache/webdav/HttpManagerFactory.java

@@ -3,11 +3,14 @@
 import io.milton.config.HttpManagerBuilder;
 import io.milton.http.HttpManager;
 import io.milton.http.webdav.DefaultWebDavResponseHandler;
+import io.milton.http.webdav.WebDavResponseHandler;
 import org.springframework.beans.factory.FactoryBean;
 import org.springframework.core.io.Resource;
 
 import java.io.IOException;
 
+import org.dcache.webdav.federation.FederationResponseHandler;
+
 public class HttpManagerFactory extends HttpManagerBuilder implements FactoryBean
 {
     private Resource _templateResource;
@@ -17,7 +20,8 @@ public class HttpManagerFactory extends HttpManagerBuilder implements FactoryBea
     public Object getObject() throws Exception
     {
         DcacheResponseHandler dcacheResponseHandler = new DcacheResponseHandler();
-        setWebdavResponseHandler(dcacheResponseHandler);
+        WebDavResponseHandler handler = new FederationResponseHandler(dcacheResponseHandler);
+        setWebdavResponseHandler(handler);
 
         init();
 

modules/dcache-webdav/src/main/java/org/dcache/webdav/federation/FederationResponseHandler.java

@@ -0,0 +1,70 @@
+package org.dcache.webdav.federation;
+
+import io.milton.http.AbstractWrappingResponseHandler;
+import io.milton.http.Request;
+import io.milton.http.Response;
+import io.milton.http.webdav.WebDavResponseHandler;
+import io.milton.resource.Resource;
+
+/**
+ * This class implements support for the "Global Access Service" feature that
+ * allows seamless access to a global federation of storage.  This is achieved
+ * by the storage system responding to certain local failures by redirecting
+ * the client to the next replica from a supplied stack of replicas.  This
+ * process is described here:
+ *
+ *     https://svnweb.cern.ch/trac/lcgdm/wiki/Dpm/WebDAV/Extensions
+ *
+ * The client supplies a carefully crafted URL with additional metadata encoded
+ * as a query string.  The format for this query string is described in the
+ * ReplicaInfo class.  If dCache would return a NOT_FOUND or FORBIDDEN
+ * response, this is replaced by a MOVED_TEMPORARILY response with the next
+ * replica in the stack as the location.  A list of the failures that the
+ * client has already experienced are propagated as part of the next request.
+ * <p>
+ * Normally, the final replica in the stack of supplied replicas is the
+ * catalogue itself.  This is to allow the catalogue to update its internal
+ * cache with the failed responses and suggest additional replicas to the
+ * client, if any.
+ * <p>
+ * Because of this, we do not anticipate receiving a federated request that
+ * fails with no next replica to redirect; this scenario is not described in the
+ * above wiki page.  Under these circumstances, dCache will allow the FORBIDDEN
+ * or NOT_FOUND error response to propagate back to the client.
+ * <p>
+ * If the client supplies no query string, or the query string is malformed then
+ * dCache will treat the request as a normal request and all errors will be
+ * propagated back to the client.
+ */
+public class FederationResponseHandler extends AbstractWrappingResponseHandler
+{
+
+    public FederationResponseHandler(WebDavResponseHandler wrapped)
+    {
+        super(wrapped);
+    }
+
+    @Override
+    public void respondNotFound(Response response, Request request)
+    {
+        ReplicaInfo info = ReplicaInfo.forRequest(request);
+
+        if (info.hasNext()) {
+            super.respondRedirect(response, request, info.buildLocationWhenNotFound());
+        } else {
+            super.respondNotFound(response, request);
+        }
+    }
+
+    @Override
+    public void respondForbidden(Resource resource, Response response, Request request)
+    {
+        ReplicaInfo info = ReplicaInfo.forRequest(request);
+
+        if (info.hasNext()) {
+            super.respondRedirect(response, request, info.buildLocationWhenForbidden());
+        } else {
+            super.respondForbidden(resource, response, request);
+        }
+    }
+}

modules/dcache-webdav/src/main/java/org/dcache/webdav/federation/ReplicaInfo.java

@@ -0,0 +1,223 @@
+package org.dcache.webdav.federation;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.escape.Escaper;
+import com.google.common.net.PercentEscaper;
+import io.milton.http.Request;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+/**
+ * This class represents the information passed by the client in the URL when
+ * a request comes from the GlobalAccessService.  The format is documented here:
+ * <p>
+ * https://svnweb.cern.ch/trac/lcgdm/wiki/Dpm/WebDAV/Extensions#GlobalAccessService
+ * <p>
+ * Format is that the query part of a URL contains a list key-value pairs; the
+ * each key and value is joined by '=' and the key-value pairs are joined by
+ * '&'s.  This is the normal format used by web-forms when submitting results
+ * via a GET request.
+ * <p>
+ * Certain keys are recognised and the values have the following semantics:
+ * <pre>
+ *   rid        the replica-id for the replica being requested,
+ *   forbidden  a comma-list of replica-id for replicas that have been tried
+ *              and that would have failed with a 403 FORBIDDEN response,
+ *   notfound   a comma-list of replicas-id for replicas that have been tried
+ *              and that would have failed with a 404 NOT FOUND response,
+ *   r&lt;index>   a comma-separated pair of replica-id and URL, respectively.
+ *              This item repeats a replica that is still to be attempted.
+ * </pre>
+ * <p>
+ * For the {@literal r&lt;index>} fields, {@literal &lt;index>} is some
+ * positive integer.  The values of {@literal r&lt;index>} do not repeat.
+ * Considered together, all {@literal r&lt;index>} fields represent a stack of
+ * replicas that are still to be attempted, with {@literal r1} representing the
+ * next replica.
+ * <p>
+ * Note that creating an object is a light-weight operation.  The computational
+ * effort of parsing the supplied information happens when {@code #hasNext}
+ * method is called the first time.  This call must happen before
+ * {@code #buildLocationWhenNotFound} or {@code #buildLocationWhenForbidden} is
+ * called.
+ */
+public class ReplicaInfo
+{
+    private static final Logger LOG = LoggerFactory.getLogger(ReplicaInfo.class);
+
+    private static final ReplicaInfo EMPTY_INFO = new ReplicaInfo();
+    private static final Splitter ON_FIRST_COMMA = Splitter.on(',')
+            .trimResults().limit(2);
+
+    // Equivalent to Guava's uriQueryStringEscaper(false), but this hasn't been
+    // released yet.
+    private static final Escaper QUERY_STRING_ESCAPER =
+            new PercentEscaper("-._~!$'()*,;@:/?", false);
+
+    // Used to escape the schema and path part of the redirected URI.
+    private static final Escaper SCHEMA_AND_PATH_ESCAPER =
+            new PercentEscaper("-._~!$'()*,;@:/", false);
+
+    private final Map<String,String> _parameters;
+
+    private boolean _isParsed;
+    private String _nextReplica;
+    private String _ourId;
+    private List<String> _remainingReplicas = new ArrayList<>();
+
+    public static ReplicaInfo forRequest(Request request)
+    {
+        Map<String,String> parameters = request.getParams();
+        String r1 = parameters.get("r1");
+
+        if (isNullOrEmpty(parameters.get("rid"))
+                || isNullOrEmpty(r1)
+                || r1.indexOf(',') == -1) {
+            LOG.trace("returning empty QueryStringInfo for request");
+            return EMPTY_INFO;
+        } else {
+            LOG.trace("returning non-empty QueryStringInfo for request");
+            return new ReplicaInfo(request);
+        }
+    }
+
+    private ReplicaInfo()
+    {
+        _parameters = null;
+        _isParsed = true;
+    }
+
+    private ReplicaInfo(Request request)
+    {
+        _parameters = request.getParams();
+    }
+
+    private void parseParameters()
+    {
+        _ourId = _parameters.get("rid");
+
+        String replica;
+        for (int index = 1;
+                (replica = _parameters.get("r"+index)) != null;
+                index++) {
+            if (_nextReplica == null) {
+                _nextReplica = replica;
+            } else {
+                _remainingReplicas.add(replica);
+            }
+        }
+
+        _isParsed = true;
+    }
+
+    public boolean hasNext()
+    {
+        if (!_isParsed) {
+            parseParameters();
+        }
+
+        return _ourId != null && _nextReplica != null;
+    }
+
+    private StringBuilder buildNextReplicaLocation()
+    {
+        StringBuilder sb = new StringBuilder();
+
+        List<String> nextReplica =
+                Lists.newArrayList(ON_FIRST_COMMA.split(_nextReplica));
+        sb.append(SCHEMA_AND_PATH_ESCAPER.escape(nextReplica.get(1)));
+        sb.append("?rid=").append(QUERY_STRING_ESCAPER.escape(nextReplica.get(0)));
+
+        return sb;
+    }
+
+    private StringBuilder addRemainingReplicas(StringBuilder sb)
+    {
+        int index = 1;
+
+        for(String url : this._remainingReplicas) {
+            sb.append('&').append('r').append(index++);
+            sb.append('=').append(QUERY_STRING_ESCAPER.escape(url));
+        }
+
+        return sb;
+    }
+
+    public String buildLocationWhenNotFound()
+    {
+        checkState(_isParsed && _ourId != null && _nextReplica != null);
+
+        StringBuilder sb = buildNextReplicaLocation();
+
+        sb.append(ampersandValueOrEmpty("forbidden"));
+        sb.append('&').append(getAppendedField("notfound", _ourId));
+
+        return addRemainingReplicas(sb).toString();
+    }
+
+    public String buildLocationWhenForbidden()
+    {
+        checkState(_isParsed && _ourId != null && _nextReplica != null);
+
+        StringBuilder sb = buildNextReplicaLocation();
+
+        sb.append('&').append(getAppendedField("forbidden", _ourId));
+        sb.append(ampersandValueOrEmpty("notfound"));
+
+        return addRemainingReplicas(sb).toString();
+    }
+
+    private CharSequence ampersandValueOrEmpty(String name)
+    {
+        String item = _parameters.get(name);
+
+        if (item == null) {
+            return "";
+        }
+
+        StringBuilder sb = new StringBuilder();
+        sb.append('&').append(name);
+        sb.append('=').append(QUERY_STRING_ESCAPER.escape(item));
+        return sb;
+    }
+
+    private CharSequence getAppendedField(String name, String item)
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(name).append('=');
+
+        String existing = _parameters.get(name);
+        if (existing != null) {
+            sb.append(QUERY_STRING_ESCAPER.escape(existing)).append(',');
+        }
+
+        sb.append(item);
+
+        return sb;
+    }
+
+    @Override
+    public String toString()
+    {
+        if (_parameters == null) {
+            return "<EMPTY>";
+        }
+
+        if (!_isParsed) {
+            return "<NOT PARSED>";
+        }
+
+        return "ourId=" + _ourId + ", next="+ _nextReplica + ", remaining=" +
+                _remainingReplicas;
+    }
+}

modules/dcache-webdav/src/main/resources/org/dcache/webdav/webdav.xml

@@ -103,7 +103,6 @@
       <property name="rootPath" value="${webdav.root}"/>
   </bean>
 
-
   <bean id="logging-filter" class="org.dcache.webdav.LoggingFilter">
       <description>Logs all requests</description>
   </bean>