dcache-frontend: support VO- or user-root relative paths in requests

Motivation:

See #7072
dcache-frontend: add extraction of effective root
for user to include in REST API for bulk/stage.

Modification:

The frontend resources had to be modified to
use the user root derived from the login
attributes in order to determine the
`target-prefix` which would allow for
resolution of the relative paths.  In
the case of a user-provided prefix
(`/api/v1/bulk-requests`) this is done
by trying to union the root and prefix
paths; for the `TAPE` resources, we
simply set the prefix to the root
under the covers.

On the bulk end, a little be of re-engineering
was necessary to make sure the users can
retrieve paths according to their
root expectations.  Thus, if the submitted
paths are originally relative, they
will get back all relative paths, including
those discovered through recursion. If
they submitted absolute paths, they should
see those instead.  The consistency is
arrived at by allowing the initial paths
to go into the database as given, but
derived/discovered paths are stored
as absolute and then given back through
the REST API according to the prefix
that has been set.

Result:

The RESTful interface supports
paths relative to the user root
as well as absolute paths for requests
to the bulk service.

Note:  a separate solution will be necessary
for the one-off `namespace` resource operations.

Target: master
Request: 9.0
Request: 8.2
Patch: https://rb.dcache.org/r/13937
Requires-notes: yes
Closes: #7072
Acked-by: Tigran

modules/dcache-bulk/src/main/java/org/dcache/services/bulk/job/PrestoreRequestContainerJob.java

@@ -295,6 +295,10 @@ private ListenableFuture perform(BulkRequestTarget target)
         checkForRequestCancellation();
         Long id = target.getId();
         FsPath path = target.getPath();
+        if (targetPrefix != null && !path.contains(targetPrefix)) {
+            path = computeFsPath(targetPrefix, target.getPath().toString());
+        }
+
         FileAttributes attributes = target.getAttributes();
 
         if (hasBeenCancelled(id, target.getPid(), path, attributes)) {

modules/dcache-bulk/src/main/java/org/dcache/services/bulk/store/jdbc/request/JdbcBulkRequestStore.java

@@ -66,17 +66,20 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import static org.dcache.services.bulk.BulkRequestStatus.STARTED;
 import static org.dcache.services.bulk.activity.BulkActivity.MINIMALLY_REQUIRED_ATTRIBUTES;
 import static org.dcache.services.bulk.util.BulkRequestTarget.NON_TERMINAL;
+import static org.dcache.services.bulk.util.BulkRequestTarget.PID.DISCOVERED;
 import static org.dcache.services.bulk.util.BulkRequestTarget.PLACEHOLDER_PNFSID;
 import static org.dcache.services.bulk.util.BulkRequestTarget.ROOT_REQUEST_PATH;
 import static org.dcache.services.bulk.util.BulkRequestTarget.State.CREATED;
 
+import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
 import diskCacheV111.util.CacheException;
+import diskCacheV111.util.FsPath;
 import diskCacheV111.util.PnfsHandler;
 import diskCacheV111.util.PnfsId;
 import java.util.ArrayList;
@@ -456,7 +459,7 @@ public void load() throws BulkStorageException {
          */
         requestTargetDao.delete(requestTargetDao.where().pids(PID.ROOT.ordinal()));
         requestTargetDao.delete(
-              requestTargetDao.where().pids(PID.DISCOVERED.ordinal()).state(NON_TERMINAL));
+              requestTargetDao.where().pids(DISCOVERED.ordinal()).state(NON_TERMINAL));
         requestTargetDao.update(
               requestTargetDao.where().pids(PID.INITIAL.ordinal()).state(NON_TERMINAL),
               requestTargetDao.set().state(CREATED).errorType(null).errorMessage(null));
@@ -489,7 +492,7 @@ public void reset(String uid) throws BulkStorageException {
          */
         LOGGER.trace("reset {}.", uid);
         requestTargetDao.delete(requestTargetDao.where().pids(PID.ROOT.ordinal()).ruids(uid));
-        requestTargetDao.delete(requestTargetDao.where().pids(PID.DISCOVERED.ordinal()).ruids(uid));
+        requestTargetDao.delete(requestTargetDao.where().pids(DISCOVERED.ordinal()).ruids(uid));
         requestTargetDao.update(requestTargetDao.where().pids(PID.INITIAL.ordinal()).ruids(uid),
               requestTargetDao.set().state(CREATED).errorType(null).errorMessage(null));
         requestDao.update(requestDao.where().uids(uid),
@@ -754,21 +757,29 @@ private BulkRequestInfo processInfo(BulkRequest stored, long offset) {
         BulkRequestInfo info = new BulkRequestInfo();
         BulkRequestStatusInfo status = stored.getStatusInfo();
         String uid = stored.getUid();
+        String prefixString = stored.getTargetPrefix();
         info.setUid(uid);
         info.setStatus(status.getStatus());
         info.setArrivedAt(status.getCreatedAt());
         info.setLastModified(status.getLastModified());
         info.setStartedAt(status.getStartedAt());
-        info.setTargetPrefix(stored.getTargetPrefix());
+        info.setTargetPrefix(prefixString);
+
         /*
          *  Order by id from offset.  Limit is 10000 per swatch.
          */
-        List<BulkRequestTargetInfo> targets =
-              requestTargetDao.get(
-                          requestTargetDao.where().rid(stored.getId()).offset(offset).notRootRequest()
-                                .sorter("request_target.id"), FETCH_SIZE).stream()
-                    .map(this::toRequestTargetInfo).collect(Collectors.toList());
-        info.setTargets(targets);
+        List<BulkRequestTarget> targets = new ArrayList<>(requestTargetDao.get(
+              requestTargetDao.where().rid(stored.getId()).offset(offset).notRootRequest()
+                    .sorter("request_target.id"), FETCH_SIZE));
+
+        FsPath prefix = Strings.emptyToNull(prefixString) == null ? null
+              : FsPath.create(prefixString);
+        boolean stripRecursive = targets.stream().anyMatch(t -> t.getPid() == DISCOVERED);
+
+        List<BulkRequestTargetInfo> targetInfo = targets.stream()
+              .map(t -> toRequestTargetInfo(t, stripRecursive? prefix : null)).collect(Collectors.toList());
+
+        info.setTargets(targetInfo);
         int size = targets.size();
         if (size == FETCH_SIZE) {
             info.setNextId(targets.get(size - 1).getId() + 1);
@@ -778,10 +789,14 @@ private BulkRequestInfo processInfo(BulkRequest stored, long offset) {
         return info;
     }
 
-    private BulkRequestTargetInfo toRequestTargetInfo(BulkRequestTarget target) {
+    private BulkRequestTargetInfo toRequestTargetInfo(BulkRequestTarget target,  FsPath prefix) {
         BulkRequestTargetInfo info = new BulkRequestTargetInfo();
         info.setId(target.getId());
-        info.setTarget(target.getPath().toString());
+        if (prefix != null && target.getPid() == DISCOVERED) {
+            info.setTarget(target.getPath().stripPrefix(prefix));
+        } else {
+            info.setTarget(target.getPath().toString());
+        }
         info.setState(target.getState().name());
         info.setSubmittedAt(target.getCreatedAt());
         info.setStartedAt(target.getStartedAt());

modules/dcache-bulk/src/main/java/org/dcache/services/bulk/store/jdbc/rtarget/JdbcBulkTargetStore.java

@@ -60,6 +60,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 package org.dcache.services.bulk.store.jdbc.rtarget;
 
 import static org.dcache.services.bulk.util.BulkRequestTarget.NON_TERMINAL;
+import static org.dcache.services.bulk.util.BulkRequestTarget.PID.DISCOVERED;
 import static org.dcache.services.bulk.util.BulkRequestTarget.State.CREATED;
 
 import java.util.List;
@@ -204,9 +205,13 @@ public void update(Long id, State state, String errorType, String errorMessage)
 
     private JdbcRequestTargetUpdate prepareUpdate(BulkRequestTarget target) {
         JdbcRequestTargetUpdate update = targetDao.set().pid(target.getPid())
-              .rid(target.getRid()).pnfsid(target.getPnfsId()).path(target.getPath())
+              .rid(target.getRid()).pnfsid(target.getPnfsId())
               .type(target.getType()).state(target.getState());
 
+        if (target.getId() == null || target.getPid() == DISCOVERED) {
+            update.path(target.getPath());
+        }
+
         switch (target.getState()) {
             case COMPLETED:
             case FAILED:

modules/dcache-frontend/src/main/java/org/dcache/restful/resources/bulk/BulkResources.java

@@ -59,6 +59,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  */
 package org.dcache.restful.resources.bulk;
 
+import static org.dcache.restful.util.HttpServletRequests.getUserRootAwareTargetPrefix;
 import static org.dcache.restful.util.JSONUtils.newBadRequestException;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -224,7 +225,7 @@ public Response submit(
         Subject subject = getSubject();
         Restriction restriction = getRestriction();
 
-        BulkRequest request = toBulkRequest(requestPayload);
+        BulkRequest request = toBulkRequest(requestPayload, this.request);
 
         /*
          *  Frontend sets the URL.  The backend service provides the UUID.
@@ -413,7 +414,7 @@ public static Restriction getRestriction() {
      * they are defined in the Bulk service as well.
      */
     @VisibleForTesting
-    static BulkRequest toBulkRequest(String requestPayload) {
+    static BulkRequest toBulkRequest(String requestPayload, HttpServletRequest httpServletRequest) {
         if (Strings.emptyToNull(requestPayload) == null) {
             throw new BadRequestException("empty request payload.");
         }
@@ -446,7 +447,11 @@ static BulkRequest toBulkRequest(String requestPayload) {
 
         string = removeEntry(map, String.class, "target_prefix", "target-prefix",
               "targetPrefix");
-        request.setTargetPrefix(string);
+        if (httpServletRequest != null) {
+            request.setTargetPrefix(getUserRootAwareTargetPrefix(httpServletRequest, string));
+        } else {
+            request.setTargetPrefix(string);
+        }
 
         string = removeEntry(map, String.class, "expand_directories", "expand-directories",
               "expandDirectories");

modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/ReleaseResources.java

@@ -59,6 +59,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  */
 package org.dcache.restful.resources.tape;
 
+import static org.dcache.restful.util.HttpServletRequests.getUserRootAwareTargetPrefix;
 import static org.dcache.restful.util.JSONUtils.newBadRequestException;
 import static org.dcache.restful.util.RequestUser.getRestriction;
 import static org.dcache.restful.util.RequestUser.getSubject;
@@ -172,6 +173,7 @@ public Response release(
          *  Frontend sets the URL.  The backend service provides the UUID.
          */
         request.setUrlPrefix(this.request.getRequestURL().toString());
+        request.setTargetPrefix(getUserRootAwareTargetPrefix(this.request, null));
 
         BulkRequestMessage message = new BulkRequestMessage(request, restriction);
         message.setSubject(subject);

modules/dcache-frontend/src/main/java/org/dcache/restful/resources/tape/StageResources.java

@@ -61,6 +61,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 
 import static org.dcache.restful.resources.bulk.BulkResources.getRestriction;
 import static org.dcache.restful.resources.bulk.BulkResources.getSubject;
+import static org.dcache.restful.util.HttpServletRequests.getUserRootAwareTargetPrefix;
 import static org.dcache.restful.util.JSONUtils.newBadRequestException;
 
 import com.google.common.base.Strings;
@@ -340,6 +341,7 @@ private BulkRequest toBulkRequest(String requestPayload) {
         request.setClearOnFailure(false);
         request.setClearOnSuccess(false);
         request.setActivity("STAGE");
+        request.setTargetPrefix(getUserRootAwareTargetPrefix(this.request, null));
 
         try {
             JSONObject reqPayload = new JSONObject(requestPayload);

modules/dcache-frontend/src/main/java/org/dcache/restful/util/HttpServletRequests.java

@@ -18,6 +18,9 @@
  */
 package org.dcache.restful.util;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import diskCacheV111.util.FsPath;
 import java.util.Set;
 import javax.security.auth.Subject;
 import javax.servlet.http.HttpServletRequest;
@@ -26,6 +29,7 @@
 import org.dcache.auth.attributes.LoginAttributes;
 import org.dcache.auth.attributes.Restriction;
 import org.dcache.auth.attributes.Restrictions;
+import org.dcache.auth.attributes.RootDirectory;
 import org.dcache.http.AuthenticationHandler;
 
 /**
@@ -57,4 +61,51 @@ public static Subject roleAwareSubject(HttpServletRequest request) {
     public static Restriction roleAwareRestriction(HttpServletRequest request) {
         return isAdmin(request) ? Restrictions.none() : getRestriction(request);
     }
+
+    public static String getUserRootAwareTargetPrefix(HttpServletRequest request,
+          String includedPrefix) {
+        FsPath userRootPath = getLoginAttributes(request).stream()
+              .filter(RootDirectory.class::isInstance)
+              .findFirst()
+              .map(RootDirectory.class::cast)
+              .map(RootDirectory::getRoot)
+              .map(FsPath::create)
+              .orElse(FsPath.ROOT);
+        return getTargetPrefixFromUserRoot(userRootPath, includedPrefix);
+    }
+
+    @VisibleForTesting
+    static String getTargetPrefixFromUserRoot(FsPath userRootPath, String includedPrefix) {
+        if (userRootPath == null) {
+            return includedPrefix;
+        }
+
+        if (Strings.emptyToNull(includedPrefix) == null) {
+            return userRootPath.toString();
+        }
+
+        return pathUnion(userRootPath, includedPrefix).toString();
+    }
+
+    private static FsPath pathUnion(FsPath root, String path) {
+        FsPath rootPath = dovetail(root, FsPath.create(path));
+        if (rootPath == null) {
+            rootPath = root.chroot(path);
+        }
+        return rootPath;
+    }
+
+    private static FsPath dovetail(FsPath root, FsPath path) {
+        FsPath prefix = FsPath.create(FsPath.ROOT + root.name());
+        if (path.hasPrefix(prefix)) {
+            String relative = path.stripPrefix(prefix);
+            return root.chroot(relative);
+        }
+
+        if (root.parent().isRoot()) {
+            return null;
+        }
+
+        return dovetail(root.parent(), path);
+    }
 }

modules/dcache-frontend/src/test/java/org/dcache/restful/resources/bulk/BulkRequestJsonParseTest.java

@@ -167,6 +167,6 @@ private void givenJsonWithArrayTargetAttribute() {
     }
 
     private void whenParsed() {
-        bulkRequest = toBulkRequest(requestJson);
+        bulkRequest = toBulkRequest(requestJson, null);
     }
 }