Skip to content

Commit

Permalink
dcache-frontend: add backend filtering and sorting for snapshot servi…
Browse files Browse the repository at this point in the history 
…ces*

Motivation:

In order to achieve infinite scrolling with lazy-loaded
data from the REST service, backend filtering and
sorting needs to be added.

Modification:

Change the APIs to carry the necessary extra parameters.
Make small modifications to the underlying data objects
to make filtering easier.
Add sorter and filter implementations in the respective
service implementations.
Modify the service implementations as necessary.
Fix unit test.

Result:
The transfers and restores GET supports filtering and multi-sort.

Target: master
Require-notes: no
Require-book: no
Acked-by: Paul
  • Loading branch information
@alrossi
alrossi committed Oct 25, 2017
1 parent 3df441d commit f910393
Show file tree
Hide file tree
Showing 11 changed files with 408 additions and 188 deletions.
7 changes: 6 additions & 1 deletion modules/dcache-frontend/src/main/java/org/dcache/restful/providers/restores/RestoreInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
/**
* <p>Restore/stage request metadata.</p>
*/
public class RestoreInfo implements Serializable {
public class RestoreInfo implements Comparable<RestoreInfo>, Serializable {
private String key;
private PnfsId pnfsId;
private String path;
Expand Down Expand Up @@ -98,6 +98,11 @@ public RestoreInfo(RestoreHandlerInfo info) {
errorMessage = info.getErrorMessage();
}

@Override
public int compareTo(RestoreInfo o) {
return key.compareTo(o.key);
}

public Integer getClients() {
return clients;
}
Expand Down
38 changes: 22 additions & 16 deletions ...dcache-frontend/src/main/java/org/dcache/restful/resources/restores/RestoreResources.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,9 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;

import java.lang.reflect.InvocationTargetException;
import java.util.UUID;

import diskCacheV111.util.PnfsId;

import diskCacheV111.util.CacheException;
import org.dcache.restful.providers.SnapshotList;
import org.dcache.restful.providers.restores.RestoreInfo;
import org.dcache.restful.services.restores.RestoresInfoService;
Expand Down Expand Up @@ -103,9 +100,13 @@ public final class RestoreResources {
* the current list, the service will return a null token and
* an empty list, and the client will need to recall the method
* without a token (refresh).
* @param offset Return restores beginning at this index.
* @param limit Return at most this number of items.
* @param pnfsid Return only restores for this file.
* @param offset Return items beginning at this index.
* @param limit Return at most this number of items.
* @param pnfsid Filter on pnfsid.
* @param subnet Filter on subnet.
* @param pool Filter on pool.
* @param status Filter on status.
* @param sort comma-delimited orderd list of fields to sort on.
* @return object containing list of restores, along with token and
* offset information.
*/
Expand All @@ -114,17 +115,22 @@ public final class RestoreResources {
public SnapshotList<RestoreInfo> getRestores(@QueryParam("token") UUID token,
@QueryParam("offset") Integer offset,
@QueryParam("limit") Integer limit,
@QueryParam("pnfsid") PnfsId pnfsid) {
@QueryParam("pnfsid") String pnfsid,
@QueryParam("subnet") String subnet,
@QueryParam("pool") String pool,
@QueryParam("status") String status,
@QueryParam("sort") String sort) {
try {
return service.get(token, offset, limit, pnfsid);
} catch (InvocationTargetException | IllegalAccessException e) {
return service.get(token,
offset,
limit,
pnfsid,
subnet,
pool,
status,
sort);
} catch (CacheException e) {
throw new InternalServerErrorException(e);
} catch (NoSuchMethodException e) {
/*
* This should not happen unless the API has changed, in which
* case there is either a bug or a class loader issue.
*/
throw new RuntimeException(e);
}
}
}
77 changes: 51 additions & 26 deletions ...ache-frontend/src/main/java/org/dcache/restful/resources/transfers/TransferResources.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,10 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;

import java.lang.reflect.InvocationTargetException;
import java.util.UUID;

import diskCacheV111.util.PnfsId;
import diskCacheV111.util.CacheException;
import diskCacheV111.util.TransferInfo;

import org.dcache.restful.providers.SnapshotList;
import org.dcache.restful.services.transfers.TransferInfoService;

Expand All @@ -95,37 +92,65 @@ public final class TransferResources {
* (not pool-to-pool) transfer operations that are currently
* queued or running.</p>
*
* @param token Use the snapshot corresponding to this UUID. The contract
* with the service is that if the parameter value is null, the
* snapshot will be used, regardless of whether offset and limit
* are still valid. Initial/refresh calls should always be
* without a token. Subsequent calls should send back the
* current token; in the case that it no longer corresponds to
* the current list, the service will return a null token and
* an empty list, and the client will need to recall the method
* without a token (refresh).
* @param offset Return transfers beginning at this index.
* @param limit Return at most this number of items.
* @param pnfsid Return only transfers for this file.
* @param token Use the snapshot corresponding to this UUID. The contract
* with the service is that if the parameter value is null, the
* snapshot will be used, regardless of whether offset and limit
* are still valid. Initial/refresh calls should always be
* without a token. Subsequent calls should send back the
* current token; in the case that it no longer corresponds to
* the current list, the service will return a null token and
* an empty list, and the client will need to recall the method
* without a token (refresh).
* @param offset Return transfers beginning at this index.
* @param limit Return at most this number of items.
* @param state Filter on state.
* @param door Filter on door.
* @param domain Filter on domain.
* @param protocol Filter on protocol.
* @param uid Filter on uid.
* @param gid Filter on gid.
* @param vomsgroup Filter on vomsgroup.
* @param protocol Filter on protocol.
* @param pnfsid Filter on pnfsid.
* @param pool Filter on pool.
* @param pool Filter on client.
* @param sort comma-delimited orderd list of fields to sort on.
* @return object containing list of transfers, along with token and
* offset information.
* offset information.
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
public SnapshotList<TransferInfo> getTransfers(@QueryParam("token") UUID token,
@QueryParam("offset") Integer offset,
@QueryParam("limit") Integer limit,
@QueryParam("pnfsid") PnfsId pnfsid) {
@QueryParam("state") String state,
@QueryParam("door") String door,
@QueryParam("domain") String domain,
@QueryParam("prot") String protocol,
@QueryParam("uid") String uid,
@QueryParam("gid") String gid,
@QueryParam("vomsgroup") String vomsgroup,
@QueryParam("pnfsid") String pnfsid,
@QueryParam("pool") String pool,
@QueryParam("client") String client,
@QueryParam("sort") String sort) {
try {
return service.get(token, offset, limit, pnfsid);
} catch (InvocationTargetException | IllegalAccessException e) {
return service.get(token,
offset,
limit,
state,
door,
domain,
protocol,
uid,
gid,
vomsgroup,
pnfsid,
pool,
client,
sort);
} catch (CacheException e) {
throw new InternalServerErrorException(e);
} catch (NoSuchMethodException e) {
/*
* This should not happen unless the API has changed, in which
* case there is either a bug or a class loader issue.
*/
throw new RuntimeException(e);
}
}
}
25 changes: 14 additions & 11 deletions ...ache-frontend/src/main/java/org/dcache/restful/services/restores/RestoresInfoService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,10 +59,9 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
*/
package org.dcache.restful.services.restores;

import java.lang.reflect.InvocationTargetException;
import java.util.UUID;

import diskCacheV111.util.PnfsId;
import diskCacheV111.util.CacheException;
import org.dcache.restful.providers.SnapshotList;
import org.dcache.restful.providers.restores.RestoreInfo;

Expand All @@ -73,18 +72,22 @@ public interface RestoresInfoService {
/**
* <p>Return the metadata objects.</p>
*
* @param token specifying the current snapshot held by caller
* (can be <code>null</code>).
* @param offset specifying the index in the snapshot at which to begin.
* @param limit maximum number of elements to include.
* @param pnfsId to use as filter.
* @param offset Return items beginning at this index.
* @param limit Return at most this number of items.
* @param pnfsid Filter on pnfsid.
* @param subnet Filter on subnet.
* @param pool Filter on pool.
* @param status Filter on status.
* @param sort comma-delimited orderd list of fields to sort on.
* @return {@link SnapshotList<RestoreInfo>} containing list of beans.
*/
SnapshotList<RestoreInfo> get(UUID token,
Integer offset,
Integer limit,
PnfsId pnfsId)
throws InvocationTargetException,
IllegalAccessException,
NoSuchMethodException;
String pnfsid,
String subnet,
String pool,
String status,
String sort)
throws CacheException;
}
97 changes: 86 additions & 11 deletions ...-frontend/src/main/java/org/dcache/restful/services/restores/RestoresInfoServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,24 +59,29 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
*/
package org.dcache.restful.services.restores;

import com.google.common.base.Strings;
import com.google.common.util.concurrent.ListenableFuture;

import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import diskCacheV111.util.CacheException;
import diskCacheV111.util.PnfsId;
import diskCacheV111.vehicles.RestoreHandlerInfo;
import dmg.util.command.Command;
import org.dcache.restful.providers.SnapshotList;
import org.dcache.restful.providers.restores.RestoreInfo;
import org.dcache.restful.util.admin.SnapshotDataAccess;
import org.dcache.restful.util.restores.RestoreCollector;
import org.dcache.services.collector.CellDataCollectingService;
import org.dcache.util.FieldSort;

/**
* <p>Service layer responsible for collecting information from
Expand Down Expand Up @@ -105,6 +110,68 @@ class RestoresSetTimeoutCommand extends SetTimeoutCommand {
class RestoresRefreshCommand extends RefreshCommand {
}

private static Function<FieldSort, Comparator<RestoreInfo>> nextComparator() {
return (sort) -> {
Comparator<RestoreInfo> comparator;

switch (sort.getName()) {
case "pnfsid":
comparator = Comparator.comparing(RestoreInfo::getPnfsId);
break;
case "subnet":
comparator = Comparator.comparing(RestoreInfo::getSubnet);
break;
case "pool":
comparator = Comparator.comparing(
RestoreInfo::getPoolCandidate);
break;
case "status":
comparator = Comparator.comparing(RestoreInfo::getStatus);
break;
case "started":
comparator = Comparator.comparing(RestoreInfo::getStarted);
break;
case "clients":
comparator = Comparator.comparing(RestoreInfo::getClients);
break;
case "retries":
comparator = Comparator.comparing(RestoreInfo::getRetries);
break;
default:
throw new IllegalArgumentException(
"sort field " + sort.getName()
+ " not supported.");
}

if (sort.isReverse()) {
return comparator.reversed();
}

return comparator;
};
}

private static Predicate<RestoreInfo> getFilter(String pnfsid,
String subnet,
String pool,
String status) {
Predicate<RestoreInfo> matchesPnfsid =
(info) -> pnfsid == null || Strings.nullToEmpty
(String.valueOf(info.getPnfsId()))
.contains(pnfsid);
Predicate<RestoreInfo> matchesSubnet =
(info) -> subnet == null || Strings.nullToEmpty(info.getSubnet())
.contains(subnet);
Predicate<RestoreInfo> matchesPool =
(info) -> pool == null || Strings.nullToEmpty(info.getPoolCandidate())
.contains(pool);
Predicate<RestoreInfo> matchesStatus =
(info) -> status == null || Strings.nullToEmpty(info.getStatus())
.contains(status);
return matchesPnfsid.and(matchesSubnet).and(matchesPool).and(matchesStatus);
}


/**
* <p>Data store providing snapshots.</p>
*/
Expand All @@ -115,14 +182,22 @@ class RestoresRefreshCommand extends RefreshCommand {
public SnapshotList<RestoreInfo> get(UUID token,
Integer offset,
Integer limit,
PnfsId pnfsid)
throws InvocationTargetException,
IllegalAccessException,
NoSuchMethodException {
Method getPnfsid = RestoreInfo.class.getMethod("getPnfsId");
Method[] methods = pnfsid == null? null : new Method[]{getPnfsid};
Object[] values = pnfsid == null? null : new Object[]{pnfsid};
return access.getSnapshot(token, offset, limit, methods, values);
String pnfsid,
String subnet,
String pool,
String status,
String sort)
throws CacheException {
Predicate<RestoreInfo> filter = getFilter(pnfsid, subnet, pool, status);
List<FieldSort> fields = null;
if (sort != null) {
fields= Arrays.stream(sort.split(","))
.map(FieldSort::new)
.collect(Collectors.toList());
}
Comparator<RestoreInfo> sorter
= FieldSort.getSorter(fields, nextComparator());
return access.getSnapshot(token, offset, limit, filter, sorter);
}

@Override
Expand Down
Loading

0 comments on commit f910393

Please sign in to comment.