Add endpoint to list secrets in batches

cli/src/main/java/keywhiz/cli/commands/ListAction.java

@@ -24,8 +24,9 @@
 import keywhiz.cli.configs.ListActionConfig;
 import keywhiz.client.KeywhizClient;
 
-public class ListAction implements Runnable {
+import static com.google.common.base.Preconditions.checkArgument;
 
+public class ListAction implements Runnable {
   private final ListActionConfig listActionConfig;
   private final KeywhizClient keywhizClient;
   private final Printing printing;
@@ -37,7 +38,7 @@ public ListAction(ListActionConfig listActionConfig, KeywhizClient client, Print
   }
 
   @Override public void run() {
-    List<String> listOptions = listActionConfig.listOptions;
+    List<String> listOptions = listActionConfig.listType;
     if (listOptions == null) {
       try {
         printing.printAllSanitizedSecrets(keywhizClient.allSecrets());
@@ -61,7 +62,19 @@ public ListAction(ListActionConfig listActionConfig, KeywhizClient client, Print
           break;
 
         case "secrets":
-          printing.printAllSanitizedSecrets(keywhizClient.allSecrets());
+          if (listActionConfig.idx == null && listActionConfig.num == null) {
+            printing.printAllSanitizedSecrets(keywhizClient.allSecrets());
+          } else if (listActionConfig.idx != null && listActionConfig.num != null) {
+            checkArgument(listActionConfig.idx >= 0);
+            checkArgument(listActionConfig.num >= 0);
+            if (listActionConfig.newestFirst == null) {
+              printing.printAllSanitizedSecrets(keywhizClient.allSecretsBatched(listActionConfig.idx, listActionConfig.num, true));
+            }  else {
+              printing.printAllSanitizedSecrets(keywhizClient.allSecretsBatched(listActionConfig.idx, listActionConfig.num, listActionConfig.newestFirst));
+            }
+          } else {
+            throw new AssertionError("Both idx and num must be provided for batched secret queries");
+          }
           break;
 
         default:

cli/src/main/java/keywhiz/cli/configs/ListActionConfig.java

@@ -24,5 +24,14 @@
 public class ListActionConfig {
 
   @Parameter(description = "[<groups|clients|secrets>]")
-  public List<String> listOptions;
+  public List<String> listType;
+
+  @Parameter(names = "--idx", description = "Index to start retrieving secrets (valid only with 'secrets'; requires --num to also be specified)")
+  public Integer idx;
+
+  @Parameter(names = "--num", description = "Number of secrets to retrieve after index (valid only with 'secrets'; requires --idx to also be specified)")
+  public Integer num;
+
+  @Parameter(names = "--newestFirst", description = "Whether to batch the secrets from newest creation date first.  Defaults to 'true' (valid only with 'secrets'; requires --idx and --num to also be specified)")
+  public Boolean newestFirst;
 }

cli/src/test/java/keywhiz/cli/commands/ListActionTest.java

@@ -46,38 +46,83 @@ public void setUp() {
 
   @Test
   public void listCallsPrintForListAll() throws Exception {
-    listActionConfig.listOptions = null;
+    listActionConfig.listType = null;
     listAction.run();
     verify(printing).printAllSanitizedSecrets(keywhizClient.allSecrets());
   }
 
   @Test
   public void listCallsPrintForListGroups() throws Exception {
-    listActionConfig.listOptions = Arrays.asList("groups");
+    listActionConfig.listType = Arrays.asList("groups");
     listAction.run();
 
     verify(printing).printAllGroups(keywhizClient.allGroups());
   }
 
   @Test
   public void listCallsPrintForListClients() throws Exception {
-    listActionConfig.listOptions = Arrays.asList("clients");
+    listActionConfig.listType = Arrays.asList("clients");
     listAction.run();
 
     verify(printing).printAllClients(keywhizClient.allClients());
   }
 
   @Test
   public void listCallsPrintForListSecrets() throws Exception {
-    listActionConfig.listOptions = Arrays.asList("secrets");
+    listActionConfig.listType = Arrays.asList("secrets");
     listAction.run();
 
     verify(printing).printAllSanitizedSecrets(keywhizClient.allSecrets());
   }
 
+  @Test
+  public void listCallsPrintForListSecretsBatched() throws Exception {
+    listActionConfig.listType = Arrays.asList("secrets");
+    listActionConfig.idx = 0;
+    listActionConfig.num = 10;
+    listActionConfig.newestFirst = false;
+    listAction.run();
+
+    verify(printing).printAllSanitizedSecrets(keywhizClient.allSecretsBatched(0, 10, false));
+  }
+
+  @Test
+  public void listCallsPrintForListSecretsBatchedWithDefault() throws Exception {
+    listActionConfig.listType = Arrays.asList("secrets");
+    listActionConfig.idx = 5;
+    listActionConfig.num = 10;
+    listAction.run();
+
+    verify(printing).printAllSanitizedSecrets(keywhizClient.allSecretsBatched(5, 10, true));
+  }
+
+  @Test
+  public void listCallsErrorsCorrectly() throws Exception {
+    listActionConfig.listType = Arrays.asList("secrets");
+    listActionConfig.idx = 5;
+    boolean error = false;
+    try {
+      listAction.run();
+    } catch (AssertionError e) {
+      error = true;
+    }
+    assert(error);
+
+    listActionConfig.listType = Arrays.asList("secrets");
+    listActionConfig.idx = 5;
+    listActionConfig.num = -5;
+    error = false;
+    try {
+      listAction.run();
+    } catch (IllegalArgumentException e) {
+      error = true;
+    }
+    assert(error);
+  }
+
   @Test(expected = AssertionError.class)
   public void listThrowsIfInvalidType() throws Exception {
-    listActionConfig.listOptions = Arrays.asList("invalid_type");
+    listActionConfig.listType = Arrays.asList("invalid_type");
     listAction.run();
   }
 }

client/src/main/java/keywhiz/client/KeywhizClient.java

@@ -145,6 +145,11 @@ public List<SanitizedSecret> allSecrets() throws IOException {
     return mapper.readValue(response, new TypeReference<List<SanitizedSecret>>() {});
   }
 
+  public List<SanitizedSecret> allSecretsBatched(int idx, int num, boolean newestFirst) throws IOException {
+    String response = httpGet(baseUrl.resolve(String.format("/admin/secrets?idx=%d&num=%d&newestFirst=%s", idx, num, newestFirst)));
+    return mapper.readValue(response, new TypeReference<List<SanitizedSecret>>() {});
+  }
+
   public SecretDetailResponse createSecret(String name, String description, byte[] content,
       ImmutableMap<String, String> metadata, long expiry) throws IOException {
     checkArgument(!name.isEmpty());

server/src/main/java/keywhiz/service/daos/SecretController.java

@@ -83,6 +83,20 @@ public List<SanitizedSecret> getSecretsNameOnly() {
         .collect(toList());
   }
 
+  /**
+   * @param idx the first index to select in a list of secrets sorted by creation time
+   * @param num the number of secrets after idx to select in the list of secrets
+   * @param newestFirst if true, order the secrets from newest creation time to oldest
+   * @return A list of secret names
+   */
+  public List<SanitizedSecret> getSecretsBatched(int idx, int num, boolean newestFirst) {
+    checkArgument(idx >= 0, "Index must be positive when getting batched secret names!");
+    checkArgument(num >= 0, "Num must be positive when getting batched secret names!");
+    return secretDAO.getSecretsBatched(idx, num, newestFirst).stream()
+        .map(SanitizedSecret::fromSecretSeriesAndContent)
+        .collect(toList());
+  }
+
   public SecretBuilder builder(String name, String secret, String creator, long expiry) {
     checkArgument(!name.isEmpty());
     checkArgument(!secret.isEmpty());

server/src/main/java/keywhiz/service/daos/SecretDAO.java

@@ -33,6 +33,7 @@
 import keywhiz.api.model.SecretSeriesAndContent;
 import keywhiz.api.model.SecretVersion;
 import keywhiz.jooq.tables.Secrets;
+import keywhiz.jooq.tables.records.SecretsRecord;
 import keywhiz.service.config.Readonly;
 import keywhiz.service.daos.SecretContentDAO.SecretContentDAOFactory;
 import keywhiz.service.daos.SecretSeriesDAO.SecretSeriesDAOFactory;
@@ -220,6 +221,29 @@ public ImmutableList<SimpleEntry<Long, String>> getSecretsNameOnly() {
     return ImmutableList.copyOf(results);
   }
 
+  /**
+   * @param idx the first index to select in a list of secrets sorted by creation time
+   * @param num the number of secrets after idx to select in the list of secrets
+   * @param newestFirst if true, order the secrets from newest creation time to oldest
+   * @return A list of secrets
+   */
+  public ImmutableList<SecretSeriesAndContent> getSecretsBatched(int idx, int num, boolean newestFirst) {
+    return dslContext.transactionResult(configuration -> {
+      SecretContentDAO secretContentDAO = secretContentDAOFactory.using(configuration);
+      SecretSeriesDAO secretSeriesDAO = secretSeriesDAOFactory.using(configuration);
+
+      ImmutableList.Builder<SecretSeriesAndContent> secretsBuilder = ImmutableList.builder();
+
+      for (SecretSeries series : secretSeriesDAO.getSecretSeriesBatched(idx, num, newestFirst)) {
+        SecretContent content = secretContentDAO.getSecretContentById(series.currentVersion().get()).get();
+        SecretSeriesAndContent seriesAndContent = SecretSeriesAndContent.of(series, content);
+        secretsBuilder.add(seriesAndContent);
+      }
+
+      return secretsBuilder.build();
+    });
+  }
+
   /**
    * @param name of secret series to look up secrets by.
    * @param versionIdx the first index to select in a list of versions sorted by creation time

server/src/main/java/keywhiz/service/daos/SecretSeriesDAO.java

@@ -58,7 +58,8 @@ public class SecretSeriesDAO {
   private final ObjectMapper mapper;
   private final SecretSeriesMapper secretSeriesMapper;
 
-  private SecretSeriesDAO(DSLContext dslContext, ObjectMapper mapper, SecretSeriesMapper secretSeriesMapper) {
+  private SecretSeriesDAO(DSLContext dslContext, ObjectMapper mapper,
+      SecretSeriesMapper secretSeriesMapper) {
     this.dslContext = dslContext;
     this.mapper = mapper;
     this.secretSeriesMapper = secretSeriesMapper;
@@ -92,8 +93,9 @@ long createSecretSeries(String name, String creator, String description, @Nullab
     return r.getId();
   }
 
-  void updateSecretSeries(long secretId, String name, String creator, String description, @Nullable String type,
-                          @Nullable Map<String, String> generationOptions) {
+  void updateSecretSeries(long secretId, String name, String creator, String description,
+      @Nullable String type,
+      @Nullable Map<String, String> generationOptions) {
     long now = OffsetDateTime.now().toEpochSecond();
     if (generationOptions == null) {
       generationOptions = ImmutableMap.of();
@@ -189,6 +191,25 @@ public ImmutableList<SecretSeries> getSecretSeries(@Nullable Long expireMaxTime,
     return ImmutableList.copyOf(r);
   }
 
+  public ImmutableList<SecretSeries> getSecretSeriesBatched(int idx, int num, boolean newestFirst) {
+    SelectQuery<Record> select = dslContext
+        .select()
+        .from(SECRETS)
+        .join(SECRETS_CONTENT)
+        .on(SECRETS.CURRENT.equal(SECRETS_CONTENT.ID))
+        .where(SECRETS.CURRENT.isNotNull())
+        .getQuery();
+    if (newestFirst) {
+      select.addOrderBy(SECRETS.CREATEDAT.desc());
+    } else {
+      select.addOrderBy(SECRETS.CREATEDAT.asc());
+    }
+    select.addLimit(idx, num);
+
+    List<SecretSeries> r = select.fetchInto(SECRETS).map(secretSeriesMapper);
+    return ImmutableList.copyOf(r);
+  }
+
   public void deleteSecretSeriesByName(String name) {
     long now = OffsetDateTime.now().toEpochSecond();
     dslContext.transaction(configuration -> {
@@ -214,16 +235,16 @@ public void deleteSecretSeriesById(long id) {
     dslContext.transaction(configuration -> {
       DSL.using(configuration)
           .update(SECRETS)
-          .set(SECRETS.CURRENT, (Long)null)
+          .set(SECRETS.CURRENT, (Long) null)
           .set(SECRETS.UPDATEDAT, now)
           .where(SECRETS.ID.eq(id))
           .execute();
 
-       DSL.using(configuration)
-           .delete(ACCESSGRANTS)
-           .where(ACCESSGRANTS.SECRETID.eq(id))
-           .execute();
-      });
+      DSL.using(configuration)
+          .delete(ACCESSGRANTS)
+          .where(ACCESSGRANTS.SECRETID.eq(id))
+          .execute();
+    });
   }
 
   public static class SecretSeriesDAOFactory implements DAOFactory<SecretSeriesDAO> {

server/src/main/java/keywhiz/service/resources/admin/SecretsResource.java

@@ -30,6 +30,7 @@
 import java.util.Optional;
 import javax.inject.Inject;
 import javax.validation.Valid;
+import javax.ws.rs.BadRequestException;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.DefaultValue;
@@ -105,6 +106,9 @@ public class SecretsResource {
    * @param name the name of the Secret to retrieve, if provided
    * @optionalParams version
    * @param nameOnly if set, the result only contains the id and name for the secrets.
+   * @param idx if set, the desired starting index in a list of secrets to be retrieved
+   * @param num if set, the number of secrets to retrieve
+   * @param newestFirst whether to order the secrets by creation date with newest first; defaults to true
    *
    * @description Returns a single Secret or a set of all Secrets for this user.
    * Used by Keywhiz CLI and the web ui.
@@ -114,17 +118,41 @@ public class SecretsResource {
   @Timed @ExceptionMetered
   @GET
   public Response findSecrets(@Auth User user, @DefaultValue("") @QueryParam("name") String name,
-      @DefaultValue("") @QueryParam("nameOnly") String nameOnly) {
+      @DefaultValue("") @QueryParam("nameOnly") String nameOnly, @QueryParam("idx") Integer idx,
+      @QueryParam("num") Integer num,
+      @DefaultValue("true") @QueryParam("newestFirst") Boolean newestFirst) {
+    if (!name.isEmpty() && idx != null && num != null) {
+      throw new BadRequestException("Name and idx/num cannot both be specified");
+    }
+
+    validateArguments(name, nameOnly, idx, num);
+
     if (name.isEmpty()) {
       if (nameOnly.isEmpty()) {
-        return Response.ok().entity(listSecrets(user)).build();
+        if (idx == null || num == null) {
+          return Response.ok().entity(listSecrets(user)).build();
+        } else {
+          return Response.ok().entity(listSecretsBatched(user, idx, num, newestFirst)).build();
+        }
       } else {
         return Response.ok().entity(listSecretsNameOnly(user)).build();
       }
     }
     return Response.ok().entity(retrieveSecret(user, name)).build();
   }
 
+  private void validateArguments(String name, String nameOnly, Integer idx, Integer num) {
+    if (idx == null && num != null || idx != null && num != null) {
+      throw new IllegalArgumentException("Both idx and num must be specified");
+    }
+    if (!name.isEmpty() && idx != null && num != null) {
+      throw new IllegalArgumentException("Name, idx, and num must not all be specified");
+    }
+    if (nameOnly.isEmpty() && idx != null && num != null) {
+      throw new IllegalArgumentException("nameOnly option is not valid for batched secret retrieval");
+    }
+  }
+
   protected List<SanitizedSecret> listSecrets(@Auth User user) {
     logger.info("User '{}' listing secrets.", user);
     return secretController.getSanitizedSecrets(null, null);
@@ -135,6 +163,11 @@ protected List<SanitizedSecret> listSecretsNameOnly(@Auth User user) {
     return secretController.getSecretsNameOnly();
   }
 
+  protected List<SanitizedSecret> listSecretsBatched(@Auth User user, int idx, int num, boolean newestFirst) {
+    logger.info("User '{}' listing secrets with idx '{}', num '{}', newestFirst '{}'.", user, idx, num, newestFirst);
+    return secretController.getSecretsBatched(idx, num, newestFirst);
+  }
+
   protected SanitizedSecret retrieveSecret(@Auth User user, String name) {
     logger.info("User '{}' retrieving secret name={}.", user, name);
     return sanitizedSecretFromName(name);