Merge pull request #218 from scireum/feature/jvo/issues/214

List Objects V2

src/main/java/ninja/Bucket.java

@@ -168,14 +168,18 @@ public boolean delete() {
     }
 
     /**
-     * Returns a list of at most the provided number of stored objects
+     * Sends a list of at most the provided number of stored objects using
+     * <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV1.html">V1</a> format.
      *
      * @param output the xml structured output the list of objects should be written to
      * @param limit  controls the maximum number of objects returned
      * @param marker the key to start with when listing objects in a bucket
      * @param prefix limits the response to keys that begin with the specified prefix
      */
-    public void outputObjects(XMLStructuredOutput output, int limit, @Nullable String marker, @Nullable String prefix) {
+    public void outputObjectsV1(XMLStructuredOutput output,
+                                int limit,
+                                @Nullable String marker,
+                                @Nullable String prefix) {
         ListFileTreeVisitor visitor = new ListFileTreeVisitor(output, limit, marker, prefix);
 
         output.beginOutput("ListBucketResult", Attribute.set("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/"));
@@ -192,6 +196,36 @@ public void outputObjects(XMLStructuredOutput output, int limit, @Nullable Strin
         output.endOutput();
     }
 
+    /**
+     * Sends a list of at most the provided number of stored objects using
+     * <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html">V2</a> format.
+     *
+     * @param output the xml structured output the list of objects should be written to
+     * @param limit  controls the maximum number of objects returned
+     * @param marker the key to start with when listing objects in a bucket
+     * @param prefix limits the response to keys that begin with the specified prefix
+     */
+    public void outputObjectsV2(XMLStructuredOutput output,
+                                int limit,
+                                @Nullable String marker,
+                                @Nullable String prefix) {
+        ListFileTreeVisitor visitor = new ListFileTreeVisitor(output, limit, marker, prefix);
+
+        output.beginOutput("ListBucketResult", Attribute.set("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/"));
+        output.property("Name", getName());
+        output.property("MaxKeys", limit);
+        output.property("StartAfter", marker);
+        output.property("Prefix", prefix);
+        try {
+            walkFileTreeOurWay(folder.toPath(), visitor);
+        } catch (IOException e) {
+            throw Exceptions.handle(e);
+        }
+        output.property("IsTruncated", limit > 0 && visitor.getCount() > limit);
+        output.property("KeyCount", visitor.getCount());
+        output.endOutput();
+    }
+
     /**
      * Very simplified stand-in for {@link Files#walkFileTree(Path, FileVisitor)} where we control the traversal order.
      *

src/main/java/ninja/S3Dispatcher.java

@@ -617,20 +617,42 @@ private boolean objectCheckAuth(WebContext webContext, Bucket bucket, String key
     }
 
     /**
-     * Handles GET /bucket
+     * Handles {@code GET /bucket} requests as triggered by
+     * <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html">{@code ListObjects}</a>
+     * and <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html">{@code ListObjectsV2}</a>
+     * calls.
      *
      * @param webContext the context describing the current request
      * @param bucket     the bucket of which the contents should be listed
      */
     private void listObjects(WebContext webContext, Bucket bucket) {
+        if (webContext.get("list-type").asInt(1) == 2) {
+            listObjectsV2(webContext, bucket);
+        } else {
+            listObjectsV1(webContext, bucket);
+        }
+    }
+
+    private void listObjectsV1(WebContext webContext, Bucket bucket) {
         int maxKeys = webContext.get("max-keys").asInt(1000);
         String marker = webContext.get("marker").asString();
         String prefix = webContext.get("prefix").asString();
 
         Response response = webContext.respondWith();
         response.setHeader(HTTP_HEADER_NAME_CONTENT_TYPE, CONTENT_TYPE_XML);
 
-        bucket.outputObjects(response.xml(), maxKeys, marker, prefix);
+        bucket.outputObjectsV1(response.xml(), maxKeys, marker, prefix);
+    }
+
+    private void listObjectsV2(WebContext webContext, Bucket bucket) {
+        int maxKeys = webContext.get("max-keys").asInt(1000);
+        String marker = webContext.get("start-after").asString();
+        String prefix = webContext.get("prefix").asString();
+
+        Response response = webContext.respondWith();
+        response.setHeader(HTTP_HEADER_NAME_CONTENT_TYPE, CONTENT_TYPE_XML);
+
+        bucket.outputObjectsV2(response.xml(), maxKeys, marker, prefix);
     }
 
     /**

src/main/resources/templates/api.html.pasta

@@ -6,9 +6,10 @@
     <div class="row">
         <div class="col-md-12">
             <div class="well">
-                <p>Basically all object methods are supported. However no ACLs are checked. If the bucket is public,
+                <p>
+                    Basically all object methods are supported. However, no ACLs are checked. If the bucket is public,
                     everyone can access its contents.
-                    Otherwise a valid hash has to be provided as Authorization header. The hash will be checked as
+                    Otherwise, a valid hash has to be provided as Authorization header. The hash will be checked as
                     expected by amazon, but no multiline-headers are supported yet. (Multi-value headers are supported).
                 </p>
                 <legend>Supported Methods</legend>

src/main/resources/templates/header.html.pasta

@@ -1,6 +1,7 @@
 <i:arg name="title" type="String" default=""/>
 <i:arg name="subtitle" type="String" default=""/>
 <i:arg name="notice" type="String" default=""/>
+
 <div class="row">
     <div class="col-md-8">
         <div class="header">

src/test/java/BaseAWSSpec.groovy

@@ -10,8 +10,8 @@ import com.amazonaws.HttpMethod
 import com.amazonaws.services.s3.AmazonS3Client
 import com.amazonaws.services.s3.model.AmazonS3Exception
 import com.amazonaws.services.s3.model.DeleteObjectsRequest
-import com.amazonaws.services.s3.model.DeleteObjectsResult
 import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest
+import com.amazonaws.services.s3.model.ListObjectsV2Request
 import com.amazonaws.services.s3.model.ObjectMetadata
 import com.amazonaws.services.s3.model.ResponseHeaderOverrides
 import com.amazonaws.services.s3.transfer.TransferManagerBuilder
@@ -389,18 +389,52 @@ abstract class BaseAWSSpec extends BaseSpecification {
                 key3,
                 new ByteArrayInputStream("Drei".getBytes(Charsets.UTF_8)),
                 new ObjectMetadata())
-        List<DeleteObjectsResult.DeletedObject> deletedObjects = client
-                .deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(key1, key2)).getDeletedObjects()
+        def result = client.deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(key1, key2))
         then:
-        deletedObjects.size() == 2
-        deletedObjects.get(0).getKey() == key1
-        deletedObjects.get(1).getKey() == key2
+        result.getDeletedObjects().size() == 2
+        result.getDeletedObjects().get(0).getKey() == key1
+        result.getDeletedObjects().get(1).getKey() == key2
         and:
         def listing = client.listObjects(bucketName)
-        def summaries = listing.getObjectSummaries()
-        summaries.size() == 1
-        summaries.get(0).getKey() == key3
+        listing.getObjectSummaries().size() == 1
+        listing.getObjectSummaries().get(0).getKey() == key3
+        and:
+        client.deleteObject(bucketName, key3)
+    }
+
+    // reported in https://github.com/scireum/s3ninja/issues/214
+    def "ListObjectsV2 works as expected"() {
+        given:
+        def bucketName = DEFAULT_BUCKET_NAME
+        def key1 = DEFAULT_KEY + "/Eins"
+        def key2 = DEFAULT_KEY + "/Eins-Eins"
+        def key3 = DEFAULT_KEY + "/Drei"
+        def client = getClient()
+        when:
+        client.putObject(
+                bucketName,
+                key1,
+                new ByteArrayInputStream("Eins".getBytes(Charsets.UTF_8)),
+                new ObjectMetadata())
+        client.putObject(
+                bucketName,
+                key2,
+                new ByteArrayInputStream("Zwei".getBytes(Charsets.UTF_8)),
+                new ObjectMetadata())
+        client.putObject(
+                bucketName,
+                key3,
+                new ByteArrayInputStream("Drei".getBytes(Charsets.UTF_8)),
+                new ObjectMetadata())
+        def result = client.listObjectsV2(new ListObjectsV2Request().withBucketName(bucketName).withPrefix(key1))
+        then:
+        result.getKeyCount() == 2
+        result.getObjectSummaries().size() == 2
+        result.getObjectSummaries().get(0).getKey() == key1
+        result.getObjectSummaries().get(1).getKey() == key2
         and:
+        client.deleteObject(bucketName, key1)
+        client.deleteObject(bucketName, key2)
         client.deleteObject(bucketName, key3)
     }
 }