Adding the ability to filter objects from S3 opendir

This change the listFilter stream context option that can be used when using
opendir and the Amazon S3 stream wrapper. This function can be used to filter
out specific objects from the files yielded from the stream wrapper. This is
utilized in the download sync builder to not attempt to download objects that
have a GLACIER storage class. Closes aws#322.

src/Aws/S3/StreamWrapper.php

@@ -428,6 +428,11 @@ public function rmdir($path)
     /**
      * Support for opendir().
      *
+     * The opendir() method of the Amazon S3 stream wrapper supports a stream
+     * context option of "listFilter". listFilter must be a callable that
+     * accepts an associative array of object data and returns true if the
+     * object should be yielded when iterating the keys in a bucket.
+     *
      * @param string $path    The path to the directory (e.g. "s3://dir[</prefix>]")
      * @param string $options Whether or not to enforce safe_mode (0x04). Unused.
      *
@@ -440,6 +445,7 @@ public function dir_opendir($path, $options)
         $this->clearStatInfo();
         $params = $this->getParams($path);
         $delimiter = $this->getOption('delimiter');
+        $filterFn = $this->getOption('listFilter');
 
         if ($delimiter === null) {
             $delimiter = '/';
@@ -462,12 +468,14 @@ public function dir_opendir($path, $options)
             'sort_results'    => true
         ));
 
-        // Filter our "/" keys added by the console as directories
+        // Filter our "/" keys added by the console as directories, and ensure
+        // that if a filter function is provided that it passes the filter.
         $this->objectIterator = new FilterIterator(
             $objectIterator,
-            function ($key) {
+            function ($key) use ($filterFn) {
                 // Each yielded results can contain a "Key" or "Prefix"
-                return !isset($key['Key']) || substr($key['Key'], -1, 1) !== '/';
+                return (!$filterFn || call_user_func($filterFn, $key)) &&
+                    (!isset($key['Key']) || substr($key['Key'], -1, 1) !== '/');
             }
         );
 
@@ -725,7 +733,7 @@ protected function triggerError($errors, $flags = null)
     {
         if ($flags & STREAM_URL_STAT_QUIET) {
           // This is triggered with things like file_exists()
-          
+
           if ($flags & STREAM_URL_STAT_LINK) {
             // This is triggered for things like is_link()
             return $this->formatUrlStat(false);

src/Aws/S3/Sync/AbstractSyncBuilder.php

@@ -414,7 +414,16 @@ protected function createS3Iterator()
         }
 
         // Use opendir so that we can pass stream context to the iterator
-        $dh = opendir($dir, stream_context_create(array('s3' => array('delimiter' => ''))));
+        $dh = opendir($dir, stream_context_create(array(
+            's3' => array(
+                'delimiter'  => '',
+                'listFilter' => function ($obj) {
+                    // Ensure that we do not try to download a glacier object.
+                    return !isset($obj['StorageClass']) ||
+                        $obj['StorageClass'] != 'GLACIER';
+                }
+            )
+        )));
 
         // Add the trailing slash for the OpendirIterator concatenation
         if (!$this->keyPrefix) {

tests/Aws/Tests/S3/Sync/DownloadSyncBuilderTest.php

@@ -16,10 +16,13 @@
 
 namespace Aws\Tests\S3\Sync;
 
+use Aws\S3\S3Client;
 use Aws\S3\Command\S3Command;
 use Aws\S3\ResumableDownload;
 use Aws\S3\Sync\DownloadSync;
 use Aws\S3\Sync\DownloadSyncBuilder;
+use Guzzle\Http\Message\Response;
+use Guzzle\Plugin\Mock\MockPlugin;
 
 /**
  * @covers Aws\S3\Sync\DownloadSyncBuilder
@@ -93,4 +96,70 @@ public function testAddsDebugListenerForResumable()
         rewind($out);
         $this->assertContains('Resuming Foo/Bar -> ' . __FILE__, stream_get_contents($out));
     }
+
+    public function testDoesNotDownloadGlacierStorageObjects()
+    {
+        $res = <<<EOT
+HTTP/1.1 200 OK
+x-amz-id-2: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+x-amz-request-id: XXXXXXXXXXXXXXXX
+Date: Thu, 04 Jul 2012 12:00:00 GMT
+Content-Type: application/xml
+Server: AmazonS3
+
+<?xml version="1.0" encoding="UTF-8"?>
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+    <Name>bucket-1</Name>
+    <Prefix></Prefix>
+    <Marker></Marker>
+    <MaxKeys></MaxKeys>
+    <Delimiter>/</Delimiter>
+    <IsTruncated>false</IsTruncated>
+    <Contents>
+        <Key>e</Key>
+        <LastModified>2012-04-07T12:00:00.000Z</LastModified>
+        <ETag>"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"</ETag>
+        <Size>0</Size>
+        <Owner>
+            <ID>XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX</ID>
+            <DisplayName>XXXXXXXXXX</DisplayName>
+        </Owner>
+        <StorageClass>GLACIER</StorageClass>
+    </Contents>
+    <Contents>
+        <Key>f</Key>
+        <LastModified>2012-04-07T12:00:00.000Z</LastModified>
+        <ETag>"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"</ETag>
+        <Size>0</Size>
+        <Owner>
+            <ID>XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX</ID>
+            <DisplayName>XXXXXXXXXX</DisplayName>
+        </Owner>
+        <StorageClass>STANDARD</StorageClass>
+    </Contents>
+</ListBucketResult>
+EOT;
+
+        $s3 = S3Client::factory(array('key' => 'foo', 'secret' => 'bar'));
+        $s3->getEventDispatcher()->addSubscriber(new MockPlugin(array(
+            Response::fromMessage($res),
+            new Response(200)
+        )));
+
+        $dir = __DIR__ . '/../../../../../build/artifacts';
+        if (!is_dir($dir)) {
+            mkdir($dir);
+        }
+
+        DownloadSyncBuilder::getInstance()
+            ->setClient($s3)
+            ->setBucket('Foo')
+            ->setDirectory($dir)
+            ->build()
+            ->transfer();
+
+        $this->assertFileNotExists($dir . '/e');
+        $this->assertFileExists($dir . '/f');
+        unlink($dir . '/f');
+    }
 }