Merge pull request #1064 from travis-ci/sf-s3-verbose

add s3 --verbose (ports #882)

lib/dpl/providers/s3.rb

@@ -37,15 +37,16 @@ class S3 < Provider
       opt '--index_document_suffix SUFFIX', 'Index document suffix of a S3 website'
       opt '--default_text_charset CHARSET', 'Default character set to append to the content-type of text files'
       opt '--max_threads NUM', 'The number of threads to use for S3 file uploads', default: 5, max: 15, type: :integer
+      opt '--verbose', 'Be verbose about uploading files'
       # how come there is no glob or file option?
 
       msgs login:                 'Using Access Key: %{access_key_id}',
            default_uri_schema:    'S3 endpoint does not specify a scheme; defaulting to https',
            access_denied:         'It looks like you tried to write to a bucket that is not yours or does not exist. Please create the bucket before trying to write to it.',
            checksum_error:        'AWS secret key does not match the access key id',
            invalid_access_key_id: 'Invalid S3 access key id',
-           upload:                'Uploading %s files with up to %s threads.',
-           upload_file:           'Uploading file %s to %s with %s',
+           upload:                'Uploading %s files with up to %s threads ...',
+           upload_file:           'Uploading %s to %s with %s',
            upload_failed:         'Failed to upload %s',
            index_document_suffix: 'Setting index document suffix to %s'
 
@@ -78,17 +79,26 @@ def upload
           info :upload, files.length, max_threads
           threads = max_threads.times.map { |i| Thread.new(&method(:upload_files)) }
           threads.each(&:join)
+          info "\n" unless verbose?
         end
 
         def upload_files
           while file = files.pop
             data = content_data(file)
-            info :upload_file, file, upload_dir || '/', to_pairs(data)
+            progress(file, data)
             object = s3.bucket(bucket).object(upload_path(file))
             object.upload_file(file, data) || warn(:upload_failed, file)
           end
         end
 
+        def progress(file, data)
+          if verbose?
+            info :upload_file, file, upload_dir || '/', to_pairs(data)
+          else
+            print '.'
+          end
+        end
+
         def index_document_suffix
           info :index_document_suffix, super
           body = { website_configuration: { index_document: { suffix: super } } }

spec/dpl/providers/s3_spec.rb

@@ -42,8 +42,8 @@
 
   describe 'by default', record: true do
     it { should have_run '[info] Using Access Key: ac******************' }
-    it { should have_run '[info] Uploading 1 files with up to 5 threads.' }
-    it { should have_run '[info] Uploading file one.txt to / with acl=private content_type=text/plain cache_control=no-cache storage_class=STANDARD' }
+    it { should have_run '[info] Uploading 1 files with up to 5 threads ...' }
+    it { should have_run '[print] .' }
     it { should have_run_in_order }
     it { should put_object 'one.txt', host: 'bucket.s3.amazonaws.com', 'x-amz-acl': 'private', 'cache-control': 'no-cache', 'x-amz-storage-class': 'STANDARD' }
   end
@@ -57,32 +57,26 @@
   end
 
   describe 'given --upload_dir dir' do
-    it { should have_run %r(Uploading file one.txt to dir) }
     it { should put_object 'one.txt', path: '/dir/one.txt' }
   end
 
   describe 'given --dot_match' do
-    it { should have_run %r(.hidden.txt) }
     it { should put_object '.hidden.txt' }
   end
 
   describe 'given --storage_class STANDARD_IA' do
-    it { should have_run %r(one.txt.* storage_class=STANDARD_IA) }
     it { should put_object 'one.txt', 'x-amz-storage-class': 'STANDARD_IA' }
   end
 
   describe 'given --acl public_read' do
-    it { should have_run %r(one.txt.* acl=public-read) }
     it { should put_object 'one.txt', 'x-amz-acl': 'public-read' }
   end
 
   describe 'given --cache_control public' do
-    it { should have_run %r(one.txt.* cache_control=public) }
     it { should put_object 'one.txt', 'cache-control': 'public' }
   end
 
   describe 'given --cache_control max-age=60' do
-    it { should have_run %r(one.txt.* cache_control=max-age=60) }
     it { should put_object 'one.txt', 'cache-control': 'max-age=60' }
   end
 
@@ -92,7 +86,6 @@
   end
 
   describe 'given --expires "2020-01-01 00:00:00 UTC"' do
-    it { should have_run %r(one.txt.* expires=2020-01-01 00:00:00 UTC) }
     it { should put_object 'one.txt', 'expires': 'Wed, 01 Jan 2020 00:00:00 GMT' }
   end
 
@@ -102,17 +95,14 @@
   end
 
   describe 'given --default_text_charset utf-8' do
-    it { should have_run %r(one.txt.* content_type=text/plain; charset=utf-8) }
     it { should put_object 'one.txt', 'content-type': 'text/plain; charset=utf-8' }
   end
 
   describe 'given --server_side_encryption' do
-    it { should have_run %r(one.txt.* server_side_encryption=AES256) }
     it { should put_object 'one.txt', 'x-amz-server-side-encryption': 'AES256' }
   end
 
   describe 'given --detect_encoding' do
-    it { should have_run %r(one.txt.* content_encoding=text) }
     it { should put_object 'one.txt', 'content-encoding': 'text' }
   end
 
@@ -126,9 +116,7 @@
     file 'two/two.txt'
 
     before { subject.run }
-    it { should_not have_run %r(Uploading file one.txt) }
     it { should_not put_object 'one.txt' }
-    it { should have_run %r(Uploading file two.txt) }
     it { should put_object 'two.txt' }
   end