diff --git a/lib/dpl/providers/s3.rb b/lib/dpl/providers/s3.rb index 82988f087..2fe7d37f3 100644 --- a/lib/dpl/providers/s3.rb +++ b/lib/dpl/providers/s3.rb @@ -37,6 +37,7 @@ class S3 < Provider opt '--index_document_suffix SUFFIX', 'Index document suffix of a S3 website' opt '--default_text_charset CHARSET', 'Default character set to append to the content-type of text files' opt '--max_threads NUM', 'The number of threads to use for S3 file uploads', default: 5, max: 15, type: :integer + opt '--verbose', 'Be verbose about uploading files' # how come there is no glob or file option? msgs login: 'Using Access Key: %{access_key_id}', @@ -44,8 +45,8 @@ class S3 < Provider access_denied: 'It looks like you tried to write to a bucket that is not yours or does not exist. Please create the bucket before trying to write to it.', checksum_error: 'AWS secret key does not match the access key id', invalid_access_key_id: 'Invalid S3 access key id', - upload: 'Uploading %s files with up to %s threads.', - upload_file: 'Uploading file %s to %s with %s', + upload: 'Uploading %s files with up to %s threads ...', + upload_file: 'Uploading %s to %s with %s', upload_failed: 'Failed to upload %s', index_document_suffix: 'Setting index document suffix to %s' @@ -78,17 +79,26 @@ def upload info :upload, files.length, max_threads threads = max_threads.times.map { |i| Thread.new(&method(:upload_files)) } threads.each(&:join) + info "\n" unless verbose? end def upload_files while file = files.pop data = content_data(file) - info :upload_file, file, upload_dir || '/', to_pairs(data) + progress(file, data) object = s3.bucket(bucket).object(upload_path(file)) object.upload_file(file, data) || warn(:upload_failed, file) end end + def progress(file, data) + if verbose? + info :upload_file, file, upload_dir || '/', to_pairs(data) + else + print '.' + end + end + def index_document_suffix info :index_document_suffix, super body = { website_configuration: { index_document: { suffix: super } } } diff --git a/spec/dpl/providers/s3_spec.rb b/spec/dpl/providers/s3_spec.rb index 3c6869cb4..7b469fd73 100644 --- a/spec/dpl/providers/s3_spec.rb +++ b/spec/dpl/providers/s3_spec.rb @@ -42,8 +42,8 @@ describe 'by default', record: true do it { should have_run '[info] Using Access Key: ac******************' } - it { should have_run '[info] Uploading 1 files with up to 5 threads.' } - it { should have_run '[info] Uploading file one.txt to / with acl=private content_type=text/plain cache_control=no-cache storage_class=STANDARD' } + it { should have_run '[info] Uploading 1 files with up to 5 threads ...' } + it { should have_run '[print] .' } it { should have_run_in_order } it { should put_object 'one.txt', host: 'bucket.s3.amazonaws.com', 'x-amz-acl': 'private', 'cache-control': 'no-cache', 'x-amz-storage-class': 'STANDARD' } end @@ -57,32 +57,26 @@ end describe 'given --upload_dir dir' do - it { should have_run %r(Uploading file one.txt to dir) } it { should put_object 'one.txt', path: '/dir/one.txt' } end describe 'given --dot_match' do - it { should have_run %r(.hidden.txt) } it { should put_object '.hidden.txt' } end describe 'given --storage_class STANDARD_IA' do - it { should have_run %r(one.txt.* storage_class=STANDARD_IA) } it { should put_object 'one.txt', 'x-amz-storage-class': 'STANDARD_IA' } end describe 'given --acl public_read' do - it { should have_run %r(one.txt.* acl=public-read) } it { should put_object 'one.txt', 'x-amz-acl': 'public-read' } end describe 'given --cache_control public' do - it { should have_run %r(one.txt.* cache_control=public) } it { should put_object 'one.txt', 'cache-control': 'public' } end describe 'given --cache_control max-age=60' do - it { should have_run %r(one.txt.* cache_control=max-age=60) } it { should put_object 'one.txt', 'cache-control': 'max-age=60' } end @@ -92,7 +86,6 @@ end describe 'given --expires "2020-01-01 00:00:00 UTC"' do - it { should have_run %r(one.txt.* expires=2020-01-01 00:00:00 UTC) } it { should put_object 'one.txt', 'expires': 'Wed, 01 Jan 2020 00:00:00 GMT' } end @@ -102,17 +95,14 @@ end describe 'given --default_text_charset utf-8' do - it { should have_run %r(one.txt.* content_type=text/plain; charset=utf-8) } it { should put_object 'one.txt', 'content-type': 'text/plain; charset=utf-8' } end describe 'given --server_side_encryption' do - it { should have_run %r(one.txt.* server_side_encryption=AES256) } it { should put_object 'one.txt', 'x-amz-server-side-encryption': 'AES256' } end describe 'given --detect_encoding' do - it { should have_run %r(one.txt.* content_encoding=text) } it { should put_object 'one.txt', 'content-encoding': 'text' } end @@ -126,9 +116,7 @@ file 'two/two.txt' before { subject.run } - it { should_not have_run %r(Uploading file one.txt) } it { should_not put_object 'one.txt' } - it { should have_run %r(Uploading file two.txt) } it { should put_object 'two.txt' } end