[match] Add s3_object_prefix option to match's S3Storage. (fastlane…

…#16682)

* Adds `s3_object_prefix option to match.

This option is used to add a prefix to all objects managed by match.
It does not automatically add a trailing `/` character to specify a
directory.

* Updates match runner to pass `s3_object_prefix` option to Storage

* Updates S3Storage to handle `s3_object_prefix` option

Will prepend this value to all target_paths of uploaded/deleted files
Will strip this value from the object_key when downloading files so the
path in the working directory is maintained.

* Update regex when stripping s3_object_prefix from object_path

* Reverts changes to Match runner spec

match/lib/match/options.rb

@@ -186,6 +186,10 @@ def self.available_options
                                      env_name: "MATCH_S3_BUCKET",
                                      description: "Name of the S3 bucket",
                                      optional: true),
+        FastlaneCore::ConfigItem.new(key: :s3_object_prefix,
+                                     env_name: "MATCH_S3_OBJECT_PREFIX",
+                                     description: "Prefix to be used on all objects uploaded to S3",
+                                     optional: true),
 
         # Keychain
         FastlaneCore::ConfigItem.new(key: :keychain_name,

match/lib/match/runner.rb

@@ -50,6 +50,7 @@ def run(params)
         s3_access_key: params[:s3_access_key].to_s,
         s3_secret_access_key: params[:s3_secret_access_key].to_s,
         s3_bucket: params[:s3_bucket].to_s,
+        s3_object_prefix: params[:s3_object_prefix],
         readonly: params[:readonly],
         username: params[:readonly] ? nil : params[:username], # only pass username if not readonly
         team_id: params[:team_id],

match/lib/match/storage/s3_storage.rb

@@ -14,6 +14,7 @@ class S3Storage < Interface
       attr_reader :s3_bucket
       attr_reader :s3_region
       attr_reader :s3_client
+      attr_reader :s3_object_prefix
       attr_reader :readonly
       attr_reader :username
       attr_reader :team_id
@@ -24,6 +25,7 @@ def self.configure(params)
         s3_access_key = params[:s3_access_key]
         s3_secret_access_key = params[:s3_secret_access_key]
         s3_bucket = params[:s3_bucket]
+        s3_object_prefix = params[:s3_object_prefix]
 
         if params[:git_url].to_s.length > 0
           UI.important("Looks like you still define a `git_url` somewhere, even though")
@@ -37,6 +39,7 @@ def self.configure(params)
           s3_access_key: s3_access_key,
           s3_secret_access_key: s3_secret_access_key,
           s3_bucket: s3_bucket,
+          s3_object_prefix: s3_object_prefix,
           readonly: params[:readonly],
           username: params[:username],
           team_id: params[:team_id],
@@ -48,13 +51,15 @@ def initialize(s3_region: nil,
                      s3_access_key: nil,
                      s3_secret_access_key: nil,
                      s3_bucket: nil,
+                     s3_object_prefix: nil,
                      readonly: nil,
                      username: nil,
                      team_id: nil,
                      team_name: nil)
         @s3_bucket = s3_bucket
         @s3_region = s3_region
         @s3_client = Fastlane::Helper::S3ClientHelper.new(access_key: s3_access_key, secret_access_key: s3_secret_access_key, region: s3_region)
+        @s3_object_prefix = s3_object_prefix.to_s
         @readonly = readonly
         @username = username
         @team_id = team_id
@@ -88,8 +93,10 @@ def download
         # No existing working directory, creating a new one now
         self.working_directory = Dir.mktmpdir
 
-        s3_client.find_bucket!(s3_bucket).objects.each do |object|
-          file_path = object.key # :team_id/path/to/file
+        s3_client.find_bucket!(s3_bucket).objects(prefix: s3_object_prefix).each do |object|
+          file_path = strip_s3_object_prefix(object.key) # :s3_object_prefix:team_id/path/to/file
+
+          # strip s3_prefix from file_path
           download_path = File.join(self.working_directory, file_path)
 
           FileUtils.mkdir_p(File.expand_path("..", download_path))
@@ -113,12 +120,11 @@ def upload_files(files_to_upload: [], custom_message: nil)
 
         files_to_upload.each do |file_name|
           # Go from
-          #   "/var/folders/px/bz2kts9n69g8crgv4jpjh6b40000gn/T/d20181026-96528-1av4gge/profiles/development/Development_me.mobileprovision"
+          #   "/var/folders/px/bz2kts9n69g8crgv4jpjh6b40000gn/T/d20181026-96528-1av4gge/:team_id/profiles/development/Development_me.mobileprovision"
           # to
-          #   "profiles/development/Development_me.mobileprovision"
+          #   ":s3_object_prefix:team_id/profiles/development/Development_me.mobileprovision"
           #
-
-          target_path = sanitize_file_name(file_name)
+          target_path = s3_object_path(file_name)
           UI.verbose("Uploading '#{target_path}' to S3 Storage...")
 
           body = File.read(file_name)
@@ -130,7 +136,8 @@ def upload_files(files_to_upload: [], custom_message: nil)
 
       def delete_files(files_to_delete: [], custom_message: nil)
         files_to_delete.each do |file_name|
-          target_path = sanitize_file_name(file_name)
+          target_path = s3_object_path(file_name)
+          UI.verbose("Deleting '#{target_path}' from S3 Storage...")
           s3_client.delete_file(s3_bucket, target_path)
         end
       end
@@ -152,6 +159,17 @@ def generate_matchfile_content(template: nil)
 
       private
 
+      def s3_object_path(file_name)
+        santized = sanitize_file_name(file_name)
+        return santized if santized.start_with?(s3_object_prefix)
+
+        s3_object_prefix + santized
+      end
+
+      def strip_s3_object_prefix(object_path)
+        object_path.gsub(/^#{s3_object_prefix}/, "")
+      end
+
       def sanitize_file_name(file_name)
         file_name.gsub(self.working_directory + "/", "")
       end

match/spec/runner_spec.rb

@@ -58,6 +58,7 @@
             s3_access_key: "",
             s3_secret_access_key: "",
             s3_bucket: "",
+            s3_object_prefix: nil,
             readonly: false,
             username: values[:username],
             team_id: nil,
@@ -139,6 +140,7 @@
             s3_access_key: "",
             s3_secret_access_key: "",
             s3_bucket: "",
+            s3_object_prefix: nil,
             readonly: false,
             username: values[:username],
             team_id: nil,
@@ -219,6 +221,7 @@
             s3_access_key: "",
             s3_secret_access_key: "",
             s3_bucket: "",
+            s3_object_prefix: nil,
             readonly: false,
             username: values[:username],
             team_id: nil,
@@ -285,6 +288,7 @@
             s3_access_key: "",
             s3_secret_access_key: "",
             s3_bucket: "",
+            s3_object_prefix: nil,
             readonly: false,
             username: values[:username],
             team_id: nil,

match/spec/storage/s3_storage_spec.rb

@@ -31,6 +31,14 @@
         expect(s3_client).to receive(:upload_file).with('foobar', 'ABCDEFG/certs/development/ABCDEFG.p12', 'body', 'private')
         subject.upload_files(files_to_upload: files_to_upload)
       end
+
+      it 'uploads files with s3_object_prefix if set' do
+        allow(subject).to receive(:s3_object_prefix).and_return('123456/')
+
+        expect(s3_client).to receive(:upload_file).with('foobar', '123456/ABCDEFG/certs/development/ABCDEFG.cer', 'body', 'private')
+        expect(s3_client).to receive(:upload_file).with('foobar', '123456/ABCDEFG/certs/development/ABCDEFG.p12', 'body', 'private')
+        subject.upload_files(files_to_upload: files_to_upload)
+      end
     end
 
     describe '#delete_files' do
@@ -48,6 +56,15 @@
 
         subject.delete_files(files_to_delete: files_to_upload)
       end
+
+      it 'deletes files with s3_object_prefix if set' do
+        allow(subject).to receive(:s3_object_prefix).and_return('123456/')
+
+        expect(s3_client).to receive(:delete_file).with('foobar', '123456/ABCDEFG/certs/development/ABCDEFG.cer')
+        expect(s3_client).to receive(:delete_file).with('foobar', '123456/ABCDEFG/certs/development/ABCDEFG.p12')
+
+        subject.delete_files(files_to_delete: files_to_upload)
+      end
     end
 
     describe '#download' do
@@ -68,6 +85,16 @@
 
         subject.download
       end
+
+      it 'downloads files and strips the s3_object_prefix for working_directory path' do
+        allow(subject).to receive(:s3_object_prefix).and_return('123456/')
+
+        files_to_download.each do |file_object|
+          expect(file_object).to receive(:download_file).with("#{working_directory}/#{file_object.key}")
+        end
+
+        subject.download
+      end
     end
   end
 end