Update fastlane spaceship to use the new TestFlight (#8871)

.gitignore

@@ -67,3 +67,6 @@ xcuserdata/
 *.moved-aside
 *.xccheckout
 *.xcscmblueprint
+
+# Dotenv
+.env

fastlane.gemspec

@@ -71,6 +71,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec_junit_formatter', '~> 0.2.3'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'pry-rescue'
+  spec.add_development_dependency 'pry-stack_explorer'
   spec.add_development_dependency 'yard', '~> 0.8.7.4'
   spec.add_development_dependency 'webmock', '~> 2.3.2'
   spec.add_development_dependency 'coveralls', '~> 0.8.13'

fastlane_core/lib/fastlane_core.rb

@@ -30,6 +30,7 @@
 require 'fastlane_core/fastlane_folder'
 require 'fastlane_core/keychain_importer'
 require 'fastlane_core/swag'
+require 'fastlane_core/build_watcher'
 
 # Third Party code
 require 'colored'

fastlane_core/lib/fastlane_core/build_watcher.rb

@@ -0,0 +1,37 @@
+module FastlaneCore
+  class BuildWatcher
+    # @return The build we waited for. This method will always return a build
+    def self.wait_for_build_processing_to_be_complete(app_id: nil, platform: nil)
+      # First, find the train and build version we want to watch for
+      processing_builds = Spaceship::TestFlight::Build.all_processing_builds(app_id: app_id, platform: platform)
+
+      watching_build = processing_builds.sort_by(&:upload_date).last # either it's still processing
+      watching_build ||= Spaceship::TestFlight::Build.latest(app_id: app_id, platform: platform) # or we fallback to the most recent uplaod
+
+      loop do
+        UI.message("Waiting for iTunes Connect to finish processing the new build (#{watching_build.train_version} - #{watching_build.build_version})")
+
+        # Due to iTunes Connect, builds disappear from the build list alltogether
+        # after they finished processing. Before returning this build, we have to
+        # wait for the build to appear in the build list again
+        # As this method is very often used to wait for a build, and then do something
+        # with it, we have to be sure that the build actually is ready
+
+        matching_builds = Spaceship::TestFlight::Build.builds_for_train(app_id: app_id, platform: platform, train_version: watching_build.train_version)
+        matching_build = matching_builds.find { |build| build.build_version == watching_build.build_version }
+
+        if matching_build.nil?
+          UI.message("Build doesn't show up in the build list any more, waiting for it to appear again")
+        elsif matching_build.active?
+          UI.success("Build #{matching_build.train_version} - #{matching_build.build_version} is already being tested")
+          return matching_build
+        elsif matching_build.ready_to_submit?
+          UI.success("Successfully finished processing the build #{matching_build.train_version} - #{matching_build.build_version}")
+          return matching_build
+        end
+
+        sleep 10
+      end
+    end
+  end
+end

pilot/lib/pilot/build_manager.rb

@@ -35,40 +35,17 @@ def upload(options)
       end
 
       UI.message("If you want to skip waiting for the processing to be finished, use the `skip_waiting_for_build_processing` option")
-      uploaded_build = wait_for_processing_build(options, platform) # this might take a while
+      latest_build = FastlaneCore::BuildWatcher.wait_for_build_processing_to_be_complete(app_id: app.apple_id, platform: platform)
 
-      distribute(options, uploaded_build)
+      distribute(options, latest_build)
     end
 
-    def distribute(options, build = nil)
+    def distribute(options, build)
       start(options)
       if config[:apple_id].to_s.length == 0 and config[:app_identifier].to_s.length == 0
         config[:app_identifier] = UI.input("App Identifier: ")
       end
 
-      if build.nil?
-        platform = fetch_app_platform(required: false)
-        builds = app.all_processing_builds(platform: platform) + app.builds(platform: platform)
-        # sort by upload_date
-        builds.sort! { |a, b| a.upload_date <=> b.upload_date }
-        build = builds.last
-        if build.nil?
-          UI.user_error!("No builds found.")
-          return
-        end
-        if build.processing
-          UI.user_error!("Build #{build.train_version}(#{build.build_version}) is still processing.")
-          return
-        end
-        if build.testing_status == "External"
-          UI.user_error!("Build #{build.train_version}(#{build.build_version}) has already been distributed.")
-          return
-        end
-
-        type = options[:distribute_external] ? 'External' : 'Internal'
-        UI.message("Distributing build #{build.train_version}(#{build.build_version}) from #{build.testing_status} -> #{type}")
-      end
-
       unless config[:update_build_info_on_upload]
         if should_update_build_information(options)
           build.update_build_information!(whats_new: options[:changelog], description: options[:beta_app_description], feedback_email: options[:beta_app_feedback_email])
@@ -128,88 +105,34 @@ def should_update_build_information(options)
       options[:changelog].to_s.length > 0 or options[:beta_app_description].to_s.length > 0 or options[:beta_app_feedback_email].to_s.length > 0
     end
 
-    # This method will takes care of checking for the processing builds every few seconds
-    # @return [Build] The build that we just uploaded
-    def wait_for_processing_build(options, platform)
-      # the upload date of the new buid
-      # we use it to identify the build
-      start = Time.now
-      wait_processing_interval = config[:wait_processing_interval].to_i
-      latest_build = nil
-      UI.message("Waiting for iTunes Connect to process the new build")
-      must_update_build_info = config[:update_build_info_on_upload]
-      loop do
-        sleep(wait_processing_interval)
-
-        # before we look for processing builds, we need to ensure that there
-        #  is a build train for this application; new applications don't
-        #  build trains right away, and if we don't do this check, we will
-        #  get break out of this loop and then generate an error later when we
-        #  have a nil build
-        if app.build_trains(platform: platform).count == 0
-          UI.message("New application; waiting for build train to appear on iTunes Connect")
-        else
-          builds = app.all_processing_builds(platform: platform)
-          latest_build = builds.last unless latest_build
-          break unless builds.include?(latest_build)
-
-          if latest_build.valid and must_update_build_info
-            # Set the changelog and/or description if necessary
-            if should_update_build_information(options)
-              latest_build.update_build_information!(whats_new: options[:changelog], description: options[:beta_app_description], feedback_email: options[:beta_app_feedback_email])
-              UI.success "Successfully set the changelog and/or description for build"
-            end
-            must_update_build_info = false
-          end
-
-          UI.message("Waiting for iTunes Connect to finish processing the new build (#{latest_build.train_version} - #{latest_build.build_version})")
-        end
-      end
+    def distribute_build(uploaded_build, options)
+      UI.message("Distributing new build to testers: #{uploaded_build.train_version} - #{uploaded_build.build_version}")
 
-      UI.user_error!("Error receiving the newly uploaded binary, please check iTunes Connect") if latest_build.nil?
-      full_build = nil
+      # This is where we could add a check to see if encryption is required and has been updated
+      uploaded_build.export_compliance.encryption_updated = false
+      uploaded_build.beta_review_info.demo_account_required = false
+      uploaded_build.submit_for_testflight_review!
 
-      while full_build.nil? || full_build.processing
-        # The build's processing state should go from true to false, and be done. But sometimes it goes true -> false ->
-        # true -> false, where the second true is transient. This causes a spurious failure. Find build by build_version
-        # and ensure it's not processing before proceeding - it had to have already been false before, to get out of the
-        # previous loop.
-        full_build = app.build_trains(platform: platform)[latest_build.train_version].builds.find do |b|
-          b.build_version == latest_build.build_version
-        end
+      if options[:distribute_external]
+        external_group = Spaceship::TestFlight::Group.default_external_group(app_id: uploaded_build.app_id)
+        uploaded_build.add_group!(external_group) unless external_group.nil?
 
-        UI.message("Waiting for iTunes Connect to finish processing the new build (#{latest_build.train_version} - #{latest_build.build_version})")
-        sleep(wait_processing_interval)
-      end
+        if external_group.nil? && options[:groups].nil?
+          UI.user_error!("You must specify at least one group using the `:groups` option to distribute externally")
+        end
 
-      if full_build && !full_build.processing && full_build.valid
-        minutes = ((Time.now - start) / 60).round
-        UI.success("Successfully finished processing the build")
-        UI.message("You can now tweet: ")
-        UI.important("iTunes Connect #iosprocessingtime #{minutes} minutes")
-        return full_build
-      else
-        UI.user_error!("Error: Seems like iTunes Connect didn't properly pre-process the binary")
       end
-    end
 
-    def distribute_build(uploaded_build, options)
-      UI.message("Distributing new build to testers")
-
-      # Submit for review before external testflight is available
-      if options[:distribute_external]
-        uploaded_build.client.submit_testflight_build_for_review!(
-          app_id: uploaded_build.build_train.application.apple_id,
-          train: uploaded_build.build_train.version_string,
-          build_number: uploaded_build.build_version,
-          platform: uploaded_build.platform
-        )
+      if options[:groups]
+        groups = Group.filter_groups(app_id: uploaded_build.app_id) do |group|
+          options[:groups].include?(group.name)
+        end
+        groups.each do |group|
+          uploaded_build.add_group!(group)
+        end
       end
 
-      # Submit for beta testing
-      type = options[:distribute_external] ? 'external' : 'internal'
-      uploaded_build.build_train.update_testing_status!(true, type, uploaded_build)
-      return true
+      true
     end
   end
 end

pilot/lib/pilot/tester_manager.rb

@@ -8,14 +8,8 @@ def add_tester(options)
       start(options)
 
       if config[:groups]
-        groups = Spaceship::Tunes::Tester::External.groups
-        selected_groups = []
-        config[:groups].each do |group|
-          group_id = groups.find { |k, v| v == group || k == group }
-          raise "Group '#{group}' not found for #{config[:email]}" unless group_id
-          selected_groups.push(group_id[0])
-        end
-        config[:groups] = selected_groups
+        UI.important("Currently pilot doesn't support groups yet, we're working on restoring that functionality")
+        config[:groups] = nil
       end
 
       begin
@@ -27,8 +21,7 @@ def add_tester(options)
         else
           tester = Spaceship::Tunes::Tester::External.create!(email: config[:email],
                                                               first_name: config[:first_name],
-                                                              last_name: config[:last_name],
-                                                              groups: config[:groups])
+                                                              last_name: config[:last_name])
           UI.success("Successfully invited tester: #{tester.email}")
         end
 
@@ -37,7 +30,7 @@ def add_tester(options)
           begin
             app = Spaceship::Application.find(app_filter)
             UI.user_error!("Couldn't find app with '#{app_filter}'") unless app
-            tester.add_to_app!(app.apple_id)
+            app.default_external_group.add_tester!(tester)
             UI.success("Successfully added tester to app #{app_filter}")
           rescue => ex
             UI.error("Could not add #{tester.email} to app: #{ex}")
@@ -74,7 +67,7 @@ def remove_tester(options)
           begin
             app = Spaceship::Application.find(app_filter)
             UI.user_error!("Couldn't find app with '#{app_filter}'") unless app
-            tester.remove_from_app!(app.apple_id)
+            app.default_external_group.remove_tester!(tester)
             UI.success("Successfully removed tester #{tester.email} from app #{app_filter}")
           rescue => ex
             UI.error("Could not remove #{tester.email} from app: #{ex}")

spaceship/lib/spaceship.rb

@@ -10,6 +10,7 @@
 # iTunes Connect
 require 'spaceship/tunes/tunes'
 require 'spaceship/tunes/spaceship'
+require 'spaceship/test_flight'
 
 # To support legacy code
 module Spaceship

spaceship/lib/spaceship/base.rb

@@ -20,6 +20,8 @@ module Spaceship
   # When you want to instantiate a model pass in the parsed response: `Widget.new(widget_json)`
   class Base
     class DataHash
+      include Enumerable
+
       def initialize(hash)
         @hash = hash || {}
       end
@@ -50,11 +52,19 @@ def lookup(keys)
         end
       end
 
+      def each(&block)
+        @hash.each(&block)
+      end
+
       def to_json(*a)
         h = @hash.dup
         h.delete(:application)
         h.to_json(*a)
       end
+
+      def to_h
+        @hash.dup
+      end
     end
 
     class << self

spaceship/lib/spaceship/client.rb

@@ -19,6 +19,7 @@
 end
 
 module Spaceship
+  # rubocop:disable Metrics/ClassLength
   class Client
     PROTOCOL_VERSION = "QH65B2"
     USER_AGENT = "Spaceship #{Fastlane::VERSION}"
@@ -126,14 +127,81 @@ def self.hostname
       raise "You must implement self.hostname"
     end
 
-    def initialize
+    # @return (Array) A list of all available teams
+    def teams
+      user_details_data['associatedAccounts'].sort_by do |team|
+        [
+          team['contentProvider']['name'],
+          team['contentProvider']['contentProviderId']
+        ]
+      end
+    end
+
+    def user_details_data
+      return @_cached_user_details if @_cached_user_details
+      r = request(:get, '/WebObjects/iTunesConnect.woa/ra/user/detail')
+      @_cached_user_details = parse_response(r, 'data')
+    end
+
+    # @return (String) The currently selected Team ID
+    def team_id
+      return @current_team_id if @current_team_id
+
+      if teams.count > 1
+        puts "The current user is in #{teams.count} teams. Pass a team ID or call `select_team` to choose a team. Using the first one for now."
+      end
+      @current_team_id ||= teams[0]['contentProvider']['contentProviderId']
+    end
+
+    # Set a new team ID which will be used from now on
+    def team_id=(team_id)
+      # First, we verify the team actually exists, because otherwise iTC would return the
+      # following confusing error message
+      #
+      #     invalid content provider id
+      #
+      available_teams = teams.collect do |team|
+        (team["contentProvider"] || {})["contentProviderId"]
+      end
+
+      result = available_teams.find do |available_team_id|
+        team_id.to_s == available_team_id.to_s
+      end
+
+      unless result
+        raise ITunesConnectError.new, "Could not set team ID to '#{team_id}', only found the following available teams: #{available_teams.join(', ')}"
+      end
+
+      response = request(:post) do |req|
+        req.url "ra/v1/session/webSession"
+        req.body = {
+          contentProviderId: team_id,
+          dsId: user_detail_data.ds_id # https://github.com/fastlane/fastlane/issues/6711
+        }.to_json
+        req.headers['Content-Type'] = 'application/json'
+      end
+
+      handle_itc_response(response.body)
+
+      @current_team_id = team_id
+    end
+
+    # Instantiates a client but with a cookie derived from another client.
+    #
+    # HACK: since the `@cookie` is not exposed, we use this hacky way of sharing the instance.
+    def self.client_with_authorization_from(another_client)
+      self.new(cookie: another_client.instance_variable_get(:@cookie), current_team_id: another_client.team_id)
+    end
+
+    def initialize(cookie: nil, current_team_id: nil)
       options = {
        request: {
           timeout:       (ENV["SPACESHIP_TIMEOUT"] || 300).to_i,
           open_timeout:  (ENV["SPACESHIP_TIMEOUT"] || 300).to_i
         }
       }
-      @cookie = HTTP::CookieJar.new
+      @current_team_id = current_team_id
+      @cookie = cookie || HTTP::CookieJar.new
       @client = Faraday.new(self.class.hostname, options) do |c|
         c.response :json, content_type: /\bjson$/
         c.response :xml, content_type: /\bxml$/
@@ -569,6 +637,7 @@ def encode_params(params, headers)
       return params, headers
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end
 
 require 'spaceship/two_step_client'