[match] add caching layer to significantly improve performance by up …

…to 100x (#21694)

* [match] increase match speed with caching

* chore: skip new profile validation

* chore: don’t install a nonexistent profile

* chore: code readability improvements

* chore: improve cert & device difference

* chore: fix variable naming

* chore: remove redundant var init

* chore: check for unique profiles

* fix: typo

* chore: match portal bundle id fetcher expect only arrays

* fix: only uuids with one hyphen supported for silicon macs

* chore: add comment about cert type downcasing

Co-authored-by: Roger Oba <rogerluan.oba@gmail.com>

* chore: use standard syntax for multiline blocks

* chore: remove empty it

* chore: update cache returns

* chore: extra check for cached certs after reset

---------

Co-authored-by: Roger Oba <rogerluan.oba@gmail.com>

match/lib/match.rb

@@ -12,3 +12,6 @@
 require_relative 'match/storage'
 require_relative 'match/encryption'
 require_relative 'match/module'
+require_relative 'match/portal_cache'
+require_relative 'match/portal_fetcher'
+require_relative 'match/profile_includes'

match/lib/match/generator.rb

@@ -1,4 +1,5 @@
 require_relative 'module'
+require_relative 'profile_includes'
 
 module Match
   # Generate missing resources
@@ -57,7 +58,7 @@ def self.generate_certificate(params, cert_type, working_directory, specific_cer
     end
 
     # @return (String) The UUID of the newly generated profile
-    def self.generate_provisioning_profile(params: nil, prov_type: nil, certificate_id: nil, app_identifier: nil, force: true, working_directory: nil)
+    def self.generate_provisioning_profile(params: nil, prov_type: nil, certificate_id: nil, app_identifier: nil, force: true, cache: nil, working_directory: nil)
       require 'sigh/manager'
       require 'sigh/options'
 
@@ -104,6 +105,13 @@ def self.generate_provisioning_profile(params: nil, prov_type: nil, certificate_
         values[:development] = true
       end
 
+      if cache
+        values[:cached_certificates] = cache.certificates
+        values[:cached_devices] = cache.devices
+        values[:cached_bundle_ids] = cache.bundle_ids
+        values[:cached_profiles] = cache.profiles
+      end
+
       arguments = FastlaneCore::Configuration.create(Sigh::Options.available_options, values)
 
       Sigh.config = arguments

match/lib/match/module.rb

@@ -23,7 +23,8 @@ def self.profile_type_sym(type)
   end
 
   def self.cert_type_sym(type)
-    type = type.to_s
+    # To determine certificate types to fetch from the portal, we use `Sigh.certificate_types_for_profile_and_platform`, and it returns typed `Spaceship::ConnectAPI::Certificate::CertificateType` with the same values but uppercased, so we downcase them here
+    type = type.to_s.downcase
     return :mac_installer_distribution if type == "mac_installer_distribution"
     return :developer_id_installer if type == "developer_id_installer"
     return :developer_id_application if type == "developer_id"

match/lib/match/portal_cache.rb

@@ -0,0 +1,106 @@
+require 'fastlane_core/provisioning_profile'
+require 'spaceship/client'
+require_relative 'portal_fetcher'
+module Match
+  class Portal
+    class Cache
+      def self.build(params:, bundle_id_identifiers:)
+        require_relative 'profile_includes'
+        require 'sigh'
+
+        profile_type = Sigh.profile_type_for_distribution_type(
+          platform: params[:platform],
+          distribution_type: params[:type]
+        )
+
+        cache = Portal::Cache.new(
+          platform: params[:platform],
+          profile_type: profile_type,
+          additional_cert_types: params[:additional_cert_types],
+          bundle_id_identifiers: bundle_id_identifiers,
+          needs_profiles_devices: ProfileIncludes.can_force_include?(params: params, notify: true) && !params[:force] && !params[:readonly],
+          needs_profiles_certificate_content: !ProfileIncludes.can_force_include_all_certificates?(params: params),
+          include_mac_in_profiles: params[:include_mac_in_profiles]
+        )
+
+        return cache
+      end
+
+      attr_reader :platform, :profile_type, :bundle_id_identifiers, :additional_cert_types, :needs_profiles_devices, :needs_profiles_certificate_content, :include_mac_in_profiles
+
+      def initialize(platform:, profile_type:, additional_cert_types:, bundle_id_identifiers:, needs_profiles_devices:, needs_profiles_certificate_content:, include_mac_in_profiles:)
+        @platform = platform
+        @profile_type = profile_type
+
+        # Bundle Ids
+        @bundle_id_identifiers = bundle_id_identifiers
+
+        # Certs
+        @additional_cert_types = additional_cert_types
+
+        # Profiles
+        @needs_profiles_devices = needs_profiles_devices
+        @needs_profiles_certificate_content = needs_profiles_certificate_content
+
+        # Devices
+        @include_mac_in_profiles = include_mac_in_profiles
+      end
+
+      def portal_profile(stored_profile_path:, keychain_path:)
+        parsed = FastlaneCore::ProvisioningProfile.parse(stored_profile_path, keychain_path)
+        uuid = parsed["UUID"]
+
+        portal_profile = self.profiles.detect { |i| i.uuid == uuid }
+
+        portal_profile
+      end
+
+      def reset_certificates
+        @certificates = nil
+      end
+
+      def forget_portal_profile(portal_profile)
+        return unless @profiles && portal_profile
+
+        @profiles -= [portal_profile]
+      end
+
+      def bundle_ids
+        @bundle_ids ||= Match::Portal::Fetcher.bundle_ids(
+          bundle_id_identifiers: @bundle_id_identifiers
+        )
+
+        return @bundle_ids.dup
+      end
+
+      def certificates
+        @certificates ||= Match::Portal::Fetcher.certificates(
+          platform: @platform,
+          profile_type: @profile_type,
+          additional_cert_types: @additional_cert_types
+        )
+
+        return @certificates.dup
+      end
+
+      def profiles
+        @profiles ||= Match::Portal::Fetcher.profiles(
+          profile_type: @profile_type,
+          needs_profiles_devices: @needs_profiles_devices,
+          needs_profiles_certificate_content: @needs_profiles_certificate_content
+        )
+
+        return @profiles.dup
+      end
+
+      def devices
+        @devices ||= Match::Portal::Fetcher.devices(
+          platform: @platform,
+          include_mac_in_profiles: @include_mac_in_profiles
+        )
+
+        return @devices.dup
+      end
+    end
+  end
+end

match/lib/match/portal_fetcher.rb

@@ -0,0 +1,72 @@
+require 'fastlane_core/provisioning_profile'
+require 'spaceship/client'
+require 'spaceship/connect_api/models/profile'
+
+module Match
+  class Portal
+    module Fetcher
+      def self.profiles(profile_type:, needs_profiles_devices: false, needs_profiles_certificate_content: false, name: nil)
+        includes = ['bundleId']
+
+        if needs_profiles_devices
+          includes += ['devices', 'certificates']
+        end
+
+        if needs_profiles_certificate_content
+          includes += ['certificates']
+        end
+
+        profiles = Spaceship::ConnectAPI::Profile.all(
+          filter: { profileType: profile_type, name: name }.compact,
+          includes: includes.uniq.join(',')
+        )
+
+        profiles
+      end
+
+      def self.certificates(platform:, profile_type:, additional_cert_types:)
+        require 'sigh'
+        certificate_types = Sigh.certificate_types_for_profile_and_platform(platform: platform, profile_type: profile_type)
+
+        additional_cert_types ||= []
+        additional_cert_types.map! do |cert_type|
+          case Match.cert_type_sym(cert_type)
+          when :mac_installer_distribution
+            Spaceship::ConnectAPI::Certificate::CertificateType::MAC_INSTALLER_DISTRIBUTION
+          when :developer_id_installer
+            Spaceship::ConnectAPI::Certificate::CertificateType::DEVELOPER_ID_INSTALLER
+          end
+        end
+
+        certificate_types += additional_cert_types
+
+        filter = { certificateType: certificate_types.uniq.sort.join(',') } unless certificate_types.empty?
+
+        certificates = Spaceship::ConnectAPI::Certificate.all(
+          filter: filter
+        ).select(&:valid?)
+
+        certificates
+      end
+
+      def self.devices(platform: nil, include_mac_in_profiles: false)
+        devices = Spaceship::ConnectAPI::Device.devices_for_platform(
+          platform: platform,
+          include_mac_in_profiles: include_mac_in_profiles
+        )
+
+        devices
+      end
+
+      def self.bundle_ids(bundle_id_identifiers: nil)
+        filter = { identifier: bundle_id_identifiers.join(',') } if bundle_id_identifiers
+
+        bundle_ids = Spaceship::ConnectAPI::BundleId.all(
+          filter: filter
+        )
+
+        bundle_ids
+      end
+    end
+  end
+end

match/lib/match/profile_includes.rb

@@ -0,0 +1,120 @@
+require_relative 'portal_fetcher'
+require_relative 'module'
+
+module Match
+  class ProfileIncludes
+    PROV_TYPES_WITH_DEVICES = [:adhoc, :development]
+    PROV_TYPES_WITH_MULTIPLE_CERTIFICATES = [:development]
+
+    def self.can_force_include?(params:, notify:)
+      self.can_force_include_all_devices?(params: params, notify: notify) &&
+        self.can_force_include_all_certificates?(params: params, notify: notify)
+    end
+
+    ###############
+    #
+    # DEVICES
+    #
+    ###############
+
+    def self.should_force_include_all_devices?(params:, portal_profile:, cached_devices:)
+      return false unless self.can_force_include_all_devices?(params: params)
+
+      force = devices_differ?(portal_profile: portal_profile, platform: params[:platform], include_mac_in_profiles: params[:include_mac_in_profiles], cached_devices: cached_devices)
+
+      return force
+    end
+
+    def self.can_force_include_all_devices?(params:, notify: false)
+      return false if params[:readonly] || params[:force]
+      return false unless params[:force_for_new_devices]
+
+      provisioning_type = params[:type].to_sym
+
+      can_force = PROV_TYPES_WITH_DEVICES.include?(provisioning_type)
+
+      if !can_force && notify
+        # App Store provisioning profiles don't contain device identifiers and
+        # thus shouldn't be renewed if the device count has changed.
+        UI.important("Warning: `force_for_new_devices` is set but is ignored for #{provisioning_type}.")
+        UI.important("You can safely stop specifying `force_for_new_devices` when running Match for type '#{provisioning_type}'.")
+      end
+
+      can_force
+    end
+
+    def self.devices_differ?(portal_profile:, platform:, include_mac_in_profiles:, cached_devices:)
+      return false unless portal_profile
+
+      profile_devices = portal_profile.devices
+
+      portal_devices = cached_devices
+      portal_devices ||= Match::Portal::Fetcher.devices(platform: platform, include_mac_in_profiles: include_mac_in_profiles)
+
+      profile_device_ids = profile_devices.map(&:id).sort
+      portal_devices_ids = portal_devices.map(&:id).sort
+
+      devices_differs = profile_device_ids != portal_devices_ids
+
+      UI.important("Devices in the profile and available on the portal differ. Recreating a profile") if devices_differs
+
+      return devices_differs
+    end
+
+    ###############
+    #
+    # CERTIFICATES
+    #
+    ###############
+
+    def self.should_force_include_all_certificates?(params:, portal_profile:, cached_certificates:)
+      return false unless self.can_force_include_all_certificates?(params: params)
+
+      force = certificates_differ?(portal_profile: portal_profile, platform: params[:platform], cached_certificates: cached_certificates)
+
+      return force
+    end
+
+    def self.can_force_include_all_certificates?(params:, notify: false)
+      return false if params[:readonly] || params[:force]
+      return false unless params[:force_for_new_certificates]
+
+      unless params[:include_all_certificates]
+        UI.important("You specified 'force_for_new_certificates: true', but new certificates will not be added, cause 'include_all_certificates' is 'false'") if notify
+        return false
+      end
+
+      provisioning_type = params[:type].to_sym
+
+      can_force = PROV_TYPES_WITH_MULTIPLE_CERTIFICATES.include?(provisioning_type)
+
+      if !can_force && notify
+        # All other (not development) provisioning profiles don't contain
+        # multiple certificates, thus shouldn't be renewed
+        # if the certificates  count has changed.
+        UI.important("Warning: `force_for_new_certificates` is set but is ignored for non-'development' provisioning profiles.")
+        UI.important("You can safely stop specifying `force_for_new_certificates` when running Match for '#{provisioning_type}' provisioning profiles.")
+      end
+
+      can_force
+    end
+
+    def self.certificates_differ?(portal_profile:, platform:, cached_certificates:)
+      return false unless portal_profile
+
+      profile_certs = portal_profile.certificates
+
+      portal_certs = cached_certificates
+      portal_certs ||= Match::Portal::Fetcher.certificates(platform: platform, profile_type: portal_profile.profile_type)
+
+      profile_certs_ids = profile_certs.map(&:id).sort
+      portal_certs_ids = portal_certs.map(&:id).sort
+
+      certificates_differ = profile_certs_ids != portal_certs_ids
+
+      UI.important("Certificates in the profile and available on the portal differ. Recreating a profile") if certificates_differ
+
+      return certificates_differ
+    end
+  end
+end