Fixes #13987 - Capsule Sync Optimization

app/lib/actions/katello/capsule_content/configure_capsule.rb

@@ -5,7 +5,7 @@ class ConfigureCapsule < ::Actions::EntryAction
         def plan(capsule, environment, content_view)
           sequence do
             plan_action(RemoveUnneededRepos, capsule)
-            plan_action(CreateOrUpdate, capsule, environment, content_view)
+            plan_action(CreateRepos, capsule, environment, content_view)
           end
         end
       end

app/lib/actions/katello/capsule_content/create_or_update.rb → app/lib/actions/katello/capsule_content/create_repos.rb

@@ -1,23 +1,18 @@
 module Actions
   module Katello
     module CapsuleContent
-      class CreateOrUpdate < ::Actions::EntryAction
+      class CreateRepos < ::Actions::EntryAction
         # @param capsule_content [::Katello::CapsuleContent]
         def plan(capsule_content, environment = nil, content_view = nil)
           fail _("Action not allowed for the default capsule.") if capsule_content.default_capsule?
 
           current_repos_on_capsule = capsule_content.current_repositories(environment, content_view)
           list_of_repos_to_sync = capsule_content.repos_available_to_capsule(environment, content_view)
           need_creation =  list_of_repos_to_sync - current_repos_on_capsule
-          need_update = current_repos_on_capsule & list_of_repos_to_sync
 
           need_creation.each do |repo|
             create_repo_in_pulp(capsule_content, repo)
           end
-
-          need_update.each do |repo|
-            plan_action(Pulp::Repository::Refresh, repo, capsule_id: capsule_content.capsule.id)
-          end
         end
 
         def create_repo_in_pulp(capsule_content, repository)
@@ -37,7 +32,7 @@ def create_repo_in_pulp(capsule_content, repository)
                       checksum_type: checksum_type,
                       path: relative_path,
                       with_importer: true,
-                      docker_upstream_name: repository.try(:docker_upstream_name),
+                      docker_upstream_name: repository.pulp_id,
                       download_policy: repository.capsule_download_policy,
                       capsule_id: capsule_content.capsule.id)
         end

app/lib/actions/katello/capsule_content/sync.rb

@@ -19,6 +19,11 @@ def plan(capsule_content, options = {})
 
           fail _("Action not allowed for the default capsule.") if capsule_content.default_capsule?
 
+          need_updates = repos_needing_updates(capsule_content, environment, content_view)
+          need_updates.each do |repo|
+            plan_action(Pulp::Repository::Refresh, repo, capsule_id: capsule_content.capsule.id)
+          end
+
           repository_ids = get_repository_ids(capsule_content, environment, content_view, repository)
           unless repository_ids.blank?
             sequence do
@@ -54,6 +59,32 @@ def get_repository_ids(capsule, environment, content_view, repository)
           repository_ids
         end
 
+        def repos_needing_updates(capsule_content, environment, content_view)
+          need_importer_update = repos_needing_importer_updates(capsule_content, environment, content_view)
+          need_distributor_update = repos_needing_distributor_updates(capsule_content, environment, content_view)
+          (need_distributor_update && need_importer_update).uniq
+        end
+
+        def repos_needing_distributor_updates(capsule, environment, content_view)
+          repos = capsule.repos_available_to_capsule(environment, content_view)
+          repos.select do |repo|
+            repo_details = capsule.pulp_repo_facts(repo.pulp_id)
+            next unless repo_details
+            capsule_distributors = repo_details["distributors"]
+            !(repo.distributors_match?(capsule_distributors))
+          end
+        end
+
+        def repos_needing_importer_updates(capsule, environment, content_view)
+          repos = capsule.repos_available_to_capsule(environment, content_view)
+          repos.select do |repo|
+            repo_details = capsule.pulp_repo_facts(repo.pulp_id)
+            next unless repo_details
+            capsule_importer = repo_details["importers"][0]["config"]
+            !(repo.importer_matches?(capsule_importer))
+          end
+        end
+
         def rescue_strategy
           Dynflow::Action::Rescue::Skip
         end

app/lib/katello/capsule_content.rb

@@ -140,6 +140,12 @@ def pulp_url
       self.capsule.url + "/pulp/api/v2/"
     end
 
+    def pulp_repo_facts(pulp_id)
+      self.pulp_server.extensions.repository.retrieve_with_details(pulp_id)
+    rescue RestClient::ResourceNotFound
+      nil
+    end
+
     def self.with_environment(environment, include_default = false)
       features = [SmartProxy::PULP_NODE_FEATURE]
       features << SmartProxy::PULP_FEATURE if include_default

app/models/katello/glue/pulp/repo.rb

@@ -130,7 +130,7 @@ def generate_importer(capsule = false)
           Runcible::Models::IsoImporter.new(importer_ssl_options(capsule).merge(:feed => importer_feed_url(capsule)))
         when Repository::PUPPET_TYPE
           options = {}
-          options[:feed] = importer_feed_url(capsule) if self.respond_to?(:url)
+          options[:feed] = importer_feed_url(capsule)
           Runcible::Models::PuppetImporter.new(options)
         when Repository::DOCKER_TYPE
           options = {}
@@ -777,8 +777,33 @@ def capsule_download_policy
         end
       end
 
+      def distributors_match?(capsule_distributors)
+        generated_distributors = self.generate_distributors.map(&:as_json)
+        capsule_distributors.each do |dist|
+          dist.merge!(dist["config"])
+          dist.delete("config")
+        end
+
+        generated_distributors.any? do |gen_dist|
+          capsule_distributors.any? do |cap_dist|
+            (gen_dist.to_a - cap_dist.to_a).empty?
+          end
+        end
+      end
+
+      def importer_matches?(capsule_importer)
+        generated_importer = self.generate_importer(true).as_json
+        (generated_importer.to_a - capsule_importer.to_a).empty?
+      end
+
       protected
 
+      def object_to_hash(object)
+        hash = {}
+        object.instance_variables.each { |var| hash[var.to_s.delete("@")] = object.instance_variable_get(var) }
+        hash
+      end
+
       def _get_most_recent_sync_status
         begin
           history = Katello.pulp_server.extensions.repository.sync_status(pulp_id)

test/actions/katello/capsule_content_test.rb

@@ -53,14 +53,22 @@ class SyncTest < TestBase
 
     it 'plans' do
       capsule_content.add_lifecycle_environment(environment)
+      action_class.any_instance.expects(:repos_needing_updates).returns([repository])
       capsule_content_sync = plan_action(action, capsule_content)
+
       synced_repos = synced_repos(capsule_content_sync, capsule_content.repos_available_to_capsule)
 
       assert_equal synced_repos.sort.uniq, capsule_content.repos_available_to_capsule.map { |repo| repo.pulp_id }.sort.uniq
+      assert_action_planed_with(action,
+                                ::Actions::Pulp::Repository::Refresh,
+                                repository,
+                                :capsule_id => capsule_content.capsule.id
+                                )
     end
 
     it 'allows limiting scope of the syncing to one environment' do
       capsule_content.add_lifecycle_environment(dev_environment)
+      action_class.any_instance.expects(:repos_needing_updates).returns([])
       capsule_content_sync = plan_action(action, capsule_content, :environment => dev_environment)
       synced_repos = synced_repos(capsule_content_sync, capsule_content.repos_available_to_capsule)
 
@@ -73,6 +81,7 @@ class SyncTest < TestBase
         plan_action(action, capsule_content, :environment => dev_environment)
       end
     end
+
     it 'fails when trying to sync a lifecyle environment that is not attached' do
       capsule_content.add_lifecycle_environment(environment)
 
@@ -83,10 +92,10 @@ class SyncTest < TestBase
     end
   end
 
-  class CreateOrUpdateTest < TestBase
+  class CreateReposTest < TestBase
     include VCR::TestCase
 
-    let(:action_class) { ::Actions::Katello::CapsuleContent::CreateOrUpdate }
+    let(:action_class) { ::Actions::Katello::CapsuleContent::CreateRepos }
 
     let(:cert) do
       {
@@ -132,23 +141,11 @@ class CreateOrUpdateTest < TestBase
                          checksum_type: repository.checksum_type,
                          path: repository.relative_path,
                          with_importer: true,
-                         docker_upstream_name: repository.try(:docker_upstream_name),
+                         docker_upstream_name: repository.pulp_id,
                          capsule_id: capsule_content.capsule.id
                         )
       end
     end
-
-    it "updates repos on the capsule" do
-      capsule_content.stubs(:current_repositories).returns([repository])
-      capsule_content.stubs(:repos_available_to_capsule).returns([repository])
-
-      action = create_and_plan_action(action_class, capsule_content)
-      assert_action_planed_with(action,
-                                ::Actions::Pulp::Repository::Refresh,
-                                repository,
-                                :capsule_id => capsule_content.capsule.id
-                                )
-    end
   end
 
   class RemoveUnneededReposTest < TestBase