Fixes #24902 - pass in list of packages when publishing CV

Previously, the only way to control which packages land in content
view version was by using filters, or by managing the contents of the
Library versions of the repos.

This commit lets you optionally specify the exact package set you want
in your repository. It will look at the Library version of each repo,
and copy the list of packages into the CV version.

Example value for `repos_units` when calling
`/katello/api/v2/content_views/2/publish`:

```json
[
    {
        "repo_label": "zoo",
        "rpm_filenames": [
            "walrus-5.21-1.noarch.rpm",
            "gorilla-0.62-1.noarch.rpm"
        ]
    },
    {
        "repo_label": "a_longer_label",
        "rpm_filenames": [
            "facter-2.4.6-3.el7sat.x86_64.rpm",
            "pulp-rpm-handlers-2.13.4.9-1.el7sat.noarch.rpm"
        ]
    }
]
```

This works the same for custom and RH repos.

When used with Katello#7594, you can
set the CV version number as well as the list of packages in each
repo. This is useful for cloning a CV version from one Katello to
another.

app/controllers/katello/api/v2/content_views_controller.rb

@@ -81,11 +81,23 @@ def update
                                                            "in the content view version")
     param :major, :number, :desc => N_("Override the major version number"), :required => false
     param :minor, :number, :desc => N_("Override the minor version number"), :required => false
+
+    param :repos_units, Hash, :desc => N_("Specify the list of units in each repo"), :required => false do
+      param :repo_label, String
+      param :rpm_filenames, Array do
+        param :filename, String
+      end
+    end
     def publish
+      if params[:repos_units].present? && @view.composite?
+        fail HttpErrors::BadRequest, _("Directly setting package lists on composite content views is not allowed. Please " \
+                                     "update the components, then re-publish the composite.")
+      end
       task = async_task(::Actions::Katello::ContentView::Publish, @view, params[:description],
                         :force_yum_metadata_regeneration => params[:force_yum_metadata_regeneration],
                         :major => params[:major],
-                        :minor => params[:minor])
+                        :minor => params[:minor],
+                        :repos_units => params[:repos_units])
       respond_for_async :resource => task
     end
 

app/lib/actions/katello/content_view/publish.rb

@@ -9,11 +9,26 @@ class Publish < Actions::EntryAction
         def plan(content_view, description = "", options = {})
           action_subject(content_view)
           content_view.check_ready_to_publish!
+
+          if options[:repos_units].present?
+            valid_labels_from_cv = content_view.library_repos.map { |r| r.label }
+            labels_from_repos_units = options[:repos_units].map { |l| l[:repo_label] }
+
+            labels_from_repos_units.each do |label|
+              fail _("Repository label '%s' is not associated with content view.") % label unless valid_labels_from_cv.include? label
+            end
+
+            valid_labels_from_cv.each do |label|
+              fail _("Content view has repository label '%s' which is not specified in repos_units parameter.") % label unless labels_from_repos_units.include? label
+            end
+          end
+
           if options[:minor] && options[:major]
             version = content_view.create_new_version(options[:major], options[:minor])
           else
             version = content_view.create_new_version
           end
+
           library = content_view.organization.library
           history = ::Katello::ContentViewHistory.create!(:content_view_version => version,
                                                           :user => ::User.current.login,
@@ -29,7 +44,7 @@ def plan(content_view, description = "", options = {})
             concurrence do
               content_view.publish_repositories do |repositories|
                 sequence do
-                  clone_to_version = plan_action(Repository::CloneToVersion, repositories, version)
+                  clone_to_version = plan_action(Repository::CloneToVersion, repositories, version, :repos_units => options[:repos_units])
                   plan_action(Repository::CloneToEnvironment, clone_to_version.new_repository, library,
                               :force_yum_metadata_regeneration => options[:force_yum_metadata_regeneration])
                 end

app/lib/actions/katello/content_view_version/incremental_update.rb

@@ -66,7 +66,7 @@ def repos_to_copy(old_version, new_components)
         def copy_repos(source_repos, new_version, content, dep_solve)
           copy_output = []
           sequence do
-            new_repo = plan_action(Repository::CloneToVersion, source_repos, new_version, true).new_repository
+            new_repo = plan_action(Repository::CloneToVersion, source_repos, new_version, :incremental => true).new_repository
             copy_output = copy_yum_content(new_repo, dep_solve, content[:package_ids], content[:errata_ids])
 
             plan_action(Katello::Repository::MetadataGenerate, new_repo)

app/lib/actions/katello/repository/clone_to_version.rb

@@ -5,13 +5,16 @@ class CloneToVersion < Actions::Base
         # allows accessing the build object from the superior action
         attr_accessor :new_repository
 
-        def plan(repositories, content_view_version, incremental = false)
+        def plan(repositories, content_view_version, options = {})
+          incremental = options.fetch(:incremental, false)
           content_view = content_view_version.content_view
           filters = incremental ? [] : content_view.filters.applicable(repositories.first)
 
           self.new_repository = repositories.first.build_clone(content_view: content_view,
                                                                version: content_view_version)
 
+          rpm_filenames = extract_rpm_filenames(options.fetch(:repos_units, nil), repositories.first.label)
+
           sequence do
             plan_action(Repository::Create, new_repository, true, false)
 
@@ -22,8 +25,8 @@ def plan(repositories, content_view_version, incremental = false)
             else
               repositories.each do |repository|
                 if new_repository.yum?
-                  plan_action(Repository::CloneYumContent, repository, new_repository, filters, !incremental,
-                              :generate_metadata => !incremental, :index_content => !incremental, :simple_clone => incremental)
+                  plan_action(Repository::CloneYumContent, repository, new_repository, filters, :purge_empty_units => !incremental,
+                              :generate_metadata => !incremental, :index_content => !incremental, :simple_clone => incremental, :rpm_filenames => rpm_filenames)
                 elsif new_repository.deb?
                   plan_action(Repository::CloneDebContent, repository, new_repository, filters, !incremental,
                               :generate_metadata => !incremental, :index_content => !incremental, :simple_clone => incremental)
@@ -38,6 +41,15 @@ def plan(repositories, content_view_version, incremental = false)
             end
           end
         end
+
+        def extract_rpm_filenames(repos_units, repo_label)
+          return if repos_units.blank?
+
+          repo_units = repos_units.detect { |r| r[:repo_label] == repo_label }
+          return if repo_units.blank?
+
+          repo_units.fetch(:rpm_filenames, nil)
+        end
       end
     end
   end

app/lib/actions/katello/repository/clone_yum_content.rb

@@ -3,9 +3,13 @@ module Katello
     module Repository
       class CloneYumContent < Actions::Base
         # rubocop:disable MethodLength
-        def plan(source_repo, target_repo, filters, purge_empty_units, options = {})
+        # rubocop:disable Metrics/CyclomaticComplexity TODO: refactor to push everything into options hash
+        # rubocop:disable Metrics/PerceivedComplexity
+        def plan(source_repo, target_repo, filters, options = {})
           generate_metadata = options.fetch(:generate_metadata, true)
           index_content = options.fetch(:index_content, true)
+          rpm_filenames = options.fetch(:rpm_filenames, {})
+          purge_empty_units = options.fetch(:purge_empty_units, {})
 
           copy_clauses = nil
           remove_clauses = nil
@@ -19,6 +23,19 @@ def plan(source_repo, target_repo, filters, purge_empty_units, options = {})
             remove_clauses = clause_gen.remove_clause
           end
 
+          # if we are providing a list of files, process that here and override filters
+          if rpm_filenames.present?
+            # log a warning if we are overriding the list of RPMs and using a filter
+            # ensure we have all the files we want to copy
+            rpms_available = source_repo.rpms.pluck(:filename)
+            rpm_filenames.each do |filename|
+              fail "%s not available in repository %s" % [filename, source_repo.label] unless rpms_available.include? filename
+            end
+            copy_clauses = { 'filename' => { '$in' => rpm_filenames } }
+            remove_clauses = nil
+            Rails.logger.warn("Filters on content view have been overridden by passed-in filename list during publish") if filters.any?
+          end
+
           sequence do
             plan_copy(Pulp::Repository::CopySrpm, source_repo, target_repo)
 
@@ -60,7 +77,7 @@ def plan(source_repo, target_repo, filters, purge_empty_units, options = {})
               plan_action(Katello::Repository::IndexPackageGroups, target_repo)
             end
 
-            source_repository = filters.empty? ? source_repo : nil
+            source_repository = filters.empty? && rpm_filenames.empty? ? source_repo : nil
 
             if generate_metadata
               plan_action(Katello::Repository::MetadataGenerate,

app/lib/actions/katello/repository/export.rb

@@ -71,7 +71,7 @@ def copy_units(repos)
               sequence do
                 if repo.link?
                   plan_action(Katello::Repository::Clear, repo)
-                  plan_action(Katello::Repository::CloneYumContent, repo.target_repository, repo, [], false,
+                  plan_action(Katello::Repository::CloneYumContent, repo.target_repository, repo, [], :purge_empty_units => false,
                               :generate_metadata => false, :index_content => false)
                 end
               end

test/actions/katello/repository/clone_to_version_test.rb

@@ -22,23 +22,25 @@ def setup
       action = create_action(action_class)
       cloned_repo.expects(:link?).returns(false)
       yum_repo.expects(:build_clone).returns(cloned_repo)
+      options = {}
 
-      plan_action(action, [yum_repo], version)
+      plan_action(action, [yum_repo], version, options)
 
       assert_action_planed_with(action, Actions::Katello::Repository::CloneYumContent,
-                                yum_repo, cloned_repo, [], true, :generate_metadata => true,
-                                :index_content => true, :simple_clone => false)
+                                yum_repo, cloned_repo, [], :purge_empty_units => true, :generate_metadata => true,
+                                :index_content => true, :simple_clone => false, :rpm_filenames => nil)
     end
 
     it 'plans to clone yum metadata' do
       cloned_repo = ::Katello::Repository.new
 
       action = create_action(action_class)
       cloned_repo.expects(:link?).returns(true)
+      options = {}
 
       yum_repo.expects(:build_clone).returns(cloned_repo)
 
-      plan_action(action, [yum_repo], version)
+      plan_action(action, [yum_repo], version, options)
 
       assert_action_planed_with(action, Actions::Katello::Repository::CloneYumMetadata,
                                 yum_repo, cloned_repo, :force_yum_metadata_regeneration => true)
@@ -50,8 +52,9 @@ def setup
 
       action = create_action(action_class)
       docker_repo.expects(:build_clone).returns(cloned_repo)
+      options = {}
 
-      plan_action(action, [docker_repo], version)
+      plan_action(action, [docker_repo], version, options)
 
       assert_action_planed_with(action, Actions::Katello::Repository::CloneDockerContent,
                                 docker_repo, cloned_repo, [])
@@ -62,8 +65,9 @@ def setup
                                             version: version)
       action = create_action(action_class)
       file_repo.expects(:build_clone).returns(cloned_repo)
+      options = {}
 
-      plan_action(action, [file_repo], version)
+      plan_action(action, [file_repo], version, options)
 
       assert_action_planed_with(action, Actions::Katello::Repository::CloneFileContent,
                                 file_repo, cloned_repo)

test/actions/katello/repository/clone_yum_content_test.rb

@@ -0,0 +1,43 @@
+require 'katello_test_helper'
+
+module Actions
+  describe Katello::Repository::CloneYumContent do
+    include Dynflow::Testing
+    include Support::Actions::Fixtures
+    include FactoryBot::Syntax::Methods
+
+    let(:action_class) { ::Actions::Katello::Repository::CloneYumContent }
+    let(:source_repo) { katello_repositories(:rhel_6_x86_64_dev_archive) }
+    let(:target_repo) { katello_repositories(:rhel_6_x86_64_dev) }
+
+    it 'plans to copy rpms' do
+      action = create_action(action_class)
+      source_repo = katello_repositories(:rhel_6_x86_64_dev_archive)
+      target_repo = katello_repositories(:rhel_6_x86_64_dev)
+
+      plan_action(action, source_repo, target_repo, [], :purge_empty_units => false)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopySrpm, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyRpm, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyYumMetadataFile, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyDistribution, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyModuleStream, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyModuleDefault, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+    end
+
+    it 'plans to copy rpms with rpm_filenames' do
+      action = create_action(action_class)
+      source_repo = katello_repositories(:rhel_6_x86_64_dev_archive)
+      source_repo.stubs(:rpms).returns([{ :filename => "rpm1.rpm" }, { :filename => "rpm2.rpm" }])
+      target_repo = katello_repositories(:rhel_6_x86_64_dev)
+
+      plan_action(action, source_repo, target_repo, [], :purge_empty_units => false, :rpm_filenames => ["rpm1.rpm", "rpm2.rpm"])
+
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopySrpm, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyRpm, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => {"filename" => {"$in" => ["rpm1.rpm", "rpm2.rpm"]}})
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyYumMetadataFile, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyDistribution, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyModuleStream, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+      assert_action_planed_with(action, ::Actions::Pulp::Repository::CopyModuleDefault, :source_pulp_id => source_repo.pulp_id, :target_pulp_id => target_repo.pulp_id, :clauses => nil)
+    end
+  end
+end

test/actions/katello/repository_test.rb

@@ -561,7 +561,7 @@ class ExportRepositoryTest < TestBase
       plan_action(action, [repository], false, nil, 0, "8")
       assert_action_planed_with(action, ::Actions::Katello::Repository::Clear,
                                 repository)
-      assert_action_planed_with(action, ::Actions::Katello::Repository::CloneYumContent, custom_repository, repository, [], false,
+      assert_action_planed_with(action, ::Actions::Katello::Repository::CloneYumContent, custom_repository, repository, [], :purge_empty_units => false,
                                     :generate_metadata => false, :index_content => false)
     end
 

test/controllers/api/v2/content_views_controller_test.rb

@@ -196,13 +196,13 @@ def test_update_components
 
     def test_duplicate_component_error_message
       view = katello_content_views(:library_view)
-      view_version = create(:katello_content_view_version, :content_view => view)
+      view_version = create(:katello_content_view_version, :content_view => view, :major => 8999)
 
       composite = katello_content_views(:composite_view)
       ContentViewComponent.create!(:composite_content_view => composite,
                                    :content_view_version => view_version, :latest => false)
 
-      view_version2 = create(:katello_content_view_version, :content_view => view)
+      view_version2 = create(:katello_content_view_version, :content_view => view, :major => 9001)
 
       put :update, params: { id: composite.id, content_view: { component_ids: [view_version.id, view_version2.id] } }
       display_message = JSON.parse(response.body)['displayMessage']
@@ -307,6 +307,12 @@ def test_publish_default_view
       assert_equal version_count, view.versions.reload.count
     end
 
+    def test_publish_composite_with_repos_units
+      composite = ContentView.find(katello_content_views(:composite_view).id)
+      post :publish, params: { :id => composite.id, :repos_units => "{\"hello\": 1}" }
+      assert_response 400
+    end
+
     test_attributes :pid => 'd582f1b3-8118-4e78-a639-237c6f9d27c6'
     def test_destroy
       view = ContentView.create!(:name => "Cat",