Fixes #29421 - add pulp3 debian support

create, update, sync and delete debian repositories on pulp3

Add tests

Add signing-service

Add verification of deb-repos in pulp3

app/lib/actions/katello/repository/filtered_index_content.rb

@@ -10,6 +10,7 @@ class FilteredIndexContent < Actions::EntryAction
           param :upload_actions
         end
 
+        # rubocop:disable Metrics/MethodLength
         def run
           repo = ::Katello::Repository.find(input[:id])
           if repo.puppet?
@@ -21,7 +22,15 @@ def run
           elsif repo.file?
             ::Katello::FileUnit.import_for_repository(repo)
           elsif repo.deb?
-            unit_ids = search_units(repo)
+            if input[:import_upload_task] && input[:import_upload_task][:content_unit_href]
+              unit_ids = [input[:import_upload_task][:content_unit_href]]
+            elsif input[:upload_actions]&.any? { |action| action.try(:[], "content_unit_href") }
+              uploaded_content_unit_hrefs = []
+              input[:upload_actions].each { |action| uploaded_content_unit_hrefs << action.try(:[], "content_unit_href") }
+              unit_ids = uploaded_content_unit_hrefs.compact
+            else
+              unit_ids = search_units(repo)
+            end
             ::Katello::Deb.import_all(unit_ids, repo)
           elsif repo.yum?
             if input[:import_upload_task] && input[:import_upload_task][:content_unit_href]

app/lib/actions/pulp3/repository/save_artifact.rb

@@ -11,7 +11,7 @@ def invoke_external_task
           artifact_href = input[:tasks].last[:created_resources].first
           content_type = input[:unit_type_id]
           content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
-          output[:pulp_tasks] = [content_backend_service.content_api.create(input[:options][:file_name], artifact: artifact_href)]
+          output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name], artifact: artifact_href)]
         end
       end
     end

app/services/katello/pulp3/api/apt.rb

@@ -0,0 +1,57 @@
+require "pulpcore_client"
+
+module Katello
+  module Pulp3
+    module Api
+      class Apt < Core
+        def self.api_exception_class
+          PulpDebClient::ApiError
+        end
+
+        def self.client_module
+          PulpDebClient
+        end
+
+        def self.remote_class
+          PulpDebClient::DebAptRemote
+        end
+
+        def self.distribution_class
+          PulpDebClient::DebAptDistribution
+        end
+
+        def self.publication_class
+          PulpDebClient::DebAptPublication
+        end
+
+        def self.repository_sync_url_class
+          PulpDebClient::RepositorySyncURL
+        end
+
+        def api_client
+          PulpDebClient::ApiClient.new(smart_proxy.pulp3_configuration(PulpDebClient::Configuration))
+        end
+
+        def repositories_api
+          PulpDebClient::RepositoriesAptApi.new(api_client)
+        end
+
+        def repository_versions_api
+          PulpDebClient::RepositoriesDebVersionsApi.new(api_client)
+        end
+
+        def remotes_api
+          PulpDebClient::RemotesAptApi.new(api_client)
+        end
+
+        def publications_api
+          PulpDebClient::PublicationsAptApi.new(api_client)
+        end
+
+        def distributions_api
+          PulpDebClient::DistributionsAptApi.new(api_client)
+        end
+      end
+    end
+  end
+end

app/services/katello/pulp3/api/core.rb

@@ -90,6 +90,10 @@ def upload_commit_class
           PulpcoreClient::UploadCommit
         end
 
+        def signing_services_api
+          PulpcoreClient::SigningServicesApi.new(core_api_client)
+        end
+
         def tasks_api
           PulpcoreClient::TasksApi.new(core_api_client)
         end

app/services/katello/pulp3/deb.rb

@@ -0,0 +1,38 @@
+module Katello
+  module Pulp3
+    class Deb < PulpContentUnit
+      include LazyAccessor
+      CONTENT_TYPE = "deb".freeze
+
+      def self.content_api
+        PulpDebClient::ContentPackagesApi.new(Katello::Pulp3::Api::Apt.new(SmartProxy.pulp_primary!).api_client)
+      end
+
+      def self.content_api_create(opts = {})
+        self.content_api.create(opts)
+      end
+
+      def self.create_content(options)
+        fail _("Artifact Id and relative path are needed to create content") unless options.dig(:file_name) && options.dig(:artifact)
+        PulpDebClient::DebContent.new(relative_path: options[:file_name], artifact: options[:artifact])
+      end
+
+      def self.ids_for_repository(repo_id)
+        repo = Katello::Pulp3::Repository::Apt.new(Katello::Repository.find(repo_id), SmartProxy.pulp_primary)
+        repo_content_list = repo.content_list
+        repo_content_list.map { |content| content.try(:pulp_href) }
+      end
+
+      def update_model(model)
+        custom_json = {}
+        custom_json['checksum'] = backend_data['sha256']
+        custom_json['filename'] = backend_data['relative_path']
+        custom_json['name'] = backend_data['package']
+        custom_json['version'] = backend_data['version']
+        custom_json['description'] = backend_data['description']
+        custom_json['architecture'] = backend_data['architecture']
+        model.update!(custom_json)
+      end
+    end
+  end
+end

app/services/katello/pulp3/pulp_content_unit.rb

@@ -11,6 +11,11 @@ def self.content_api
         fail NotImplementedError
       end
 
+      def self.content_api_create(opts = {})
+        relative_path = opts.delete(:relative_path)
+        self.content_api.create(relative_path, opts)
+      end
+
       def self.create_content
         fail NotImplementedError
       end

app/services/katello/pulp3/repository.rb

@@ -205,10 +205,16 @@ def sync_url_params(_sync_options)
       end
 
       def create_publication
-        publication_data = api.class.publication_class.new(repository_version: repo.version_href)
+        publication_data = api.class.publication_class.new(publication_options(repo.version_href))
         api.publications_api.create(publication_data)
       end
 
+      def publication_options(repository_version)
+        {
+          repository_version: repository_version
+        }
+      end
+
       def relative_path
         repo.relative_path.sub(/^\//, '')
       end

app/services/katello/pulp3/repository/apt.rb

@@ -0,0 +1,63 @@
+require 'pulp_deb_client'
+
+module Katello
+  module Pulp3
+    class Repository
+      class Apt < ::Katello::Pulp3::Repository
+        SIGNING_SERVICE_NAME = 'katello_deb_sign'.freeze
+
+        def remote_options
+          deb_remote_options = {
+            distributions: root.deb_releases
+          }
+          deb_remote_options[:components] = root.deb_components.present? ? root.deb_components : nil
+          deb_remote_options[:architectures] = root.deb_architectures.present? ? root.deb_architectures : nil
+
+          if root.url.blank?
+            deb_remote_options[:url] = nil
+          end
+
+          deb_remote_options[:gpgkey] = root.gpg_key.present? ? root.gpg_key.content : nil
+
+          common_remote_options.merge(deb_remote_options)
+        end
+
+        def publication_options(repository_version)
+          ss = api.signing_services_api.list(name: SIGNING_SERVICE_NAME).results
+          popts = super(repository_version)
+          popts.merge!(
+            {
+              # structured is not necessary for subscription-manager
+              #structured: true, # publish real suites (e.g. 'stable')
+              simple: true # publish all into 'default'-suite
+            }
+          )
+          popts[:signing_service] = ss[0].pulp_href if ss && ss.length == 1
+          popts
+        end
+
+        def distribution_options(path)
+          {
+            base_path: path,
+            publication: repo.publication_href,
+            name: "#{generate_backend_object_name}"
+          }
+        end
+
+        def partial_repo_path
+          "/pulp/deb/#{repo.relative_path}/".sub('//', '/')
+        end
+
+        def copy_content_for_source
+          # TODO
+          fail NotImplementedError
+        end
+
+        def regenerate_applicability
+          # TODO
+          fail NotImplementedError
+        end
+      end
+    end
+  end
+end

katello.gemspec

@@ -54,6 +54,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "pulp_file_client", ">= 1.2.0", "< 1.4.0"
   gem.add_dependency "pulp_ansible_client", ">= 0.2", "< 0.5"
   gem.add_dependency "pulp_container_client", ">= 2.0.0", "< 2.2.0"
+  gem.add_dependency "pulp_deb_client", ">= 2.6.0", "< 2.8.0"
   gem.add_dependency "pulp_rpm_client", ">=3.6.2", "< 3.8.0"
   gem.add_dependency "pulp_2to3_migration_client", ">= 0.3.0", "< 0.6.0", "!= 0.4.0"
   gem.add_dependency "pulp_certguard_client", "< 2.0"

lib/katello/repository_types/deb.rb

@@ -1,6 +1,14 @@
 Katello::RepositoryTypeManager.register(::Katello::Repository::DEB_TYPE) do
   service_class Katello::Pulp::Repository::Deb
+  pulp3_service_class Katello::Pulp3::Repository::Apt
+  pulp3_api_class Katello::Pulp3::Api::Apt
+  pulp3_plugin 'pulp_deb'
   prevent_unneeded_metadata_publish
+
   default_managed_content_type Katello::Deb
-  content_type Katello::Deb, :pulp2_service_class => ::Katello::Pulp::Deb, :removable => true, :uploadable => true
+  content_type Katello::Deb,
+    :pulp2_service_class => ::Katello::Pulp::Deb,
+    :pulp3_service_class => ::Katello::Pulp3::Deb,
+    :removable => true,
+    :uploadable => true
 end

test/actions/katello/capsule_content_test.rb

@@ -137,6 +137,59 @@ class SyncTest < TestBase
       end
     end
 
+    it 'plans correctly for a pulp3 apt repo' do
+      with_pulp3_features(capsule_content.smart_proxy)
+      capsule_content.smart_proxy.add_lifecycle_environment(environment)
+      repo = katello_repositories(:pulp3_deb_1)
+      tree = plan_action_tree(action_class, capsule_content.smart_proxy, :repository_id => repo.id)
+      options = { smart_proxy_id: capsule_content.smart_proxy.id,
+                  content_view_id: nil,
+                  repository_id: repo.id,
+                  environment_id: nil
+                }
+
+      assert_tree_planned_with(tree, ::Actions::Pulp::Orchestration::Repository::RefreshRepos, options)
+      assert_tree_planned_with(tree, ::Actions::Pulp3::Orchestration::Repository::RefreshRepos, options)
+
+      assert_tree_planned_with(tree, ::Actions::Pulp3::CapsuleContent::Sync) do |input|
+        assert_equal capsule_content.smart_proxy.id, input[:smart_proxy_id]
+        assert_equal repo.id, input[:repository_id]
+      end
+
+      assert_tree_planned_with(tree, ::Actions::Pulp3::CapsuleContent::GenerateMetadata) do |input|
+        assert_equal capsule_content.smart_proxy.id, input[:smart_proxy_id]
+        assert_equal repo.id, input[:repository_id]
+      end
+
+      assert_tree_planned_with(tree, Actions::Pulp3::CapsuleContent::RefreshDistribution) do |input|
+        assert_equal capsule_content.smart_proxy.id, input[:smart_proxy_id]
+        assert_equal repo.id, input[:repository_id]
+        refute input[:options][:use_repository_version]
+        assert input[:options][:tasks].present?
+      end
+    end
+
+    it 'plans correctly for a pulp2 apt repo' do
+      capsule_content.smart_proxy.add_lifecycle_environment(environment)
+      SmartProxy.any_instance.stubs(:pulp3_support?).returns(false)
+      repo = katello_repositories(:debian_9_amd64)
+      tree = plan_action_tree(action_class, capsule_content.smart_proxy, :repository_id => repo.id)
+      options = { smart_proxy_id: capsule_content.smart_proxy.id,
+                  content_view_id: nil,
+                  repository_id: repo.id,
+                  environment_id: nil
+                }
+
+      assert_tree_planned_with(tree, ::Actions::Pulp::Orchestration::Repository::RefreshRepos, options)
+      assert_tree_planned_with(tree, ::Actions::Pulp3::Orchestration::Repository::RefreshRepos, options)
+
+      assert_tree_planned_with(tree, Actions::Pulp::Consumer::SyncCapsule) do |input|
+        assert_equal capsule_content.smart_proxy.id, input[:capsule_id]
+        assert_equal repo.pulp_id, input[:repo_pulp_id]
+        assert input[:sync_options][:remove_missing]
+      end
+    end
+
     it 'plans correctly for a pulp yum repo' do
       capsule_content.smart_proxy.add_lifecycle_environment(environment)
       SmartProxy.any_instance.stubs(:pulp3_support?).returns(false)

test/actions/katello/repository_test.rb

@@ -14,6 +14,7 @@ class TestBase < ActiveSupport::TestCase
     let(:repository) { katello_repositories(:rhel_6_x86_64) }
     let(:repository_pulp3) { katello_repositories(:pulp3_file_1) }
     let(:repository_ansible_collection_pulp3) { katello_repositories(:pulp3_ansible_collection_1) }
+    let(:repository_apt_pulp3) { katello_repositories(:pulp3_deb_1) }
     let(:custom_repository) { katello_repositories(:fedora_17_x86_64) }
     let(:puppet_repository) { katello_repositories(:p_forge) }
     let(:docker_repository) { katello_repositories(:redis) }
@@ -477,6 +478,23 @@ class SyncTest < TestBase
       assert_action_planed_with(action, ::Actions::Pulp3::Repository::RefreshDistribution, repository_pulp3, proxy, :contents_changed => true)
     end
 
+    it 'plans pulp3 orchestration actions with apt repo' do
+      action = create_action pulp3_action_class
+      action.stubs(:action_subject).with(repository_apt_pulp3)
+      plan_action action, repository_apt_pulp3, proxy, {}
+      assert_action_planed_with(action, ::Actions::Pulp3::Repository::Sync, repository_apt_pulp3, proxy, {})
+      assert_action_planed action, ::Actions::Pulp3::Repository::SaveVersion
+      assert_action_planed action, ::Actions::Pulp3::Orchestration::Repository::GenerateMetadata
+    end
+
+    it 'plans pulp3 apt metadata generate with contents_changed' do
+      action = create_action pulp3_metadata_generate_action_class
+      action.stubs(:action_subject).with(repository_apt_pulp3)
+      plan_action action, repository_apt_pulp3, proxy, :contents_changed => true
+      assert_action_planed_with(action, ::Actions::Pulp3::Repository::CreatePublication, repository_apt_pulp3, proxy, :contents_changed => true)
+      assert_action_planed_with(action, ::Actions::Pulp3::Repository::RefreshDistribution, repository_apt_pulp3, proxy, :contents_changed => true)
+    end
+
     it 'plans pulp3 ansible collection metadata generate without publication ' do
       action = create_action pulp3_metadata_generate_action_class
       action.stubs(:action_subject).with(repository_ansible_collection_pulp3)
@@ -589,6 +607,24 @@ class SyncTest < TestBase
         end
       end
 
+      describe 'successfully synchronized pulp3 apt repo' do
+        let(:fixture_variant) { :success_apt }
+
+        specify do
+          assert_equal action.humanized_output, "Total steps: 93/93\n"\
+                                             "--------------------------------\n"\
+                                             "Associating Content: 67/67\n"\
+                                             "Downloading Artifacts: 16/16\n"\
+                                             "Un-Associating Content: 7/7\n"\
+                                             "Update PackageIndex units: 2/2\n"\
+                                             "Update ReleaseFile units: 1/1"
+        end
+
+        specify do
+          pulp3_action.run_progress.must_be_within_delta 1
+        end
+      end
+
       describe 'syncing files in progress' do
         let(:fixture_variant) { :progress_units_file }
 

test/controllers/api/v2/repositories_controller_test.rb

@@ -147,7 +147,7 @@ def test_index_with_content_view_version_id_archived
     def test_index_available_for_content_view
       ids = @view.organization.default_content_view.versions.first.repositories.pluck(:id) - @view.repositories.pluck(:id)
 
-      response = get :index, params: { :content_view_id => @view.id, :available_for => :content_view, :organization_id => @organization.id }
+      response = get :index, params: { :content_view_id => @view.id, :available_for => :content_view, :organization_id => @organization.id, :per_page => 100 }
 
       assert_response :success
       assert_template 'api/v2/repositories/index'