[CP] Katello 4.3 rc3 change log and cp's

app/controllers/katello/api/v2/capsule_content_controller.rb

@@ -4,7 +4,7 @@ class Api::V2::CapsuleContentController < Api::V2::ApiController
       api_base_url "/katello/api"
     end
 
-    before_action :find_capsule, :except => [:sync, :cancel_sync, :add_lifecycle_environment, :remove_lifecycle_environment]
+    before_action :find_capsule, :except => [:sync, :cancel_sync, :add_lifecycle_environment, :remove_lifecycle_environment, :reclaim_space]
     before_action :find_editable_capsule, :only => [:sync, :cancel_sync, :add_lifecycle_environment, :remove_lifecycle_environment]
     before_action :find_environment, :only => [:add_lifecycle_environment, :remove_lifecycle_environment]
     before_action :find_optional_organization, :only => [:sync_status]
@@ -86,6 +86,14 @@ def cancel_sync
       end
     end
 
+    api :POST, '/capsules/:id/reclaim_space', N_('Reclaim space from all On Demand repositories on a smart proxy')
+    param :id, :number, :required => true, :desc => N_('Id of the smart proxy')
+    def reclaim_space
+      find_capsule(true)
+      task = async_task(::Actions::Pulp3::CapsuleContent::ReclaimSpace, @capsule)
+      respond_for_async :resource => task
+    end
+
     protected
 
     def respond_for_lifecycle_environments_index(environments)
@@ -104,9 +112,9 @@ def find_editable_capsule
       end
     end
 
-    def find_capsule
+    def find_capsule(primary_okay = false)
       @capsule = SmartProxy.unscoped.authorized(:view_capsule_content).find(params[:id])
-      unless @capsule&.pulp_mirror?
+      unless @capsule&.pulp_mirror? || primary_okay
         fail _("This request may only be performed on a Smart proxy that has the Pulpcore feature with mirror=true.")
       end
     end

app/controllers/katello/api/v2/content_uploads_controller.rb

@@ -10,7 +10,7 @@ class Api::V2::ContentUploadsController < Api::V2::ApiController
     param :repository_id, :number, :required => true, :desc => N_("repository id")
     param :size, :number, :required => true, :desc => N_("Size of file to upload")
     param :checksum, String, :required => false, :desc => N_("Checksum of file to upload")
-    param :content_type, RepositoryTypeManager.uploadable_content_types(false).map(&:label), :required => false, :desc => N_("content type ('deb', 'docker_manifest', 'file', 'ostree', 'rpm', 'srpm')")
+    param :content_type, RepositoryTypeManager.uploadable_content_types(false).map(&:label), :required => false, :desc => N_("content type ('deb', 'docker_manifest', 'file', 'ostree_ref', 'rpm', 'srpm')")
     def create
       fail Katello::Errors::InvalidRepositoryContent, _("Cannot upload Ansible collections.") if @repository.ansible_collection?
       content_type = params[:content_type] || ::Katello::RepositoryTypeManager.find(@repository.content_type)&.default_managed_content_type&.label

app/controllers/katello/api/v2/generic_content_units_controller.rb

@@ -1,10 +1,15 @@
 module Katello
   class Api::V2::GenericContentUnitsController < Api::V2::ApiController
+    resource_description do
+      name 'Content Units'
+      param :content_type, String, desc: N_("Possible values: #{Katello::RepositoryTypeManager.generic_content_types.join(", ")}"), required: true
+    end
+    apipie_concern_subst(:a_resource => N_("a content unit"), :resource_id => "content_units")
+
     Katello::RepositoryTypeManager.generic_content_types(false).each do |type|
-      apipie_concern_subst(:a_resource => N_(type), :resource_id => type.pluralize)
-      resource_description do
-        name type.pluralize.titleize
-      end
+      api :GET, "/#{type.pluralize}", N_("List %s" % type.pluralize)
+      api :GET, "/#{type.pluralize}/:id", N_("Show %s" % type.gsub(/_/, ' '))
+      api :GET, "/repositories/:repository_id/#{type.pluralize}/:id", N_("Show %s" % type.gsub(/_/, ' '))
     end
 
     include Katello::Concerns::Api::V2::RepositoryContentController
@@ -14,6 +19,7 @@ def default_sort
     end
 
     def resource_class
+      fail "Required param content_type is missing" unless params[:content_type]
       ::Katello::GenericContentUnit.where(content_type: params[:content_type].singularize)
     end
 

app/controllers/katello/api/v2/repositories_bulk_actions_controller.rb

@@ -50,6 +50,14 @@ def sync_repositories
       end
     end
 
+    api :POST, "/repositories/bulk/reclaim_space", N_("Reclaim space from On Demand repositories")
+    param :ids, Array, :desc => N_("List of repository ids"), :required => true
+    def reclaim_space_from_repositories
+      task = async_task(::Actions::Pulp3::Repository::ReclaimSpace, @repositories)
+
+      respond_for_async :resource => task
+    end
+
     private
 
     def find_repositories

app/controllers/katello/api/v2/repositories_controller.rb

@@ -23,7 +23,7 @@ class Api::V2::RepositoriesController < Api::V2::ApiController # rubocop:disable
     before_action :find_unauthorized_katello_resource, :only => [:gpg_key_content]
     before_action :find_authorized_katello_resource, :only => [:show, :update, :destroy, :sync,
                                                                :remove_content, :upload_content, :republish,
-                                                               :import_uploads, :verify_checksum]
+                                                               :import_uploads, :verify_checksum, :reclaim_space]
     before_action :find_content, :only => :remove_content
     before_action :find_organization_from_repo, :only => [:update]
     before_action :error_on_rh_product, :only => [:create]
@@ -234,6 +234,10 @@ def create
         fail HttpErrors::UnprocessableEntity, msg
       end
 
+      if !repo_params[:url].nil? && URI(repo_params[:url]).userinfo
+        fail "Do not include the username/password in the URL. Use the username/password settings instead."
+      end
+
       gpg_key = get_content_credential(repo_params, CONTENT_CREDENTIAL_GPG_KEY_TYPE)
       ssl_ca_cert = get_content_credential(repo_params, CONTENT_CREDENTIAL_SSL_CA_CERT_TYPE)
       ssl_client_cert = get_content_credential(repo_params, CONTENT_CREDENTIAL_SSL_CLIENT_CERT_TYPE)
@@ -317,13 +321,25 @@ def verify_checksum
       raise HttpErrors::BadRequest, e.message
     end
 
+    api :POST, "/repositories/:id/reclaim_space", N_("Reclaim space from an On Demand repository")
+    param :id, :number, :required => true, :desc => N_("repository ID")
+    def reclaim_space
+      task = async_task(::Actions::Pulp3::Repository::ReclaimSpace, @repository)
+      respond_for_async :resource => task
+    rescue Errors::InvalidActionOptionError => e
+      raise HttpErrors::BadRequest, e.message
+    end
+
     api :PUT, "/repositories/:id", N_("Update a repository")
     param :id, :number, :required => true, :desc => N_("repository ID")
     param :name, String, :required => false
     param :description, String, :desc => N_("description of the repository"), :required => false
     param_group :repo
     def update
       repo_params = repository_params
+      if !repo_params[:url].nil? && URI(repo_params[:url]).userinfo
+        fail "Do not include the username/password in the URL. Use the username/password settings instead."
+      end
 
       if @repository.generic?
         generic_remote_options = generic_remote_options_hash(repo_params)
@@ -421,10 +437,16 @@ def import_uploads
       end
 
       begin
+        upload_args = {
+          content_type: params[:content_type],
+          generate_metadata: generate_metadata,
+          sync_capsule: sync_capsule
+        }
+        upload_args.merge!(generic_content_type_import_upload_args)
+
         respond_for_async(resource: send(
           async ? :async_task : :sync_task,
-          ::Actions::Katello::Repository::ImportUpload, @repository, uploads,
-          generate_metadata: generate_metadata, sync_capsule: sync_capsule, content_type: params[:content_type]))
+          ::Actions::Katello::Repository::ImportUpload, @repository, uploads, upload_args))
       rescue => e
         raise HttpErrors::BadRequest, e.message
       end
@@ -617,6 +639,16 @@ def generic_remote_options_hash(repo_params)
       generic_remote_options
     end
 
+    def generic_content_type_import_upload_args
+      args = {}
+      @repository.repository_type&.import_attributes&.collect do |import_attribute|
+        if params[import_attribute.api_param]
+          args[import_attribute.api_param] = params[import_attribute.api_param]
+        end
+      end
+      args
+    end
+
     def check_import_parameters
       @repository.repository_type&.import_attributes&.each do |import_attribute|
         if import_attribute.required && params[import_attribute.api_param].blank?

app/lib/actions/katello/repository/import_upload.rb

@@ -4,6 +4,7 @@ module Katello
     module Repository
       class ImportUpload < Actions::EntryAction
         include Actions::Katello::PulpSelector
+        # rubocop:disable Metrics/MethodLength
         def plan(repository, uploads, options = {})
           action_subject(repository)
           repository.clear_smart_proxy_sync_histories
@@ -21,6 +22,7 @@ def plan(repository, uploads, options = {})
           else
             unit_type_id = SmartProxy.pulp_primary.content_service(options[:content_type])::CONTENT_TYPE
           end
+          content_type = ::Katello::RepositoryTypeManager.find_content_type(options[:content_type])
 
           sequence do
             upload_results = concurrence do
@@ -34,8 +36,15 @@ def plan(repository, uploads, options = {})
                   unit_metadata: unit_metadata
                 }
 
-                import_upload = plan_action(Actions::Pulp3::Orchestration::Repository::ImportUpload,
-                                  repository, SmartProxy.pulp_primary, import_upload_args)
+                import_upload_args.merge!(options)
+
+                if content_type.repository_import_on_upload
+                  action_class = ::Actions::Pulp3::Orchestration::Repository::ImportRepositoryUpload
+                else
+                  action_class = ::Actions::Pulp3::Orchestration::Repository::ImportUpload
+                end
+
+                import_upload = plan_action(action_class, repository, SmartProxy.pulp_primary, import_upload_args)
                 plan_action(FinishUpload, repository, :import_upload_task => import_upload.output,
                             generate_metadata: false, content_type: options[:content_type])
                 import_upload.output
@@ -46,6 +55,7 @@ def plan(repository, uploads, options = {})
             plan_self(repository_id: repository.id, sync_capsule: sync_capsule, upload_results: upload_results)
           end
         end
+        # rubocop:enable Metrics/MethodLength
 
         def run
           repository = ::Katello::Repository.find(input[:repository_id])

app/lib/actions/pulp3/capsule_content/reclaim_space.rb

@@ -0,0 +1,25 @@
+module Actions
+  module Pulp3
+    module CapsuleContent
+      class ReclaimSpace < Pulp3::AbstractAsyncTask
+        def plan(smart_proxy)
+          if smart_proxy.pulp_primary?
+            repository_hrefs = ::Katello::Pulp3::RepositoryReference.default_cv_repository_hrefs(::Katello::Repository.unscoped.on_demand, ::Organization.all)
+            repository_hrefs.flatten!
+          else
+            if smart_proxy.download_policy != ::Katello::RootRepository::DOWNLOAD_ON_DEMAND
+              fail _('Only On Demand smart proxies may have space reclaimed.')
+            end
+            repository_hrefs = ::Katello::Pulp3::Api::Core.new(smart_proxy).core_repositories_list_all(fields: 'pulp_href').map(&:pulp_href)
+          end
+          plan_self(repository_hrefs: repository_hrefs, smart_proxy_id: smart_proxy.id)
+        end
+
+        def invoke_external_task
+          output[:pulp_tasks] = ::Katello::Pulp3::Api::Core.new(SmartProxy.find(input[:smart_proxy_id])).
+            repositories_reclaim_space_api.reclaim(repo_hrefs: input[:repository_hrefs])
+        end
+      end
+    end
+  end
+end

app/lib/actions/pulp3/orchestration/repository/import_repository_upload.rb

@@ -0,0 +1,36 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module Repository
+        #Used for a different type of uploading where you are importing an entire repository, not a single content unit
+        # This workflow involves never actually creating a content unit directly, but instead importing the artifact directly into the repository
+        class ImportRepositoryUpload < Pulp3::Abstract
+          def plan(repository, smart_proxy, args)
+            file = {:filename => args.dig(:unit_key, :name), :sha256 => args.dig(:unit_key, :checksum) }
+            sequence do
+              upload_href = "/pulp/api/v3/uploads/#{args.dig(:upload_id)}/" if args.dig(:upload_id) && args.dig(:upload_id) != 'duplicate'
+              commit_output = plan_action(Pulp3::Repository::CommitUpload,
+                                          repository,
+                                          smart_proxy,
+                                          upload_href,
+                                          args.dig(:unit_key, :checksum)).output
+
+              artifact_output = plan_action(Pulp3::Repository::SaveArtifact,
+                                            file,
+                                            repository,
+                                            smart_proxy,
+                                            commit_output[:pulp_tasks],
+                                            args.dig(:unit_type_id), args).output
+              plan_self(:artifact_output => artifact_output)
+              plan_action(Pulp3::Repository::SaveVersion, repository, tasks: artifact_output[:pulp_tasks])
+            end
+          end
+
+          def run
+            output[:content_unit_href] = input[:artifact_output][:content_unit_href] || input[:artifact_output][:pulp_tasks].last[:created_resources].first
+          end
+        end
+      end
+    end
+  end
+end

app/lib/actions/pulp3/orchestration/repository/import_upload.rb

@@ -33,7 +33,7 @@ def plan(repository, smart_proxy, args)
                                                 repository,
                                                 smart_proxy,
                                                 commit_output[:pulp_tasks],
-                                                args.dig(:unit_type_id)).output
+                                                args.dig(:unit_type_id), args).output
                 end
 
                 plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output)

app/lib/actions/pulp3/repository/commit_upload.rb

@@ -1,6 +1,7 @@
 module Actions
   module Pulp3
     module Repository
+      #Creates an artifacts
       class CommitUpload < Pulp3::AbstractAsyncTask
         def plan(repository, smart_proxy, upload_href, sha256)
           plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :upload_href => upload_href, :sha256 => sha256)
@@ -10,10 +11,11 @@ def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
           uploads_api = repo_backend_service.core_api.uploads_api
+
           duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": input[:sha256])
           duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
           if duplicate_sha_artifact_href
-            uploads_api.delete(input[:upload_href])
+            uploads_api.delete(input[:upload_href]) if input[:upload_href]
             output[:artifact_href] = duplicate_sha_artifact_href
             output[:pulp_tasks] = nil
           else

app/lib/actions/pulp3/repository/import_upload.rb

@@ -4,10 +4,11 @@ module Actions
   module Pulp3
     module Repository
       class ImportUpload < Pulp3::AbstractAsyncTask
-        def plan(save_artifact_output, repository, smart_proxy)
+        def plan(save_artifact_output, repository, smart_proxy, options = {})
           plan_self(:save_artifact_output => save_artifact_output,
                     :repository_id => repository.id,
-                    :smart_proxy_id => smart_proxy.id)
+                    :smart_proxy_id => smart_proxy.id,
+                    :options => options)
         end
 
         def invoke_external_task
@@ -19,6 +20,7 @@ def invoke_external_task
 
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
+
           output[:content_unit_href] = content_unit_href
           output[:pulp_tasks] = [repo_backend_service.add_content(content_unit_href)]
         end

app/lib/actions/pulp3/repository/reclaim_space.rb

@@ -0,0 +1,25 @@
+module Actions
+  module Pulp3
+    module Repository
+      class ReclaimSpace < Pulp3::AbstractAsyncTask
+        def plan(repositories, smart_proxy = SmartProxy.pulp_primary)
+          repositories = [repositories] if repositories.is_a?(::Katello::Repository)
+          if repositories.empty?
+            fail _("No repositories selected.")
+          end
+          repositories = repositories.select { |repo| repo.download_policy == ::Katello::RootRepository::DOWNLOAD_ON_DEMAND }
+          if repositories.empty?
+            fail _("Only On Demand repositories may have space reclaimed.")
+          end
+          repository_hrefs = ::Katello::Pulp3::RepositoryReference.default_cv_repository_hrefs(repositories, Organization.current)
+          plan_self(repository_hrefs: repository_hrefs, smart_proxy_id: smart_proxy.id)
+        end
+
+        def invoke_external_task
+          output[:pulp_tasks] = ::Katello::Pulp3::Api::Core.new(SmartProxy.find(input[:smart_proxy_id])).
+            repositories_reclaim_space_api.reclaim(repo_hrefs: input[:repository_hrefs])
+        end
+      end
+    end
+  end
+end

app/lib/actions/pulp3/repository/save_artifact.rb

@@ -2,6 +2,7 @@ module Actions
   module Pulp3
     module Repository
       class SaveArtifact < Pulp3::AbstractAsyncTask
+        #This task creates a content unit and may or may not create a new repository version in the process
         def plan(file, repository, smart_proxy, tasks, unit_type_id, options = {})
           options[:file_name] = file[:filename]
           options[:sha256] = file[:sha256] || Digest::SHA256.hexdigest(File.read(file[:path]))
@@ -15,17 +16,20 @@ def invoke_external_task
           content_type = input[:unit_type_id]
           content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
 
-          existing_content = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, input['unit_type_id'], {filename: input[:options][:file_name]}, input[:options][:sha256])
+          existing_content = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, input[:unit_type_id], {filename: input[:options][:file_name]}, input[:options][:sha256])
           existing_content_href = existing_content&.results&.first&.pulp_href
 
-          if existing_content_href
-            output[:content_unit_href] = existing_content_href
-            []
+          if ::Katello::RepositoryTypeManager.find_content_type(input[:unit_type_id]).repository_import_on_upload
+            output[:pulp_tasks] = [repository.backend_service(smart_proxy).repository_import_content(artifact_href, input[:options])]
           else
-            output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
-                                                                              artifact: artifact_href,
-                                                                              repository_id: input[:repository_id],
-                                                                              content_type: content_type)]
+            if existing_content_href
+              output[:content_unit_href] = existing_content_href
+              []
+            else
+              output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
+                                                                                artifact: artifact_href,
+                                                                                content_type: content_type)]
+            end
           end
         end
 

app/models/katello/concerns/smart_proxy_extensions.rb

@@ -186,7 +186,7 @@ def pulp3_configuration(config_class)
       end
 
       def pulp3_ssl_configuration(config)
-        legacy_pulp_cert = !self.setting(PULP3_FEATURE, 'client_authentication')&.include?('client_certificates')
+        legacy_pulp_cert = !self.setting(PULP3_FEATURE, 'client_authentication')&.include?('client_certificate')
 
         if Faraday.default_adapter == :excon
           config.ssl_client_cert = ::Cert::Certs.ssl_client_cert_filename(use_admin_as_cn_cert: legacy_pulp_cert)