Fixes #33292 - ostree repo upload (#9808)

* Fixes #33292 - modifications to enable ostree commit ref content uploads via cli

Co-authored-by: James Jeffers <jjeffers@redhat.com>

app/controllers/katello/api/v2/content_uploads_controller.rb

@@ -10,7 +10,7 @@ class Api::V2::ContentUploadsController < Api::V2::ApiController
     param :repository_id, :number, :required => true, :desc => N_("repository id")
     param :size, :number, :required => true, :desc => N_("Size of file to upload")
     param :checksum, String, :required => false, :desc => N_("Checksum of file to upload")
-    param :content_type, RepositoryTypeManager.uploadable_content_types(false).map(&:label), :required => false, :desc => N_("content type ('deb', 'docker_manifest', 'file', 'ostree', 'rpm', 'srpm')")
+    param :content_type, RepositoryTypeManager.uploadable_content_types(false).map(&:label), :required => false, :desc => N_("content type ('deb', 'docker_manifest', 'file', 'ostree_ref', 'rpm', 'srpm')")
     def create
       fail Katello::Errors::InvalidRepositoryContent, _("Cannot upload Ansible collections.") if @repository.ansible_collection?
       content_type = params[:content_type] || ::Katello::RepositoryTypeManager.find(@repository.content_type)&.default_managed_content_type&.label

app/controllers/katello/api/v2/repositories_controller.rb

@@ -421,10 +421,16 @@ def import_uploads
       end
 
       begin
+        upload_args = {
+          content_type: params[:content_type],
+          generate_metadata: generate_metadata,
+          sync_capsule: sync_capsule
+        }
+        upload_args.merge!(generic_content_type_import_upload_args)
+
         respond_for_async(resource: send(
           async ? :async_task : :sync_task,
-          ::Actions::Katello::Repository::ImportUpload, @repository, uploads,
-          generate_metadata: generate_metadata, sync_capsule: sync_capsule, content_type: params[:content_type]))
+          ::Actions::Katello::Repository::ImportUpload, @repository, uploads, upload_args))
       rescue => e
         raise HttpErrors::BadRequest, e.message
       end
@@ -617,6 +623,16 @@ def generic_remote_options_hash(repo_params)
       generic_remote_options
     end
 
+    def generic_content_type_import_upload_args
+      args = {}
+      @repository.repository_type&.import_attributes&.collect do |import_attribute|
+        if params[import_attribute.api_param]
+          args[import_attribute.api_param] = params[import_attribute.api_param]
+        end
+      end
+      args
+    end
+
     def check_import_parameters
       @repository.repository_type&.import_attributes&.each do |import_attribute|
         if import_attribute.required && params[import_attribute.api_param].blank?

app/lib/actions/katello/repository/import_upload.rb

@@ -4,6 +4,7 @@ module Katello
     module Repository
       class ImportUpload < Actions::EntryAction
         include Actions::Katello::PulpSelector
+        # rubocop:disable Metrics/MethodLength
         def plan(repository, uploads, options = {})
           action_subject(repository)
           repository.clear_smart_proxy_sync_histories
@@ -21,6 +22,7 @@ def plan(repository, uploads, options = {})
           else
             unit_type_id = SmartProxy.pulp_primary.content_service(options[:content_type])::CONTENT_TYPE
           end
+          content_type = ::Katello::RepositoryTypeManager.find_content_type(options[:content_type])
 
           sequence do
             upload_results = concurrence do
@@ -34,8 +36,15 @@ def plan(repository, uploads, options = {})
                   unit_metadata: unit_metadata
                 }
 
-                import_upload = plan_action(Actions::Pulp3::Orchestration::Repository::ImportUpload,
-                                  repository, SmartProxy.pulp_primary, import_upload_args)
+                import_upload_args.merge!(options)
+
+                if content_type.repository_import_on_upload
+                  action_class = ::Actions::Pulp3::Orchestration::Repository::ImportRepositoryUpload
+                else
+                  action_class = ::Actions::Pulp3::Orchestration::Repository::ImportUpload
+                end
+
+                import_upload = plan_action(action_class, repository, SmartProxy.pulp_primary, import_upload_args)
                 plan_action(FinishUpload, repository, :import_upload_task => import_upload.output,
                             generate_metadata: false, content_type: options[:content_type])
                 import_upload.output
@@ -46,6 +55,7 @@ def plan(repository, uploads, options = {})
             plan_self(repository_id: repository.id, sync_capsule: sync_capsule, upload_results: upload_results)
           end
         end
+        # rubocop:enable Metrics/MethodLength
 
         def run
           repository = ::Katello::Repository.find(input[:repository_id])

app/lib/actions/pulp3/orchestration/repository/import_repository_upload.rb

@@ -0,0 +1,36 @@
+module Actions
+  module Pulp3
+    module Orchestration
+      module Repository
+        #Used for a different type of uploading where you are importing an entire repository, not a single content unit
+        # This workflow involves never actually creating a content unit directly, but instead importing the artifact directly into the repository
+        class ImportRepositoryUpload < Pulp3::Abstract
+          def plan(repository, smart_proxy, args)
+            file = {:filename => args.dig(:unit_key, :name), :sha256 => args.dig(:unit_key, :checksum) }
+            sequence do
+              upload_href = "/pulp/api/v3/uploads/#{args.dig(:upload_id)}/" if args.dig(:upload_id) && args.dig(:upload_id) != 'duplicate'
+              commit_output = plan_action(Pulp3::Repository::CommitUpload,
+                                          repository,
+                                          smart_proxy,
+                                          upload_href,
+                                          args.dig(:unit_key, :checksum)).output
+
+              artifact_output = plan_action(Pulp3::Repository::SaveArtifact,
+                                            file,
+                                            repository,
+                                            smart_proxy,
+                                            commit_output[:pulp_tasks],
+                                            args.dig(:unit_type_id), args).output
+              plan_self(:artifact_output => artifact_output)
+              plan_action(Pulp3::Repository::SaveVersion, repository, tasks: artifact_output[:pulp_tasks])
+            end
+          end
+
+          def run
+            output[:content_unit_href] = input[:artifact_output][:content_unit_href] || input[:artifact_output][:pulp_tasks].last[:created_resources].first
+          end
+        end
+      end
+    end
+  end
+end

app/lib/actions/pulp3/orchestration/repository/import_upload.rb

@@ -33,7 +33,7 @@ def plan(repository, smart_proxy, args)
                                                 repository,
                                                 smart_proxy,
                                                 commit_output[:pulp_tasks],
-                                                args.dig(:unit_type_id)).output
+                                                args.dig(:unit_type_id), args).output
                 end
 
                 plan_self(:commit_output => commit_output[:pulp_tasks], :artifact_output => artifact_output)

app/lib/actions/pulp3/repository/commit_upload.rb

@@ -1,6 +1,7 @@
 module Actions
   module Pulp3
     module Repository
+      #Creates an artifacts
       class CommitUpload < Pulp3::AbstractAsyncTask
         def plan(repository, smart_proxy, upload_href, sha256)
           plan_self(:repository_id => repository.id, :smart_proxy_id => smart_proxy.id, :upload_href => upload_href, :sha256 => sha256)
@@ -10,10 +11,11 @@ def invoke_external_task
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
           uploads_api = repo_backend_service.core_api.uploads_api
+
           duplicate_sha_artifact_list = ::Katello::Pulp3::Api::Core.new(smart_proxy).artifacts_api.list("sha256": input[:sha256])
           duplicate_sha_artifact_href = duplicate_sha_artifact_list&.results&.first&.pulp_href
           if duplicate_sha_artifact_href
-            uploads_api.delete(input[:upload_href])
+            uploads_api.delete(input[:upload_href]) if input[:upload_href]
             output[:artifact_href] = duplicate_sha_artifact_href
             output[:pulp_tasks] = nil
           else

app/lib/actions/pulp3/repository/import_upload.rb

@@ -4,10 +4,11 @@ module Actions
   module Pulp3
     module Repository
       class ImportUpload < Pulp3::AbstractAsyncTask
-        def plan(save_artifact_output, repository, smart_proxy)
+        def plan(save_artifact_output, repository, smart_proxy, options = {})
           plan_self(:save_artifact_output => save_artifact_output,
                     :repository_id => repository.id,
-                    :smart_proxy_id => smart_proxy.id)
+                    :smart_proxy_id => smart_proxy.id,
+                    :options => options)
         end
 
         def invoke_external_task
@@ -19,6 +20,7 @@ def invoke_external_task
 
           repo = ::Katello::Repository.find(input[:repository_id])
           repo_backend_service = repo.backend_service(smart_proxy)
+
           output[:content_unit_href] = content_unit_href
           output[:pulp_tasks] = [repo_backend_service.add_content(content_unit_href)]
         end

app/lib/actions/pulp3/repository/save_artifact.rb

@@ -2,6 +2,7 @@ module Actions
   module Pulp3
     module Repository
       class SaveArtifact < Pulp3::AbstractAsyncTask
+        #This task creates a content unit and may or may not create a new repository version in the process
         def plan(file, repository, smart_proxy, tasks, unit_type_id, options = {})
           options[:file_name] = file[:filename]
           options[:sha256] = file[:sha256] || Digest::SHA256.hexdigest(File.read(file[:path]))
@@ -15,17 +16,20 @@ def invoke_external_task
           content_type = input[:unit_type_id]
           content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
 
-          existing_content = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, input['unit_type_id'], {filename: input[:options][:file_name]}, input[:options][:sha256])
+          existing_content = ::Katello::Pulp3::PulpContentUnit.find_duplicate_unit(repository, input[:unit_type_id], {filename: input[:options][:file_name]}, input[:options][:sha256])
           existing_content_href = existing_content&.results&.first&.pulp_href
 
-          if existing_content_href
-            output[:content_unit_href] = existing_content_href
-            []
+          if ::Katello::RepositoryTypeManager.find_content_type(input[:unit_type_id]).repository_import_on_upload
+            output[:pulp_tasks] = [repository.backend_service(smart_proxy).repository_import_content(artifact_href, input[:options])]
           else
-            output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
-                                                                              artifact: artifact_href,
-                                                                              repository_id: input[:repository_id],
-                                                                              content_type: content_type)]
+            if existing_content_href
+              output[:content_unit_href] = existing_content_href
+              []
+            else
+              output[:pulp_tasks] = [content_backend_service.content_api_create(relative_path: input[:options][:file_name],
+                                                                                artifact: artifact_href,
+                                                                                content_type: content_type)]
+            end
           end
         end
 

app/models/katello/root_repository.rb

@@ -73,7 +73,6 @@ class RootRepository < Katello::Model
     validate :ensure_valid_authentication_token, :if => :yum?
     validate :ensure_valid_deb_constraints, :if => :deb?
     validate :ensure_no_checksum_on_demand
-    validates :url, presence: true, if: :ostree?
     validates :checksum_type, :inclusion => {:in => CHECKSUM_TYPES}, :allow_blank => true
     validates :product_id, :presence => true
     validates :content_type, :inclusion => {

app/services/katello/pulp3/content.rb

@@ -6,10 +6,12 @@ class Content
       class << self
         def create_upload(size = 0, checksum = nil, content_type = nil, repository = nil)
           content_unit_href = nil
-          if checksum
+          content_type = ::Katello::RepositoryTypeManager.find_content_type(content_type)
+
+          if checksum && !content_type.repository_import_on_upload
             content_backend_service = SmartProxy.pulp_primary.content_service(content_type)
             if repository&.generic?
-              content_list = content_backend_service.content_api(repository.repository_type, content_type).list("sha256": checksum)
+              content_list = content_backend_service.content_api(repository.repository_type, content_type).list('sha256': checksum)
             else
               content_list = content_backend_service.content_api.list("sha256": checksum)
             end

app/services/katello/pulp3/pulp_content_unit.rb

@@ -116,24 +116,30 @@ def self.fetch_content_list(page_opts)
         content_unit_list page_opts
       end
 
+      # rubocop:disable Lint/UselessAssignment
       def self.find_duplicate_unit(repository, unit_type_id, file, checksum)
+        filter_label = 'sha256'
+        if unit_type_id == 'ostree_ref'
+          filter_label = 'checksum'
+        end
         content_backend_service = SmartProxy.pulp_primary.content_service(unit_type_id)
         duplicates_allowed = ::Katello::RepositoryTypeManager.find_content_type(unit_type_id).try(:duplicates_allowed)
         if repository.generic? && duplicates_allowed
           filename_key = ::Katello::RepositoryTypeManager.find_content_type(unit_type_id).filename_key
           duplicate_sha_path_content_list = content_backend_service.content_api(repository.repository_type, unit_type_id).list(
-            "sha256": checksum,
+            filter_label: checksum,
             filename_key => file[:filename])
         elsif repository.generic?
           duplicate_sha_path_content_list = content_backend_service.content_api(repository.repository_type, unit_type_id).list(
-            "sha256": checksum)
+            filter_label: checksum)
         else
           duplicate_sha_path_content_list = content_backend_service.content_api.list(
-            "sha256": checksum,
+            filter_label: checksum,
             "relative_path": file[:filename])
         end
         duplicate_sha_path_content_list
       end
+      # rubocop:enable Lint/UselessAssignment
     end
   end
 end

app/services/katello/pulp3/repository.rb

@@ -474,6 +474,15 @@ def remove_content(content_units)
         end
       end
 
+      def repository_import_content(artifact_href, options = {})
+        ostree_import = PulpOstreeClient::OstreeRepoImport.new
+        ostree_import.artifact = artifact_href
+        ostree_import.repository_name = options[:ostree_repository_name]
+        ostree_import.ref = options[:ostree_ref]
+        ostree_import.parent_commit = options[:ostree_parent_commit]
+        api.repositories_api.import_commits(repository_reference.repository_href, ostree_import)
+      end
+
       def add_content(content_unit_href, remove_all_units = false)
         content_unit_href = [content_unit_href] unless content_unit_href.is_a?(Array)
         if remove_all_units

app/services/katello/repository_type.rb

@@ -145,7 +145,7 @@ def pulp3_api(smart_proxy)
 
     class ContentType
       attr_accessor :model_class, :priority, :pulp2_service_class, :pulp3_service_class, :index, :uploadable, :removable,
-                    :primary_content, :index_on_pulp3, :generic_browser, :content_type
+                    :primary_content, :index_on_pulp3, :generic_browser, :content_type, :repository_import_on_upload
 
       def initialize(options)
         self.model_class = options[:model_class]
@@ -159,6 +159,7 @@ def initialize(options)
         self.removable = options[:removable] || false
         self.primary_content = options[:primary_content] || false
         self.generic_browser = options[:generic_browser]
+        self.repository_import_on_upload = options[:repository_import_on_upload]
       end
 
       def label

lib/katello/repository_types/ostree.rb

@@ -25,7 +25,9 @@
                        pulp3_api: PulpOstreeClient::ContentRefsApi,
                        pulp3_service_class: Katello::Pulp3::GenericContentUnit,
                        model_name: lambda { |pulp_unit| pulp_unit["name"] },
-                       model_version: lambda { |pulp_unit| pulp_unit["version"] }
+                       model_version: lambda { |pulp_unit| pulp_unit["checksum"] },
+                       uploadable: true,
+                       repository_import_on_upload: true
 
   import_attribute :ref, :content_type => 'ostree_ref',
                         :api_param => :ostree_ref,

test/actions/katello/repository/import_upload_test.rb

@@ -20,7 +20,8 @@ module Actions
         unit_type_id: repo.unit_type_id,
         unit_key: upload.except('id'),
         upload_id: '1',
-        unit_metadata: nil
+        unit_metadata: nil,
+        content_type: 'rpm'
       }
 
       assert_action_planed_with(action, pulp3_import_class,

test/actions/katello/repository_test.rb

@@ -399,7 +399,10 @@ class UploadDockerTest < TestBase
         unit_type_id: 'docker_manifest',
         unit_key: {'size' => '12333', 'checksum' => 'asf23421324', 'name' => 'test'},
         upload_id: 1,
-        unit_metadata: nil
+        unit_metadata: nil,
+        generate_metadata: true,
+        sync_capsule: true,
+        content_type: 'docker_manifest'
       }
       assert_action_planned_with(action, ::Actions::Pulp3::Orchestration::Repository::ImportUpload,
                                  docker_repository, SmartProxy.pulp_primary,
@@ -422,7 +425,10 @@ class UploadDockerTest < TestBase
         unit_type_id: 'docker_tag',
         unit_key: unit_keys[0],
         upload_id: 1,
-        unit_metadata: unit_keys[0]
+        unit_metadata: unit_keys[0],
+        generate_metadata: true,
+        sync_capsule: true,
+        content_type: 'docker_tag'
       }
       assert_action_planned_with(action, ::Actions::Pulp3::Orchestration::Repository::ImportUpload,
                                  docker_repository, SmartProxy.pulp_primary,