Fixes #32802 - Sync/Index Python repo with remote options

app/controllers/katello/api/v2/repositories_controller.rb

@@ -2,7 +2,14 @@ module Katello
   class Api::V2::RepositoriesController < Api::V2::ApiController # rubocop:disable Metrics/ClassLength
     include Katello::Concerns::FilteredAutoCompleteSearch
 
-    wrap_parameters :repository, :include => RootRepository.attribute_names.concat([:ignore_global_proxy])
+    generic_repo_wrap_params = []
+    RepositoryTypeManager.generic_remote_options.each do |option|
+      generic_repo_wrap_params << option.name
+    end
+
+    repo_wrap_params = RootRepository.attribute_names.concat([:ignore_global_proxy]) + generic_repo_wrap_params
+
+    wrap_parameters :repository, :include => repo_wrap_params
 
     CONTENT_CREDENTIAL_GPG_KEY_TYPE = "gpg_key".freeze
     CONTENT_CREDENTIAL_SSL_CA_CERT_TYPE = "ssl_ca_cert".freeze
@@ -62,6 +69,10 @@ def custom_index_relation(collection)
       param :http_proxy_id, :number, :desc => N_("ID of a HTTP Proxy")
       param :arch, String, :desc => N_("Architecture of content in the repository")
       param :retain_package_versions_count, :number, :desc => N_("The maximum number of versions of each package to keep.")
+
+      RepositoryTypeManager.generic_remote_options(defined_only: true).each do |option|
+        param option.name, option.type, :desc => N_(option.description)
+      end
     end
 
     def_param_group :repo_create do
@@ -310,6 +321,14 @@ def verify_checksum
     def update
       repo_params = repository_params
 
+      if @repository.generic?
+        generic_remote_options = generic_remote_options_hash(repo_params)
+        repo_params[:generic_remote_options] = generic_remote_options.to_json
+        RepositoryTypeManager.generic_remote_options.each do |option|
+          repo_params&.delete(option.name)
+        end
+      end
+
       sync_task(::Actions::Katello::Repository::Update, @repository.root, repo_params)
       respond_for_show(:resource => @repository)
     end
@@ -440,6 +459,7 @@ def find_content_credential(content_type)
       end
     end
 
+    # rubocop:disable Metrics/CyclomaticComplexity
     def repository_params
       keys = [:download_policy, :mirror_on_sync, :arch, :verify_ssl_on_sync, :upstream_password, :upstream_username, :download_concurrency,
               :ostree_upstream_sync_depth, :ostree_upstream_sync_policy, {:os_versions => []},
@@ -450,6 +470,18 @@ def repository_params
       keys += [{:docker_tags_whitelist => []}, :docker_upstream_name] if params[:action] == 'create' || @repository&.docker?
       keys += [:ansible_collection_requirements, :ansible_collection_auth_url, :ansible_collection_auth_token] if params[:action] == 'create' || @repository&.ansible_collection?
       keys += [:label, :content_type] if params[:action] == "create"
+
+      if params[:action] == 'create' || @repository&.generic?
+        RepositoryTypeManager.generic_remote_options.each do |option|
+          if option.type == Array
+            keys += [{option.name => []}]
+          elsif option.type == Hash
+            keys += [{option.name => {}}]
+          else
+            keys += [option.name]
+          end
+        end
+      end
       if params[:action] == 'create' || @repository.custom?
         keys += [:url, :gpg_key_id, :ssl_ca_cert_id, :ssl_client_cert_id, :ssl_client_key_id, :unprotected, :name,
                  :checksum_type]
@@ -467,8 +499,8 @@ def get_content_credential(repo_params, content_type)
       credential_value
     end
 
-    # rubocop:disable Metrics/CyclomaticComplexity
     # rubocop:disable Metrics/PerceivedComplexity
+    # rubocop:disable Metrics/MethodLength
     def construct_repo_from_params(repo_params) # rubocop:disable Metrics/AbcSize
       root = @product.add_repo(repo_params.slice(:label, :name, :description, :url, :content_type, :arch, :unprotected,
                                                             :gpg_key, :ssl_ca_cert, :ssl_client_cert, :ssl_client_key,
@@ -485,6 +517,11 @@ def construct_repo_from_params(repo_params) # rubocop:disable Metrics/AbcSize
       root.os_versions = repo_params.fetch(:os_versions, []) if root.yum?
       root.retain_package_versions_count = repo_params[:retain_package_versions_count] if root.yum? && repo_params.key?(:retain_package_versions_count)
 
+      if root.generic?
+        generic_remote_options = generic_remote_options_hash(repo_params)
+        root.generic_remote_options = generic_remote_options.to_json
+      end
+
       if root.ostree?
         root.ostree_upstream_sync_policy = repo_params[:ostree_upstream_sync_policy]
         root.ostree_upstream_sync_depth = repo_params[:ostree_upstream_sync_depth]
@@ -549,5 +586,13 @@ def filter_by_content_view(query, content_view_id, environment_id, is_available_
       end
       query
     end
+
+    def generic_remote_options_hash(repo_params)
+      generic_remote_options = {}
+      RepositoryTypeManager.generic_remote_options(content_type: repo_params[:content_type]).each do |option|
+        generic_remote_options[option.name] = repo_params[option.name]
+      end
+      generic_remote_options
+    end
   end
 end

app/models/katello/concerns/pulp_database_unit.rb

@@ -127,7 +127,7 @@ def unit_id_field
         "#{self.name.demodulize.underscore}_id"
       end
 
-      def import_all(pulp_ids = nil, repository = nil)
+      def import_all(pulp_ids = nil, repository = nil, content_type_name = nil)
         ids_to_associate = []
         service_class = SmartProxy.pulp_primary!.content_service(content_type)
         service_class.pulp_units_batch_all(pulp_ids).each do |units|
@@ -145,21 +145,25 @@ def import_all(pulp_ids = nil, repository = nil)
             end
             service = service_class.new(model.pulp_id)
             service.backend_data = unit
-            service.update_model(model)
+            if repository&.generic?
+              service.update_model(model, repository.repository_type, content_type_name)
+            else
+              service.update_model(model)
+            end
             ids_to_associate << model.pulp_id
           end
         end
         sync_repository_associations(repository, :pulp_ids => ids_to_associate, :additive => true) if self.many_repository_associations && repository && ids_to_associate.present?
       end
 
-      def import_for_repository(repository)
+      def import_for_repository(repository, content_type_name = nil)
         pulp_id_href_map = {}
         service_class = SmartProxy.pulp_primary!.content_service(content_type)
         fetch_only_ids = !repository.content_view.default? &&
                          !repository.repository_type.unique_content_per_repo &&
                          service_class.supports_id_fetch?
 
-        service_class.pulp_units_batch_for_repo(repository, fetch_identifiers: fetch_only_ids).each do |units|
+        service_class.pulp_units_batch_for_repo(repository, fetch_identifiers: fetch_only_ids, content_type: content_type_name).each do |units|
           units.each do |unit|
             unit = unit.with_indifferent_access
             pulp_id = unit[service_class.unit_identifier]
@@ -171,7 +175,11 @@ def import_for_repository(repository)
               service = service_class.new(model.pulp_id)
               service.backend_data = unit
               model.repository_id = repository.id unless many_repository_associations
-              service.update_model(model)
+              if repository.generic?
+                service.update_model(model, repository.repository_type, content_type_name)
+              else
+                service.update_model(model)
+              end
             end
             pulp_id_href_map[pulp_id] = backend_identifier
           end

app/models/katello/generic_content_unit.rb

@@ -0,0 +1,16 @@
+module Katello
+  class GenericContentUnit < Katello::Model
+    self.table_name = 'katello_generic_content_units'
+    include Concerns::PulpDatabaseUnit
+
+    CONTENT_TYPE = 'generic_content_unit'.freeze
+
+    def self.default_sort
+      order(:name)
+    end
+
+    def self.total_for_repositories(repos)
+      self.in_repositories(repos).count
+    end
+  end
+end

app/models/katello/repository.rb

@@ -29,6 +29,7 @@ class Repository < Katello::Model
     DOCKER_TYPE = 'docker'.freeze
     OSTREE_TYPE = 'ostree'.freeze
     ANSIBLE_COLLECTION_TYPE = 'ansible_collection'.freeze
+    GENERIC_TYPE = 'generic'.freeze
 
     EXPORTABLE_TYPES = [YUM_TYPE, FILE_TYPE, ANSIBLE_COLLECTION_TYPE].freeze
 
@@ -57,6 +58,9 @@ class Repository < Katello::Model
     has_many :repository_srpms, :class_name => "Katello::RepositorySrpm", :dependent => :delete_all
     has_many :srpms, :through => :repository_srpms
 
+    has_many :repository_generic_content_units, :class_name => "Katello::RepositoryGenericContentUnit", :dependent => :delete_all
+    has_many :generic_content_units, :through => :repository_generic_content_units
+
     has_many :repository_file_units, :class_name => "Katello::RepositoryFileUnit", :dependent => :delete_all
     has_many :files, :through => :repository_file_units, :source => :file_unit
     alias_attribute :file_units, :files
@@ -130,6 +134,7 @@ class Repository < Katello::Model
     scope :docker_type, -> { with_type(DOCKER_TYPE) }
     scope :ostree_type, -> { with_type(OSTREE_TYPE) }
     scope :ansible_collection_type, -> { with_type(ANSIBLE_COLLECTION_TYPE) }
+    scope :generic_type, -> { with_type(Katello::RepositoryTypeManager.enabled_repository_types.select { |_, v| v.pulp3_service_class == Katello::Pulp3::Repository::Generic }.keys) }
     scope :non_archived, -> { where('environment_id is not NULL') }
     scope :archived, -> { where('environment_id is NULL') }
     scope :in_published_environments, -> { in_content_views(Katello::ContentView.non_default).where.not(:environment_id => nil) }
@@ -166,7 +171,7 @@ class Repository < Katello::Model
     scoped_search :on => :content_label, :ext_method => :search_by_content_label
 
     delegate :product, :redhat?, :custom?, :to => :root
-    delegate :yum?, :docker?, :deb?, :file?, :ostree?, :ansible_collection?, :to => :root
+    delegate :yum?, :docker?, :deb?, :file?, :ostree?, :ansible_collection?, :generic?, :to => :root
     delegate :name, :label, :docker_upstream_name, :url, :download_concurrency, :to => :root
 
     delegate :name, :created_at, :updated_at, :major, :minor, :gpg_key_id, :gpg_key, :arch, :label, :url, :unprotected,
@@ -178,6 +183,7 @@ class Repository < Katello::Model
              :ansible_collection_auth_url, :ansible_collection_auth_token, :http_proxy_policy, :http_proxy_id, :to => :root
 
     delegate :content_id, to: :root, allow_nil: true
+    delegate :repository_type, to: :root
 
     def self.with_type(content_type)
       joins(:root).where("#{RootRepository.table_name}.content_type" => content_type)
@@ -872,10 +878,6 @@ def custom_docker_repo_path
       parts.map { |x| x.gsub(/[^-\w]/, "_") }.join("-").downcase
     end
 
-    def repository_type
-      RepositoryTypeManager.find(self.content_type)
-    end
-
     def copy_indexed_data(source_repository)
       repository_type.content_types_to_index.each do |type|
         type.model_class.copy_repository_associations(source_repository, self)
@@ -901,7 +903,11 @@ def index_content(options = {})
       else
         repository_type.content_types_to_index.each do |type|
           Katello::Logging.time("CONTENT_INDEX", data: {type: type.model_class}) do
-            type.model_class.import_for_repository(self)
+            if self.generic?
+              type.model_class.import_for_repository(self, type.content_type)
+            else
+              type.model_class.import_for_repository(self)
+            end
           end
         end
         repository_type.index_additional_data_proc&.call(self)

app/models/katello/repository_generic_content_unit.rb

@@ -0,0 +1,7 @@
+module Katello
+  class RepositoryGenericContentUnit < Katello::Model
+    # Do not use active record callbacks in this join model.  Direct INSERTs and DELETEs are done
+    belongs_to :repository, :inverse_of => :repository_generic_content_units, :class_name => 'Katello::Repository'
+    belongs_to :generic_content_unit, :inverse_of => :repository_generic_content_units, :class_name => 'Katello::GenericContentUnit'
+  end
+end

app/models/katello/root_repository.rb

@@ -1,4 +1,5 @@
 module Katello
+  # rubocop:disable Metrics/ClassLength
   class RootRepository < Katello::Model
     audited :except => [:content_id]
     serialize :ignorable_content
@@ -120,6 +121,10 @@ def self.repositories
       Repository.where(:root => self)
     end
 
+    def repository_type
+      RepositoryTypeManager.find(self.content_type)
+    end
+
     def custom?
       !redhat?
     end
@@ -313,6 +318,10 @@ def ansible_collection?
       self.content_type == Repository::ANSIBLE_COLLECTION_TYPE
     end
 
+    def generic?
+      Katello::RepositoryTypeManager.generic_repository_types(enabled_only: false).values.map(&:id).map(&:to_s).flatten.include? self.content_type
+    end
+
     def metadata_generate_needed?
       (%w(unprotected checksum_type container_repsoitory_name) & previous_changes.keys).any?
     end

app/services/katello/pulp3/api/generic.rb

@@ -1,4 +1,5 @@
 require "pulpcore_client"
+
 module Katello
   module Pulp3
     module Api
@@ -14,24 +15,24 @@ def self.api_exception_class
           fail NotImplementedError
         end
 
-        def self.client_module
-          fail NotImplementedError
+        def self.client_module(repository_type)
+          repository_type.client_module_class
         end
 
-        def self.remote_class
-          fail NotImplementedError
+        def self.remote_class(repository_type)
+          repository_type.remote_class
         end
 
-        def self.distribution_class
-          fail NotImplementedError
+        def self.distribution_class(repository_type)
+          repository_type.distribution_class
         end
 
-        def self.publication_class
-          fail NotImplementedError
+        def self.publication_class(repository_type)
+          repository_type.publication_class
         end
 
-        def self.repository_sync_url_class
-          fail NotImplementedError
+        def self.repository_sync_url_class(repository_type)
+          repository_type.repo_sync_url_class
         end
 
         def self.add_remove_content_class
@@ -55,7 +56,7 @@ def remotes_api
         end
 
         def publications_api
-          fail NotImplementedError
+          @repository_type.publications_api_class.new(api_client)
         end
 
         def distributions_api

app/services/katello/pulp3/generic_content_unit.rb

@@ -0,0 +1,29 @@
+module Katello
+  module Pulp3
+    class GenericContentUnit < PulpContentUnit
+      include LazyAccessor
+      CONTENT_TYPE = "generic_content_unit".freeze
+
+      def self.fetch_content_list(page_opts, repository_type, content_type)
+        content_unit_list page_opts, repository_type, content_type
+      end
+
+      def self.content_unit_list(page_opts, repository_type, content_type)
+        self.content_api(repository_type, content_type).list page_opts
+      end
+
+      def self.content_api(repository_type, content_type)
+        repository_type.content_types.find { |type| type.content_type == content_type }.pulp3_api.new(repository_type.pulp3_api_class.new(SmartProxy.pulp_primary!, repository_type).api_client)
+      end
+
+      def update_model(model, repository_type, content_type)
+        custom_json = {}
+        custom_json['pulp_id'] = backend_data['pulp_href']
+        custom_json['name'] = repository_type.model_name.call(backend_data)
+        custom_json['version'] = repository_type.model_version.call(backend_data)
+        custom_json['content_type'] = content_type
+        model.update!(custom_json)
+      end
+    end
+  end
+end

app/services/katello/pulp3/pulp_content_unit.rb

@@ -72,7 +72,11 @@ def self.pulp_units_batch_for_repo(repository, options = {})
               (response["count"] && page_opts["offset"] < response["count"]) ||
               page_opts["offset"] == 0)
             page_opts = page_options page_opts
-            response = fetch_content_list page_opts
+            if repository.generic?
+              response = fetch_content_list page_opts, repository.repository_type, options[:content_type]
+            else
+              response = fetch_content_list page_opts
+            end
             response = response.as_json.with_indifferent_access
             yielder.yield response[:results]
             page_opts[:offset] += page_size