i672 Create an object factory that supports Valkyrie

app/factories/bulkrax/valkyrie_object_factory.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Bulkrax
+  class ValkyrieObjectFactory < ObjectFactory
+    ##
+    # Retrieve properties from M3 model
+    # @param klass the model
+    # return Array<string>
+    def self.schema_properties(klass)
+      @schema_properties_map ||= {}
+
+      klass_key = klass.name
+      unless @schema_properties_map.has_key?(klass_key)
+        @schema_properties_map[klass_key] = klass.schema.map { |k| k.name.to_s }
+      end
+
+      @schema_properties_map[klass_key]
+    end
+
+    def run!
+      run
+      return object if object.persisted?
+
+      raise(RecordInvalid, object)
+    end
+
+    def find_by_id
+      Hyrax.query_service.find_by(id: attributes[:id]) if attributes.key? :id
+    end
+
+    def search_by_identifier
+      # Query can return partial matches (something6 matches both something6 and something68)
+      # so we need to weed out any that are not the correct full match. But other items might be
+      # in the multivalued field, so we have to go through them one at a time.
+      match = Hyrax.query_service.find_by_alternate_identifier(alternate_identifier: source_identifier_value)
+
+      return match if match
+    rescue => err
+      Hyrax.logger.error(err)
+      false
+    end
+
+    # An ActiveFedora bug when there are many habtm <-> has_many associations means they won't all get saved.
+    # https://github.com/projecthydra/active_fedora/issues/874
+    # 2+ years later, still open!
+    def create
+      attrs = transform_attributes
+        .merge(alternate_ids: [source_identifier_value])
+        .symbolize_keys
+
+      cx = Hyrax::Forms::ResourceForm.for(klass.new).prepopulate!
+      cx.validate(attrs)
+
+      result = transaction
+        .with_step_args(
+          # "work_resource.add_to_parent" => {parent_id: @related_parents_parsed_mapping, user: @user},
+          "work_resource.add_bulkrax_files" => {files: get_s3_files(remote_files: attributes["remote_files"]), user: @user},
+          "change_set.set_user_as_depositor" => {user: @user},
+          "work_resource.change_depositor" => {user: @user}
+          # TODO: uncomment when we upgrade Hyrax 4.x
+          # 'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
+        )
+        .call(cx)
+
+      @object = result.value!
+
+      @object
+    end
+
+    def update
+      raise "Object doesn't exist" unless @object
+
+      destroy_existing_files if @replace_files && ![Collection, FileSet].include?(klass)
+
+      attrs = transform_attributes(update: true)
+
+      cx = Hyrax::Forms::ResourceForm.for(@object)
+      cx.validate(attrs)
+
+      result = update_transaction
+        .with_step_args(
+          "work_resource.add_bulkrax_files" => {files: get_s3_files(remote_files: attributes["remote_files"]), user: @user}
+
+          # TODO: uncomment when we upgrade Hyrax 4.x
+          # 'work_resource.save_acl' => { permissions_params: [attrs.try('visibility') || 'open'].compact }
+        )
+        .call(cx)
+
+      @object = result.value!
+    end
+
+    def get_s3_files(remote_files: {})
+      if remote_files.blank?
+        Hyrax.logger.info "No remote files listed for #{attributes["source_identifier"]}"
+        return []
+      end
+
+      s3_bucket_name = ENV.fetch("STAGING_AREA_S3_BUCKET", "comet-staging-area-#{Rails.env}")
+      s3_bucket = Rails.application.config.staging_area_s3_connection
+        .directories.get(s3_bucket_name)
+
+      remote_files.map { |r| r["url"] }.map do |key|
+        s3_bucket.files.get(key)
+      end.compact
+    end
+
+    ##
+    # TODO: What else fields are necessary: %i[id edit_users edit_groups read_groups work_members_attributes]?
+    # Regardless of what the Parser gives us, these are the properties we are prepared to accept.
+    def permitted_attributes
+      Bulkrax::ValkyrieObjectFactory.schema_properties(klass) +
+        %i[
+          admin_set_id
+          title
+          visibility
+        ]
+    end
+
+    def apply_depositor_metadata(object, user)
+      object.depositor = user.email
+      object = Hyrax.persister.save(resource: object)
+      Hyrax.publisher.publish("object.metadata.updated", object: object, user: @user)
+      object
+    end
+
+    # @Override remove branch for FileSets replace validation with errors
+    def new_remote_files
+      @new_remote_files ||= if @object.is_a? FileSet
+        parsed_remote_files.select do |file|
+          # is the url valid?
+          is_valid = file[:url]&.match(URI::ABS_URI)
+          # does the file already exist
+          is_existing = @object.import_url && @object.import_url == file[:url]
+          is_valid && !is_existing
+        end
+      else
+        parsed_remote_files.select do |file|
+          file[:url]&.match(URI::ABS_URI)
+        end
+      end
+    end
+
+    # @Override Destroy existing files with Hyrax::Transactions
+    def destroy_existing_files
+      existing_files = fetch_child_file_sets(resource: @object)
+
+      existing_files.each do |fs|
+        Hyrax::Transactions::Container["file_set.destroy"]
+          .with_step_args("file_set.remove_from_work" => {user: @user},
+            "file_set.delete" => {user: @user})
+          .call(fs)
+          .value!
+      end
+
+      @object.member_ids = @object.member_ids.reject { |m| existing_files.detect { |f| f.id == m } }
+      @object.rendering_ids = []
+      @object.representative_id = nil
+      @object.thumbnail_id = nil
+    end
+
+    private
+
+    def transaction
+      Hyrax::Transactions::Container["work_resource.create_with_bulk_behavior"]
+    end
+
+    # Customize Hyrax::Transactions::WorkUpdate transaction with bulkrax
+    def update_transaction
+      Hyrax::Transactions::Container["work_resource.update_with_bulk_behavior"]
+    end
+
+    # Query child FileSet in the resource/object
+    def fetch_child_file_sets(resource:)
+      Hyrax.custom_queries.find_child_file_sets(resource: resource)
+    end
+  end
+
+  class RecordInvalid < StandardError
+  end
+end

app/helpers/bulkrax/importers_helper.rb

@@ -5,8 +5,13 @@ module ImportersHelper
     # borrowed from batch-importer https://github.com/samvera-labs/hyrax-batch_ingest/blob/main/app/controllers/hyrax/batch_ingest/batches_controller.rb
     def available_admin_sets
       # Restrict available_admin_sets to only those current user can deposit to.
-      @available_admin_sets ||= Hyrax::Collections::PermissionsService.source_ids_for_deposit(ability: current_ability, source_type: 'admin_set').map do |admin_set_id|
-        [AdminSet.find(admin_set_id).title.first, admin_set_id]
+      # TODO: key off of something more reliable than Bulkrax.object_factory
+      if Bulkrax.object_factory.to_s == 'Bulkrax::ValkyrieObjectFactory'
+        @available_admin_sets ||= Hyrax.metadata_adapter.query_service.find_all_of_model(model: Hyrax::AdministrativeSet).to_a
+      else
+        @available_admin_sets ||= Hyrax::Collections::PermissionsService.source_ids_for_deposit(ability: current_ability, source_type: 'admin_set').map do |admin_set_id|
+          [AdminSet.find(admin_set_id).title.first, admin_set_id]
+        end
       end
     end
   end

app/models/concerns/bulkrax/has_matchers.rb

@@ -125,7 +125,9 @@ def field_supported?(field)
 
       return false if excluded?(field)
       return true if supported_bulkrax_fields.include?(field)
-      return factory_class.method_defined?(field) && factory_class.properties[field].present?
+      property_defined = factory_class.singleton_methods.include?(:properties) && factory_class.properties[field].present?
+
+      factory_class.method_defined?(field) && (Bulkrax::ValkyrieObjectFactory.schema_properties(factory_class).include?(field) || property_defined)
     end
 
     def supported_bulkrax_fields
@@ -155,7 +157,18 @@ def multiple?(field)
       return true if @multiple_bulkrax_fields.include?(field)
       return false if field == 'model'
 
-      field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')
+      # TODO: key off of something more reliable than Bulkrax.object_factory
+      if Bulkrax.object_factory.to_s == 'Bulkrax::ValkyrieObjectFactory'
+        field_supported?(field) && (multiple_field?(field) || factory_class.singleton_methods.include?(:properties) && factory_class&.properties&.[](field)&.[]("multiple"))
+      else
+        field_supported?(field) && factory_class&.properties&.[](field)&.[]('multiple')
+      end
+    end
+
+    def multiple_field?(field)
+      Hyrax::Forms::ResourceForm # TODO: this prevents `NoMethodError: undefined method `ResourceForm' for Hyrax::Forms:Module`, why?
+      form_class = "#{factory_class}Form".constantize
+      form_class.definitions[field.to_s][:multiple].present?
     end
 
     def get_object_name(field)

app/services/bulkrax/transactions/steps/add_files.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "dry/monads"
+
+module Bulkrax
+  module Transactions
+    module Steps
+      class AddFiles
+        include Dry::Monads[:result]
+
+        ##
+        # @param [Class] handler
+        def initialize(handler: Hyrax::WorkUploadsHandler)
+          @handler = handler
+        end
+
+        ##
+        # @param [Hyrax::Work] obj
+        # @param [Array<Fog::AWS::Storage::File>] file
+        # @param [User] user
+        #
+        # @return [Dry::Monads::Result]
+        def call(obj, files:, user:)
+          if files && user
+            begin
+              files.each do |file|
+                FileIngest.upload(
+                  content_type: file.content_type,
+                  file_body: StringIO.new(file.body),
+                  filename: Pathname.new(file.key).basename,
+                  last_modified: file.last_modified,
+                  permissions: Hyrax::AccessControlList.new(resource: obj),
+                  size: file.content_length,
+                  user: user,
+                  work: obj
+                )
+              end
+            rescue => e
+              Hyrax.logger.error(e)
+              return Failure[:failed_to_attach_file_sets, files]
+            end
+          end
+
+          Success(obj)
+        end
+      end
+    end
+  end
+end

bulkrax.gemspec

@@ -19,9 +19,9 @@ Gem::Specification.new do |s|
   s.files = Dir["{app,config,db,lib}/**/*", "LICENSE", "Rakefile", "README.md"]
 
   s.add_dependency 'rails', '>= 5.1.6'
-  s.add_dependency 'bagit', '~> 0.4'
+  # s.add_dependency 'bagit', '~> 0.4'
   s.add_dependency 'coderay'
-  s.add_dependency 'dry-monads', '~> 1.4.0'
+  s.add_dependency 'dry-monads', '~> 1.5'
   s.add_dependency 'iso8601', '~> 0.9.0'
   s.add_dependency 'kaminari'
   s.add_dependency 'language_list', '~> 1.2', '>= 1.2.1'

lib/generators/bulkrax/templates/config/initializers/bulkrax.rb

@@ -12,6 +12,8 @@
 
   # Factory Class to use when generating and saving objects
   config.object_factory = Bulkrax::ObjectFactory
+  # Use this for a Postgres-backed Valkyrized Hyrax
+  # config.object_factory = Bulkrax::ValkyrieObjectFactory
 
   # Path to store pending imports
   # config.import_path = 'tmp/imports'
@@ -83,3 +85,98 @@
 if Object.const_defined?(:Hyrax) && ::Hyrax::DashboardController&.respond_to?(:sidebar_partials)
   Hyrax::DashboardController.sidebar_partials[:repository_content] << "hyrax/dashboard/sidebar/bulkrax_sidebar_additions"
 end
+
+# TODO: move outside of initializer?
+class BulkraxTransactionContainer
+  extend Dry::Container::Mixin
+
+  namespace "work_resource" do |ops|
+    ops.register "create_with_bulk_behavior" do
+      steps = Hyrax::Transactions::WorkCreate::DEFAULT_STEPS.dup
+      steps[steps.index("work_resource.add_file_sets")] = "work_resource.add_bulkrax_files"
+
+      Hyrax::Transactions::WorkCreate.new(steps: steps)
+    end
+
+    ops.register "update_with_bulk_behavior" do
+      steps = Hyrax::Transactions::WorkUpdate::DEFAULT_STEPS.dup
+      steps[steps.index("work_resource.add_file_sets")] = "work_resource.add_bulkrax_files"
+
+      Hyrax::Transactions::WorkUpdate.new(steps: steps)
+    end
+
+    # TODO: uninitialized constant BulkraxTransactionContainer::InlineUploadHandler
+    # ops.register "add_file_sets" do
+    #   Hyrax::Transactions::Steps::AddFileSets.new(handler: InlineUploadHandler)
+    # end
+
+    ops.register "add_bulkrax_files" do
+      Bulkrax::Transactions::Steps::AddFiles.new
+    end
+  end
+end
+Hyrax::Transactions::Container.merge(BulkraxTransactionContainer)
+
+# TODO: move outside of initializer?
+module HasMappingExt
+  ##
+  # Field of the model that can be supported
+  def field_supported?(field)
+    field = field.gsub("_attributes", "")
+
+    return false if excluded?(field)
+    return true if supported_bulkrax_fields.include?(field)
+
+    property_defined = factory_class.singleton_methods.include?(:properties) && factory_class.properties[field].present?
+
+    factory_class.method_defined?(field) && (Bulkrax::ValkyrieObjectFactory.schema_properties(factory_class).include?(field) || property_defined)
+  end
+
+  ##
+  # Determine a multiple properties field
+  def multiple?(field)
+    @multiple_bulkrax_fields ||=
+      %W[
+        file
+        remote_files
+        rights_statement
+        #{related_parents_parsed_mapping}
+        #{related_children_parsed_mapping}
+      ]
+
+    return true if @multiple_bulkrax_fields.include?(field)
+    return false if field == "model"
+
+    field_supported?(field) && (multiple_field?(field) || factory_class.singleton_methods.include?(:properties) && factory_class&.properties&.[](field)&.[]("multiple"))
+  end
+
+  def multiple_field?(field)
+    form_definition = schema_form_definitions[field.to_sym]
+    form_definition.nil? ? false : form_definition.multiple?
+  end
+
+  # override: we want to directly infer from a property being multiple that we should split when it's a String
+  # def multiple_metadata(content)
+  #   return unless content
+
+  #   case content
+  #   when Nokogiri::XML::NodeSet
+  #     content&.content
+  #   when Array
+  #     content
+  #   when Hash
+  #     Array.wrap(content)
+  #   when String
+  #     String(content).strip.split(Bulkrax.multi_value_element_split_on)
+  #   else
+  #     Array.wrap(content)
+  #   end
+  # end
+
+  def schema_form_definitions
+    @schema_form_definitions ||= ::SchemaLoader.new.form_definitions_for(factory_class.name.underscore.to_sym)
+  end
+end
+[Bulkrax::HasMatchers, Bulkrax::HasMatchers.singleton_class].each do |mod|
+  mod.prepend HasMappingExt
+end