More detailed file status

This adds a status field to `PreIngestFile`s and
`PreIngestWorks` that defaults to 'preingest'.

The `ContentDepositEventJob` and `FileSetAttachedEventJob` classes
are preprended with logic that updates their status to 'attached'.

The logic requires that `deduplication_key` is present in the
CSV so that field is added to the default mapper and validator.

There is a helper which outputs a success icon (a green checkmark)
if a `PreIngestFile` or `PreIngestWork` has an 'attached' status.

README.md

@@ -60,7 +60,7 @@ to create a new application with Zizia installed.
 8. Add a deduplication_key to your default work type's medata:
 
 ```
-  property :deduplication_key, predicate: "http://curationexperts.com/vocab/predicates#deduplicationKey", multiple: false do |index|
+  property :deduplication_key, predicate: ::RDF::Vocab::BF2::identifiedBy, multiple: false do |index|
     index.as :stored_searchable
   end
 ```
@@ -104,7 +104,7 @@ environment variables called `IMPORT_PATH`. If `IMPORT_PATH` is not set, `HyraxR
 
 To run Solr and Fedora for testing purposes, open a new terminal session for each and run the following commads:
 
-`solr_wrapper --config spec/dummy/config/solr_wrapper_test.yml`  
+`solr_wrapper --config spec/dummy/config/solr_wrapper_test.yml`
 `fcrepo_wrapper --config spec/dummy/config/fcrepo_wrapper_test.yml`
 
 After this you can run the whole suite, or individual specs. System specs are located

app/helpers/zizia/application_helper.rb

@@ -30,5 +30,11 @@ def collections_for_select
     def collections?
       !ActiveFedora::SolrService.query('has_model_ssim:Collection').empty?
     end
+
+    def status_icon(status)
+      # rubocop:disable Rails/OutputSafety
+      return "<span class='glyphicon glyphicon-ok-sign text-success' aria-label='Status: Complete'></span>".html_safe if status == 'attached'
+      "<span class='glyphicon glyphicon-question-sign' aria-label='Status: Unknown'></span>".html_safe
+    end
   end
 end

app/uploaders/zizia/csv_manifest_validator.rb

@@ -63,7 +63,7 @@ def valid_headers
          'publisher', 'date created', 'subject',
          'language', 'identifier', 'location',
          'related url', 'bibliographic_citation',
-         'source', 'visibility']
+         'source', 'visibility', 'deduplication_key', 'type']
       end
 
       def parse_csv
@@ -82,7 +82,7 @@ def missing_headers
       end
 
       def required_headers
-        ['title', 'creator', 'keyword', 'rights statement', 'visibility', 'files']
+        ['title', 'creator', 'keyword', 'rights statement', 'visibility', 'files', 'deduplication_key']
       end
 
       def duplicate_headers

app/views/zizia/csv_import_details/_pre_ingest_files_table.html.erb

@@ -8,6 +8,7 @@
       <th>Size</th>
       <th>Row Number</th>
       <th>Date Created</th>
+      <th>Status</th>
     </tr>
     <% pre_ingest_work.pre_ingest_files.each do |pre_ingest_file| %>
       <tr>
@@ -23,6 +24,9 @@
         <td>
           <%= pre_ingest_file.created_at.strftime("%B %-d, %Y %H:%M") %>
         </td>
+        <td>
+          <%= status_icon(pre_ingest_file.status) %>
+        </td>
       </tr>
     <% end %>
   </table>

app/views/zizia/csv_import_details/show.html.erb

@@ -30,6 +30,9 @@
       <td>
         <%= pre_ingest_work.created_at.strftime("%B %-d, %Y %H:%M") %>
       </td>
+      <td>
+        <%= status_icon(pre_ingest_work.status) %>
+      </td>
     </tr>
   <% end %>
   </table>

config/initializers/content_deposit_event_job_prepends.rb

@@ -0,0 +1,10 @@
+module ContentDepositEventJobPrepends
+  def action
+    super
+    if repo_object.deduplication_key
+      pre_ingest_work = Zizia::PreIngestWork.find_by(deduplication_key: repo_object.deduplication_key)
+      pre_ingest_work.status = 'attached'
+      pre_ingest_work.save
+    end
+  end
+end

config/initializers/file_set_attached_event_job_prepends.rb

@@ -0,0 +1,12 @@
+module FileSetAttachedEventJobPrepends
+  def action
+    if repo_object.kind_of?(FileSet)
+      pre_ingest_work_id = Zizia::PreIngestWork.find_by(deduplication_key: curation_concern.deduplication_key)
+      pre_ingest_file = Zizia::PreIngestFile.find_by(size: repo_object.files.first.size,
+                                                     filename: repo_object.files.first.original_name,
+                                                     pre_ingest_work_id: pre_ingest_work_id)
+      pre_ingest_file.status = 'attached'
+      pre_ingest_file.save
+    end
+  end
+end

config/initializers/prepends.rb

@@ -0,0 +1,2 @@
+ContentDepositEventJob.prepend(ContentDepositEventJobPrepends)
+FileSetAttachedEventJob.prepend(FileSetAttachedEventJobPrepends)

db/migrate/20191114021032_add_status_to_pre_ingest_file.rb

@@ -0,0 +1,5 @@
+class AddStatusToPreIngestFile < ActiveRecord::Migration[5.1]
+  def change
+    add_column :zizia_pre_ingest_files, :status, :string, default: 'preingest'
+  end
+end

db/migrate/20191210223833_add_status_to_pre_ingest_work.rb

@@ -0,0 +1,5 @@
+class AddStatusToPreIngestWork < ActiveRecord::Migration[5.1]
+  def change
+    add_column :zizia_pre_ingest_works, :status, :string, default: 'preingest'
+  end
+end

lib/zizia/hyrax/hyrax_basic_metadata_mapper.rb

@@ -31,7 +31,7 @@ class HyraxBasicMetadataMapper < HashMapper
     ##
     # @return [Enumerable<Symbol>] The fields the mapper can process.
     def fields
-      core_fields + basic_fields + [:visibility, :files]
+      core_fields + basic_fields + [:visibility, :files] + zizia_fields
     end
 
     # Properties defined with `multiple: false` in
@@ -57,6 +57,10 @@ def import_url
       single_value('import_url')
     end
 
+    def deduplication_key
+      single_value('deduplication_key')
+    end
+
     # We should accept visibility values that match the UI and transform them into
     # the controlled vocabulary term expected by Hyrax
     def visibility
@@ -152,5 +156,10 @@ def basic_fields
          :based_near, :related_url,
          :bibliographic_citation, :source]
       end
+
+      # Properties requires for zizia
+      def zizia_fields
+        [:deduplication_key]
+      end
   end
 end

spec/dummy/app/models/work.rb

@@ -9,7 +9,7 @@ class Work < ActiveFedora::Base
   validates :title, presence: { message: 'Your work must have a title.' }
 
 
-  property :deduplication_key, predicate: 'http://metadata.example.com/vocab/predicates#deduplicationKey', multiple: false do |index|
+  property :deduplication_key, predicate: ::RDF::Vocab::BF2::identifiedBy, multiple: false do |index|
     index.as :stored_searchable
   end
   # This must be included at the end, because it finalizes the metadata

spec/dummy/db/schema.rb

@@ -595,6 +595,8 @@
     t.integer "pre_ingest_work_id"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.string "status", default: "preingest"
+    t.string "access_control_id"
     t.index ["pre_ingest_work_id"], name: "index_zizia_pre_ingest_files_on_pre_ingest_work_id"
   end
 
@@ -604,6 +606,7 @@
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.string "deduplication_key"
+    t.string "status", default: "preingest"
     t.index ["csv_import_detail_id"], name: "index_zizia_pre_ingest_works_on_csv_import_detail_id"
     t.index ["deduplication_key"], name: "index_zizia_pre_ingest_works_on_deduplication_key"
   end

spec/dummy/spec/fixtures/csv_import/csv_files_with_problems/extra - headers.csv

@@ -1,4 +1,4 @@
-rights_statement,another_header_2,rights statement,title,files,label,relative_path,import url,resource type,creator,contributor,abstract or summary,keyword,license,publisher,date created,subject,language,identifier,location,related url,bibliographic_citation,source,visibility
-http://rightsstatements.org/vocab/InC/1.0/,,http://rightsstatements.org/vocab/InC/1.0/,Work 1 Title,cat.jpg,,,,,creator,,,cat,,,,,,,,,,,open
-http://rightsstatements.org/vocab/InC/1.0/,,http://rightsstatements.org/vocab/InC/1.0/,Work 2 Title,cat.jpg,,,,,creator,,,cat,,,,,,,,,,,open
-http://rightsstatements.org/vocab/InC/1.0/,,http://rightsstatements.org/vocab/InC/1.0/,Work 3 Title,cat.jpg,,,,,creator,,,cat,,,,,,,,,,,open
+deduplication_key,rights_statement,another_header_2,rights statement,title,files,label,relative_path,import url,resource type,creator,contributor,abstract or summary,keyword,license,publisher,date created,subject,language,identifier,location,related url,bibliographic_citation,source,visibility
+1,http://rightsstatements.org/vocab/InC/1.0/,,http://rightsstatements.org/vocab/InC/1.0/,Work 1 Title,cat.jpg,,,,,creator,,,cat,,,,,,,,,,,open
+2,http://rightsstatements.org/vocab/InC/1.0/,,http://rightsstatements.org/vocab/InC/1.0/,Work 2 Title,cat.jpg,,,,,creator,,,cat,,,,,,,,,,,open
+3,http://rightsstatements.org/vocab/InC/1.0/,,http://rightsstatements.org/vocab/InC/1.0/,Work 3 Title,cat.jpg,,,,,creator,,,cat,,,,,,,,,,,open

spec/dummy/spec/fixtures/csv_import/good/all_fields.csv

@@ -1,2 +1,2 @@
-identifier,license,deduplication_key,visibility,location,keyword,rights statement,creator,title,files
-abc/123,https://creativecommons.org/licenses/by/4.0/,abc/123,PUBlic,http://www.geonames.org/5667009/montana.html|~|http://www.geonames.org/6252001/united-states.html,Clothing stores $z California $z Los Angeles|~|Interior design $z California $z Los Angeles,http://rightsstatements.org/vocab/InC/1.0/,"Connell, Will, $d 1898-1961","Interior view of The Bachelors haberdashery designed by Julius Ralph Davidson, Los Angeles, circa 1929",dog.jpg
+identifier,type,license,deduplication_key,visibility,location,keyword,rights statement,creator,title,files
+abc/123,work,https://creativecommons.org/licenses/by/4.0/,abc/123,PUBlic,http://www.geonames.org/5667009/montana.html|~|http://www.geonames.org/6252001/united-states.html,Clothing stores $z California $z Los Angeles|~|Interior design $z California $z Los Angeles,http://rightsstatements.org/vocab/InC/1.0/,"Connell, Will, $d 1898-1961","Interior view of The Bachelors haberdashery designed by Julius Ralph Davidson, Los Angeles, circa 1929",dog.jpg

spec/dummy/spec/fixtures/csv_import/good/all_fields_multi.csv

@@ -0,0 +1,2 @@
+identifier,type,license,deduplication_key,visibility,location,keyword,rights statement,creator,title,files
+abc/123,work,https://creativecommons.org/licenses/by/4.0/,abc/123,PUBlic,http://www.geonames.org/5667009/montana.html|~|http://www.geonames.org/6252001/united-states.html,Clothing stores $z California $z Los Angeles|~|Interior design $z California $z Los Angeles,http://rightsstatements.org/vocab/InC/1.0/,"Connell, Will, $d 1898-1961","Interior view of The Bachelors haberdashery designed by Julius Ralph Davidson, Los Angeles, circa 1929",dog.jpg|~|cat.jpg

spec/dummy/spec/support/capybara.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+Capybara.default_max_wait_time = 20
 
 # Setup chrome headless driver
 Capybara.server = :puma, { Silent: true }

spec/dummy/spec/system/import_file_status_spec.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+require 'rails_helper'
+include Warden::Test::Helpers
+
+RSpec.describe 'Importing records from a CSV file', :perform_jobs, :clean, type: :system, js: true do
+  before do
+    allow(CharacterizeJob).to receive(:perform_later)
+    ENV['IMPORT_PATH'] = File.join(fixture_path, 'images')
+  end
+
+  let(:csv_file) { File.join(fixture_path, 'csv_import', 'good', 'all_fields_multi.csv') }
+
+  context 'logged in as an admin user' do
+    let(:collection) { FactoryBot.build(:collection, title: ['Testing Collection']) }
+    let(:admin_user) { FactoryBot.create(:admin) }
+
+    let(:admin_set_id) { AdminSet.find_or_create_default_admin_set_id }
+    let(:permission_template) { Hyrax::PermissionTemplate.find_or_create_by!(source_id: admin_set_id) }
+    let(:workflow) { Sipity::Workflow.create!(active: true, name: 'test-workflow', permission_template: permission_template) }
+
+    before do
+      # Create a single action that can be taken
+      Sipity::WorkflowAction.create!(name: 'submit', workflow: workflow)
+
+      # Grant the user access to deposit into the admin set.
+      Hyrax::PermissionTemplateAccess.create!(
+        permission_template_id: permission_template.id,
+        agent_type: 'user',
+        agent_id: admin_user.user_key,
+        access: 'deposit'
+      )
+
+      collection.save!
+      login_as admin_user
+    end
+
+    it 'starts the import' do
+      visit '/csv_imports/new'
+      expect(page).to have_content 'Testing Collection'
+      expect(page).not_to have_content '["Testing Collection"]'
+
+      # Fill in and submit the form
+      select 'Testing Collection', from: "csv_import[fedora_collection_id]"
+      select 'Update Existing Metadata, create new works', from: "csv_import[update_actor_stack]"
+      attach_file('csv_import[manifest]', csv_file, make_visible: true)
+
+      expect(page).to have_content('You sucessfully uploaded this CSV: all_fields_multi.csv')
+
+      click_on 'Preview Import'
+
+      # We expect to see the title of the collection on the page
+      expect(page).to have_content 'Testing Collection'
+
+      expect(page).to have_content 'This import will process 1 row(s).'
+
+      # There is a link so the user can cancel.
+      expect(page).to have_link 'Cancel', href: '/csv_imports/new?locale=en'
+
+      expect(page).not_to have_content 'deduplication_key'
+
+      # After reading the warnings, the user decides
+      # to continue with the import.
+      click_on 'Start Import'
+
+      # The show page for the CsvImport
+      expect(page).to have_content 'all_fields_multi.csv'
+      expect(page).to have_content 'Start time'
+
+      # We expect to see the title of the collection on the page
+      expect(page).to have_content 'Testing Collection'
+      expect(Work.count).to eq 1
+
+      # Ensure that all the fields got assigned as expected
+      work = Work.where(title: "*haberdashery*").first
+      expect(work.title.first).to match(/haberdashery/)
+      expect(Zizia::PreIngestFile.last.status).to eq('attached')
+
+      expect(Zizia::PreIngestWork.find_by(deduplication_key: work.deduplication_key).pre_ingest_files.count).to eq(2)
+
+      # Everything should have the attached status now
+      expect(Zizia::PreIngestWork.find_by(deduplication_key: work.deduplication_key)
+               .pre_ingest_files.first.status).to eq('attached')
+      expect(Zizia::PreIngestWork.find_by(deduplication_key: work.deduplication_key).pre_ingest_files.last.status).to eq('attached')
+      expect(Zizia::PreIngestWork.find_by(deduplication_key: work.deduplication_key).status).to eq('attached')
+      visit('/csv_import_details/index')
+      click_on '1'
+      expect(page).to have_content 'View Files'
+      click_on 'View Files'
+      expect(page.html).not_to match(/glyphicon-question-sign/)
+      expect(page.html).to match(/glyphicon-ok-sign/)
+      expect(page).to have_content('dog.jpg')
+      expect(page).to have_content('cat.jpg')
+    end
+  end
+end

spec/dummy/spec/system/import_from_csv_spec.rb

@@ -4,6 +4,7 @@
 
 RSpec.describe 'Importing records from a CSV file', :perform_jobs, :clean, type: :system, js: true do
   before do
+    allow(CharacterizeJob).to receive(:perform_later)
     ENV['IMPORT_PATH'] = File.join(fixture_path, 'images')
   end
 
@@ -32,7 +33,6 @@
         access: 'deposit'
       )
 
-      allow(CharacterizeJob).to receive(:perform_later) # There is no fits installed on travis-ci
       collection.save!
       login_as admin_user
     end
@@ -251,7 +251,7 @@
       expect(page).to have_content('cat.jpg')
       expect(page).to have_content('5.74 MB')
       expect(page).to have_content('abc/123')
-      expect(page).to have_content('This work\'s metadata has not been indexed yet.')
+      expect(page).to have_content('haberdashery')
       expect(page).to have_content('Date Created')
     end
   end

spec/models/zizia/application_job_spec.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+require 'rails_helper'
+
+RSpec.describe Zizia::ApplicationJob do
+  let(:application_job) { described_class.new }
+
+  context 'when including Zizia' do
+    it 'has its own ApplicationJob class' do
+      expect(application_job).to be_kind_of(ActiveJob::Base)
+    end
+  end
+end

spec/support/hyrax/basic_metadata.rb

@@ -25,6 +25,9 @@ def self.included(work)
       work.property :related_url, predicate: ::RDF::RDFS.seeAlso
       work.property :bibliographic_citation, predicate: ::RDF::Vocab::DC.bibliographicCitation
       work.property :source, predicate: ::RDF::Vocab::DC.source
+
+      # From Zizia
+      work.property :deduplication_key, predicate: ::RDF::Vocab::BF2::identifiedBy
     end
   end
 end

spec/zizia/hyrax/hyrax_basic_metadata_mapper_spec.rb

@@ -23,12 +23,16 @@
     [:visibility, :files]
   end
 
+  let(:zizia_fields) do
+    [:deduplication_key]
+  end
+
   it_behaves_like 'a Zizia::Mapper' do
     let(:metadata) do
       { title: ['A Title for a Record'],
         my_custom_field: ['This gets ignored'] }
     end
-    let(:expected_fields) { core_fields + basic_fields + tenejo_fields }
+    let(:expected_fields) { core_fields + basic_fields + tenejo_fields + zizia_fields }
   end
 
   context 'with metadata, but some missing fields' do