Merge pull request #2631 from archivesspace/clean-up-bulk-import-temp…

…lates

fix column key publish -> digital

backend/app/lib/bulk_import/import_archival_objects.rb

@@ -216,7 +216,8 @@ def create_archival_object(parent_uri)
       begin
         normalize_publish_column(@row_hash, 'digital_object_link_publish')
         normalize_publish_column(@row_hash, 'thumbnail_publish')
-        dig_instance = @doh.create(@row_hash["digital_object_title"], @row_hash["thumbnail"], @row_hash["digital_object_link"], @row_hash["digital_object_id"], @row_hash["publish"], ao, @report, @row_hash['digital_object_link_publish'], @row_hash['thumbnail_publish'])
+        normalize_publish_column(@row_hash, 'digital_object_publish')
+        dig_instance = @doh.create(@row_hash["digital_object_title"], @row_hash["thumbnail"], @row_hash["digital_object_link"], @row_hash["digital_object_id"], @row_hash["digital_object_publish"], ao, @report, @row_hash['digital_object_link_publish'], @row_hash['thumbnail_publish'])
       rescue Exception => e
         @report.add_errors(e.message)
       end

backend/spec/fixtures/bulk_import/bulk_import_VFIRST01_test01.csv

@@ -1,4 +1,4 @@
-ArchivesSpace field code (please don't edit this row),collection_id,ead,res_uri,ref_id,title,unit_id,hierarchy,level,other_level,publish,restrictions_flag,processing_note,l_lang,l_langscript,n_langmaterial,p_langmaterial,l_lang_2,l_langscript_2,n_langmaterial_2,p_langmaterial_2,dates_label,begin,end,date_type,expression,date_certainty,dates_label_2,begin_2,end_2,date_type_2,expression_2,date_certainty_2,portion,number,extent_type,container_summary,physical_details,dimensions,portion_2,number_2,extent_type_2,container_summary_2,physical_details_2,dimensions_2,cont_instance_type,type_1,indicator_1,barcode,type_2,indicator_2,type_3,indicator_3,cont_instance_type_2,type_1_2,indicator_1_2,barcode_2,type_2_2,indicator_2_2,type_3_2,indicator_3_2,digital_object_title,digital_object_link,thumbnail,people_agent_record_id_1,people_agent_header_1,people_agent_role_1,people_agent_relator_1,people_agent_record_id_2,people_agent_header_2,people_agent_role_2,people_agent_relator_2,people_agent_record_id_3,people_agent_header_3,people_agent_role_3,people_agent_relator_3,people_agent_record_id_4,people_agent_header_4,people_agent_role_4,people_agent_relator_4,people_agent_record_id_5,people_agent_header_5,people_agent_role_5,people_agent_relator_5,families_agent_record_id_1,families_agent_header_1,families_agent_role_1,families_agent_relator_1,families_agent_record_id_2,families_agent_header_2,families_agent_role_2,families_agent_relator_2,corporate_entities_agent_record_id_1,corporate_entities_agent_header_1,corporate_entities_agent_role_1,corporate_entities_agent_relator_1,corporate_entities_agent_record_id_2,corporate_entities_agent_header_2,corporate_entities_agent_role_2,corporate_entities_agent_relator_2,corporate_entities_agent_record_id_3,corporate_entities_agent_header_3,corporate_entities_agent_role_3,corporate_entities_agent_relator_3,subject_1_record_id,subject_1_term,subject_1_type,subject_1_source,subject_2_record_id,subject_2_term,subject_2_type,subject_2_source,n_abstract,p_abstract,n_accessrestrict,p_accessrestrict,n_acqinfo,p_acqinfo,n_arrangement,p_arrangement,n_bioghist,p_bioghist,n_custodhist,p_custodhist,n_dimensions,p_dimensions,n_odd,p_odd,n_physdesc,p_physdesc,n_physfacet,p_physfacet,n_physloc,p_physloc,n_prefercite,p_prefercite,n_processinfo,p_processinfo,n_relatedmaterial,p_relatedmaterial,n_scopecontent,p_scopecontent,n_separatedmaterial,p_separatedmaterial,n_userestrict,p_userestrict
-Field name ,Resource Identifier,EAD ID (either this or URI is *REQUIRED*),Resource URI (either this or EAD ID is *REQUIRED*),Ref ID ,Title (either or Date is *REQUIRED*),Component Unique Identifier,Hierarchical Relationship *REQUIRED*,Description Level *REQUIRED*,Other Level,Publish?,Restrictions Apply?,Processing Note,Language,Script,Language Note,Publish?,Language (2),Script (2),Language Note (2),Publish?,Date(1) Label,Date(1) Begin,Date(1) end,Date(1) Type  *REQUIRED*,Date(1) expression,Date(1) Certainty,Date(2) Label,Date(2) Begin,Date(2) end,Date(2) Type  *REQUIRED* if specifying a second Date,Date(2) expression,Date(2) Certainty,Extent portion,Extent number *REQUIRED* if specifying Extent,"Extent type: cubic feet, cds, etc.*REQUIRED* if specifying Extent",Container Summary,Physical Details,Dimensions,Extent portion(2),Extent number(2) *REQUIRED* if specifying second Extent,"Extent type(2): cubic feet, cds, etc. *REQUIRED* if specifying second Extent",Container Summary(2),Physical Details(2),Dimensions(2),Container Instance Type,Top Container type,Top Container [indicator],barcode,Child type,Child indicator,Grandchild type,Grandchild indicator,Container Instance Type(2),Top Container type(2),Top Container [indicator](2),barcode(2),Child type(2),Child indicator(2),Grandchild type(2),Grandchild indicator(2),Digital Object Title,URL of Linked-out digital object,URL of thumbnail,Agent (1) Record ID,Agent(1) header string,Agent(1) Role,Agent(1) Relator,Agent(2) Record ID,Agent(2) header string,Agent(2) Role,Agent(2) Relator,Agent(3) Record ID,Agent(3) header string,Agent(3) Role,Agent(3) Relator,Agent(4) Record ID,Agent(4) header string,Agent(4) Role,Agent(4) Relator,Agent(5) Record ID,Agent(5) header string,Agent(5) Role,Agent(5) Relator,Family Agent(1) Record ID,Family Agent(1) header string,Family Agent(1) Role,Family Agent(1) Relator,Family Agent(2) Record ID,Family Agent(2) header string,Family Agent(2) Role,Family Agent(2) Relator,Corporate Agent Record ID(1),Corporate Agent header string(1),Corporate Agent Role(1),CorporateRelator(1),Corporate Agent Record ID(2),Corporate Agent header string(2),Corporate Agent Role(2),CorporateRelator(2),Corporate Agent Record ID(3),Corporate Agent header string(3),Corporate Agent Role(3),CorporateRelator(3),Subject (1) Record ID,Subject (1) Term,Subject (1) Type,Subject (1) Source,Subject (2) Record ID,Subject (2) Term,Subject (2) Type,Subject (2) Source,Abstract,Publish Abstract,Access Restrictions,Publish Access Restrictions,Acquisition Information,Publish Acquisition Information?,Arrangement,Publish Arrangement?,Biography/History,Publish Biographical History?,Custodial History,Publish Custodial History?,Dimensions,Publish Dimensions?,General,Publish General?,Physical Description,Publish Physical Description?,Physical Facet,Publish Physical Facet?,Physical Location,Publish Physical Location?,Preferred Citation,Publish Preferred Citation?,Processing Information,Publish Process Info?,Related Materials,Publish Related materials?,Scope and Contents,Publish Scope and contents?,Separated Materials,Publish Separated Materials?,Use Restrictions,Publish Use Restrictions?
-,,VFIRST01,,,The first series,,1,Series,,TRUE,,,English,Latin,This is a <strong>Language</strong> note,TRUE,Spanish; Castilian,,,,,2010,2020,bulk,,,Copyright,2020,,single,,,,2,Cubic Feet,,,,,2,Reels,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,A Publishable Abstract,TRUE,,,,,,,,,,,,,,,,,,,,,,,,,,,"Reels of audio discussions, and transcripts of same",TRUE,Stuff,FALSE,,
-,,VFIRST01,,, A subseries,,2,Sub-Series,,TRUE,,,,,,,,,,,,2010,2011,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+ArchivesSpace field code (please don't edit this row),collection_id,ead,res_uri,ref_id,title,unit_id,hierarchy,level,other_level,publish,restrictions_flag,processing_note,l_lang,l_langscript,n_langmaterial,p_langmaterial,l_lang_2,l_langscript_2,n_langmaterial_2,p_langmaterial_2,dates_label,begin,end,date_type,expression,date_certainty,dates_label_2,begin_2,end_2,date_type_2,expression_2,date_certainty_2,portion,number,extent_type,container_summary,physical_details,dimensions,portion_2,number_2,extent_type_2,container_summary_2,physical_details_2,dimensions_2,cont_instance_type,type_1,indicator_1,barcode,type_2,indicator_2,type_3,indicator_3,cont_instance_type_2,type_1_2,indicator_1_2,barcode_2,type_2_2,indicator_2_2,type_3_2,indicator_3_2,digital_object_id,digital_object_title,digital_object_publish,digital_object_link,digital_object_link_publish,thumbnail,thumbnail_publish,people_agent_record_id_1,people_agent_header_1,people_agent_role_1,people_agent_relator_1,people_agent_record_id_2,people_agent_header_2,people_agent_role_2,people_agent_relator_2,people_agent_record_id_3,people_agent_header_3,people_agent_role_3,people_agent_relator_3,people_agent_record_id_4,people_agent_header_4,people_agent_role_4,people_agent_relator_4,people_agent_record_id_5,people_agent_header_5,people_agent_role_5,people_agent_relator_5,families_agent_record_id_1,families_agent_header_1,families_agent_role_1,families_agent_relator_1,families_agent_record_id_2,families_agent_header_2,families_agent_role_2,families_agent_relator_2,corporate_entities_agent_record_id_1,corporate_entities_agent_header_1,corporate_entities_agent_role_1,corporate_entities_agent_relator_1,corporate_entities_agent_record_id_2,corporate_entities_agent_header_2,corporate_entities_agent_role_2,corporate_entities_agent_relator_2,corporate_entities_agent_record_id_3,corporate_entities_agent_header_3,corporate_entities_agent_role_3,corporate_entities_agent_relator_3,subject_1_record_id,subject_1_term,subject_1_type,subject_1_source,subject_2_record_id,subject_2_term,subject_2_type,subject_2_source,n_abstract,p_abstract,n_accessrestrict,p_accessrestrict,b_accessrestrict,e_accessrestrict,n_acqinfo,p_acqinfo,n_arrangement,p_arrangement,n_bioghist,p_bioghist,n_custodhist,p_custodhist,n_dimensions,p_dimensions,n_odd,p_odd,n_physdesc,p_physdesc,n_physfacet,p_physfacet,n_physloc,p_physloc,n_prefercite,p_prefercite,n_processinfo,p_processinfo,n_relatedmaterial,p_relatedmaterial,n_scopecontent,p_scopecontent,n_separatedmaterial,p_separatedmaterial,n_userestrict,p_userestrict
+Field name ,Resource Identifier,EAD ID (either this or URI is *REQUIRED*),Resource URI (either this or EAD ID is *REQUIRED*),Ref ID ,Title (either or Date is *REQUIRED*),Component Unique Identifier,Hierarchical Relationship *REQUIRED*,Description Level *REQUIRED*,Other Level,Publish?,Restrictions Apply?,Processing Note,Language,Script,Language Note,Publish?,Language (2),Script (2),Language Note (2),Publish?,Date(1) Label,Date(1) Begin,Date(1) end,Date(1) Type  *REQUIRED*,Date(1) expression,Date(1) Certainty,Date(2) Label,Date(2) Begin,Date(2) end,Date(2) Type  *REQUIRED* if specifying a second Date,Date(2) expression,Date(2) Certainty,Extent portion,Extent number *REQUIRED* if specifying Extent,"Extent type: cubic feet, cds, etc.*REQUIRED* if specifying Extent",Container Summary,Physical Details,Dimensions,Extent portion(2),Extent number(2) *REQUIRED* if specifying second Extent,"Extent type(2): cubic feet, cds, etc. *REQUIRED* if specifying second Extent",Container Summary(2),Physical Details(2),Dimensions(2),Container Instance Type,Top Container type,Top Container [indicator],barcode,Child type,Child indicator,Grandchild type,Grandchild indicator,Container Instance Type(2),Top Container type(2),Top Container [indicator](2),barcode(2),Child type(2),Child indicator(2),Grandchild type(2),Grandchild indicator(2),Digital Object ID,Digital Object Title,Publish Digital Object Record,URL of Linked-out digital object,Publish linked-out digital object?,URL of thumbnail,Publish thumbnail?,Agent (1) Record ID,Agent(1) header string,Agent(1) Role,Agent(1) Relator,Agent(2) Record ID,Agent(2) header string,Agent(2) Role,Agent(2) Relator,Agent(3) Record ID,Agent(3) header string,Agent(3) Role,Agent(3) Relator,Agent(4) Record ID,Agent(4) header string,Agent(4) Role,Agent(4) Relator,Agent(5) Record ID,Agent(5) header string,Agent(5) Role,Agent(5) Relator,Family Agent(1) Record ID,Family Agent(1) header string,Family Agent(1) Role,Family Agent(1) Relator,Family Agent(2) Record ID,Family Agent(2) header string,Family Agent(2) Role,Family Agent(2) Relator,Corporate Agent Record ID(1),Corporate Agent header string(1),Corporate Agent Role(1),CorporateRelator(1),Corporate Agent Record ID(2),Corporate Agent header string(2),Corporate Agent Role(2),CorporateRelator(2),Corporate Agent Record ID(3),Corporate Agent header string(3),Corporate Agent Role(3),CorporateRelator(3),Subject (1) Record ID,Subject (1) Term,Subject (1) Type,Subject (1) Source,Subject (2) Record ID,Subject (2) Term,Subject (2) Type,Subject (2) Source,Abstract,Publish Abstract,Access Restrictions,Publish Access Restrictions,Access Restrictions Begin Date,Access Restrictions End Date,Acquisition Information,Publish Acquisition Information?,Arrangement,Publish Arrangement?,Biography/History,Publish Biographical History?,Custodial History,Publish Custodial History?,Dimensions,Publish Dimensions?,General,Publish General?,Physical Description,Publish Physical Description?,Physical Facet,Publish Physical Facet?,Physical Location,Publish Physical Location?,Preferred Citation,Publish Preferred Citation?,Processing Information,Publish Process Info?,Related Materials,Publish Related materials?,Scope and Contents,Publish Scope and contents?,Separated Materials,Publish Separated Materials?,Use Restrictions,Publish Use Restrictions?
+,,VFIRST01,,,The first series,,1,Series,,TRUE,,,English,Latin,This is a <strong>Language</strong> note,TRUE,Spanish; Castilian,,,,,2010,2020,bulk,,,Copyright,2020,,single,,,,2,Cubic Feet,,,,,2,Reels,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,A Publishable Abstract,TRUE,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"Reels of audio discussions, and transcripts of same",TRUE,Stuff,FALSE,,
+,,VFIRST01,,, A subseries,,2,Sub-Series,,TRUE,,,,,,,,,,,,2010,2011,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,

backend/spec/lib_bulk_import_import_archival_objects_spec.rb

@@ -4,6 +4,7 @@
 
 describe "Import Archival Objects" do
   BULK_FIXTURES_DIR = File.join(File.dirname(__FILE__), "fixtures", "bulk_import")
+  TEMPLATES_DIR = File.join(File.dirname(__FILE__), "../", "../", "templates")
   before(:each) do
     @current_user = User.find(:username => "admin")
     # create the resource
@@ -33,6 +34,38 @@
     id = resource.save
     @resource = Resource.get_or_die(id)
   end
+
+  # we don't want to test fixtures with different keys than the real templates!
+  it "keeps the example fixtures up to date with the templates" do
+    xlsx_fixture = BULK_FIXTURES_DIR + "/bulk_import_VFIRST01_test01.xlsx"
+    xlsx_template = TEMPLATES_DIR + "/bulk_import_template.xlsx"
+    xlsx_fixture = RubyXL::Parser.parse(xlsx_fixture)
+    xlsx_template = RubyXL::Parser.parse(xlsx_template)
+
+    fixture_keys = xlsx_fixture.worksheets[0][3].cells.map { |c| c.value }
+    template_keys = xlsx_template.worksheets[0][3].cells.map { |c| c.value }
+    expect(template_keys - fixture_keys).to be_empty
+    expect(fixture_keys - template_keys).to be_empty
+
+    csv_fixture = BULK_FIXTURES_DIR + "/bulk_import_VFIRST01_test01.csv"
+    csv_template = TEMPLATES_DIR + "/bulk_import_template.csv"
+    csv_fixture = CSV.read(csv_fixture)
+    csv_template = CSV.read(csv_template)
+
+    fixture_keys = csv_fixture.to_a.first
+    template_keys = csv_template.to_a.first
+    expect(template_keys - fixture_keys).to be_empty
+    expect(fixture_keys - template_keys).to be_empty
+  end
+
+  it "doesn't have any duplicate keys in the template" do
+    xlsx_template = TEMPLATES_DIR + "/bulk_import_template.xlsx"
+    xlsx_template = RubyXL::Parser.parse(xlsx_template)
+
+    template_keys = xlsx_template.worksheets[0][3].cells.map { |c| c.value }
+    expect(template_keys.size).to eq template_keys.uniq.size
+  end
+
   it "reads in excel spreadsheet at the resource level" do
     opts = { :repo_id => @resource[:repo_id],
              :rid => @resource[:id],
@@ -52,12 +85,14 @@
     expect(report.rows[1].archival_object_display).to eq("A subseries, 2010 - 2011")
     tree = JSONModel(:resource_tree).find(nil, :resource_id => @resource.id).to_hash
     children = tree["children"]
-    #    aos = []
     children.each_with_index do |child, index|
       expect(child["record_uri"]).to eq(report.rows[index].archival_object_id)
-      #      ao = JSONModel(:archival_object).find_by_uri(child["record_uri"]).to_hash
-      #      aos << ao
     end
+    child = JSONModel(:archival_object).find(children[0]['id'])
+    dig_obj = JSONModel(:digital_object).find(
+      JSONModel(:digital_object).id_for(child.instances[0]['digital_object']['ref'])
+    )
+    expect(dig_obj.publish).to be false
   end
   it "reads in CSV spreadsheet at the resource level" do
     opts = { :repo_id => @resource[:repo_id],