Modify restrictions to include the new repository published flag

backend/app/lib/oai/aspace_oai_repository.rb

@@ -23,7 +23,6 @@ class ArchivesSpaceOAIRepository < OAI::Provider::Model
 
   DELETES_PER_PAGE = 100
 
-  VISIBILITY_RESTRICTIONS = {:publish => 1, :suppressed => 0}
   RESOLVE = ['repository', 'subjects', 'linked_agents', 'digital_object', 'top_container', 'ancestors', 'linked_agents', 'resource']
 
 
@@ -82,7 +81,7 @@ def fetch_single_record(uri, options = {})
     repo_id = JSONModel.parse_reference(parsed_ref.fetch(:repository)).fetch(:id)
 
     RequestContext.open(:repo_id => repo_id) do
-      obj = model.filter(:id => parsed_ref[:id]).filter(VISIBILITY_RESTRICTIONS).first
+      obj = add_visibility_restrictions(model.filter(:id => parsed_ref[:id])).first
 
       raise IdException.new unless obj
 
@@ -120,6 +119,25 @@ def find(selector, options = {})
 
   private
 
+  def add_visibility_restrictions(dataset)
+    unpublished_repos = Repository.exclude(:publish => 1).select(:id).map {|row| row[:id]}
+
+    dataset.exclude(:repo_id => unpublished_repos).filter(:publish => 1, :suppressed => 0)
+  end
+
+  # Don't show deletes for repositories that aren't published.
+  def restrict_tombstones_to_published_repositories(dataset)
+    unpublished_repos = Repository.exclude(:publish => 1).select(:id).map {|row| row[:id]}
+
+    result = dataset
+
+    unpublished_repos.each do |repo_id|
+      result = result.exclude(Sequel.like(:uri, JSONModel(:repository).uri_for(repo_id) + '/%'))
+    end
+
+    result
+  end
+
   def options_for_type(metadata_prefix)
     AVAILABLE_RECORD_TYPES.fetch(metadata_prefix)
   end
@@ -155,9 +173,7 @@ def produce_next_record_set(resumption_token, options)
 
       # Request one extra record (limit + 1) to determine whether we've hit
       # the end of the stream or not
-      matches = record_type
-                  .any_repo
-                  .filter(VISIBILITY_RESTRICTIONS)
+      matches = add_visibility_restrictions(record_type.any_repo)
                   .where { id > last_id }
                   .order(:id)
                   .limit(limit + 1)
@@ -203,7 +219,10 @@ def produce_next_delete_set(resumption_token, options)
     # Get a dataset that will pull back all tombstones for the record types that
     # this metadata type supports.
     matching_tombstones = format_options.record_types.map {|record_type|
-      DELETE_LOOKUPS[record_type]
+      tombstone_ds = DELETE_LOOKUPS[record_type]
+      if tombstone_ds
+        restrict_tombstones_to_published_repositories(tombstone_ds)
+      end
     }.compact.reduce {|deletes, tombstone_ds|
       deletes.union(tombstone_ds)
     }

backend/spec/fixtures/oai/responses/getrecord_oai_dc.xml

@@ -1,7 +1,7 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:34Z</responseDate>
-  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_dc" verb="GetRecord">http://to-be-determined.example.com</request>
+  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_dc" verb="GetRecord">http://your-public-oai-url.example.com</request>
   <GetRecord>
     <record>
       <header>

backend/spec/fixtures/oai/responses/getrecord_oai_dcterms.xml

@@ -1,7 +1,7 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:34Z</responseDate>
-  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_dcterms" verb="GetRecord">http://to-be-determined.example.com</request>
+  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_dcterms" verb="GetRecord">http://your-public-oai-url.example.com</request>
   <GetRecord>
     <record>
       <header>

backend/spec/fixtures/oai/responses/getrecord_oai_ead.xml

@@ -1,7 +1,7 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:33Z</responseDate>
-  <request identifier="oai:archivesspace//repositories/3/resources/1" metadataPrefix="oai_ead" verb="GetRecord">http://to-be-determined.example.com</request>
+  <request identifier="oai:archivesspace//repositories/3/resources/1" metadataPrefix="oai_ead" verb="GetRecord">http://your-public-oai-url.example.com</request>
   <GetRecord>
     <record>
       <header>

backend/spec/fixtures/oai/responses/getrecord_oai_marc.xml

@@ -1,7 +1,7 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:32Z</responseDate>
-  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_marc" verb="GetRecord">http://to-be-determined.example.com</request>
+  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_marc" verb="GetRecord">http://your-public-oai-url.example.com</request>
   <GetRecord>
     <record>
       <header>

backend/spec/fixtures/oai/responses/getrecord_oai_mods.xml

@@ -1,7 +1,7 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:34Z</responseDate>
-  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_mods" verb="GetRecord">http://to-be-determined.example.com</request>
+  <request identifier="oai:archivesspace//repositories/3/archival_objects/1" metadataPrefix="oai_mods" verb="GetRecord">http://your-public-oai-url.example.com</request>
   <GetRecord>
     <record>
       <header>

backend/spec/fixtures/oai/responses/identify.xml

@@ -1,10 +1,10 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:34Z</responseDate>
-  <request verb="Identify">http://to-be-determined.example.com</request>
+  <request verb="Identify">http://your-public-oai-url.example.com</request>
   <Identify>
     <repositoryName>ArchivesSpace OAI Provider</repositoryName>
-    <baseURL>http://to-be-determined.example.com</baseURL>
+    <baseURL>http://your-public-oai-url.example.com</baseURL>
     <protocolVersion>2.0</protocolVersion>
     <adminEmail>admin@example.com</adminEmail>
     <earliestDatestamp>1970-01-01T00:00:00Z</earliestDatestamp>

backend/spec/fixtures/oai/responses/list_metadata_formats.xml

@@ -1,7 +1,7 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
   <responseDate>2017-06-06T03:23:33Z</responseDate>
-  <request verb="ListMetadataFormats">http://to-be-determined.example.com</request>
+  <request verb="ListMetadataFormats">http://your-public-oai-url.example.com</request>
   <ListMetadataFormats>
     <metadataFormat>
       <metadataPrefix>oai_dc</metadataPrefix>

backend/spec/fixtures/oai/responses/list_sets.xml

@@ -1,26 +1,8 @@
 <!-- THIS FILE IS AUTOMATICALLY GENERATED - DO NOT EDIT.  To update it, just delete it and it will be regenerated from your test data. -->
 <OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd">
-  <responseDate>2017-06-06T03:23:32Z</responseDate>
-  <request verb="ListSets">http://to-be-determined.example.com</request>
+  <responseDate>2017-06-18T23:31:20Z</responseDate>
+  <request verb="ListSets">http://your-public-oai-url.example.com</request>
   <ListSets>
-    <set>
-      <setSpec>repository_set</setSpec>
-      <setName>repository_set</setName>
-      <setDescription>
-        <oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
-          <oai_dc:description>A set of one or more repositories</oai_dc:description>
-        </oai_dc:dc>
-      </setDescription>
-    </set>
-    <set>
-      <setSpec>sponsor_set</setSpec>
-      <setName>sponsor_set</setName>
-      <setDescription>
-        <oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
-          <oai_dc:description>A set of one or more sponsors</oai_dc:description>
-        </oai_dc:dc>
-      </setDescription>
-    </set>
     <set>
       <setSpec>class</setSpec>
       <setName>class</setName>

backend/spec/model_oai_spec.rb

@@ -295,18 +295,25 @@ def list_identifiers(prefix)
 
 
     it "supports OAI sets based on sponsors" do
+      allow(AppConfig).to receive(:has_key?).with(any_args).and_call_original
+      allow(AppConfig).to receive(:has_key?).with(:oai_sets).and_return(true)
+
       allow(AppConfig).to receive(:[]).with(any_args).and_call_original
       allow(AppConfig).to receive(:[]).with(:oai_sets)
                             .and_return('sponsor_0' => {
                                           :sponsors => ['sponsor_0']
                                         })
 
       response = oai_repo.find(:all, {:metadata_prefix => "oai_dc", :set => 'sponsor_0'})
+
       response.records.all? {|record| record.jsonmodel_record.resource['ref'] == @test_resource_record}
         .should be(true)
     end
 
     it "supports OAI sets based on repositories" do
+      allow(AppConfig).to receive(:has_key?).with(any_args).and_call_original
+      allow(AppConfig).to receive(:has_key?).with(:oai_sets).and_return(true)
+
       allow(AppConfig).to receive(:[]).with(any_args).and_call_original
       allow(AppConfig).to receive(:[]).with(:oai_sets)
                             .and_return('by_repo' => {

backend/spec/oai_response_checker.rb

@@ -46,7 +46,11 @@ def compare_or_fail(ours, theirs, path = [])
       end
 
       unless ours.children.length == theirs.children.length
-        raise MismatchError.new("Child count mismatch", ours.children.length, theirs.children.length, new_path)
+        our_xml = ours.to_xml
+        their_xml = theirs.to_xml
+
+        raise MismatchError.new("Child count mismatch", ours.children.length, theirs.children.length, new_path,
+                               "Ours: #{our_xml}\n\nTheirs: #{their_xml}")
       end
 
       # Compare all children
@@ -103,9 +107,14 @@ def skip_element?(element)
   end
 
   class MismatchError < StandardError
-    def initialize(description, expected, got, path)
+    def initialize(description, expected, got, path, additional_information = nil)
       msg = "#{description}:\nexpected '#{expected}'\ngot: '#{got}'\nat: #{path.join('/')}"
 
+      if additional_information
+        msg += "\n"
+        msg += additional_information
+      end
+
       $stderr.puts(msg)
       super(msg)
     end