Fixity dashboard (#890)

* Generate dashboard controller

* Add query to return lazy enumerator for most recently updated filesets

* Update least_recently_updpated_file_sets query to access all file sets

* Add custom query for fixity failures

* Add controller for fixity dashboard

* Refactor: combine custom queries for file sets sorted by updated_at

* Move reports controller into dashboards controller

.rubocop.yml

@@ -120,5 +120,6 @@ RSpec/AnyInstance:
     - 'spec/services/manifest_builder_spec.rb'
     - 'spec/services/messaging_client_spec.rb'
     - 'spec/jobs/check_fixity_recursive_job_spec.rb'
+    - 'spec/controllers/dashboard_controller_spec.rb'
 Lint/UnusedMethodArgument:
   AllowUnusedKeywordArguments: true

app/controllers/reports_controller.rb → app/controllers/dashboard_controller.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
-class ReportsController < ApplicationController
+class DashboardController < ApplicationController
+  delegate :query_service, to: :metadata_adapter
+
   def identifiers_to_reconcile
     authorize! :show, Report
     @resources = Valkyrie.config.metadata_adapter.query_service.custom_queries.find_identifiers_to_reconcile
@@ -12,6 +14,12 @@ def identifiers_to_reconcile
     end
   end
 
+  def fixity
+    @failures = query_service.custom_queries.find_fixity_failures
+    @recents = query_service.custom_queries.file_sets_sorted_by_updated(sort: 'desc', limit: 10)
+    @upcoming = query_service.custom_queries.file_sets_sorted_by_updated(limit: 20)
+  end
+
   private
 
     def to_csv(records, fields:)
@@ -22,4 +30,8 @@ def to_csv(records, fields:)
         end
       end
     end
+
+    def metadata_adapter
+      Valkyrie::MetadataAdapter.find(:indexing_persister)
+    end
 end

app/jobs/check_fixity_recursive_job.rb

@@ -3,7 +3,7 @@ class CheckFixityRecursiveJob < ApplicationJob
   delegate :query_service, to: :metadata_adapter
 
   def perform
-    file_set = query_service.custom_queries.least_recently_updated_file_set
+    file_set = query_service.custom_queries.file_sets_sorted_by_updated(limit: 1).first
     CheckFixityJob.perform_now(file_set.id)
     CheckFixityRecursiveJob.set(queue: queue_name).perform_later
   end

app/queries/least_recently_updated_file_set.rb → app/queries/file_sets_sorted_by_updated.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
-class LeastRecentlyUpdatedFileSet
+class FileSetsSortedByUpdated
   def self.queries
-    [:least_recently_updated_file_set]
+    [:file_sets_sorted_by_updated]
   end
 
   attr_reader :query_service
@@ -11,19 +11,25 @@ def initialize(query_service:)
     @query_service = query_service
   end
 
-  def least_recently_updated_file_set
-    run_query(query).first
+  def file_sets_sorted_by_updated(sort: 'asc', limit: 50)
+    run_query("#{query} #{order(sort)} #{number(limit)}")
   end
 
   def query
     <<-SQL
       select * FROM orm_resources WHERE
       internal_resource='FileSet'
-      order by updated_at
-      limit 1
     SQL
   end
 
+  def order(sort)
+    "order by updated_at #{sort}"
+  end
+
+  def number(limit)
+    "limit #{limit}"
+  end
+
   def run_query(query)
     orm_class.find_by_sql([query]).lazy.map do |object|
       resource_factory.to_resource(object: object)

app/queries/find_fixity_failures.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+class FindFixityFailures
+  def self.queries
+    [:find_fixity_failures]
+  end
+
+  attr_reader :query_service
+  delegate :resource_factory, to: :query_service
+  delegate :orm_class, to: :resource_factory
+  def initialize(query_service:)
+    @query_service = query_service
+  end
+
+  def find_fixity_failures
+    internal_array = { file_metadata: [{ fixity_success: 0 }] }.to_json
+    run_query(query, internal_array)
+  end
+
+  def query
+    <<-SQL
+      select * FROM orm_resources WHERE
+      metadata @> ?
+    SQL
+  end
+
+  def run_query(query, *args)
+    orm_class.find_by_sql(([query] + args)).lazy.map do |object|
+      resource_factory.to_resource(object: object)
+    end
+  end
+end

app/views/dashboard/fixity.html.erb

@@ -0,0 +1,2 @@
+<h1>Dashboard#fixity</h1>
+<p>Find me in app/views/dashboard/fixity.html.erb</p>

config/initializers/valkyrie.rb

@@ -164,7 +164,8 @@
     MemoryEfficientAllQuery,
     FindProjectFolders,
     FindIdentifiersToReconcile,
-    LeastRecentlyUpdatedFileSet
+    FileSetsSortedByUpdated,
+    FindFixityFailures
   ].each do |query_handler|
     Valkyrie.config.metadata_adapter.query_service.custom_queries.register_query_handler(query_handler)
   end

config/routes.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 Rails.application.routes.draw do
+  get 'dashboard/fixity'
+
   resources :auth_tokens
   default_url_options Rails.application.config.action_mailer.default_url_options
   concern :exportable, Blacklight::Routes::Exportable.new
@@ -168,7 +170,7 @@
   get '/catalog/parent/:parent_id/:id', to: 'catalog#show', as: :parent_solr_document
   get "/iiif/lookup/:prefix/:naan/:arkid", to: 'catalog#lookup_manifest', as: :lookup_manifest
 
-  get "/reports/identifiers_to_reconcile", to: "reports#identifiers_to_reconcile", as: :identifiers_to_reconcile
+  get "/reports/identifiers_to_reconcile", to: "dashboard#identifiers_to_reconcile", as: :identifiers_to_reconcile
 
   mount BrowseEverything::Engine => '/browse'
   mount Valhalla::Engine => '/'

spec/controllers/dashboard_controller_spec.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+require 'rails_helper'
+
+RSpec.describe DashboardController, type: :controller do
+  describe "fixity dashboard" do
+    let(:adapter) { Valkyrie::MetadataAdapter.find(:indexing_persister) }
+    let(:storage_adapter) { Valkyrie.config.storage_adapter }
+    let(:query_service) { adapter.query_service }
+
+    let(:resource) { FactoryBot.create_for_repository(:scanned_resource) }
+    let(:resource2) { FactoryBot.create_for_repository(:scanned_resource) }
+    let(:resource3) { FactoryBot.create_for_repository(:scanned_resource) }
+
+    let(:file_metadata) { FileMetadata.new(fixity_success: 0) }
+
+    let(:file) { fixture_file_upload('files/example.tif', 'image/tiff') }
+    let(:change_set_persister) { PlumChangeSetPersister.new(metadata_adapter: adapter, storage_adapter: storage_adapter) }
+    let(:change_set) { ScannedResourceChangeSet.new(resource) }
+    let(:output) do
+      change_set.files = [file]
+      change_set_persister.save(change_set: change_set)
+    end
+
+    before do
+      # they have to be saved to come out in the query
+      file_set = query_service.find_members(resource: output).first
+      file_set_change_set = FileSetChangeSet.new(file_set)
+      file_set.file_metadata = file_metadata
+      change_set_persister.save(change_set: file_set_change_set)
+    end
+
+    it "returns http success" do
+      get :fixity
+      expect(response).to have_http_status(:success)
+    end
+
+    it "sets fixity failures variable" do
+      get :fixity
+      expect(assigns[:failures].size).to eq 1
+    end
+
+    context "most-recently-upated filesets" do
+      before do
+        allow_any_instance_of(FileSetsSortedByUpdated).to receive(:file_sets_sorted_by_updated).and_return [resource, resource2, resource3]
+      end
+      it "sets recents variable" do
+        get :fixity
+        expect(assigns[:recents].size).to eq 3
+      end
+    end
+
+    context "least-recently-upated filesets" do
+      before do
+        allow_any_instance_of(FileSetsSortedByUpdated).to receive(:file_sets_sorted_by_updated).and_return [resource3, resource2, resource]
+      end
+      it "sets upcoming variable" do
+        get :fixity
+        expect(assigns[:upcoming].size).to eq 3
+      end
+    end
+  end
+
+  describe 'reports dashboard' do
+    let(:user) { FactoryBot.create(:admin) }
+    let(:resource) { FactoryBot.build(:scanned_resource, title: []) }
+    let(:resource2) { FactoryBot.create_for_repository(:scanned_resource, title: []) }
+    let(:change_set_persister) { PlumChangeSetPersister.new(metadata_adapter: Valkyrie.config.metadata_adapter, storage_adapter: Valkyrie.config.storage_adapter) }
+    let(:data) { "bibid,ark,title\n123456,ark:/99999/fk48675309,Earth rites : fertility rites in pre-industrial Britain\n" }
+
+    before do
+      sign_in user
+      stub_bibdata(bib_id: '123456')
+      stub_ezid(shoulder: "99999/fk4", blade: "8675309")
+
+      change_set = ScannedResourceChangeSet.new(resource)
+      change_set.validate(source_metadata_identifier: '123456')
+      change_set.sync
+      change_set_persister.save(change_set: change_set)
+    end
+
+    describe "GET #identifiers_to_reconcile" do
+      it "displays a html view" do
+        get :identifiers_to_reconcile
+        expect(response).to render_template :identifiers_to_reconcile
+      end
+      it "allows downloading a CSV file" do
+        get :identifiers_to_reconcile, format: 'csv'
+        expect(response.body).to eq(data)
+      end
+    end
+  end
+end

spec/queries/file_sets_sorted_by_updated_spec.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+require 'rails_helper'
+include ActionDispatch::TestProcess
+
+RSpec.describe FileSetsSortedByUpdated do
+  let(:file) { fixture_file_upload('files/example.tif', 'image/tiff') }
+  let(:file2) { fixture_file_upload('files/example.tif', 'image/tiff') }
+  let(:file3) { fixture_file_upload('files/example.tif', 'image/tiff') }
+  let(:resource) { FactoryBot.build(:scanned_resource) }
+  let(:resource2) { FactoryBot.build(:scanned_resource) }
+  let(:resource3) { FactoryBot.build(:scanned_resource) }
+  let(:adapter) { Valkyrie::MetadataAdapter.find(:indexing_persister) }
+  let(:storage_adapter) { Valkyrie.config.storage_adapter }
+  let(:query_service) { adapter.query_service }
+  let(:change_set_persister) { PlumChangeSetPersister.new(metadata_adapter: adapter, storage_adapter: storage_adapter) }
+  let(:change_set) { ScannedResourceChangeSet.new(resource) }
+  let(:change_set2) { ScannedResourceChangeSet.new(resource2) }
+  let(:change_set3) { ScannedResourceChangeSet.new(resource3) }
+  let(:output) do
+    change_set.files = [file]
+    change_set_persister.save(change_set: change_set)
+  end
+  let(:output2) do
+    change_set2.files = [file2]
+    change_set_persister.save(change_set: change_set2)
+  end
+  let(:output3) do
+    change_set3.files = [file3]
+    change_set_persister.save(change_set: change_set3)
+  end
+  before do
+    # they have to be saved to come out in the query
+    Timecop.freeze(Time.now.utc - 10.minutes) do
+      query_service.find_members(resource: output).first
+    end
+    Timecop.freeze(Time.now.utc - 5.minutes) do
+      query_service.find_members(resource: output2).first
+    end
+    query_service.find_members(resource: output3).first
+  end
+
+  describe "#file_sets_sorted_by_updated" do
+    it "finds least recently updated file sets with the given limit" do
+      result = query_service.custom_queries.file_sets_sorted_by_updated
+      expect(result.count).to eq 3
+      expect(result.next.id.to_s).to eq query_service.find_members(resource: output).first.id.to_s
+      expect(result.next.id.to_s).to eq query_service.find_members(resource: output2).first.id.to_s
+      expect(result.next.id.to_s).to eq query_service.find_members(resource: output3).first.id.to_s
+    end
+
+    it "returns a lazy enumerator" do
+      result = query_service.custom_queries.file_sets_sorted_by_updated
+      expect(result).to be_a Enumerator::Lazy
+    end
+  end
+
+  it "finds most recently updated file sets with the given limit" do
+    result = query_service.custom_queries.file_sets_sorted_by_updated(sort: 'desc', limit: 2)
+    expect(result.count).to eq 2
+    expect(result.next.id.to_s).to eq query_service.find_members(resource: output3).first.id.to_s
+    expect(result.next.id.to_s).to eq query_service.find_members(resource: output2).first.id.to_s
+  end
+end

spec/queries/find_fixity_failures_spec.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+require 'rails_helper'
+
+RSpec.describe FindFixityFailures do
+  subject(:query) { described_class.new(query_service: query_service) }
+  let(:file_metadata) { FileMetadata.new(fixity_success: 0) }
+  let(:file_set) { FactoryBot.create_for_repository(:file_set, file_metadata: file_metadata) }
+  let(:query_service) { Valkyrie.config.metadata_adapter.query_service }
+
+  describe "#find_fixity_failures" do
+    it "can find file_sets with metadata fixity_success == 0" do
+      file_set
+      output = query.find_fixity_failures.first
+      expect(output.id).to eq file_set.id
+    end
+  end
+end