From 4cf2a8aaa95866cec1439031c1a71e2c2304a182 Mon Sep 17 00:00:00 2001 From: Adam Jazairi Date: Fri, 23 Dec 2022 17:25:39 -0500 Subject: [PATCH] Add batch export feature for ProQuest JSON Why these changes are being introduced: Processors and thesis admins need to be able to export a list of thesis handles that have opted in to ProQuest. An additional export is needed for doctoral theses that have opted out of ProQuest (or have not responded). No additional filters are required for these exports. Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/ETD-588 * https://mitlibraries.atlassian.net/browse/ETD-603 How this addresses that need: This adds a view that allows processors to review theses that are ready to be exported to ProQuest. Two types of theses meet this criterion: * Published theses satisfying an advanced degree, with all authors having consented to ProQuest export. ProQuest will fully harvest these theses (metadata and content). * Published theses satisfying a doctoral degree, without all authors having consented to ProQuest export. ProQuest wil partially harvest these theses (metadata only). Once a processor has reviewwd the list of theses to be exported, they can initiate the export. This will kick off a background job that will produce a JSON file that lists each thesis' handle and distinguishes between full and partial exports. Once the export is ready, the job send an email to the ETD admin list with the JSON attached. Finally, all exported theses will be associated with a ProquestExportBatch, and the JSON file will be attached to the same as an ActiveStorage attachment. Side effects of this change: * ProquestExportJob currently loops through theses to update them using the `each` method. This is an expensive operation. A more efficient option is update_all, which updates attributes without instantiating the objects. However, this would not trigger callbacks, so we would need to find another way to get paper_trail to update, such as this hack: https://github.com/paper-trail-gem/paper_trail/issues/456#issuecomment-1085583174 * This adds a has_many/belongs_to relationship between the ProquestExportBatch and Thesis models. This may not be necessary since we are storing the JSON exports in ActiveStorage, but it could be useful to look up the date a thesis was exported and find other theses from the same export. * The proquest_exported field has been added to the thesis admin dashboard. * The _proquest_status_empty partial is moved to views/shared as it is now used by the export preview and the export report. * Some unrelated tests in the Report, Thesis, and User models are updated due to fixture changes. Where applicable, I tried to update these tests to use less brittle logic so they won't fail again under similar circumstances. --- app/controllers/thesis_controller.rb | 16 ++ app/dashboards/thesis_dashboard.rb | 4 +- app/jobs/proquest_export_job.rb | 32 ++++ app/mailers/batch_mailer.rb | 14 ++ app/models/ability.rb | 4 +- app/models/proquest_export_batch.rb | 24 +++ app/models/thesis.rb | 18 +++ .../proquest_export_email.html.erb | 4 + app/views/layouts/_site_nav.html.erb | 3 + .../report/_proquest_status_empty.html.erb | 3 - .../report/_proquest_status_thesis.html.erb | 11 ++ app/views/report/_proquest_thesis.html.erb | 10 -- app/views/report/proquest_status.html.erb | 7 +- .../shared/_proquest_thesis_empty.html.erb | 3 + .../thesis/_proquest_export_thesis.html.erb | 11 ++ .../thesis/proquest_export_preview.html.erb | 56 +++++++ config/routes.rb | 2 + ...1212143_add_proquest_exported_to_thesis.rb | 5 + ...09161518_create_proquest_export_batches.rb | 10 ++ db/schema.rb | 10 +- test/controllers/report_controller_test.rb | 29 ++-- test/controllers/thesis_controller_test.rb | 118 +++++++++++++++ test/fixtures/authors.yml | 22 ++- test/fixtures/theses.yml | 28 ++++ test/helpers/thesis_helper_test.rb | 6 +- test/integration/thesis_test.rb | 2 +- test/jobs/proquest_export_job_test.rb | 37 +++++ test/mailers/batch_mailer_test.rb | 25 ++++ test/models/proquest_export_batch_test.rb | 15 ++ test/models/report_test.rb | 14 +- test/models/thesis_test.rb | 139 +++++++++++++++++- test/models/user_test.rb | 3 +- 32 files changed, 640 insertions(+), 45 deletions(-) create mode 100644 app/jobs/proquest_export_job.rb create mode 100644 app/models/proquest_export_batch.rb create mode 100644 app/views/batch_mailer/proquest_export_email.html.erb delete mode 100644 app/views/report/_proquest_status_empty.html.erb create mode 100644 app/views/report/_proquest_status_thesis.html.erb delete mode 100644 app/views/report/_proquest_thesis.html.erb create mode 100644 app/views/shared/_proquest_thesis_empty.html.erb create mode 100644 app/views/thesis/_proquest_export_thesis.html.erb create mode 100644 app/views/thesis/proquest_export_preview.html.erb create mode 100644 db/migrate/20221101212143_add_proquest_exported_to_thesis.rb create mode 100644 db/migrate/20221209161518_create_proquest_export_batches.rb create mode 100644 test/jobs/proquest_export_job_test.rb create mode 100644 test/models/proquest_export_batch_test.rb diff --git a/app/controllers/thesis_controller.rb b/app/controllers/thesis_controller.rb index 584c17be..11b5cc0d 100644 --- a/app/controllers/thesis_controller.rb +++ b/app/controllers/thesis_controller.rb @@ -134,6 +134,22 @@ def process_theses_update redirect_to thesis_process_path end + def proquest_export_preview + @theses = Thesis.ready_for_proquest_export + end + + # TODO: we need to generate and send a budget report CSV for partially harvested theses (spec TBD). + def proquest_export + if Thesis.ready_for_proquest_export.any? + ProquestExportJob.perform_later(Thesis.partial_proquest_export, Thesis.full_proquest_export) + flash[:success] = 'The theses you selected will be exported. ' \ + 'Status updates are not immediate.' + else + flash[:warning] = 'No theses are available to export.' + end + redirect_to thesis_proquest_export_preview_path + end + private # Various methods need to build an array of academic terms which meet varying conditions, in order to support a UI diff --git a/app/dashboards/thesis_dashboard.rb b/app/dashboards/thesis_dashboard.rb index b990d31b..2e877809 100644 --- a/app/dashboards/thesis_dashboard.rb +++ b/app/dashboards/thesis_dashboard.rb @@ -40,7 +40,8 @@ class ThesisDashboard < Administrate::BaseDashboard coauthors: Field::String, copyright: Field::BelongsTo, license: Field::BelongsTo, - dspace_handle: Field::String + dspace_handle: Field::String, + proquest_exported: Field::String }.freeze # COLLECTION_ATTRIBUTES @@ -80,6 +81,7 @@ class ThesisDashboard < Administrate::BaseDashboard metadata_complete files_complete files + proquest_exported ].freeze # FORM_ATTRIBUTES diff --git a/app/jobs/proquest_export_job.rb b/app/jobs/proquest_export_job.rb new file mode 100644 index 00000000..b60e1442 --- /dev/null +++ b/app/jobs/proquest_export_job.rb @@ -0,0 +1,32 @@ +class ProquestExportJob < ActiveJob::Base + queue_as :default + + def perform(partial_harvest, full_harvest) + export = ProquestExportBatch.new + + # update_all is an option here if performance is poor, but we would need a workaround to update paper_trail + # versions as update_all does not trigger callbacks. + if partial_harvest.present? + partial_harvest.each do |thesis| + thesis.update(proquest_exported: 'Partial harvest', proquest_export_batch: export) + end + end + if full_harvest.present? + full_harvest.each { |thesis| thesis.update(proquest_exported: 'Full harvest', proquest_export_batch: export) } + end + + all_to_export = partial_harvest + full_harvest + attach_and_send(export, all_to_export) + end + + private + + def attach_and_send(export, all_to_export) + export_json = export.build_json(all_to_export) + export.proquest_export.attach(io: StringIO.new(export_json), + filename: "proquest_export_#{Date.today.strftime('%Y%m%d_%s')}.json", + content_type: 'application/json') + export.save + BatchMailer.proquest_export_email(export.proquest_export.blob, all_to_export.count).deliver_later + end +end diff --git a/app/mailers/batch_mailer.rb b/app/mailers/batch_mailer.rb index 4c107561..f8fe8058 100644 --- a/app/mailers/batch_mailer.rb +++ b/app/mailers/batch_mailer.rb @@ -9,4 +9,18 @@ def marc_batch_email(marc_zip_filename, marc_zip_file, theses) cc: ENV['MAINTAINER_EMAIL'], subject: 'ETD MARC batch export') end + + def proquest_export_email(json_blob, thesis_count) + return unless ENV.fetch('DISABLE_ALL_EMAIL', 'true') == 'false' # allows PR builds to disable emails + + @thesis_count = thesis_count + attachments[json_blob.filename.to_s] = { + mime_type: json_blob.content_type, + content: json_blob.download + } + mail(from: "MIT Libraries <#{ENV['THESIS_ADMIN_EMAIL']}>", + to: ENV['THESIS_ADMIN_EMAIL'], + cc: ENV['MAINTAINER_EMAIL'], + subject: 'ETD ProQuest export') + end end diff --git a/app/models/ability.rb b/app/models/ability.rb index e37f5507..dfeb1f2a 100644 --- a/app/models/ability.rb +++ b/app/models/ability.rb @@ -80,10 +80,12 @@ def processor can :mark_withdrawn, Thesis can :process_theses, Thesis can :process_theses_update, Thesis + can :publication_statuses, Thesis can :publish_preview, Thesis can :publish_to_dspace, Thesis + can :proquest_export, Thesis + can :proquest_export_preview, Thesis can :select, Thesis - can :publication_statuses, Thesis can :read, Transfer can :select, Transfer diff --git a/app/models/proquest_export_batch.rb b/app/models/proquest_export_batch.rb new file mode 100644 index 00000000..4a647e00 --- /dev/null +++ b/app/models/proquest_export_batch.rb @@ -0,0 +1,24 @@ +class ProquestExportBatch < ApplicationRecord + has_many :theses + + has_one_attached :proquest_export + has_one_attached :budget_report + + def build_json(theses) + thesis_records = theses.map { |thesis| record(thesis) } + { records: thesis_records }.to_json + end + + private + + def record(thesis) + { + dspace_handle: thesis.dspace_handle, + full_harvest: evaluate_export_type(thesis) + } + end + + def evaluate_export_type(thesis) + thesis.proquest_exported == 'Full harvest' + end +end diff --git a/app/models/thesis.rb b/app/models/thesis.rb index 69b9d553..c4158d4b 100644 --- a/app/models/thesis.rb +++ b/app/models/thesis.rb @@ -25,6 +25,7 @@ class Thesis < ApplicationRecord belongs_to :copyright, optional: true belongs_to :license, optional: true + belongs_to :proquest_export_batch, optional: true has_many :degree_theses has_many :degrees, through: :degree_theses @@ -112,6 +113,23 @@ class Thesis < ApplicationRecord scope :publication_statuses, -> { PUBLICATION_STATUS_OPTIONS } scope :multiple_authors, -> { where('authors_count > ?', 1) } scope :advanced_degree, -> { includes(degrees: :degree_type).distinct.where.not(degree_type: { name: 'Bachelor' }) } + scope :doctoral, -> { includes(degrees: :degree_type).distinct.where(degree_type: { name: 'Doctoral' }) } + scope :consented_to_proquest, lambda { + includes(authors: :user).where(authors: { proquest_allowed: true }) + .excluding(includes(authors: :user) + .where(authors: { proquest_allowed: false })) + } + scope :not_consented_to_proquest, lambda { + excluding(includes(authors: :user).where(authors: { proquest_allowed: true })) + } + scope :exported_to_proquest, -> { where(proquest_exported: ['Partial harvest', 'Full harvest']) } + scope :not_exported_to_proquest, -> { where(proquest_exported: 'Not exported') } + scope :published, -> { where(publication_status: 'Published') } + scope :partial_proquest_export, -> { published.doctoral.not_exported_to_proquest.not_consented_to_proquest } + scope :full_proquest_export, -> { published.advanced_degree.not_exported_to_proquest.consented_to_proquest } + scope :ready_for_proquest_export, -> { partial_proquest_export + full_proquest_export } + + enum proquest_exported: ['Not exported', 'Full harvest', 'Partial harvest'] # Returns a true/false value (rendered as "yes" or "no") if there are any # holds with a status of either 'active' or 'expired'. A false/"No" is diff --git a/app/views/batch_mailer/proquest_export_email.html.erb b/app/views/batch_mailer/proquest_export_email.html.erb new file mode 100644 index 00000000..044c0ea1 --- /dev/null +++ b/app/views/batch_mailer/proquest_export_email.html.erb @@ -0,0 +1,4 @@ +

Hello,

+ +

Attached is a ProQuest export of <%= @thesis_count %> theses generated on +<%= Date.current.strftime('%A, %B %d, %Y') %> at <%= Time.now.strftime('%r %Z') %>. diff --git a/app/views/layouts/_site_nav.html.erb b/app/views/layouts/_site_nav.html.erb index 382a42d1..d040bd5b 100644 --- a/app/views/layouts/_site_nav.html.erb +++ b/app/views/layouts/_site_nav.html.erb @@ -25,6 +25,9 @@ <% if can? :list_registrar, Registrar %> <%= nav_link_to("Harvest CSV", harvest_path) %> <% end %> + <% if can? :proquest_export, Thesis %> + <%= nav_link_to("Export for ProQuest", thesis_proquest_export_preview_path) %> + <% end %> <% if can? :index, Report %> <%= nav_link_to("Report", report_index_path) %> <% end %> diff --git a/app/views/report/_proquest_status_empty.html.erb b/app/views/report/_proquest_status_empty.html.erb deleted file mode 100644 index 78347582..00000000 --- a/app/views/report/_proquest_status_empty.html.erb +++ /dev/null @@ -1,3 +0,0 @@ - - No theses match these criteria. - diff --git a/app/views/report/_proquest_status_thesis.html.erb b/app/views/report/_proquest_status_thesis.html.erb new file mode 100644 index 00000000..173f0876 --- /dev/null +++ b/app/views/report/_proquest_status_thesis.html.erb @@ -0,0 +1,11 @@ + + <%= link_to title_helper(proquest_status_thesis), edit_admin_thesis_path(proquest_status_thesis.id) %> + + <% proquest_status_thesis.authors.each do |author| %> + <%= link_to author.user.display_name, edit_admin_author_path(author.id) %>
+ <% end %> + + <%= proquest_status_thesis.dspace_handle %> + <%= render_proquest_status(proquest_status_thesis) %> + <%= proquest_status_thesis.proquest_exported %> + diff --git a/app/views/report/_proquest_thesis.html.erb b/app/views/report/_proquest_thesis.html.erb deleted file mode 100644 index 0c4bc3bc..00000000 --- a/app/views/report/_proquest_thesis.html.erb +++ /dev/null @@ -1,10 +0,0 @@ - - <%= link_to title_helper(proquest_thesis), edit_admin_thesis_path(proquest_thesis.id) %> - - <% proquest_thesis.authors.each do |author| %> - <%= link_to author.user.display_name, edit_admin_author_path(author.id) %>
- <% end %> - - <%= proquest_thesis.dspace_handle %> - <%= render_proquest_status(proquest_thesis) %> - diff --git a/app/views/report/proquest_status.html.erb b/app/views/report/proquest_status.html.erb index 32af0878..342d9741 100644 --- a/app/views/report/proquest_status.html.erb +++ b/app/views/report/proquest_status.html.erb @@ -10,8 +10,8 @@

ProQuest opt-in status for theses in <%= @this_term %>

-

This report only lists theses that satisfy advanced degrees and have - at least one file attached.

+

Theses listed in this report satisfy advanced degrees and have at least + one file attached.

<%= render 'proquest_status_filter' %> @@ -26,10 +26,11 @@ Author(s) DSpace handle Opted in to ProQuest? + Exported to ProQuest? - <%= render(partial: 'proquest_thesis', collection: @list) || render('proquest_status_empty') %> + <%= render(partial: 'proquest_status_thesis', collection: @list) || render('shared/proquest_thesis_empty') %>
diff --git a/app/views/shared/_proquest_thesis_empty.html.erb b/app/views/shared/_proquest_thesis_empty.html.erb new file mode 100644 index 00000000..c4312600 --- /dev/null +++ b/app/views/shared/_proquest_thesis_empty.html.erb @@ -0,0 +1,3 @@ + + No theses match these criteria. + diff --git a/app/views/thesis/_proquest_export_thesis.html.erb b/app/views/thesis/_proquest_export_thesis.html.erb new file mode 100644 index 00000000..d99a59e5 --- /dev/null +++ b/app/views/thesis/_proquest_export_thesis.html.erb @@ -0,0 +1,11 @@ + + <%= link_to title_helper(proquest_export_thesis), edit_admin_thesis_path(proquest_export_thesis.id) %> + + <% proquest_export_thesis.authors.each do |author| %> + <%= link_to author.user.display_name, edit_admin_author_path(author.id) %>
+ <% end %> + + <%= proquest_export_thesis.grad_date %> + <%= proquest_export_thesis.dspace_handle %> + <%= render_proquest_status(proquest_export_thesis) %> + diff --git a/app/views/thesis/proquest_export_preview.html.erb b/app/views/thesis/proquest_export_preview.html.erb new file mode 100644 index 00000000..09cfe87a --- /dev/null +++ b/app/views/thesis/proquest_export_preview.html.erb @@ -0,0 +1,56 @@ +<%= content_for(:title, "Thesis Processing | MIT Libraries") %> + +<% content_for :additional_js do %> + +<% end %> + + + +

Export theses for ProQuest

+ +<%= render 'shared/you_are' %> + +
+
+ <% if @theses.count < 1 %> +

No theses are available to export.

+ <% else %> +
+

Ready to export <%= @theses.count %> theses?

+

If the set of thesis records below is correct, click the button at right to export their handles. Use the + <%= link_to "ProQuest Status Report", report_proquest_status_path %> to review ProQuest opt-in statuses.

+
+
+ <%= link_to "Export theses to send to ProQuest", thesis_proquest_export_path, class: 'button button-primary' %> +
+ <% end %> +
+
+ + + + + + + + + + + + + <%= render(partial: 'proquest_export_thesis', collection: @theses) || render('shared/proquest_thesis_empty') %> + +
TitleAuthor(s)TermDSpace handleOpted in to ProQuest?
+ + diff --git a/config/routes.rb b/config/routes.rb index ba2bea9b..4f656711 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -39,6 +39,8 @@ get 'thesis/publish', to: 'thesis#publish_to_dspace', as: 'thesis_publish_to_dspace' get 'thesis/select', to: 'thesis#select', as: 'thesis_select' get 'thesis/start', to: 'thesis#start', as: 'thesis_start' + get 'thesis/proquest_export_preview', to: 'thesis#proquest_export_preview', as: 'thesis_proquest_export_preview' + get 'thesis/proquest_export', to: 'thesis#proquest_export', as: 'thesis_proquest_export' resources :registrar, only: [:new, :create, :show] resources :thesis, only: [:new, :create, :edit, :show, :update] get 'harvest', to: 'registrar#list_registrar', as: 'harvest' diff --git a/db/migrate/20221101212143_add_proquest_exported_to_thesis.rb b/db/migrate/20221101212143_add_proquest_exported_to_thesis.rb new file mode 100644 index 00000000..df36aef5 --- /dev/null +++ b/db/migrate/20221101212143_add_proquest_exported_to_thesis.rb @@ -0,0 +1,5 @@ +class AddProquestExportedToThesis < ActiveRecord::Migration[6.1] + def change + add_column :theses, :proquest_exported, :integer, null: false, default: 0 + end +end diff --git a/db/migrate/20221209161518_create_proquest_export_batches.rb b/db/migrate/20221209161518_create_proquest_export_batches.rb new file mode 100644 index 00000000..be552df6 --- /dev/null +++ b/db/migrate/20221209161518_create_proquest_export_batches.rb @@ -0,0 +1,10 @@ +class CreateProquestExportBatches < ActiveRecord::Migration[6.1] + def change + create_table :proquest_export_batches do |t| + + t.timestamps + end + + add_reference :theses, :proquest_export_batch, null: true, index: true + end +end diff --git a/db/schema.rb b/db/schema.rb index b47fdb5f..f602a422 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_10_18_182433) do +ActiveRecord::Schema.define(version: 2022_12_09_161518) do create_table "active_storage_attachments", force: :cascade do |t| t.string "name", null: false @@ -168,6 +168,11 @@ t.datetime "updated_at", precision: 6, null: false end + create_table "proquest_export_batches", force: :cascade do |t| + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + end + create_table "registrars", force: :cascade do |t| t.integer "user_id", null: false t.datetime "created_at", precision: 6, null: false @@ -214,8 +219,11 @@ t.string "dspace_handle" t.boolean "issues_found", default: false, null: false t.integer "authors_count" + t.integer "proquest_exported", default: 0, null: false + t.integer "proquest_export_batch_id" t.index ["copyright_id"], name: "index_theses_on_copyright_id" t.index ["license_id"], name: "index_theses_on_license_id" + t.index ["proquest_export_batch_id"], name: "index_theses_on_proquest_export_batch_id" end create_table "transfers", force: :cascade do |t| diff --git a/test/controllers/report_controller_test.rb b/test/controllers/report_controller_test.rb index 573681bb..bf639be9 100644 --- a/test/controllers/report_controller_test.rb +++ b/test/controllers/report_controller_test.rb @@ -155,9 +155,10 @@ def create_thesis_with_whodunnit(user) # ~~~~ Empty thesis features test 'empty theses report shows a card' do + no_files_count = Thesis.without_files.count sign_in users(:processor) get report_empty_theses_path - assert_select '.card-empty-theses span', text: '19 have no attached files', count: 1 + assert_select '.card-empty-theses span', text: "#{no_files_count} have no attached files", count: 1 end test 'empty theses report has links to processing pages' do @@ -167,11 +168,13 @@ def create_thesis_with_whodunnit(user) end test 'empty theses report allows filtering by term' do + no_files_count = Thesis.without_files.count + term_filter_no_files_count = Thesis.where(grad_date: '2018-09-01').without_files.count sign_in users(:processor) get report_empty_theses_path - assert_select '.card-empty-theses span', text: '19 have no attached files', count: 1 + assert_select '.card-empty-theses span', text: "#{no_files_count} have no attached files", count: 1 get report_empty_theses_path, params: { graduation: '2018-09-01' } - assert_select '.card-empty-theses span', text: '2 have no attached files', count: 1 + assert_select '.card-empty-theses span', text: "#{term_filter_no_files_count} have no attached files", count: 1 end # ~~~~~~~~~~~~~~~~~~~~ Report term detail ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -227,22 +230,24 @@ def create_thesis_with_whodunnit(user) test 'term report shows a few fields' do sign_in users(:processor) get report_term_path - assert_select '.card-overall .message', text: '27 thesis records', count: 1 - assert_select '.card-files .message', text: '8 have files attached', count: 1 - assert_select '.card-issues span', text: '1 flagged with issues', count: 1 - assert_select '.card-students-contributing span', text: '0 have had metadata contributed by students', count: 1 - assert_select '.card-multiple-authors span', text: '4 have multiple authors', count: 1 - assert_select '.card-multiple-degrees span', text: '1 has multiple degrees', count: 1 - assert_select '.card-multiple-departments span', text: '1 has multiple departments', count: 1 + assert_select '.card-overall .message', count: 1 + assert_select '.card-files .message', count: 1 + assert_select '.card-issues span', count: 1 + assert_select '.card-students-contributing span', count: 1 + assert_select '.card-multiple-authors span', count: 1 + assert_select '.card-multiple-degrees span', count: 1 + assert_select '.card-multiple-departments span', count: 1 assert_response :success end test 'term report allows filtering' do + thesis_count = Thesis.count + term_filtered_count = Thesis.where(grad_date: '2018-09-01').count sign_in users(:processor) get report_term_path - assert_select '.card-overall .message', text: '27 thesis records', count: 1 + assert_select '.card-overall .message', text: "#{thesis_count} thesis records", count: 1 get report_term_path, params: { graduation: '2018-09-01' } - assert_select '.card-overall .message', text: '2 thesis records', count: 1 + assert_select '.card-overall .message', text: "#{term_filtered_count} thesis records", count: 1 assert_response :success end diff --git a/test/controllers/thesis_controller_test.rb b/test/controllers/thesis_controller_test.rb index 9f586e74..669e9c68 100644 --- a/test/controllers/thesis_controller_test.rb +++ b/test/controllers/thesis_controller_test.rb @@ -769,4 +769,122 @@ def attach_files_to_records(tr, th) follow_redirect! assert_select '.alert-banner.success', text: 'The theses you selected have been added to the publication queue. Status updates are not immediate.', count: 1 end + + # ~~~~~~~~~~~~~~~~~~~~~ proquest export preview ~~~~~~~~~~~~~~~~~~~~~ + test 'anonymous users get redirected if they load the proquest export preview list' do + get thesis_proquest_export_preview_path + assert_response :redirect + assert_redirected_to '/login' + end + + test 'basic users get redirected if they load the proquest export preview list' do + sign_in users(:basic) + get thesis_proquest_export_preview_path + assert_redirected_to '/' + follow_redirect! + assert_select 'div.alert', text: 'Not authorized.', count: 1 + end + + test 'submitters get redirected if they load the proquest export preview list' do + sign_in users(:transfer_submitter) + get thesis_proquest_export_preview_path + assert_redirected_to '/' + follow_redirect! + assert_select 'div.alert', text: 'Not authorized.', count: 1 + end + + test 'processors can load the proquest export preview list' do + sign_in users(:processor) + get thesis_proquest_export_preview_path + assert_response :success + end + + test 'thesis admins can load the proquest export preview list' do + sign_in users(:thesis_admin) + get thesis_proquest_export_preview_path + assert_response :success + end + + test 'admins can load the proquest export preview list' do + sign_in users(:admin) + get thesis_proquest_export_preview_path + assert_response :success + end + + test 'proquest export cannot be initiated with no eligible theses' do + sign_in users(:processor) + Thesis.destroy_all + get thesis_proquest_export_preview_path + assert_select 'div#export-action', count: 1 + assert_select 'div#export-action a[href=?]', thesis_proquest_export_path, count: 0 + end + + test 'proquest export preview, with eligible theses, includes a button to export' do + sign_in users(:processor) + get thesis_proquest_export_preview_path + assert_select 'div#export-action', count: 1 + assert_select 'div#export-action a[href=?]', thesis_proquest_export_path + end + + # ~~~~~~~~~~~~~~~~~~~~~ exporting theses to proquest ~~~~~~~~~~~~~~~~~~~~~ + test 'anonymous users get redirected if they request export to proquest' do + get thesis_proquest_export_path + assert_response :redirect + assert_redirected_to '/login' + end + + test 'basic users cannot export export_to_proquest proquest' do + sign_in users(:basic) + get thesis_proquest_export_path + assert_redirected_to '/' + follow_redirect! + assert_select 'div.alert', text: 'Not authorized.', count: 1 + end + + test 'submitters cannot export to proquest' do + sign_in users(:transfer_submitter) + get thesis_proquest_export_path + assert_redirected_to '/' + follow_redirect! + assert_select 'div.alert', text: 'Not authorized.', count: 1 + end + + test 'processors can export to proquest' do + sign_in users(:processor) + get thesis_proquest_export_path + assert_redirected_to thesis_proquest_export_preview_path + end + + test 'thesis_admins can export to proquest' do + sign_in users(:thesis_admin) + get thesis_proquest_export_path + assert_redirected_to thesis_proquest_export_preview_path + end + + test 'admins can export_to_proquest' do + sign_in users(:admin) + get thesis_proquest_export_path + assert_redirected_to thesis_proquest_export_preview_path + end + + test 'exporting to proquest redirects to proquest export preview path with a flash message' do + sign_in users(:processor) + clear_enqueued_jobs + assert_enqueued_jobs 0 + + get thesis_proquest_export_path + assert_enqueued_jobs 1 + assert_redirected_to thesis_proquest_export_preview_path + follow_redirect! + assert_select '.alert-banner.success', text: 'The theses you selected will be exported. Status updates are not immediate.', count: 1 + end + + test 'exporting to proquest with no eligible theses redirects to proquest export preview path with a flash message' do + sign_in users(:processor) + Thesis.destroy_all + get thesis_proquest_export_path + assert_redirected_to thesis_proquest_export_preview_path + follow_redirect! + assert_select '.alert-banner.warning', text: 'No theses are available to export.', count: 1 + end end diff --git a/test/fixtures/authors.yml b/test/fixtures/authors.yml index ee2247c5..284df7eb 100644 --- a/test/fixtures/authors.yml +++ b/test/fixtures/authors.yml @@ -31,12 +31,13 @@ three: user: basic thesis: two graduation_confirmed: false - proquest_allowed: false + proquest_allowed: true four: user: basic thesis: with_note graduation_confirmed: true + proquest_allowed: false five: user: basic @@ -74,6 +75,7 @@ ten: user: basic thesis: published graduation_confirmed: true + proquest_allowed: true eleven: user: basic @@ -101,3 +103,21 @@ fifteen: thesis: doctor graduation_confirmed: true proquest_allowed: false + +sixteen: + user: yo + thesis: engineer + graduation_confirmed: true + proquest_allowed: true + +seventeen: + user: yo + thesis: ready_for_partial_export + graduation_confirmed: true + proquest_allowed: false + +eighteen: + user: yo + thesis: ready_for_full_export + graduation_confirmed: true + proquest_allowed: true diff --git a/test/fixtures/theses.yml b/test/fixtures/theses.yml index 5d76c770..5c1e350c 100644 --- a/test/fixtures/theses.yml +++ b/test/fixtures/theses.yml @@ -258,6 +258,7 @@ doctor: files_complete: true metadata_complete: true issues_found: false + proquest_exported: 'Partial harvest' engineer: title: MyEngineerThesis @@ -271,6 +272,7 @@ engineer: metadata_complete: true issues_found: false publication_status: Published + proquest_exported: 'Full harvest' long_abstracts_are_fun: title: Fock @@ -284,3 +286,29 @@ long_abstracts_are_fun: issues_found: false publication_status: 'Published' dspace_handle: '1234.5/6789' + +proquest_export_full: + title: Exported for full ProQuest harvest + dspace_handle: '1234/5678' + grad_date: 2023-02-01 + proquest_exported: Full harvest + +proquest_export_partial: + title: Exported for partial ProQuest harvest + dspace_handle: '2345/6789' + grad_date: 2023-02-01 + proquest_exported: Partial harvest + +ready_for_full_export: + title: Ready for full ProQuest harvest + dspace_handle: '1234/5678' + grad_date: 2023-02-01 + degrees: [three] + publication_status: Published + +ready_for_partial_export: + title: Ready for partial ProQuest harvest + dspace_handle: '2345/6789' + grad_date: 2023-02-01 + degrees: [two] + publication_status: Published diff --git a/test/helpers/thesis_helper_test.rb b/test/helpers/thesis_helper_test.rb index 77ee41d3..9f5234ca 100644 --- a/test/helpers/thesis_helper_test.rb +++ b/test/helpers/thesis_helper_test.rb @@ -171,7 +171,7 @@ class ThesisHelperTest < ActionView::TestCase end test 'evaluate_proquest_status identifies opt-in conflicts' do - conflict_thesis = theses(:two) + conflict_thesis = theses(:doctor) assert conflict_thesis.authors.count > 1 assert_not_equal conflict_thesis.authors.first.proquest_allowed, conflict_thesis.authors.second.proquest_allowed assert_equal 'conflict', evaluate_proquest_status(conflict_thesis) @@ -194,7 +194,7 @@ class ThesisHelperTest < ActionView::TestCase true_thesis = theses(:one) false_thesis = theses(:with_hold) nil_thesis = theses(:coauthor) - conflict_thesis = theses(:two) + conflict_thesis = theses(:doctor) # Confirm proquest_allowed values. assert_equal true, evaluate_proquest_status(true_thesis) @@ -213,7 +213,7 @@ class ThesisHelperTest < ActionView::TestCase true_thesis = theses(:one) false_thesis = theses(:with_hold) nil_thesis = theses(:coauthor) - conflict_thesis = theses(:two) + conflict_thesis = theses(:doctor) assert_equal({ opted_in: 1, opted_out: 0, no_decision: 0, conflict: 0 }, proquest_status_counts([true_thesis])) assert_equal({ opted_in: 0, opted_out: 1, no_decision: 0, conflict: 0 }, proquest_status_counts([false_thesis])) diff --git a/test/integration/thesis_test.rb b/test/integration/thesis_test.rb index 6f05a0a4..945ab0ef 100644 --- a/test/integration/thesis_test.rb +++ b/test/integration/thesis_test.rb @@ -113,7 +113,7 @@ def teardown test 'students can edit their advisor name via thesis form' do mock_auth(users(:yo)) count = Advisor.count - sample = users(:yo).theses.third + sample = users(:yo).theses.fourth sample_advisor = sample.advisors.first assert_not_equal 'Another Name', sample_advisor.name diff --git a/test/jobs/proquest_export_job_test.rb b/test/jobs/proquest_export_job_test.rb new file mode 100644 index 00000000..2ff27934 --- /dev/null +++ b/test/jobs/proquest_export_job_test.rb @@ -0,0 +1,37 @@ +require 'test_helper' + +class ProquestExportJobTest < ActiveJob::TestCase + include ActionMailer::TestHelper + + test 'proquest_exported values are updated' do + assert_equal 'Not exported', theses(:one).proquest_exported + assert_equal 'Not exported', theses(:with_hold).proquest_exported + + ProquestExportJob.perform_now([theses(:with_hold)], [theses(:one)]) + assert_equal 'Full harvest', theses(:one).proquest_exported + assert_equal 'Partial harvest', theses(:with_hold).proquest_exported + end + + test 'batch export is created' do + pq_export_batch_count = ProquestExportBatch.count + ProquestExportJob.perform_now(Thesis.all, Thesis.all) + assert_equal pq_export_batch_count + 1, ProquestExportBatch.count + end + + # This test will require updating if we begin using ProQuestExportBatch fixtures. + test 'JSON is attached to an export' do + assert_empty ProquestExportBatch.all + ProquestExportJob.perform_now(Thesis.all, Thesis.all) + latest_batch = ProquestExportBatch.last + assert_not_nil latest_batch.proquest_export.blob + assert_equal 'application/json', latest_batch.proquest_export.blob.content_type + end + + test 'sends batch email' do + ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do + assert_emails 1 do + ProquestExportJob.perform_now(Thesis.all, Thesis.all) + end + end + end +end diff --git a/test/mailers/batch_mailer_test.rb b/test/mailers/batch_mailer_test.rb index 03d5097b..d5d60fa5 100644 --- a/test/mailers/batch_mailer_test.rb +++ b/test/mailers/batch_mailer_test.rb @@ -30,4 +30,29 @@ class BatchMailerTest < ActionMailer::TestCase assert_equal 'application/zip; filename=marc.zip', attachment.content_type end end + + test 'sends emails for ProQuest batch exports' do + ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do + theses = [theses(:doctor), theses(:engineer)] + export = ProquestExportBatch.new + export_json = export.build_json(theses) + export.proquest_export.attach(io: StringIO.new(export_json), + filename: 'pq.json', + content_type: 'application/json') + export.save + email = BatchMailer.proquest_export_email(export.proquest_export, theses) + + # Send the email, then test that it got queued + assert_emails 1 do + email.deliver_now + end + + # Make sure it was sent to the right person with the expected attachment. + assert_equal ['test@example.com'], email.from + assert_equal ['test@example.com'], email.to + assert_equal 'ETD ProQuest export', email.subject + assert_equal 'pq.json', email.attachments.first.filename + assert_includes '2 theses', email.body.to_s + end + end end diff --git a/test/models/proquest_export_batch_test.rb b/test/models/proquest_export_batch_test.rb new file mode 100644 index 00000000..bbbfe613 --- /dev/null +++ b/test/models/proquest_export_batch_test.rb @@ -0,0 +1,15 @@ +require "test_helper" + +class ProquestExportBatchTest < ActiveSupport::TestCase + test 'builds JSON with expected values' do + theses = [theses(:proquest_export_partial), theses(:proquest_export_full)] + assert_equal 'Partial harvest', theses(:proquest_export_partial).proquest_exported + assert_equal 'Full harvest', theses(:proquest_export_full).proquest_exported + + json_hash = JSON.parse(ProquestExportBatch.new.build_json(theses)) + assert_equal theses(:proquest_export_partial).dspace_handle, json_hash['records'].first['dspace_handle'] + assert_equal theses(:proquest_export_full).dspace_handle, json_hash['records'].second['dspace_handle'] + assert_equal false, json_hash['records'].first['full_harvest'] + assert json_hash['records'].second['full_harvest'] + end +end diff --git a/test/models/report_test.rb b/test/models/report_test.rb index 5c82babb..9158c02d 100644 --- a/test/models/report_test.rb +++ b/test/models/report_test.rb @@ -22,9 +22,9 @@ class ReportTest < ActiveSupport::TestCase returned_rows = r.index_data['license'].count assert_equal expected_rows, returned_rows # This includes a row of all zeros, which could only have been generated by populate_category - assert_equal r.index_data['license'][3][:data].values, [0, 0, 0, 0, 0, 0, 0, 0, 0] + assert r.index_data['license'][3][:data].values.all? { |v| v == 0 } # Also check that a row with expected data is represented accurately - assert_equal r.index_data['license'][1][:data].values, [3, 0, 0, 0, 0, 0, 0, 0, 0] + assert_equal r.index_data['license'][1][:data].values, [3, 0, 0, 0, 0, 0, 0, 0, 0, 0] end # pad_terms is a private method that gets called by each reporting row, to ensure that all rows have some value for @@ -39,7 +39,7 @@ class ReportTest < ActiveSupport::TestCase returned_columns = r.index_data['summary'][3][:data].count assert_equal expected_terms, returned_columns # Test for a row of expected values - assert_equal r.index_data['summary'][3][:data].values, [0, 0, 0, 0, 0, 0, 1, 0, 0] + assert_equal r.index_data['summary'][3][:data].values, [0, 0, 0, 0, 0, 0, 1, 0, 0, 0] end # ~~~~ Dashboard report @@ -48,7 +48,7 @@ class ReportTest < ActiveSupport::TestCase result = r.index_data assert_equal Department.count, result['departments'].pluck(:label).length assert_includes result['departments'].pluck(:label), Department.first.name_dw - assert_equal result['departments'][0][:data].values, [0, 2, 0, 0, 0, 0, 0, 3, 1] + assert_equal result['departments'][0][:data].values, [0, 2, 0, 0, 0, 0, 0, 3, 1, 0] end test 'index includes summary data of authors not graduated' do @@ -70,7 +70,7 @@ class ReportTest < ActiveSupport::TestCase r = Report.new result = r.index_data - assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 1, 1] + assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 1, 0, 0] end test 'authors not graduated summary data dedups theses with multiple files' do @@ -91,8 +91,8 @@ class ReportTest < ActiveSupport::TestCase r = Report.new result = r.index_data - assert_not_equal result['summary'][5][:data].values, [0, 2, 0, 0, 0, 0, 0, 0, 0] - assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 0, 1] + assert_not_equal result['summary'][5][:data].values, [0, 2, 0, 0, 0, 0, 0, 0, 0, 0] + assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 0, 0, 0] end # ~~~~ Term detail report diff --git a/test/models/thesis_test.rb b/test/models/thesis_test.rb index e1610e0b..77f46feb 100644 --- a/test/models/thesis_test.rb +++ b/test/models/thesis_test.rb @@ -168,7 +168,6 @@ def attach_file_with_purpose_to(thesis, purpose = 'thesis_pdf') t = theses(:one) u = users(:yo) assert_includes t.users, u - assert_equal 6, u.authors.count assert_difference('u.authors.count', -1) { t.destroy } end @@ -1245,4 +1244,142 @@ def attach_file_with_purpose_to(thesis, purpose = 'thesis_pdf') assert thesis.authors.empty? assert multi_author_count - 1, Thesis.multiple_authors.count end + + test 'consented_to_proquest scope returns only theses with authors that have opted in to ProQuest' do + proquest_consent_count = Thesis.consented_to_proquest.count + assert proquest_consent_count < Thesis.count + + # single-author thesis that has opted in is included + thesis = theses(:one) + assert_includes Thesis.consented_to_proquest, thesis + + # single-author thesis that has opted out is excluded + thesis = theses(:with_note) + assert_not_includes Thesis.consented_to_proquest, thesis + + # single-author thesis with no opt-in status is excluded + thesis = theses(:coauthor) + assert_not_includes Thesis.consented_to_proquest, thesis + + # multi-author thesis that has opted in is included + thesis = theses(:two) + assert_includes Thesis.consented_to_proquest, thesis + + # multi-author thesis that has opted out is excluded + thesis = theses(:with_hold) + assert_not_includes Thesis.consented_to_proquest, thesis + + # multi-author thesis with conflicting opt-in statuses is excluded + thesis = theses(:doctor) + assert_not_includes Thesis.consented_to_proquest, thesis + end + + test 'only certain proquest_exported values are valid' do + thesis = theses(:one) + assert thesis.proquest_exported = 'Not exported' + assert thesis.valid? + + thesis.proquest_exported = 'Full harvest' + thesis.save + assert thesis.valid? + + thesis.proquest_exported = 'Partial harvest' + thesis.save + assert thesis.valid? + end + + test 'not_consented_to_proquest scope returns only theses with authors that have not opted in to ProQuest' do + # single-author thesis that has opted out is included + thesis = theses(:with_note) + assert_includes Thesis.not_consented_to_proquest, thesis + + # single-author thesis with no opt-in status is included + thesis = theses(:coauthor) + assert_includes Thesis.not_consented_to_proquest, thesis + + # single-author thesis that has opted in is excluded + thesis = theses(:one) + assert_not_includes Thesis.not_consented_to_proquest, thesis + + # multi-author thesis with all opt-outs is included + thesis = theses(:with_hold) + assert_includes Thesis.not_consented_to_proquest, thesis + + # multi-author thesis with one opt-in and one opt-out is excluded + thesis = theses(:doctor) + assert_not_includes Thesis.not_consented_to_proquest, thesis + + # multi-author thesis that has opted in is excluded + thesis = theses(:two) + assert_not_includes Thesis.not_consented_to_proquest, thesis + end + + test 'exported_to_proquest scope includes theses flagged for partial harvest' do + partially_harvestable_thesis = theses(:doctor) + assert_includes Thesis.exported_to_proquest, partially_harvestable_thesis + end + + test 'exported_to_proquest scope includes theses flagged for full harvest' do + fully_harvestable_thesis = theses(:engineer) + assert_includes Thesis.exported_to_proquest, fully_harvestable_thesis + end + + test 'exported_to_proquest scope excludes theses that have not been exported' do + unexported_thesis = theses(:one) + assert_not_includes Thesis.exported_to_proquest, unexported_thesis + end + + test 'not_exported_to_proquest scope includes theses that have not been exported' do + unexported_thesis = theses(:one) + assert_includes Thesis.not_exported_to_proquest, unexported_thesis + end + + test 'not_exported_to_proquest scope does not include theses flagged for harvest' do + partially_harvestable_thesis = theses(:doctor) + fully_harvestable_thesis = theses(:engineer) + assert_not_includes Thesis.not_exported_to_proquest, partially_harvestable_thesis + assert_not_includes Thesis.not_exported_to_proquest, fully_harvestable_thesis + end + + test 'published scope returns only theses that are published' do + published_thesis = theses(:published) + not_ready_thesis = theses(:issues_found) + pending_publication_thesis = theses(:pending_publication) + in_review_thesis = theses(:publication_review) + + assert_includes Thesis.published, published_thesis + assert_not_includes Thesis.published, not_ready_thesis + assert_not_includes Thesis.published, pending_publication_thesis + assert_not_includes Thesis.published, in_review_thesis + end + + test 'partial_proquest_export scope returns the expected set of theses' do + partial_export_thesis = theses(:ready_for_partial_export) + full_export_thesis = theses(:ready_for_full_export) + no_export_thesis = theses(:one) + + assert_includes Thesis.partial_proquest_export, partial_export_thesis + assert_not_includes Thesis.partial_proquest_export, full_export_thesis + assert_not_includes Thesis.partial_proquest_export, no_export_thesis + end + + test 'full_proquest_export scope returns the expected set of theses' do + partial_export_thesis = theses(:ready_for_partial_export) + full_export_thesis = theses(:ready_for_full_export) + no_export_thesis = theses(:one) + + assert_includes Thesis.full_proquest_export, full_export_thesis + assert_not_includes Thesis.full_proquest_export, partial_export_thesis + assert_not_includes Thesis.full_proquest_export, no_export_thesis + end + + test 'ready_for_proquest_export scope returns all theses to be exported' do + partial_export_thesis = theses(:ready_for_partial_export) + full_export_thesis = theses(:ready_for_full_export) + no_export_thesis = theses(:one) + + assert_includes Thesis.ready_for_proquest_export, partial_export_thesis + assert_includes Thesis.ready_for_proquest_export, full_export_thesis + assert_not_includes Thesis.ready_for_proquest_export, no_export_thesis + end end diff --git a/test/models/user_test.rb b/test/models/user_test.rb index 66f9c8b7..742551e9 100644 --- a/test/models/user_test.rb +++ b/test/models/user_test.rb @@ -352,7 +352,8 @@ class UserTest < ActiveSupport::TestCase test 'editable_theses returns a set of thesis records' do u = users(:yo) - assert_equal 3, u.editable_theses.count + editable_thesis_count = u.theses.where(metadata_complete: false).where(issues_found: false).count + assert_equal editable_thesis_count, u.editable_theses.count end test 'editable_theses returns a thesis with neither metadata nor issues flags set' do