Add batch export feature for ProQuest JSON

app/controllers/thesis_controller.rb

@@ -134,6 +134,22 @@ def process_theses_update
     redirect_to thesis_process_path
   end
 
+  def proquest_export_preview
+    @theses = Thesis.ready_for_proquest_export
+  end
+
+  # TODO: we need to generate and send a budget report CSV for partially harvested theses (spec TBD).
+  def proquest_export
+    if Thesis.ready_for_proquest_export.any?
+      ProquestExportJob.perform_later(Thesis.partial_proquest_export, Thesis.full_proquest_export)
+      flash[:success] = 'The theses you selected will be exported. ' \
+                        'Status updates are not immediate.'
+    else
+      flash[:warning] = 'No theses are available to export.'
+    end
+    redirect_to thesis_proquest_export_preview_path
+  end
+
   private
 
   # Various methods need to build an array of academic terms which meet varying conditions, in order to support a UI

app/dashboards/thesis_dashboard.rb

@@ -40,7 +40,8 @@ class ThesisDashboard < Administrate::BaseDashboard
     coauthors: Field::String,
     copyright: Field::BelongsTo,
     license: Field::BelongsTo,
-    dspace_handle: Field::String
+    dspace_handle: Field::String,
+    proquest_exported: Field::String
   }.freeze
 
   # COLLECTION_ATTRIBUTES
@@ -80,6 +81,7 @@ class ThesisDashboard < Administrate::BaseDashboard
     metadata_complete
     files_complete
     files
+    proquest_exported
   ].freeze
 
   # FORM_ATTRIBUTES

app/jobs/proquest_export_job.rb

@@ -0,0 +1,32 @@
+class ProquestExportJob < ActiveJob::Base
+  queue_as :default
+
+  def perform(partial_harvest, full_harvest)
+    export = ProquestExportBatch.new
+
+    # update_all is an option here if performance is poor, but we would need a workaround to update paper_trail
+    # versions as update_all does not trigger callbacks.
+    if partial_harvest.present?
+      partial_harvest.each do |thesis|
+        thesis.update(proquest_exported: 'Partial harvest', proquest_export_batch: export)
+      end
+    end
+    if full_harvest.present?
+      full_harvest.each { |thesis| thesis.update(proquest_exported: 'Full harvest', proquest_export_batch: export) }
+    end
+
+    all_to_export = partial_harvest + full_harvest
+    attach_and_send(export, all_to_export)
+  end
+
+  private
+
+  def attach_and_send(export, all_to_export)
+    export_json = export.build_json(all_to_export)
+    export.proquest_export.attach(io: StringIO.new(export_json),
+                                  filename: "proquest_export_#{Date.today.strftime('%Y%m%d_%s')}.json",
+                                  content_type: 'application/json')
+    export.save
+    BatchMailer.proquest_export_email(export.proquest_export.blob, all_to_export.count).deliver_later
+  end
+end

app/mailers/batch_mailer.rb

@@ -9,4 +9,18 @@ def marc_batch_email(marc_zip_filename, marc_zip_file, theses)
          cc: ENV['MAINTAINER_EMAIL'],
          subject: 'ETD MARC batch export')
   end
+
+  def proquest_export_email(json_blob, thesis_count)
+    return unless ENV.fetch('DISABLE_ALL_EMAIL', 'true') == 'false' # allows PR builds to disable emails
+
+    @thesis_count = thesis_count
+    attachments[json_blob.filename.to_s] = {
+      mime_type: json_blob.content_type,
+      content: json_blob.download
+    }
+    mail(from: "MIT Libraries <#{ENV['THESIS_ADMIN_EMAIL']}>",
+         to: ENV['THESIS_ADMIN_EMAIL'],
+         cc: ENV['MAINTAINER_EMAIL'],
+         subject: 'ETD ProQuest export')
+  end
 end

app/models/ability.rb

@@ -80,10 +80,12 @@ def processor
     can :mark_withdrawn, Thesis
     can :process_theses, Thesis
     can :process_theses_update, Thesis
+    can :publication_statuses, Thesis
     can :publish_preview, Thesis
     can :publish_to_dspace, Thesis
+    can :proquest_export, Thesis
+    can :proquest_export_preview, Thesis
     can :select, Thesis
-    can :publication_statuses, Thesis
 
     can :read, Transfer
     can :select, Transfer

app/models/proquest_export_batch.rb

@@ -0,0 +1,24 @@
+class ProquestExportBatch < ApplicationRecord
+  has_many :theses
+
+  has_one_attached :proquest_export
+  has_one_attached :budget_report
+
+  def build_json(theses)
+    thesis_records = theses.map { |thesis| record(thesis) }
+    { records: thesis_records }.to_json
+  end
+
+  private
+
+  def record(thesis)
+    {
+      dspace_handle: thesis.dspace_handle,
+      full_harvest: evaluate_export_type(thesis)
+    }
+  end
+
+  def evaluate_export_type(thesis)
+    thesis.proquest_exported == 'Full harvest'
+  end
+end

app/models/thesis.rb

@@ -25,6 +25,7 @@ class Thesis < ApplicationRecord
 
   belongs_to :copyright, optional: true
   belongs_to :license, optional: true
+  belongs_to :proquest_export_batch, optional: true
 
   has_many :degree_theses
   has_many :degrees, through: :degree_theses
@@ -112,6 +113,23 @@ class Thesis < ApplicationRecord
   scope :publication_statuses, -> { PUBLICATION_STATUS_OPTIONS }
   scope :multiple_authors, -> { where('authors_count > ?', 1) }
   scope :advanced_degree, -> { includes(degrees: :degree_type).distinct.where.not(degree_type: { name: 'Bachelor' }) }
+  scope :doctoral, -> { includes(degrees: :degree_type).distinct.where(degree_type: { name: 'Doctoral' }) }
+  scope :consented_to_proquest, lambda {
+                                  includes(authors: :user).where(authors: { proquest_allowed: true })
+                                                          .excluding(includes(authors: :user)
+                                                                     .where(authors: { proquest_allowed: false }))
+                                }
+  scope :not_consented_to_proquest, lambda {
+                                      excluding(includes(authors: :user).where(authors: { proquest_allowed: true }))
+                                    }
+  scope :exported_to_proquest, -> { where(proquest_exported: ['Partial harvest', 'Full harvest']) }
+  scope :not_exported_to_proquest, -> { where(proquest_exported: 'Not exported') }
+  scope :published, -> { where(publication_status: 'Published') }
+  scope :partial_proquest_export, -> { published.doctoral.not_exported_to_proquest.not_consented_to_proquest }
+  scope :full_proquest_export, -> { published.advanced_degree.not_exported_to_proquest.consented_to_proquest }
+  scope :ready_for_proquest_export, -> { partial_proquest_export + full_proquest_export }
+
+  enum proquest_exported: ['Not exported', 'Full harvest', 'Partial harvest']
 
   # Returns a true/false value (rendered as "yes" or "no") if there are any
   # holds with a status of either 'active' or 'expired'. A false/"No" is

app/views/batch_mailer/proquest_export_email.html.erb

@@ -0,0 +1,4 @@
+<p>Hello,</p>
+
+<p>Attached is a ProQuest export of <%= @thesis_count %> theses generated on
+<%= Date.current.strftime('%A, %B %d, %Y') %> at <%= Time.now.strftime('%r %Z') %>.

app/views/layouts/_site_nav.html.erb

@@ -25,6 +25,9 @@
             <% if can? :list_registrar, Registrar %>
               <%= nav_link_to("Harvest CSV", harvest_path) %>
             <% end %>
+            <% if can? :proquest_export, Thesis %>
+              <%= nav_link_to("Export for ProQuest", thesis_proquest_export_preview_path) %>
+            <% end %>
             <% if can? :index, Report %>
               <%= nav_link_to("Report", report_index_path) %>
             <% end %>

app/views/report/_proquest_status_thesis.html.erb

@@ -0,0 +1,11 @@
+<tr>
+  <td><%= link_to title_helper(proquest_status_thesis), edit_admin_thesis_path(proquest_status_thesis.id) %></td>
+  <td>
+    <% proquest_status_thesis.authors.each do |author| %>
+      <%= link_to author.user.display_name, edit_admin_author_path(author.id) %><br>
+    <% end %>
+  </td>
+  <td><%= proquest_status_thesis.dspace_handle %></td>
+  <td><%= render_proquest_status(proquest_status_thesis) %></td>
+  <td><%= proquest_status_thesis.proquest_exported %></td>
+</tr>

app/views/report/proquest_status.html.erb

@@ -10,8 +10,8 @@
   <div class="col3q">
     <h3 class="title title-page">ProQuest opt-in status for theses in <%= @this_term %></h3>
     <div class="alert alert-banner">
-      <p><i class="fa fa-info-circle fa-lg"></i> This report only lists theses that satisfy advanced degrees and have
-      at least one file attached.</p>
+      <p><i class="fa fa-info-circle fa-lg"></i> Theses listed in this report satisfy advanced degrees and have at least
+      one file attached.</p>
     </div>
 
     <%= render 'proquest_status_filter' %>
@@ -26,10 +26,11 @@
           <th scope="col">Author(s)</th>
           <th scope="col">DSpace handle</th>
           <th scope="col">Opted in to ProQuest?</th>
+          <th scope="col">Exported to ProQuest?</th>
         </tr>
       </thead>
       <tbody>
-        <%= render(partial: 'proquest_thesis', collection: @list) || render('proquest_status_empty') %>
+        <%= render(partial: 'proquest_status_thesis', collection: @list) || render('shared/proquest_thesis_empty') %>
       </tbody>
     </table>
   </div>

app/views/shared/_proquest_thesis_empty.html.erb

@@ -0,0 +1,3 @@
+<tr>
+  <td colspan="5">No theses match these criteria.</td>
+</tr>

app/views/thesis/_proquest_export_thesis.html.erb

@@ -0,0 +1,11 @@
+<tr>
+  <td><%= link_to title_helper(proquest_export_thesis), edit_admin_thesis_path(proquest_export_thesis.id) %></td>
+  <td>
+    <% proquest_export_thesis.authors.each do |author| %>
+      <%= link_to author.user.display_name, edit_admin_author_path(author.id) %><br>
+    <% end %>
+  </td>
+  <td><%= proquest_export_thesis.grad_date %></td>
+  <td><%= proquest_export_thesis.dspace_handle %></td>
+  <td><%= render_proquest_status(proquest_export_thesis) %></td>
+</tr>

app/views/thesis/proquest_export_preview.html.erb

@@ -0,0 +1,56 @@
+<%= content_for(:title, "Thesis Processing | MIT Libraries") %>
+
+<% content_for :additional_js do %>
+  <script src="https://cdn.datatables.net/1.11.3/js/jquery.dataTables.min.js"></script>
+<% end %>
+
+<link href="https://cdn.datatables.net/1.11.3/css/jquery.dataTables.min.css" rel="stylesheet">
+
+<h3 class="title title-page">Export theses for ProQuest</h3>
+
+<%= render 'shared/you_are' %>
+
+<div class="gridband" id="export-action" aria-live="polite" role="region">
+  <div class="inline-action well warning">
+    <% if @theses.count < 1 %>
+      <p>No theses are available to export.</p>
+    <% else %>
+      <div class="message">
+        <h4 class="title">Ready to export <%= @theses.count %> theses?</h4>
+        <p>If the set of thesis records below is correct, click the button at right to export their handles. Use the
+           <%= link_to "ProQuest Status Report", report_proquest_status_path %> to review ProQuest opt-in statuses.</p>
+      </div>
+      <div class="actions">
+        <%= link_to "Export theses to send to ProQuest", thesis_proquest_export_path, class: 'button button-primary' %>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<table class="table" id="proquestPreview" summary="This table presents a list of theses with their ProQuest opt-in status."
+ title="ProQuest opt-in status">
+  <thead>
+    <tr>
+      <th scope="col">Title</th>
+      <th scope="col">Author(s)</th>
+      <th scope="col">Term</th>
+      <th scope="col">DSpace handle</th>
+      <th scope="col">Opted in to ProQuest?</th>
+    </tr>
+  </thead>
+  <tbody>
+    <%= render(partial: 'proquest_export_thesis', collection: @theses) || render('shared/proquest_thesis_empty') %>
+  </tbody>
+</table>
+
+<script type="text/javascript">
+$(document).ready(function () {
+  $('#proquestPreview').DataTable({
+    pageLength: -1,
+    lengthMenu: [
+      [10, 25, 50, -1],
+      [10, 25, 50, 'All'],
+    ],
+  });
+});
+</script>

config/routes.rb

@@ -39,6 +39,8 @@
   get 'thesis/publish', to: 'thesis#publish_to_dspace', as: 'thesis_publish_to_dspace'
   get 'thesis/select', to: 'thesis#select', as: 'thesis_select'
   get 'thesis/start', to: 'thesis#start', as: 'thesis_start'
+  get 'thesis/proquest_export_preview', to: 'thesis#proquest_export_preview', as: 'thesis_proquest_export_preview'
+  get 'thesis/proquest_export', to: 'thesis#proquest_export', as: 'thesis_proquest_export'
   resources :registrar, only: [:new, :create, :show]
   resources :thesis, only: [:new, :create, :edit, :show, :update]
   get 'harvest', to: 'registrar#list_registrar', as: 'harvest'

db/migrate/20221101212143_add_proquest_exported_to_thesis.rb

@@ -0,0 +1,5 @@
+class AddProquestExportedToThesis < ActiveRecord::Migration[6.1]
+  def change
+    add_column :theses, :proquest_exported, :integer, null: false, default: 0
+  end
+end

db/migrate/20221209161518_create_proquest_export_batches.rb

@@ -0,0 +1,10 @@
+class CreateProquestExportBatches < ActiveRecord::Migration[6.1]
+  def change
+    create_table :proquest_export_batches do |t|
+
+      t.timestamps
+    end
+
+    add_reference :theses, :proquest_export_batch, null: true, index: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2022_10_18_182433) do
+ActiveRecord::Schema.define(version: 2022_12_09_161518) do
 
   create_table "active_storage_attachments", force: :cascade do |t|
     t.string "name", null: false
@@ -168,6 +168,11 @@
     t.datetime "updated_at", precision: 6, null: false
   end
 
+  create_table "proquest_export_batches", force: :cascade do |t|
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+  end
+
   create_table "registrars", force: :cascade do |t|
     t.integer "user_id", null: false
     t.datetime "created_at", precision: 6, null: false
@@ -214,8 +219,11 @@
     t.string "dspace_handle"
     t.boolean "issues_found", default: false, null: false
     t.integer "authors_count"
+    t.integer "proquest_exported", default: 0, null: false
+    t.integer "proquest_export_batch_id"
     t.index ["copyright_id"], name: "index_theses_on_copyright_id"
     t.index ["license_id"], name: "index_theses_on_license_id"
+    t.index ["proquest_export_batch_id"], name: "index_theses_on_proquest_export_batch_id"
   end
 
   create_table "transfers", force: :cascade do |t|