Generate budget report CSV for partially harvested theses

app/jobs/proquest_export_job.rb

@@ -16,17 +16,22 @@ def perform(partial_harvest, full_harvest)
     end
 
     all_to_export = partial_harvest + full_harvest
-    attach_and_send(export, all_to_export)
+    attach_and_send(export, all_to_export, partial_harvest)
   end
 
   private
 
-  def attach_and_send(export, all_to_export)
+  def attach_and_send(export, all_to_export, partial_harvest)
     export_json = export.build_json(all_to_export)
+    budget_report = export.build_budget_report(partial_harvest)
     export.proquest_export.attach(io: StringIO.new(export_json),
                                   filename: "proquest_export_#{Date.today.strftime('%Y%m%d_%s')}.json",
                                   content_type: 'application/json')
+    export.budget_report.attach(io: StringIO.new(budget_report),
+                                filename: "proquest_budget_report_#{Date.today.strftime('%Y%m%d_%s')}.csv",
+                                content_type: 'application/csv')
     export.save
-    BatchMailer.proquest_export_email(export.proquest_export.blob, all_to_export.count).deliver_later
+    BatchMailer.proquest_export_email(export.proquest_export.blob, export.budget_report.blob, all_to_export.count,
+                                      partial_harvest.count).deliver_later
   end
 end

app/mailers/batch_mailer.rb

@@ -10,14 +10,20 @@ def marc_batch_email(marc_zip_filename, marc_zip_file, theses)
          subject: 'ETD MARC batch export')
   end
 
-  def proquest_export_email(json_blob, thesis_count)
+  def proquest_export_email(json_blob, csv_blob, thesis_count, budget_report_count)
     return unless ENV.fetch('DISABLE_ALL_EMAIL', 'true') == 'false' # allows PR builds to disable emails
 
     @thesis_count = thesis_count
+    @budget_report_count = budget_report_count
+
     attachments[json_blob.filename.to_s] = {
       mime_type: json_blob.content_type,
       content: json_blob.download
     }
+    attachments[csv_blob.filename.to_s] = {
+      mime_type: csv_blob.content_type,
+      content: csv_blob.download
+    }
     mail(from: "MIT Libraries <#{ENV['ETD_APP_EMAIL']}>",
          to: ENV['THESIS_ADMIN_EMAIL'],
          cc: ENV['MAINTAINER_EMAIL'],

app/models/proquest_export_batch.rb

@@ -1,4 +1,6 @@
 class ProquestExportBatch < ApplicationRecord
+  require 'csv'
+
   has_many :theses
 
   has_one_attached :proquest_export
@@ -9,6 +11,16 @@ def build_json(theses)
     { records: thesis_records }.to_json
   end
 
+  def build_budget_report(partial_export_theses)
+    CSV.generate do |csv|
+      csv << ['author name(s)', 'department(s)', 'degree type(s)', 'degree period', 'handle', 'export date']
+      partial_export_theses.each do |thesis|
+        csv << [author_names(thesis), departments(thesis), degree_types(thesis), thesis.grad_date, thesis.dspace_handle,
+                Date.today.strftime('%m-%d-%Y')].flatten
+      end
+    end
+  end
+
   private
 
   def record(thesis)
@@ -21,4 +33,16 @@ def record(thesis)
   def evaluate_export_type(thesis)
     thesis.proquest_exported == 'Full harvest'
   end
+
+  def author_names(thesis)
+    thesis.users.map(&:name).join('; ')
+  end
+
+  def departments(thesis)
+    thesis.departments.map(&:name_dw).join('; ')
+  end
+
+  def degree_types(thesis)
+    thesis.degrees.map { |degree| degree.degree_type.name }.join('; ')
+  end
 end

app/views/batch_mailer/proquest_export_email.html.erb

@@ -1,4 +1,11 @@
 <p>Hello,</p>
 
-<p>Attached is a ProQuest export of <%= @thesis_count %> theses generated on
-<%= Date.current.strftime('%A, %B %d, %Y') %> at <%= Time.now.strftime('%r %Z') %>.
+<p>Attached is a report of <%= @thesis_count %> theses exported to ProQuest from the
+<a href="https://thesis-submit.mit.edu/">ETD application</a> on <%= Date.current.strftime('%A, %B %d, %Y') %> at
+<%= Time.now.strftime('%r %Z') %>.</p>
+
+<p>Included in this export are <%= @budget_report_count %> doctoral theses with authors that have not opted in to have
+their theses submitted to ProQuest. ProQuest will harvest these theses' metadata. Please use the attached budget report
+for budget reconciliation purposes.</p>
+
+<p>Please contact the ETD team at <%= ENV['THESIS_ADMIN_EMAIL'] %> with any questions.</p>

test/fixtures/authors.yml

@@ -145,3 +145,15 @@ twentytwo:
   thesis: pq_conflict_false_nil
   graduation_confirmed: true
   proquest_allowed: nil
+
+twentythree:
+  user: yo
+  thesis: budget_report_multiple
+  graduation_confirmed: true
+  proquest_allowed: false
+
+twentyfour:
+  user: basic
+  thesis: budget_report_multiple
+  graduation_confirmed: true
+  proquest_allowed: false

test/fixtures/theses.yml

@@ -311,6 +311,8 @@ ready_for_partial_export:
   dspace_handle: '2345/6789'
   grad_date: 2023-02-01
   degrees: [two]
+  departments: [one]
+  dspace_handle: 1234/5678
   publication_status: Published
 
 pq_conflict_true_nil:
@@ -328,3 +330,11 @@ pq_conflict_false_nil:
   degrees: [two]
   departments: [one]
   publication_status: Published
+
+budget_report_multiple:
+  title: Budget export with multiple departments, degrees, authors
+  grad_date: 2022-09-01
+  degrees: [two, three]
+  departments: [one, two]
+  dspace_handle: 1234/5678
+  publication_status: Published

test/jobs/proquest_export_job_test.rb

@@ -18,7 +18,6 @@ class ProquestExportJobTest < ActiveJob::TestCase
     assert_equal pq_export_batch_count + 1, ProquestExportBatch.count
   end
 
-  # This test will require updating if we begin using ProQuestExportBatch fixtures.
   test 'JSON is attached to an export' do
     assert_empty ProquestExportBatch.all
     ProquestExportJob.perform_now(Thesis.all, Thesis.all)
@@ -27,7 +26,23 @@ class ProquestExportJobTest < ActiveJob::TestCase
     assert_equal 'application/json', latest_batch.proquest_export.blob.content_type
   end
 
-  test 'sends batch email' do
+  test 'budget report is attached to an export' do
+    assert_empty ProquestExportBatch.all
+    ProquestExportJob.perform_now(Thesis.all, Thesis.all)
+    latest_batch = ProquestExportBatch.last
+    assert_not_nil latest_batch.budget_report.blob
+    assert_equal 'application/csv', latest_batch.budget_report.blob.content_type
+  end
+
+  test 'budget report includes only theses exported for partial harvest' do
+    ProquestExportJob.perform_now([theses(:proquest_export_partial)], [theses(:proquest_export_full)])
+    csv = CSV.parse(ProquestExportBatch.last.budget_report.blob.download)
+    assert_equal 2, csv.length
+    assert_not_equal theses(:proquest_export_full).dspace_handle, csv[1][4]
+    assert_equal theses(:proquest_export_partial).dspace_handle, csv[1][4]
+  end
+
+  test 'sends batch and budget report emails' do
     ClimateControl.modify DISABLE_ALL_EMAIL: 'false' do
       assert_emails 1 do
         ProquestExportJob.perform_now(Thesis.all, Thesis.all)

test/mailers/batch_mailer_test.rb

@@ -36,11 +36,16 @@ class BatchMailerTest < ActionMailer::TestCase
       theses = [theses(:doctor), theses(:engineer)]
       export = ProquestExportBatch.new
       export_json = export.build_json(theses)
+      export_csv = export.build_budget_report(theses)
       export.proquest_export.attach(io: StringIO.new(export_json),
                                     filename: 'pq.json',
                                     content_type: 'application/json')
+      export.budget_report.attach(io: StringIO.new(export_csv),
+                                    filename: 'pq.csv',
+                                    content_type: 'application/csv')
       export.save
-      email = BatchMailer.proquest_export_email(export.proquest_export, theses)
+      email = BatchMailer.proquest_export_email(export.proquest_export, export.budget_report,  theses.count,
+                                                theses.count)
 
       # Send the email, then test that it got queued
       assert_emails 1 do
@@ -52,7 +57,9 @@ class BatchMailerTest < ActionMailer::TestCase
       assert_equal ['test@example.com'], email.to
       assert_equal 'ETD ProQuest export', email.subject
       assert_equal 'pq.json', email.attachments.first.filename
+      assert_equal 'pq.csv', email.attachments.second.filename
       assert_includes '2 theses', email.body.to_s
+      assert_includes '1 doctoral theses', email.body.to_s
     end
   end
 end

test/models/proquest_export_batch_test.rb

@@ -1,4 +1,5 @@
-require "test_helper"
+require 'test_helper'
+require 'csv'
 
 class ProquestExportBatchTest < ActiveSupport::TestCase
   test 'builds JSON with expected values' do
@@ -12,4 +13,31 @@ class ProquestExportBatchTest < ActiveSupport::TestCase
     assert_equal false, json_hash['records'].first['full_harvest']
     assert json_hash['records'].second['full_harvest']
   end
+
+  test 'builds CSV with expected headers' do
+    csv = CSV.parse(ProquestExportBatch.new.build_budget_report([theses(:ready_for_partial_export)]))
+    assert_equal ['author name(s)', 'department(s)', 'degree type(s)', 'degree period', 'handle', 'export date'], csv[0]
+  end
+
+  test 'CSV has expected values in the correct positions (single values)' do
+    thesis = theses(:ready_for_partial_export)
+    csv = CSV.parse(ProquestExportBatch.new.build_budget_report([thesis]))
+    assert_equal thesis.users.first.name, csv[1][0]
+    assert_equal thesis.departments.first.name_dw, csv[1][1]
+    assert_equal thesis.degrees.first.degree_type.name, csv[1][2]
+    assert_equal thesis.grad_date.to_s, csv[1][3]
+    assert_equal thesis.dspace_handle, csv[1][4]
+    assert_equal Date.today.strftime("%m-%d-%Y"), csv[1][5]
+  end
+
+  test 'CSV has expected values in the correct positions (multiple values)' do
+    thesis = theses(:budget_report_multiple)
+    csv = CSV.parse(ProquestExportBatch.new.build_budget_report([thesis]))
+    assert_equal thesis.users.map { |user| user.name }.join('; '), csv[1][0]
+    assert_equal thesis.departments.map { |dept| dept.name_dw }.join('; '), csv[1][1]
+    assert_equal thesis.degrees.map { |degree| degree.degree_type.name }.join('; '), csv[1][2]
+    assert_equal thesis.grad_date.to_s, csv[1][3]
+    assert_equal thesis.dspace_handle, csv[1][4]
+    assert_equal Date.today.strftime("%m-%d-%Y"), csv[1][5]
+  end
 end

test/models/report_test.rb

@@ -24,7 +24,7 @@ class ReportTest < ActiveSupport::TestCase
     # This includes a row of all zeros, which could only have been generated by populate_category
     assert r.index_data['license'][3][:data].values.all? { |v| v == 0 }
     # Also check that a row with expected data is represented accurately
-    assert_equal r.index_data['license'][1][:data].values, [3, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+    assert_equal r.index_data['license'][1][:data].values, [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
   end
 
   # pad_terms is a private method that gets called by each reporting row, to ensure that all rows have some value for
@@ -39,7 +39,7 @@ class ReportTest < ActiveSupport::TestCase
     returned_columns = r.index_data['summary'][3][:data].count
     assert_equal expected_terms, returned_columns
     # Test for a row of expected values
-    assert_equal r.index_data['summary'][3][:data].values, [0, 0, 0, 0, 0, 0, 1, 0, 0, 0]
+    assert_equal r.index_data['summary'][3][:data].values, [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0]
   end
 
   # ~~~~ Dashboard report
@@ -48,7 +48,7 @@ class ReportTest < ActiveSupport::TestCase
     result = r.index_data
     assert_equal Department.count, result['departments'].pluck(:label).length
     assert_includes result['departments'].pluck(:label), Department.first.name_dw
-    assert_equal result['departments'][0][:data].values, [0, 2, 0, 0, 0, 0, 0, 3, 1, 0]
+    assert_equal result['departments'][0][:data].values, [0, 2, 0, 0, 0, 0, 0, 3, 1, 0, 0]
   end
 
   test 'index includes summary data of authors not graduated' do
@@ -70,7 +70,7 @@ class ReportTest < ActiveSupport::TestCase
 
     r = Report.new
     result = r.index_data
-    assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 1, 0, 0]
+    assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0]
   end
 
   test 'authors not graduated summary data dedups theses with multiple files' do
@@ -92,7 +92,7 @@ class ReportTest < ActiveSupport::TestCase
     r = Report.new
     result = r.index_data
     assert_not_equal result['summary'][5][:data].values, [0, 2, 0, 0, 0, 0, 0, 0, 0, 0]
-    assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]
+    assert_equal result['summary'][5][:data].values, [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
   end
 
   # ~~~~ Term detail report