Merge pull request #113 from MITLibraries/106_submission_delete

106 submission delete

app/controllers/submissions_controller.rb

@@ -64,6 +64,12 @@ def resubmit
   def show
   end
 
+  def destroy
+    @submission.destroy
+    flash.notice = 'Submission has been deleted'
+    redirect_to submissions_path
+  end
+
   private
 
   def require_user

app/models/s3.rb

@@ -0,0 +1,26 @@
+class S3
+  # Returns a list of keys for content in S3 that has been abandoned
+  # @api private
+  # @return list of abandoned keys in the S3_BUCKET
+  # @note Content can become abandoned if a Submission event is not completed.
+  #   For example, a user that comes to the form and attaches a document but
+  #   does not complete the form Submission will leave abandoned documents in
+  #   S3. This cleans those up.
+  def self.abandoned_keys
+    keys.reject { |k| Submission.local_document_keys.include?(k) }
+  end
+
+  # Returns a list of all keys in the S3_BUCKET
+  # @api private
+  # @return list of all keys in the S3_BUCKET
+  def self.keys
+    S3_BUCKET.objects.map(&:key)
+  end
+
+  # Delete all S3 content not associated with any Submission
+  # @api private
+  # @return nil
+  def self.remove_abandoned_content
+    abandoned_keys.map { |key| S3_BUCKET.object(key).delete }
+  end
+end

app/models/submission.rb

@@ -32,6 +32,7 @@ class Submission < ActiveRecord::Base
   serialize :documents, JSON
   serialize :funders, JSON
   before_create :generate_uuid
+  before_destroy :delete_documents_from_s3
 
   SUBMITTABLE_FUNDERS = ['Department of Agriculture (USDA)',
                          'Department of Defense (DoD)',
@@ -112,6 +113,22 @@ def document_uri(document)
     end
   end
 
+  # Deletes all S3 content associated with a Submission
+  def delete_documents_from_s3
+    documents.each do |doc|
+      obj = Submission.document_key(doc)
+      S3_BUCKET.object(obj).delete
+    end
+  end
+
+  def self.local_document_keys
+    Submission.all.map(&:documents).flatten.map { |k| document_key(k) }
+  end
+
+  def self.document_key(doc)
+    "uploads#{doc.split('uploads').last}"
+  end
+
   # An array of funders that does not include UI only funders
   def funders_minus_ui_only_funders
     funders - ui_only_funders

app/views/submissions/index.html.erb

@@ -39,6 +39,8 @@ View:
           <%= link_to('View Sword Package', submission_package_path(sub)) %>
           --
           <%= link_to('Resubmit Package', resubmit_path(sub), method: :post) %>
+          --
+          <%= link_to('Delete Submission', submission_path(sub), method: :delete) %>
         </div>
       <% end %>
     </div>

config/environments/test.rb

@@ -3,7 +3,7 @@
   ENV['EMAIL_FROM'] = 'some_from@example.com'
   ENV['AWS_ACCESS_KEY_ID'] = 'fakeaccessid'
   ENV['AWS_SECRET_ACCESS_KEY'] = 'fakeaccesskey'
-  ENV['S3_BUCKET'] = 'fakebucket'
+  ENV['S3_BUCKET'] = 'testbucket'
   # The test environment is used exclusively to run your application's
   # test suite. You never need to work with it otherwise. Remember that
   # your test database is "scratch space" for the test suite and is wiped

config/routes.rb

@@ -1,5 +1,5 @@
 Rails.application.routes.draw do
-  resources :submissions, only: [:new, :create, :index, :show]
+  resources :submissions, only: [:new, :create, :index, :show, :destroy]
   get 'submissions/package/:id', to: 'submissions#package', as: :submission_package
   post 'submissions/resubmit/:id', to: 'submissions#resubmit', as: :resubmit
   post 'callbacks/status/:uuid', to: 'callbacks#status', as: :callback_submission_status

lib/tasks/submission_cleanup.rake

@@ -0,0 +1,19 @@
+namespace :submission do
+  desc 'Delete Deposited Submissions older than One Month'
+  task cleanup: :environment do
+    logger           = Logger.new(STDOUT)
+    logger.level     = Logger::INFO
+    old_submissions = Submission.where(status: 'approved')
+                                .where('updated_at < ?', 1.month.ago)
+    logger.debug("Deleting #{old_submissions.count} old Submissions")
+    old_submissions.each(&:destroy)
+  end
+
+  desc 'Delete abandoned S3 content'
+  task cleanup_abandoned: :environment do
+    logger           = Logger.new(STDOUT)
+    logger.level     = Logger::INFO
+    logger.info("Deleting #{S3.abandoned_keys.count} abandoned files")
+    S3.remove_abandoned_content
+  end
+end

test/controllers/submissions_controller_test.rb

@@ -134,4 +134,29 @@ class SubmissionsControllerTest < ActionController::TestCase
     get :show, id: submissions(:sub_one)
     assert_response :success
   end
+
+  test 'admin users can delete submission' do
+    sign_in users(:admin)
+    s = submissions(:sub_one)
+    delete :destroy, id: s
+    assert_raises(ActiveRecord::RecordNotFound) do
+      s.reload
+    end
+    assert_redirected_to submissions_path
+  end
+
+  test 'non admin users cannot delete submission' do
+    sign_in users(:one)
+    s = submissions(:sub_one)
+    delete :destroy, id: s
+    s.reload
+    assert_redirected_to root_path
+  end
+
+  test 'non-authenticated users cannot delete submission' do
+    s = submissions(:sub_one)
+    delete :destroy, id: s
+    s.reload
+    assert_redirected_to user_mit_oauth2_omniauth_authorize_path
+  end
 end

test/features/s3_test.rb

@@ -0,0 +1,57 @@
+require 'test_helper'
+
+class S3PagesTest < Capybara::Rails::TestCase
+  def setup
+    Capybara.current_driver = :poltergeist
+    Capybara.server_port = 5000
+    Capybara.server_host = 'localhost'
+    auth_setup
+    FileUtils.rm_f('tmp/69b9156a124c96bbdb55cad753810e14.zip')
+    FileUtils.rm_f('tmp/40550618d6b4d97792b0773c97207186.zip')
+  end
+
+  def teardown
+    super
+    Capybara.use_default_driver
+    auth_teardown
+    @sub.documents.map(&:remove!) if @sub
+  end
+
+  def base_valid_form
+    Timecop.freeze(Time.zone.local(1999))
+    mock_auth(users(:one))
+    visit new_submission_path
+    fill_in('Journal', with: 'Super Mega Journal')
+    fill_in('Title', with: 'Alphabetical Order is Good Enough')
+    check('Department of Energy (DOE)')
+    select '1999', from: 'submission[pub_date(1i)]'
+    select 'January', from: 'submission[pub_date(2i)]'
+    Timecop.return
+  end
+
+  test 'attaching a file stores it in s3' do
+    base_valid_form
+    attach_file('submission[documents][]',
+                File.absolute_path('./test/fixtures/a_pdf.pdf'))
+    assert_text('a_pdf.pdf uploaded')
+    click_on('Create Submission')
+    @sub = Submission.last
+    doc = @sub.documents.last
+    doc_uri = @sub.document_uri(doc).split("#{ENV['S3_BUCKET']}/").last
+    assert_equal(true, S3_BUCKET.object(doc_uri).exists?)
+  end
+
+  test 'deleting a Submission deletes files from s3' do
+    base_valid_form
+    attach_file('submission[documents][]',
+                File.absolute_path('./test/fixtures/a_pdf.pdf'))
+    assert_text('a_pdf.pdf uploaded')
+    click_on('Create Submission')
+    @sub = Submission.last
+    doc = @sub.documents.last
+    doc_uri = @sub.document_uri(doc).split("#{ENV['S3_BUCKET']}/").last
+    assert_equal(true, S3_BUCKET.object(doc_uri).exists?)
+    @sub.destroy
+    assert_equal(false, S3_BUCKET.object(doc_uri).exists?)
+  end
+end

test/fixtures/submissions.yml

@@ -45,3 +45,12 @@ sub_deposited:
     user: one
     uuid: 7648b505-af79-4a2b-abcb-2727f716100c
     status: 'approved'
+
+sub_new_approved:
+    title: 'another bowl of popcorn'
+    documents: "[\"//s3.amazonaws.com/mitquicksubmitdev/uploads/ea656123-f24f-4b12-9c78-17399338a6d6/b_pdf.pdf\", \"//s3.amazonaws.com/mitquicksubmitdev/uploads/89ce4e35-8573-4def-8114-8767365c954e/a_pdf.pdf\"]"
+    funders: '["Department of Energy (DOE)"]'
+    pub_date: <%= 1.year.ago %>
+    user: one
+    uuid: 7648b505-af79-4a2b-abcb-2727f716100c
+    status: 'approved'

test/lib/submission_cleanup_test.rb

@@ -0,0 +1,62 @@
+require 'test_helper'
+require 'rake'
+
+class RakeTaskTestCase < ActiveSupport::TestCase
+  def setup
+    Rake::Task.define_task :environment
+    QuickSubmit::Application.load_tasks
+
+    Timecop.freeze(Time.zone.local(1999))
+    @sub_old_approved = create_subs('old and approved', 'approved')
+    @sub_old_notapproved = create_subs('old and unapproved', '')
+    Timecop.return
+
+    @sub_new_approved = create_subs('new and approved', 'approved')
+    @sub_new_notapproved = create_subs('new and unapproved', '')
+  end
+
+  def teardown
+    Rake::Task.clear
+  end
+
+  def create_subs(title, status)
+    Submission.create(
+      title: title,
+      documents: ['b_pdf.pdf'],
+      user: users(:one),
+      funders: ['Department of Energy (DOE)'],
+      pub_date: 1.year.ago,
+      status: status,
+      handle: 'http://example.com')
+  end
+
+  test 'submissions with approved and older than 1 month are deleted' do
+    subs = Submission.count
+    Rake::Task['submission:cleanup'].invoke
+    assert_raises(ActiveRecord::RecordNotFound) do
+      @sub_old_approved.reload
+    end
+    assert_equal(subs - 1, Submission.count)
+  end
+
+  test 'submissions without approved and older than 1 month are not deleted' do
+    subs = Submission.count
+    Rake::Task['submission:cleanup'].invoke
+    @sub_old_notapproved.reload
+    assert_equal(subs - 1, Submission.count)
+  end
+
+  test 'submissions with approved and newer than one month are not deleted' do
+    subs = Submission.count
+    Rake::Task['submission:cleanup'].invoke
+    @sub_new_approved.reload
+    assert_equal(subs - 1, Submission.count)
+  end
+
+  test 'submissions without approved and newer than 1 month are not deleted' do
+    subs = Submission.count
+    Rake::Task['submission:cleanup'].invoke
+    @sub_new_notapproved.reload
+    assert_equal(subs - 1, Submission.count)
+  end
+end

test/models/s3_test.rb

@@ -0,0 +1,47 @@
+require 'test_helper'
+
+class S3Test < ActiveSupport::TestCase
+  def setup
+    S3_BUCKET.objects.map(&:delete)
+    S3_BUCKET.put_object(key: 'uploads/12345')
+    S3_BUCKET.put_object(key: 'uploads/45678')
+    S3_BUCKET.put_object(key: 'uploads/asdfa')
+  end
+
+  def teardown
+    S3_BUCKET.object('uploads/12345').delete
+    S3_BUCKET.object('uploads/45678').delete
+    S3_BUCKET.object('uploads/asdfa').delete
+  end
+
+  test 'keys' do
+    assert_equal(['uploads/12345', 'uploads/45678', 'uploads/asdfa'], S3.keys)
+  end
+
+  test 'abandoned_keys' do
+    Submission.create(
+      title: 'title',
+      documents: '//s3.amazonaws.com/mitquicksubmitdev/uploads/12345',
+      user: users(:one),
+      funders: ['Department of Energy (DOE)'],
+      pub_date: 1.year.ago,
+      status: 'status',
+      handle: 'http://example.com')
+    assert_equal(['uploads/45678', 'uploads/asdfa'], S3.abandoned_keys)
+  end
+
+  test 'remove_abandoned_content' do
+    Submission.create(
+      title: 'title',
+      documents: '//s3.amazonaws.com/mitquicksubmitdev/uploads/12345',
+      user: users(:one),
+      funders: ['Department of Energy (DOE)'],
+      pub_date: 1.year.ago,
+      status: 'status',
+      handle: 'http://example.com')
+    assert_equal(['uploads/45678', 'uploads/asdfa'], S3.abandoned_keys)
+    S3.remove_abandoned_content
+    assert_equal([], S3.abandoned_keys)
+    assert_equal(['uploads/12345'], S3.keys)
+  end
+end

test/models/submission_test.rb

@@ -207,12 +207,12 @@ def cleaup_sword_files(sub)
   test 'document_uri with localhost' do
     sub = submissions(:sub_one)
     assert_equal(sub.document_uri('//localhost/popcorn'),
-                 'http://localhost:10001/fakebucket/popcorn')
+                 'http://localhost:10001/testbucket/popcorn')
   end
 
   test 'document_uri with spaces' do
     sub = submissions(:sub_one)
     assert_equal(sub.document_uri('//localhost/popcorn says'),
-                 'http://localhost:10001/fakebucket/popcorn%20says')
+                 'http://localhost:10001/testbucket/popcorn%20says')
   end
 end

test/test_helper.rb

@@ -4,7 +4,9 @@
   SimpleCov::Formatter::HTMLFormatter,
   Coveralls::SimpleCov::Formatter
 ]
-SimpleCov.start('rails')
+SimpleCov.start('rails') do
+  add_filter '/lib/tasks/'
+end
 ENV['RAILS_ENV'] ||= 'test'
 require File.expand_path('../../config/environment', __FILE__)
 require 'rails/test_help'
@@ -60,6 +62,7 @@ class TestCase
 module Minitest
   class Spec
     before :each do
+      FileUtils.rm_r('tmp/s3/testbucket')
       DatabaseCleaner.start
     end