Merge pull request #36 from MITLibraries/21_queue_on_submit

21 queue on submit

.rubocop.yml

@@ -1,11 +1,15 @@
 Documentation:
   Enabled: false
 
+Rails:
+  Enabled: true
+
 AllCops:
-  RunRailsCops: true
+  TargetRubyVersion: 2.3
   Exclude:
     - db/**/*
     - config/**/*
+    - bin/delayed_job
 
 ClassLength:
   Exclude:

Gemfile

@@ -5,6 +5,7 @@ gem 'rails', '4.2.5.1'
 gem 'aws-sdk', '~> 2'
 gem 'bootstrap_form'
 gem 'cancancan'
+gem 'delayed_job_active_record'
 gem 'devise'
 gem 'http_logger'
 gem 'jquery-rails'
@@ -47,7 +48,7 @@ group :test do
   gem 'minitest-reporters'
   gem 'minitest-rails'
   gem 'minitest-rails-capybara'
-  gem 'poltergeist'
+  gem 'poltergeist', git: 'git://github.com/teampoltergeist/poltergeist.git'
   gem 'vcr'
   gem 'webmock'
 end

Gemfile.lock

@@ -1,3 +1,13 @@
+GIT
+  remote: git://github.com/teampoltergeist/poltergeist.git
+  revision: c44714102f1e2c997004b614a27e69ef46307154
+  specs:
+    poltergeist (1.8.1)
+      capybara (~> 2.1)
+      cliver (~> 0.3.1)
+      multi_json (~> 1.0)
+      websocket-driver (>= 0.2.0)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -77,6 +87,11 @@ GEM
       safe_yaml (~> 1.0.0)
     database_cleaner (1.5.1)
     debug_inspector (0.0.2)
+    delayed_job (4.1.1)
+      activesupport (>= 3.0, < 5.0)
+    delayed_job_active_record (4.1.0)
+      activerecord (>= 3.0, < 5)
+      delayed_job (>= 3.0, < 5)
     devise (3.5.5)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
@@ -174,11 +189,6 @@ GEM
       rack
       rake (>= 0.8.1)
     pg (0.18.4)
-    poltergeist (1.8.1)
-      capybara (~> 2.1)
-      cliver (~> 0.3.1)
-      multi_json (~> 1.0)
-      websocket-driver (>= 0.2.0)
     powerpack (0.1.1)
     rack (1.6.4)
     rack-test (0.6.3)
@@ -296,6 +306,7 @@ DEPENDENCIES
   cancancan
   coveralls
   database_cleaner
+  delayed_job_active_record
   devise
   fakes3
   http_logger
@@ -310,7 +321,7 @@ DEPENDENCIES
   omniauth-oauth2
   passenger
   pg
-  poltergeist
+  poltergeist!
   rails (= 4.2.5.1)
   rails_12factor
   rest-client

Procfile

@@ -1 +1,2 @@
-web: bundle exec passenger start -p $PORT --max-pool-size 3
+web: bundle exec passenger start -p $PORT --max-pool-size 3
+worker: bundle exec rake jobs:work

app/controllers/submissions_controller.rb

@@ -4,11 +4,11 @@ class SubmissionsController < ApplicationController
   load_and_authorize_resource
 
   def index
-    if current_user.admin?
-      @submissions = Submission.all.order(created_at: :desc)
-    else
-      @submissions = current_user.submissions.order(created_at: :desc)
-    end
+    @submissions = if current_user.admin?
+                     Submission.all.order(created_at: :desc)
+                   else
+                     current_user.submissions.order(created_at: :desc)
+                   end
   end
 
   def new
@@ -20,6 +20,7 @@ def create
     @submission = Submission.new(submission_params)
     @submission.user = current_user
     if @submission.save
+      SwordSubmitJob.perform_later(@submission, callback_uri)
       flash.notice = 'Your Submission is now in progress'
       redirect_to submissions_path
     else
@@ -28,7 +29,9 @@ def create
   end
 
   def package
-    send_file(Submission.find_by_id(params[:id]).sword_path)
+    @submission = Submission.find_by_id(params[:id])
+    @submission.to_sword_package(callback_uri)
+    send_file(@submission.sword_path)
   end
 
   private

app/jobs/sword_submit_job.rb

@@ -1,20 +1,22 @@
 class SwordSubmitJob < ActiveJob::Base
   queue_as :default
 
-  def perform(submission)
+  def perform(submission, callback_uri)
     @submission = submission
+    @callback_uri = callback_uri
     process_submission
     @submission.send_status_email
   end
 
   def process_submission
-    callback_uri = "http://example.com/callbacks/status/#{@submission.uuid}'"
-    sword = Sword.new(@submission, callback_uri)
+    sword = Sword.new(@submission, @callback_uri)
     begin
       sword.deposit
       read_sword_response(sword)
     rescue RestClient::Unauthorized
       @submission.status = 'failed'
+    rescue RestClient::RequestFailed
+      @submission.status = 'failed'
     end
     @submission.save
   end

app/models/submission.rb

@@ -71,7 +71,8 @@ def to_sword_package(callback)
 
   def document_uri(document)
     if document.include?('localhost')
-      "http:#{document.gsub('localhost', 'localhost:10001')}"
+      swap = "localhost:10001/#{ENV['S3_BUCKET']}/"
+      "http:#{document.gsub('localhost/', swap)}"
     else
       "https:#{document}"
     end

bin/delayed_job

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'config', 'environment'))
+require 'delayed/command'
+Delayed::Command.new(ARGV).daemonize

config/application.rb

@@ -20,6 +20,8 @@ class Application < Rails::Application
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
     # config.i18n.default_locale = :de
 
+    config.active_job.queue_adapter = :delayed_job
+
     # Do not swallow errors in after_commit/after_rollback callbacks.
     config.active_record.raise_in_transactional_callbacks = true
   end

db/migrate/20160202152401_create_delayed_jobs.rb

@@ -0,0 +1,22 @@
+class CreateDelayedJobs < ActiveRecord::Migration
+  def self.up
+    create_table :delayed_jobs, force: true do |table|
+      table.integer :priority, default: 0, null: false # Allows some jobs to jump to the front of the queue
+      table.integer :attempts, default: 0, null: false # Provides for retries, but still fail eventually.
+      table.text :handler,                 null: false # YAML-encoded string of the object that will do work
+      table.text :last_error                           # reason for last failure (See Note below)
+      table.datetime :run_at                           # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+      table.datetime :locked_at                        # Set when a client is working on this object
+      table.datetime :failed_at                        # Set when all retries have failed (actually, by default, the record is deleted instead)
+      table.string :locked_by                          # Who is working on this object (if locked)
+      table.string :queue                              # The name of the queue this job is in
+      table.timestamps null: true
+    end
+
+    add_index :delayed_jobs, [:priority, :run_at], name: "delayed_jobs_priority"
+  end
+
+  def self.down
+    drop_table :delayed_jobs
+  end
+end

db/schema.rb

@@ -11,7 +11,23 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20151211200848) do
+ActiveRecord::Schema.define(version: 20160202152401) do
+
+  create_table "delayed_jobs", force: :cascade do |t|
+    t.integer  "priority",   default: 0, null: false
+    t.integer  "attempts",   default: 0, null: false
+    t.text     "handler",                null: false
+    t.text     "last_error"
+    t.datetime "run_at"
+    t.datetime "locked_at"
+    t.datetime "failed_at"
+    t.string   "locked_by"
+    t.string   "queue"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  add_index "delayed_jobs", ["priority", "run_at"], name: "delayed_jobs_priority"
 
   create_table "submissions", force: :cascade do |t|
     t.integer  "user_id"

test/controllers/submissions_controller_test.rb

@@ -64,9 +64,11 @@ class SubmissionsControllerTest < ActionController::TestCase
 
   test 'admin users can download package' do
     sub = submissions(:sub_one)
-    File.write(sub.sword_path, 'Fakey fake fake')
     sign_in users(:admin)
-    get :package, id: sub
+    VCR.use_cassette('read_a_and_b_files_from_s3',
+                     preserve_exact_body_bytes: true) do
+      get :package, id: sub
+    end
     assert_response :success
     FileUtils.rm_f(sub.sword_path)
   end

test/jobs/sword_submit_job_test.rb

@@ -3,8 +3,9 @@
 class SwordSubmitJobTest < ActiveJob::TestCase
   test 'successful sword submission with workflow enabled' do
     sub = submissions(:sub_one)
+    callback_uri = "http://example.com/callbacks/status/#{sub.uuid}"
     VCR.use_cassette('workflow_submission', preserve_exact_body_bytes: true) do
-      SwordSubmitJob.perform_now(sub)
+      SwordSubmitJob.perform_now(sub, callback_uri)
     end
     sub.reload
     assert_equal('in review queue', sub.status)
@@ -13,8 +14,9 @@ class SwordSubmitJobTest < ActiveJob::TestCase
 
   test 'successful sword submission with no workflow enabled' do
     sub = submissions(:sub_one)
+    callback_uri = "http://example.com/callbacks/status/#{sub.uuid}"
     VCR.use_cassette('deposit', preserve_exact_body_bytes: true) do
-      SwordSubmitJob.perform_now(sub)
+      SwordSubmitJob.perform_now(sub, callback_uri)
     end
     sub.reload
     assert_equal('deposited', sub.status)
@@ -23,8 +25,9 @@ class SwordSubmitJobTest < ActiveJob::TestCase
 
   test 'invalid sword credentials' do
     sub = submissions(:sub_one)
+    callback_uri = "http://example.com/callbacks/status/#{sub.uuid}"
     VCR.use_cassette('invalid_credentials', preserve_exact_body_bytes: true) do
-      SwordSubmitJob.perform_now(sub)
+      SwordSubmitJob.perform_now(sub, callback_uri)
     end
     sub.reload
     assert_equal('failed', sub.status)