Implement background worker for big uploaded files

Gemfile

@@ -78,6 +78,9 @@ gem 'ruby-nmap'
 # Command line interface
 gem 'thor', '~> 0.19.1'
 
+# Redis-based background worker
+gem 'resque', require: 'resque/status_server'
+gem 'resque-status'
 
 # ------------------------------------------------------------------ Test & Dev
 group :development do

Gemfile.lock

@@ -182,6 +182,7 @@ GEM
     mime-types (2.4.3)
     mini_portile (0.6.2)
     minitest (5.5.1)
+    mono_logger (1.1.0)
     multi_json (1.10.1)
     multipart-post (2.0.0)
     nokogiri (1.6.5)
@@ -191,6 +192,8 @@ GEM
       method_source (~> 0.8.1)
       slop (~> 3.4)
     rack (1.5.2)
+    rack-protection (1.5.3)
+      rack
     rack-test (0.6.3)
       rack (>= 1.0)
     rails (4.1.9)
@@ -216,6 +219,17 @@ GEM
       ffi (>= 0.5.0)
     rdoc (4.1.1)
       json (~> 1.4)
+    redis (3.2.1)
+    redis-namespace (1.5.1)
+      redis (~> 3.0, >= 3.0.4)
+    resque (1.25.2)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    resque-status (0.4.3)
+      resque (~> 1.19)
     rprogram (0.3.2)
     rspec (3.0.0)
       rspec-core (~> 3.0.0)
@@ -252,6 +266,10 @@ GEM
     simple_form (3.1.0)
       actionpack (~> 4.0)
       activemodel (~> 4.0)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
     slop (3.5.0)
     spring (1.1.3)
     sprockets (2.11.0)
@@ -275,6 +293,8 @@ GEM
     uglifier (2.5.1)
       execjs (>= 0.3.0)
       json (>= 1.8.0)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
     vulndbhq (0.1.0)
       faraday (~> 0.8)
       multi_json (~> 1.3)
@@ -309,6 +329,8 @@ DEPENDENCIES
   jbuilder (~> 2.0)
   nokogiri (= 1.6.5)
   rails (= 4.1.9)
+  resque
+  resque-status
   rspec-rails
   ruby-nmap
   sass-rails (~> 4.0.3)

app/workers/base_worker.rb

@@ -0,0 +1,22 @@
+# Base class from which other Resque workers inherit. It makes sure that all DB
+# connections are active (they could have timed out).
+#
+# See:
+#   http://axonflux.com/resque-to-the-rescue-but-a-gotcha-dont-forget
+#
+
+class BaseWorker
+  # All our workers will make use of the resque-status plugin
+  include Resque::Plugins::Status
+
+  # Main method for this class, it re-connects any stale AR connections and
+  # delegates to the #perform_delegate() method to perform the actual work.
+  def perform(*args)
+    ActiveRecord::Base.clear_active_connections!
+    perform_delegate(*args)
+  end
+
+  # Implementing workers will override this method.
+  def perform_delegate(*args)
+  end
+end

app/workers/upload_processor.rb

@@ -0,0 +1,38 @@
+# Resque background worker to process files uploaded via the Upload Manager.
+#
+# This worker requires the following options:
+#   plugin: Upload Plugin that needs to process the file,
+#     file: Full path to the file to be processed,
+#      uid: uuid for this job
+
+class UploadProcessor < BaseWorker
+  @queue = :dradis_upload
+
+  def perform_delegate
+    file        = options['file']
+    plugin_name = options['plugin']
+    uid         = options['uid']
+
+    logger = Dradis::Core::Log.new(uid: uid)
+
+    logger.write{ "Running Ruby version %s" % RUBY_VERSION }
+    logger.write{ 'Worker process starting background task.' }
+
+    plugin = plugin_name.constantize
+
+    # Detect new-style gemified plugins
+    if plugin::constants::include?(:Importer)
+      content_service = Dradis::Plugins::ContentService.new(plugin: plugin)
+      template_service = Dradis::Plugins::TemplateService.new(plugin: plugin)
+
+      importer = plugin::Importer.new(
+                  logger: logger,
+         content_service: content_service,
+        template_service: template_service
+      )
+
+      importer.import(file: file)
+    end
+    logger.write{ 'Worker process completed.' }
+  end
+end

config/initializers/resque.rb

@@ -0,0 +1,5 @@
+Resque::Plugins::Status::Hash.expire_in = (24 * 60 * 60) # 24hrs in seconds
+
+# Resque::Server.use(Rack::Auth::Basic) do |user, password|
+#   user == 'admin' && password == 'dradispro'
+# end

engines/frontend/app/controllers/dradis/frontend/upload_controller.rb

@@ -106,19 +106,16 @@ def process_upload_inline(args={})
       end
 
       def process_upload_background(args={})
-        raise 'unimplemented!'
         attachment = args[:attachment]
 
         # Is it worth making Bj compatible?
         # Bj.submit "ruby script/rails runner lib/upload_processing_job.rb %s \"%s\" %s" % [ params[:uploader], attachment.fullpath, params[:item_id] ]
 
-        @job_id = UploadProcessor.create(
-                                        file: attachment.fullpath,
-                                        plugin: params[:uploader],
-                                        project_id: @project.id,
-                                        uid: item_id)
+        @job_id = UploadProcessor.create file: attachment.fullpath,
+                                         plugin: params[:uploader],
+                                         uid: params[:item_id]
 
-        job_logger.write("Enqueueing job to start in the background. Job id is #{item_id}")
+        job_logger.write("Enqueueing job to start in the background. Job id is #{ @job_id }")
       end
 
       # Ensure that the requested :uploader is valid
@@ -135,4 +132,4 @@ def validate_uploader()
     end
 
   end
-end
+end

lib/tasks/resque.rake

@@ -0,0 +1,8 @@
+require 'resque/tasks'
+
+# We may want to include only the stuff we need, instead of loading the full
+# See:
+#   http://railscasts.com/episodes/271-resque
+task "resque:setup" => :environment do
+  ENV['QUEUE'] = 'dradis_upload,dradis_export'
+end