Merge 6791dad into e026294

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+ - 2016-08-03
+   - Feature: Analysis Jobs items integration. Analysis jobs have been setup and their complete workflows tested and 
+     integrated. See [#300](https://github.com/QutBioacoustics/baw-server/pull/300)
+
 ## [Release 0.19.2](https://github.com/QutBioacoustics/baw-server/releases/tag/0.19.2) (2016-06-26)
 
  - 2016-06-21

Gemfile

@@ -96,7 +96,11 @@ gem 'jc-validates_timeliness', '~> 3.1.1'
 
 gem 'enumerize', '~> 1.0'
 gem 'uuidtools', '~> 2.1.5'
-gem 'acts_as_paranoid', git: 'https://github.com/ActsAsParanoid/acts_as_paranoid.git', branch: :master, ref: 'ab31723bc1'
+gem 'acts_as_paranoid', git: 'https://github.com/ActsAsParanoid/acts_as_paranoid.git', branch: :master, ref: 'c2db19554ddaedcac0a2b8d6a0563dea83c972c5'
+
+
+# for state machines
+gem 'aasm'
 
 # SETTINGS
 # -------------------------------------
@@ -204,4 +208,9 @@ end
 group :test do
   gem 'webmock', '~> 2.1.0'
   gem 'shoulda-matchers', '< 3.0.0', require: false
+
+  gem 'rspec-mocks', '~>3.5.0'
+
+  # use to mock time in tests - currently not needed
+  #gem 'timecop'
 end

Gemfile.lock

@@ -1,12 +1,12 @@
 GIT
   remote: https://github.com/ActsAsParanoid/acts_as_paranoid.git
-  revision: ab31723bc11c82bac144cfabd4631b26307bfde1
-  ref: ab31723bc1
+  revision: c2db19554ddaedcac0a2b8d6a0563dea83c972c5
+  ref: c2db19554ddaedcac0a2b8d6a0563dea83c972c5
   branch: master
   specs:
-    acts_as_paranoid (0.5.0.beta2)
-      activerecord (~> 4.0)
-      activesupport (~> 4.0)
+    acts_as_paranoid (0.5.0.rc1)
+      activerecord (>= 4.0, < 5.1)
+      activesupport (>= 4.0, < 5.1)
 
 GIT
   remote: https://github.com/QutBioacoustics/baw-audio-tools.git
@@ -52,6 +52,7 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    aasm (4.11.0)
     actionmailer (4.2.7)
       actionpack (= 4.2.7)
       actionview (= 4.2.7)
@@ -410,7 +411,7 @@ GEM
       rack-protection (~> 1.4)
       tilt (>= 1.3, < 3)
     slop (3.6.0)
-    sprockets (3.6.3)
+    sprockets (3.5.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.1.1)
@@ -460,6 +461,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aasm
   activesupport-json_encoder!
   acts_as_paranoid!
   baw-audio-tools!
@@ -512,6 +514,7 @@ DEPENDENCIES
   resque-status (~> 0.5.0)
   role_model (~> 0.8.1)
   rspec (~> 3.5.0)
+  rspec-mocks (~> 3.5.0)
   rspec-rails (~> 3.5.0)
   rspec_api_documentation (~> 4.8.0)
   rubocop (~> 0.41.0)

app/controllers/analysis_jobs_controller.rb

@@ -41,11 +41,12 @@ def create
     do_set_attributes(analysis_job_create_params)
     do_authorize_instance
 
+    # runs first step in analysis_job workflow (`initialize_workflow`) and then saves
     if @analysis_job.save
 
       # now create and enqueue job items (which updates status attributes again)
       # needs to be called after save as it makes use of the analysis_job id.
-      @analysis_job.begin_work(current_user)
+      @analysis_job.prepare!
 
       respond_create_success
     else
@@ -60,7 +61,16 @@ def update
     do_load_resource
     do_authorize_instance
 
-    if @analysis_job.update_attributes(analysis_job_update_params)
+    parameters = analysis_job_update_params
+
+    # allow the API to transition this analysis job to a new state.
+    # Used for suspending and resuming an analysis_job
+    if parameters.key?(:overall_status)
+      @analysis_job.transition_to_state(parameters[:overall_status].to_sym)
+      parameters = parameters.except(:overall_status)
+    end
+
+    if @analysis_job.update_attributes(parameters)
       respond_show
     else
       respond_change_fail
@@ -74,13 +84,23 @@ def destroy
     do_load_resource
     do_authorize_instance
 
-    @analysis_job.destroy
-    add_archived_at_header(@analysis_job)
+    # only allow deleting from suspended or completed states
+    can_delete = @analysis_job.completed? || @analysis_job.suspended?
 
-    # TODO: delete pending analysis jobs from worker message queue
-    # TODO: change all pending analysis_job_items to :cancelled
+    # also allow from processing, since we can suspend
+    if @analysis_job.processing? && @analysis_job.may_suspend?
+      can_delete = true
+      @analysis_job.suspend!
+    end
 
-    respond_destroy
+    unless can_delete
+      respond_error(:conflict, "Cannot be deleted while `overall_status` is `#{@analysis_job.overall_status}`")
+    else
+      @analysis_job.destroy
+      add_archived_at_header(@analysis_job)
+
+      respond_destroy
+    end
   end
 
   # GET|POST /analysis_jobs/filter
@@ -127,7 +147,7 @@ def analysis_job_create_params
   def analysis_job_update_params
     # Only name and description can be updated via API.
     # Other properties are updated by the processing system.
-    params.require(:analysis_job).permit(:name, :description)
+    params.require(:analysis_job).permit(:name, :description, :overall_status)
   end
 
   def get_analysis_jobs

app/controllers/analysis_jobs_items_controller.rb

@@ -73,9 +73,41 @@ def update
     end
 
     do_load_resource
+    do_get_analysis_job
     do_authorize_instance
 
-    if @analysis_jobs_item.update_attributes(analysis_jobs_item_update_params)
+    parameters = analysis_jobs_item_update_params
+    desired_state = parameters[:status].to_sym
+
+    client_cancelled = desired_state == :cancelled
+    should_cancel = @analysis_jobs_item.may_confirm_cancel?(desired_state)
+    valid_transition = @analysis_jobs_item.may_transition_to_state(desired_state)
+
+    if valid_transition
+      @analysis_jobs_item.transition_to_state(desired_state)
+    elsif should_cancel
+      @analysis_jobs_item.confirm_cancel(desired_state)
+    end
+
+    saved = @analysis_jobs_item.save
+
+    if saved
+      # update progress statistics and check if job has been completed
+      @analysis_job.check_progress
+    end
+
+    if should_cancel && !client_cancelled && saved
+      # If someone tried to :cancelling-->:working instead of :cancelling-->:cancelled then it is an error
+      # However if client :cancelled when we expected :cancelling-->:cancelled then well behaved
+      respond_error(
+          :unprocessable_entity,
+          "This entity has been cancelled - can not set new state to `#{desired_state}`")
+
+    elsif !valid_transition
+      respond_error(
+          :unprocessable_entity,
+          "Cannot transition from `#{@analysis_jobs_item.status}` to `#{desired_state}`")
+    elsif saved
       respond_show
     else
       respond_change_fail
@@ -130,7 +162,9 @@ def do_get_opts
   end
 
   def do_get_analysis_job
-    @analysis_job = AnalysisJob.find(@analysis_job_id) unless @is_system_job
+    # We use with deleted here because we always need to be able to load AnalysisJobsItem even if the parent
+    # AnalysisJob has been deleted.
+    @analysis_job = AnalysisJob.with_deleted.find(@analysis_job_id) unless @is_system_job
   end
 
   def get_query
@@ -362,7 +396,7 @@ def normalised_name(path)
   end
 
   def normalise_path(path)
-    analysis_base_paths = BawWorkers::Config.analysis_cache_helper.existing_dirs
+    analysis_base_paths = BawWorkers::Config.analysis_cache_helper.possible_dirs
     matching_base_path = analysis_base_paths.select { |abp| path.start_with?(abp) }
     if matching_base_path.size == 1
       path_without_base = path.gsub(/#{matching_base_path[0].gsub('/', '\/')}\/[^\/]+\/[^\/]+\/[^\/]+\/?/, '')

app/controllers/audio_recordings_controller.rb

@@ -295,12 +295,14 @@ def audio_recording_params
     ]
 
     # can't permit arbitrary hash
-    # https://github.com/rails/rails/issues/9454#issuecomment-14167664
+    # ~~https://github.com/rails/rails/issues/9454#issuecomment-14167664~~
+    # http://stackoverflow.com/questions/19172893/rails-hashes-with-unknown-keys-and-strong-parameters/24752108#24752108
     # http://guides.rubyonrails.org/action_controller_overview.html#more-examples
     # add arbitrary hash for notes manually
+    properties = params[:audio_recording].delete(:notes)
     params.require(:audio_recording).permit(*permitted_attributes).tap do |allowed_params|
-      if params[:audio_recording][:notes]
-        allowed_params[:notes] = params[:audio_recording][:notes]
+      if properties
+        allowed_params[:notes] = properties
       end
     end
   end

app/controllers/permissions_controller.rb

@@ -100,13 +100,14 @@ def permission_params
   end
 
   def update_permissions_params
-    params.permit(project_wide: [:logged_in, :anonymous], per_user: [:none, :reader, :writer, :owner])
+    params.slice(:project_wide, :per_user).permit(project_wide: [:logged_in, :anonymous], per_user: [:none, :reader, :writer, :owner])
   end
 
   def update_permissions
+    return nil if !params.include?(:project_wide) && !params.include?(:per_user)
+
     request_params = update_permissions_params
 
-    return nil if !request_params.include?(:project_wide) && !request_params.include?(:per_user)
     if request_params.include?(:project_wide) && request_params[:project_wide].include?(:logged_in)
       permission = Permission.where(project: @project, user: nil, allow_logged_in: true, allow_anonymous: false).first
       permission = Permission.new(project: @project, user: nil, allow_logged_in: true, allow_anonymous: false) if permission.blank?

app/controllers/saved_searches_controller.rb

@@ -75,11 +75,13 @@ def filter
 
   def saved_search_params
     # can't permit arbitrary hash
-    # https://github.com/rails/rails/issues/9454#issuecomment-14167664
+    # ~~https://github.com/rails/rails/issues/9454#issuecomment-14167664~~
+    # http://stackoverflow.com/questions/19172893/rails-hashes-with-unknown-keys-and-strong-parameters/24752108#24752108
     # add arbitrary hash for stored_query manually
+    properties = params[:saved_search].delete(:stored_query)
     params.require(:saved_search).permit(:id, :name, :description).tap do |allowed_params|
-      if params[:saved_search][:stored_query]
-        allowed_params[:stored_query] = params[:saved_search][:stored_query]
+      if properties
+        allowed_params[:stored_query] = properties
       end
     end
   end

app/mailers/analysis_job_mailer.rb

@@ -0,0 +1,45 @@
+class AnalysisJobMailer < ActionMailer::Base
+  default from: Settings.mailer.emails.sender_address
+
+  # @param [AnalysisJob] analysis_job
+  # @param [ActionDispatch::Request] rails_request
+  def new_job_message(analysis_job, rails_request)
+    send_message(analysis_job, rails_request, 'New analysis job', 'analysis_job_new_message')
+  end
+
+  # @param [AnalysisJob] analysis_job
+  # @param [ActionDispatch::Request] rails_request
+  def completed_job_message(analysis_job, rails_request)
+    send_message(analysis_job, rails_request, 'Completed analysis job', 'analysis_job_complete_message')
+  end
+
+  # @param [AnalysisJob] analysis_job
+  # @param [ActionDispatch::Request] rails_request
+  def retry_job_message(analysis_job, rails_request)
+    send_message(analysis_job, rails_request, 'Retrying analysis job', 'analysis_job_retry_message')
+  end
+
+  private
+
+  # Construct the email.
+  # @param [AnalysisJob] analysis_job
+  # @param [ActionDispatch::Request] rails_request
+  # @param [string] subject_prefix
+  # @param [string] template_name
+  def send_message(analysis_job, rails_request, subject_prefix, template_name)
+    user_emails = User.find([analysis_job.creator_id, analysis_job.updater_id]).map {|u| u.email }.uniq
+
+    @info = {
+        analysis_job: analysis_job,
+        datestamp: Time.zone.now.utc.iso8601
+    }
+
+    # email gets sent to required recipients (creator, updater, admins)
+    mail(
+        to: user_emails.concat(Settings.mailer.emails.required_recipients).uniq,
+        subject: "#{Settings.mailer.emails.email_prefix} [#{subject_prefix}] #{@info[:analysis_job].name}.",
+        template_path: 'analysis_jobs_mailer',
+        template_name: template_name)
+  end
+
+end

app/models/ability.rb

@@ -128,7 +128,7 @@ def for_harvester
 
     # omitted: :new, :create,
     # applied by default: :index, :show, :filter
-    can [ :update ], AnalysisJobsItem
+    can [ :show, :update ], AnalysisJobsItem
   end
 
   def create_guest_user