Merge branch 'develop' into feature/bulk_delete

Gemfile

@@ -42,16 +42,19 @@ gem 'uuidtools'
 gem 'kaminari'
 gem 'bootstrap-sass', '~> 3.3.7'
 
-# These gems aren't required directly but is required in dependencies and needs specific updating due to a security warning
+# These gems aren't required directly but are required in various dependencies and
+# needed specific updating due to security warnings
 gem 'mail', '>= 2.7.0'
 gem 'sprockets', '>= 3.7.2'
+gem 'loofah', '~> 2.2.3'
 
 # S3 connector
 #gem 'aws-sdk-core'
 
 # Graylog logging gems
 gem 'gelf'
 gem 'lograge'
+gem 'logged'
 gem 'activerecord-nulldb-adapter'
 gem 'puma'
 

Gemfile.lock

@@ -145,12 +145,14 @@ GEM
       kaminari-core (= 1.1.1)
     kaminari-core (1.1.1)
     libv8 (3.16.14.19)
+    logged (0.1.2)
+      railties (>= 4.0, < 6.0)
     lograge (0.10.0)
       actionpack (>= 4)
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.2.2)
+    loofah (2.2.3)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
@@ -168,7 +170,7 @@ GEM
     minitest (5.11.3)
     multi_json (1.13.1)
     nio4r (2.3.1)
-    nokogiri (1.8.4)
+    nokogiri (1.8.5)
       mini_portile2 (~> 2.3.0)
     orm_adapter (0.5.0)
     pg (1.1.3)
@@ -335,7 +337,9 @@ DEPENDENCIES
   jquery-rails
   jquery-ui-rails
   kaminari
+  logged
   lograge
+  loofah (~> 2.2.3)
   mail (>= 2.7.0)
   meta_request (= 0.6.0)
   pg (= 1.1.3)

Makefile

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Docker_start.sh
+# Script to provision a development environment
+# with docker-ce and start pharos in a container as per docker-compose.yml
+
+# 1. ID OS - Linux or OSX
+# 2. If OSX, install homebrew
+# 3. Install Docker-CE on osx (brew cask install docker/linux: apt-get install docker)
+# 4. Run build.sh to build the latest version of Pharos
+# 5. docker-compose up -f docker-compose-dev.yml
+# 6. Print out URL for user
+
+# -  make restart: docker-compose up -d -f docker-compose-dev.yml
+#
+
+registry = registry.gitlab.com/aptrust
+repository = container-registry
+name = pharos
+version = latest
+tag = $(name):$(version)
+revision=$(shell git rev-parse --short=2 HEAD)
+
+# HELP
+# This will output the help for each task
+# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+.PHONY: help build publish
+
+help: ## This help.
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+.DEFAULT_GOAL := help
+
+revision: ## Show me the git hash
+	echo "$(revision)"
+
+build: ## Build the Pharos container
+	docker build -t $(tag) -t $(name):$(revision) -t $(registry)/$(repository)/$(tag) .
+
+up: ## Start containers for Pharos, Postgresql, Nginx
+	docker-compose up -d
+
+run: ## Just run Pharos in foreground
+	docker run aptrust/$(tag) .
+
+publish:
+	docker tag aptrust/pharos registry.gitlab.com/aptrust/container-registry/pharos && \
+	docker push registry.gitlab.com/aptrust/container-registry/pharos
+
+# Docker release - build, tag and push the container
+release: build publish ## Make a release by building and publishing the `{version}` as `latest` tagged containers to Gitlab
+
+
+push: ## Push the Docker image up to the registry
+	docker push  $(registry)/$(repository)/$(tag)
+
+clean: ## Clean the generated/compiles files
+

app/assets/javascripts/requeue.js

@@ -1,5 +1,6 @@
 function restoreRequeue(){
     $('#restore_form').removeClass('hidden');
+    $('#requeue-btn').addClass('hidden');
     $('#restore_form_submit').on("click", function() {
         var checked = '';
         if ($('#state_item').is(':checked')) {
@@ -8,18 +9,17 @@ function restoreRequeue(){
             checked = 'false';
         }
         var id = $('#work_item_id').text();
-        $.get('/items/'+id+'/requeue', {delete_state_item: checked},
-            function(data) {
-                alert('Item has been requeued.');
-            });
+        $.get('/items/'+id+'/requeue', {delete_state_item: checked});
     });
     $('#restore_form_cancel').on("click", function() {
         $('#restore_form').addClass('hidden');
+        $('#requeue-btn').removeClass('hidden');
     });
 }
 
 function glacierRestoreRequeue(){
     $('#glacier_restore_form').removeClass('hidden');
+    $('#requeue-btn').addClass('hidden');
     $('#glacier_restore_form_submit').on("click", function() {
         var checked = '';
         if ($('#state_item').is(':checked')) {
@@ -28,62 +28,62 @@ function glacierRestoreRequeue(){
             checked = 'false';
         }
         var id = $('#work_item_id').text();
-        $.get('/items/'+id+'/requeue', {delete_state_item: checked},
-            function(data) {
-                alert('Item has been requeued.');
-            });
+        $.get('/items/'+id+'/requeue', {delete_state_item: checked});
     });
     $('#glacier_restore_form_cancel').on("click", function() {
         $('#glacier_restore_form').addClass('hidden');
+        $('#requeue-btn').removeClass('hidden');
     });
 }
 
 function ingestRequeue(){
     $('#ingest_form').removeClass('hidden');
+    $('#requeue-btn').addClass('hidden');
     $('#ingest_form_submit').on("click", function() {
         var stage = $('input[name="stage"]:checked').val();
         if (stage == "Fetch" || stage == "Store" || stage == "Record") {
             if (!$('#ingest_error').hasClass('hidden')) {
                 $('#ingest_error').addClass('hidden');
             }
             var id = $('#work_item_id').text();
-            $.get('/items/'+id+'/requeue', {item_stage: stage},
+            $.get('/items/'+id+'/requeue', { item_stage: stage },
                 function(data) {
-                    alert('Item has been requeued.');
+                    window.location.replace(id);
                 });
         } else {
             $('#ingest_error').removeClass('hidden');
         }
     });
     $('#ingest_form_cancel').on("click", function() {
         $('#ingest_form').addClass('hidden');
+        $('#requeue-btn').removeClass('hidden');
     });
 }
 
 function dpnRequeue() {
     $('#dpn_form').removeClass('hidden');
+    $('#requeue-btn').addClass('hidden');
     $('#dpn_form_submit').on("click", function () {
         var stage = $('input[name="stage"]:checked').val();
         if (stage == "Package" || stage == "Store" || stage == "Record") {
             if (!$('#ingest_error').hasClass('hidden')) {
                 $('#ingest_error').addClass('hidden');
             }
             var id = $('#work_item_id').text();
-            $.get('/items/' + id + '/requeue', {item_stage: stage},
-                function (data) {
-                    alert('Item has been requeued.');
-                });
+            $.get('/items/' + id + '/requeue', {item_stage: stage});
         } else {
             $('#dpn_error').removeClass('hidden');
         }
     });
     $('#dpn_form_cancel').on("click", function () {
         $('#dpn_form').addClass('hidden');
+        $('#requeue-btn').removeClass('hidden');
     });
 }
 
 function dpnItemRequeue() {
     $('#dpn_item_form').removeClass('hidden');
+    $('#requeue-btn').addClass('hidden');
     $('#dpn_item_form_submit').on("click", function () {
         var checked = '';
         if ($('#dpn_state_item').is(':checked')) {
@@ -97,15 +97,34 @@ function dpnItemRequeue() {
                 $('#dpn_item_error').addClass('hidden');
             }
             var id = $('#dpn_item_id').text();
-            $.get('/dpn_items/' + id + '/requeue', {task: task, delete_state_item: checked},
-                function (data) {
-                    alert('DPN Item has been requeued.');
-                });
+            $.get('/dpn_items/' + id + '/requeue', {task: task, delete_state_item: checked});
         } else {
             $('#dpn_item_error').removeClass('hidden');
         }
     });
     $('#dpn_item_form_cancel').on("click", function () {
         $('#dpn_item_form').addClass('hidden');
+        $('#requeue-btn').removeClass('hidden');
+    });
+}
+
+function dpnItemFixityRequeue() {
+    $('#dpn_item_fixity_form').removeClass('hidden');
+    $('#requeue-btn').addClass('hidden');
+    $('#dpn_item_fixity_form_submit').on("click", function () {
+        var stage = $('input[name="stage"]:checked').val();
+        if (stage == "requested" || stage == "validate" || stage == "available_in_s3") {
+            if (!$('#dpn_item_fixity_error').hasClass('hidden')) {
+                $('#dpn_item_fixity_error').addClass('hidden');
+            }
+            var id = $('#dpn_item_id').text();
+            $.get('/dpn_items/' + id + '/requeue', {stage: stage});
+        } else {
+            $('#dpn_item_fixity_error').removeClass('hidden');
+        }
+    });
+    $('#dpn_item_fixity_form_cancel').on("click", function () {
+        $('#dpn_item_fixity_form').addClass('hidden');
+        $('#requeue-btn').removeClass('hidden');
     });
 }

app/controllers/dpn_work_items_controller.rb

@@ -64,10 +64,16 @@ def show
   def requeue
     if @dpn_item
       authorize @dpn_item
-      if @dpn_item.task == 'ingest' || @dpn_item.task == 'replication'
-        (params[:delete_state_item] && params[:delete_state_item] == 'true') ? delete_state = 'true' : delete_state = 'false'
-        @dpn_item.requeue_item(delete_state)
-        response = issue_requeue_http_post(params[:task])
+      if @dpn_item.task == 'ingest' || @dpn_item.task == 'replication' || @dpn_item.task == 'fixity'
+        if @dpn_item.task == 'ingest' || @dpn_item.task == 'replication'
+          (params[:delete_state_item] && params[:delete_state_item] == 'true') ? delete_state = 'true' : delete_state = 'false'
+          @dpn_item.requeue_item(delete_state)
+          response = issue_requeue_http_post(params[:task])
+        elsif @dpn_item.task == 'fixity'
+          stage = params[:stage]
+          @dpn_item.fixity_requeue(stage)
+          response = issue_requeue_http_post(stage)
+        end
         respond_to do |format|
           format.json { render json: { status: response.code, body: response.body } }
           format.html {
@@ -111,23 +117,31 @@ def set_item
   rescue ActiveRecord::RecordNotFound
   end
 
-  def issue_requeue_http_post(task)
+  def issue_requeue_http_post(task_or_stage)
     if @dpn_item.task == 'replication'
-      if task == 'copy'
+      if task_or_stage == 'copy'
         uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_copy_topic")
-      elsif task == 'validation'
+      elsif task_or_stage == 'validation'
         uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_validation_topic")
-      elsif task == 'store'
+      elsif task_or_stage == 'store'
         uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_replication_store_topic")
       end
     elsif @dpn_item.task == 'ingest'
-      if task == 'package'
+      if task_or_stage == 'package'
         uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_package_topic")
-      elsif task == 'store'
+      elsif task_or_stage == 'store'
         uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_ingest_store_topic")
-      elsif task == 'record'
+      elsif task_or_stage == 'record'
         uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_record_topic")
       end
+    elsif @dpn_item.task == 'fixity'
+      if task_or_stage == Pharos::Application::PHAROS_STAGES['requested']
+        uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_glacier_restore_topic")
+      elsif task_or_stage == Pharos::Application::PHAROS_STAGES['validate']
+        uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_fixity_topic")
+      elsif task_or_stage == Pharos::Application::PHAROS_STAGES['available_in_s3']
+        uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=dpn_s3_download_topic")
+      end
     end
     http = Net::HTTP.new(uri.host, uri.port)
     request = Net::HTTP::Post.new(uri)

app/controllers/work_items_controller.rb

@@ -27,7 +27,11 @@ def index
       authorize @items
       respond_to do |format|
         format.json {
-          json_list = @paged_results.map { |item| item.serializable_hash(except: [:node, :pid]) }
+          if current_user.admin?
+            json_list = @paged_results.map { |item| item.serializable_hash }
+          else
+            json_list = @paged_results.map { |item| item.serializable_hash(except: [:node, :pid]) }
+          end
           render json: {count: @count, next: @next, previous: @previous, results: json_list}
         }
         format.html { render 'old_index' }
@@ -80,13 +84,28 @@ def requeue
         options[:stage] = params[:item_stage] if params[:item_stage]
         options[:work_item_state_delete] = 'true' if params[:delete_state_item] && params[:delete_state_item] == 'true'
         @work_item.requeue_item(options)
-        options[:stage] ? response = issue_requeue_http_post(options[:stage]) : response = issue_requeue_http_post('')
-        respond_to do |format|
-          format.json { render json: { status: response.code, body: response.body } }
-          format.html {
-            render 'show'
-            flash[:notice] = "The response from NSQ to the requeue request is as follows: Status: #{response.code}, Body: #{response.body}"
-          }
+        if Rails.env.development?
+          flash[:notice] = 'The response from NSQ to the requeue request is as follows: Status: 200, Body: ok'
+          flash.keep(:notice)
+          respond_to do |format|
+            format.json { render json: { status: 200, body: 'ok' } }
+            format.html {
+              redirect_to @work_item
+              flash[:notice] = 'The response from NSQ to the requeue request is as follows: Status: 200, Body: ok'
+            }
+            format.js {
+              render js: "window.location = '#{work_item_path(@work_item)}'"
+            }
+          end
+        else
+          options[:stage] ? response = issue_requeue_http_post(options[:stage]) : response = issue_requeue_http_post('')
+          respond_to do |format|
+            format.json { render json: { status: response.code, body: response.body } }
+            format.html {
+              redirect_to @work_item
+              flash[:notice] = "The response from NSQ to the requeue request is as follows: Status: #{response.code}, Body: #{response.body}"
+            }
+          end
         end
       end
     else
@@ -441,7 +460,6 @@ def issue_requeue_http_post(stage)
     elsif @work_item.action == Pharos::Application::PHAROS_ACTIONS['glacier_restore']
       uri = URI("#{Pharos::Application::NSQ_BASE_URL}/pub?topic=apt_glacier_restore_init_topic")
     end
-
     http = Net::HTTP.new(uri.host, uri.port)
     request = Net::HTTP::Post.new(uri)
     request.body = @work_item.id.to_s
@@ -474,8 +492,11 @@ def filter_count_and_sort
                  .with_action(params[:item_action])
                  .queued(params[:queued])
                  .with_node(params[:node])
+                 .with_pid(params[:pid])
                  .with_unempty_node(params[:node_not_empty])
                  .with_empty_node(params[:node_empty])
+                 .with_unempty_pid(params[:pid_not_empty])
+                 .with_empty_pid(params[:pid_empty])
                  .with_retry(params[:retry])
     @selected = {}
     get_status_counts(@items)