Merge pull request #511 from sul-dlss/workflow_indexer

Workflow indexer

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2019-03-13 11:39:01 -0500 using RuboCop version 0.65.0.
+# on 2019-04-03 11:25:54 -0500 using RuboCop version 0.65.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -63,41 +63,41 @@ Lint/Void:
   Exclude:
     - 'lib/dor/models/concerns/identifiable.rb'
 
-# Offense count: 82
+# Offense count: 84
 Metrics/AbcSize:
   Max: 106
 
-# Offense count: 199
+# Offense count: 201
 # Configuration parameters: CountComments, ExcludedMethods.
 # ExcludedMethods: refine
 Metrics/BlockLength:
   Max: 394
 
-# Offense count: 13
+# Offense count: 12
 # Configuration parameters: CountComments.
 Metrics/ClassLength:
   Max: 253
 
-# Offense count: 17
+# Offense count: 16
 Metrics/CyclomaticComplexity:
   Max: 25
 
-# Offense count: 78
+# Offense count: 77
 # Configuration parameters: CountComments, ExcludedMethods.
 Metrics/MethodLength:
   Max: 52
 
 # Offense count: 3
 # Configuration parameters: CountComments.
 Metrics/ModuleLength:
-  Max: 148
+  Max: 146
 
 # Offense count: 2
 # Configuration parameters: CountKeywordArgs.
 Metrics/ParameterLists:
   Max: 7
 
-# Offense count: 17
+# Offense count: 16
 Metrics/PerceivedComplexity:
   Max: 25
 
@@ -108,7 +108,7 @@ Naming/AccessorMethodName:
     - 'lib/dor/models/concerns/describable.rb'
     - 'lib/dor/models/concerns/governable.rb'
 
-# Offense count: 16
+# Offense count: 8
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: lowercase, uppercase
 Naming/HeredocDelimiterCase:
@@ -238,7 +238,7 @@ RSpec/DescribeClass:
     - 'spec/services/cleanup_service_filesystem_spec.rb'
     - 'spec/services/digital_stacks_service_spec.rb'
 
-# Offense count: 208
+# Offense count: 207
 # Configuration parameters: Max.
 RSpec/ExampleLength:
   Enabled: false
@@ -262,7 +262,7 @@ RSpec/ExpectInHook:
     - 'spec/services/shelving_service_spec.rb'
     - 'spec/services/status_service_spec.rb'
 
-# Offense count: 67
+# Offense count: 69
 # Configuration parameters: CustomTransform, IgnoreMethods.
 RSpec/FilePath:
   Enabled: false
@@ -277,7 +277,7 @@ RSpec/MessageChain:
   Exclude:
     - 'spec/services/suri_service_spec.rb'
 
-# Offense count: 276
+# Offense count: 277
 # Configuration parameters: EnforcedStyle.
 # SupportedStyles: have_received, receive
 RSpec/MessageSpies:
@@ -316,7 +316,7 @@ RSpec/RepeatedDescription:
     - 'spec/services/abiltiy_spec.rb'
     - 'spec/services/release_tag_service_spec.rb'
 
-# Offense count: 22
+# Offense count: 21
 RSpec/ScatteredLet:
   Exclude:
     - 'spec/indexers/describable_indexer_spec.rb'
@@ -402,7 +402,7 @@ Style/ConditionalAssignment:
     - 'lib/dor/services/merge_service.rb'
     - 'spec/support/foxml_helper.rb'
 
-# Offense count: 77
+# Offense count: 78
 Style/Documentation:
   Enabled: false
 
@@ -436,19 +436,14 @@ Style/MultilineBlockChain:
   Exclude:
     - 'lib/dor/datastreams/workflow_definition_ds.rb'
 
-# Offense count: 1
-Style/NestedTernaryOperator:
-  Exclude:
-    - 'lib/dor/workflow/document.rb'
-
 # Offense count: 36
 # Cop supports --auto-correct.
 # Configuration parameters: AutoCorrect, EnforcedStyle, IgnoredMethods.
 # SupportedStyles: predicate, comparison
 Style/NumericPredicate:
   Enabled: false
 
-# Offense count: 33
+# Offense count: 32
 # Cop supports --auto-correct.
 # Configuration parameters: PreferredDelimiters.
 Style/PercentLiteralDelimiters:
@@ -459,7 +454,6 @@ Style/PercentLiteralDelimiters:
     - 'lib/dor/models/concerns/releaseable.rb'
     - 'lib/dor/services/cleanup_service.rb'
     - 'lib/dor/services/metadata_handlers/catalog_handler.rb'
-    - 'lib/dor/services/metadata_service.rb'
     - 'lib/dor/services/publish_metadata_service.rb'
     - 'lib/dor/utils/sdr_client.rb'
     - 'spec/datastreams/events_ds_spec.rb'

lib/dor-services.rb

@@ -91,6 +91,7 @@ def logger
     autoload :ProcessableIndexer
     autoload :ReleasableIndexer
     autoload :WorkflowIndexer
+    autoload :WorkflowsIndexer
   end
 
   # datastreams
@@ -144,6 +145,7 @@ def logger
       autoload :Collection
       autoload :AdminPolicyObject
       autoload :WorkflowObject
+      autoload :WorkflowSolrDocument
     end
   end
 

lib/dor/datastreams/workflow_ds.rb

@@ -83,7 +83,7 @@ def current_priority
     end
 
     def to_solr(solr_doc = {}, *_args)
-      # noop - indexing is done by the WorkflowIndexer
+      # noop - indexing is done by the WorkflowsIndexer
       solr_doc
     end
 

lib/dor/indexers/workflow_indexer.rb

@@ -3,25 +3,82 @@
 module Dor
   # Indexes the objects position in workflows
   class WorkflowIndexer
-    include SolrDocHelper
+    ERROR_OMISSION = '... (continued)'
+    private_constant :ERROR_OMISSION
 
-    attr_reader :resource
-    def initialize(resource:)
-      @resource = resource
+    # see https://lucene.apache.org/core/7_3_1/core/org/apache/lucene/util/BytesRefHash.MaxBytesLengthExceededException.html
+    MAX_ERROR_LENGTH = 32_768 - 2 - ERROR_OMISSION.length
+    private_constant :MAX_ERROR_LENGTH
+
+    # @param [Dor::WorkflowDocument] document the workflow document to index
+    def initialize(document:)
+      @document = document
     end
 
-    # @return [Hash] the partial solr document for workflow concerns
+    # @return [Hash] the partial solr document for the workflow document
     def to_solr
-      {}.tap do |solr_doc|
-        workflows.each { |wf| solr_doc = wf.to_solr(solr_doc) }
+      WorkflowSolrDocument.new do |solr_doc|
+        wf_name = document.workflowId.first
+        solr_doc.name = wf_name
+        errors = processes.count(&:error?)
+
+        repo = document.repository.first
+        solr_doc.status = [wf_name, workflow_status, errors, repo].join('|')
+
+        processes.each do |process|
+          index_process(solr_doc, wf_name, process)
+        end
       end
     end
 
     private
 
-    # @return [Array<Dor::WorkflowDocument>]
-    def workflows
-      resource.workflows.workflows
+    attr_reader :document
+    delegate :processes, to: :document
+
+    def index_process(solr_doc, wf_name, process)
+      return unless process.status.present?
+
+      # add a record of the robot having operated on this item, so we can track robot activity
+      solr_doc.add_process_time(wf_name, process.name, Time.parse(process.date_time)) if process_has_time?(process)
+
+      index_error_message(solr_doc, wf_name, process)
+
+      # workflow name, process status then process name
+      solr_doc.add_wps("#{wf_name}:#{process.status}", "#{wf_name}:#{process.status}:#{process.name}")
+
+      # workflow name, process name then process status
+      solr_doc.add_wps("#{wf_name}:#{process.name}", "#{wf_name}:#{process.name}:#{process.status}")
+
+      # process status, workflowname then process name
+      solr_doc.add_swp(process.status.to_s, "#{process.status}:#{wf_name}", "#{process.status}:#{wf_name}:#{process.name}")
+      return if process.state == process.status
+
+      solr_doc.add_wsp("#{wf_name}:#{process.state}:#{process.name}")
+      solr_doc.add_wps("#{wf_name}:#{process.name}:#{process.state}")
+      solr_doc.add_swp(process.state.to_s, "#{process.state}:#{wf_name}", "#{process.state}:#{wf_name}:#{process.name}")
+    end
+
+    def process_has_time?(process)
+      !process.date_time.blank? && process.status && (process.status == 'completed' || process.status == 'error')
+    end
+
+    def workflow_status
+      return 'empty' if processes.empty?
+
+      workflow_should_show_completed?(processes) ? 'completed' : 'active'
+    end
+
+    def workflow_should_show_completed?(processes)
+      processes.all? { |p| ['skipped', 'completed', '', nil].include?(p.status) }
+    end
+
+    # index the error message without the druid so we hopefully get some overlap
+    # truncate to avoid org.apache.lucene.util.BytesRefHash$MaxBytesLengthExceededException
+    def index_error_message(solr_doc, wf_name, process)
+      return unless process.error_message
+
+      solr_doc.error = "#{wf_name}:#{process.name}:#{process.error_message}".truncate(MAX_ERROR_LENGTH, omission: ERROR_OMISSION)
     end
   end
 end

lib/dor/indexers/workflows_indexer.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Dor
+  # Indexes the objects position in workflows
+  class WorkflowsIndexer
+    attr_reader :resource
+    def initialize(resource:)
+      @resource = resource
+    end
+
+    # @return [Hash] the partial solr document for workflow concerns
+    def to_solr
+      WorkflowSolrDocument.new do |combined_doc|
+        workflows.each do |wf|
+          doc = WorkflowIndexer.new(document: wf).to_solr
+          combined_doc.merge!(doc)
+        end
+      end.to_h
+    end
+
+    private
+
+    # @return [Array<Dor::WorkflowDocument>]
+    def workflows
+      resource.workflows.workflows
+    end
+  end
+end

lib/dor/models/admin_policy_object.rb

@@ -15,7 +15,7 @@ class AdminPolicyObject < Dor::Abstract
       EditableIndexer,
       IdentifiableIndexer,
       ProcessableIndexer,
-      WorkflowIndexer
+      WorkflowsIndexer
     )
 
     CREATIVE_COMMONS_USE_LICENSES = ActiveSupport::Deprecation::DeprecatedConstantProxy.new('CREATIVE_COMMONS_USE_LICENSES', 'Dor::CreativeCommonsLicenseService')

lib/dor/models/collection.rb

@@ -12,7 +12,7 @@ class Collection < Dor::Set
       IdentifiableIndexer,
       ProcessableIndexer,
       ReleasableIndexer,
-      WorkflowIndexer
+      WorkflowsIndexer
     )
   end
 end

lib/dor/models/item.rb

@@ -19,7 +19,7 @@ class Item < Dor::Abstract
       IdentifiableIndexer,
       ProcessableIndexer,
       ReleasableIndexer,
-      WorkflowIndexer
+      WorkflowsIndexer
     )
 
     has_metadata name: 'technicalMetadata', type: TechnicalMetadataDS, label: 'Technical Metadata', control_group: 'M'

lib/dor/models/set.rb

@@ -12,7 +12,7 @@ class Set < Dor::Abstract
       DescribableIndexer,
       IdentifiableIndexer,
       ProcessableIndexer,
-      WorkflowIndexer
+      WorkflowsIndexer
     )
   end
 end

lib/dor/models/workflow_object.rb

@@ -15,7 +15,7 @@ class WorkflowObject < Dor::Abstract
       DescribableIndexer,
       IdentifiableIndexer,
       ProcessableIndexer,
-      WorkflowIndexer
+      WorkflowsIndexer
     )
 
     def self.find_by_name(name)