feature/MMT-2326: Merge loss report feature into master

.gitignore

@@ -58,3 +58,7 @@ nohup.out
 
 #backup files
 *~
+
+#ignore package
+package-lock.json
+package.json

Gemfile

@@ -59,6 +59,8 @@ gem 'kaminari'
 gem 'momentjs-rails' # js lib for dates
 gem 'pundit'
 
+gem 'nokogiri-diff', '~> 0.2.0' # for comparing xml documents
+
 gem 'activerecord-import' # bulk insertion of data
 
 gem 'activerecord-session_store'

Gemfile.lock

@@ -195,6 +195,9 @@ GEM
     nio4r (2.5.3)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
+    nokogiri-diff (0.2.0)
+      nokogiri (~> 1.5)
+      tdiff (~> 0.3, >= 0.3.2)
     parallel (1.19.1)
     parser (2.7.1.2)
       ast (~> 2.4.0)
@@ -297,6 +300,7 @@ GEM
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
+    tdiff (0.3.4)
     thor (1.0.1)
     thread_safe (0.3.6)
     tilt (2.0.10)
@@ -365,6 +369,7 @@ DEPENDENCIES
   mini_racer
   momentjs-rails
   multi_xml
+  nokogiri-diff (~> 0.2.0)
   pg
   pundit
   rack_session_access

app/assets/javascripts/change_current_provider.coffee

@@ -4,6 +4,9 @@ $(document).ready ->
     overlay: 0.6
     closeButton: '.modal-close'
 
+  $('a.loss-report').on 'click', (element) ->
+    $('#loss-report-modal').hide()
+
   # Handle not-current-provider-modal
   $('a.not-current-provider').on 'click', (element) ->
     provider = $(element.target).data('provider')

app/assets/stylesheets/components/_buttons.scss

@@ -26,7 +26,23 @@
   &:hover {
     background-color: darken($red, 6%);
   }
-}  
+}
+
+.eui-btn--steel-blue {
+  @extend .eui-btn;
+
+  background-color: $steel-blue;
+  color: $white !important;
+  padding: 0.5em 1em;
+
+  &:hover {
+    background-color: darken($steel-blue, 6%);
+  }
+}
+
+.button-spacing {
+  margin-right: 0.625em;
+}
 
 button {
   &:disabled {
@@ -36,4 +52,4 @@ button {
       background-color: lighten($dolphin-grey, 10%);
     }
   }
-}
+}

app/assets/stylesheets/components/_search.scss

@@ -64,12 +64,12 @@
         }
       }
     }
-    .search-dropdown-short {   /* smaller dropdown if radio buttons are hidden in DMMT */ 
+    .search-dropdown-short {   /* smaller dropdown if radio buttons are hidden in DMMT */
       width: 100%;
       height: 45px;
     }
-    
-    .search-dropdown-wide {   /* wider dropdown if UMM-T is enabled */ 
+
+    .search-dropdown-wide {   /* wider dropdown if UMM-T is enabled */
       right: 0px; // causes dropdown width to extend to the left
       max-width: 440px;
     }
@@ -79,12 +79,12 @@
       border-right: 1px solid $dolphin-grey;
       border-bottom: 1px solid $steel-blue;
     }
-    
+
     #search-submit-button {
       text-align: left;
       width: 172px; // 11.92em
     }
-    
+
     .search-field-wide {
       width: 241px;
     }

app/controllers/collections_controller.rb

@@ -2,6 +2,7 @@ class CollectionsController < ManageCollectionsController
   include ManageMetadataHelper
   include CMRCollectionsHelper
   include CollectionsHelper
+  include LossReportHelper
 
   before_action :set_collection
   before_action :ensure_correct_collection_provider, only: [:edit, :clone, :revert, :destroy]
@@ -117,6 +118,35 @@ def create_update_proposal
     redirect_to collection_draft_proposal_path(proposal)
   end
 
+  def loss_report
+    # When a user wants to use MMT to edit metadata that currently exists in a non-UMM form,
+    # it's important that they're able to see if any data loss occurs in the translation to umm.
+    # This method is needed to reference the appropriate helper and view for the lossiness report.
+    # If translated_collections contains an :error field, the error message will appear.
+
+    # this checks the 'hide_items' url parameter that is can be manually added. Its primary use is for developers
+    # that need to debug using the text_output
+    if params[:hide_items].nil? || params[:hide_items].downcase == 'true'
+      hide_items = true
+    elsif params[:hide_items].downcase == 'false'
+      hide_items = false
+    else
+      translated_collections = { error: 'Unknown value for the hide_items parameter. The format should be: ".../loss_report.text?hide_items=true" or ".../loss_report.text?hide_items=false"' }
+    end
+
+    translated_collections ||= prepare_translated_collections
+
+    respond_to do |format|
+      if translated_collections[:error]
+        format.text { render plain: translated_collections[:error] }
+        format.json { render json: JSON.pretty_generate(translated_collections) }
+      else
+        format.text { render plain: loss_report_output(translated_collections: translated_collections, hide_items: hide_items, display: 'text') }
+        format.json { render json: JSON.pretty_generate(loss_report_output(translated_collections: translated_collections, hide_items: hide_items, display: 'json')) }
+      end
+    end
+  end
+
   private
 
   def ensure_correct_collection_provider
@@ -131,6 +161,25 @@ def ensure_correct_collection_provider
     render :show
   end
 
+  def prepare_translated_collections
+    original_collection_native_xml = cmr_client.get_concept(params[:id],token, {})
+    return { error: 'Failed to retrieve collection from CMR' } unless original_collection_native_xml.success?
+
+    content_type = original_collection_native_xml.headers.fetch('content-type').split(';')[0]
+    return { error: 'This collection is already in UMM format so there is no loss report' } if content_type.include?('application/vnd.nasa.cmr.umm+json')
+
+    translated_collection_native_xml = cmr_client.translate_collection(JSON.pretty_generate(@collection), "application/#{Rails.configuration.umm_c_version}; charset=utf-8", content_type,  skip_validation=true)
+    return { error: 'Failed to translate collection from UMM back to native format' } unless translated_collection_native_xml.success?
+
+    return {
+      original_collection_native_xml: original_collection_native_xml.body,
+      translated_collection_native_xml: translated_collection_native_xml.body,
+      original_collection_native_hash: Hash.from_xml(original_collection_native_xml.body),
+      translated_collection_native_hash: Hash.from_xml(translated_collection_native_xml.body),
+      native_format: content_type
+    }
+  end
+
   def set_collection
     @concept_id = params[:id]
     @revision_id = params[:revision_id]

app/helpers/loss_report_helper.rb

@@ -0,0 +1,191 @@
+module LossReportHelper
+
+  def loss_report_output(translated_collections:, hide_items:, display:)
+    # depending on the input selection (json or text) a comparison string/hash is created and displayed in-browser
+    # this display feature could be a good candidate for dependency injection
+
+    orig_h = translated_collections[:original_collection_native_hash]
+    conv_h = translated_collections[:translated_collection_native_hash]
+
+    # ISO and DIF collections (in XML form) contain namespaces that cause errors in the below comparison.
+    # Specifically, when nodes are evaluated individually, (their namespace definitions remaining at the top of the xml)
+    # their prefixes are undefined in the scope of the evaluation and therefore raise errors. Removing the namespaces
+    # eliminates this issue.
+    if translated_collections[:native_format].include?('iso') || translated_collections[:native_format].include?('dif')
+      orig = Nokogiri::XML(translated_collections[:original_collection_native_xml]) { |config| config.strict.noblanks }.remove_namespaces!
+      conv = Nokogiri::XML(translated_collections[:translated_collection_native_xml]) { |config| config.strict.noblanks }.remove_namespaces!
+    else
+      orig = Nokogiri::XML(translated_collections[:original_collection_native_xml]) { |config| config.strict.noblanks }
+      conv = Nokogiri::XML(translated_collections[:translated_collection_native_xml]) { |config| config.strict.noblanks }
+    end
+
+    # This array is used to keep track of the paths that lead to arrays that have already been mapped
+    arr_paths = Array.new
+
+    if display == 'text'
+      text_output = String.new
+      json_output = nil
+      text_output += (translated_collections[:native_format] + "\n\n")
+    elsif display == 'json'
+      json_output = Hash.new
+      text_output = nil
+      json_output['format'] = translated_collections[:native_format]
+    end
+
+    # Below is the Nokogiri#diff method that is used to compare Nokogiri::XML objects.
+    # The 'change' item is either '+' or '-'; the 'node' item is the Nokogiri::XML::Node object
+    orig.diff(conv, {:added => true, :removed => true}) do |change,node|
+
+      element = node.to_xml
+      path = node.parent.path.split('[')[0]
+      arr_path = top_level_arr_path(path, orig_h, conv_h)
+
+      # the first layer of the following if/else structure is used to separately evaluate explicit array changes.
+      # This is why arr_path will evaluate true if the element in question is an array
+      if arr_path && path_not_checked?(arr_path, arr_paths)
+        arr_paths << arr_path
+        array_comparison(arr_path, orig_h, conv_h).each { |item| add_to_report(item[0], item[1], item[2], hide_items, display, json_output, text_output) }
+      elsif path_not_checked?(path, arr_paths)
+        # this layer of if/else separates items that contain xml (this is a nokogiri oddity that occurs where
+        # Nokogiri does not directly map to an item that is changed thus it still contains xml - this is the
+        # purpose of hash_values_and_paths), items that represent xml attribute changes, and normal changes.
+        if is_xml?(element)
+          element = Hash.from_xml(element)
+          hash_values_and_paths(element).each do |item|
+            arr_path = top_level_arr_path("#{path}/#{item['path']}", orig_h, conv_h)
+            # this layer of if/else structure is used to separately evaluate implicit array changes in the xml.
+            # This is why arr_path will evaluate true if the element in question is an array
+            if arr_path && path_not_checked?("#{path}/#{item['path']}", arr_paths)
+              if path_not_checked?(arr_path, arr_paths)
+                arr_paths << arr_path
+                array_comparison(arr_path, orig_h, conv_h).each { |item| add_to_report(item[0], item[1], item[2], hide_items, display, json_output, text_output) }
+              end
+            elsif path_not_checked?("#{path}/#{item['path']}", arr_paths)
+              add_to_report(change, item['value'], "#{path}/#{item['path']}", hide_items, display, json_output, text_output)
+            end
+          end
+        elsif (attr,val = is_attribute?(element))
+          add_to_report(change, val, "#{path}/#{attr}" , hide_items, display, json_output, text_output)
+        else
+          add_to_report(change, element, path, hide_items, display, json_output, text_output)
+        end
+      end
+    end
+    return text_output if display == 'text'
+    return json_output if display == 'json'
+  end
+
+  def is_xml?(element)
+    # checks if the element being passed is xml
+    # may be beneficial to add more checks
+    element.include?('<' && '</' && '>')
+  end
+
+  def is_attribute?(element)
+    # this method checks if the element being passed is an attribute change;
+    # TODO: it may be beneficial to add more conditions to improve accuracy
+    if element.include?('=') && !element.include?(' = ')
+      attr_val = Array.new
+      element.split('=').each {|item| attr_val << item.strip.delete('\\"')}
+      attr_val
+    else
+      false
+    end
+  end
+
+  def path_not_checked?(arr_path, arr_paths)
+    # this method checks the arr_paths array to see if the path being added to
+    # the report has already been previously evaluated and added
+    arr_paths.each { |path| return false if arr_path.include?(path) }
+    true
+  end
+
+  def top_level_arr_path(path, orig_h, conv_h)
+    # if an array is passed that passes through an array ie. /Contacts/Contact[0]/Role/Name
+    # this method would return /Contacts/Contact because Contact is the outermost array (or false if the path doesn't contain an array)
+    pre_translation_array, pre_translation_path = hash_navigation(path, orig_h)
+    post_translation_array, post_translation_path = hash_navigation(path, conv_h)
+
+    # the following line handles a scenario where hash_navigation returns false for both pre_ and post_translation_arrays
+    # which means that the path passed does not exist in the original or converted collections
+    return path_exists = false if pre_translation_array == false && post_translation_array == false
+
+    return pre_translation_path if pre_translation_array.is_a?(Array)
+    return post_translation_path if post_translation_array.is_a?(Array)
+
+    # the number of keys must be 1 because all arrays in echo10, dif10, and iso19115 are tagged similar to:
+    # <Contacts><Contact>contact</Contact></Contacts> and so all array-containing tags will be the plural
+    # of the array name. This clause serves to identify array-containing tags when their paths aren't properly
+    # displayed by nokogiri
+    if pre_translation_array.is_a?(Hash) && pre_translation_array.keys.length == 1 && pre_translation_array[pre_translation_array.keys[0]].is_a?(Array)
+      return "#{pre_translation_path}/#{pre_translation_array.keys[0]}"
+    elsif post_translation_array.is_a?(Hash) && post_translation_array.keys.length == 1 && post_translation_array[post_translation_array.keys[0]].is_a?(Array)
+      return "#{post_translation_path}/#{post_translation_array.keys[0]}"
+    end
+    path_contains_array = false
+  end
+
+  def add_to_report(change, element, path, hide_items, display, json_output, text_output)
+    @counter ||= 0 and @counter += 1
+    # this function serves to preclude complex nests from forming in loss_report_output the
+    # following 'if' structure is intended to increase readability by eliminating nests
+    return text_output.concat("#{@counter}.".ljust(4)+"#{change}: #{element}".ljust(60) + path + "\n") if hide_items == false && display == 'text'
+    return text_output.concat("#{@counter}.".ljust(4)+"#{change}: ".ljust(3) + path + "\n") if hide_items == true && display == 'text'
+    return json_output["#{@counter}. #{change}: #{path}"] = element if display == 'json'
+  end
+
+  def hash_values_and_paths(hash)
+    buckets = Array.new
+    hash.each do |key,val|
+      if val.is_a? Hash then hash_values_and_paths(val).each do |item|
+        item['path'] = key + '/' + item['path']
+        buckets << item end
+      else
+        buckets << {'path'=> key, 'value'=> val}
+      end
+    end
+    buckets
+  end
+
+  def hash_navigation(path, hash)
+    # Passed a path string and the hash being navigated. This method parses the path string and
+    # returns the array/value at the end of the path
+    current_path = String.new
+    path.split('/').each do |key|
+      if hash.is_a?(Array)
+        return hash, current_path
+      elsif hash.key?(key) && hash.is_a?(Hash)
+        current_path += "/#{key}"
+        hash = hash[key]
+      elsif !hash.key?(key) && key != ''
+        return path_exists = false, "#{current_path}/#{key}"
+      end
+    end
+    return hash, current_path
+  end
+
+  def array_comparison(path, original_hash, converted_hash)
+    pre_translation_array = hash_navigation(path, original_hash)[0]
+    post_translation_array = hash_navigation(path, converted_hash)[0]
+
+    pre_translation_array == false ? pre_translation_array = Array.new : pre_translation_array = Array.wrap(pre_translation_array)
+    post_translation_array == false ? post_translation_array = Array.new : post_translation_array = Array.wrap(post_translation_array)
+
+    output = Array.new
+    (pre_translation_array - post_translation_array).each do |item|
+      path_with_index = path + "[#{pre_translation_array.index(item)}]"
+      # the following line is used to eliminate indexing confusion when there is more than one occurrence of a particular item in an array
+      pre_translation_array[pre_translation_array.index(item)] = item.to_s + 'item indexed'
+      output << ['-', item, path_with_index]
+    end
+
+    (post_translation_array - pre_translation_array).each do |item|
+      path_with_index = path + "[#{post_translation_array.index(item)}]"
+      # the following line is used to eliminate indexing confusion when there is more than one occurrence of a particular item in an array
+      post_translation_array[post_translation_array.index(item)] = item.to_s + 'item indexed'
+      output << ['+', item, path_with_index]
+    end
+    output
+  end
+
+end

app/helpers/tools_helper.rb

@@ -15,4 +15,4 @@ def render_change_provider_tool_action_link(tool_action, concept_id, revision_id
   def tool_action_text(tool_action)
     tool_action
   end
-end
+end